Browse Source

Use berkley db instead of plaintext files for mailbox maps

- Move reading of mailbox maps to storage module
- Migrate to berkley db file reader
- Remove deprecated code from auth module
Alexey Edelev 5 years ago
parent
commit
b3d26c62ba
8 changed files with 110 additions and 136 deletions
  1. 2 38
      auth/authenticator.go
  2. 0 26
      auth/registrar.go
  3. 2 2
      build.sh
  4. 2 2
      config/config.go
  5. 86 16
      db/db.go
  6. 2 0
      go.mod
  7. 10 0
      go.sum
  8. 6 52
      scanner/mailscanner.go

+ 2 - 38
auth/authenticator.go

@@ -26,20 +26,15 @@
 package auth
 
 import (
-	"bufio"
 	"log"
-	"os"
-	"strings"
 
-	config "git.semlanik.org/semlanik/gostfix/config"
 	db "git.semlanik.org/semlanik/gostfix/db"
 	utils "git.semlanik.org/semlanik/gostfix/utils"
 	uuid "github.com/google/uuid"
 )
 
 type Authenticator struct {
-	storage  *db.Storage
-	mailMaps map[string]string //TODO: temporary here. Later should be part of mailscanner and never accessed from here
+	storage *db.Storage
 }
 
 func NewAuthenticator() (a *Authenticator) {
@@ -51,8 +46,7 @@ func NewAuthenticator() (a *Authenticator) {
 	}
 
 	a = &Authenticator{
-		mailMaps: readMailMaps(), //TODO: temporary here. Later should be part of mailscanner and never accessed from here
-		storage:  storage,
+		storage: storage,
 	}
 	return
 }
@@ -78,33 +72,3 @@ func (a *Authenticator) Verify(user, token string) bool {
 
 	return a.storage.CheckToken(user, token) == nil
 }
-
-func (a *Authenticator) MailPath(user string) string { //TODO: temporary here. Later should be part of mailscanner and never accessed from here
-	return a.mailMaps[user]
-}
-
-func readMailMaps() map[string]string { //TODO: temporary here. Later should be part of mailscanner and never accessed from here
-	mailMaps := make(map[string]string)
-	mapsFile := config.ConfigInstance().VMailboxMaps
-	if !utils.FileExists(mapsFile) {
-		return mailMaps
-	}
-
-	file, err := os.Open(mapsFile)
-	if err != nil {
-		log.Fatalf("Unable to open virtual mailbox maps %s\n", mapsFile)
-	}
-
-	scanner := bufio.NewScanner(file)
-
-	for scanner.Scan() {
-		mailPathPair := strings.Split(scanner.Text(), " ")
-		if len(mailPathPair) != 2 {
-			log.Printf("Invalid record in virtual mailbox maps %s", scanner.Text())
-			continue
-		}
-		mailMaps[mailPathPair[0]] = mailPathPair[1]
-	}
-
-	return mailMaps
-}

+ 0 - 26
auth/registrar.go

@@ -1,26 +0,0 @@
-/*
- * MIT License
- *
- * Copyright (c) 2020 Alexey Edelev <semlanik@gmail.com>
- *
- * This file is part of gostfix project https://git.semlanik.org/semlanik/gostfix
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy of this
- * software and associated documentation files (the "Software"), to deal in the Software
- * without restriction, including without limitation the rights to use, copy, modify,
- * merge, publish, distribute, sublicense, and/or sell copies of the Software, and
- * to permit persons to whom the Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be included in all copies
- * or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
- * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
- * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
- * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
- * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- */
-
-package auth

+ 2 - 2
build.sh

@@ -9,14 +9,14 @@ go get -u github.com/amsokol/protoc-gen-gotag
 rm -f $RPC_PATH/*.pb.go
 protoc -I$RPC_PATH --go_out=plugins=grpc:$RPC_PATH $RPC_PATH/gostfix.proto
 
-protoc -I$RPC_PATH --gotag_out=xxx="bson+\"-\"",output_path=$RPC_PATH:. $RPC_PATH/gostfix.proto 
+protoc -I$RPC_PATH --gotag_out=xxx="bson+\"-\"",output_path=$RPC_PATH:. $RPC_PATH/gostfix.proto
 
 echo "Installing data"
 rm -rf data
 mkdir data
 cp -a main.ini data/
 cp -a main.cf data/
-cp -a vmailbox data/
+cp -a vmailbox.db data/
 cp -a web/assets data/
 cp -a web/css data/
 cp -a web/js data/

+ 2 - 2
config/config.go

@@ -118,7 +118,7 @@ func newConfig() (config *gostfixConfig, err error) {
 		return
 	}
 
-	if !utils.FileExists(mapsList[1]) {
+	if !utils.FileExists(mapsList[1] + ".db") {
 		log.Fatalf("Virtual mailbox map %s doesn't exist, postfix is not configured proper way, check %s in %s\n", mapsList[1], PostfixKeyVirtualMailboxMaps, postfixConfigPath)
 		return
 	}
@@ -164,7 +164,7 @@ func newConfig() (config *gostfixConfig, err error) {
 	config = &gostfixConfig{
 		MyDomain:            myDomain,
 		VMailboxBase:        baseDir,
-		VMailboxMaps:        mapsList[1],
+		VMailboxMaps:        mapsList[1] + ".db",
 		VMailboxDomains:     validDomains,
 		MongoUser:           mongoUser,
 		MongoPassword:       mongoPassword,

+ 86 - 16
db/db.go

@@ -32,12 +32,12 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"os"
-	"os/exec"
 	"strings"
 	"time"
 
 	common "git.semlanik.org/semlanik/gostfix/common"
+	"git.semlanik.org/semlanik/gostfix/utils"
+	"github.com/jsimonetti/berkeleydb"
 	bcrypt "golang.org/x/crypto/bcrypt"
 
 	bson "go.mongodb.org/mongo-driver/bson"
@@ -134,7 +134,6 @@ func (s *Storage) AddUser(user, password, fullName string) error {
 		return err
 	}
 
-	//TODO: Update postfix virtual map here
 	return nil
 }
 
@@ -164,26 +163,26 @@ func (s *Storage) addEmail(user string, email string, upsert bool) error {
 		}
 	}
 
-	file, err := os.OpenFile(config.ConfigInstance().VMailboxMaps, os.O_APPEND|os.O_WRONLY, 0664)
-	if err != nil {
-		return errors.New("Unable to add email to maps" + err.Error())
-	}
-
 	emailParts := strings.Split(email, "@")
 
 	if len(emailParts) != 2 {
 		return errors.New("Invalid email format")
 	}
 
-	_, err = file.WriteString(email + " " + emailParts[1] + "/" + emailParts[0] + "\n")
+	db, err := berkeleydb.NewDB()
 	if err != nil {
-		return errors.New("Unable to add email to maps" + err.Error())
+		log.Fatal(err)
+	}
+
+	err = db.Open(config.ConfigInstance().VMailboxMaps, berkeleydb.DbHash, 0)
+	if err != nil {
+		log.Fatalf("Unable to open virtual mailbox maps %s %s\n", config.ConfigInstance().VMailboxMaps, err)
 	}
+	defer db.Close()
 
-	cmd := exec.Command("postmap", config.ConfigInstance().VMailboxMaps)
-	err = cmd.Run()
+	err = db.Put(email, emailParts[1]+"/"+emailParts[0])
 	if err != nil {
-		return errors.New("Unable to execute postmap")
+		return errors.New("Unable to add email to maps" + err.Error())
 	}
 
 	_, err = s.emailsCollection.UpdateOne(context.Background(),
@@ -196,11 +195,25 @@ func (s *Storage) addEmail(user string, email string, upsert bool) error {
 
 func (s *Storage) RemoveEmail(user string, email string) error {
 
-	_, err := s.emailsCollection.UpdateOne(context.Background(),
+	db, err := berkeleydb.NewDB()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = db.Open(config.ConfigInstance().VMailboxMaps, berkeleydb.DbHash, 0)
+	if err != nil {
+		log.Fatalf("Unable to open virtual mailbox maps %s %s\n", config.ConfigInstance().VMailboxMaps, err)
+	}
+	defer db.Close()
+
+	err = db.Delete(email)
+	if err != nil {
+		return errors.New("Unable to remove email from maps" + err.Error())
+	}
+
+	_, err = s.emailsCollection.UpdateOne(context.Background(),
 		bson.M{"user": user},
 		bson.M{"$pull": bson.M{"email": email}})
-
-	//TODO: Update postfix virtual map here
 	return err
 }
 
@@ -575,3 +588,60 @@ func (s *Storage) GetFolders(email string) (folders []*common.Folder) {
 	}
 	return
 }
+
+func (s *Storage) ReadEmailMaps() (map[string]string, error) {
+	registredEmails, err := s.GetAllEmails()
+	if err != nil {
+		return nil, err
+	}
+
+	mailPath := config.ConfigInstance().VMailboxBase
+
+	mapsFile := config.ConfigInstance().VMailboxMaps
+	if !utils.FileExists(mapsFile) {
+		return nil, errors.New("Could not read virtual mailbox maps")
+	}
+
+	db, err := berkeleydb.NewDB()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = db.Open(config.ConfigInstance().VMailboxMaps, berkeleydb.DbHash, berkeleydb.DbRdOnly)
+	if err != nil {
+		return nil, errors.New("Unable to open virtual mailbox maps " + mapsFile + " " + err.Error())
+	}
+	defer db.Close()
+
+	cursor, err := db.Cursor()
+	if err != nil {
+		return nil, errors.New("Unable to read virtual mailbox maps " + mapsFile + " " + err.Error())
+	}
+
+	emailMaps := make(map[string]string)
+
+	for true {
+		email, path, dberr := cursor.GetNext()
+		if dberr != nil {
+			break
+		}
+		found := false
+		for _, registredEmail := range registredEmails {
+			if email == registredEmail {
+				found = true
+			}
+		}
+		if !found {
+			return nil, errors.New("Found non-registred mailbox <" + email + "> in mail maps. Database has inconsistancy")
+		}
+		emailMaps[email] = mailPath + "/" + path
+	}
+
+	for _, registredEmail := range registredEmails {
+		if _, exists := emailMaps[registredEmail]; !exists {
+			return nil, errors.New("Found existing mailbox <" + registredEmail + "> in database. Mail maps has inconsistancy")
+		}
+	}
+
+	return emailMaps, nil
+}

+ 2 - 0
go.mod

@@ -11,8 +11,10 @@ require (
 	github.com/gorilla/sessions v1.2.0
 	github.com/gorilla/websocket v1.4.1
 	github.com/jhillyerd/enmime v0.8.0
+	github.com/jsimonetti/berkeleydb v0.0.0-20170815141343-5cde5eaaf78c
 	github.com/lyft/protoc-gen-star v0.4.14 // indirect
 	github.com/pkg/profile v1.4.0
+	github.com/semlanik/berkeleydb v0.0.0-20200324010932-906a014940d9
 	github.com/spf13/afero v1.2.2 // indirect
 	go.mongodb.org/mongo-driver v1.3.0
 	golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d

+ 10 - 0
go.sum

@@ -60,6 +60,8 @@ github.com/jaytaylor/html2text v0.0.0-20190408195923-01ec452cbe43/go.mod h1:CVKl
 github.com/jhillyerd/enmime v0.8.0 h1:PHc/2LXtnDmCDm0V4+5NlBx+MoubmufhuNXwpKSV2o8=
 github.com/jhillyerd/enmime v0.8.0/go.mod h1:MBHs3ugk03NGjMM6PuRynlKf+HA5eSillZ+TRCm73AE=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/jsimonetti/berkeleydb v0.0.0-20170815141343-5cde5eaaf78c h1:/5ja7kk4aMhLJiRTtOhbPYLb9Y95VlqFiRTAoIhrHI4=
+github.com/jsimonetti/berkeleydb v0.0.0-20170815141343-5cde5eaaf78c/go.mod h1:MzZzsYRTOJXLM04/edhWq3Y89ktkIWH0OcenhMCSHfs=
 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
 github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -91,6 +93,14 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
+github.com/semlanik/berkeleydb v0.0.0-20200324005721-8981f6974e5f h1:9RyoumR7T4Q4tIp3Y3t3h47m8KYVdHKQm5UJCK5Fk1A=
+github.com/semlanik/berkeleydb v0.0.0-20200324005721-8981f6974e5f/go.mod h1:HwjJPwsF6GOUdyxwWx1MICyYgt3ppFp2nwOkgr5XHIA=
+github.com/semlanik/berkeleydb v0.0.0-20200324005859-f1ad1141be4d h1:WYw+xVcv5XgzS+dVRp56ydK697rrtB7JhQkC1W5erpo=
+github.com/semlanik/berkeleydb v0.0.0-20200324005859-f1ad1141be4d/go.mod h1:HwjJPwsF6GOUdyxwWx1MICyYgt3ppFp2nwOkgr5XHIA=
+github.com/semlanik/berkeleydb v0.0.0-20200324010442-d983bd2b507f h1:BHwg1t1/FB2JKTS4pXaNs6lPlG+xftf/jqZtTbnKYrs=
+github.com/semlanik/berkeleydb v0.0.0-20200324010442-d983bd2b507f/go.mod h1:HwjJPwsF6GOUdyxwWx1MICyYgt3ppFp2nwOkgr5XHIA=
+github.com/semlanik/berkeleydb v0.0.0-20200324010932-906a014940d9 h1:2zqdU1t/hLKuKuVZspcAUX3rhqeGAZLBj37T6We5EWk=
+github.com/semlanik/berkeleydb v0.0.0-20200324010932-906a014940d9/go.mod h1:HwjJPwsF6GOUdyxwWx1MICyYgt3ppFp2nwOkgr5XHIA=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=

+ 6 - 52
scanner/mailscanner.go

@@ -26,11 +26,9 @@
 package scanner
 
 import (
-	"bufio"
 	"fmt"
 	"log"
 	"os"
-	"strings"
 	"sync"
 
 	"git.semlanik.org/semlanik/gostfix/common"
@@ -105,56 +103,12 @@ func (ms *MailScanner) checkEmailRegistred(email string) bool {
 	return false
 }
 
-func (ms *MailScanner) readEmailMaps() {
-	registredEmails, err := ms.storage.GetAllEmails()
+func (ms *MailScanner) reconfigure() {
+	var err error
+	ms.emailMaps, err = ms.storage.ReadEmailMaps()
 	if err != nil {
-		log.Fatal(err)
-		return
-	}
-
-	mailPath := config.ConfigInstance().VMailboxBase
-
-	emailMaps := make(map[string]string)
-	mapsFile := config.ConfigInstance().VMailboxMaps
-	if !utils.FileExists(mapsFile) {
-		log.Fatal("Could not read virtual mailbox maps")
-		return
-	}
-
-	file, err := os.Open(mapsFile)
-	if err != nil {
-		log.Fatalf("Unable to open virtual mailbox maps %s\n", mapsFile)
-	}
-
-	scanner := bufio.NewScanner(file)
-
-	for scanner.Scan() {
-		emailMapPair := strings.Split(scanner.Text(), " ")
-		if len(emailMapPair) != 2 {
-			log.Printf("Invalid record in virtual mailbox maps %s\n", scanner.Text())
-			continue
-		}
-
-		found := false
-		email := emailMapPair[0]
-		for _, registredEmail := range registredEmails {
-			if email == registredEmail {
-				found = true
-			}
-		}
-		if !found {
-			log.Fatalf("Found non-registred mailbox <%s> in mail maps. Database has inconsistancy.\n", email)
-			return
-		}
-		emailMaps[email] = mailPath + "/" + emailMapPair[1]
-	}
-
-	for _, registredEmail := range registredEmails {
-		if _, exists := emailMaps[registredEmail]; !exists {
-			log.Fatalf("Found existing mailbox <%s> in database. Mail maps has inconsistancy.\n", registredEmail)
-		}
+		log.Fatal(err.Error())
 	}
-	ms.emailMaps = emailMaps
 
 	for mailbox, mailPath := range ms.emailMaps {
 		if !utils.FileExists(mailPath) {
@@ -183,14 +137,14 @@ func (ms *MailScanner) readEmailMaps() {
 
 func (ms *MailScanner) Run() {
 	go func() {
-		ms.readEmailMaps()
+		ms.reconfigure()
 
 		for {
 			select {
 			case signal := <-ms.signalChannel:
 				switch signal {
 				case SignalReconfigure:
-					ms.readEmailMaps()
+					ms.reconfigure()
 				}
 			case event, ok := <-ms.watcher.Events:
 				if !ok {