Add mailboxes integrity check

- Implement mailboxes integriry check
- Make mailscanner responsible for emails parsing and collecting
- Update html templates
- Add mail id to html
- Add view for email addresses

auth/authenticator.go

@@ -32,16 +32,27 @@ import (
 	"strings"
 
 	config "git.semlanik.org/semlanik/gostfix/config"
+	db "git.semlanik.org/semlanik/gostfix/db"
 	utils "git.semlanik.org/semlanik/gostfix/utils"
+	uuid "github.com/google/uuid"
 )
 
 type Authenticator struct {
+	storage  *db.Storage
 	mailMaps map[string]string //TODO: temporary here. Later should be part of mailscanner and never accessed from here
 }
 
 func NewAuthenticator() (a *Authenticator) {
+	storage, err := db.NewStorage()
+
+	if err != nil {
+		log.Fatalf("Unable to intialize user storage %s", err)
+		return nil
+	}
+
 	a = &Authenticator{
 		mailMaps: readMailMaps(), //TODO: temporary here. Later should be part of mailscanner and never accessed from here
+		storage:  storage,
 	}
 	return
 }
@@ -50,18 +61,22 @@ func (a *Authenticator) Authenticate(user, password string) (string, bool) {
 	if !utils.RegExpUtilsInstance().EmailChecker.MatchString(user) {
 		return "", false
 	}
-	_, ok := a.mailMaps[user]
 
-	return "", ok
+	if a.storage.CheckUser(user, password) != nil {
+		return "", false
+	}
+
+	token := uuid.New().String()
+	a.storage.AddToken(user, token)
+	return token, true
 }
 
 func (a *Authenticator) Verify(user, token string) bool {
 	if !utils.RegExpUtilsInstance().EmailChecker.MatchString(user) {
 		return false
 	}
-	_, ok := a.mailMaps[user]
 
-	return ok
+	return a.storage.CheckToken(user, token) == nil
 }
 
 func (a *Authenticator) MailPath(user string) string { //TODO: temporary here. Later should be part of mailscanner and never accessed from here

build.sh

@@ -1,13 +1,16 @@
 export PATH=$PATH:$PWD/bin
 export GOBIN=$PWD/bin
-export RPC_PATH=$PWD/common
+export RPC_PATH=common
 
 go get -u github.com/golang/protobuf/protoc-gen-go@v1.3.4
+go get -u github.com/amsokol/protoc-gen-gotag
 
 # mkdir -p $RPC_PATH
 rm -f $RPC_PATH/*.pb.go
 protoc -I$RPC_PATH --go_out=plugins=grpc:$RPC_PATH $RPC_PATH/gostfix.proto
 
+protoc -I$RPC_PATH --gotag_out=xxx="bson+\"-\"",output_path=$RPC_PATH:. $RPC_PATH/gostfix.proto 
+
 echo "Installing data"
 rm -rf data
 mkdir data

common/gostfix.pb.go

@@ -24,9 +24,9 @@ type MailBody struct {
 	PlainText            string              `protobuf:"bytes,1,opt,name=plainText,proto3" json:"plainText,omitempty"`
 	RichText             string              `protobuf:"bytes,2,opt,name=richText,proto3" json:"richText,omitempty"`
 	Attachments          []*AttachmentHeader `protobuf:"bytes,3,rep,name=attachments,proto3" json:"attachments,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}            `json:"-"`
-	XXX_unrecognized     []byte              `json:"-"`
-	XXX_sizecache        int32               `json:"-"`
+	XXX_NoUnkeyedLiteral struct{}            `json:"-" bson:"-"`
+	XXX_unrecognized     []byte              `json:"-" bson:"-"`
+	XXX_sizecache        int32               `json:"-" bson:"-"`
 }
 
 func (m *MailBody) Reset()         { *m = MailBody{} }
@@ -82,9 +82,9 @@ type MailHeader struct {
 	Bcc                  string   `protobuf:"bytes,4,opt,name=bcc,proto3" json:"bcc,omitempty"`
 	Date                 string   `protobuf:"bytes,5,opt,name=date,proto3" json:"date,omitempty"`
 	Subject              string   `protobuf:"bytes,6,opt,name=subject,proto3" json:"subject,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-" bson:"-"`
+	XXX_unrecognized     []byte   `json:"-" bson:"-"`
+	XXX_sizecache        int32    `json:"-" bson:"-"`
 }
 
 func (m *MailHeader) Reset()         { *m = MailHeader{} }
@@ -157,9 +157,9 @@ func (m *MailHeader) GetSubject() string {
 type Mail struct {
 	Header               *MailHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"`
 	Body                 *MailBody   `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
-	XXX_unrecognized     []byte      `json:"-"`
-	XXX_sizecache        int32       `json:"-"`
+	XXX_NoUnkeyedLiteral struct{}    `json:"-" bson:"-"`
+	XXX_unrecognized     []byte      `json:"-" bson:"-"`
+	XXX_sizecache        int32       `json:"-" bson:"-"`
 }
 
 func (m *Mail) Reset()         { *m = Mail{} }
@@ -204,9 +204,9 @@ func (m *Mail) GetBody() *MailBody {
 type Attachment struct {
 	Header               *AttachmentHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"`
 	Data                 []byte            `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
-	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
-	XXX_unrecognized     []byte            `json:"-"`
-	XXX_sizecache        int32             `json:"-"`
+	XXX_NoUnkeyedLiteral struct{}          `json:"-" bson:"-"`
+	XXX_unrecognized     []byte            `json:"-" bson:"-"`
+	XXX_sizecache        int32             `json:"-" bson:"-"`
 }
 
 func (m *Attachment) Reset()         { *m = Attachment{} }
@@ -251,9 +251,9 @@ func (m *Attachment) GetData() []byte {
 type AttachmentHeader struct {
 	FileName             string   `protobuf:"bytes,1,opt,name=fileName,proto3" json:"fileName,omitempty"`
 	ContentType          string   `protobuf:"bytes,2,opt,name=contentType,proto3" json:"contentType,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-" bson:"-"`
+	XXX_unrecognized     []byte   `json:"-" bson:"-"`
+	XXX_sizecache        int32    `json:"-" bson:"-"`
 }
 
 func (m *AttachmentHeader) Reset()         { *m = AttachmentHeader{} }

common/metadata.go

@@ -0,0 +1,33 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Alexey Edelev <semlanik@gmail.com>
+ *
+ * This file is part of gostfix project https://git.semlanik.org/semlanik/gostfix
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the "Software"), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify,
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies
+ * or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+ * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+package common
+
+type MailMetadata struct {
+	Id   string `bson:"_id"`
+	User string
+	Mail *Mail
+	Read bool
+}

db/db.go

@@ -27,7 +27,10 @@ package db
 
 import (
 	"context"
+	"crypto/sha1"
+	"encoding/hex"
 	"errors"
+	"log"
 	"time"
 
 	common "git.semlanik.org/semlanik/gostfix/common"
@@ -41,9 +44,17 @@ import (
 )
 
 type Storage struct {
-	usersCollection  *mongo.Collection
-	tokensCollection *mongo.Collection
-	emailsCollection *mongo.Collection
+	db                  *mongo.Database
+	usersCollection     *mongo.Collection
+	tokensCollection    *mongo.Collection
+	emailsCollection    *mongo.Collection
+	allEmailsCollection *mongo.Collection
+	mailsCollection     *mongo.Collection
+}
+
+func qualifiedMailCollection(user string) string {
+	sum := sha1.Sum([]byte(user))
+	return "mb" + hex.EncodeToString(sum[:])
 }
 
 func NewStorage() (s *Storage, err error) {
@@ -81,15 +92,19 @@ func NewStorage() (s *Storage, err error) {
 	}
 
 	s = &Storage{
-		usersCollection:  db.Collection("users"),
-		tokensCollection: db.Collection("tokens"),
-		emailsCollection: db.Collection("emails"),
+		db:                  db,
+		usersCollection:     db.Collection("users"),
+		tokensCollection:    db.Collection("tokens"),
+		emailsCollection:    db.Collection("emails"),
+		allEmailsCollection: db.Collection("allEmails"),
+		mailsCollection:     db.Collection("mails"),
 	}
 
 	//Initial database setup
 	s.usersCollection.Indexes().CreateOne(context.Background(), index)
 	s.tokensCollection.Indexes().CreateOne(context.Background(), index)
 	s.emailsCollection.Indexes().CreateOne(context.Background(), index)
+
 	return
 }
 
@@ -133,6 +148,19 @@ func (s *Storage) addEmail(user string, email string, upsert bool) error {
 	if err != nil {
 		return err
 	}
+
+	emails, err := s.GetAllEmails()
+
+	if err != nil {
+		return err
+	}
+
+	for _, existingEmail := range emails {
+		if existingEmail == email {
+			return errors.New("Email exists")
+		}
+	}
+
 	_, err = s.emailsCollection.UpdateOne(context.Background(),
 		bson.M{"user": user},
 		bson.M{"$addToSet": bson.M{"email": email}},
@@ -153,6 +181,7 @@ func (s *Storage) RemoveEmail(user string, email string) error {
 }
 
 func (s *Storage) CheckUser(user, password string) error {
+	log.Printf("Check user: %s %s", user, password)
 	result := struct {
 		User     string
 		Password string
@@ -162,21 +191,85 @@ func (s *Storage) CheckUser(user, password string) error {
 		return errors.New("Invalid user or password")
 	}
 
-	if bcrypt.CompareHashAndPassword([]byte(password), []byte(result.Password)) != nil {
+	if bcrypt.CompareHashAndPassword([]byte(result.Password), []byte(password)) != nil {
 		return errors.New("Invalid user or password")
 	}
 	return nil
 }
 
 func (s *Storage) AddToken(user, token string) error {
+	log.Printf("add token: %s, %s", user, token)
+	s.tokensCollection.UpdateOne(context.Background(),
+		bson.M{"user": user},
+		bson.M{
+			"$addToSet": bson.M{
+				"token": bson.M{
+					"token":  token,
+					"expire": time.Now().Add(time.Hour * 96).Unix(),
+				},
+			},
+		},
+		options.Update().SetUpsert(true))
 	return nil
 }
 
 func (s *Storage) CheckToken(user, token string) error {
-	return nil
+	log.Printf("Check token: %s %s", user, token)
+	if token == "" {
+		return errors.New("Invalid token")
+	}
+
+	cur, err := s.tokensCollection.Aggregate(context.Background(),
+		bson.A{
+			bson.M{"$match": bson.M{"user": user}},
+			bson.M{"$unwind": "$token"},
+			bson.M{"$match": bson.M{"token.token": token}},
+			bson.M{"$project": bson.M{"_id": 0, "token.expire": 1}},
+		})
+
+	if err != nil {
+		log.Fatalln(err)
+		return err
+	}
+
+	defer cur.Close(context.Background())
+	if cur.Next(context.Background()) {
+		result := struct {
+			Token struct {
+				Expire int64
+			}
+		}{}
+
+		err = cur.Decode(&result)
+
+		if err == nil && result.Token.Expire >= time.Now().Unix() {
+			log.Printf("Check token %s expire: %d", user, result.Token.Expire)
+			return nil
+		}
+	}
+
+	return errors.New("Token expired")
 }
 
-func (s *Storage) SaveMail(user string, m *common.Mail) error {
+func (s *Storage) SaveMail(email, folder string, m *common.Mail) error {
+	result := &struct {
+		User string
+	}{}
+
+	s.emailsCollection.FindOne(context.Background(), bson.M{"email": email}).Decode(result)
+
+	mailsCollection := s.db.Collection(qualifiedMailCollection(result.User))
+	mailsCollection.InsertOne(context.Background(), &struct {
+		Email  string
+		Mail   *common.Mail
+		Folder string
+		Read   bool
+	}{
+		Email:  email,
+		Mail:   m,
+		Folder: folder,
+		Read:   false,
+	}, options.InsertOne().SetBypassDocumentValidation(true))
 	return nil
 }
 
@@ -184,8 +277,28 @@ func (s *Storage) RemoveMail(user string, m *common.Mail) error {
 	return nil
 }
 
-func (s *Storage) MailList(user string) ([]*common.MailHeader, error) {
-	return nil, nil
+func (s *Storage) MailList(user, email, folder string) ([]*common.MailMetadata, error) {
+	mailsCollection := s.db.Collection(qualifiedMailCollection(user))
+	cur, err := mailsCollection.Find(context.Background(), bson.M{"email": email})
+
+	if err != nil {
+		return nil, err
+	}
+
+	var headers []*common.MailMetadata
+	for cur.Next(context.Background()) {
+		result := &common.MailMetadata{}
+		err = cur.Decode(result)
+		if err != nil {
+			log.Printf("Unable to read database mail record: %s", err)
+			continue
+		}
+		log.Printf("Add message: %s", result.Id)
+		headers = append(headers, result)
+	}
+
+	log.Printf("Mails read from database: %v", headers)
+	return headers, nil
 }
 
 func (s *Storage) GetMail(user string, header *common.MailHeader) (m *common.Mail, err error) {
@@ -205,5 +318,15 @@ func (s *Storage) GetEmails(user []string) (emails []string, err error) {
 }
 
 func (s *Storage) GetAllEmails() (emails []string, err error) {
-	return nil, nil
+	cur, err := s.allEmailsCollection.Find(context.Background(), bson.M{})
+	if cur.Next(context.Background()) {
+		result := struct {
+			Emails []string
+		}{}
+		err = cur.Decode(&result)
+		if err == nil {
+			return result.Emails, nil
+		}
+	}
+	return nil, err
 }

go.mod

@@ -3,9 +3,14 @@ module git.semlanik.org/semlanik/gostfix
 go 1.12
 
 require (
+	github.com/amsokol/protoc-gen-gotag v0.2.1 // indirect
+	github.com/fatih/structtag v1.2.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/golang/protobuf v1.3.4
+	github.com/google/uuid v1.1.1
 	github.com/gorilla/sessions v1.2.0
+	github.com/lyft/protoc-gen-star v0.4.14 // indirect
+	github.com/spf13/afero v1.2.2 // indirect
 	go.mongodb.org/mongo-driver v1.3.0
 	golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d
 	golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae

go.sum

@@ -1,6 +1,11 @@
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/amsokol/protoc-gen-gotag v0.2.1 h1:N3ovy0PyxxUBXz+jkuJJUA1iNlQ+7a/QVSW361XF0kc=
+github.com/amsokol/protoc-gen-gotag v0.2.1/go.mod h1:VGStdb9DuXf/2T+bgrIqBouu8Hl81egNzVxD/DyEPAg=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4=
+github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
@@ -33,7 +38,10 @@ github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk
 github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYbQ=
@@ -50,6 +58,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lyft/protoc-gen-star v0.4.14 h1:HUkD4H4dYFIgu3Bns/3N6J5GmKHCEGnhYBwNu3fvXgA=
+github.com/lyft/protoc-gen-star v0.4.14/go.mod h1:mE8fbna26u7aEA2QCVvvfBU/ZrPgocG1206xAFPcs94=
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
@@ -57,6 +67,7 @@ github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUr
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -64,12 +75,16 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=

main.go

@@ -26,6 +26,7 @@
 package main
 
 import (
+	"git.semlanik.org/semlanik/gostfix/db"
 	scanner "git.semlanik.org/semlanik/gostfix/scanner"
 	web "git.semlanik.org/semlanik/gostfix/web"
 )
@@ -51,6 +52,16 @@ func (e *GofixEngine) Run() {
 }
 
 func main() {
+	//Bad
+	storage, _ := db.NewStorage()
+	storage.AddUser("semlanik@semlanik.org", "test", "Alexey Edelev")
+	storage.AddUser("junkmail@semlanik.org", "test", "Alexey Edelev")
+	storage.AddUser("git@semlanik.org", "test", "Alexey Edelev")
+	storage.AddEmail("semlanik@semlanik.org", "ci@semlanik.org")
+	storage.AddEmail("semlanik@semlanik.org", "shopping@semlanik.org")
+	storage.AddEmail("semlanik@semlanik.org", "junkmail@semlanik.org")
+	storage.AddEmail("junkmail@semlanik.org", "qqqqq@semlanik.org")
+	storage.AddEmail("junkmail@semlanik.org", "main@semlanik.org")
 	engine := NewGofixEngine()
 	engine.Run()
 }

scanner/mailscanner.go

@@ -32,45 +32,78 @@ import (
 	"os"
 	"strings"
 
+	"git.semlanik.org/semlanik/gostfix/common"
 	config "git.semlanik.org/semlanik/gostfix/config"
+	db "git.semlanik.org/semlanik/gostfix/db"
 	utils "git.semlanik.org/semlanik/gostfix/utils"
 	fsnotify "github.com/fsnotify/fsnotify"
 )
 
+func NewEmail() *common.Mail {
+	return &common.Mail{
+		Header: &common.MailHeader{},
+		Body:   &common.MailBody{},
+	}
+}
+
 type MailScanner struct {
-	watcher  *fsnotify.Watcher
-	mailMaps map[string]string
+	watcher   *fsnotify.Watcher
+	emailMaps map[string]string
+	storage   *db.Storage
 }
 
 func NewMailScanner() (ms *MailScanner) {
-	mailPath := config.ConfigInstance().VMailboxBase
-	fmt.Printf("Add mail folder %s for watching\n", mailPath)
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		log.Fatal(err)
+		return
+	}
+
+	storage, err := db.NewStorage()
+
+	if err != nil {
+		log.Fatal(err)
+		return
 	}
 
 	ms = &MailScanner{
-		watcher:  watcher,
-		mailMaps: readMailMaps(),
+		watcher: watcher,
+		storage: storage,
+	}
+
+	return
+}
+
+func (ms *MailScanner) checkEmailRegistred(email string) bool {
+	emails, err := ms.storage.GetAllEmails()
+
+	if err != nil {
+		return false
 	}
 
-	for _, f := range ms.mailMaps {
-		fullPath := mailPath + "/" + f
-		if utils.FileExists(fullPath) {
-			fmt.Printf("Add mail file %s for watching\n", fullPath)
-			watcher.Add(fullPath)
+	for _, e := range emails {
+		if email == e {
+			return true
 		}
 	}
 
-	return
+	return false
 }
 
-func readMailMaps() map[string]string {
-	mailMaps := make(map[string]string)
+func (ms *MailScanner) readEmailMaps() {
+	registredEmails, err := ms.storage.GetAllEmails()
+	if err != nil {
+		log.Fatal(err)
+		return
+	}
+
+	mailPath := config.ConfigInstance().VMailboxBase
+
+	emailMaps := make(map[string]string)
 	mapsFile := config.ConfigInstance().VMailboxMaps
 	if !utils.FileExists(mapsFile) {
-		return mailMaps
+		log.Fatal("Could not read virtual mailbox maps")
+		return
 	}
 
 	file, err := os.Open(mapsFile)
@@ -81,19 +114,62 @@ func readMailMaps() map[string]string {
 	scanner := bufio.NewScanner(file)
 
 	for scanner.Scan() {
-		mailPathPair := strings.Split(scanner.Text(), " ")
-		if len(mailPathPair) != 2 {
-			log.Printf("Invalid record in virtual mailbox maps %s", scanner.Text())
+		emailMapPair := strings.Split(scanner.Text(), " ")
+		if len(emailMapPair) != 2 {
+			log.Printf("Invalid record in virtual mailbox maps %s\n", scanner.Text())
 			continue
 		}
-		mailMaps[mailPathPair[0]] = mailPathPair[1]
+
+		found := false
+		email := emailMapPair[0]
+		for _, registredEmail := range registredEmails {
+			if email == registredEmail {
+				found = true
+			}
+		}
+		if !found {
+			log.Fatalf("Found non-registred mailbox <%s> in mail maps. Database has inconsistancy.\n", email)
+			return
+		}
+		emailMaps[email] = mailPath + "/" + emailMapPair[1]
 	}
 
-	return mailMaps
+	for _, registredEmail := range registredEmails {
+		if _, exists := emailMaps[registredEmail]; !exists {
+			log.Fatalf("Found existing mailbox <%s> in database. Mail maps has inconsistancy.\n", registredEmail)
+		}
+	}
+	ms.emailMaps = emailMaps
 }
 
 func (ms *MailScanner) Run() {
 	go func() {
+		ms.readEmailMaps()
+
+		for mailbox, mailPath := range ms.emailMaps {
+			if !utils.FileExists(mailPath) {
+				file, err := os.Create(mailPath)
+				if err != nil {
+					fmt.Printf("Unable to create mailbox for watching %s\n", err)
+					continue
+				}
+				file.Close()
+			}
+
+			mails := ms.readMailFile(mailPath)
+			for _, mail := range mails {
+				ms.storage.SaveMail(mailbox, "Inbox", mail)
+			}
+			log.Printf("New email for %s, emails read %d", mailPath, len(mails))
+
+			err := ms.watcher.Add(mailPath)
+			if err != nil {
+				fmt.Printf("Unable to add mailbox for watching\n")
+			} else {
+				fmt.Printf("Add mail file %s for watching\n", mailPath)
+			}
+		}
+
 		for {
 			select {
 			case event, ok := <-ms.watcher.Events:
@@ -101,7 +177,23 @@ func (ms *MailScanner) Run() {
 					return
 				}
 				if event.Op&fsnotify.Write == fsnotify.Write {
-					log.Println("New email for", event.Name)
+					mailPath := event.Name
+					mailbox := ""
+					for k, v := range ms.emailMaps {
+						if v == mailPath {
+							mailbox = k
+						}
+					}
+
+					if mailbox != "" {
+						mails := ms.readMailFile(mailPath)
+						for _, mail := range mails {
+							ms.storage.SaveMail(mailbox, "Inbox", mail)
+						}
+						log.Printf("New email for %s, emails read %d", mailPath, len(mails))
+					} else {
+						log.Printf("Invalid path update triggered: %s", mailPath)
+					}
 				}
 			case err, ok := <-ms.watcher.Errors:
 				if !ok {
@@ -116,3 +208,22 @@ func (ms *MailScanner) Run() {
 func (ms *MailScanner) Stop() {
 	defer ms.watcher.Close()
 }
+
+func (ms *MailScanner) readMailFile(mailPath string) (mails []*common.Mail) {
+	if !utils.FileExists(mailPath) {
+		return nil
+	}
+
+	file, err := utils.OpenAndLockWait(mailPath)
+	if err != nil {
+		return nil
+	}
+	defer file.CloseAndUnlock()
+
+	mails = parseFile(file)
+	if len(mails) > 0 {
+		file.Truncate(0)
+	}
+
+	return mails
+}

scanner/parser.go

@@ -0,0 +1,166 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Alexey Edelev <semlanik@gmail.com>
+ *
+ * This file is part of gostfix project https://git.semlanik.org/semlanik/gostfix
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the "Software"), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify,
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies
+ * or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+ * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+package scanner
+
+import (
+	"bufio"
+	"strings"
+
+	"git.semlanik.org/semlanik/gostfix/common"
+	utils "git.semlanik.org/semlanik/gostfix/utils"
+)
+
+const (
+	StateHeaderScan = iota
+	StateBodyScan
+)
+
+const (
+	AtLeastOneHeaderMask = 1 << iota
+	FromHeaderMask
+	DateHeaderMask
+	ToHeaderMask
+	AllHeaderMask = 15
+)
+
+type parseData struct {
+	state            int
+	mandatoryHeaders int
+	previousHeader   *string
+	email            *common.Mail
+	bodyContentType  string
+	bodyData         string
+	activeBoundary   string
+}
+
+func (pd *parseData) reset() {
+	*pd = parseData{
+		state:            StateHeaderScan,
+		previousHeader:   nil,
+		mandatoryHeaders: 0,
+		email:            NewEmail(),
+		bodyContentType:  "plain/text",
+		bodyData:         "",
+		activeBoundary:   "",
+	}
+}
+
+func parseFile(file *utils.LockedFile) []*common.Mail {
+	var emails []*common.Mail
+
+	pd := &parseData{}
+	pd.reset()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		switch pd.state {
+		case StateHeaderScan:
+			if scanner.Text() == "" {
+				if pd.mandatoryHeaders&AtLeastOneHeaderMask == AtLeastOneHeaderMask { //Cause we read at least one header
+					pd.previousHeader = nil
+					boundaryCapture := utils.RegExpUtilsInstance().BoundaryFinder.FindStringSubmatch(pd.bodyContentType)
+					if len(boundaryCapture) == 2 {
+						pd.activeBoundary = boundaryCapture[1]
+					} else {
+						pd.activeBoundary = ""
+					}
+					pd.state = StateBodyScan
+				}
+			} else {
+				pd.parseHeader(scanner.Text())
+			}
+		case StateBodyScan:
+			if scanner.Text() == "" {
+				if pd.state == StateBodyScan && pd.activeBoundary == "" {
+					if pd.mandatoryHeaders == AllHeaderMask {
+						emails = append(emails, pd.email)
+					}
+					pd.reset()
+					continue
+				}
+			}
+
+			if pd.activeBoundary != "" {
+				pd.bodyData += scanner.Text() + "\n"
+				capture := utils.RegExpUtilsInstance().BoundaryEndFinder.FindStringSubmatch(scanner.Text())
+				if len(capture) == 2 && pd.activeBoundary == capture[1] {
+					pd.state = StateBodyScan
+					pd.activeBoundary = ""
+				}
+			}
+		}
+	}
+
+	if pd.state == StateBodyScan {
+		if pd.mandatoryHeaders == AllHeaderMask {
+			emails = append(emails, pd.email)
+		}
+		pd.reset()
+	}
+	return emails
+}
+
+func (pd *parseData) parseHeader(headerRaw string) {
+	capture := utils.RegExpUtilsInstance().HeaderFinder.FindStringSubmatch(headerRaw)
+	//Parse header
+	if len(capture) == 3 {
+		// fmt.Printf("capture Header %s : %s\n", strings.ToLower(capture[0]), strings.ToLower(capture[1]))
+		header := strings.ToLower(capture[1])
+		pd.mandatoryHeaders |= AtLeastOneHeaderMask
+		switch header {
+		case "from":
+			pd.previousHeader = &pd.email.Header.From
+			pd.mandatoryHeaders |= FromHeaderMask
+		case "to":
+			pd.previousHeader = &pd.email.Header.To
+			pd.mandatoryHeaders |= ToHeaderMask
+		case "cc":
+			pd.previousHeader = &pd.email.Header.Cc
+		case "bcc":
+			pd.previousHeader = &pd.email.Header.Bcc
+			pd.mandatoryHeaders |= ToHeaderMask
+		case "subject":
+			pd.previousHeader = &pd.email.Header.Subject
+		case "date":
+			pd.previousHeader = &pd.email.Header.Date
+			pd.mandatoryHeaders |= DateHeaderMask
+		case "content-type":
+			pd.previousHeader = &pd.bodyContentType
+		default:
+			pd.previousHeader = nil
+		}
+		if pd.previousHeader != nil {
+			*pd.previousHeader += capture[2]
+		}
+		return
+	}
+
+	//Parse folding
+	capture = utils.RegExpUtilsInstance().FoldingFinder.FindStringSubmatch(headerRaw)
+	if len(capture) == 2 && pd.previousHeader != nil {
+		*pd.previousHeader += capture[1]
+	}
+}

utils/fileutils.go

@@ -76,6 +76,10 @@ func (f *LockedFile) Read(p []byte) (n int, err error) {
 	return f.file.Read(p)
 }
 
+func (f *LockedFile) Truncate(size int64) error {
+	return f.file.Truncate(size)
+}
+
 func (f *LockedFile) CloseAndUnlock() error {
 	err1 := unix.FcntlFlock(f.file.Fd(), unix.F_SETLKW, f.lock)
 	err2 := f.file.Close()

views

@@ -0,0 +1 @@
+db.createView("allEmails", "emails", [{$unwind:"$email"},{$group: {_id: null, emails: {$addToSet: "$email"}}}]

web/server.go

@@ -26,16 +26,13 @@
 package web
 
 import (
-	"bufio"
 	"fmt"
 	template "html/template"
 	"log"
 	"net/http"
-	"strings"
 
 	auth "git.semlanik.org/semlanik/gostfix/auth"
 	common "git.semlanik.org/semlanik/gostfix/common"
-	config "git.semlanik.org/semlanik/gostfix/config"
 	db "git.semlanik.org/semlanik/gostfix/db"
 	utils "git.semlanik.org/semlanik/gostfix/utils"
 
@@ -92,12 +89,6 @@ func NewServer() *Server {
 		storage:       storage,
 	}
 
-	s.storage.AddUser("semlanik@semlanik.org", "testpassword", "Alexey Edelev")
-	s.storage.AddUser("junkmail@semlanik.org", "testpassword", "Alexey Edelev")
-	err = s.storage.AddEmail("semlanik@semlanik.org", "ci@semlanik.org")
-	err = s.storage.AddEmail("semlanik@semlanik.org", "shopping@semlanik.org")
-	err = s.storage.AddEmail("semlanik@semlanik.org", "junkmail@semlanik.org")
-	err = s.storage.AddEmail("junkmail@semlanik.org", "qqqqq@semlanik.org")
 	return s
 }
 
@@ -192,141 +183,22 @@ func (s *Server) handleMailbox(w http.ResponseWriter, r *http.Request) {
 	if !s.authenticator.Verify(user, token) {
 		s.logout(w, r)
 		http.Redirect(w, r, "/login", http.StatusTemporaryRedirect)
-	}
-
-	mailPath := config.ConfigInstance().VMailboxBase + "/" + s.authenticator.MailPath(user)
-	if !utils.FileExists(mailPath) {
-		s.logout(w, r)
-		s.error(http.StatusInternalServerError, "Unable to access your mailbox. Please contact Administrator.", w, r)
 		return
 	}
 
-	file, err := utils.OpenAndLockWait(mailPath)
+	mailList, err := s.storage.MailList(user, user, "Inbox")
+
 	if err != nil {
-		s.logout(w, r)
-		s.error(http.StatusInternalServerError, "Unable to access your mailbox. Please contact Administrator.", w, r)
+		s.error(http.StatusInternalServerError, "Couldn't read email database", w, r)
 		return
 	}
-	defer file.CloseAndUnlock()
-
-	scanner := bufio.NewScanner(file)
-	activeBoundary := ""
-	var previousHeader *string = nil
-	var emails []*common.Mail
-	mandatoryHeaders := 0
-	email := NewEmail()
-	state := StateHeaderScan
-	contentType := "plain/text"
-	for scanner.Scan() {
-		if scanner.Text() == "" {
-			if state == StateHeaderScan && mandatoryHeaders&AtLeastOneHeaderMask == AtLeastOneHeaderMask {
-				boundaryCapture := utils.RegExpUtilsInstance().BoundaryFinder.FindStringSubmatch(contentType)
-				if len(boundaryCapture) == 2 {
-					activeBoundary = boundaryCapture[1]
-				} else {
-					activeBoundary = ""
-				}
-				state = StateBodyScan
-				// fmt.Printf("--------------------------Start body scan content type:%s boundary: %s -------------------------\n", contentType, activeBoundary)
-			} else if state == StateBodyScan {
-				// fmt.Printf("--------------------------Previous email-------------------------\n%v\n", email)
-				if activeBoundary == "" {
-					previousHeader = nil
-					activeBoundary = ""
-					// fmt.Printf("Actual headers: %d\n", mandatoryHeaders)
-					if mandatoryHeaders == AllHeaderMask {
-						emails = append(emails, email)
-					}
-					email = NewEmail()
-					contentType = "plain/text"
-					state = StateHeaderScan
-					mandatoryHeaders = 0
-				} else {
-					// fmt.Printf("Still in body scan\n")
-					continue
-				}
-			} else {
-				// fmt.Printf("Empty line in state %d\n", state)
-			}
-		}
-
-		if state == StateHeaderScan {
-			capture := utils.RegExpUtilsInstance().HeaderFinder.FindStringSubmatch(scanner.Text())
-			if len(capture) == 3 {
-				// fmt.Printf("capture Header %s : %s\n", strings.ToLower(capture[0]), strings.ToLower(capture[1]))
-				header := strings.ToLower(capture[1])
-				mandatoryHeaders |= AtLeastOneHeaderMask
-				switch header {
-				case "from":
-					previousHeader = &email.Header.From
-					mandatoryHeaders |= FromHeaderMask
-				case "to":
-					previousHeader = &email.Header.To
-					mandatoryHeaders |= ToHeaderMask
-				case "cc":
-					previousHeader = &email.Header.Cc
-				case "bcc":
-					previousHeader = &email.Header.Bcc
-					mandatoryHeaders |= ToHeaderMask
-				case "subject":
-					previousHeader = &email.Header.Subject
-				case "date":
-					previousHeader = &email.Header.Date
-					mandatoryHeaders |= DateHeaderMask
-				case "content-type":
-					previousHeader = &contentType
-				default:
-					previousHeader = nil
-				}
-				if previousHeader != nil {
-					*previousHeader += capture[2]
-				}
-				continue
-			}
-
-			capture = utils.RegExpUtilsInstance().FoldingFinder.FindStringSubmatch(scanner.Text())
-			if len(capture) == 2 && previousHeader != nil {
-				*previousHeader += capture[1]
-				continue
-			}
-		} else {
-			// email.Body.Content += scanner.Text() + "\n"
-			if activeBoundary != "" {
-				capture := utils.RegExpUtilsInstance().BoundaryEndFinder.FindStringSubmatch(scanner.Text())
-				if len(capture) == 2 {
-					// fmt.Printf("capture Boundary End %s\n", capture[1])
-					if activeBoundary == capture[1] {
-						state = StateBodyScan
-						activeBoundary = ""
-					}
-
-					continue
-				}
-				// capture = boundaryStartFinder.FindStringSubmatch(scanner.Text())
-				// if len(capture) == 2 && activeBoundary == capture[1] {
-				// 	// fmt.Printf("capture Boundary Start %s\n", capture[1])
-				// 	state = StateContentScan
-				// 	continue
-				// }
-			}
-		}
-	}
-
-	if state == StateBodyScan && mandatoryHeaders == AllHeaderMask { //Finalize if body read till EOF
-		// fmt.Printf("--------------------------Previous email-------------------------\n%v\n", email)
-
-		previousHeader = nil
-		activeBoundary = ""
-		emails = append(emails, email)
-		state = StateHeaderScan
-	}
 
 	fmt.Fprint(w, s.templater.ExecuteIndex(&struct {
 		Folders  template.HTML
 		MailList template.HTML
 		Version  template.HTML
 	}{
-		MailList: template.HTML(s.templater.ExecuteMailList(emails)),
+		MailList: template.HTML(s.templater.ExecuteMailList(mailList)),
 		Folders:  "Folders",
 		Version:  common.Version,
 	}))
@@ -350,7 +222,7 @@ func (s *Server) login(user, token string, w http.ResponseWriter, r *http.Reques
 }
 
 func (s *Server) error(code int, text string, w http.ResponseWriter, r *http.Request) {
-	w.WriteHeader(http.StatusInternalServerError)
+	w.WriteHeader(code)
 	fmt.Fprint(w, s.templater.ExecuteError(&struct {
 		Code    int
 		Text    string

web/templates/maillist.html

@@ -3,13 +3,13 @@
         <table class="mailList" >
             <tbody>
                 {{range .}}
-                <tr onclick="openEmail('someid');">
-                    <td class="fromCol">{{.Header.From}}</td>
-                    <td class="subjCol">{{.Header.Subject}}</td>
-                    <td class="dateCol">{{.Header.Date}}</td>
+                <tr id="mail{{.Id}}" class="{{if .Read}}read{{else}}unread{{end}}" onclick="openEmail('{{.Id}}');">
+                    <td class="fromCol">{{.Mail.Header.From}}</td>
+                    <td class="subjCol">{{.Mail.Header.Subject}}</td>
+                    <td class="dateCol">{{.Mail.Header.Date}}</td>
                 </tr>
                 {{else}}
-                <tr><td><b>Mail folder is empty</b></td>></tr>
+                <tr><td><b>Mail folder is empty</b></td></tr>
                 {{end}}
             </tbody>
         </table>