/* * MIT License * * Copyright (c) 2020 Alexey Edelev * * This file is part of gostfix project https://git.semlanik.org/semlanik/gostfix * * Permission is hereby granted, free of charge, to any person obtaining a copy of this * software and associated documentation files (the "Software"), to deal in the Software * without restriction, including without limitation the rights to use, copy, modify, * merge, publish, distribute, sublicense, and/or sell copies of the Software, and * to permit persons to whom the Software is furnished to do so, subject to the following * conditions: * * The above copyright notice and this permission notice shall be included in all copies * or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ package db import ( "context" "crypto/sha1" "encoding/hex" "errors" "fmt" "log" "os" "strings" "sync" "time" common "git.semlanik.org/semlanik/gostfix/common" "git.semlanik.org/semlanik/gostfix/utils" "github.com/semlanik/berkeleydb" bcrypt "golang.org/x/crypto/bcrypt" bson "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo "go.mongodb.org/mongo-driver/mongo" options "go.mongodb.org/mongo-driver/mongo/options" config "git.semlanik.org/semlanik/gostfix/config" ) type StuctNotifiers struct { notifiers []common.Notifier notifiersLock sync.Mutex } var notifiers StuctNotifiers = StuctNotifiers{ notifiers: []common.Notifier{}, } type Storage struct { db *mongo.Database usersCollection *mongo.Collection tokensCollection *mongo.Collection emailsCollection *mongo.Collection allEmailsCollection *mongo.Collection } func qualifiedMailCollection(user string) string { sum := sha1.Sum([]byte(user)) return "mb" + hex.EncodeToString(sum[:]) } func NewStorage() (s *Storage, err error) { fullUrl := "mongodb://" if config.ConfigInstance().MongoUser != "" { fullUrl += config.ConfigInstance().MongoUser if config.ConfigInstance().MongoPassword != "" { fullUrl += ":" + config.ConfigInstance().MongoPassword } fullUrl += "@" } fullUrl += config.ConfigInstance().MongoAddress client, err := mongo.NewClient(options.Client().ApplyURI(fullUrl)) if err != nil { return nil, err } ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() err = client.Connect(ctx) if err != nil { return nil, err } db := client.Database("gostfix") index := mongo.IndexModel{ Keys: bson.M{ "user": 1, }, Options: options.Index().SetUnique(true), } collections, err := db.ListCollectionNames(context.Background(), bson.D{}) if err != nil { log.Fatal(err) } createAllEmails := true for _, collection := range collections { if collection == "allEmails" { createAllEmails = false } log.Println(collection) } if createAllEmails { unwindStage := bson.D{{"$unwind", bson.D{{"path", "$email"}}}} groupStage := bson.D{{"$group", bson.D{{"_id", "null"}, {"emails", bson.D{{"$addToSet", "$email"}}}}}} pipeline := mongo.Pipeline{unwindStage, groupStage} err = db.CreateView(context.Background(), "allEmails", "emails", pipeline) if err != nil { log.Fatal(err) } } s = &Storage{ db: db, usersCollection: db.Collection("users"), tokensCollection: db.Collection("tokens"), emailsCollection: db.Collection("emails"), allEmailsCollection: db.Collection("allEmails"), } //Initial database setup s.usersCollection.Indexes().CreateOne(context.Background(), index) s.tokensCollection.Indexes().CreateOne(context.Background(), index) s.emailsCollection.Indexes().CreateOne(context.Background(), index) return } func (s *Storage) AddUser(user, password, fullName string) error { hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) if err != nil { return err } hashString := string(hash) userInfo := bson.M{ "user": user, "password": hashString, "fullName": fullName, } _, err = s.usersCollection.InsertOne(context.Background(), userInfo) if err != nil { return err } err = s.addEmail(user, user, true) if err != nil { s.usersCollection.DeleteOne(context.Background(), bson.M{"user": user}) return err } return nil } func (s *Storage) UpdateUser(user, password, fullName string) error { userInfo := bson.M{} if len(password) > 0 && len(password) < 128 { hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) if err != nil { return err } hashString := string(hash) userInfo["password"] = hashString } if len(fullName) > 0 && len(fullName) < 128 && utils.RegExpUtilsInstance().FullNameChecker.MatchString(fullName) { userInfo["fullName"] = fullName } if len(userInfo) > 0 { _, err := s.usersCollection.UpdateOne(context.Background(), bson.M{"user": user}, bson.M{"$set": userInfo}) if err != nil { return err } } return nil } func (s *Storage) AddEmail(user string, email string) error { return s.addEmail(user, email, false) } func (s *Storage) addEmail(user string, email string, upsert bool) error { result := struct { User string }{} err := s.usersCollection.FindOne(context.Background(), bson.M{"user": user}).Decode(&result) if err != nil { return err } emails, err := s.GetAllEmails() if err != nil { return err } for _, existingEmail := range emails { if existingEmail == email { return errors.New("Email exists") } } emailParts := strings.Split(email, "@") if len(emailParts) != 2 { return errors.New("Invalid email format") } db, err := berkeleydb.NewDB() if err != nil { log.Fatal(err) } err = db.Open(config.ConfigInstance().VMailboxMaps, berkeleydb.DbHash, 0) if err != nil { log.Fatalf("Unable to open virtual mailbox maps %s %s\n", config.ConfigInstance().VMailboxMaps, err) } defer db.Close() err = db.Put(email, emailParts[1]+"/"+emailParts[0]) if err != nil { return errors.New("Unable to add email to maps" + err.Error()) } _, err = s.emailsCollection.UpdateOne(context.Background(), bson.M{"user": user}, bson.M{"$addToSet": bson.M{"email": email}}, options.Update().SetUpsert(upsert)) return err } func (s *Storage) RemoveEmail(user string, email string) error { log.Printf("User %s removes email %s", user, email) result := s.emailsCollection.FindOne(context.Background(), bson.M{ "user": user, "email": email, }) if result.Err() != nil { return result.Err() } db, err := berkeleydb.NewDB() if err != nil { log.Fatal(err) } err = db.Open(config.ConfigInstance().VMailboxMaps, berkeleydb.DbHash, 0) if err != nil { log.Fatalf("Unable to open virtual mailbox maps %s %s\n", config.ConfigInstance().VMailboxMaps, err) } defer db.Close() err = db.Delete(email) if err != nil { return errors.New("Unable to remove email from maps" + err.Error()) } err = s.cleanupAttachments(user, email) if err != nil { log.Printf("Unable to cleanup attachments for %s %s\n", email, err) } mailsCollection := s.db.Collection(qualifiedMailCollection(user)) mailsCollection.DeleteMany(context.Background(), bson.M{"email": email}) _, err = s.emailsCollection.UpdateOne(context.Background(), bson.M{"user": user}, bson.M{"$pull": bson.M{"email": email}}) return err } func (s *Storage) SaveMail(email, folder string, m *common.Mail, read bool) error { user := &struct { User string }{} s.emailsCollection.FindOne(context.Background(), bson.M{"email": email}).Decode(user) mailsCollection := s.db.Collection(qualifiedMailCollection(user.User)) result, err := mailsCollection.InsertOne(context.Background(), &struct { Email string Mail *common.Mail Folder string Read bool Trash bool }{ Email: email, Mail: m, Folder: folder, Read: read, Trash: false, }, options.InsertOne().SetBypassDocumentValidation(true)) if err != nil { return err } mail := *m //deep copy for multithreading s.notifyNewMail(email, common.MailMetadata{ Id: result.InsertedID.(primitive.ObjectID).Hex(), Read: false, Trash: false, Folder: folder, User: user.User, Mail: &mail, }) stats, err := s.GetEmailStats(user.User, email, folder) if err == nil { s.notifyMailboxUpdate(email, []common.FolderStat{stats}) } return nil } func (s *Storage) DeleteMail(user string, mailId string) error { mailsCollection := s.db.Collection(qualifiedMailCollection(user)) oId, err := primitive.ObjectIDFromHex(mailId) if err != nil { return err } var result common.MailMetadata err = mailsCollection.FindOne(context.Background(), bson.M{"_id": oId}).Decode(&result) if err != nil { return err } for _, attachment := range result.Mail.Body.Attachments { removeAttachment(attachment.Id) } _, err = mailsCollection.DeleteOne(context.Background(), bson.M{"_id": oId}) stats, errTemp := s.GetEmailStats(user, result.Email, common.Trash) if errTemp == nil { s.notifyMailboxUpdate(result.Email, []common.FolderStat{stats}) } return err } func (s *Storage) GetMailList(user, email, folder string, frame common.Frame) ([]*common.MailMetadata, error) { mailsCollection := s.db.Collection(qualifiedMailCollection(user)) matchFilter := bson.M{"email": email} if folder == common.Trash { matchFilter["$or"] = bson.A{ bson.M{"trash": true}, bson.M{"folder": folder}, //TODO: Legacy for old databases remove soon } } else { matchFilter["folder"] = folder matchFilter["$or"] = bson.A{ bson.M{"trash": false}, bson.M{"trash": bson.M{"$exists": false}}, //TODO: Legacy for old databases remove soon } } request := bson.A{ bson.M{"$match": matchFilter}, bson.M{"$sort": bson.M{"mail.header.date": -1}}, } if frame.Skip > 0 { request = append(request, bson.M{"$skip": frame.Skip}) } fmt.Printf("Trying limit number of mails: %v\n", frame) if frame.Limit > 0 { fmt.Printf("Limit number of mails: %v\n", frame) request = append(request, bson.M{"$limit": frame.Limit}) } cur, err := mailsCollection.Aggregate(context.Background(), request) if err != nil { log.Println(err.Error()) return nil, err } var headers []*common.MailMetadata for cur.Next(context.Background()) { result := &common.MailMetadata{} err = cur.Decode(result) if err != nil { log.Printf("Unable to read database mail record: %s", err) continue } // fmt.Printf("Add mail: %s", result.Id) headers = append(headers, result) } // fmt.Printf("Mails read from database: %v", headers) return headers, nil } func (s *Storage) GetUserInfo(user string) (*common.UserInfo, error) { result := &common.UserInfo{} err := s.usersCollection.FindOne(context.Background(), bson.M{"user": user}).Decode(result) return result, err } func (s *Storage) GetEmailStats(user string, email string, folder string) (stat common.FolderStat, err error) { stat = common.FolderStat{ Folder: folder, } mailsCollection := s.db.Collection(qualifiedMailCollection(user)) matchFilter := bson.M{"email": email} if folder == common.Trash { matchFilter["$or"] = bson.A{ bson.M{"trash": true}, bson.M{"folder": folder}, //TODO: Legacy for old databases remove soon } } else { matchFilter["folder"] = folder matchFilter["$or"] = bson.A{ bson.M{"trash": false}, bson.M{"trash": bson.M{"$exists": false}}, //TODO: Legacy for old databases remove soon } } cur, err := mailsCollection.Aggregate(context.Background(), bson.A{bson.M{"$match": matchFilter}, bson.M{"$count": "total"}}) if err == nil && cur.Next(context.Background()) { cur.Decode(&stat) } else { return } matchFilter["read"] = false cur, err = mailsCollection.Aggregate(context.Background(), bson.A{bson.M{"$match": matchFilter}, bson.M{"$count": "unread"}}) if err == nil && cur.Next(context.Background()) { cur.Decode(&stat) } else { return } return } func (s *Storage) GetMail(user string, id string) (metadata *common.MailMetadata, err error) { mailsCollection := s.db.Collection(qualifiedMailCollection(user)) oId, err := primitive.ObjectIDFromHex(id) if err != nil { return nil, err } metadata = &common.MailMetadata{ Mail: common.NewMail(), } err = mailsCollection.FindOne(context.Background(), bson.M{"_id": oId}).Decode(metadata) if err != nil { return nil, err } return metadata, nil } func (s *Storage) SetRead(user string, id string, read bool) error { mailsCollection := s.db.Collection(qualifiedMailCollection(user)) oId, err := primitive.ObjectIDFromHex(id) if err != nil { return err } _, err = mailsCollection.UpdateOne(context.Background(), bson.M{"_id": oId}, bson.M{"$set": bson.M{"read": read}}) s.notifyMailboxUpdateForMail(user, id) return err } func (s *Storage) UpdateMail(user string, id string, mailMap interface{}) error { mailsCollection := s.db.Collection(qualifiedMailCollection(user)) oId, err := primitive.ObjectIDFromHex(id) if err != nil { return err } fromFolder := "" metadata, err := s.GetMail(user, id) if err == nil { if metadata.Trash { fromFolder = common.Trash } else { fromFolder = metadata.Folder } } else { log.Printf("Unable to get mail info to update folder statistics %s", err) } _, err = mailsCollection.UpdateOne(context.Background(), bson.M{"_id": oId}, bson.M{"$set": mailMap}) s.notifyMailboxUpdateForMail(user, id, fromFolder) return err } func (s *Storage) GetUsers() (users []string, err error) { return nil, nil } func (s *Storage) GetEmails(user string) (emails []string, err error) { result := &struct { Email []string }{} err = s.emailsCollection.FindOne(context.Background(), bson.M{"user": user}).Decode(result) if err != nil { return nil, err } return result.Email, nil } func (s *Storage) GetAllEmails() (emails []string, err error) { cur, err := s.allEmailsCollection.Find(context.Background(), bson.M{}) if cur.Next(context.Background()) { result := struct { Emails []string }{} err = cur.Decode(&result) if err == nil { return result.Emails, nil } } return nil, err } func (s *Storage) CheckEmailExists(email string) bool { result := s.allEmailsCollection.FindOne(context.Background(), bson.M{"emails": email}) return result.Err() == nil } func (s *Storage) GetFolders(email string) (folders []*common.Folder) { folders = []*common.Folder{ {Name: common.Inbox, Custom: false}, {Name: common.Sent, Custom: false}, {Name: common.Trash, Custom: false}, {Name: common.Spam, Custom: false}, } return } func (s *Storage) ReadEmailMaps() (map[string]string, error) { registredEmails, err := s.GetAllEmails() if err != nil { return nil, err } mailPath := config.ConfigInstance().VMailboxBase mapsFile := config.ConfigInstance().VMailboxMaps db, err := berkeleydb.NewDB() if err != nil { log.Fatal(err) } if !utils.FileExists(config.ConfigInstance().VMailboxMaps) { err = db.Open(config.ConfigInstance().VMailboxMaps, berkeleydb.DbHash, berkeleydb.DbCreate) } else { err = db.Open(config.ConfigInstance().VMailboxMaps, berkeleydb.DbHash, berkeleydb.DbRdOnly) } if err != nil { return nil, errors.New("Unable to open virtual mailbox maps " + mapsFile + " " + err.Error()) } defer db.Close() cursor, err := db.Cursor() if err != nil { return nil, errors.New("Unable to read virtual mailbox maps " + mapsFile + " " + err.Error()) } emailMaps := make(map[string]string) for true { email, path, dberr := cursor.GetNext() if dberr != nil { break } found := false for _, registredEmail := range registredEmails { if email == registredEmail { found = true } } if !found { return nil, errors.New("Found non-registred mailbox <" + email + "> in mail maps. Database has inconsistancy") } emailMaps[email] = mailPath + "/" + path } for _, registredEmail := range registredEmails { if _, exists := emailMaps[registredEmail]; !exists { return nil, errors.New("Found existing mailbox <" + registredEmail + "> in database. Mail maps has inconsistancy") } } return emailMaps, nil } func removeAttachment(attachmentId string) error { attachmentPath := config.ConfigInstance().AttachmentsPath + "/" + attachmentId err := os.Remove(attachmentPath) if err != nil { log.Printf("Unable to remove attachment file: %s. Database inconsistency", attachmentPath) } return err } func (s *Storage) cleanupAttachments(user, email string) error { mailsCollection := s.db.Collection(qualifiedMailCollection(user)) cur, err := mailsCollection.Aggregate(context.Background(), bson.A{ bson.M{"$match": bson.M{"email": email}}, bson.M{"$project": bson.M{"mail.body.attachments": 1}}, bson.M{"$unwind": "$mail.body.attachments"}, bson.M{"$replaceRoot": bson.M{"newRoot": "$mail.body.attachments"}}, }) if err != nil { return err } for cur.Next(context.Background()) { var attachment common.AttachmentHeader err = cur.Decode(&attachment) if err != nil { log.Printf("Unable to decode attachment") } removeAttachment(attachment.Id) } return nil } func (s *Storage) CheckAttachment(user, attachment string) bool { mailsCollection := s.db.Collection(qualifiedMailCollection(user)) result := mailsCollection.FindOne(context.Background(), bson.M{"mail.body.attachments.id": attachment}) return result.Err() == nil } func (s *Storage) RegisterNotifier(notifier common.Notifier) { if notifier != nil { notifiers.notifiersLock.Lock() defer notifiers.notifiersLock.Unlock() notifiers.notifiers = append(notifiers.notifiers, notifier) } } func (s *Storage) notifyNewMail(email string, mail common.MailMetadata) { notifiers.notifiersLock.Lock() defer notifiers.notifiersLock.Unlock() for _, notifier := range notifiers.notifiers { notifier.NotifyNewMail(email, mail) } } func (s *Storage) notifyMailboxUpdate(email string, stats []common.FolderStat) { notifiers.notifiersLock.Lock() defer notifiers.notifiersLock.Unlock() for _, notifier := range notifiers.notifiers { notifier.NotifyMaiboxUpdate(email, stats) } } func (s *Storage) notifyMailboxUpdateForMail(user, id string, folders ...string) { metadata, err := s.GetMail(user, id) if err != nil { log.Printf("Unable to get mail metadata to update mailbox stat %v\n", err) return } if metadata.Trash { folders = append(folders, common.Trash) } var stats []common.FolderStat stat, err := s.GetEmailStats(user, metadata.Email, metadata.Folder) stats = append(stats, stat) for _, folder := range folders { if folder == metadata.Folder { continue } stat, err = s.GetEmailStats(user, metadata.Email, folder) if err == nil { stats = append(stats, stat) } else { log.Printf("Unable to update mailbox stat %v\n", err) } } s.notifyMailboxUpdate(metadata.Email, stats) }