Add websocket notifier

- Add websocket based mailbox update
- Move NewMail function to common package
- Add notification interface for mailscanner updates
- Migrate to go net/mail parser for date
TODO: migrate to js based mail parsing and maillist generator

README.md

@@ -1,5 +1,5 @@
-# gostfix
-
+# gostfix
+
 gostfix is simple go-based mail-manager for postfix with web interface
 
 Supported features:
@@ -8,4 +8,30 @@ Supported features:
 - Web mail interface
 - gRPC admin interface
 - POP3 inteface
-- IMAP interface
+- IMAP interface
+
+# Nginx
+
+```
+    listen 443 ssl;
+    server_name mail.example.com;
+
+    # Add proxy micro-web services
+    location / {
+        proxy_pass http://localhost:65200;
+    }
+
+    # Add web sockets proxy
+    location ~ ^/m[\d]+/notifierSubscribe$ {
+        proxy_pass http://localhost:65200;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "Upgrade";
+        proxy_set_header Host $host;
+    }
+
+
+    # SSL configuration
+    ssl_certificate /path/to/cert.pem;
+    ssl_certificate_key /path/to/privkey.pem;
+```

common/metadata.go → common/mailutils.go

@@ -25,6 +25,13 @@
 
 package common
 
+func NewMail() *Mail {
+	return &Mail{
+		Header: &MailHeader{},
+		Body:   &MailBody{},
+	}
+}
+
 type MailMetadata struct {
 	Id     string `bson:"_id"`
 	User   string

common/notifier.go

@@ -0,0 +1,31 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Alexey Edelev <semlanik@gmail.com>
+ *
+ * This file is part of gostfix project https://git.semlanik.org/semlanik/gostfix
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the "Software"), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify,
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies
+ * or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+ * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+package common
+
+type Notifier interface {
+	NotifyMaiboxUpdate(email string)
+	NotifyNewMail(email string, m Mail)
+}

db/db.go

@@ -483,7 +483,7 @@ func (s *Storage) GetMail(user string, id string) (metadata *common.MailMetadata
 	}
 
 	metadata = &common.MailMetadata{
-		Mail: &common.Mail{},
+		Mail: common.NewMail(),
 	}
 
 	err = mailsCollection.FindOne(context.Background(), bson.M{"_id": oId}).Decode(metadata)

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/golang/protobuf v1.3.4
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/sessions v1.2.0
+	github.com/gorilla/websocket v1.4.1
 	github.com/jhillyerd/enmime v0.8.0
 	github.com/lyft/protoc-gen-star v0.4.14 // indirect
 	github.com/pkg/profile v1.4.0

go.sum

@@ -52,6 +52,8 @@ github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyC
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
 github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYbQ=
 github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
+github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
+github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jaytaylor/html2text v0.0.0-20190408195923-01ec452cbe43 h1:jTkyeF7NZ5oIr0ESmcrpiDgAfoidCBF4F5kJhjtaRwE=
 github.com/jaytaylor/html2text v0.0.0-20190408195923-01ec452cbe43/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=

main.go

@@ -45,6 +45,7 @@ func NewGofixEngine() (e *GofixEngine) {
 		sasl:    sasl.NewSaslServer(),
 	}
 
+	e.scanner.RegisterNotifier(e.web.Notifier)
 	return
 }
 

sasl/sasl.go

@@ -110,7 +110,8 @@ func (s *SaslServer) handleRequest(conn net.Conn) {
 		}
 
 		if err != nil {
-			fmt.Printf("Read error %s\n", err)
+			log.Printf("Read error %s\n", err)
+			break
 		}
 
 		currentMessage := fullbuf

scanner/mailscanner.go

@@ -31,6 +31,7 @@ import (
 	"log"
 	"os"
 	"strings"
+	"sync"
 
 	"git.semlanik.org/semlanik/gostfix/common"
 	config "git.semlanik.org/semlanik/gostfix/config"
@@ -43,18 +44,13 @@ const (
 	SignalReconfigure = iota
 )
 
-func NewEmail() *common.Mail {
-	return &common.Mail{
-		Header: &common.MailHeader{},
-		Body:   &common.MailBody{},
-	}
-}
-
 type MailScanner struct {
 	watcher       *fsnotify.Watcher
 	emailMaps     map[string]string
 	storage       *db.Storage
 	signalChannel chan int
+	notifiers     []common.Notifier
+	notifiersLock sync.Mutex
 }
 
 func NewMailScanner() (ms *MailScanner) {
@@ -83,6 +79,7 @@ func NewMailScanner() (ms *MailScanner) {
 		watcher:       watcher,
 		storage:       storage,
 		signalChannel: make(chan int),
+		notifiers:     []common.Notifier{},
 	}
 
 	return
@@ -208,13 +205,18 @@ func (ms *MailScanner) Run() {
 
 					if mailbox != "" {
 						mails := ms.readMailFile(mailPath)
+						if len(mails) > 0 {
+							ms.notifyMailboxUpdate(mailbox)
+						}
 						for _, mail := range mails {
 							ms.storage.SaveMail(mailbox, common.Inbox, mail, false)
+							ms.notifyNewMail(mailbox, *mail)
 						}
 						log.Printf("New email for %s, emails read %d", mailPath, len(mails))
 					} else {
 						log.Printf("Invalid path update triggered: %s", mailPath)
 					}
+
 				}
 			case err, ok := <-ms.watcher.Errors:
 				if !ok {
@@ -250,3 +252,27 @@ func (ms *MailScanner) readMailFile(mailPath string) (mails []*common.Mail) {
 
 	return mails
 }
+
+func (ms *MailScanner) RegisterNotifier(notifier common.Notifier) {
+	if notifier != nil {
+		ms.notifiersLock.Lock()
+		defer ms.notifiersLock.Unlock()
+		ms.notifiers = append(ms.notifiers, notifier)
+	}
+}
+
+func (ms *MailScanner) notifyNewMail(email string, mail common.Mail) {
+	ms.notifiersLock.Lock()
+	defer ms.notifiersLock.Unlock()
+	for _, notifier := range ms.notifiers {
+		notifier.NotifyNewMail(email, mail)
+	}
+}
+
+func (ms *MailScanner) notifyMailboxUpdate(email string) {
+	ms.notifiersLock.Lock()
+	defer ms.notifiersLock.Unlock()
+	for _, notifier := range ms.notifiers {
+		notifier.NotifyMaiboxUpdate(email)
+	}
+}

scanner/parser.go

@@ -36,6 +36,8 @@ import (
 	"strings"
 	"time"
 
+	"net/mail"
+
 	"git.semlanik.org/semlanik/gostfix/common"
 	"git.semlanik.org/semlanik/gostfix/config"
 	utils "git.semlanik.org/semlanik/gostfix/utils"
@@ -71,7 +73,7 @@ func (pd *parseData) reset() {
 		state:            StateHeaderScan,
 		previousHeader:   nil,
 		mandatoryHeaders: 0,
-		email:            NewEmail(),
+		email:            common.NewMail(),
 		bodyContentType:  "plain/text",
 		bodyData:         "",
 		activeBoundary:   "",
@@ -179,9 +181,10 @@ func (pd *parseData) parseHeader(headerRaw string) {
 			pd.previousHeader = &pd.email.Header.Subject
 		case "date":
 			pd.previousHeader = nil
-			unixTime, err := parseDate(strings.Trim(capture[2], " \t"))
+
+			unixTime, err := mail.ParseDate(strings.Trim(capture[2], " \t")) //parseDate(strings.Trim(capture[2], " \t"))
 			if err == nil {
-				pd.email.Header.Date = unixTime
+				pd.email.Header.Date = unixTime.Unix()
 				pd.mandatoryHeaders |= DateHeaderMask
 			} else {
 				log.Printf("Unable to parse message: %s\n", err)

web/js/index.js

@@ -26,12 +26,11 @@
 var currentFolder = ""
 var currentPage = 0
 var currentMail = ""
-var updateTimerId = null
-var updateInterval = 50000
 var mailbox = ""
 var pageMax = 10
 const mailboxRegex = /^(\/m\d+)/g
 var folders = new Array()
+var notifierSocket = null
 
 $(window).click(function(e){
     var target = $(e.target)
@@ -65,11 +64,8 @@ $(document).ready(function(){
     loadFolders()
     loadStatusLine()
 
-    if (mailbox != "") {
-        clearInterval(updateTimerId)
-    }
-
     $("#mailNewButton").click(mailNew)
+    connectNotifier()
 })
 
 function mailNew(e) {
@@ -116,10 +112,8 @@ function onHashChanged() {
 
     hashParts = hashLocation.split("/")
     if (hashParts.length == 2 && hashParts[1] == "mailNew") {
-        console.log("hashParts: " + hashParts + " length" + hashParts.length + " hashParts[1] " + hashParts[1])
         setMailNewVisible(true)
     } else {
-        console.log("!hashParts: " + hashParts)
         setMailNewVisible(false)
     }
 }
@@ -312,7 +306,6 @@ function setDetailsVisible(visible) {
     if (visible) {
         $("#mailDetails").show()
         $("#mailList").css({pointerEvents: "none"})
-        clearInterval(updateTimerId)
     } else {
         currentMail = ""
         $("#mailDetails").hide()
@@ -325,7 +318,6 @@ function setMailNewVisible(visible) {
     if (visible) {
         $("#mailNew").show()
         $("#mailList").css({pointerEvents: "none"})
-        clearInterval(updateTimerId)
     } else {
         currentMail = ""
         $("#mailNew").hide()
@@ -405,4 +397,33 @@ function sendNewMail() {
 
 function logout() {
     window.location.href = "/logout"
-}
+}
+
+function connectNotifier() {
+    if (notifierSocket != null) {
+        return
+    }
+
+    var protocol = "wss://"
+    if(window.location.protocol  !== "https:") {
+        protocol = "ws://"
+    }
+    notifierSocket = new WebSocket(protocol + window.location.host + mailbox + "/notifierSubscribe")
+    notifierSocket.onopen = function() {
+    };
+    notifierSocket.onmessage = function (e) {
+        for (var i = 0; i < folders.length; i++) {
+            folderStat(folders[i])
+        }
+        updateMailList(currentFolder, currentPage)
+    }
+    notifierSocket.onclose = function () {
+    }
+}
+
+window.onbeforeunload = function() {
+    if (notifierSocket != null) {
+        notifierSocket.onclose = function () {}; // disable onclose handler first
+        notifierSocket.close();
+    }
+};

web/mailbox.go

@@ -87,6 +87,8 @@ func (s *Server) handleMailboxRequest(path, user string, mailbox int, w http.Res
 		s.handleMailList(w, r, user, emails[mailbox])
 	case "sendNewMail":
 		s.handleNewMail(w, r, user, emails[mailbox])
+	case "notifierSubscribe":
+		s.Notifier.handleNotifierRequest(w, r, emails[mailbox])
 	default:
 		http.Redirect(w, r, "/m0", http.StatusTemporaryRedirect)
 	}
@@ -231,7 +233,7 @@ func (s *Server) extractFolder(email string, r *http.Request) string {
 
 func (s *Server) handleNewMail(w http.ResponseWriter, r *http.Request, user, email string) {
 
-	rawMail := common.Mail{
+	rawMail := &common.Mail{
 		Header: &common.MailHeader{
 			From:    email,
 			To:      r.FormValue("to"),
@@ -327,7 +329,7 @@ func (s *Server) handleNewMail(w http.ResponseWriter, r *http.Request, user, ema
 
 	client.Quit()
 
-	s.storage.SaveMail(email, common.Sent, &rawMail, true)
+	s.storage.SaveMail(email, common.Sent, rawMail, true)
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte{0})
 }

web/server.go

@@ -57,19 +57,13 @@ const (
 	CookieSessionToken = "gostfix_session"
 )
 
-func NewEmail() *common.Mail {
-	return &common.Mail{
-		Header: &common.MailHeader{},
-		Body:   &common.MailBody{},
-	}
-}
-
 type Server struct {
 	authenticator *auth.Authenticator
 	fileServer    http.Handler
 	templater     *Templater
 	sessionStore  *sessions.CookieStore
 	storage       *db.Storage
+	Notifier      *webNotifier
 }
 
 func NewServer() *Server {
@@ -87,6 +81,7 @@ func NewServer() *Server {
 		fileServer:    http.FileServer(http.Dir("data")),
 		sessionStore:  sessions.NewCookieStore(make([]byte, 32)),
 		storage:       storage,
+		Notifier:      NewWebNotifier(),
 	}
 
 	return s

web/webnotifier.go

@@ -0,0 +1,127 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Alexey Edelev <semlanik@gmail.com>
+ *
+ * This file is part of gostfix project https://git.semlanik.org/semlanik/gostfix
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the "Software"), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify,
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies
+ * or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+ * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+package web
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"sync"
+
+	"git.semlanik.org/semlanik/gostfix/common"
+	"github.com/gorilla/websocket"
+)
+
+type websocketChannel struct {
+	connection *websocket.Conn
+	channel    chan *common.Mail
+}
+
+type webNotifier struct {
+	notifiers     map[string]*websocketChannel
+	notifiersLock sync.Mutex
+}
+
+func NewWebNotifier() *webNotifier {
+	return &webNotifier{
+		notifiers: make(map[string]*websocketChannel),
+	}
+}
+
+func (wn *webNotifier) NotifyMaiboxUpdate(email string) {
+	if channel, ok := wn.getNotifier(email); ok {
+		channel.channel <- &common.Mail{} //TODO: Dummy notificator for now, later need to make separate interface to handle this
+	}
+}
+
+func (wn *webNotifier) NotifyNewMail(email string, m common.Mail) {
+	// if channel, ok := wn.getNotifier(email); ok {
+	// 	channel.channel <- &m
+	// }
+	//TODO: this functionality needs JS support to create new mails from templates
+}
+
+var upgrader = websocket.Upgrader{
+	ReadBufferSize:  1024,
+	WriteBufferSize: 1024,
+}
+
+func (wn *webNotifier) handleNotifierRequest(w http.ResponseWriter, r *http.Request, email string) {
+	fmt.Printf("New web socket session start %s\n", email)
+	conn, err := upgrader.Upgrade(w, r, nil)
+	if err != nil {
+		http.Error(w, "Could not open websocket connection", http.StatusBadRequest)
+		return
+	}
+
+	c := &websocketChannel{
+		connection: conn,
+		channel:    make(chan *common.Mail, 10),
+	}
+	wn.addNotifier(email, c)
+
+	conn.SetCloseHandler(func(code int, text string) error {
+		fmt.Printf("Web socket session end %s\n", email)
+		wn.removeNotifier(email)
+		conn.Close()
+		return nil
+	})
+
+	go wn.handleNotifications(c)
+}
+
+func (wn *webNotifier) handleNotifications(c *websocketChannel) {
+	//Do nothing for now
+	for {
+		select {
+		case newMail := <-c.channel:
+			err := c.connection.WriteJSON(newMail)
+			if err != nil {
+				log.Println(err.Error())
+				return
+			}
+		}
+	}
+}
+
+func (wn *webNotifier) getNotifier(email string) (channel *websocketChannel, ok bool) {
+	wn.notifiersLock.Lock()
+	defer wn.notifiersLock.Unlock()
+	channel, ok = wn.notifiers[email]
+	return
+}
+
+func (wn *webNotifier) addNotifier(email string, channel *websocketChannel) {
+	wn.notifiersLock.Lock()
+	defer wn.notifiersLock.Unlock()
+	wn.notifiers[email] = channel
+}
+
+func (wn *webNotifier) removeNotifier(email string) {
+	wn.notifiersLock.Lock()
+	defer wn.notifiersLock.Unlock()
+	delete(wn.notifiers, email)
+}