feat: save email contents to local files

backends/p_content_parser.go

@@ -0,0 +1,44 @@
+package backends
+
+import (
+	"github.com/phires/go-guerrilla/mail"
+)
+
+// ----------------------------------------------------------------------------------
+// Processor Name: contentparser
+// ----------------------------------------------------------------------------------
+// Description   : Parses and decodes the content
+// ----------------------------------------------------------------------------------
+// Config Options: none
+// --------------:-------------------------------------------------------------------
+// Input         : envelope
+// ----------------------------------------------------------------------------------
+// Output        : Content will be populated in e.EnvelopeBody
+// ----------------------------------------------------------------------------------
+func init() {
+	processors["contentparser"] = func() Decorator {
+		return ContentParser()
+	}
+}
+
+func ContentParser() Decorator {
+
+
+	return func(p Processor) Processor {
+		return ProcessWith(func(e *mail.Envelope, task SelectTask) (Result, error) {
+			if task == TaskSaveMail {
+				if err := e.ParseContent(); err != nil {
+					Log().WithError(err).Error("parse content error")
+				} else {
+					Log().Info("Parsed Content is: ", e.EnvelopeBody)
+				}
+				// next processor
+				return p.Process(e, task)
+			} else {
+				// next processor
+				return p.Process(e, task)
+			}
+		})
+	}
+}
+

backends/p_local_files.go

@@ -0,0 +1,60 @@
+package backends
+
+import (
+	"github.com/phires/go-guerrilla/mail"
+)
+
+// ----------------------------------------------------------------------------------
+// Processor Name: localfiles
+// ----------------------------------------------------------------------------------
+// Description   : Dump the decoded content to local files
+// ----------------------------------------------------------------------------------
+// Config Options: Specify the location path to save the parts of the email
+// --------------:-------------------------------------------------------------------
+// Input         : envelope
+// ----------------------------------------------------------------------------------
+// Output        : Saved paths will be populated in e.LocalFilesPaths
+// ----------------------------------------------------------------------------------
+func init() {
+	processors["localfiles"] = func() Decorator {
+		return LocalFiles()
+	}
+}
+
+type LocalFilesProcessorConfig struct {
+	LocalStoragePath     string `json:"local_storage_path"`
+}
+
+func LocalFiles() Decorator {
+
+	var config *LocalFilesProcessorConfig
+
+	Svc.AddInitializer(InitializeWith(func(backendConfig BackendConfig) error {
+		configType := BaseConfig(&LocalFilesProcessorConfig{})
+		bcfg, err := Svc.ExtractConfig(backendConfig, configType)
+		if err != nil {
+			return err
+		}
+		config = bcfg.(*LocalFilesProcessorConfig)
+		return nil
+	}))
+
+
+	return func(p Processor) Processor {
+		return ProcessWith(func(e *mail.Envelope, task SelectTask) (Result, error) {
+			if task == TaskSaveMail {
+				if err := e.SaveLocalFiles(config.LocalStoragePath); err != nil {
+					Log().WithError(err).Error("save local file error")
+				}  else {
+					Log().Info("Dumped Content is: ", e.LocalFilesPaths)
+				}
+				// next processor
+				return p.Process(e, task)
+			} else {
+				// next processor
+				return p.Process(e, task)
+			}
+		})
+	}
+}
+

go.mod

@@ -15,8 +15,17 @@ require (
 
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect
+	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 // indirect
+	github.com/jhillyerd/enmime v1.2.0 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/stretchr/testify v1.9.0 // indirect
 	golang.org/x/sys v0.21.0 // indirect
 	golang.org/x/text v0.16.0 // indirect

go.sum

@@ -2,6 +2,8 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM=
 github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII=
+github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI=
+github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -11,16 +13,32 @@ github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrt
 github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
+github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
+github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
 github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
 github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
 github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s=
 github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQyktQ5+f3dMVZfwD2KWJUgm7M0gdL9NGr8KA=
+github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
+github.com/jhillyerd/enmime v1.2.0 h1:dIu1IPEymQgoT2dzuB//ttA/xcV40NMPpQtmd4wslHk=
+github.com/jhillyerd/enmime v1.2.0/go.mod h1:FRFuUPCLh8PByQv+8xRcLO9QHqaqTqreYhopv5eyk4I=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/pires/go-proxyproto v0.7.0 h1:IukmRewDQFWC7kfnb66CSomk2q/seBuilHBYFwyq0Hs=
 github.com/pires/go-proxyproto v0.7.0/go.mod h1:Vz/1JPY/OACxWGQNIRY2BeyDmpoaWmEP40O9LbuiFR4=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
+github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
@@ -30,6 +48,8 @@ github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
+github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=

goguerrilla.conf.sample

@@ -15,7 +15,8 @@
         "save_process" : "HeadersParser|Header|Debugger",
         "primary_mail_host" : "mail.example.com",
         "gw_save_timeout" : "30s",
-        "gw_val_rcpt_timeout" : "3s"
+        "gw_val_rcpt_timeout" : "3s",
+        "local_storage_path" : "/tmp/go-guerrilla/"
     },
     "servers" : [
         {

mail/envelope.go

@@ -10,10 +10,13 @@ import (
 	"mime"
 	"net"
 	"net/textproto"
+	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/jhillyerd/enmime"
 	"github.com/phires/go-guerrilla/mail/rfc5321"
 )
 
@@ -119,6 +122,12 @@ func NewAddress(str string) (*Address, error) {
 	return a, nil
 }
 
+type LocalFilesPaths struct {
+	PreferredDisplay string
+	Path             string
+}
+
+
 // Envelope of Email represents a single SMTP message.
 type Envelope struct {
 	// Remote IP address
@@ -133,6 +142,10 @@ type Envelope struct {
 	Data bytes.Buffer
 	// Subject stores the subject of the email, extracted and decoded after calling ParseHeaders()
 	Subject string
+    // EnvelopeBody stores the parsed email content
+	EnvelopeBody enmime.Envelope
+	// LocalFilesPaths stores the path to the part files, extracted after calling ParseContent
+	LocalFilesPaths []LocalFilesPaths
 	// TLS is true if the email was received using a TLS connection
 	TLS bool
 	// Header stores the results from ParseHeaders()
@@ -203,6 +216,100 @@ func (e *Envelope) ParseHeaders() error {
 	return err
 }
 
+// Retrieve the content of the email and save
+// each raw or MIME part to the local file system
+func (e *Envelope) ParseContent() error {
+	if e.Header == nil {
+		return errors.New("headers not parsed")
+	}
+
+	// Clear the Content to prevent accumulation
+	e.EnvelopeBody = enmime.Envelope{}
+
+	// Parse message body with enmime.
+	env, err := enmime.ReadEnvelope(bytes.NewReader(e.Data.Bytes()))
+	if err != nil {
+		fmt.Print(err)
+		return err
+	}
+
+	e.EnvelopeBody = *env
+	return nil
+}
+
+
+// Retrieve the content of the email and save
+// each raw or MIME part to the local file system
+func (e *Envelope) SaveLocalFiles(storagePath string) error {
+	if e.Header == nil {
+		return errors.New("headers not parsed")
+	}
+
+	if len(e.EnvelopeBody.Text) == 0 && len(e.EnvelopeBody.HTML) == 0 && len(e.EnvelopeBody.Inlines) == 0 && len(e.EnvelopeBody.Attachments) == 0 {
+		return errors.New("content not parsed")
+	}
+
+	// Clear LocalFilesPaths to prevent accumulation
+	e.LocalFilesPaths = []LocalFilesPaths{}
+
+	path := storagePath + "/" + e.QueuedId
+
+	// Sanitize the path
+	path = filepath.Clean(path)
+
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		os.MkdirAll(path, 0755)
+	}
+
+	var localFilesPaths []LocalFilesPaths
+
+	if len(e.EnvelopeBody.Text) > 0 {
+		file_path := path + "/body.txt"
+		err := WriteFile(file_path, []byte(e.EnvelopeBody.Text))
+		if err != nil {
+			return err
+		}
+		localFilesPaths = append(localFilesPaths, LocalFilesPaths{PreferredDisplay: "PLAIN", Path: file_path})
+	}
+
+	if len(e.EnvelopeBody.HTML) > 0 {
+		file_path := path + "/body.html"
+		err := WriteFile(file_path, []byte(e.EnvelopeBody.HTML))
+		if err != nil {
+			return err
+		}
+		localFilesPaths = append(localFilesPaths, LocalFilesPaths{PreferredDisplay: "HTML", Path: file_path})
+	}
+
+	if len(e.EnvelopeBody.Inlines) > 0 {
+		for i, inline := range e.EnvelopeBody.Inlines {
+			fileName := BuildFileName(inline, "inline_"+fmt.Sprintf("%d", i), i)
+			file_path := path + "/" + fileName
+			err := WriteFile(file_path, inline.Content)
+			if err != nil {
+				return err
+			}
+			localFilesPaths = append(localFilesPaths, LocalFilesPaths{PreferredDisplay: "INLINE", Path: file_path})
+		}
+	}
+
+	if len(e.EnvelopeBody.Attachments) > 0 {
+		for i, attachment := range e.EnvelopeBody.Attachments {
+			fileName := BuildFileName(attachment, "attachment_"+fmt.Sprintf("%d", i), i)
+			file_path := path + "/" + fileName
+			err := WriteFile(file_path, attachment.Content)
+			if err != nil {
+				return err
+			}
+			localFilesPaths = append(localFilesPaths, LocalFilesPaths{PreferredDisplay: "ATTACHMENT", Path: file_path})
+		}
+	}
+
+	e.LocalFilesPaths = localFilesPaths
+
+	return nil
+}
+
 // Len returns the number of bytes that would be in the reader returned by NewReader()
 func (e *Envelope) Len() int {
 	return len(e.DeliveryHeader) + e.Data.Len()

mail/utils.go

@@ -0,0 +1,60 @@
+package mail
+
+import (
+	"os"
+	"mime"
+	"strings"
+	"github.com/jhillyerd/enmime"
+	"fmt"
+)
+
+
+// Wite a file with the given content
+func WriteFile(file_path string, decodedContent []byte) error {
+	file, err := os.Create(file_path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, err = file.WriteString(string(decodedContent))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// BuildFileName builds a file name for a MIME part
+// If the name is provided in the Content-Disposition header, it will be used.
+// Otherwise, a name will be generated based on the radix and index.
+func BuildFileName(part *enmime.Part, radix string, index int) (filename string) {
+
+	filename = part.FileName
+	if len(filename) > 0 {
+		return filename
+	}
+
+	mediaType, _, err := mime.ParseMediaType(part.Header.Get("Content-Type"))
+	if err == nil {
+		mime_type, e := mime.ExtensionsByType(mediaType)
+
+		// Remove any extension not starting with a dot - it should not happen but it might happen
+		for i, e := range mime_type {
+			if !strings.HasPrefix(e, ".") {
+				mime_type = append(mime_type[:i], mime_type[i+1:]...)
+			}
+		}
+
+		if e == nil {
+			if len(mime_type) > 0 { // Arbitrary take the first extension found
+				return fmt.Sprintf("%s-%d-autogeneratedfilename%s", radix, index, mime_type[0])
+			} else {
+				return fmt.Sprintf("%s-%d-autogeneratedfilename.unidentified", radix, index)
+			}
+		} else { // No extension found
+			return fmt.Sprintf("%s-%d-autogeneratedfilename.unknown", radix, index)
+		}
+	}
+
+	return fmt.Sprintf("%s-%d-autogeneratedfilename.unspecified", radix, index)
+}