mox/webapisrv/server_test.go

509 lines
20 KiB
Go
Raw Normal View History

add a webapi and webhooks for a simple http/json-based api for applications to compose/send messages, receive delivery feedback, and maintain suppression lists. this is an alternative to applications using a library to compose messages, submitting those messages using smtp, and monitoring a mailbox with imap for DSNs, which can be processed into the equivalent of suppression lists. but you need to know about all these standards/protocols and find libraries. by using the webapi & webhooks, you just need a http & json library. unfortunately, there is no standard for these kinds of api, so mox has made up yet another one... matching incoming DSNs about deliveries to original outgoing messages requires keeping history of "retired" messages (delivered from the queue, either successfully or failed). this can be enabled per account. history is also useful for debugging deliveries. we now also keep history of each delivery attempt, accessible while still in the queue, and kept when a message is retired. the queue webadmin pages now also have pagination, to show potentially large history. a queue of webhook calls is now managed too. failures are retried similar to message deliveries. webhooks can also be saved to the retired list after completing. also configurable per account. messages can be sent with a "unique smtp mail from" address. this can only be used if the domain is configured with a localpart catchall separator such as "+". when enabled, a queued message gets assigned a random "fromid", which is added after the separator when sending. when DSNs are returned, they can be related to previously sent messages based on this fromid. in the future, we can implement matching on the "envid" used in the smtp dsn extension, or on the "message-id" of the message. using a fromid can be triggered by authenticating with a login email address that is configured as enabling fromid. suppression lists are automatically managed per account. if a delivery attempt results in certain smtp errors, the destination address is added to the suppression list. future messages queued for that recipient will immediately fail without a delivery attempt. suppression lists protect your mail server reputation. submitted messages can carry "extra" data through the queue and webhooks for outgoing deliveries. through webapi as a json object, through smtp submission as message headers of the form "x-mox-extra-<key>: value". to make it easy to test webapi/webhooks locally, the "localserve" mode actually puts messages in the queue. when it's time to deliver, it still won't do a full delivery attempt, but just delivers to the sender account. unless the recipient address has a special form, simulating a failure to deliver. admins now have more control over the queue. "hold rules" can be added to mark newly queued messages as "on hold", pausing delivery. rules can be about certain sender or recipient domains/addresses, or apply to all messages pausing the entire queue. also useful for (local) testing. new config options have been introduced. they are editable through the admin and/or account web interfaces. the webapi http endpoints are enabled for newly generated configs with the quickstart, and in localserve. existing configurations must explicitly enable the webapi in mox.conf. gopherwatch.org was created to dogfood this code. it initially used just the compose/smtpclient/imapclient mox packages to send messages and process delivery feedback. it will get a config option to use the mox webapi/webhooks instead. the gopherwatch code to use webapi/webhook is smaller and simpler, and developing that shaped development of the mox webapi/webhooks. for issue #31 by cuu508
2024-04-15 22:49:02 +03:00
package webapisrv
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"net/textproto"
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/queue"
"github.com/mjl-/mox/store"
"github.com/mjl-/mox/webapi"
"github.com/mjl-/mox/webhook"
)
var ctxbg = context.Background()
func tcheckf(t *testing.T, err error, format string, args ...any) {
t.Helper()
if err != nil {
t.Fatalf("%s: %s", fmt.Sprintf(format, args...), err)
}
}
func tcompare(t *testing.T, got, expect any) {
t.Helper()
if !reflect.DeepEqual(got, expect) {
t.Fatalf("got:\n%#v\nexpected:\n%#v", got, expect)
}
}
func terrcode(t *testing.T, err error, code string) {
t.Helper()
if err == nil {
t.Fatalf("no error, expected error with code %q", code)
}
if xerr, ok := err.(webapi.Error); !ok {
t.Fatalf("got %v, expected webapi error with code %q", err, code)
} else if xerr.Code != code {
t.Fatalf("got error code %q, expected %q", xerr.Code, code)
}
}
func TestServer(t *testing.T) {
mox.LimitersInit()
os.RemoveAll("../testdata/webapisrv/data")
mox.Context = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/webapisrv/mox.conf")
mox.MustLoadConfig(true, false)
defer store.Switchboard()()
err := queue.Init()
tcheckf(t, err, "queue init")
defer queue.Shutdown()
add a webapi and webhooks for a simple http/json-based api for applications to compose/send messages, receive delivery feedback, and maintain suppression lists. this is an alternative to applications using a library to compose messages, submitting those messages using smtp, and monitoring a mailbox with imap for DSNs, which can be processed into the equivalent of suppression lists. but you need to know about all these standards/protocols and find libraries. by using the webapi & webhooks, you just need a http & json library. unfortunately, there is no standard for these kinds of api, so mox has made up yet another one... matching incoming DSNs about deliveries to original outgoing messages requires keeping history of "retired" messages (delivered from the queue, either successfully or failed). this can be enabled per account. history is also useful for debugging deliveries. we now also keep history of each delivery attempt, accessible while still in the queue, and kept when a message is retired. the queue webadmin pages now also have pagination, to show potentially large history. a queue of webhook calls is now managed too. failures are retried similar to message deliveries. webhooks can also be saved to the retired list after completing. also configurable per account. messages can be sent with a "unique smtp mail from" address. this can only be used if the domain is configured with a localpart catchall separator such as "+". when enabled, a queued message gets assigned a random "fromid", which is added after the separator when sending. when DSNs are returned, they can be related to previously sent messages based on this fromid. in the future, we can implement matching on the "envid" used in the smtp dsn extension, or on the "message-id" of the message. using a fromid can be triggered by authenticating with a login email address that is configured as enabling fromid. suppression lists are automatically managed per account. if a delivery attempt results in certain smtp errors, the destination address is added to the suppression list. future messages queued for that recipient will immediately fail without a delivery attempt. suppression lists protect your mail server reputation. submitted messages can carry "extra" data through the queue and webhooks for outgoing deliveries. through webapi as a json object, through smtp submission as message headers of the form "x-mox-extra-<key>: value". to make it easy to test webapi/webhooks locally, the "localserve" mode actually puts messages in the queue. when it's time to deliver, it still won't do a full delivery attempt, but just delivers to the sender account. unless the recipient address has a special form, simulating a failure to deliver. admins now have more control over the queue. "hold rules" can be added to mark newly queued messages as "on hold", pausing delivery. rules can be about certain sender or recipient domains/addresses, or apply to all messages pausing the entire queue. also useful for (local) testing. new config options have been introduced. they are editable through the admin and/or account web interfaces. the webapi http endpoints are enabled for newly generated configs with the quickstart, and in localserve. existing configurations must explicitly enable the webapi in mox.conf. gopherwatch.org was created to dogfood this code. it initially used just the compose/smtpclient/imapclient mox packages to send messages and process delivery feedback. it will get a config option to use the mox webapi/webhooks instead. the gopherwatch code to use webapi/webhook is smaller and simpler, and developing that shaped development of the mox webapi/webhooks. for issue #31 by cuu508
2024-04-15 22:49:02 +03:00
log := mlog.New("webapisrv", nil)
acc, err := store.OpenAccount(log, "mjl")
tcheckf(t, err, "open account")
const pw0 = "te\u0301st \u00a0\u2002\u200a" // NFD and various unicode spaces.
const pw1 = "tést " // PRECIS normalized, with NFC.
err = acc.SetPassword(log, pw0)
tcheckf(t, err, "set password")
defer func() {
err := acc.Close()
log.Check(err, "closing account")
acc.CheckClosed()
add a webapi and webhooks for a simple http/json-based api for applications to compose/send messages, receive delivery feedback, and maintain suppression lists. this is an alternative to applications using a library to compose messages, submitting those messages using smtp, and monitoring a mailbox with imap for DSNs, which can be processed into the equivalent of suppression lists. but you need to know about all these standards/protocols and find libraries. by using the webapi & webhooks, you just need a http & json library. unfortunately, there is no standard for these kinds of api, so mox has made up yet another one... matching incoming DSNs about deliveries to original outgoing messages requires keeping history of "retired" messages (delivered from the queue, either successfully or failed). this can be enabled per account. history is also useful for debugging deliveries. we now also keep history of each delivery attempt, accessible while still in the queue, and kept when a message is retired. the queue webadmin pages now also have pagination, to show potentially large history. a queue of webhook calls is now managed too. failures are retried similar to message deliveries. webhooks can also be saved to the retired list after completing. also configurable per account. messages can be sent with a "unique smtp mail from" address. this can only be used if the domain is configured with a localpart catchall separator such as "+". when enabled, a queued message gets assigned a random "fromid", which is added after the separator when sending. when DSNs are returned, they can be related to previously sent messages based on this fromid. in the future, we can implement matching on the "envid" used in the smtp dsn extension, or on the "message-id" of the message. using a fromid can be triggered by authenticating with a login email address that is configured as enabling fromid. suppression lists are automatically managed per account. if a delivery attempt results in certain smtp errors, the destination address is added to the suppression list. future messages queued for that recipient will immediately fail without a delivery attempt. suppression lists protect your mail server reputation. submitted messages can carry "extra" data through the queue and webhooks for outgoing deliveries. through webapi as a json object, through smtp submission as message headers of the form "x-mox-extra-<key>: value". to make it easy to test webapi/webhooks locally, the "localserve" mode actually puts messages in the queue. when it's time to deliver, it still won't do a full delivery attempt, but just delivers to the sender account. unless the recipient address has a special form, simulating a failure to deliver. admins now have more control over the queue. "hold rules" can be added to mark newly queued messages as "on hold", pausing delivery. rules can be about certain sender or recipient domains/addresses, or apply to all messages pausing the entire queue. also useful for (local) testing. new config options have been introduced. they are editable through the admin and/or account web interfaces. the webapi http endpoints are enabled for newly generated configs with the quickstart, and in localserve. existing configurations must explicitly enable the webapi in mox.conf. gopherwatch.org was created to dogfood this code. it initially used just the compose/smtpclient/imapclient mox packages to send messages and process delivery feedback. it will get a config option to use the mox webapi/webhooks instead. the gopherwatch code to use webapi/webhook is smaller and simpler, and developing that shaped development of the mox webapi/webhooks. for issue #31 by cuu508
2024-04-15 22:49:02 +03:00
}()
s := NewServer(100*1024, "/webapi/", false).(server)
hs := httptest.NewServer(s)
defer hs.Close()
// server expects the mount path to be stripped already.
client := webapi.Client{BaseURL: hs.URL + "/v0/", Username: "mjl@mox.example", Password: pw0}
testHTTPHdrsBody := func(s server, method, path string, headers map[string]string, body string, expCode int, expTooMany bool, expCT, expErrCode string) {
t.Helper()
r := httptest.NewRequest(method, path, strings.NewReader(body))
for k, v := range headers {
r.Header.Set(k, v)
}
w := httptest.NewRecorder()
s.ServeHTTP(w, r)
res := w.Result()
if res.StatusCode != http.StatusTooManyRequests || !expTooMany {
tcompare(t, res.StatusCode, expCode)
}
if expCT != "" {
tcompare(t, res.Header.Get("Content-Type"), expCT)
}
if expErrCode != "" {
dec := json.NewDecoder(res.Body)
dec.DisallowUnknownFields()
var apierr webapi.Error
err := dec.Decode(&apierr)
tcheckf(t, err, "decoding json error")
tcompare(t, apierr.Code, expErrCode)
}
}
testHTTP := func(method, path string, expCode int, expCT string) {
t.Helper()
testHTTPHdrsBody(s, method, path, nil, "", expCode, false, expCT, "")
}
testHTTP("GET", "/", http.StatusSeeOther, "")
testHTTP("POST", "/", http.StatusMethodNotAllowed, "")
testHTTP("GET", "/v0/", http.StatusOK, "text/html; charset=utf-8")
testHTTP("GET", "/other/", http.StatusNotFound, "")
testHTTP("GET", "/v0/Send", http.StatusOK, "text/html; charset=utf-8")
testHTTP("GET", "/v0/MessageRawGet", http.StatusOK, "text/html; charset=utf-8")
testHTTP("GET", "/v0/Bogus", http.StatusNotFound, "")
testHTTP("PUT", "/v0/Send", http.StatusMethodNotAllowed, "")
testHTTP("POST", "/v0/Send", http.StatusUnauthorized, "")
for i := 0; i < 11; i++ {
// Missing auth doesn't trigger auth rate limiter.
testHTTP("POST", "/v0/Send", http.StatusUnauthorized, "")
}
for i := 0; i < 21; i++ {
// Bad auth does.
expCode := http.StatusUnauthorized
tooMany := i >= 10
if i == 20 {
expCode = http.StatusTooManyRequests
}
testHTTPHdrsBody(s, "POST", "/v0/Send", map[string]string{"Authorization": "Basic " + base64.StdEncoding.EncodeToString([]byte("mjl@mox.example:badpassword"))}, "", expCode, tooMany, "", "")
}
mox.LimitersInit()
// Request with missing X-Forwarded-For.
sfwd := NewServer(100*1024, "/webapi/", true).(server)
testHTTPHdrsBody(sfwd, "POST", "/v0/Send", map[string]string{"Authorization": "Basic " + base64.StdEncoding.EncodeToString([]byte("mjl@mox.example:badpassword"))}, "", http.StatusInternalServerError, false, "", "")
// Body must be form, not JSON.
authz := "Basic " + base64.StdEncoding.EncodeToString([]byte("mjl@mox.example:"+pw1))
testHTTPHdrsBody(s, "POST", "/v0/Send", map[string]string{"Content-Type": "application/json", "Authorization": authz}, "{}", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
testHTTPHdrsBody(s, "POST", "/v0/Send", map[string]string{"Content-Type": "multipart/form-data", "Authorization": authz}, "not formdata", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
formAuth := map[string]string{
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": authz,
}
testHTTPHdrsBody(s, "POST", "/v0/Send", formAuth, "not encoded\n\n", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
// Missing "request".
testHTTPHdrsBody(s, "POST", "/v0/Send", formAuth, "", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
// "request" must be JSON.
testHTTPHdrsBody(s, "POST", "/v0/Send", formAuth, "request=notjson", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
// "request" must be JSON object.
testHTTPHdrsBody(s, "POST", "/v0/Send", formAuth, "request=[]", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
// Send message. Look for the message in the queue.
now := time.Now()
yes := true
sendReq := webapi.SendRequest{
Message: webapi.Message{
From: []webapi.NameAddress{{Name: "møx", Address: "mjl@mox.example"}},
To: []webapi.NameAddress{{Name: "móx", Address: "mjl+to@mox.example"}, {Address: "mjl+to2@mox.example"}},
CC: []webapi.NameAddress{{Name: "möx", Address: "mjl+cc@mox.example"}},
BCC: []webapi.NameAddress{{Name: "møx", Address: "mjl+bcc@mox.example"}},
ReplyTo: []webapi.NameAddress{{Name: "reply1", Address: "mox+reply1@mox.example"}, {Name: "reply2", Address: "mox+reply2@mox.example"}},
MessageID: "<random@localhost>",
References: []string{"<messageid0@localhost>", "<messageid1@localhost>"},
Date: &now,
Subject: "¡hello world!",
Text: "hi ☺\n",
HTML: `<html><img src="cid:x" /></html>`, // Newline will be added.
},
Extra: map[string]string{"a": "123"},
Headers: [][2]string{{"x-custom", "header"}},
AlternativeFiles: []webapi.File{
{
Name: "x.ics",
ContentType: "text/calendar",
Data: base64.StdEncoding.EncodeToString([]byte("ics data...")),
},
},
add a webapi and webhooks for a simple http/json-based api for applications to compose/send messages, receive delivery feedback, and maintain suppression lists. this is an alternative to applications using a library to compose messages, submitting those messages using smtp, and monitoring a mailbox with imap for DSNs, which can be processed into the equivalent of suppression lists. but you need to know about all these standards/protocols and find libraries. by using the webapi & webhooks, you just need a http & json library. unfortunately, there is no standard for these kinds of api, so mox has made up yet another one... matching incoming DSNs about deliveries to original outgoing messages requires keeping history of "retired" messages (delivered from the queue, either successfully or failed). this can be enabled per account. history is also useful for debugging deliveries. we now also keep history of each delivery attempt, accessible while still in the queue, and kept when a message is retired. the queue webadmin pages now also have pagination, to show potentially large history. a queue of webhook calls is now managed too. failures are retried similar to message deliveries. webhooks can also be saved to the retired list after completing. also configurable per account. messages can be sent with a "unique smtp mail from" address. this can only be used if the domain is configured with a localpart catchall separator such as "+". when enabled, a queued message gets assigned a random "fromid", which is added after the separator when sending. when DSNs are returned, they can be related to previously sent messages based on this fromid. in the future, we can implement matching on the "envid" used in the smtp dsn extension, or on the "message-id" of the message. using a fromid can be triggered by authenticating with a login email address that is configured as enabling fromid. suppression lists are automatically managed per account. if a delivery attempt results in certain smtp errors, the destination address is added to the suppression list. future messages queued for that recipient will immediately fail without a delivery attempt. suppression lists protect your mail server reputation. submitted messages can carry "extra" data through the queue and webhooks for outgoing deliveries. through webapi as a json object, through smtp submission as message headers of the form "x-mox-extra-<key>: value". to make it easy to test webapi/webhooks locally, the "localserve" mode actually puts messages in the queue. when it's time to deliver, it still won't do a full delivery attempt, but just delivers to the sender account. unless the recipient address has a special form, simulating a failure to deliver. admins now have more control over the queue. "hold rules" can be added to mark newly queued messages as "on hold", pausing delivery. rules can be about certain sender or recipient domains/addresses, or apply to all messages pausing the entire queue. also useful for (local) testing. new config options have been introduced. they are editable through the admin and/or account web interfaces. the webapi http endpoints are enabled for newly generated configs with the quickstart, and in localserve. existing configurations must explicitly enable the webapi in mox.conf. gopherwatch.org was created to dogfood this code. it initially used just the compose/smtpclient/imapclient mox packages to send messages and process delivery feedback. it will get a config option to use the mox webapi/webhooks instead. the gopherwatch code to use webapi/webhook is smaller and simpler, and developing that shaped development of the mox webapi/webhooks. for issue #31 by cuu508
2024-04-15 22:49:02 +03:00
InlineFiles: []webapi.File{
{
Name: "x.png",
ContentType: "image/png",
ContentID: "<x>",
Data: base64.StdEncoding.EncodeToString([]byte("png data")),
},
},
AttachedFiles: []webapi.File{
{
Data: base64.StdEncoding.EncodeToString([]byte("%PDF-")), // Should be detected as PDF.
},
},
RequireTLS: &yes,
FutureRelease: &now,
SaveSent: true,
}
sendResp, err := client.Send(ctxbg, sendReq)
tcheckf(t, err, "send message")
tcompare(t, sendResp.MessageID, sendReq.Message.MessageID)
tcompare(t, len(sendResp.Submissions), 2+1+1) // 2 to, 1 cc, 1 bcc
subs := sendResp.Submissions
tcompare(t, subs[0].Address, "mjl+to@mox.example")
tcompare(t, subs[1].Address, "mjl+to2@mox.example")
tcompare(t, subs[2].Address, "mjl+cc@mox.example")
tcompare(t, subs[3].Address, "mjl+bcc@mox.example")
tcompare(t, subs[3].QueueMsgID, subs[0].QueueMsgID+3)
tcompare(t, subs[0].FromID, "")
// todo: look in queue for parameters. parse the message.
// Send a custom multipart/form-data POST, with different request parameters, and
// additional files.
var sb strings.Builder
mp := multipart.NewWriter(&sb)
fdSendReq := webapi.SendRequest{
Message: webapi.Message{
To: []webapi.NameAddress{{Address: "møx@mox.example"}},
// Let server assign date, message-id.
Subject: "test",
Text: "hi",
},
// Don't let server add its own user-agent.
Headers: [][2]string{{"User-Agent", "test"}},
}
sendReqBuf, err := json.Marshal(fdSendReq)
tcheckf(t, err, "send request")
mp.WriteField("request", string(sendReqBuf))
// One alternative file.
pw, err := mp.CreateFormFile("alternativefile", "test.ics")
tcheckf(t, err, "create alternative ics file")
_, err = fmt.Fprint(pw, "ICS...")
tcheckf(t, err, "write ics")
add a webapi and webhooks for a simple http/json-based api for applications to compose/send messages, receive delivery feedback, and maintain suppression lists. this is an alternative to applications using a library to compose messages, submitting those messages using smtp, and monitoring a mailbox with imap for DSNs, which can be processed into the equivalent of suppression lists. but you need to know about all these standards/protocols and find libraries. by using the webapi & webhooks, you just need a http & json library. unfortunately, there is no standard for these kinds of api, so mox has made up yet another one... matching incoming DSNs about deliveries to original outgoing messages requires keeping history of "retired" messages (delivered from the queue, either successfully or failed). this can be enabled per account. history is also useful for debugging deliveries. we now also keep history of each delivery attempt, accessible while still in the queue, and kept when a message is retired. the queue webadmin pages now also have pagination, to show potentially large history. a queue of webhook calls is now managed too. failures are retried similar to message deliveries. webhooks can also be saved to the retired list after completing. also configurable per account. messages can be sent with a "unique smtp mail from" address. this can only be used if the domain is configured with a localpart catchall separator such as "+". when enabled, a queued message gets assigned a random "fromid", which is added after the separator when sending. when DSNs are returned, they can be related to previously sent messages based on this fromid. in the future, we can implement matching on the "envid" used in the smtp dsn extension, or on the "message-id" of the message. using a fromid can be triggered by authenticating with a login email address that is configured as enabling fromid. suppression lists are automatically managed per account. if a delivery attempt results in certain smtp errors, the destination address is added to the suppression list. future messages queued for that recipient will immediately fail without a delivery attempt. suppression lists protect your mail server reputation. submitted messages can carry "extra" data through the queue and webhooks for outgoing deliveries. through webapi as a json object, through smtp submission as message headers of the form "x-mox-extra-<key>: value". to make it easy to test webapi/webhooks locally, the "localserve" mode actually puts messages in the queue. when it's time to deliver, it still won't do a full delivery attempt, but just delivers to the sender account. unless the recipient address has a special form, simulating a failure to deliver. admins now have more control over the queue. "hold rules" can be added to mark newly queued messages as "on hold", pausing delivery. rules can be about certain sender or recipient domains/addresses, or apply to all messages pausing the entire queue. also useful for (local) testing. new config options have been introduced. they are editable through the admin and/or account web interfaces. the webapi http endpoints are enabled for newly generated configs with the quickstart, and in localserve. existing configurations must explicitly enable the webapi in mox.conf. gopherwatch.org was created to dogfood this code. it initially used just the compose/smtpclient/imapclient mox packages to send messages and process delivery feedback. it will get a config option to use the mox webapi/webhooks instead. the gopherwatch code to use webapi/webhook is smaller and simpler, and developing that shaped development of the mox webapi/webhooks. for issue #31 by cuu508
2024-04-15 22:49:02 +03:00
// Two inline PDFs.
pw, err = mp.CreateFormFile("inlinefile", "test.pdf")
add a webapi and webhooks for a simple http/json-based api for applications to compose/send messages, receive delivery feedback, and maintain suppression lists. this is an alternative to applications using a library to compose messages, submitting those messages using smtp, and monitoring a mailbox with imap for DSNs, which can be processed into the equivalent of suppression lists. but you need to know about all these standards/protocols and find libraries. by using the webapi & webhooks, you just need a http & json library. unfortunately, there is no standard for these kinds of api, so mox has made up yet another one... matching incoming DSNs about deliveries to original outgoing messages requires keeping history of "retired" messages (delivered from the queue, either successfully or failed). this can be enabled per account. history is also useful for debugging deliveries. we now also keep history of each delivery attempt, accessible while still in the queue, and kept when a message is retired. the queue webadmin pages now also have pagination, to show potentially large history. a queue of webhook calls is now managed too. failures are retried similar to message deliveries. webhooks can also be saved to the retired list after completing. also configurable per account. messages can be sent with a "unique smtp mail from" address. this can only be used if the domain is configured with a localpart catchall separator such as "+". when enabled, a queued message gets assigned a random "fromid", which is added after the separator when sending. when DSNs are returned, they can be related to previously sent messages based on this fromid. in the future, we can implement matching on the "envid" used in the smtp dsn extension, or on the "message-id" of the message. using a fromid can be triggered by authenticating with a login email address that is configured as enabling fromid. suppression lists are automatically managed per account. if a delivery attempt results in certain smtp errors, the destination address is added to the suppression list. future messages queued for that recipient will immediately fail without a delivery attempt. suppression lists protect your mail server reputation. submitted messages can carry "extra" data through the queue and webhooks for outgoing deliveries. through webapi as a json object, through smtp submission as message headers of the form "x-mox-extra-<key>: value". to make it easy to test webapi/webhooks locally, the "localserve" mode actually puts messages in the queue. when it's time to deliver, it still won't do a full delivery attempt, but just delivers to the sender account. unless the recipient address has a special form, simulating a failure to deliver. admins now have more control over the queue. "hold rules" can be added to mark newly queued messages as "on hold", pausing delivery. rules can be about certain sender or recipient domains/addresses, or apply to all messages pausing the entire queue. also useful for (local) testing. new config options have been introduced. they are editable through the admin and/or account web interfaces. the webapi http endpoints are enabled for newly generated configs with the quickstart, and in localserve. existing configurations must explicitly enable the webapi in mox.conf. gopherwatch.org was created to dogfood this code. it initially used just the compose/smtpclient/imapclient mox packages to send messages and process delivery feedback. it will get a config option to use the mox webapi/webhooks instead. the gopherwatch code to use webapi/webhook is smaller and simpler, and developing that shaped development of the mox webapi/webhooks. for issue #31 by cuu508
2024-04-15 22:49:02 +03:00
tcheckf(t, err, "create inline pdf file")
_, err = fmt.Fprint(pw, "%PDF-")
tcheckf(t, err, "write pdf")
pw, err = mp.CreateFormFile("inlinefile", "test.pdf")
tcheckf(t, err, "create second inline pdf file")
_, err = fmt.Fprint(pw, "%PDF-")
tcheckf(t, err, "write second pdf")
// One attached PDF.
fh := textproto.MIMEHeader{}
fh.Set("Content-Disposition", `form-data; name="attachedfile"; filename="test.pdf"`)
fh.Set("Content-ID", "<testpdf>")
pw, err = mp.CreatePart(fh)
tcheckf(t, err, "create attached pdf file")
_, err = fmt.Fprint(pw, "%PDF-")
tcheckf(t, err, "write attached pdf")
fdct := mp.FormDataContentType()
err = mp.Close()
tcheckf(t, err, "close multipart")
// Perform custom POST.
req, err := http.NewRequest("POST", hs.URL+"/v0/Send", strings.NewReader(sb.String()))
tcheckf(t, err, "new request")
req.Header.Set("Content-Type", fdct)
// Use a unique MAIL FROM id when delivering.
req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte("mjl+fromid@mox.example:"+pw1)))
resp, err := http.DefaultClient.Do(req)
tcheckf(t, err, "request multipart/form-data")
tcompare(t, resp.StatusCode, http.StatusOK)
var sendRes webapi.SendResult
err = json.NewDecoder(resp.Body).Decode(&sendRes)
tcheckf(t, err, "parse send response")
tcompare(t, sendRes.MessageID != "", true)
tcompare(t, len(sendRes.Submissions), 1)
tcompare(t, sendRes.Submissions[0].FromID != "", true)
// Trigger various error conditions.
_, err = client.Send(ctxbg, webapi.SendRequest{
Message: webapi.Message{
To: []webapi.NameAddress{{Address: "mjl@mox.example"}},
Subject: "test",
},
})
terrcode(t, err, "missingBody")
_, err = client.Send(ctxbg, webapi.SendRequest{
Message: webapi.Message{
From: []webapi.NameAddress{{Address: "other@mox.example"}},
To: []webapi.NameAddress{{Address: "mjl@mox.example"}},
Subject: "test",
Text: "hi",
},
})
terrcode(t, err, "badFrom")
_, err = client.Send(ctxbg, webapi.SendRequest{
Message: webapi.Message{
From: []webapi.NameAddress{{Address: "mox@mox.example"}, {Address: "mox@mox.example"}},
To: []webapi.NameAddress{{Address: "mjl@mox.example"}},
Subject: "test",
Text: "hi",
},
})
terrcode(t, err, "multipleFrom")
_, err = client.Send(ctxbg, webapi.SendRequest{Message: webapi.Message{Subject: "test", Text: "hi"}})
terrcode(t, err, "noRecipients")
_, err = client.Send(ctxbg, webapi.SendRequest{
Message: webapi.Message{
MessageID: "missingltgt@localhost",
To: []webapi.NameAddress{{Address: "møx@mox.example"}},
Subject: "test",
Text: "hi",
},
})
terrcode(t, err, "malformedMessageID")
_, err = client.Send(ctxbg, webapi.SendRequest{
Message: webapi.Message{
MessageID: "missingltgt@localhost",
To: []webapi.NameAddress{{Address: "møx@mox.example"}},
Subject: "test",
Text: "hi",
},
})
terrcode(t, err, "malformedMessageID")
// todo: messageLimitReached, recipientLimitReached
// SuppressionList
supListRes, err := client.SuppressionList(ctxbg, webapi.SuppressionListRequest{})
tcheckf(t, err, "listing suppressions")
tcompare(t, len(supListRes.Suppressions), 0)
// SuppressionAdd
supAddReq := webapi.SuppressionAddRequest{EmailAddress: "Remote.Last-catchall@xn--74h.localhost", Manual: true, Reason: "tests"}
_, err = client.SuppressionAdd(ctxbg, supAddReq)
tcheckf(t, err, "add address to suppression list")
_, err = client.SuppressionAdd(ctxbg, supAddReq)
terrcode(t, err, "error") // Already present.
supAddReq2 := webapi.SuppressionAddRequest{EmailAddress: "remotelast@☺.localhost", Manual: false, Reason: "tests"}
_, err = client.SuppressionAdd(ctxbg, supAddReq2)
terrcode(t, err, "error") // Already present, same base address.
supAddReq3 := webapi.SuppressionAddRequest{EmailAddress: "not an address"}
_, err = client.SuppressionAdd(ctxbg, supAddReq3)
terrcode(t, err, "badAddress")
supListRes, err = client.SuppressionList(ctxbg, webapi.SuppressionListRequest{})
tcheckf(t, err, "listing suppressions")
tcompare(t, len(supListRes.Suppressions), 1)
supListRes.Suppressions[0].Created = now
tcompare(t, supListRes.Suppressions, []webapi.Suppression{
{
ID: 1,
Created: now,
Account: "mjl",
BaseAddress: "remotelast@☺.localhost",
OriginalAddress: "Remote.Last-catchall@☺.localhost",
Manual: true,
Reason: "tests",
},
})
// SuppressionPresent
supPresRes, err := client.SuppressionPresent(ctxbg, webapi.SuppressionPresentRequest{EmailAddress: "not@localhost"})
tcheckf(t, err, "address present")
tcompare(t, supPresRes.Present, false)
supPresRes, err = client.SuppressionPresent(ctxbg, webapi.SuppressionPresentRequest{EmailAddress: "remotelast@xn--74h.localhost"})
tcheckf(t, err, "address present")
tcompare(t, supPresRes.Present, true)
supPresRes, err = client.SuppressionPresent(ctxbg, webapi.SuppressionPresentRequest{EmailAddress: "Remote.Last-catchall@☺.localhost"})
tcheckf(t, err, "address present")
tcompare(t, supPresRes.Present, true)
supPresRes, err = client.SuppressionPresent(ctxbg, webapi.SuppressionPresentRequest{EmailAddress: "not an address"})
terrcode(t, err, "badAddress")
// SuppressionRemove
_, err = client.SuppressionRemove(ctxbg, webapi.SuppressionRemoveRequest{EmailAddress: "remote.LAST+more@☺.LocalHost"})
tcheckf(t, err, "remove suppressed address")
_, err = client.SuppressionRemove(ctxbg, webapi.SuppressionRemoveRequest{EmailAddress: "remote.LAST+more@☺.LocalHost"})
terrcode(t, err, "error") // Absent.
_, err = client.SuppressionRemove(ctxbg, webapi.SuppressionRemoveRequest{EmailAddress: "not an address"})
terrcode(t, err, "badAddress")
supListRes, err = client.SuppressionList(ctxbg, webapi.SuppressionListRequest{})
tcheckf(t, err, "listing suppressions")
tcompare(t, len(supListRes.Suppressions), 0)
// MessageGet, we retrieve the message we sent first.
msgRes, err := client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
tcheckf(t, err, "remove suppressed address")
sentMsg := sendReq.Message
sentMsg.Date = msgRes.Message.Date
sentMsg.HTML += "\n"
tcompare(t, msgRes.Message, sentMsg)
// The structure is: mixed (related (alternative text html) inline-png) attached-pdf).
pdfpart := msgRes.Structure.Parts[1]
tcompare(t, pdfpart.ContentType, "application/pdf")
// structure compared below, parsed again from raw message.
// todo: compare Meta
_, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1 + 999})
terrcode(t, err, "messageNotFound")
// MessageRawGet
r, err := client.MessageRawGet(ctxbg, webapi.MessageRawGetRequest{MsgID: 1})
tcheckf(t, err, "get raw message")
var b bytes.Buffer
_, err = io.Copy(&b, r)
r.Close()
tcheckf(t, err, "reading raw message")
part, err := message.EnsurePart(log.Logger, true, bytes.NewReader(b.Bytes()), int64(b.Len()))
tcheckf(t, err, "parsing raw message")
structure, err := webhook.PartStructure(log, &part)
tcheckf(t, err, "part structure")
tcompare(t, structure, msgRes.Structure)
add a webapi and webhooks for a simple http/json-based api for applications to compose/send messages, receive delivery feedback, and maintain suppression lists. this is an alternative to applications using a library to compose messages, submitting those messages using smtp, and monitoring a mailbox with imap for DSNs, which can be processed into the equivalent of suppression lists. but you need to know about all these standards/protocols and find libraries. by using the webapi & webhooks, you just need a http & json library. unfortunately, there is no standard for these kinds of api, so mox has made up yet another one... matching incoming DSNs about deliveries to original outgoing messages requires keeping history of "retired" messages (delivered from the queue, either successfully or failed). this can be enabled per account. history is also useful for debugging deliveries. we now also keep history of each delivery attempt, accessible while still in the queue, and kept when a message is retired. the queue webadmin pages now also have pagination, to show potentially large history. a queue of webhook calls is now managed too. failures are retried similar to message deliveries. webhooks can also be saved to the retired list after completing. also configurable per account. messages can be sent with a "unique smtp mail from" address. this can only be used if the domain is configured with a localpart catchall separator such as "+". when enabled, a queued message gets assigned a random "fromid", which is added after the separator when sending. when DSNs are returned, they can be related to previously sent messages based on this fromid. in the future, we can implement matching on the "envid" used in the smtp dsn extension, or on the "message-id" of the message. using a fromid can be triggered by authenticating with a login email address that is configured as enabling fromid. suppression lists are automatically managed per account. if a delivery attempt results in certain smtp errors, the destination address is added to the suppression list. future messages queued for that recipient will immediately fail without a delivery attempt. suppression lists protect your mail server reputation. submitted messages can carry "extra" data through the queue and webhooks for outgoing deliveries. through webapi as a json object, through smtp submission as message headers of the form "x-mox-extra-<key>: value". to make it easy to test webapi/webhooks locally, the "localserve" mode actually puts messages in the queue. when it's time to deliver, it still won't do a full delivery attempt, but just delivers to the sender account. unless the recipient address has a special form, simulating a failure to deliver. admins now have more control over the queue. "hold rules" can be added to mark newly queued messages as "on hold", pausing delivery. rules can be about certain sender or recipient domains/addresses, or apply to all messages pausing the entire queue. also useful for (local) testing. new config options have been introduced. they are editable through the admin and/or account web interfaces. the webapi http endpoints are enabled for newly generated configs with the quickstart, and in localserve. existing configurations must explicitly enable the webapi in mox.conf. gopherwatch.org was created to dogfood this code. it initially used just the compose/smtpclient/imapclient mox packages to send messages and process delivery feedback. it will get a config option to use the mox webapi/webhooks instead. the gopherwatch code to use webapi/webhook is smaller and simpler, and developing that shaped development of the mox webapi/webhooks. for issue #31 by cuu508
2024-04-15 22:49:02 +03:00
_, err = client.MessageRawGet(ctxbg, webapi.MessageRawGetRequest{MsgID: 1 + 999})
terrcode(t, err, "messageNotFound")
// MessagePartGet
// The structure is: mixed (related (alternative text html) inline-png) attached-pdf).
r, err = client.MessagePartGet(ctxbg, webapi.MessagePartGetRequest{MsgID: 1, PartPath: []int{0, 0, 1}})
tcheckf(t, err, "get message part")
tdata(t, r, sendReq.HTML+"\r\n") // Part returns the raw data with \r\n line endings.
r.Close()
r, err = client.MessagePartGet(ctxbg, webapi.MessagePartGetRequest{MsgID: 1, PartPath: []int{}})
tcheckf(t, err, "get message part")
r.Close()
_, err = client.MessagePartGet(ctxbg, webapi.MessagePartGetRequest{MsgID: 1, PartPath: []int{2}})
terrcode(t, err, "partNotFound")
_, err = client.MessagePartGet(ctxbg, webapi.MessagePartGetRequest{MsgID: 1 + 999, PartPath: []int{}})
terrcode(t, err, "messageNotFound")
_, err = client.MessageFlagsAdd(ctxbg, webapi.MessageFlagsAddRequest{MsgID: 1, Flags: []string{`\answered`, "$Forwarded", "custom"}})
tcheckf(t, err, "add flags")
msgRes, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
tcheckf(t, err, "get message")
tcompare(t, slices.Contains(msgRes.Meta.Flags, `\answered`), true)
tcompare(t, slices.Contains(msgRes.Meta.Flags, "$forwarded"), true)
tcompare(t, slices.Contains(msgRes.Meta.Flags, "custom"), true)
// Setting duplicate flags doesn't make a change.
_, err = client.MessageFlagsAdd(ctxbg, webapi.MessageFlagsAddRequest{MsgID: 1, Flags: []string{`\Answered`, "$forwarded", "custom"}})
tcheckf(t, err, "add flags")
msgRes2, err := client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
tcheckf(t, err, "get message")
tcompare(t, msgRes.Meta.Flags, msgRes2.Meta.Flags)
// Non-existing message gives generic user error.
_, err = client.MessageFlagsAdd(ctxbg, webapi.MessageFlagsAddRequest{MsgID: 1 + 999, Flags: []string{`\answered`, "$Forwarded", "custom"}})
terrcode(t, err, "messageNotFound")
// MessageFlagsRemove
_, err = client.MessageFlagsRemove(ctxbg, webapi.MessageFlagsRemoveRequest{MsgID: 1, Flags: []string{`\Answered`, "$forwarded", "custom"}})
tcheckf(t, err, "remove")
msgRes, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
tcheckf(t, err, "get message")
tcompare(t, slices.Contains(msgRes.Meta.Flags, `\answered`), false)
tcompare(t, slices.Contains(msgRes.Meta.Flags, "$forwarded"), false)
tcompare(t, slices.Contains(msgRes.Meta.Flags, "custom"), false)
// Can try removing again, no change.
_, err = client.MessageFlagsRemove(ctxbg, webapi.MessageFlagsRemoveRequest{MsgID: 1, Flags: []string{`\Answered`, "$forwarded", "custom"}})
tcheckf(t, err, "remove")
_, err = client.MessageFlagsRemove(ctxbg, webapi.MessageFlagsRemoveRequest{MsgID: 1 + 999, Flags: []string{`\Answered`, "$forwarded", "custom"}})
terrcode(t, err, "messageNotFound")
// MessageMove
tcompare(t, msgRes.Meta.MailboxName, "Sent")
_, err = client.MessageMove(ctxbg, webapi.MessageMoveRequest{MsgID: 1, DestMailboxName: "Inbox"})
tcheckf(t, err, "move to inbox")
msgRes, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
tcheckf(t, err, "get message")
tcompare(t, msgRes.Meta.MailboxName, "Inbox")
_, err = client.MessageMove(ctxbg, webapi.MessageMoveRequest{MsgID: 1, DestMailboxName: "Bogus"})
terrcode(t, err, "user")
_, err = client.MessageMove(ctxbg, webapi.MessageMoveRequest{MsgID: 1 + 999, DestMailboxName: "Inbox"})
terrcode(t, err, "messageNotFound")
// MessageDelete
_, err = client.MessageDelete(ctxbg, webapi.MessageDeleteRequest{MsgID: 1})
tcheckf(t, err, "delete message")
_, err = client.MessageDelete(ctxbg, webapi.MessageDeleteRequest{MsgID: 1})
terrcode(t, err, "user") // No longer.
_, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
terrcode(t, err, "messageNotFound") // No longer.
_, err = client.MessageDelete(ctxbg, webapi.MessageDeleteRequest{MsgID: 1 + 999})
terrcode(t, err, "messageNotFound")
}
func tdata(t *testing.T, r io.Reader, exp string) {
t.Helper()
buf, err := io.ReadAll(r)
tcheckf(t, err, "reading body")
tcompare(t, string(buf), exp)
}