mox/webapisrv/server_test.go
Mechiel Lukkien c9451d4d06
in webmail & webapisrv, store bcc header in sent messages
when sending a message with bcc's, prepend the bcc header to the message we
store in the sent folder. still not in the message we send to the recipients.
2024-04-16 17:57:46 +02:00

491 lines
19 KiB
Go

package webapisrv
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"net/textproto"
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/queue"
"github.com/mjl-/mox/store"
"github.com/mjl-/mox/webapi"
"github.com/mjl-/mox/webhook"
)
var ctxbg = context.Background()
func tcheckf(t *testing.T, err error, format string, args ...any) {
t.Helper()
if err != nil {
t.Fatalf("%s: %s", fmt.Sprintf(format, args...), err)
}
}
func tcompare(t *testing.T, got, expect any) {
t.Helper()
if !reflect.DeepEqual(got, expect) {
t.Fatalf("got:\n%#v\nexpected:\n%#v", got, expect)
}
}
func terrcode(t *testing.T, err error, code string) {
t.Helper()
if err == nil {
t.Fatalf("no error, expected error with code %q", code)
}
if xerr, ok := err.(webapi.Error); !ok {
t.Fatalf("got %v, expected webapi error with code %q", err, code)
} else if xerr.Code != code {
t.Fatalf("got error code %q, expected %q", xerr.Code, code)
}
}
func TestServer(t *testing.T) {
mox.LimitersInit()
os.RemoveAll("../testdata/webapisrv/data")
mox.Context = ctxbg
mox.ConfigStaticPath = filepath.FromSlash("../testdata/webapisrv/mox.conf")
mox.MustLoadConfig(true, false)
defer store.Switchboard()()
err := queue.Init()
tcheckf(t, err, "queue init")
log := mlog.New("webapisrv", nil)
acc, err := store.OpenAccount(log, "mjl")
tcheckf(t, err, "open account")
const pw0 = "te\u0301st \u00a0\u2002\u200a" // NFD and various unicode spaces.
const pw1 = "tést " // PRECIS normalized, with NFC.
err = acc.SetPassword(log, pw0)
tcheckf(t, err, "set password")
defer func() {
err := acc.Close()
log.Check(err, "closing account")
acc.CheckClosed()
}()
s := NewServer(100*1024, "/webapi/", false).(server)
hs := httptest.NewServer(s)
defer hs.Close()
// server expects the mount path to be stripped already.
client := webapi.Client{BaseURL: hs.URL + "/v0/", Username: "mjl@mox.example", Password: pw0}
testHTTPHdrsBody := func(s server, method, path string, headers map[string]string, body string, expCode int, expTooMany bool, expCT, expErrCode string) {
t.Helper()
r := httptest.NewRequest(method, path, strings.NewReader(body))
for k, v := range headers {
r.Header.Set(k, v)
}
w := httptest.NewRecorder()
s.ServeHTTP(w, r)
res := w.Result()
if res.StatusCode != http.StatusTooManyRequests || !expTooMany {
tcompare(t, res.StatusCode, expCode)
}
if expCT != "" {
tcompare(t, res.Header.Get("Content-Type"), expCT)
}
if expErrCode != "" {
dec := json.NewDecoder(res.Body)
dec.DisallowUnknownFields()
var apierr webapi.Error
err := dec.Decode(&apierr)
tcheckf(t, err, "decoding json error")
tcompare(t, apierr.Code, expErrCode)
}
}
testHTTP := func(method, path string, expCode int, expCT string) {
t.Helper()
testHTTPHdrsBody(s, method, path, nil, "", expCode, false, expCT, "")
}
testHTTP("GET", "/", http.StatusSeeOther, "")
testHTTP("POST", "/", http.StatusMethodNotAllowed, "")
testHTTP("GET", "/v0/", http.StatusOK, "text/html; charset=utf-8")
testHTTP("GET", "/other/", http.StatusNotFound, "")
testHTTP("GET", "/v0/Send", http.StatusOK, "text/html; charset=utf-8")
testHTTP("GET", "/v0/MessageRawGet", http.StatusOK, "text/html; charset=utf-8")
testHTTP("GET", "/v0/Bogus", http.StatusNotFound, "")
testHTTP("PUT", "/v0/Send", http.StatusMethodNotAllowed, "")
testHTTP("POST", "/v0/Send", http.StatusUnauthorized, "")
for i := 0; i < 11; i++ {
// Missing auth doesn't trigger auth rate limiter.
testHTTP("POST", "/v0/Send", http.StatusUnauthorized, "")
}
for i := 0; i < 21; i++ {
// Bad auth does.
expCode := http.StatusUnauthorized
tooMany := i >= 10
if i == 20 {
expCode = http.StatusTooManyRequests
}
testHTTPHdrsBody(s, "POST", "/v0/Send", map[string]string{"Authorization": "Basic " + base64.StdEncoding.EncodeToString([]byte("mjl@mox.example:badpassword"))}, "", expCode, tooMany, "", "")
}
mox.LimitersInit()
// Request with missing X-Forwarded-For.
sfwd := NewServer(100*1024, "/webapi/", true).(server)
testHTTPHdrsBody(sfwd, "POST", "/v0/Send", map[string]string{"Authorization": "Basic " + base64.StdEncoding.EncodeToString([]byte("mjl@mox.example:badpassword"))}, "", http.StatusInternalServerError, false, "", "")
// Body must be form, not JSON.
authz := "Basic " + base64.StdEncoding.EncodeToString([]byte("mjl@mox.example:"+pw1))
testHTTPHdrsBody(s, "POST", "/v0/Send", map[string]string{"Content-Type": "application/json", "Authorization": authz}, "{}", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
testHTTPHdrsBody(s, "POST", "/v0/Send", map[string]string{"Content-Type": "multipart/form-data", "Authorization": authz}, "not formdata", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
formAuth := map[string]string{
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": authz,
}
testHTTPHdrsBody(s, "POST", "/v0/Send", formAuth, "not encoded\n\n", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
// Missing "request".
testHTTPHdrsBody(s, "POST", "/v0/Send", formAuth, "", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
// "request" must be JSON.
testHTTPHdrsBody(s, "POST", "/v0/Send", formAuth, "request=notjson", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
// "request" must be JSON object.
testHTTPHdrsBody(s, "POST", "/v0/Send", formAuth, "request=[]", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
// Send message. Look for the message in the queue.
now := time.Now()
yes := true
sendReq := webapi.SendRequest{
Message: webapi.Message{
From: []webapi.NameAddress{{Name: "møx", Address: "mjl@mox.example"}},
To: []webapi.NameAddress{{Name: "móx", Address: "mjl+to@mox.example"}, {Address: "mjl+to2@mox.example"}},
CC: []webapi.NameAddress{{Name: "möx", Address: "mjl+cc@mox.example"}},
BCC: []webapi.NameAddress{{Name: "møx", Address: "mjl+bcc@mox.example"}},
ReplyTo: []webapi.NameAddress{{Name: "reply1", Address: "mox+reply1@mox.example"}, {Name: "reply2", Address: "mox+reply2@mox.example"}},
MessageID: "<random@localhost>",
References: []string{"<messageid0@localhost>", "<messageid1@localhost>"},
Date: &now,
Subject: "¡hello world!",
Text: "hi ☺\n",
HTML: `<html><img src="cid:x" /></html>`, // Newline will be added.
},
Extra: map[string]string{"a": "123"},
Headers: [][2]string{{"x-custom", "header"}},
InlineFiles: []webapi.File{
{
Name: "x.png",
ContentType: "image/png",
ContentID: "<x>",
Data: base64.StdEncoding.EncodeToString([]byte("png data")),
},
},
AttachedFiles: []webapi.File{
{
Data: base64.StdEncoding.EncodeToString([]byte("%PDF-")), // Should be detected as PDF.
},
},
RequireTLS: &yes,
FutureRelease: &now,
SaveSent: true,
}
sendResp, err := client.Send(ctxbg, sendReq)
tcheckf(t, err, "send message")
tcompare(t, sendResp.MessageID, sendReq.Message.MessageID)
tcompare(t, len(sendResp.Submissions), 2+1+1) // 2 to, 1 cc, 1 bcc
subs := sendResp.Submissions
tcompare(t, subs[0].Address, "mjl+to@mox.example")
tcompare(t, subs[1].Address, "mjl+to2@mox.example")
tcompare(t, subs[2].Address, "mjl+cc@mox.example")
tcompare(t, subs[3].Address, "mjl+bcc@mox.example")
tcompare(t, subs[3].QueueMsgID, subs[0].QueueMsgID+3)
tcompare(t, subs[0].FromID, "")
// todo: look in queue for parameters. parse the message.
// Send a custom multipart/form-data POST, with different request parameters, and
// additional files.
var sb strings.Builder
mp := multipart.NewWriter(&sb)
fdSendReq := webapi.SendRequest{
Message: webapi.Message{
To: []webapi.NameAddress{{Address: "møx@mox.example"}},
// Let server assign date, message-id.
Subject: "test",
Text: "hi",
},
// Don't let server add its own user-agent.
Headers: [][2]string{{"User-Agent", "test"}},
}
sendReqBuf, err := json.Marshal(fdSendReq)
tcheckf(t, err, "send request")
mp.WriteField("request", string(sendReqBuf))
// Two inline PDFs.
pw, err := mp.CreateFormFile("inlinefile", "test.pdf")
tcheckf(t, err, "create inline pdf file")
_, err = fmt.Fprint(pw, "%PDF-")
tcheckf(t, err, "write pdf")
pw, err = mp.CreateFormFile("inlinefile", "test.pdf")
tcheckf(t, err, "create second inline pdf file")
_, err = fmt.Fprint(pw, "%PDF-")
tcheckf(t, err, "write second pdf")
// One attached PDF.
fh := textproto.MIMEHeader{}
fh.Set("Content-Disposition", `form-data; name="attachedfile"; filename="test.pdf"`)
fh.Set("Content-ID", "<testpdf>")
pw, err = mp.CreatePart(fh)
tcheckf(t, err, "create attached pdf file")
_, err = fmt.Fprint(pw, "%PDF-")
tcheckf(t, err, "write attached pdf")
fdct := mp.FormDataContentType()
err = mp.Close()
tcheckf(t, err, "close multipart")
// Perform custom POST.
req, err := http.NewRequest("POST", hs.URL+"/v0/Send", strings.NewReader(sb.String()))
tcheckf(t, err, "new request")
req.Header.Set("Content-Type", fdct)
// Use a unique MAIL FROM id when delivering.
req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte("mjl+fromid@mox.example:"+pw1)))
resp, err := http.DefaultClient.Do(req)
tcheckf(t, err, "request multipart/form-data")
tcompare(t, resp.StatusCode, http.StatusOK)
var sendRes webapi.SendResult
err = json.NewDecoder(resp.Body).Decode(&sendRes)
tcheckf(t, err, "parse send response")
tcompare(t, sendRes.MessageID != "", true)
tcompare(t, len(sendRes.Submissions), 1)
tcompare(t, sendRes.Submissions[0].FromID != "", true)
// Trigger various error conditions.
_, err = client.Send(ctxbg, webapi.SendRequest{
Message: webapi.Message{
To: []webapi.NameAddress{{Address: "mjl@mox.example"}},
Subject: "test",
},
})
terrcode(t, err, "missingBody")
_, err = client.Send(ctxbg, webapi.SendRequest{
Message: webapi.Message{
From: []webapi.NameAddress{{Address: "other@mox.example"}},
To: []webapi.NameAddress{{Address: "mjl@mox.example"}},
Subject: "test",
Text: "hi",
},
})
terrcode(t, err, "badFrom")
_, err = client.Send(ctxbg, webapi.SendRequest{
Message: webapi.Message{
From: []webapi.NameAddress{{Address: "mox@mox.example"}, {Address: "mox@mox.example"}},
To: []webapi.NameAddress{{Address: "mjl@mox.example"}},
Subject: "test",
Text: "hi",
},
})
terrcode(t, err, "multipleFrom")
_, err = client.Send(ctxbg, webapi.SendRequest{Message: webapi.Message{Subject: "test", Text: "hi"}})
terrcode(t, err, "noRecipients")
_, err = client.Send(ctxbg, webapi.SendRequest{
Message: webapi.Message{
MessageID: "missingltgt@localhost",
To: []webapi.NameAddress{{Address: "møx@mox.example"}},
Subject: "test",
Text: "hi",
},
})
terrcode(t, err, "malformedMessageID")
_, err = client.Send(ctxbg, webapi.SendRequest{
Message: webapi.Message{
MessageID: "missingltgt@localhost",
To: []webapi.NameAddress{{Address: "møx@mox.example"}},
Subject: "test",
Text: "hi",
},
})
terrcode(t, err, "malformedMessageID")
// todo: messageLimitReached, recipientLimitReached
// SuppressionList
supListRes, err := client.SuppressionList(ctxbg, webapi.SuppressionListRequest{})
tcheckf(t, err, "listing suppressions")
tcompare(t, len(supListRes.Suppressions), 0)
// SuppressionAdd
supAddReq := webapi.SuppressionAddRequest{EmailAddress: "Remote.Last-catchall@xn--74h.localhost", Manual: true, Reason: "tests"}
_, err = client.SuppressionAdd(ctxbg, supAddReq)
tcheckf(t, err, "add address to suppression list")
_, err = client.SuppressionAdd(ctxbg, supAddReq)
terrcode(t, err, "error") // Already present.
supAddReq2 := webapi.SuppressionAddRequest{EmailAddress: "remotelast@☺.localhost", Manual: false, Reason: "tests"}
_, err = client.SuppressionAdd(ctxbg, supAddReq2)
terrcode(t, err, "error") // Already present, same base address.
supAddReq3 := webapi.SuppressionAddRequest{EmailAddress: "not an address"}
_, err = client.SuppressionAdd(ctxbg, supAddReq3)
terrcode(t, err, "badAddress")
supListRes, err = client.SuppressionList(ctxbg, webapi.SuppressionListRequest{})
tcheckf(t, err, "listing suppressions")
tcompare(t, len(supListRes.Suppressions), 1)
supListRes.Suppressions[0].Created = now
tcompare(t, supListRes.Suppressions, []webapi.Suppression{
{
ID: 1,
Created: now,
Account: "mjl",
BaseAddress: "remotelast@☺.localhost",
OriginalAddress: "Remote.Last-catchall@☺.localhost",
Manual: true,
Reason: "tests",
},
})
// SuppressionPresent
supPresRes, err := client.SuppressionPresent(ctxbg, webapi.SuppressionPresentRequest{EmailAddress: "not@localhost"})
tcheckf(t, err, "address present")
tcompare(t, supPresRes.Present, false)
supPresRes, err = client.SuppressionPresent(ctxbg, webapi.SuppressionPresentRequest{EmailAddress: "remotelast@xn--74h.localhost"})
tcheckf(t, err, "address present")
tcompare(t, supPresRes.Present, true)
supPresRes, err = client.SuppressionPresent(ctxbg, webapi.SuppressionPresentRequest{EmailAddress: "Remote.Last-catchall@☺.localhost"})
tcheckf(t, err, "address present")
tcompare(t, supPresRes.Present, true)
supPresRes, err = client.SuppressionPresent(ctxbg, webapi.SuppressionPresentRequest{EmailAddress: "not an address"})
terrcode(t, err, "badAddress")
// SuppressionRemove
_, err = client.SuppressionRemove(ctxbg, webapi.SuppressionRemoveRequest{EmailAddress: "remote.LAST+more@☺.LocalHost"})
tcheckf(t, err, "remove suppressed address")
_, err = client.SuppressionRemove(ctxbg, webapi.SuppressionRemoveRequest{EmailAddress: "remote.LAST+more@☺.LocalHost"})
terrcode(t, err, "error") // Absent.
_, err = client.SuppressionRemove(ctxbg, webapi.SuppressionRemoveRequest{EmailAddress: "not an address"})
terrcode(t, err, "badAddress")
supListRes, err = client.SuppressionList(ctxbg, webapi.SuppressionListRequest{})
tcheckf(t, err, "listing suppressions")
tcompare(t, len(supListRes.Suppressions), 0)
// MessageGet, we retrieve the message we sent first.
msgRes, err := client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
tcheckf(t, err, "remove suppressed address")
sentMsg := sendReq.Message
sentMsg.Date = msgRes.Message.Date
sentMsg.HTML += "\n"
tcompare(t, msgRes.Message, sentMsg)
// The structure is: mixed (related (alternative text html) inline-png) attached-pdf).
pdfpart := msgRes.Structure.Parts[1]
tcompare(t, pdfpart.ContentType, "application/pdf")
// structure compared below, parsed again from raw message.
// todo: compare Meta
_, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1 + 999})
terrcode(t, err, "messageNotFound")
// MessageRawGet
r, err := client.MessageRawGet(ctxbg, webapi.MessageRawGetRequest{MsgID: 1})
tcheckf(t, err, "get raw message")
var b bytes.Buffer
_, err = io.Copy(&b, r)
r.Close()
tcheckf(t, err, "reading raw message")
part, err := message.EnsurePart(log.Logger, true, bytes.NewReader(b.Bytes()), int64(b.Len()))
tcheckf(t, err, "parsing raw message")
tcompare(t, webhook.PartStructure(&part), msgRes.Structure)
_, err = client.MessageRawGet(ctxbg, webapi.MessageRawGetRequest{MsgID: 1 + 999})
terrcode(t, err, "messageNotFound")
// MessagePartGet
// The structure is: mixed (related (alternative text html) inline-png) attached-pdf).
r, err = client.MessagePartGet(ctxbg, webapi.MessagePartGetRequest{MsgID: 1, PartPath: []int{0, 0, 1}})
tcheckf(t, err, "get message part")
tdata(t, r, sendReq.HTML+"\r\n") // Part returns the raw data with \r\n line endings.
r.Close()
r, err = client.MessagePartGet(ctxbg, webapi.MessagePartGetRequest{MsgID: 1, PartPath: []int{}})
tcheckf(t, err, "get message part")
r.Close()
_, err = client.MessagePartGet(ctxbg, webapi.MessagePartGetRequest{MsgID: 1, PartPath: []int{2}})
terrcode(t, err, "partNotFound")
_, err = client.MessagePartGet(ctxbg, webapi.MessagePartGetRequest{MsgID: 1 + 999, PartPath: []int{}})
terrcode(t, err, "messageNotFound")
_, err = client.MessageFlagsAdd(ctxbg, webapi.MessageFlagsAddRequest{MsgID: 1, Flags: []string{`\answered`, "$Forwarded", "custom"}})
tcheckf(t, err, "add flags")
msgRes, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
tcheckf(t, err, "get message")
tcompare(t, slices.Contains(msgRes.Meta.Flags, `\answered`), true)
tcompare(t, slices.Contains(msgRes.Meta.Flags, "$forwarded"), true)
tcompare(t, slices.Contains(msgRes.Meta.Flags, "custom"), true)
// Setting duplicate flags doesn't make a change.
_, err = client.MessageFlagsAdd(ctxbg, webapi.MessageFlagsAddRequest{MsgID: 1, Flags: []string{`\Answered`, "$forwarded", "custom"}})
tcheckf(t, err, "add flags")
msgRes2, err := client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
tcheckf(t, err, "get message")
tcompare(t, msgRes.Meta.Flags, msgRes2.Meta.Flags)
// Non-existing message gives generic user error.
_, err = client.MessageFlagsAdd(ctxbg, webapi.MessageFlagsAddRequest{MsgID: 1 + 999, Flags: []string{`\answered`, "$Forwarded", "custom"}})
terrcode(t, err, "messageNotFound")
// MessageFlagsRemove
_, err = client.MessageFlagsRemove(ctxbg, webapi.MessageFlagsRemoveRequest{MsgID: 1, Flags: []string{`\Answered`, "$forwarded", "custom"}})
tcheckf(t, err, "remove")
msgRes, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
tcheckf(t, err, "get message")
tcompare(t, slices.Contains(msgRes.Meta.Flags, `\answered`), false)
tcompare(t, slices.Contains(msgRes.Meta.Flags, "$forwarded"), false)
tcompare(t, slices.Contains(msgRes.Meta.Flags, "custom"), false)
// Can try removing again, no change.
_, err = client.MessageFlagsRemove(ctxbg, webapi.MessageFlagsRemoveRequest{MsgID: 1, Flags: []string{`\Answered`, "$forwarded", "custom"}})
tcheckf(t, err, "remove")
_, err = client.MessageFlagsRemove(ctxbg, webapi.MessageFlagsRemoveRequest{MsgID: 1 + 999, Flags: []string{`\Answered`, "$forwarded", "custom"}})
terrcode(t, err, "messageNotFound")
// MessageMove
tcompare(t, msgRes.Meta.MailboxName, "Sent")
_, err = client.MessageMove(ctxbg, webapi.MessageMoveRequest{MsgID: 1, DestMailboxName: "Inbox"})
tcheckf(t, err, "move to inbox")
msgRes, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
tcheckf(t, err, "get message")
tcompare(t, msgRes.Meta.MailboxName, "Inbox")
_, err = client.MessageMove(ctxbg, webapi.MessageMoveRequest{MsgID: 1, DestMailboxName: "Bogus"})
terrcode(t, err, "user")
_, err = client.MessageMove(ctxbg, webapi.MessageMoveRequest{MsgID: 1 + 999, DestMailboxName: "Inbox"})
terrcode(t, err, "messageNotFound")
// MessageDelete
_, err = client.MessageDelete(ctxbg, webapi.MessageDeleteRequest{MsgID: 1})
tcheckf(t, err, "delete message")
_, err = client.MessageDelete(ctxbg, webapi.MessageDeleteRequest{MsgID: 1})
terrcode(t, err, "user") // No longer.
_, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
terrcode(t, err, "messageNotFound") // No longer.
_, err = client.MessageDelete(ctxbg, webapi.MessageDeleteRequest{MsgID: 1 + 999})
terrcode(t, err, "messageNotFound")
}
func tdata(t *testing.T, r io.Reader, exp string) {
t.Helper()
buf, err := io.ReadAll(r)
tcheckf(t, err, "reading body")
tcompare(t, string(buf), exp)
}