mox/webapisrv/server_test.go

package webapisrv

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"net/http/httptest"
	"net/textproto"
	"os"
	"path/filepath"
	"reflect"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/mjl-/mox/message"
	"github.com/mjl-/mox/mlog"
	"github.com/mjl-/mox/mox-"
	"github.com/mjl-/mox/queue"
	"github.com/mjl-/mox/store"
	"github.com/mjl-/mox/webapi"
	"github.com/mjl-/mox/webhook"
)

var ctxbg = context.Background()

func tcheckf(t *testing.T, err error, format string, args ...any) {
	t.Helper()
	if err != nil {
		t.Fatalf("%s: %s", fmt.Sprintf(format, args...), err)
	}
}

func tcompare(t *testing.T, got, expect any) {
	t.Helper()
	if !reflect.DeepEqual(got, expect) {
		t.Fatalf("got:\n%#v\nexpected:\n%#v", got, expect)
	}
}

func terrcode(t *testing.T, err error, code string) {
	t.Helper()
	if err == nil {
		t.Fatalf("no error, expected error with code %q", code)
	}
	if xerr, ok := err.(webapi.Error); !ok {
		t.Fatalf("got %v, expected webapi error with code %q", err, code)
	} else if xerr.Code != code {
		t.Fatalf("got error code %q, expected %q", xerr.Code, code)
	}
}

func TestServer(t *testing.T) {
	mox.LimitersInit()
	os.RemoveAll("../testdata/webapisrv/data")
	mox.Context = ctxbg
	mox.ConfigStaticPath = filepath.FromSlash("../testdata/webapisrv/mox.conf")
	mox.MustLoadConfig(true, false)
	defer store.Switchboard()()
	err := queue.Init()
	tcheckf(t, err, "queue init")

	log := mlog.New("webapisrv", nil)
	acc, err := store.OpenAccount(log, "mjl")
	tcheckf(t, err, "open account")
	const pw0 = "te\u0301st \u00a0\u2002\u200a" // NFD and various unicode spaces.
	const pw1 = "tést    "                      // PRECIS normalized, with NFC.
	err = acc.SetPassword(log, pw0)
	tcheckf(t, err, "set password")
	defer func() {
		err := acc.Close()
		log.Check(err, "closing account")
		acc.CheckClosed()
	}()

	s := NewServer(100*1024, "/webapi/", false).(server)
	hs := httptest.NewServer(s)
	defer hs.Close()

	// server expects the mount path to be stripped already.
	client := webapi.Client{BaseURL: hs.URL + "/v0/", Username: "mjl@mox.example", Password: pw0}

	testHTTPHdrsBody := func(s server, method, path string, headers map[string]string, body string, expCode int, expTooMany bool, expCT, expErrCode string) {
		t.Helper()

		r := httptest.NewRequest(method, path, strings.NewReader(body))
		for k, v := range headers {
			r.Header.Set(k, v)
		}
		w := httptest.NewRecorder()
		s.ServeHTTP(w, r)
		res := w.Result()
		if res.StatusCode != http.StatusTooManyRequests || !expTooMany {
			tcompare(t, res.StatusCode, expCode)
		}
		if expCT != "" {
			tcompare(t, res.Header.Get("Content-Type"), expCT)
		}
		if expErrCode != "" {
			dec := json.NewDecoder(res.Body)
			dec.DisallowUnknownFields()
			var apierr webapi.Error
			err := dec.Decode(&apierr)
			tcheckf(t, err, "decoding json error")
			tcompare(t, apierr.Code, expErrCode)
		}
	}
	testHTTP := func(method, path string, expCode int, expCT string) {
		t.Helper()
		testHTTPHdrsBody(s, method, path, nil, "", expCode, false, expCT, "")
	}

	testHTTP("GET", "/", http.StatusSeeOther, "")
	testHTTP("POST", "/", http.StatusMethodNotAllowed, "")
	testHTTP("GET", "/v0/", http.StatusOK, "text/html; charset=utf-8")
	testHTTP("GET", "/other/", http.StatusNotFound, "")
	testHTTP("GET", "/v0/Send", http.StatusOK, "text/html; charset=utf-8")
	testHTTP("GET", "/v0/MessageRawGet", http.StatusOK, "text/html; charset=utf-8")
	testHTTP("GET", "/v0/Bogus", http.StatusNotFound, "")
	testHTTP("PUT", "/v0/Send", http.StatusMethodNotAllowed, "")
	testHTTP("POST", "/v0/Send", http.StatusUnauthorized, "")

	for i := 0; i < 11; i++ {
		// Missing auth doesn't trigger auth rate limiter.
		testHTTP("POST", "/v0/Send", http.StatusUnauthorized, "")
	}
	for i := 0; i < 21; i++ {
		// Bad auth does.
		expCode := http.StatusUnauthorized
		tooMany := i >= 10
		if i == 20 {
			expCode = http.StatusTooManyRequests
		}
		testHTTPHdrsBody(s, "POST", "/v0/Send", map[string]string{"Authorization": "Basic " + base64.StdEncoding.EncodeToString([]byte("mjl@mox.example:badpassword"))}, "", expCode, tooMany, "", "")
	}
	mox.LimitersInit()

	// Request with missing X-Forwarded-For.
	sfwd := NewServer(100*1024, "/webapi/", true).(server)
	testHTTPHdrsBody(sfwd, "POST", "/v0/Send", map[string]string{"Authorization": "Basic " + base64.StdEncoding.EncodeToString([]byte("mjl@mox.example:badpassword"))}, "", http.StatusInternalServerError, false, "", "")

	// Body must be form, not JSON.
	authz := "Basic " + base64.StdEncoding.EncodeToString([]byte("mjl@mox.example:"+pw1))
	testHTTPHdrsBody(s, "POST", "/v0/Send", map[string]string{"Content-Type": "application/json", "Authorization": authz}, "{}", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
	testHTTPHdrsBody(s, "POST", "/v0/Send", map[string]string{"Content-Type": "multipart/form-data", "Authorization": authz}, "not formdata", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
	formAuth := map[string]string{
		"Content-Type":  "application/x-www-form-urlencoded",
		"Authorization": authz,
	}
	testHTTPHdrsBody(s, "POST", "/v0/Send", formAuth, "not encoded\n\n", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
	// Missing "request".
	testHTTPHdrsBody(s, "POST", "/v0/Send", formAuth, "", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
	// "request" must be JSON.
	testHTTPHdrsBody(s, "POST", "/v0/Send", formAuth, "request=notjson", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")
	// "request" must be JSON object.
	testHTTPHdrsBody(s, "POST", "/v0/Send", formAuth, "request=[]", http.StatusBadRequest, false, "application/json; charset=utf-8", "protocol")

	// Send message. Look for the message in the queue.
	now := time.Now()
	yes := true
	sendReq := webapi.SendRequest{
		Message: webapi.Message{
			From:       []webapi.NameAddress{{Name: "møx", Address: "mjl@mox.example"}},
			To:         []webapi.NameAddress{{Name: "móx", Address: "mjl+to@mox.example"}, {Address: "mjl+to2@mox.example"}},
			CC:         []webapi.NameAddress{{Name: "möx", Address: "mjl+cc@mox.example"}},
			BCC:        []webapi.NameAddress{{Name: "møx", Address: "mjl+bcc@mox.example"}},
			ReplyTo:    []webapi.NameAddress{{Name: "reply1", Address: "mox+reply1@mox.example"}, {Name: "reply2", Address: "mox+reply2@mox.example"}},
			MessageID:  "<random@localhost>",
			References: []string{"<messageid0@localhost>", "<messageid1@localhost>"},
			Date:       &now,
			Subject:    "¡hello world!",
			Text:       "hi ☺\n",
			HTML:       `<html><img src="cid:x" /></html>`, // Newline will be added.
		},
		Extra:   map[string]string{"a": "123"},
		Headers: [][2]string{{"x-custom", "header"}},
		InlineFiles: []webapi.File{
			{
				Name:        "x.png",
				ContentType: "image/png",
				ContentID:   "<x>",
				Data:        base64.StdEncoding.EncodeToString([]byte("png data")),
			},
		},
		AttachedFiles: []webapi.File{
			{
				Data: base64.StdEncoding.EncodeToString([]byte("%PDF-")), // Should be detected as PDF.
			},
		},
		RequireTLS:    &yes,
		FutureRelease: &now,
		SaveSent:      true,
	}
	sendResp, err := client.Send(ctxbg, sendReq)
	tcheckf(t, err, "send message")
	tcompare(t, sendResp.MessageID, sendReq.Message.MessageID)
	tcompare(t, len(sendResp.Submissions), 2+1+1) // 2 to, 1 cc, 1 bcc
	subs := sendResp.Submissions
	tcompare(t, subs[0].Address, "mjl+to@mox.example")
	tcompare(t, subs[1].Address, "mjl+to2@mox.example")
	tcompare(t, subs[2].Address, "mjl+cc@mox.example")
	tcompare(t, subs[3].Address, "mjl+bcc@mox.example")
	tcompare(t, subs[3].QueueMsgID, subs[0].QueueMsgID+3)
	tcompare(t, subs[0].FromID, "")
	// todo: look in queue for parameters. parse the message.

	// Send a custom multipart/form-data POST, with different request parameters, and
	// additional files.
	var sb strings.Builder
	mp := multipart.NewWriter(&sb)
	fdSendReq := webapi.SendRequest{
		Message: webapi.Message{
			To: []webapi.NameAddress{{Address: "møx@mox.example"}},
			// Let server assign date, message-id.
			Subject: "test",
			Text:    "hi",
		},
		// Don't let server add its own user-agent.
		Headers: [][2]string{{"User-Agent", "test"}},
	}
	sendReqBuf, err := json.Marshal(fdSendReq)
	tcheckf(t, err, "send request")
	mp.WriteField("request", string(sendReqBuf))
	// Two inline PDFs.
	pw, err := mp.CreateFormFile("inlinefile", "test.pdf")
	tcheckf(t, err, "create inline pdf file")
	_, err = fmt.Fprint(pw, "%PDF-")
	tcheckf(t, err, "write pdf")
	pw, err = mp.CreateFormFile("inlinefile", "test.pdf")
	tcheckf(t, err, "create second inline pdf file")
	_, err = fmt.Fprint(pw, "%PDF-")
	tcheckf(t, err, "write second pdf")

	// One attached PDF.
	fh := textproto.MIMEHeader{}
	fh.Set("Content-Disposition", `form-data; name="attachedfile"; filename="test.pdf"`)
	fh.Set("Content-ID", "<testpdf>")
	pw, err = mp.CreatePart(fh)
	tcheckf(t, err, "create attached pdf file")
	_, err = fmt.Fprint(pw, "%PDF-")
	tcheckf(t, err, "write attached pdf")
	fdct := mp.FormDataContentType()
	err = mp.Close()
	tcheckf(t, err, "close multipart")

	// Perform custom POST.
	req, err := http.NewRequest("POST", hs.URL+"/v0/Send", strings.NewReader(sb.String()))
	tcheckf(t, err, "new request")
	req.Header.Set("Content-Type", fdct)
	// Use a unique MAIL FROM id when delivering.
	req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte("mjl+fromid@mox.example:"+pw1)))
	resp, err := http.DefaultClient.Do(req)
	tcheckf(t, err, "request multipart/form-data")
	tcompare(t, resp.StatusCode, http.StatusOK)
	var sendRes webapi.SendResult
	err = json.NewDecoder(resp.Body).Decode(&sendRes)
	tcheckf(t, err, "parse send response")
	tcompare(t, sendRes.MessageID != "", true)
	tcompare(t, len(sendRes.Submissions), 1)
	tcompare(t, sendRes.Submissions[0].FromID != "", true)

	// Trigger various error conditions.
	_, err = client.Send(ctxbg, webapi.SendRequest{
		Message: webapi.Message{
			To:      []webapi.NameAddress{{Address: "mjl@mox.example"}},
			Subject: "test",
		},
	})
	terrcode(t, err, "missingBody")

	_, err = client.Send(ctxbg, webapi.SendRequest{
		Message: webapi.Message{
			From:    []webapi.NameAddress{{Address: "other@mox.example"}},
			To:      []webapi.NameAddress{{Address: "mjl@mox.example"}},
			Subject: "test",
			Text:    "hi",
		},
	})
	terrcode(t, err, "badFrom")

	_, err = client.Send(ctxbg, webapi.SendRequest{
		Message: webapi.Message{
			From:    []webapi.NameAddress{{Address: "mox@mox.example"}, {Address: "mox@mox.example"}},
			To:      []webapi.NameAddress{{Address: "mjl@mox.example"}},
			Subject: "test",
			Text:    "hi",
		},
	})
	terrcode(t, err, "multipleFrom")

	_, err = client.Send(ctxbg, webapi.SendRequest{Message: webapi.Message{Subject: "test", Text: "hi"}})
	terrcode(t, err, "noRecipients")

	_, err = client.Send(ctxbg, webapi.SendRequest{
		Message: webapi.Message{
			MessageID: "missingltgt@localhost",
			To:        []webapi.NameAddress{{Address: "møx@mox.example"}},
			Subject:   "test",
			Text:      "hi",
		},
	})
	terrcode(t, err, "malformedMessageID")

	_, err = client.Send(ctxbg, webapi.SendRequest{
		Message: webapi.Message{
			MessageID: "missingltgt@localhost",
			To:        []webapi.NameAddress{{Address: "møx@mox.example"}},
			Subject:   "test",
			Text:      "hi",
		},
	})
	terrcode(t, err, "malformedMessageID")

	// todo: messageLimitReached, recipientLimitReached

	// SuppressionList
	supListRes, err := client.SuppressionList(ctxbg, webapi.SuppressionListRequest{})
	tcheckf(t, err, "listing suppressions")
	tcompare(t, len(supListRes.Suppressions), 0)

	// SuppressionAdd
	supAddReq := webapi.SuppressionAddRequest{EmailAddress: "Remote.Last-catchall@xn--74h.localhost", Manual: true, Reason: "tests"}
	_, err = client.SuppressionAdd(ctxbg, supAddReq)
	tcheckf(t, err, "add address to suppression list")
	_, err = client.SuppressionAdd(ctxbg, supAddReq)
	terrcode(t, err, "error") // Already present.
	supAddReq2 := webapi.SuppressionAddRequest{EmailAddress: "remotelast@☺.localhost", Manual: false, Reason: "tests"}
	_, err = client.SuppressionAdd(ctxbg, supAddReq2)
	terrcode(t, err, "error") // Already present, same base address.
	supAddReq3 := webapi.SuppressionAddRequest{EmailAddress: "not an address"}
	_, err = client.SuppressionAdd(ctxbg, supAddReq3)
	terrcode(t, err, "badAddress")

	supListRes, err = client.SuppressionList(ctxbg, webapi.SuppressionListRequest{})
	tcheckf(t, err, "listing suppressions")
	tcompare(t, len(supListRes.Suppressions), 1)
	supListRes.Suppressions[0].Created = now
	tcompare(t, supListRes.Suppressions, []webapi.Suppression{
		{
			ID:              1,
			Created:         now,
			Account:         "mjl",
			BaseAddress:     "remotelast@☺.localhost",
			OriginalAddress: "Remote.Last-catchall@☺.localhost",
			Manual:          true,
			Reason:          "tests",
		},
	})

	// SuppressionPresent
	supPresRes, err := client.SuppressionPresent(ctxbg, webapi.SuppressionPresentRequest{EmailAddress: "not@localhost"})
	tcheckf(t, err, "address present")
	tcompare(t, supPresRes.Present, false)
	supPresRes, err = client.SuppressionPresent(ctxbg, webapi.SuppressionPresentRequest{EmailAddress: "remotelast@xn--74h.localhost"})
	tcheckf(t, err, "address present")
	tcompare(t, supPresRes.Present, true)
	supPresRes, err = client.SuppressionPresent(ctxbg, webapi.SuppressionPresentRequest{EmailAddress: "Remote.Last-catchall@☺.localhost"})
	tcheckf(t, err, "address present")
	tcompare(t, supPresRes.Present, true)
	supPresRes, err = client.SuppressionPresent(ctxbg, webapi.SuppressionPresentRequest{EmailAddress: "not an address"})
	terrcode(t, err, "badAddress")

	// SuppressionRemove
	_, err = client.SuppressionRemove(ctxbg, webapi.SuppressionRemoveRequest{EmailAddress: "remote.LAST+more@☺.LocalHost"})
	tcheckf(t, err, "remove suppressed address")
	_, err = client.SuppressionRemove(ctxbg, webapi.SuppressionRemoveRequest{EmailAddress: "remote.LAST+more@☺.LocalHost"})
	terrcode(t, err, "error") // Absent.
	_, err = client.SuppressionRemove(ctxbg, webapi.SuppressionRemoveRequest{EmailAddress: "not an address"})
	terrcode(t, err, "badAddress")

	supListRes, err = client.SuppressionList(ctxbg, webapi.SuppressionListRequest{})
	tcheckf(t, err, "listing suppressions")
	tcompare(t, len(supListRes.Suppressions), 0)

	// MessageGet, we retrieve the message we sent first.
	msgRes, err := client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
	tcheckf(t, err, "remove suppressed address")
	sentMsg := sendReq.Message
	sentMsg.BCC = []webapi.NameAddress{} // todo: the Sent message should contain the BCC. for webmail too.
	sentMsg.Date = msgRes.Message.Date
	sentMsg.HTML += "\n"
	tcompare(t, msgRes.Message, sentMsg)
	// The structure is: mixed (related (alternative text html) inline-png) attached-pdf).
	pdfpart := msgRes.Structure.Parts[1]
	tcompare(t, pdfpart.ContentType, "application/pdf")
	// structure compared below, parsed again from raw message.
	// todo: compare Meta

	_, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1 + 999})
	terrcode(t, err, "messageNotFound")

	// MessageRawGet
	r, err := client.MessageRawGet(ctxbg, webapi.MessageRawGetRequest{MsgID: 1})
	tcheckf(t, err, "get raw message")
	var b bytes.Buffer
	_, err = io.Copy(&b, r)
	r.Close()
	tcheckf(t, err, "reading raw message")
	part, err := message.EnsurePart(log.Logger, true, bytes.NewReader(b.Bytes()), int64(b.Len()))
	tcheckf(t, err, "parsing raw message")
	tcompare(t, webhook.PartStructure(&part), msgRes.Structure)

	_, err = client.MessageRawGet(ctxbg, webapi.MessageRawGetRequest{MsgID: 1 + 999})
	terrcode(t, err, "messageNotFound")

	// MessagePartGet
	// The structure is: mixed (related (alternative text html) inline-png) attached-pdf).
	r, err = client.MessagePartGet(ctxbg, webapi.MessagePartGetRequest{MsgID: 1, PartPath: []int{0, 0, 1}})
	tcheckf(t, err, "get message part")
	tdata(t, r, sendReq.HTML+"\r\n") // Part returns the raw data with \r\n line endings.
	r.Close()

	r, err = client.MessagePartGet(ctxbg, webapi.MessagePartGetRequest{MsgID: 1, PartPath: []int{}})
	tcheckf(t, err, "get message part")
	r.Close()

	_, err = client.MessagePartGet(ctxbg, webapi.MessagePartGetRequest{MsgID: 1, PartPath: []int{2}})
	terrcode(t, err, "partNotFound")

	_, err = client.MessagePartGet(ctxbg, webapi.MessagePartGetRequest{MsgID: 1 + 999, PartPath: []int{}})
	terrcode(t, err, "messageNotFound")

	_, err = client.MessageFlagsAdd(ctxbg, webapi.MessageFlagsAddRequest{MsgID: 1, Flags: []string{`\answered`, "$Forwarded", "custom"}})
	tcheckf(t, err, "add flags")

	msgRes, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
	tcheckf(t, err, "get message")
	tcompare(t, slices.Contains(msgRes.Meta.Flags, `\answered`), true)
	tcompare(t, slices.Contains(msgRes.Meta.Flags, "$forwarded"), true)
	tcompare(t, slices.Contains(msgRes.Meta.Flags, "custom"), true)

	// Setting duplicate flags doesn't make a change.
	_, err = client.MessageFlagsAdd(ctxbg, webapi.MessageFlagsAddRequest{MsgID: 1, Flags: []string{`\Answered`, "$forwarded", "custom"}})
	tcheckf(t, err, "add flags")
	msgRes2, err := client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
	tcheckf(t, err, "get message")
	tcompare(t, msgRes.Meta.Flags, msgRes2.Meta.Flags)

	// Non-existing message gives generic user error.
	_, err = client.MessageFlagsAdd(ctxbg, webapi.MessageFlagsAddRequest{MsgID: 1 + 999, Flags: []string{`\answered`, "$Forwarded", "custom"}})
	terrcode(t, err, "messageNotFound")

	// MessageFlagsRemove
	_, err = client.MessageFlagsRemove(ctxbg, webapi.MessageFlagsRemoveRequest{MsgID: 1, Flags: []string{`\Answered`, "$forwarded", "custom"}})
	tcheckf(t, err, "remove")
	msgRes, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
	tcheckf(t, err, "get message")
	tcompare(t, slices.Contains(msgRes.Meta.Flags, `\answered`), false)
	tcompare(t, slices.Contains(msgRes.Meta.Flags, "$forwarded"), false)
	tcompare(t, slices.Contains(msgRes.Meta.Flags, "custom"), false)
	// Can try removing again, no change.
	_, err = client.MessageFlagsRemove(ctxbg, webapi.MessageFlagsRemoveRequest{MsgID: 1, Flags: []string{`\Answered`, "$forwarded", "custom"}})
	tcheckf(t, err, "remove")

	_, err = client.MessageFlagsRemove(ctxbg, webapi.MessageFlagsRemoveRequest{MsgID: 1 + 999, Flags: []string{`\Answered`, "$forwarded", "custom"}})
	terrcode(t, err, "messageNotFound")

	// MessageMove
	tcompare(t, msgRes.Meta.MailboxName, "Sent")
	_, err = client.MessageMove(ctxbg, webapi.MessageMoveRequest{MsgID: 1, DestMailboxName: "Inbox"})
	tcheckf(t, err, "move to inbox")
	msgRes, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
	tcheckf(t, err, "get message")
	tcompare(t, msgRes.Meta.MailboxName, "Inbox")
	_, err = client.MessageMove(ctxbg, webapi.MessageMoveRequest{MsgID: 1, DestMailboxName: "Bogus"})
	terrcode(t, err, "user")
	_, err = client.MessageMove(ctxbg, webapi.MessageMoveRequest{MsgID: 1 + 999, DestMailboxName: "Inbox"})
	terrcode(t, err, "messageNotFound")

	// MessageDelete
	_, err = client.MessageDelete(ctxbg, webapi.MessageDeleteRequest{MsgID: 1})
	tcheckf(t, err, "delete message")
	_, err = client.MessageDelete(ctxbg, webapi.MessageDeleteRequest{MsgID: 1})
	terrcode(t, err, "user") // No longer.
	_, err = client.MessageGet(ctxbg, webapi.MessageGetRequest{MsgID: 1})
	terrcode(t, err, "messageNotFound") // No longer.
	_, err = client.MessageDelete(ctxbg, webapi.MessageDeleteRequest{MsgID: 1 + 999})
	terrcode(t, err, "messageNotFound")
}

func tdata(t *testing.T, r io.Reader, exp string) {
	t.Helper()
	buf, err := io.ReadAll(r)
	tcheckf(t, err, "reading body")
	tcompare(t, string(buf), exp)
}