mox/webops/xops.go

// Package webops implements shared functionality between webapisrv and webmail.
package webops

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"sort"

	"golang.org/x/exp/maps"

	"github.com/mjl-/bstore"

	"github.com/mjl-/mox/message"
	"github.com/mjl-/mox/mlog"
	"github.com/mjl-/mox/store"
)

var ErrMessageNotFound = errors.New("no such message")

type XOps struct {
	DBWrite    func(ctx context.Context, acc *store.Account, fn func(tx *bstore.Tx))
	Checkf     func(ctx context.Context, err error, format string, args ...any)
	Checkuserf func(ctx context.Context, err error, format string, args ...any)
}

func (x XOps) mailboxID(ctx context.Context, tx *bstore.Tx, mailboxID int64) store.Mailbox {
	if mailboxID == 0 {
		x.Checkuserf(ctx, errors.New("invalid zero mailbox ID"), "getting mailbox")
	}
	mb := store.Mailbox{ID: mailboxID}
	err := tx.Get(&mb)
	if err == bstore.ErrAbsent {
		x.Checkuserf(ctx, err, "getting mailbox")
	}
	x.Checkf(ctx, err, "getting mailbox")
	return mb
}

// messageID returns a non-expunged message or panics with a sherpa error.
func (x XOps) messageID(ctx context.Context, tx *bstore.Tx, messageID int64) store.Message {
	if messageID == 0 {
		x.Checkuserf(ctx, errors.New("invalid zero message id"), "getting message")
	}
	m := store.Message{ID: messageID}
	err := tx.Get(&m)
	if err == bstore.ErrAbsent {
		x.Checkuserf(ctx, ErrMessageNotFound, "getting message")
	} else if err == nil && m.Expunged {
		x.Checkuserf(ctx, errors.New("message was removed"), "getting message")
	}
	x.Checkf(ctx, err, "getting message")
	return m
}

func (x XOps) MessageDelete(ctx context.Context, log mlog.Log, acc *store.Account, messageIDs []int64) {
	acc.WithWLock(func() {
		var changes []store.Change

		x.DBWrite(ctx, acc, func(tx *bstore.Tx) {
			_, changes = x.MessageDeleteTx(ctx, log, tx, acc, messageIDs, 0)
		})

		store.BroadcastChanges(acc, changes)
	})

	for _, mID := range messageIDs {
		p := acc.MessagePath(mID)
		err := os.Remove(p)
		log.Check(err, "removing message file for expunge")
	}
}

func (x XOps) MessageDeleteTx(ctx context.Context, log mlog.Log, tx *bstore.Tx, acc *store.Account, messageIDs []int64, modseq store.ModSeq) (store.ModSeq, []store.Change) {
	removeChanges := map[int64]store.ChangeRemoveUIDs{}
	changes := make([]store.Change, 0, len(messageIDs)+1) // n remove, 1 mailbox counts

	var mb store.Mailbox
	remove := make([]store.Message, 0, len(messageIDs))

	var totalSize int64
	for _, mid := range messageIDs {
		m := x.messageID(ctx, tx, mid)
		totalSize += m.Size

		if m.MailboxID != mb.ID {
			if mb.ID != 0 {
				err := tx.Update(&mb)
				x.Checkf(ctx, err, "updating mailbox counts")
				changes = append(changes, mb.ChangeCounts())
			}
			mb = x.mailboxID(ctx, tx, m.MailboxID)
		}

		qmr := bstore.QueryTx[store.Recipient](tx)
		qmr.FilterEqual("MessageID", m.ID)
		_, err := qmr.Delete()
		x.Checkf(ctx, err, "removing message recipients")

		mb.Sub(m.MailboxCounts())

		if modseq == 0 {
			modseq, err = acc.NextModSeq(tx)
			x.Checkf(ctx, err, "assigning next modseq")
		}
		m.Expunged = true
		m.ModSeq = modseq
		err = tx.Update(&m)
		x.Checkf(ctx, err, "marking message as expunged")

		ch := removeChanges[m.MailboxID]
		ch.UIDs = append(ch.UIDs, m.UID)
		ch.MailboxID = m.MailboxID
		ch.ModSeq = modseq
		removeChanges[m.MailboxID] = ch
		remove = append(remove, m)
	}

	if mb.ID != 0 {
		err := tx.Update(&mb)
		x.Checkf(ctx, err, "updating count in mailbox")
		changes = append(changes, mb.ChangeCounts())
	}

	err := acc.AddMessageSize(log, tx, -totalSize)
	x.Checkf(ctx, err, "updating disk usage")

	// Mark removed messages as not needing training, then retrain them, so if they
	// were trained, they get untrained.
	for i := range remove {
		remove[i].Junk = false
		remove[i].Notjunk = false
	}
	err = acc.RetrainMessages(ctx, log, tx, remove, true)
	x.Checkf(ctx, err, "untraining deleted messages")

	for _, ch := range removeChanges {
		sort.Slice(ch.UIDs, func(i, j int) bool {
			return ch.UIDs[i] < ch.UIDs[j]
		})
		changes = append(changes, ch)
	}

	return modseq, changes
}

func (x XOps) MessageFlagsAdd(ctx context.Context, log mlog.Log, acc *store.Account, messageIDs []int64, flaglist []string) {
	flags, keywords, err := store.ParseFlagsKeywords(flaglist)
	x.Checkuserf(ctx, err, "parsing flags")

	acc.WithRLock(func() {
		var changes []store.Change

		x.DBWrite(ctx, acc, func(tx *bstore.Tx) {
			var modseq store.ModSeq
			var retrain []store.Message
			var mb, origmb store.Mailbox

			for _, mid := range messageIDs {
				m := x.messageID(ctx, tx, mid)

				if mb.ID != m.MailboxID {
					if mb.ID != 0 {
						err := tx.Update(&mb)
						x.Checkf(ctx, err, "updating mailbox")
						if mb.MailboxCounts != origmb.MailboxCounts {
							changes = append(changes, mb.ChangeCounts())
						}
						if mb.KeywordsChanged(origmb) {
							changes = append(changes, mb.ChangeKeywords())
						}
					}
					mb = x.mailboxID(ctx, tx, m.MailboxID)
					origmb = mb
				}
				mb.Keywords, _ = store.MergeKeywords(mb.Keywords, keywords)

				mb.Sub(m.MailboxCounts())
				oflags := m.Flags
				m.Flags = m.Flags.Set(flags, flags)
				var kwChanged bool
				m.Keywords, kwChanged = store.MergeKeywords(m.Keywords, keywords)
				mb.Add(m.MailboxCounts())

				if m.Flags == oflags && !kwChanged {
					continue
				}

				if modseq == 0 {
					modseq, err = acc.NextModSeq(tx)
					x.Checkf(ctx, err, "assigning next modseq")
				}
				m.ModSeq = modseq
				err = tx.Update(&m)
				x.Checkf(ctx, err, "updating message")

				changes = append(changes, m.ChangeFlags(oflags))
				retrain = append(retrain, m)
			}

			if mb.ID != 0 {
				err := tx.Update(&mb)
				x.Checkf(ctx, err, "updating mailbox")
				if mb.MailboxCounts != origmb.MailboxCounts {
					changes = append(changes, mb.ChangeCounts())
				}
				if mb.KeywordsChanged(origmb) {
					changes = append(changes, mb.ChangeKeywords())
				}
			}

			err = acc.RetrainMessages(ctx, log, tx, retrain, false)
			x.Checkf(ctx, err, "retraining messages")
		})

		store.BroadcastChanges(acc, changes)
	})
}

func (x XOps) MessageFlagsClear(ctx context.Context, log mlog.Log, acc *store.Account, messageIDs []int64, flaglist []string) {
	flags, keywords, err := store.ParseFlagsKeywords(flaglist)
	x.Checkuserf(ctx, err, "parsing flags")

	acc.WithRLock(func() {
		var retrain []store.Message
		var changes []store.Change

		x.DBWrite(ctx, acc, func(tx *bstore.Tx) {
			var modseq store.ModSeq
			var mb, origmb store.Mailbox

			for _, mid := range messageIDs {
				m := x.messageID(ctx, tx, mid)

				if mb.ID != m.MailboxID {
					if mb.ID != 0 {
						err := tx.Update(&mb)
						x.Checkf(ctx, err, "updating counts for mailbox")
						if mb.MailboxCounts != origmb.MailboxCounts {
							changes = append(changes, mb.ChangeCounts())
						}
						// note: cannot remove keywords from mailbox by removing keywords from message.
					}
					mb = x.mailboxID(ctx, tx, m.MailboxID)
					origmb = mb
				}

				oflags := m.Flags
				mb.Sub(m.MailboxCounts())
				m.Flags = m.Flags.Set(flags, store.Flags{})
				var changed bool
				m.Keywords, changed = store.RemoveKeywords(m.Keywords, keywords)
				mb.Add(m.MailboxCounts())

				if m.Flags == oflags && !changed {
					continue
				}

				if modseq == 0 {
					modseq, err = acc.NextModSeq(tx)
					x.Checkf(ctx, err, "assigning next modseq")
				}
				m.ModSeq = modseq
				err = tx.Update(&m)
				x.Checkf(ctx, err, "updating message")

				changes = append(changes, m.ChangeFlags(oflags))
				retrain = append(retrain, m)
			}

			if mb.ID != 0 {
				err := tx.Update(&mb)
				x.Checkf(ctx, err, "updating keywords in mailbox")
				if mb.MailboxCounts != origmb.MailboxCounts {
					changes = append(changes, mb.ChangeCounts())
				}
				// note: cannot remove keywords from mailbox by removing keywords from message.
			}

			err = acc.RetrainMessages(ctx, log, tx, retrain, false)
			x.Checkf(ctx, err, "retraining messages")
		})

		store.BroadcastChanges(acc, changes)
	})
}

// MessageMove moves messages to the mailbox represented by mailboxName, or to mailboxID if mailboxName is empty.
func (x XOps) MessageMove(ctx context.Context, log mlog.Log, acc *store.Account, messageIDs []int64, mailboxName string, mailboxID int64) {
	acc.WithRLock(func() {
		var changes []store.Change

		x.DBWrite(ctx, acc, func(tx *bstore.Tx) {
			if mailboxName != "" {
				mb, err := acc.MailboxFind(tx, mailboxName)
				x.Checkf(ctx, err, "looking up mailbox name")
				if mb == nil {
					x.Checkuserf(ctx, errors.New("not found"), "looking up mailbox name")
				} else {
					mailboxID = mb.ID
				}
			}

			mbDst := x.mailboxID(ctx, tx, mailboxID)

			if len(messageIDs) == 0 {
				return
			}

			_, changes = x.MessageMoveTx(ctx, log, acc, tx, messageIDs, mbDst, 0)
		})

		store.BroadcastChanges(acc, changes)
	})
}

func (x XOps) MessageMoveTx(ctx context.Context, log mlog.Log, acc *store.Account, tx *bstore.Tx, messageIDs []int64, mbDst store.Mailbox, modseq store.ModSeq) (store.ModSeq, []store.Change) {
	retrain := make([]store.Message, 0, len(messageIDs))
	removeChanges := map[int64]store.ChangeRemoveUIDs{}
	// n adds, 1 remove, 2 mailboxcounts, optimistic and at least for a single message.
	changes := make([]store.Change, 0, len(messageIDs)+3)

	var mbSrc store.Mailbox

	keywords := map[string]struct{}{}

	for _, mid := range messageIDs {
		m := x.messageID(ctx, tx, mid)

		// We may have loaded this mailbox in the previous iteration of this loop.
		if m.MailboxID != mbSrc.ID {
			if mbSrc.ID != 0 {
				err := tx.Update(&mbSrc)
				x.Checkf(ctx, err, "updating source mailbox counts")
				changes = append(changes, mbSrc.ChangeCounts())
			}
			mbSrc = x.mailboxID(ctx, tx, m.MailboxID)
		}

		if mbSrc.ID == mbDst.ID {
			// Client should filter out messages that are already in mailbox.
			x.Checkuserf(ctx, errors.New("already in destination mailbox"), "moving message")
		}

		var err error
		if modseq == 0 {
			modseq, err = acc.NextModSeq(tx)
			x.Checkf(ctx, err, "assigning next modseq")
		}

		ch := removeChanges[m.MailboxID]
		ch.UIDs = append(ch.UIDs, m.UID)
		ch.ModSeq = modseq
		ch.MailboxID = m.MailboxID
		removeChanges[m.MailboxID] = ch

		// Copy of message record that we'll insert when UID is freed up.
		om := m
		om.PrepareExpunge()
		om.ID = 0 // Assign new ID.
		om.ModSeq = modseq

		mbSrc.Sub(m.MailboxCounts())

		if mbDst.Trash {
			m.Seen = true
		}
		conf, _ := acc.Conf()
		m.MailboxID = mbDst.ID
		if m.IsReject && m.MailboxDestinedID != 0 {
			// Incorrectly delivered to Rejects mailbox. Adjust MailboxOrigID so this message
			// is used for reputation calculation during future deliveries.
			m.MailboxOrigID = m.MailboxDestinedID
			m.IsReject = false
			m.Seen = false
		}
		m.UID = mbDst.UIDNext
		m.ModSeq = modseq
		mbDst.UIDNext++
		m.JunkFlagsForMailbox(mbDst, conf)
		err = tx.Update(&m)
		x.Checkf(ctx, err, "updating moved message in database")

		// Now that UID is unused, we can insert the old record again.
		err = tx.Insert(&om)
		x.Checkf(ctx, err, "inserting record for expunge after moving message")

		mbDst.Add(m.MailboxCounts())

		changes = append(changes, m.ChangeAddUID())
		retrain = append(retrain, m)

		for _, kw := range m.Keywords {
			keywords[kw] = struct{}{}
		}
	}

	err := tx.Update(&mbSrc)
	x.Checkf(ctx, err, "updating source mailbox counts")

	changes = append(changes, mbSrc.ChangeCounts(), mbDst.ChangeCounts())

	// Ensure destination mailbox has keywords of the moved messages.
	var mbKwChanged bool
	mbDst.Keywords, mbKwChanged = store.MergeKeywords(mbDst.Keywords, maps.Keys(keywords))
	if mbKwChanged {
		changes = append(changes, mbDst.ChangeKeywords())
	}

	err = tx.Update(&mbDst)
	x.Checkf(ctx, err, "updating mailbox with uidnext")

	err = acc.RetrainMessages(ctx, log, tx, retrain, false)
	x.Checkf(ctx, err, "retraining messages after move")

	// Ensure UIDs of the removed message are in increasing order. It is quite common
	// for all messages to be from a single source mailbox, meaning this is just one
	// change, for which we preallocated space.
	for _, ch := range removeChanges {
		sort.Slice(ch.UIDs, func(i, j int) bool {
			return ch.UIDs[i] < ch.UIDs[j]
		})
		changes = append(changes, ch)
	}

	return modseq, changes
}

func isText(p message.Part) bool {
	return p.MediaType == "" && p.MediaSubType == "" || p.MediaType == "TEXT" && p.MediaSubType == "PLAIN"
}

func isHTML(p message.Part) bool {
	return p.MediaType == "" && p.MediaSubType == "" || p.MediaType == "TEXT" && p.MediaSubType == "HTML"
}

func isAlternative(p message.Part) bool {
	return p.MediaType == "MULTIPART" && p.MediaSubType == "ALTERNATIVE"
}

func readPart(p message.Part, maxSize int64) (string, error) {
	buf, err := io.ReadAll(io.LimitReader(p.ReaderUTF8OrBinary(), maxSize))
	if err != nil {
		return "", fmt.Errorf("reading part contents: %v", err)
	}
	return string(buf), nil
}

// ReadableParts returns the contents of the first text and/or html parts,
// descending into multiparts, truncated to maxSize bytes if longer.
func ReadableParts(p message.Part, maxSize int64) (text string, html string, found bool, err error) {
	// todo: may want to merge this logic with webmail's message parsing.

	// For non-multipart messages, top-level part.
	if isText(p) {
		data, err := readPart(p, maxSize)
		return data, "", true, err
	} else if isHTML(p) {
		data, err := readPart(p, maxSize)
		return "", data, true, err
	}

	// Look in sub-parts. Stop when we have a readable part, don't continue with other
	// subparts unless we have a multipart/alternative.
	// todo: we may have to look at disposition "inline".
	var haveText, haveHTML bool
	for _, pp := range p.Parts {
		if isText(pp) {
			haveText = true
			text, err = readPart(pp, maxSize)
			if !isAlternative(p) {
				break
			}
		} else if isHTML(pp) {
			haveHTML = true
			html, err = readPart(pp, maxSize)
			if !isAlternative(p) {
				break
			}
		}
	}
	if haveText || haveHTML {
		return text, html, true, err
	}

	// Descend into the subparts.
	for _, pp := range p.Parts {
		text, html, found, err = ReadableParts(pp, maxSize)
		if found {
			break
		}
	}
	return
}