serv/mox
1
0
Fork 0
mirror of https://github.com/mjl-/mox.git synced 2024-12-25 16:03:48 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions 1
mox/store/account.go
Mechiel Lukkien 3f727cf380
webmail: move 2 config options from localstorage to the settings popup, storing their values on the server 
these settings are applied anywhere the webmail is open.  the settings are for
showing keyboard shortcuts in the lower right after a mouse interaction, and
showing additional headers.  the shorcuts were configurable in the "help" popup
before.  the additional headers were only configurable through the developer
console before.

the "mailto:" (un)register buttons are now in the settings popup too.
2024-12-07 12:32:54 +01:00

2812 lines
89 KiB
Go
Raw Permalink Blame History

/*
Package store implements storage for accounts, their mailboxes, IMAP
subscriptions and messages, and broadcasts updates (e.g. mail delivery) to
interested sessions (e.g. IMAP connections).
Layout of storage for accounts:
<DataDir>/accounts/<name>/index.db
<DataDir>/accounts/<name>/msg/[a-zA-Z0-9_-]+/<id>
Index.db holds tables for user information, mailboxes, and messages. Messages
are stored in the msg/ subdirectory, each in their own file. The on-disk message
does not contain headers generated during an incoming SMTP transaction, such as
Received and Authentication-Results headers. Those are in the database to
prevent having to rewrite incoming messages (e.g. Authentication-Result for DKIM
signatures can only be determined after having read the message). Messages must
be read through MsgReader, which transparently adds the prefix from the
database.
*/
package store
// todo: make up a function naming scheme that indicates whether caller should broadcast changes.
import (
"context"
"crypto/md5"
cryptorand "crypto/rand"
"crypto/sha1"
"crypto/sha256"
"encoding"
"encoding/json"
"errors"
"fmt"
"hash"
"io"
"log/slog"
"os"
"path/filepath"
"runtime/debug"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
"golang.org/x/crypto/bcrypt"
"golang.org/x/text/secure/precis"
"golang.org/x/text/unicode/norm"
"github.com/mjl-/bstore"
"github.com/mjl-/mox/config"
"github.com/mjl-/mox/dns"
"github.com/mjl-/mox/message"
"github.com/mjl-/mox/metrics"
"github.com/mjl-/mox/mlog"
"github.com/mjl-/mox/mox-"
"github.com/mjl-/mox/moxio"
"github.com/mjl-/mox/publicsuffix"
"github.com/mjl-/mox/scram"
"github.com/mjl-/mox/smtp"
)
// If true, each time an account is closed its database file is checked for
// consistency. If an inconsistency is found, panic is called. Set by default
// because of all the packages with tests, the mox main function sets it to
// false again.
var CheckConsistencyOnClose = true
var (
ErrUnknownMailbox = errors.New("no such mailbox")
ErrUnknownCredentials = errors.New("credentials not found")
ErrAccountUnknown = errors.New("no such account")
ErrOverQuota = errors.New("account over quota")
)
var DefaultInitialMailboxes = config.InitialMailboxes{
SpecialUse: config.SpecialUseMailboxes{
Sent: "Sent",
Archive: "Archive",
Trash: "Trash",
Draft: "Drafts",
Junk: "Junk",
},
}
type SCRAM struct {
Salt []byte
Iterations int
SaltedPassword []byte
}
// CRAMMD5 holds HMAC ipad and opad hashes that are initialized with the first
// block with (a derivation of) the key/password, so we don't store the password in plain
// text.
type CRAMMD5 struct {
Ipad hash.Hash
Opad hash.Hash
}
// BinaryMarshal is used by bstore to store the ipad/opad hash states.
func (c CRAMMD5) MarshalBinary() ([]byte, error) {
if c.Ipad == nil || c.Opad == nil {
return nil, nil
}
ipad, err := c.Ipad.(encoding.BinaryMarshaler).MarshalBinary()
if err != nil {
return nil, fmt.Errorf("marshal ipad: %v", err)
}
opad, err := c.Opad.(encoding.BinaryMarshaler).MarshalBinary()
if err != nil {
return nil, fmt.Errorf("marshal opad: %v", err)
}
buf := make([]byte, 2+len(ipad)+len(opad))
ipadlen := uint16(len(ipad))
buf[0] = byte(ipadlen >> 8)
buf[1] = byte(ipadlen >> 0)
copy(buf[2:], ipad)
copy(buf[2+len(ipad):], opad)
return buf, nil
}
// BinaryUnmarshal is used by bstore to restore the ipad/opad hash states.
func (c *CRAMMD5) UnmarshalBinary(buf []byte) error {
if len(buf) == 0 {
*c = CRAMMD5{}
return nil
}
if len(buf) < 2 {
return fmt.Errorf("short buffer")
}
ipadlen := int(uint16(buf[0])<<8 | uint16(buf[1])<<0)
if len(buf) < 2+ipadlen {
return fmt.Errorf("buffer too short for ipadlen")
}
ipad := md5.New()
opad := md5.New()
if err := ipad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2 : 2+ipadlen]); err != nil {
return fmt.Errorf("unmarshal ipad: %v", err)
}
if err := opad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2+ipadlen:]); err != nil {
return fmt.Errorf("unmarshal opad: %v", err)
}
*c = CRAMMD5{ipad, opad}
return nil
}
// Password holds credentials in various forms, for logging in with SMTP/IMAP.
type Password struct {
Hash string // bcrypt hash for IMAP LOGIN, SASL PLAIN and HTTP basic authentication.
CRAMMD5 CRAMMD5 // For SASL CRAM-MD5.
SCRAMSHA1 SCRAM // For SASL SCRAM-SHA-1.
SCRAMSHA256 SCRAM // For SASL SCRAM-SHA-256.
}
// Subjectpass holds the secret key used to sign subjectpass tokens.
type Subjectpass struct {
Email string // Our destination address (canonical, with catchall localpart stripped).
Key string
}
// NextUIDValidity is a singleton record in the database with the next UIDValidity
// to use for the next mailbox.
type NextUIDValidity struct {
ID int // Just a single record with ID 1.
Next uint32
}
// SyncState track ModSeqs.
type SyncState struct {
ID int // Just a single record with ID 1.
// Last used, next assigned will be one higher. The first value we hand out is 2.
// That's because 0 (the default value for old existing messages, from before the
// Message.ModSeq field) is special in IMAP, so we return it as 1.
LastModSeq ModSeq `bstore:"nonzero"`
// Highest ModSeq of expunged record that we deleted. When a clients synchronizes
// and requests changes based on a modseq before this one, we don't have the
// history to provide information about deletions. We normally keep these expunged
// records around, but we may periodically truly delete them to reclaim storage
// space. Initially set to -1 because we don't want to match with any ModSeq in the
// database, which can be zero values.
HighestDeletedModSeq ModSeq
}
// Mailbox is collection of messages, e.g. Inbox or Sent.
type Mailbox struct {
ID int64
// "Inbox" is the name for the special IMAP "INBOX". Slash separated
// for hierarchy.
Name string `bstore:"nonzero,unique"`
// If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing
// name, UIDValidity must be changed. Used by IMAP for synchronization.
UIDValidity uint32
// UID likely to be assigned to next message. Used by IMAP to detect messages
// delivered to a mailbox.
UIDNext UID
SpecialUse
// Keywords as used in messages. Storing a non-system keyword for a message
// automatically adds it to this list. Used in the IMAP FLAGS response. Only
// "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in
// lower case (for JMAP), sorted.
Keywords []string
HaveCounts bool // Whether MailboxCounts have been initialized.
MailboxCounts // Statistics about messages, kept up to date whenever a change happens.
}
// MailboxCounts tracks statistics about messages for a mailbox.
type MailboxCounts struct {
Total int64 // Total number of messages, excluding \Deleted. For JMAP.
Deleted int64 // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.
Unread int64 // Messages without \Seen, excluding those with \Deleted, for JMAP.
Unseen int64 // Messages without \Seen, including those with \Deleted, for IMAP.
Size int64 // Number of bytes for all messages.
}
func (mc MailboxCounts) String() string {
return fmt.Sprintf("%d total, %d deleted, %d unread, %d unseen, size %d bytes", mc.Total, mc.Deleted, mc.Unread, mc.Unseen, mc.Size)
}
// Add increases mailbox counts mc with those of delta.
func (mc *MailboxCounts) Add(delta MailboxCounts) {
mc.Total += delta.Total
mc.Deleted += delta.Deleted
mc.Unread += delta.Unread
mc.Unseen += delta.Unseen
mc.Size += delta.Size
}
// Add decreases mailbox counts mc with those of delta.
func (mc *MailboxCounts) Sub(delta MailboxCounts) {
mc.Total -= delta.Total
mc.Deleted -= delta.Deleted
mc.Unread -= delta.Unread
mc.Unseen -= delta.Unseen
mc.Size -= delta.Size
}
// SpecialUse identifies a specific role for a mailbox, used by clients to
// understand where messages should go.
type SpecialUse struct {
Archive bool
Draft bool
Junk bool
Sent bool
Trash bool
}
// CalculateCounts calculates the full current counts for messages in the mailbox.
func (mb *Mailbox) CalculateCounts(tx *bstore.Tx) (mc MailboxCounts, err error) {
q := bstore.QueryTx[Message](tx)
q.FilterNonzero(Message{MailboxID: mb.ID})
q.FilterEqual("Expunged", false)
err = q.ForEach(func(m Message) error {
mc.Add(m.MailboxCounts())
return nil
})
return
}
// ChangeSpecialUse returns a change for special-use flags, for broadcasting to
// other connections.
func (mb Mailbox) ChangeSpecialUse() ChangeMailboxSpecialUse {
return ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse}
}
// ChangeKeywords returns a change with new keywords for a mailbox (e.g. after
// setting a new keyword on a message in the mailbox), for broadcasting to other
// connections.
func (mb Mailbox) ChangeKeywords() ChangeMailboxKeywords {
return ChangeMailboxKeywords{mb.ID, mb.Name, mb.Keywords}
}
// KeywordsChanged returns whether the keywords in a mailbox have changed.
func (mb Mailbox) KeywordsChanged(origmb Mailbox) bool {
if len(mb.Keywords) != len(origmb.Keywords) {
return true
}
// Keywords are stored sorted.
for i, kw := range mb.Keywords {
if origmb.Keywords[i] != kw {
return true
}
}
return false
}
// CountsChange returns a change with mailbox counts.
func (mb Mailbox) ChangeCounts() ChangeMailboxCounts {
return ChangeMailboxCounts{mb.ID, mb.Name, mb.MailboxCounts}
}
// Subscriptions are separate from existence of mailboxes.
type Subscription struct {
Name string
}
// Flags for a mail message.
type Flags struct {
Seen bool
Answered bool
Flagged bool
Forwarded bool
Junk bool
Notjunk bool
Deleted bool
Draft bool
Phishing bool
MDNSent bool
}
// FlagsAll is all flags set, for use as mask.
var FlagsAll = Flags{true, true, true, true, true, true, true, true, true, true}
// Validation of "message From" domain.
type Validation uint8
const (
ValidationUnknown Validation = 0
ValidationStrict Validation = 1 // Like DMARC, with strict policies.
ValidationDMARC Validation = 2 // Actual DMARC policy.
ValidationRelaxed Validation = 3 // Like DMARC, with relaxed policies.
ValidationPass Validation = 4 // For SPF.
ValidationNeutral Validation = 5 // For SPF.
ValidationTemperror Validation = 6
ValidationPermerror Validation = 7
ValidationFail Validation = 8
ValidationSoftfail Validation = 9 // For SPF.
ValidationNone Validation = 10 // E.g. No records.
)
// Message stored in database and per-message file on disk.
//
// Contents are always the combined data from MsgPrefix and the on-disk file named
// based on ID.
//
// Messages always have a header section, even if empty. Incoming messages without
// header section must get an empty header section added before inserting.
type Message struct {
// ID, unchanged over lifetime, determines path to on-disk msg file.
// Set during deliver.
ID int64
UID UID `bstore:"nonzero"` // UID, for IMAP. Set during deliver.
MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`
// Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.
// ModSeq is the last modification. CreateSeq is the Seq the message was inserted,
// always <= ModSeq. If Expunged is set, the message has been removed and should not
// be returned to the user. In this case, ModSeq is the Seq where the message is
// removed, and will never be changed again.
// We have an index on both ModSeq (for JMAP that synchronizes per account) and
// MailboxID+ModSeq (for IMAP that synchronizes per mailbox).
// The index on CreateSeq helps efficiently finding created messages for JMAP.
// The value of ModSeq is special for IMAP. Messages that existed before ModSeq was
// added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If
// we get modseq 1 from a client, the IMAP server will translate it to 0. When we
// return modseq to clients, we turn 0 into 1.
ModSeq ModSeq `bstore:"index"`
CreateSeq ModSeq `bstore:"index"`
Expunged bool
// If set, this message was delivered to a Rejects mailbox. When it is moved to a
// different mailbox, its MailboxOrigID is set to the destination mailbox and this
// flag cleared.
IsReject bool
// If set, this is a forwarded message (through a ruleset with IsForward). This
// causes fields used during junk analysis to be moved to their Orig variants, and
// masked IP fields cleared, so they aren't used in junk classifications for
// incoming messages. This ensures the forwarded messages don't cause negative
// reputation for the forwarding mail server, which may also be sending regular
// messages.
IsForward bool
// MailboxOrigID is the mailbox the message was originally delivered to. Typically
// Inbox or Rejects, but can also be a mailbox configured in a Ruleset, or
// Postmaster, TLS/DMARC reporting addresses. MailboxOrigID is not changed when the
// message is moved to another mailbox, e.g. Archive/Trash/Junk. Used for
// per-mailbox reputation.
//
// MailboxDestinedID is normally 0, but when a message is delivered to the Rejects
// mailbox, it is set to the intended mailbox according to delivery rules,
// typically that of Inbox. When such a message is moved out of Rejects, the
// MailboxOrigID is corrected by setting it to MailboxDestinedID. This ensures the
// message is used for reputation calculation for future deliveries to that
// mailbox.
//
// These are not bstore references to prevent having to update all messages in a
// mailbox when the original mailbox is removed. Use of these fields requires
// checking if the mailbox still exists.
MailboxOrigID int64
MailboxDestinedID int64
Received time.Time `bstore:"default now,index"`
// Full IP address of remote SMTP server. Empty if not delivered over SMTP. The
// masked IPs are used to classify incoming messages. They are left empty for
// messages matching a ruleset for forwarded messages.
RemoteIP string
RemoteIPMasked1 string `bstore:"index RemoteIPMasked1+Received"` // For IPv4 /32, for IPv6 /64, for reputation.
RemoteIPMasked2 string `bstore:"index RemoteIPMasked2+Received"` // For IPv4 /26, for IPv6 /48.
RemoteIPMasked3 string `bstore:"index RemoteIPMasked3+Received"` // For IPv4 /21, for IPv6 /32.
// Only set if present and not an IP address. Unicode string. Empty for forwarded
// messages.
EHLODomain string `bstore:"index EHLODomain+Received"`
MailFrom string // With localpart and domain. Can be empty.
MailFromLocalpart smtp.Localpart // SMTP "MAIL FROM", can be empty.
// Only set if it is a domain, not an IP. Unicode string. Empty for forwarded
// messages, but see OrigMailFromDomain.
MailFromDomain string `bstore:"index MailFromDomain+Received"`
RcptToLocalpart smtp.Localpart // SMTP "RCPT TO", can be empty.
RcptToDomain string // Unicode string.
// Parsed "From" message header, used for reputation along with domain validation.
MsgFromLocalpart smtp.Localpart
MsgFromDomain string `bstore:"index MsgFromDomain+Received"` // Unicode string.
MsgFromOrgDomain string `bstore:"index MsgFromOrgDomain+Received"` // Unicode string.
// Simplified statements of the Validation fields below, used for incoming messages
// to check reputation.
EHLOValidated bool
MailFromValidated bool
MsgFromValidated bool
EHLOValidation Validation // Validation can also take reverse IP lookup into account, not only SPF.
MailFromValidation Validation // Can have SPF-specific validations like ValidationSoftfail.
MsgFromValidation Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.
// Domains with verified DKIM signatures. Unicode string. For forwarded messages, a
// DKIM domain that matched a ruleset's verified domain is left out, but included
// in OrigDKIMDomains.
DKIMDomains []string `bstore:"index DKIMDomains+Received"`
// For forwarded messages,
OrigEHLODomain string
OrigDKIMDomains []string
// Canonicalized Message-Id, always lower-case and normalized quoting, without
// <>'s. Empty if missing. Used for matching message threads, and to prevent
// duplicate reject delivery.
MessageID string `bstore:"index"`
// lower-case: ../rfc/5256:495
// For matching threads in case there is no References/In-Reply-To header. It is
// lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.
SubjectBase string `bstore:"index"`
// ../rfc/5256:90
// Hash of message. For rejects delivery in case there is no Message-ID, only set
// when delivered as reject.
MessageHash []byte
// ID of message starting this thread.
ThreadID int64 `bstore:"index"`
// IDs of parent messages, from closest parent to the root message. Parent messages
// may be in a different mailbox, or may no longer exist. ThreadParentIDs must
// never contain the message id itself (a cycle), and parent messages must
// reference the same ancestors.
ThreadParentIDs []int64
// ThreadMissingLink is true if there is no match with a direct parent. E.g. first
// ID in ThreadParentIDs is not the direct ancestor (an intermediate message may
// have been deleted), or subject-based matching was done.
ThreadMissingLink bool
// If set, newly delivered child messages are automatically marked as read. This
// field is copied to new child messages. Changes are propagated to the webmail
// client.
ThreadMuted bool
// If set, this (sub)thread is collapsed in the webmail client, for threading mode
// "on" (mode "unread" ignores it). This field is copied to new child message.
// Changes are propagated to the webmail client.
ThreadCollapsed bool
// If received message was known to match a mailing list rule (with modified junk
// filtering).
IsMailingList bool
// If this message is a DSN, generated by us or received. For DSNs, we don't look
// at the subject when matching threads.
DSN bool
ReceivedTLSVersion uint16 // 0 if unknown, 1 if plaintext/no TLS, otherwise TLS cipher suite.
ReceivedTLSCipherSuite uint16
ReceivedRequireTLS bool // Whether RequireTLS was known to be used for incoming delivery.
Flags
// For keywords other than system flags or the basic well-known $-flags. Only in
// "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case
// (for JMAP), sorted.
Keywords []string `bstore:"index"`
Size int64
TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.
MsgPrefix []byte // Typically holds received headers and/or header separator.
// ParsedBuf message structure. Currently saved as JSON of message.Part because bstore
// cannot yet store recursive types. Created when first needed, and saved in the
// database.
// todo: once replaced with non-json storage, remove date fixup in ../message/part.go.
ParsedBuf []byte
}
// MailboxCounts returns the delta to counts this message means for its
// mailbox.
func (m Message) MailboxCounts() (mc MailboxCounts) {
if m.Expunged {
return
}
if m.Deleted {
mc.Deleted++
} else {
mc.Total++
}
if !m.Seen {
mc.Unseen++
if !m.Deleted {
mc.Unread++
}
}
mc.Size += m.Size
return
}
func (m Message) ChangeAddUID() ChangeAddUID {
return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords}
}
func (m Message) ChangeFlags(orig Flags) ChangeFlags {
mask := m.Flags.Changed(orig)
return ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: m.ModSeq, Mask: mask, Flags: m.Flags, Keywords: m.Keywords}
}
func (m Message) ChangeThread() ChangeThread {
return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed}
}
// ModSeq represents a modseq as stored in the database. ModSeq 0 in the
// database is sent to the client as 1, because modseq 0 is special in IMAP.
// ModSeq coming from the client are of type int64.
type ModSeq int64
func (ms ModSeq) Client() int64 {
if ms == 0 {
return 1
}
return int64(ms)
}
// ModSeqFromClient converts a modseq from a client to a modseq for internal
// use, e.g. in a database query.
// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).
func ModSeqFromClient(modseq int64) ModSeq {
if modseq == 1 {
return 0
}
return ModSeq(modseq)
}
// PrepareExpunge clears fields that are no longer needed after an expunge, so
// almost all fields. Does not change ModSeq, but does set Expunged.
func (m *Message) PrepareExpunge() {
*m = Message{
ID: m.ID,
UID: m.UID,
MailboxID: m.MailboxID,
CreateSeq: m.CreateSeq,
ModSeq: m.ModSeq,
Expunged: true,
ThreadID: m.ThreadID,
}
}
// PrepareThreading sets MessageID, SubjectBase and DSN (used in threading) based
// on the part.
func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) {
m.DSN = part.IsDSN()
if part.Envelope == nil {
return
}
messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)
if err != nil {
log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID))
} else if raw {
log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID))
}
m.MessageID = messageID
m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)
}
// LoadPart returns a message.Part by reading from m.ParsedBuf.
func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {
if m.ParsedBuf == nil {
return message.Part{}, fmt.Errorf("message not parsed")
}
var p message.Part
err := json.Unmarshal(m.ParsedBuf, &p)
if err != nil {
return p, fmt.Errorf("unmarshal message part")
}
p.SetReaderAt(r)
return p, nil
}
// NeedsTraining returns whether message needs a training update, based on
// TrainedJunk (current training status) and new Junk/Notjunk flags.
func (m Message) NeedsTraining() bool {
untrain := m.TrainedJunk != nil
untrainJunk := untrain && *m.TrainedJunk
train := m.Junk || m.Notjunk && !(m.Junk && m.Notjunk)
trainJunk := m.Junk
return untrain != train || untrain && train && untrainJunk != trainJunk
}
// JunkFlagsForMailbox sets Junk and Notjunk flags based on mailbox name if configured. Often
// used when delivering/moving/copying messages to a mailbox. Mail clients are not
// very helpful with setting junk/notjunk flags. But clients can move/copy messages
// to other mailboxes. So we set flags when clients move a message.
func (m *Message) JunkFlagsForMailbox(mb Mailbox, conf config.Account) {
if mb.Junk {
m.Junk = true
m.Notjunk = false
return
}
if !conf.AutomaticJunkFlags.Enabled {
return
}
lmailbox := strings.ToLower(mb.Name)
if conf.JunkMailbox != nil && conf.JunkMailbox.MatchString(lmailbox) {
m.Junk = true
m.Notjunk = false
} else if conf.NeutralMailbox != nil && conf.NeutralMailbox.MatchString(lmailbox) {
m.Junk = false
m.Notjunk = false
} else if conf.NotJunkMailbox != nil && conf.NotJunkMailbox.MatchString(lmailbox) {
m.Junk = false
m.Notjunk = true
} else if conf.JunkMailbox == nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox != nil {
m.Junk = true
m.Notjunk = false
} else if conf.JunkMailbox != nil && conf.NeutralMailbox == nil && conf.NotJunkMailbox != nil {
m.Junk = false
m.Notjunk = false
} else if conf.JunkMailbox != nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox == nil {
m.Junk = false
m.Notjunk = true
}
}
// Recipient represents the recipient of a message. It is tracked to allow
// first-time incoming replies from users this account has sent messages to. When a
// mailbox is added to the Sent mailbox the message is parsed and recipients are
// inserted as recipient. Recipients are never removed other than for removing the
// message. On move/copy of a message, recipients aren't modified either. For IMAP,
// this assumes a client simply appends messages to the Sent mailbox (as opposed to
// copying messages from some place).
type Recipient struct {
ID int64
MessageID int64 `bstore:"nonzero,ref Message"` // Ref gives it its own index, useful for fast removal as well.
Localpart string `bstore:"nonzero"` // Encoded localpart.
Domain string `bstore:"nonzero,index Domain+Localpart"` // Unicode string.
OrgDomain string `bstore:"nonzero,index"` // Unicode string.
Sent time.Time `bstore:"nonzero"`
}
// Outgoing is a message submitted for delivery from the queue. Used to enforce
// maximum outgoing messages.
type Outgoing struct {
ID int64
Recipient string `bstore:"nonzero,index"` // Canonical international address with utf8 domain.
Submitted time.Time `bstore:"nonzero,default now"`
}
// RecipientDomainTLS stores TLS capabilities of a recipient domain as encountered
// during most recent connection (delivery attempt).
type RecipientDomainTLS struct {
Domain string // Unicode.
Updated time.Time `bstore:"default now"`
STARTTLS bool // Supports STARTTLS.
RequireTLS bool // Supports RequireTLS SMTP extension.
}
// DiskUsage tracks quota use.
type DiskUsage struct {
ID int64 // Always one record with ID 1.
MessageSize int64 // Sum of all messages, for quota accounting.
}
// SessionToken and CSRFToken are types to prevent mixing them up.
// Base64 raw url encoded.
type SessionToken string
type CSRFToken string
// LoginSession represents a login session. We keep a limited number of sessions
// for a user, removing the oldest session when a new one is created.
type LoginSession struct {
ID int64
Created time.Time `bstore:"nonzero,default now"` // Of original login.
Expires time.Time `bstore:"nonzero"` // Extended each time it is used.
SessionTokenBinary [16]byte `bstore:"nonzero"` // Stored in cookie, like "webmailsession" or "webaccountsession".
CSRFTokenBinary [16]byte // For API requests, in "x-mox-csrf" header.
AccountName string `bstore:"nonzero"`
LoginAddress string `bstore:"nonzero"`
// Set when loading from database.
sessionToken SessionToken
csrfToken CSRFToken
}
// Quoting is a setting for how to quote in replies/forwards.
type Quoting string
const (
Default Quoting = "" // Bottom-quote if text is selected, top-quote otherwise.
Bottom Quoting = "bottom"
Top Quoting = "top"
)
// Settings are webmail client settings.
type Settings struct {
ID uint8 // Singleton ID 1.
Signature string
Quoting Quoting
// Whether to show the bars underneath the address input fields indicating
// starttls/dnssec/dane/mtasts/requiretls support by address.
ShowAddressSecurity bool
// Show HTML version of message by default, instead of plain text.
ShowHTML bool
// If true, don't show shortcuts in webmail after mouse interaction.
NoShowShortcuts bool
// Additional headers to display in message view. E.g. Delivered-To, User-Agent, X-Mox-Reason.
ShowHeaders []string
}
// ViewMode how a message should be viewed: its text parts, html parts, or html
// with loading external resources.
type ViewMode string
const (
ModeText ViewMode = "text"
ModeHTML ViewMode = "html"
ModeHTMLExt ViewMode = "htmlext" // HTML with external resources.
)
// FromAddressSettings are webmail client settings per "From" address.
type FromAddressSettings struct {
FromAddress string // Unicode.
ViewMode ViewMode
}
// RulesetNoListID records a user "no" response to the question of
// creating/removing a ruleset after moving a message with list-id header from/to
// the inbox.
type RulesetNoListID struct {
ID int64
RcptToAddress string `bstore:"nonzero"`
ListID string `bstore:"nonzero"`
ToInbox bool // Otherwise from Inbox to other mailbox.
}
// RulesetNoMsgFrom records a user "no" response to the question of
// creating/moveing a ruleset after moving a mesage with message "from" address
// from/to the inbox.
type RulesetNoMsgFrom struct {
ID int64
RcptToAddress string `bstore:"nonzero"`
MsgFromAddress string `bstore:"nonzero"` // Unicode.
ToInbox bool // Otherwise from Inbox to other mailbox.
}
// RulesetNoMailbox represents a "never from/to this mailbox" response to the
// question of adding/removing a ruleset after moving a message.
type RulesetNoMailbox struct {
ID int64
// The mailbox from/to which the move has happened.
// Not a references, if mailbox is deleted, an entry becomes ineffective.
MailboxID int64 `bstore:"nonzero"`
ToMailbox bool // Whether MailboxID is the destination of the move (instead of source).
}
// Types stored in DB.
var DBTypes = []any{
NextUIDValidity{},
Message{},
Recipient{},
Mailbox{},
Subscription{},
Outgoing{},
Password{},
Subjectpass{},
SyncState{},
Upgrade{},
RecipientDomainTLS{},
DiskUsage{},
LoginSession{},
Settings{},
FromAddressSettings{},
RulesetNoListID{},
RulesetNoMsgFrom{},
RulesetNoMailbox{},
}
// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.
type Account struct {
Name string // Name, according to configuration.
Dir string // Directory where account files, including the database, bloom filter, and mail messages, are stored for this account.
DBPath string // Path to database with mailboxes, messages, etc.
DB *bstore.DB // Open database connection.
// Channel that is closed if/when account has/gets "threads" accounting (see
// Upgrade.Threads).
threadsCompleted chan struct{}
// If threads upgrade completed with error, this is set. Used for warning during
// delivery, or aborting when importing.
threadsErr error
// Write lock must be held for account/mailbox modifications including message delivery.
// Read lock for reading mailboxes/messages.
// When making changes to mailboxes/messages, changes must be broadcasted before
// releasing the lock to ensure proper UID ordering.
sync.RWMutex
nused int // Reference count, while >0, this account is alive and shared.
}
type Upgrade struct {
ID byte
Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed.
}
// InitialUIDValidity returns a UIDValidity used for initializing an account.
// It can be replaced during tests with a predictable value.
var InitialUIDValidity = func() uint32 {
return uint32(time.Now().Unix() >> 1) // A 2-second resolution will get us far enough beyond 2038.
}
var openAccounts = struct {
names map[string]*Account
sync.Mutex
}{
names: map[string]*Account{},
}
func closeAccount(acc *Account) (rerr error) {
openAccounts.Lock()
acc.nused--
defer openAccounts.Unlock()
if acc.nused == 0 {
// threadsCompleted must be closed now because it increased nused.
rerr = acc.DB.Close()
acc.DB = nil
delete(openAccounts.names, acc.Name)
}
return
}
// OpenAccount opens an account by name.
//
// No additional data path prefix or ".db" suffix should be added to the name.
// A single shared account exists per name.
func OpenAccount(log mlog.Log, name string) (*Account, error) {
openAccounts.Lock()
defer openAccounts.Unlock()
if acc, ok := openAccounts.names[name]; ok {
acc.nused++
return acc, nil
}
if _, ok := mox.Conf.Account(name); !ok {
return nil, ErrAccountUnknown
}
acc, err := openAccount(log, name)
if err != nil {
return nil, err
}
openAccounts.names[name] = acc
return acc, nil
}
// openAccount opens an existing account, or creates it if it is missing.
func openAccount(log mlog.Log, name string) (a *Account, rerr error) {
dir := filepath.Join(mox.DataDirPath("accounts"), name)
return OpenAccountDB(log, dir, name)
}
// OpenAccountDB opens an account database file and returns an initialized account
// or error. Only exported for use by subcommands that verify the database file.
// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.
func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) {
dbpath := filepath.Join(accountDir, "index.db")
// Create account if it doesn't exist yet.
isNew := false
if _, err := os.Stat(dbpath); err != nil && os.IsNotExist(err) {
isNew = true
os.MkdirAll(accountDir, 0770)
}
opts := bstore.Options{Timeout: 5 * time.Second, Perm: 0660, RegisterLogger: log.Logger}
db, err := bstore.Open(context.TODO(), dbpath, &opts, DBTypes...)
if err != nil {
return nil, err
}
defer func() {
if rerr != nil {
db.Close()
if isNew {
os.Remove(dbpath)
}
}
}()
acc := &Account{
Name: accountName,
Dir: accountDir,
DBPath: dbpath,
DB: db,
nused: 1,
threadsCompleted: make(chan struct{}),
}
if isNew {
if err := initAccount(db); err != nil {
return nil, fmt.Errorf("initializing account: %v", err)
}
close(acc.threadsCompleted)
return acc, nil
}
// Ensure singletons are present. Mailbox counts and total message size, Settings.
var mentioned bool
err = db.Write(context.TODO(), func(tx *bstore.Tx) error {
if tx.Get(&Settings{ID: 1}) == bstore.ErrAbsent {
if err := tx.Insert(&Settings{ID: 1, ShowAddressSecurity: true}); err != nil {
return err
}
}
err := bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {
if !mentioned {
mentioned = true
log.Info("first calculation of mailbox counts for account", slog.String("account", accountName))
}
mc, err := mb.CalculateCounts(tx)
if err != nil {
return err
}
mb.HaveCounts = true
mb.MailboxCounts = mc
return tx.Update(&mb)
})
if err != nil {
return err
}
du := DiskUsage{ID: 1}
err = tx.Get(&du)
if err == nil || !errors.Is(err, bstore.ErrAbsent) {
return err
}
// No DiskUsage record yet, calculate total size and insert.
err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {
du.MessageSize += mb.Size
return nil
})
if err != nil {
return err
}
return tx.Insert(&du)
})
if err != nil {
return nil, fmt.Errorf("calculating counts for mailbox or inserting settings: %v", err)
}
// Start adding threading if needed.
up := Upgrade{ID: 1}
err = db.Write(context.TODO(), func(tx *bstore.Tx) error {
err := tx.Get(&up)
if err == bstore.ErrAbsent {
if err := tx.Insert(&up); err != nil {
return fmt.Errorf("inserting initial upgrade record: %v", err)
}
err = nil
}
return err
})
if err != nil {
return nil, fmt.Errorf("checking message threading: %v", err)
}
if up.Threads == 2 {
close(acc.threadsCompleted)
return acc, nil
}
// Increase account use before holding on to account in background.
// Caller holds the lock. The goroutine below decreases nused by calling
// closeAccount.
acc.nused++
// Ensure all messages have a MessageID and SubjectBase, which are needed when
// matching threads.
// Then assign messages to threads, in the same way we do during imports.
log.Info("upgrading account for threading, in background", slog.String("account", acc.Name))
go func() {
defer func() {
err := closeAccount(acc)
log.Check(err, "closing use of account after upgrading account storage for threads", slog.String("account", a.Name))
// Mark that upgrade has finished, possibly error is indicated in threadsErr.
close(acc.threadsCompleted)
}()
defer func() {
x := recover() // Should not happen, but don't take program down if it does.
if x != nil {
log.Error("upgradeThreads panic", slog.Any("err", x))
debug.PrintStack()
metrics.PanicInc(metrics.Upgradethreads)
acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)
}
}()
err := upgradeThreads(mox.Shutdown, log, acc, &up)
if err != nil {
a.threadsErr = err
log.Errorx("upgrading account for threading, aborted", err, slog.String("account", a.Name))
} else {
log.Info("upgrading account for threading, completed", slog.String("account", a.Name))
}
}()
return acc, nil
}
// ThreadingWait blocks until the one-time account threading upgrade for the
// account has completed, and returns an error if not successful.
//
// To be used before starting an import of messages.
func (a *Account) ThreadingWait(log mlog.Log) error {
select {
case <-a.threadsCompleted:
return a.threadsErr
default:
}
log.Debug("waiting for account upgrade to complete")
<-a.threadsCompleted
return a.threadsErr
}
func initAccount(db *bstore.DB) error {
return db.Write(context.TODO(), func(tx *bstore.Tx) error {
uidvalidity := InitialUIDValidity()
if err := tx.Insert(&Upgrade{ID: 1, Threads: 2}); err != nil {
return err
}
if err := tx.Insert(&DiskUsage{ID: 1}); err != nil {
return err
}
if err := tx.Insert(&Settings{ID: 1}); err != nil {
return err
}
if len(mox.Conf.Static.DefaultMailboxes) > 0 {
// Deprecated in favor of InitialMailboxes.
defaultMailboxes := mox.Conf.Static.DefaultMailboxes
mailboxes := []string{"Inbox"}
for _, name := range defaultMailboxes {
if strings.EqualFold(name, "Inbox") {
continue
}
mailboxes = append(mailboxes, name)
}
for _, name := range mailboxes {
mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, HaveCounts: true}
if strings.HasPrefix(name, "Archive") {
mb.Archive = true
} else if strings.HasPrefix(name, "Drafts") {
mb.Draft = true
} else if strings.HasPrefix(name, "Junk") {
mb.Junk = true
} else if strings.HasPrefix(name, "Sent") {
mb.Sent = true
} else if strings.HasPrefix(name, "Trash") {
mb.Trash = true
}
if err := tx.Insert(&mb); err != nil {
return fmt.Errorf("creating mailbox: %w", err)
}
if err := tx.Insert(&Subscription{name}); err != nil {
return fmt.Errorf("adding subscription: %w", err)
}
}
} else {
mailboxes := mox.Conf.Static.InitialMailboxes
var zerouse config.SpecialUseMailboxes
if mailboxes.SpecialUse == zerouse && len(mailboxes.Regular) == 0 {
mailboxes = DefaultInitialMailboxes
}
add := func(name string, use SpecialUse) error {
mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, SpecialUse: use, HaveCounts: true}
if err := tx.Insert(&mb); err != nil {
return fmt.Errorf("creating mailbox: %w", err)
}
if err := tx.Insert(&Subscription{name}); err != nil {
return fmt.Errorf("adding subscription: %w", err)
}
return nil
}
addSpecialOpt := func(nameOpt string, use SpecialUse) error {
if nameOpt == "" {
return nil
}
return add(nameOpt, use)
}
l := []struct {
nameOpt string
use SpecialUse
}{
{"Inbox", SpecialUse{}},
{mailboxes.SpecialUse.Archive, SpecialUse{Archive: true}},
{mailboxes.SpecialUse.Draft, SpecialUse{Draft: true}},
{mailboxes.SpecialUse.Junk, SpecialUse{Junk: true}},
{mailboxes.SpecialUse.Sent, SpecialUse{Sent: true}},
{mailboxes.SpecialUse.Trash, SpecialUse{Trash: true}},
}
for _, e := range l {
if err := addSpecialOpt(e.nameOpt, e.use); err != nil {
return err
}
}
for _, name := range mailboxes.Regular {
if err := add(name, SpecialUse{}); err != nil {
return err
}
}
}
uidvalidity++
if err := tx.Insert(&NextUIDValidity{1, uidvalidity}); err != nil {
return fmt.Errorf("inserting nextuidvalidity: %w", err)
}
return nil
})
}
// CheckClosed asserts that the account has a zero reference count. For use in tests.
func (a *Account) CheckClosed() {
openAccounts.Lock()
defer openAccounts.Unlock()
if a.nused != 0 {
panic(fmt.Sprintf("account still in use, %d refs", a.nused))
}
}
// Close reduces the reference count, and closes the database connection when
// it was the last user.
func (a *Account) Close() error {
if CheckConsistencyOnClose {
xerr := a.CheckConsistency()
err := closeAccount(a)
if xerr != nil {
panic(xerr)
}
return err
}
return closeAccount(a)
}
// CheckConsistency checks the consistency of the database and returns a non-nil
// error for these cases:
//
// - Missing on-disk file for message.
// - Mismatch between message size and length of MsgPrefix and on-disk file.
// - Missing HaveCounts.
// - Incorrect mailbox counts.
// - Incorrect total message size.
// - Message with UID >= mailbox uid next.
// - Mailbox uidvalidity >= account uid validity.
// - ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq.
// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail.
func (a *Account) CheckConsistency() error {
var uidErrors []string // With a limit, could be many.
var modseqErrors []string // With limit.
var fileErrors []string // With limit.
var threadidErrors []string // With limit.
var threadParentErrors []string // With limit.
var threadAncestorErrors []string // With limit.
var errors []string
err := a.DB.Read(context.Background(), func(tx *bstore.Tx) error {
nuv := NextUIDValidity{ID: 1}
err := tx.Get(&nuv)
if err != nil {
return fmt.Errorf("fetching next uid validity: %v", err)
}
mailboxes := map[int64]Mailbox{}
err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {
mailboxes[mb.ID] = mb
if mb.UIDValidity >= nuv.Next {
errmsg := fmt.Sprintf("mailbox %q (id %d) has uidvalidity %d >= account next uidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, nuv.Next)
errors = append(errors, errmsg)
}
return nil
})
if err != nil {
return fmt.Errorf("listing mailboxes: %v", err)
}
counts := map[int64]MailboxCounts{}
err = bstore.QueryTx[Message](tx).ForEach(func(m Message) error {
mc := counts[m.MailboxID]
mc.Add(m.MailboxCounts())
counts[m.MailboxID] = mc
mb := mailboxes[m.MailboxID]
if (m.ModSeq == 0 || m.CreateSeq == 0 || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 {
modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)
modseqErrors = append(modseqErrors, modseqerr)
}
if m.UID >= mb.UIDNext && len(uidErrors) < 20 {
uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)
uidErrors = append(uidErrors, uiderr)
}
if m.Expunged {
return nil
}
p := a.MessagePath(m.ID)
st, err := os.Stat(p)
if err != nil {
existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err)
fileErrors = append(fileErrors, existserr)
} else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {
sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size())
fileErrors = append(fileErrors, sizeerr)
}
if m.ThreadID <= 0 && len(threadidErrors) < 20 {
err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID)
threadidErrors = append(threadidErrors, err)
}
if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 {
err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID)
threadParentErrors = append(threadParentErrors, err)
}
for i, pid := range m.ThreadParentIDs {
am := Message{ID: pid}
if err := tx.Get(&am); err == bstore.ErrAbsent {
continue
} else if err != nil {
return fmt.Errorf("get ancestor message: %v", err)
} else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 {
err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)
threadAncestorErrors = append(threadAncestorErrors, err)
} else {
break
}
}
return nil
})
if err != nil {
return fmt.Errorf("reading messages: %v", err)
}
var totalSize int64
for _, mb := range mailboxes {
totalSize += mb.Size
if !mb.HaveCounts {
errmsg := fmt.Sprintf("mailbox %q (id %d) does not have counts, should be %#v", mb.Name, mb.ID, counts[mb.ID])
errors = append(errors, errmsg)
} else if mb.MailboxCounts != counts[mb.ID] {
mbcounterr := fmt.Sprintf("mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, counts[mb.ID])
errors = append(errors, mbcounterr)
}
}
du := DiskUsage{ID: 1}
if err := tx.Get(&du); err != nil {
return fmt.Errorf("get diskusage")
}
if du.MessageSize != totalSize {
errmsg := fmt.Sprintf("total message size in database is %d, sum of mailbox message sizes is %d", du.MessageSize, totalSize)
errors = append(errors, errmsg)
}
return nil
})
if err != nil {
return err
}
errors = append(errors, uidErrors...)
errors = append(errors, modseqErrors...)
errors = append(errors, fileErrors...)
errors = append(errors, threadidErrors...)
errors = append(errors, threadParentErrors...)
errors = append(errors, threadAncestorErrors...)
if len(errors) > 0 {
return fmt.Errorf("%s", strings.Join(errors, "; "))
}
return nil
}
// Conf returns the configuration for this account if it still exists. During
// an SMTP session, a configuration update may drop an account.
func (a *Account) Conf() (config.Account, bool) {
return mox.Conf.Account(a.Name)
}
// NextUIDValidity returns the next new/unique uidvalidity to use for this account.
func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {
nuv := NextUIDValidity{ID: 1}
if err := tx.Get(&nuv); err != nil {
return 0, err
}
v := nuv.Next
nuv.Next++
if err := tx.Update(&nuv); err != nil {
return 0, err
}
return v, nil
}
// NextModSeq returns the next modification sequence, which is global per account,
// over all types.
func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {
v := SyncState{ID: 1}
if err := tx.Get(&v); err == bstore.ErrAbsent {
// We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so
// already used.
// HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value
// makes sense.
v = SyncState{1, 2, -1}
return v.LastModSeq, tx.Insert(&v)
} else if err != nil {
return 0, err
}
v.LastModSeq++
return v.LastModSeq, tx.Update(&v)
}
func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {
v := SyncState{ID: 1}
err := tx.Get(&v)
if err == bstore.ErrAbsent {
return 0, nil
}
return v.HighestDeletedModSeq, err
}
// WithWLock runs fn with account writelock held. Necessary for account/mailbox
// modification. For message delivery, a read lock is required.
func (a *Account) WithWLock(fn func()) {
a.Lock()
defer a.Unlock()
fn()
}
// WithRLock runs fn with account read lock held. Needed for message delivery.
func (a *Account) WithRLock(fn func()) {
a.RLock()
defer a.RUnlock()
fn()
}
// DeliverMessage delivers a mail message to the account.
//
// The message, with msg.MsgPrefix and msgFile combined, must have a header
// section. The caller is responsible for adding a header separator to
// msg.MsgPrefix if missing from an incoming message.
//
// If the destination mailbox has the Sent special-use flag, the message is parsed
// for its recipients (to/cc/bcc). Their domains are added to Recipients for use in
// dmarc reputation.
//
// If sync is true, the message file and its directory are synced. Should be true
// for regular mail delivery, but can be false when importing many messages.
//
// If updateDiskUsage is true, the account total message size (for quota) is
// updated. Callers must check if a message can be added within quota before
// calling DeliverMessage.
//
// If CreateSeq/ModSeq is not set, it is assigned automatically.
//
// Must be called with account rlock or wlock.
//
// Caller must broadcast new message.
//
// Caller must update mailbox counts.
func (a *Account) DeliverMessage(log mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads, updateDiskUsage bool) error {
if m.Expunged {
return fmt.Errorf("cannot deliver expunged message")
}
mb := Mailbox{ID: m.MailboxID}
if err := tx.Get(&mb); err != nil {
return fmt.Errorf("get mailbox: %w", err)
}
m.UID = mb.UIDNext
mb.UIDNext++
if err := tx.Update(&mb); err != nil {
return fmt.Errorf("updating mailbox nextuid: %w", err)
}
if updateDiskUsage {
du := DiskUsage{ID: 1}
if err := tx.Get(&du); err != nil {
return fmt.Errorf("get disk usage: %v", err)
}
du.MessageSize += m.Size
if err := tx.Update(&du); err != nil {
return fmt.Errorf("update disk usage: %v", err)
}
}
conf, _ := a.Conf()
m.JunkFlagsForMailbox(mb, conf)
mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.
var part *message.Part
if m.ParsedBuf == nil {
p, err := message.EnsurePart(log.Logger, false, mr, m.Size)
if err != nil {
log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID))
// We continue, p is still valid.
}
part = &p
buf, err := json.Marshal(part)
if err != nil {
return fmt.Errorf("marshal parsed message: %w", err)
}
m.ParsedBuf = buf
} else {
var p message.Part
if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {
log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))
} else {
part = &p
}
}
// If we are delivering to the originally intended mailbox, no need to store the mailbox ID again.
if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {
m.MailboxDestinedID = 0
}
if m.CreateSeq == 0 || m.ModSeq == 0 {
modseq, err := a.NextModSeq(tx)
if err != nil {
return fmt.Errorf("assigning next modseq: %w", err)
}
m.CreateSeq = modseq
m.ModSeq = modseq
}
if part != nil && m.MessageID == "" && m.SubjectBase == "" {
m.PrepareThreading(log, part)
}
// Assign to thread (if upgrade has completed).
noThreadID := nothreads
if m.ThreadID == 0 && !nothreads && part != nil {
select {
case <-a.threadsCompleted:
if a.threadsErr != nil {
log.Info("not assigning threads for new delivery, upgrading to threads failed")
noThreadID = true
} else {
if err := assignThread(log, tx, m, part); err != nil {
return fmt.Errorf("assigning thread: %w", err)
}
}
default:
// note: since we have a write transaction to get here, we can't wait for the
// thread upgrade to finish.
// If we don't assign a threadid the upgrade process will do it.
log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message")
noThreadID = true
}
}
if err := tx.Insert(m); err != nil {
return fmt.Errorf("inserting message: %w", err)
}
if !noThreadID && m.ThreadID == 0 {
m.ThreadID = m.ID
if err := tx.Update(m); err != nil {
return fmt.Errorf("updating message for its own thread id: %w", err)
}
}
// todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's if the mail client doesn't save a message that includes the bcc header in the sent mailbox.
if mb.Sent && part != nil && part.Envelope != nil {
e := part.Envelope
sent := e.Date
if sent.IsZero() {
sent = m.Received
}
if sent.IsZero() {
sent = time.Now()
}
addrs := append(append(e.To, e.CC...), e.BCC...)
for _, addr := range addrs {
if addr.User == "" {
// Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.
log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr))
continue
}
d, err := dns.ParseDomain(addr.Host)
if err != nil {
log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr))
continue
}
lp, err := smtp.ParseLocalpart(addr.User)
if err != nil {
log.Debugx("parsing localpart in to/cc/bcc address", err, slog.Any("address", addr))
continue
}
mr := Recipient{
MessageID: m.ID,
Localpart: lp.String(),
Domain: d.Name(),
OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(),
Sent: sent,
}
if err := tx.Insert(&mr); err != nil {
return fmt.Errorf("inserting sent message recipients: %w", err)
}
}
}
msgPath := a.MessagePath(m.ID)
msgDir := filepath.Dir(msgPath)
os.MkdirAll(msgDir, 0770)
// Sync file data to disk.
if sync {
if err := msgFile.Sync(); err != nil {
return fmt.Errorf("fsync message file: %w", err)
}
}
if err := moxio.LinkOrCopy(log, msgPath, msgFile.Name(), &moxio.AtReader{R: msgFile}, true); err != nil {
return fmt.Errorf("linking/copying message to new file: %w", err)
}
if sync {
if err := moxio.SyncDir(log, msgDir); err != nil {
xerr := os.Remove(msgPath)
log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))
return fmt.Errorf("sync directory: %w", err)
}
}
if !notrain && m.NeedsTraining() {
l := []Message{*m}
if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil {
xerr := os.Remove(msgPath)
log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))
return fmt.Errorf("training junkfilter: %w", err)
}
*m = l[0]
}
return nil
}
// SetPassword saves a new password for this account. This password is used for
// IMAP, SMTP (submission) sessions and the HTTP account web page.
func (a *Account) SetPassword(log mlog.Log, password string) error {
password, err := precis.OpaqueString.String(password)
if err != nil {
return fmt.Errorf(`password not allowed by "precis"`)
}
if len(password) < 8 {
// We actually check for bytes...
return fmt.Errorf("password must be at least 8 characters long")
}
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
return fmt.Errorf("generating password hash: %w", err)
}
err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {
return fmt.Errorf("deleting existing password: %v", err)
}
var pw Password
pw.Hash = string(hash)
// CRAM-MD5 calculates an HMAC-MD5, with the password as key, over a per-attempt
// unique text that includes a timestamp. HMAC performs two hashes. Both times, the
// first block is based on the key/password. We hash those first blocks now, and
// store the hash state in the database. When we actually authenticate, we'll
// complete the HMAC by hashing only the text. We cannot store crypto/hmac's hash,
// because it does not expose its internal state and isn't a BinaryMarshaler.
// ../rfc/2104:121
pw.CRAMMD5.Ipad = md5.New()
pw.CRAMMD5.Opad = md5.New()
key := []byte(password)
if len(key) > 64 {
t := md5.Sum(key)
key = t[:]
}
ipad := make([]byte, md5.BlockSize)
opad := make([]byte, md5.BlockSize)
copy(ipad, key)
copy(opad, key)
for i := range ipad {
ipad[i] ^= 0x36
opad[i] ^= 0x5c
}
pw.CRAMMD5.Ipad.Write(ipad)
pw.CRAMMD5.Opad.Write(opad)
pw.SCRAMSHA1.Salt = scram.MakeRandom()
pw.SCRAMSHA1.Iterations = 2 * 4096
pw.SCRAMSHA1.SaltedPassword = scram.SaltPassword(sha1.New, password, pw.SCRAMSHA1.Salt, pw.SCRAMSHA1.Iterations)
pw.SCRAMSHA256.Salt = scram.MakeRandom()
pw.SCRAMSHA256.Iterations = 4096
pw.SCRAMSHA256.SaltedPassword = scram.SaltPassword(sha256.New, password, pw.SCRAMSHA256.Salt, pw.SCRAMSHA256.Iterations)
if err := tx.Insert(&pw); err != nil {
return fmt.Errorf("inserting new password: %v", err)
}
return sessionRemoveAll(context.TODO(), log, tx, a.Name)
})
if err == nil {
log.Info("new password set for account", slog.String("account", a.Name))
}
return err
}
// Subjectpass returns the signing key for use with subjectpass for the given
// email address with canonical localpart.
func (a *Account) Subjectpass(email string) (key string, err error) {
return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
v := Subjectpass{Email: email}
err := tx.Get(&v)
if err == nil {
key = v.Key
return nil
}
if !errors.Is(err, bstore.ErrAbsent) {
return fmt.Errorf("get subjectpass key from accounts database: %w", err)
}
key = ""
const chars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
buf := make([]byte, 16)
if _, err := cryptorand.Read(buf); err != nil {
return err
}
for _, b := range buf {
key += string(chars[int(b)%len(chars)])
}
v.Key = key
return tx.Insert(&v)
})
}
// Ensure mailbox is present in database, adding records for the mailbox and its
// parents if they aren't present.
//
// If subscribe is true, any mailboxes that were created will also be subscribed to.
// Caller must hold account wlock.
// Caller must propagate changes if any.
func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool) (mb Mailbox, changes []Change, rerr error) {
if norm.NFC.String(name) != name {
return Mailbox{}, nil, fmt.Errorf("mailbox name not normalized")
}
// Quick sanity check.
if strings.EqualFold(name, "inbox") && name != "Inbox" {
return Mailbox{}, nil, fmt.Errorf("bad casing for inbox")
}
elems := strings.Split(name, "/")
q := bstore.QueryTx[Mailbox](tx)
q.FilterFn(func(mb Mailbox) bool {
return mb.Name == elems[0] || strings.HasPrefix(mb.Name, elems[0]+"/")
})
l, err := q.List()
if err != nil {
return Mailbox{}, nil, fmt.Errorf("list mailboxes: %v", err)
}
mailboxes := map[string]Mailbox{}
for _, xmb := range l {
mailboxes[xmb.Name] = xmb
}
p := ""
for _, elem := range elems {
if p != "" {
p += "/"
}
p += elem
var ok bool
mb, ok = mailboxes[p]
if ok {
continue
}
uidval, err := a.NextUIDValidity(tx)
if err != nil {
return Mailbox{}, nil, fmt.Errorf("next uid validity: %v", err)
}
mb = Mailbox{
Name: p,
UIDValidity: uidval,
UIDNext: 1,
HaveCounts: true,
}
err = tx.Insert(&mb)
if err != nil {
return Mailbox{}, nil, fmt.Errorf("creating new mailbox: %v", err)
}
var flags []string
if subscribe {
if tx.Get(&Subscription{p}) != nil {
err := tx.Insert(&Subscription{p})
if err != nil {
return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox: %v", err)
}
}
flags = []string{`\Subscribed`}
}
changes = append(changes, ChangeAddMailbox{mb, flags})
}
return mb, changes, nil
}
// MailboxExists checks if mailbox exists.
// Caller must hold account rlock.
func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) {
q := bstore.QueryTx[Mailbox](tx)
q.FilterEqual("Name", name)
return q.Exists()
}
// MailboxFind finds a mailbox by name, returning a nil mailbox and nil error if mailbox does not exist.
func (a *Account) MailboxFind(tx *bstore.Tx, name string) (*Mailbox, error) {
q := bstore.QueryTx[Mailbox](tx)
q.FilterEqual("Name", name)
mb, err := q.Get()
if err == bstore.ErrAbsent {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("looking up mailbox: %w", err)
}
return &mb, nil
}
// SubscriptionEnsure ensures a subscription for name exists. The mailbox does not
// have to exist. Any parents are not automatically subscribed.
// Changes are returned and must be broadcasted by the caller.
func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, error) {
if err := tx.Get(&Subscription{name}); err == nil {
return nil, nil
}
if err := tx.Insert(&Subscription{name}); err != nil {
return nil, fmt.Errorf("inserting subscription: %w", err)
}
q := bstore.QueryTx[Mailbox](tx)
q.FilterEqual("Name", name)
_, err := q.Get()
if err == nil {
return []Change{ChangeAddSubscription{name, nil}}, nil
} else if err != bstore.ErrAbsent {
return nil, fmt.Errorf("looking up mailbox for subscription: %w", err)
}
return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil
}
// MessageRuleset returns the first ruleset (if any) that matches the message
// represented by msgPrefix and msgFile, with smtp and validation fields from m.
func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {
if len(dest.Rulesets) == 0 {
return nil
}
mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.
p, err := message.Parse(log.Logger, false, mr)
if err != nil {
log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", ""))
// note: part is still set.
}
// todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.
header, err := p.Header()
if err != nil {
log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", ""))
// todo: reject message?
return nil
}
ruleset:
for _, rs := range dest.Rulesets {
if rs.SMTPMailFromRegexpCompiled != nil {
if !rs.SMTPMailFromRegexpCompiled.MatchString(m.MailFrom) {
continue ruleset
}
}
if rs.MsgFromRegexpCompiled != nil {
if m.MsgFromLocalpart == "" && m.MsgFromDomain == "" || !rs.MsgFromRegexpCompiled.MatchString(m.MsgFromLocalpart.String()+"@"+m.MsgFromDomain) {
continue ruleset
}
}
if !rs.VerifiedDNSDomain.IsZero() {
d := rs.VerifiedDNSDomain.Name()
suffix := "." + d
matchDomain := func(s string) bool {
return s == d || strings.HasSuffix(s, suffix)
}
var ok bool
if m.EHLOValidated && matchDomain(m.EHLODomain) {
ok = true
}
if m.MailFromValidated && matchDomain(m.MailFromDomain) {
ok = true
}
for _, d := range m.DKIMDomains {
if matchDomain(d) {
ok = true
break
}
}
if !ok {
continue ruleset
}
}
header:
for _, t := range rs.HeadersRegexpCompiled {
for k, vl := range header {
k = strings.ToLower(k)
if !t[0].MatchString(k) {
continue
}
for _, v := range vl {
v = strings.ToLower(strings.TrimSpace(v))
if t[1].MatchString(v) {
continue header
}
}
}
continue ruleset
}
return &rs
}
return nil
}
// MessagePath returns the file system path of a message.
func (a *Account) MessagePath(messageID int64) string {
return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), string(filepath.Separator))
}
// MessageReader opens a message for reading, transparently combining the
// message prefix with the original incoming message.
func (a *Account) MessageReader(m Message) *MsgReader {
return &MsgReader{prefix: m.MsgPrefix, path: a.MessagePath(m.ID), size: m.Size}
}
// DeliverDestination delivers an email to dest, based on the configured rulesets.
//
// Returns ErrOverQuota when account would be over quota after adding message.
//
// Caller must hold account wlock (mailbox may be created).
// Message delivery, possible mailbox creation, and updated mailbox counts are
// broadcasted.
func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {
var mailbox string
rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)
if rs != nil {
mailbox = rs.Mailbox
} else if dest.Mailbox == "" {
mailbox = "Inbox"
} else {
mailbox = dest.Mailbox
}
return a.DeliverMailbox(log, mailbox, m, msgFile)
}
// DeliverMailbox delivers an email to the specified mailbox.
//
// Returns ErrOverQuota when account would be over quota after adding message.
//
// Caller must hold account wlock (mailbox may be created).
// Message delivery, possible mailbox creation, and updated mailbox counts are
// broadcasted.
func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFile *os.File) error {
var changes []Change
err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
if ok, _, err := a.CanAddMessageSize(tx, m.Size); err != nil {
return err
} else if !ok {
return ErrOverQuota
}
mb, chl, err := a.MailboxEnsure(tx, mailbox, true)
if err != nil {
return fmt.Errorf("ensuring mailbox: %w", err)
}
m.MailboxID = mb.ID
m.MailboxOrigID = mb.ID
// Update count early, DeliverMessage will update mb too and we don't want to fetch
// it again before updating.
mb.MailboxCounts.Add(m.MailboxCounts())
if err := tx.Update(&mb); err != nil {
return fmt.Errorf("updating mailbox for delivery: %w", err)
}
if err := a.DeliverMessage(log, tx, m, msgFile, true, false, false, true); err != nil {
return err
}
changes = append(changes, chl...)
changes = append(changes, m.ChangeAddUID(), mb.ChangeCounts())
return nil
})
// todo: if rename succeeded but transaction failed, we should remove the file.
if err != nil {
return err
}
BroadcastChanges(a, changes)
return nil
}
// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery.
//
// Caller most hold account wlock.
// Changes are broadcasted.
func (a *Account) TidyRejectsMailbox(log mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) {
var changes []Change
var remove []Message
defer func() {
for _, m := range remove {
p := a.MessagePath(m.ID)
err := os.Remove(p)
log.Check(err, "removing rejects message file", slog.String("path", p))
}
}()
err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
mb, err := a.MailboxFind(tx, rejectsMailbox)
if err != nil {
return fmt.Errorf("finding mailbox: %w", err)
}
if mb == nil {
// No messages have been delivered yet.
hasSpace = true
return nil
}
// Gather old messages to remove.
old := time.Now().Add(-14 * 24 * time.Hour)
qdel := bstore.QueryTx[Message](tx)
qdel.FilterNonzero(Message{MailboxID: mb.ID})
qdel.FilterEqual("Expunged", false)
qdel.FilterLess("Received", old)
remove, err = qdel.List()
if err != nil {
return fmt.Errorf("listing old messages: %w", err)
}
changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)
if err != nil {
return fmt.Errorf("removing messages: %w", err)
}
// We allow up to n messages.
qcount := bstore.QueryTx[Message](tx)
qcount.FilterNonzero(Message{MailboxID: mb.ID})
qcount.FilterEqual("Expunged", false)
qcount.Limit(1000)
n, err := qcount.Count()
if err != nil {
return fmt.Errorf("counting rejects: %w", err)
}
hasSpace = n < 1000
return nil
})
if err != nil {
remove = nil // Don't remove files on failure.
return false, err
}
BroadcastChanges(a, changes)
return hasSpace, nil
}
func (a *Account) rejectsRemoveMessages(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {
if len(l) == 0 {
return nil, nil
}
ids := make([]int64, len(l))
anyids := make([]any, len(l))
for i, m := range l {
ids[i] = m.ID
anyids[i] = m.ID
}
// Remove any message recipients. Should not happen, but a user can move messages
// from a Sent mailbox to the rejects mailbox...
qdmr := bstore.QueryTx[Recipient](tx)
qdmr.FilterEqual("MessageID", anyids...)
if _, err := qdmr.Delete(); err != nil {
return nil, fmt.Errorf("deleting from message recipient: %w", err)
}
// Assign new modseq.
modseq, err := a.NextModSeq(tx)
if err != nil {
return nil, fmt.Errorf("assign next modseq: %w", err)
}
// Expunge the messages.
qx := bstore.QueryTx[Message](tx)
qx.FilterIDs(ids)
var expunged []Message
qx.Gather(&expunged)
if _, err := qx.UpdateNonzero(Message{ModSeq: modseq, Expunged: true}); err != nil {
return nil, fmt.Errorf("expunging messages: %w", err)
}
var totalSize int64
for _, m := range expunged {
m.Expunged = false // Was set by update, but would cause wrong count.
mb.MailboxCounts.Sub(m.MailboxCounts())
totalSize += m.Size
}
if err := tx.Update(mb); err != nil {
return nil, fmt.Errorf("updating mailbox counts: %w", err)
}
if err := a.AddMessageSize(log, tx, -totalSize); err != nil {
return nil, fmt.Errorf("updating disk usage: %w", err)
}
// Mark as neutral and train so junk filter gets untrained with these (junk) messages.
for i := range expunged {
expunged[i].Junk = false
expunged[i].Notjunk = false
}
if err := a.RetrainMessages(ctx, log, tx, expunged, true); err != nil {
return nil, fmt.Errorf("retraining expunged messages: %w", err)
}
changes := make([]Change, len(l), len(l)+1)
for i, m := range l {
changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}, modseq}
}
changes = append(changes, mb.ChangeCounts())
return changes, nil
}
// RejectsRemove removes a message from the rejects mailbox if present.
// Caller most hold account wlock.
// Changes are broadcasted.
func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error {
var changes []Change
var remove []Message
defer func() {
for _, m := range remove {
p := a.MessagePath(m.ID)
err := os.Remove(p)
log.Check(err, "removing rejects message file", slog.String("path", p))
}
}()
err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
mb, err := a.MailboxFind(tx, rejectsMailbox)
if err != nil {
return fmt.Errorf("finding mailbox: %w", err)
}
if mb == nil {
return nil
}
q := bstore.QueryTx[Message](tx)
q.FilterNonzero(Message{MailboxID: mb.ID, MessageID: messageID})
q.FilterEqual("Expunged", false)
remove, err = q.List()
if err != nil {
return fmt.Errorf("listing messages to remove: %w", err)
}
changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)
if err != nil {
return fmt.Errorf("removing messages: %w", err)
}
return nil
})
if err != nil {
remove = nil // Don't remove files on failure.
return err
}
BroadcastChanges(a, changes)
return nil
}
// AddMessageSize adjusts the DiskUsage.MessageSize by size.
func (a *Account) AddMessageSize(log mlog.Log, tx *bstore.Tx, size int64) error {
du := DiskUsage{ID: 1}
if err := tx.Get(&du); err != nil {
return fmt.Errorf("get diskusage: %v", err)
}
du.MessageSize += size
if du.MessageSize < 0 {
log.Error("negative total message size", slog.Int64("delta", size), slog.Int64("newtotalsize", du.MessageSize))
}
if err := tx.Update(&du); err != nil {
return fmt.Errorf("update total message size: %v", err)
}
return nil
}
// QuotaMessageSize returns the effective maximum total message size for an
// account. Returns 0 if there is no maximum.
func (a *Account) QuotaMessageSize() int64 {
conf, _ := a.Conf()
size := conf.QuotaMessageSize
if size == 0 {
size = mox.Conf.Static.QuotaMessageSize
}
if size < 0 {
size = 0
}
return size
}
// CanAddMessageSize checks if a message of size bytes can be added, depending on
// total message size and configured quota for account.
func (a *Account) CanAddMessageSize(tx *bstore.Tx, size int64) (ok bool, maxSize int64, err error) {
maxSize = a.QuotaMessageSize()
if maxSize <= 0 {
return true, 0, nil
}
du := DiskUsage{ID: 1}
if err := tx.Get(&du); err != nil {
return false, maxSize, fmt.Errorf("get diskusage: %v", err)
}
return du.MessageSize+size <= maxSize, maxSize, nil
}
// We keep a cache of recent successful authentications, so we don't have to bcrypt successful calls each time.
var authCache = struct {
sync.Mutex
success map[authKey]string
}{
success: map[authKey]string{},
}
type authKey struct {
email, hash string
}
// StartAuthCache starts a goroutine that regularly clears the auth cache.
func StartAuthCache() {
go manageAuthCache()
}
func manageAuthCache() {
for {
authCache.Lock()
authCache.success = map[authKey]string{}
authCache.Unlock()
time.Sleep(15 * time.Minute)
}
}
// OpenEmailAuth opens an account given an email address and password.
//
// The email address may contain a catchall separator.
func OpenEmailAuth(log mlog.Log, email string, password string) (acc *Account, rerr error) {
password, err := precis.OpaqueString.String(password)
if err != nil {
return nil, ErrUnknownCredentials
}
acc, _, rerr = OpenEmail(log, email)
if rerr != nil {
return
}
defer func() {
if rerr != nil && acc != nil {
err := acc.Close()
log.Check(err, "closing account after open auth failure")
acc = nil
}
}()
pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()
if err != nil {
if err == bstore.ErrAbsent {
return acc, ErrUnknownCredentials
}
return acc, fmt.Errorf("looking up password: %v", err)
}
authCache.Lock()
ok := len(password) >= 8 && authCache.success[authKey{email, pw.Hash}] == password
authCache.Unlock()
if ok {
return
}
if err := bcrypt.CompareHashAndPassword([]byte(pw.Hash), []byte(password)); err != nil {
rerr = ErrUnknownCredentials
} else {
authCache.Lock()
authCache.success[authKey{email, pw.Hash}] = password
authCache.Unlock()
}
return
}
// OpenEmail opens an account given an email address.
//
// The email address may contain a catchall separator.
func OpenEmail(log mlog.Log, email string) (*Account, config.Destination, error) {
addr, err := smtp.ParseAddress(email)
if err != nil {
return nil, config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)
}
accountName, _, _, dest, err := mox.LookupAddress(addr.Localpart, addr.Domain, false, false)
if err != nil && (errors.Is(err, mox.ErrAddressNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {
return nil, config.Destination{}, ErrUnknownCredentials
} else if err != nil {
return nil, config.Destination{}, fmt.Errorf("looking up address: %v", err)
}
acc, err := OpenAccount(log, accountName)
if err != nil {
return nil, config.Destination{}, err
}
return acc, dest, nil
}
// 64 characters, must be power of 2 for MessagePath
const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-_"
// MessagePath returns the filename of the on-disk filename, relative to the
// containing directory such as <account>/msg or queue.
// Returns names like "AB/1".
func MessagePath(messageID int64) string {
return strings.Join(messagePathElems(messageID), string(filepath.Separator))
}
// messagePathElems returns the elems, for a single join without intermediate
// string allocations.
func messagePathElems(messageID int64) []string {
v := messageID >> 13 // 8k files per directory.
dir := ""
for {
dir += string(msgDirChars[int(v)&(len(msgDirChars)-1)])
v >>= 6
if v == 0 {
break
}
}
return []string{dir, strconv.FormatInt(messageID, 10)}
}
// Set returns a copy of f, with each flag that is true in mask set to the
// value from flags.
func (f Flags) Set(mask, flags Flags) Flags {
set := func(d *bool, m, v bool) {
if m {
*d = v
}
}
r := f
set(&r.Seen, mask.Seen, flags.Seen)
set(&r.Answered, mask.Answered, flags.Answered)
set(&r.Flagged, mask.Flagged, flags.Flagged)
set(&r.Forwarded, mask.Forwarded, flags.Forwarded)
set(&r.Junk, mask.Junk, flags.Junk)
set(&r.Notjunk, mask.Notjunk, flags.Notjunk)
set(&r.Deleted, mask.Deleted, flags.Deleted)
set(&r.Draft, mask.Draft, flags.Draft)
set(&r.Phishing, mask.Phishing, flags.Phishing)
set(&r.MDNSent, mask.MDNSent, flags.MDNSent)
return r
}
// Changed returns a mask of flags that have been between f and other.
func (f Flags) Changed(other Flags) (mask Flags) {
mask.Seen = f.Seen != other.Seen
mask.Answered = f.Answered != other.Answered
mask.Flagged = f.Flagged != other.Flagged
mask.Forwarded = f.Forwarded != other.Forwarded
mask.Junk = f.Junk != other.Junk
mask.Notjunk = f.Notjunk != other.Notjunk
mask.Deleted = f.Deleted != other.Deleted
mask.Draft = f.Draft != other.Draft
mask.Phishing = f.Phishing != other.Phishing
mask.MDNSent = f.MDNSent != other.MDNSent
return
}
// Strings returns the flags that are set in their string form.
func (f Flags) Strings() []string {
fields := []struct {
word string
have bool
}{
{`$forwarded`, f.Forwarded},
{`$junk`, f.Junk},
{`$mdnsent`, f.MDNSent},
{`$notjunk`, f.Notjunk},
{`$phishing`, f.Phishing},
{`\answered`, f.Answered},
{`\deleted`, f.Deleted},
{`\draft`, f.Draft},
{`\flagged`, f.Flagged},
{`\seen`, f.Seen},
}
var l []string
for _, fh := range fields {
if fh.have {
l = append(l, fh.word)
}
}
return l
}
var systemWellKnownFlags = map[string]bool{
`\answered`: true,
`\flagged`: true,
`\deleted`: true,
`\seen`: true,
`\draft`: true,
`$junk`: true,
`$notjunk`: true,
`$forwarded`: true,
`$phishing`: true,
`$mdnsent`: true,
}
// ParseFlagsKeywords parses a list of textual flags into system/known flags, and
// other keywords. Keywords are lower-cased and sorted and check for valid syntax.
func ParseFlagsKeywords(l []string) (flags Flags, keywords []string, rerr error) {
fields := map[string]*bool{
`\answered`: &flags.Answered,
`\flagged`: &flags.Flagged,
`\deleted`: &flags.Deleted,
`\seen`: &flags.Seen,
`\draft`: &flags.Draft,
`$junk`: &flags.Junk,
`$notjunk`: &flags.Notjunk,
`$forwarded`: &flags.Forwarded,
`$phishing`: &flags.Phishing,
`$mdnsent`: &flags.MDNSent,
}
seen := map[string]bool{}
for _, f := range l {
f = strings.ToLower(f)
if field, ok := fields[f]; ok {
*field = true
} else if seen[f] {
if mox.Pedantic {
return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f)
}
} else {
if err := CheckKeyword(f); err != nil {
return Flags{}, nil, fmt.Errorf("invalid keyword %s", f)
}
keywords = append(keywords, f)
seen[f] = true
}
}
sort.Strings(keywords)
return flags, keywords, nil
}
// RemoveKeywords removes keywords from l, returning whether any modifications were
// made, and a slice, a new slice in case of modifications. Keywords must have been
// validated earlier, e.g. through ParseFlagKeywords or CheckKeyword. Should only
// be used with valid keywords, not with system flags like \Seen.
func RemoveKeywords(l, remove []string) ([]string, bool) {
var copied bool
var changed bool
for _, k := range remove {
if i := slices.Index(l, k); i >= 0 {
if !copied {
l = append([]string{}, l...)
copied = true
}
copy(l[i:], l[i+1:])
l = l[:len(l)-1]
changed = true
}
}
return l, changed
}
// MergeKeywords adds keywords from add into l, returning whether it added any
// keyword, and the slice with keywords, a new slice if modifications were made.
// Keywords are only added if they aren't already present. Should only be used with
// keywords, not with system flags like \Seen.
func MergeKeywords(l, add []string) ([]string, bool) {
var copied bool
var changed bool
for _, k := range add {
if !slices.Contains(l, k) {
if !copied {
l = append([]string{}, l...)
copied = true
}
l = append(l, k)
changed = true
}
}
if changed {
sort.Strings(l)
}
return l, changed
}
// CheckKeyword returns an error if kw is not a valid keyword. Kw should
// already be in lower-case.
func CheckKeyword(kw string) error {
if kw == "" {
return fmt.Errorf("keyword cannot be empty")
}
if systemWellKnownFlags[kw] {
return fmt.Errorf("cannot use well-known flag as keyword")
}
for _, c := range kw {
// ../rfc/9051:6334
if c <= ' ' || c > 0x7e || c >= 'A' && c <= 'Z' || strings.ContainsRune(`(){%*"\]`, c) {
return errors.New(`not a valid keyword, must be lower-case ascii without spaces and without any of these characters: (){%*"\]`)
}
}
return nil
}
// SendLimitReached checks whether sending a message to recipients would reach
// the limit of outgoing messages for the account. If so, the message should
// not be sent. If the returned numbers are >= 0, the limit was reached and the
// values are the configured limits.
//
// To limit damage to the internet and our reputation in case of account
// compromise, we limit the max number of messages sent in a 24 hour window, both
// total number of messages and number of first-time recipients.
func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msglimit, rcptlimit int, rerr error) {
conf, _ := a.Conf()
msgmax := conf.MaxOutgoingMessagesPerDay
if msgmax == 0 {
// For human senders, 1000 recipients in a day is quite a lot.
msgmax = 1000
}
rcptmax := conf.MaxFirstTimeRecipientsPerDay
if rcptmax == 0 {
// Human senders may address a new human-sized list of people once in a while. In
// case of a compromise, a spammer will probably try to send to many new addresses.
rcptmax = 200
}
rcpts := map[string]time.Time{}
n := 0
err := bstore.QueryTx[Outgoing](tx).FilterGreater("Submitted", time.Now().Add(-24*time.Hour)).ForEach(func(o Outgoing) error {
n++
if rcpts[o.Recipient].IsZero() || o.Submitted.Before(rcpts[o.Recipient]) {
rcpts[o.Recipient] = o.Submitted
}
return nil
})
if err != nil {
return -1, -1, fmt.Errorf("querying message recipients in past 24h: %w", err)
}
if n+len(recipients) > msgmax {
return msgmax, -1, nil
}
// Only check if max first-time recipients is reached if there are enough messages
// to trigger the limit.
if n+len(recipients) < rcptmax {
return -1, -1, nil
}
isFirstTime := func(rcpt string, before time.Time) (bool, error) {
exists, err := bstore.QueryTx[Outgoing](tx).FilterNonzero(Outgoing{Recipient: rcpt}).FilterLess("Submitted", before).Exists()
return !exists, err
}
firsttime := 0
now := time.Now()
for _, r := range recipients {
if first, err := isFirstTime(r.XString(true), now); err != nil {
return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)
} else if first {
firsttime++
}
}
for r, t := range rcpts {
if first, err := isFirstTime(r, t); err != nil {
return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)
} else if first {
firsttime++
}
}
if firsttime > rcptmax {
return -1, rcptmax, nil
}
return -1, -1, nil
}
// MailboxCreate creates a new mailbox, including any missing parent mailboxes,
// the total list of created mailboxes is returned in created. On success, if
// exists is false and rerr nil, the changes must be broadcasted by the caller.
//
// Name must be in normalized form.
func (a *Account) MailboxCreate(tx *bstore.Tx, name string) (changes []Change, created []string, exists bool, rerr error) {
elems := strings.Split(name, "/")
var p string
for i, elem := range elems {
if i > 0 {
p += "/"
}
p += elem
exists, err := a.MailboxExists(tx, p)
if err != nil {
return nil, nil, false, fmt.Errorf("checking if mailbox exists")
}
if exists {
if i == len(elems)-1 {
return nil, nil, true, fmt.Errorf("mailbox already exists")
}
continue
}
_, nchanges, err := a.MailboxEnsure(tx, p, true)
if err != nil {
return nil, nil, false, fmt.Errorf("ensuring mailbox exists")
}
changes = append(changes, nchanges...)
created = append(created, p)
}
return changes, created, false, nil
}
// MailboxRename renames mailbox mbsrc to dst, and any missing parents for the
// destination, and any children of mbsrc and the destination.
//
// Names must be normalized and cannot be Inbox.
func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (changes []Change, isInbox, notExists, alreadyExists bool, rerr error) {
if mbsrc.Name == "Inbox" || dst == "Inbox" {
return nil, true, false, false, fmt.Errorf("inbox cannot be renamed")
}
// We gather existing mailboxes that we need for deciding what to create/delete/update.
q := bstore.QueryTx[Mailbox](tx)
srcPrefix := mbsrc.Name + "/"
dstRoot := strings.SplitN(dst, "/", 2)[0]
dstRootPrefix := dstRoot + "/"
q.FilterFn(func(mb Mailbox) bool {
return mb.Name == mbsrc.Name || strings.HasPrefix(mb.Name, srcPrefix) || mb.Name == dstRoot || strings.HasPrefix(mb.Name, dstRootPrefix)
})
q.SortAsc("Name") // We'll rename the parents before children.
l, err := q.List()
if err != nil {
return nil, false, false, false, fmt.Errorf("listing relevant mailboxes: %v", err)
}
mailboxes := map[string]Mailbox{}
for _, mb := range l {
mailboxes[mb.Name] = mb
}
if _, ok := mailboxes[mbsrc.Name]; !ok {
return nil, false, true, false, fmt.Errorf("mailbox does not exist")
}
uidval, err := a.NextUIDValidity(tx)
if err != nil {
return nil, false, false, false, fmt.Errorf("next uid validity: %v", err)
}
// Ensure parent mailboxes for the destination paths exist.
var parent string
dstElems := strings.Split(dst, "/")
for i, elem := range dstElems[:len(dstElems)-1] {
if i > 0 {
parent += "/"
}
parent += elem
mb, ok := mailboxes[parent]
if ok {
continue
}
omb := mb
mb = Mailbox{
ID: omb.ID,
Name: parent,
UIDValidity: uidval,
UIDNext: 1,
HaveCounts: true,
}
if err := tx.Insert(&mb); err != nil {
return nil, false, false, false, fmt.Errorf("creating parent mailbox %q: %v", mb.Name, err)
}
if err := tx.Get(&Subscription{Name: parent}); err != nil {
if err := tx.Insert(&Subscription{Name: parent}); err != nil {
return nil, false, false, false, fmt.Errorf("creating subscription for %q: %v", parent, err)
}
}
changes = append(changes, ChangeAddMailbox{Mailbox: mb, Flags: []string{`\Subscribed`}})
}
// Process src mailboxes, renaming them to dst.
for _, srcmb := range l {
if srcmb.Name != mbsrc.Name && !strings.HasPrefix(srcmb.Name, srcPrefix) {
continue
}
srcName := srcmb.Name
dstName := dst + srcmb.Name[len(mbsrc.Name):]
if _, ok := mailboxes[dstName]; ok {
return nil, false, false, true, fmt.Errorf("destination mailbox %q already exists", dstName)
}
srcmb.Name = dstName
srcmb.UIDValidity = uidval
if err := tx.Update(&srcmb); err != nil {
return nil, false, false, false, fmt.Errorf("renaming mailbox: %v", err)
}
var dstFlags []string
if tx.Get(&Subscription{Name: dstName}) == nil {
dstFlags = []string{`\Subscribed`}
}
changes = append(changes, ChangeRenameMailbox{MailboxID: srcmb.ID, OldName: srcName, NewName: dstName, Flags: dstFlags})
}
// If we renamed e.g. a/b to a/b/c/d, and a/b/c to a/b/c/d/c, we'll have to recreate a/b and a/b/c.
srcElems := strings.Split(mbsrc.Name, "/")
xsrc := mbsrc.Name
for i := 0; i < len(dstElems) && strings.HasPrefix(dst, xsrc+"/"); i++ {
mb := Mailbox{
UIDValidity: uidval,
UIDNext: 1,
Name: xsrc,
HaveCounts: true,
}
if err := tx.Insert(&mb); err != nil {
return nil, false, false, false, fmt.Errorf("creating mailbox at old path %q: %v", mb.Name, err)
}
xsrc += "/" + dstElems[len(srcElems)+i]
}
return changes, false, false, false, nil
}
// MailboxDelete deletes a mailbox by ID. If it has children, the return value
// indicates that and an error is returned.
//
// Caller should broadcast the changes and remove files for the removed message IDs.
func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) {
// Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about
// NoInferior and NoSelect. We just require only leaf mailboxes are deleted.
qmb := bstore.QueryTx[Mailbox](tx)
mbprefix := mailbox.Name + "/"
qmb.FilterFn(func(mb Mailbox) bool {
return strings.HasPrefix(mb.Name, mbprefix)
})
if childExists, err := qmb.Exists(); err != nil {
return nil, nil, false, fmt.Errorf("checking if mailbox has child: %v", err)
} else if childExists {
return nil, nil, true, fmt.Errorf("mailbox has a child, only leaf mailboxes can be deleted")
}
// todo jmap: instead of completely deleting a mailbox and its messages, we need to mark them all as expunged.
qm := bstore.QueryTx[Message](tx)
qm.FilterNonzero(Message{MailboxID: mailbox.ID})
remove, err := qm.List()
if err != nil {
return nil, nil, false, fmt.Errorf("listing messages to remove: %v", err)
}
if len(remove) > 0 {
removeIDs := make([]any, len(remove))
for i, m := range remove {
removeIDs[i] = m.ID
}
qmr := bstore.QueryTx[Recipient](tx)
qmr.FilterEqual("MessageID", removeIDs...)
if _, err = qmr.Delete(); err != nil {
return nil, nil, false, fmt.Errorf("removing message recipients for messages: %v", err)
}
qm = bstore.QueryTx[Message](tx)
qm.FilterNonzero(Message{MailboxID: mailbox.ID})
if _, err := qm.Delete(); err != nil {
return nil, nil, false, fmt.Errorf("removing messages: %v", err)
}
var totalSize int64
for _, m := range remove {
if !m.Expunged {
removeMessageIDs = append(removeMessageIDs, m.ID)
totalSize += m.Size
}
}
if err := a.AddMessageSize(log, tx, -totalSize); err != nil {
return nil, nil, false, fmt.Errorf("updating disk usage: %v", err)
}
// Mark messages as not needing training. Then retrain them, so they are untrained if they were.
n := 0
o := 0
for _, m := range remove {
if !m.Expunged {
remove[o] = m
remove[o].Junk = false
remove[o].Notjunk = false
n++
}
}
remove = remove[:n]
if err := a.RetrainMessages(ctx, log, tx, remove, true); err != nil {
return nil, nil, false, fmt.Errorf("untraining deleted messages: %v", err)
}
}
if err := tx.Delete(&Mailbox{ID: mailbox.ID}); err != nil {
return nil, nil, false, fmt.Errorf("removing mailbox: %v", err)
}
return []Change{ChangeRemoveMailbox{MailboxID: mailbox.ID, Name: mailbox.Name}}, removeMessageIDs, false, nil
}
// CheckMailboxName checks if name is valid, returning an INBOX-normalized name.
// I.e. it changes various casings of INBOX and INBOX/* to Inbox and Inbox/*.
// Name is invalid if it contains leading/trailing/double slashes, or when it isn't
// unicode-normalized, or when empty or has special characters.
//
// If name is the inbox, and allowInbox is false, this is indicated with the isInbox return parameter.
// For that case, and for other invalid names, an error is returned.
func CheckMailboxName(name string, allowInbox bool) (normalizedName string, isInbox bool, rerr error) {
first := strings.SplitN(name, "/", 2)[0]
if strings.EqualFold(first, "inbox") {
if len(name) == len("inbox") && !allowInbox {
return "", true, fmt.Errorf("special mailbox name Inbox not allowed")
}
name = "Inbox" + name[len("Inbox"):]
}
if norm.NFC.String(name) != name {
return "", false, errors.New("non-unicode-normalized mailbox names not allowed")
}
if name == "" {
return "", false, errors.New("empty mailbox name")
}
if strings.HasPrefix(name, "/") || strings.HasSuffix(name, "/") || strings.Contains(name, "//") {
return "", false, errors.New("bad slashes in mailbox name")
}
// "%" and "*" are difficult to use with the IMAP LIST command, but we allow mostly
// allow them. ../rfc/3501:1002 ../rfc/9051:983
if strings.HasPrefix(name, "#") {
return "", false, errors.New("mailbox name cannot start with hash due to conflict with imap namespaces")
}
// "#" and "&" are special in IMAP mailbox names. "#" for namespaces, "&" for
// IMAP-UTF-7 encoding. We do allow them. ../rfc/3501:1018 ../rfc/9051:991
for _, c := range name {
// ../rfc/3501:999 ../rfc/6855:192 ../rfc/9051:979
if c <= 0x1f || c >= 0x7f && c <= 0x9f || c == 0x2028 || c == 0x2029 {
return "", false, errors.New("control characters not allowed in mailbox name")
}
}
return name, false, nil
}
Reference in a new issue View git blame Copy permalink