mirror of
https://github.com/mjl-/mox.git
synced 2024-12-27 08:53:48 +03:00
2638 lines
85 KiB
Go
2638 lines
85 KiB
Go
/*
|
|
Package store implements storage for accounts, their mailboxes, IMAP
|
|
subscriptions and messages, and broadcasts updates (e.g. mail delivery) to
|
|
interested sessions (e.g. IMAP connections).
|
|
|
|
Layout of storage for accounts:
|
|
|
|
<DataDir>/accounts/<name>/index.db
|
|
<DataDir>/accounts/<name>/msg/[a-zA-Z0-9_-]+/<id>
|
|
|
|
Index.db holds tables for user information, mailboxes, and messages. Messages
|
|
are stored in the msg/ subdirectory, each in their own file. The on-disk message
|
|
does not contain headers generated during an incoming SMTP transaction, such as
|
|
Received and Authentication-Results headers. Those are in the database to
|
|
prevent having to rewrite incoming messages (e.g. Authentication-Result for DKIM
|
|
signatures can only be determined after having read the message). Messages must
|
|
be read through MsgReader, which transparently adds the prefix from the
|
|
database.
|
|
*/
|
|
package store
|
|
|
|
// todo: make up a function naming scheme that indicates whether caller should broadcast changes.
|
|
|
|
import (
|
|
"context"
|
|
"crypto/md5"
|
|
cryptorand "crypto/rand"
|
|
"crypto/sha1"
|
|
"crypto/sha256"
|
|
"encoding"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"hash"
|
|
"io"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime/debug"
|
|
"slices"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"golang.org/x/crypto/bcrypt"
|
|
"golang.org/x/text/unicode/norm"
|
|
|
|
"github.com/mjl-/bstore"
|
|
|
|
"github.com/mjl-/mox/config"
|
|
"github.com/mjl-/mox/dns"
|
|
"github.com/mjl-/mox/message"
|
|
"github.com/mjl-/mox/metrics"
|
|
"github.com/mjl-/mox/mlog"
|
|
"github.com/mjl-/mox/mox-"
|
|
"github.com/mjl-/mox/moxio"
|
|
"github.com/mjl-/mox/publicsuffix"
|
|
"github.com/mjl-/mox/scram"
|
|
"github.com/mjl-/mox/smtp"
|
|
)
|
|
|
|
// If true, each time an account is closed its database file is checked for
|
|
// consistency. If an inconsistency is found, panic is called. Set by default
|
|
// because of all the packages with tests, the mox main function sets it to
|
|
// false again.
|
|
var CheckConsistencyOnClose = true
|
|
|
|
var (
|
|
ErrUnknownMailbox = errors.New("no such mailbox")
|
|
ErrUnknownCredentials = errors.New("credentials not found")
|
|
ErrAccountUnknown = errors.New("no such account")
|
|
ErrOverQuota = errors.New("account over quota")
|
|
)
|
|
|
|
var DefaultInitialMailboxes = config.InitialMailboxes{
|
|
SpecialUse: config.SpecialUseMailboxes{
|
|
Sent: "Sent",
|
|
Archive: "Archive",
|
|
Trash: "Trash",
|
|
Draft: "Drafts",
|
|
Junk: "Junk",
|
|
},
|
|
}
|
|
|
|
type SCRAM struct {
|
|
Salt []byte
|
|
Iterations int
|
|
SaltedPassword []byte
|
|
}
|
|
|
|
// CRAMMD5 holds HMAC ipad and opad hashes that are initialized with the first
|
|
// block with (a derivation of) the key/password, so we don't store the password in plain
|
|
// text.
|
|
type CRAMMD5 struct {
|
|
Ipad hash.Hash
|
|
Opad hash.Hash
|
|
}
|
|
|
|
// BinaryMarshal is used by bstore to store the ipad/opad hash states.
|
|
func (c CRAMMD5) MarshalBinary() ([]byte, error) {
|
|
if c.Ipad == nil || c.Opad == nil {
|
|
return nil, nil
|
|
}
|
|
|
|
ipad, err := c.Ipad.(encoding.BinaryMarshaler).MarshalBinary()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("marshal ipad: %v", err)
|
|
}
|
|
opad, err := c.Opad.(encoding.BinaryMarshaler).MarshalBinary()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("marshal opad: %v", err)
|
|
}
|
|
buf := make([]byte, 2+len(ipad)+len(opad))
|
|
ipadlen := uint16(len(ipad))
|
|
buf[0] = byte(ipadlen >> 8)
|
|
buf[1] = byte(ipadlen >> 0)
|
|
copy(buf[2:], ipad)
|
|
copy(buf[2+len(ipad):], opad)
|
|
return buf, nil
|
|
}
|
|
|
|
// BinaryUnmarshal is used by bstore to restore the ipad/opad hash states.
|
|
func (c *CRAMMD5) UnmarshalBinary(buf []byte) error {
|
|
if len(buf) == 0 {
|
|
*c = CRAMMD5{}
|
|
return nil
|
|
}
|
|
if len(buf) < 2 {
|
|
return fmt.Errorf("short buffer")
|
|
}
|
|
ipadlen := int(uint16(buf[0])<<8 | uint16(buf[1])<<0)
|
|
if len(buf) < 2+ipadlen {
|
|
return fmt.Errorf("buffer too short for ipadlen")
|
|
}
|
|
ipad := md5.New()
|
|
opad := md5.New()
|
|
if err := ipad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2 : 2+ipadlen]); err != nil {
|
|
return fmt.Errorf("unmarshal ipad: %v", err)
|
|
}
|
|
if err := opad.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf[2+ipadlen:]); err != nil {
|
|
return fmt.Errorf("unmarshal opad: %v", err)
|
|
}
|
|
*c = CRAMMD5{ipad, opad}
|
|
return nil
|
|
}
|
|
|
|
// Password holds credentials in various forms, for logging in with SMTP/IMAP.
|
|
type Password struct {
|
|
Hash string // bcrypt hash for IMAP LOGIN, SASL PLAIN and HTTP basic authentication.
|
|
CRAMMD5 CRAMMD5 // For SASL CRAM-MD5.
|
|
SCRAMSHA1 SCRAM // For SASL SCRAM-SHA-1.
|
|
SCRAMSHA256 SCRAM // For SASL SCRAM-SHA-256.
|
|
}
|
|
|
|
// Subjectpass holds the secret key used to sign subjectpass tokens.
|
|
type Subjectpass struct {
|
|
Email string // Our destination address (canonical, with catchall localpart stripped).
|
|
Key string
|
|
}
|
|
|
|
// NextUIDValidity is a singleton record in the database with the next UIDValidity
|
|
// to use for the next mailbox.
|
|
type NextUIDValidity struct {
|
|
ID int // Just a single record with ID 1.
|
|
Next uint32
|
|
}
|
|
|
|
// SyncState track ModSeqs.
|
|
type SyncState struct {
|
|
ID int // Just a single record with ID 1.
|
|
|
|
// Last used, next assigned will be one higher. The first value we hand out is 2.
|
|
// That's because 0 (the default value for old existing messages, from before the
|
|
// Message.ModSeq field) is special in IMAP, so we return it as 1.
|
|
LastModSeq ModSeq `bstore:"nonzero"`
|
|
|
|
// Highest ModSeq of expunged record that we deleted. When a clients synchronizes
|
|
// and requests changes based on a modseq before this one, we don't have the
|
|
// history to provide information about deletions. We normally keep these expunged
|
|
// records around, but we may periodically truly delete them to reclaim storage
|
|
// space. Initially set to -1 because we don't want to match with any ModSeq in the
|
|
// database, which can be zero values.
|
|
HighestDeletedModSeq ModSeq
|
|
}
|
|
|
|
// Mailbox is collection of messages, e.g. Inbox or Sent.
|
|
type Mailbox struct {
|
|
ID int64
|
|
|
|
// "Inbox" is the name for the special IMAP "INBOX". Slash separated
|
|
// for hierarchy.
|
|
Name string `bstore:"nonzero,unique"`
|
|
|
|
// If UIDs are invalidated, e.g. when renaming a mailbox to a previously existing
|
|
// name, UIDValidity must be changed. Used by IMAP for synchronization.
|
|
UIDValidity uint32
|
|
|
|
// UID likely to be assigned to next message. Used by IMAP to detect messages
|
|
// delivered to a mailbox.
|
|
UIDNext UID
|
|
|
|
SpecialUse
|
|
|
|
// Keywords as used in messages. Storing a non-system keyword for a message
|
|
// automatically adds it to this list. Used in the IMAP FLAGS response. Only
|
|
// "atoms" are allowed (IMAP syntax), keywords are case-insensitive, only stored in
|
|
// lower case (for JMAP), sorted.
|
|
Keywords []string
|
|
|
|
HaveCounts bool // Whether MailboxCounts have been initialized.
|
|
MailboxCounts // Statistics about messages, kept up to date whenever a change happens.
|
|
}
|
|
|
|
// MailboxCounts tracks statistics about messages for a mailbox.
|
|
type MailboxCounts struct {
|
|
Total int64 // Total number of messages, excluding \Deleted. For JMAP.
|
|
Deleted int64 // Number of messages with \Deleted flag. Used for IMAP message count that includes messages with \Deleted.
|
|
Unread int64 // Messages without \Seen, excluding those with \Deleted, for JMAP.
|
|
Unseen int64 // Messages without \Seen, including those with \Deleted, for IMAP.
|
|
Size int64 // Number of bytes for all messages.
|
|
}
|
|
|
|
func (mc MailboxCounts) String() string {
|
|
return fmt.Sprintf("%d total, %d deleted, %d unread, %d unseen, size %d bytes", mc.Total, mc.Deleted, mc.Unread, mc.Unseen, mc.Size)
|
|
}
|
|
|
|
// Add increases mailbox counts mc with those of delta.
|
|
func (mc *MailboxCounts) Add(delta MailboxCounts) {
|
|
mc.Total += delta.Total
|
|
mc.Deleted += delta.Deleted
|
|
mc.Unread += delta.Unread
|
|
mc.Unseen += delta.Unseen
|
|
mc.Size += delta.Size
|
|
}
|
|
|
|
// Add decreases mailbox counts mc with those of delta.
|
|
func (mc *MailboxCounts) Sub(delta MailboxCounts) {
|
|
mc.Total -= delta.Total
|
|
mc.Deleted -= delta.Deleted
|
|
mc.Unread -= delta.Unread
|
|
mc.Unseen -= delta.Unseen
|
|
mc.Size -= delta.Size
|
|
}
|
|
|
|
// SpecialUse identifies a specific role for a mailbox, used by clients to
|
|
// understand where messages should go.
|
|
type SpecialUse struct {
|
|
Archive bool
|
|
Draft bool
|
|
Junk bool
|
|
Sent bool
|
|
Trash bool
|
|
}
|
|
|
|
// CalculateCounts calculates the full current counts for messages in the mailbox.
|
|
func (mb *Mailbox) CalculateCounts(tx *bstore.Tx) (mc MailboxCounts, err error) {
|
|
q := bstore.QueryTx[Message](tx)
|
|
q.FilterNonzero(Message{MailboxID: mb.ID})
|
|
q.FilterEqual("Expunged", false)
|
|
err = q.ForEach(func(m Message) error {
|
|
mc.Add(m.MailboxCounts())
|
|
return nil
|
|
})
|
|
return
|
|
}
|
|
|
|
// ChangeSpecialUse returns a change for special-use flags, for broadcasting to
|
|
// other connections.
|
|
func (mb Mailbox) ChangeSpecialUse() ChangeMailboxSpecialUse {
|
|
return ChangeMailboxSpecialUse{mb.ID, mb.Name, mb.SpecialUse}
|
|
}
|
|
|
|
// ChangeKeywords returns a change with new keywords for a mailbox (e.g. after
|
|
// setting a new keyword on a message in the mailbox), for broadcasting to other
|
|
// connections.
|
|
func (mb Mailbox) ChangeKeywords() ChangeMailboxKeywords {
|
|
return ChangeMailboxKeywords{mb.ID, mb.Name, mb.Keywords}
|
|
}
|
|
|
|
// KeywordsChanged returns whether the keywords in a mailbox have changed.
|
|
func (mb Mailbox) KeywordsChanged(origmb Mailbox) bool {
|
|
if len(mb.Keywords) != len(origmb.Keywords) {
|
|
return true
|
|
}
|
|
// Keywords are stored sorted.
|
|
for i, kw := range mb.Keywords {
|
|
if origmb.Keywords[i] != kw {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// CountsChange returns a change with mailbox counts.
|
|
func (mb Mailbox) ChangeCounts() ChangeMailboxCounts {
|
|
return ChangeMailboxCounts{mb.ID, mb.Name, mb.MailboxCounts}
|
|
}
|
|
|
|
// Subscriptions are separate from existence of mailboxes.
|
|
type Subscription struct {
|
|
Name string
|
|
}
|
|
|
|
// Flags for a mail message.
|
|
type Flags struct {
|
|
Seen bool
|
|
Answered bool
|
|
Flagged bool
|
|
Forwarded bool
|
|
Junk bool
|
|
Notjunk bool
|
|
Deleted bool
|
|
Draft bool
|
|
Phishing bool
|
|
MDNSent bool
|
|
}
|
|
|
|
// FlagsAll is all flags set, for use as mask.
|
|
var FlagsAll = Flags{true, true, true, true, true, true, true, true, true, true}
|
|
|
|
// Validation of "message From" domain.
|
|
type Validation uint8
|
|
|
|
const (
|
|
ValidationUnknown Validation = 0
|
|
ValidationStrict Validation = 1 // Like DMARC, with strict policies.
|
|
ValidationDMARC Validation = 2 // Actual DMARC policy.
|
|
ValidationRelaxed Validation = 3 // Like DMARC, with relaxed policies.
|
|
ValidationPass Validation = 4 // For SPF.
|
|
ValidationNeutral Validation = 5 // For SPF.
|
|
ValidationTemperror Validation = 6
|
|
ValidationPermerror Validation = 7
|
|
ValidationFail Validation = 8
|
|
ValidationSoftfail Validation = 9 // For SPF.
|
|
ValidationNone Validation = 10 // E.g. No records.
|
|
)
|
|
|
|
// Message stored in database and per-message file on disk.
|
|
//
|
|
// Contents are always the combined data from MsgPrefix and the on-disk file named
|
|
// based on ID.
|
|
//
|
|
// Messages always have a header section, even if empty. Incoming messages without
|
|
// header section must get an empty header section added before inserting.
|
|
type Message struct {
|
|
// ID, unchanged over lifetime, determines path to on-disk msg file.
|
|
// Set during deliver.
|
|
ID int64
|
|
|
|
UID UID `bstore:"nonzero"` // UID, for IMAP. Set during deliver.
|
|
MailboxID int64 `bstore:"nonzero,unique MailboxID+UID,index MailboxID+Received,index MailboxID+ModSeq,ref Mailbox"`
|
|
|
|
// Modification sequence, for faster syncing with IMAP QRESYNC and JMAP.
|
|
// ModSeq is the last modification. CreateSeq is the Seq the message was inserted,
|
|
// always <= ModSeq. If Expunged is set, the message has been removed and should not
|
|
// be returned to the user. In this case, ModSeq is the Seq where the message is
|
|
// removed, and will never be changed again.
|
|
// We have an index on both ModSeq (for JMAP that synchronizes per account) and
|
|
// MailboxID+ModSeq (for IMAP that synchronizes per mailbox).
|
|
// The index on CreateSeq helps efficiently finding created messages for JMAP.
|
|
// The value of ModSeq is special for IMAP. Messages that existed before ModSeq was
|
|
// added have 0 as value. But modseq 0 in IMAP is special, so we return it as 1. If
|
|
// we get modseq 1 from a client, the IMAP server will translate it to 0. When we
|
|
// return modseq to clients, we turn 0 into 1.
|
|
ModSeq ModSeq `bstore:"index"`
|
|
CreateSeq ModSeq `bstore:"index"`
|
|
Expunged bool
|
|
|
|
// If set, this message was delivered to a Rejects mailbox. When it is moved to a
|
|
// different mailbox, its MailboxOrigID is set to the destination mailbox and this
|
|
// flag cleared.
|
|
IsReject bool
|
|
|
|
// If set, this is a forwarded message (through a ruleset with IsForward). This
|
|
// causes fields used during junk analysis to be moved to their Orig variants, and
|
|
// masked IP fields cleared, so they aren't used in junk classifications for
|
|
// incoming messages. This ensures the forwarded messages don't cause negative
|
|
// reputation for the forwarding mail server, which may also be sending regular
|
|
// messages.
|
|
IsForward bool
|
|
|
|
// MailboxOrigID is the mailbox the message was originally delivered to. Typically
|
|
// Inbox or Rejects, but can also be a mailbox configured in a Ruleset, or
|
|
// Postmaster, TLS/DMARC reporting addresses. MailboxOrigID is not changed when the
|
|
// message is moved to another mailbox, e.g. Archive/Trash/Junk. Used for
|
|
// per-mailbox reputation.
|
|
//
|
|
// MailboxDestinedID is normally 0, but when a message is delivered to the Rejects
|
|
// mailbox, it is set to the intended mailbox according to delivery rules,
|
|
// typically that of Inbox. When such a message is moved out of Rejects, the
|
|
// MailboxOrigID is corrected by setting it to MailboxDestinedID. This ensures the
|
|
// message is used for reputation calculation for future deliveries to that
|
|
// mailbox.
|
|
//
|
|
// These are not bstore references to prevent having to update all messages in a
|
|
// mailbox when the original mailbox is removed. Use of these fields requires
|
|
// checking if the mailbox still exists.
|
|
MailboxOrigID int64
|
|
MailboxDestinedID int64
|
|
|
|
Received time.Time `bstore:"default now,index"`
|
|
|
|
// Full IP address of remote SMTP server. Empty if not delivered over SMTP. The
|
|
// masked IPs are used to classify incoming messages. They are left empty for
|
|
// messages matching a ruleset for forwarded messages.
|
|
RemoteIP string
|
|
RemoteIPMasked1 string `bstore:"index RemoteIPMasked1+Received"` // For IPv4 /32, for IPv6 /64, for reputation.
|
|
RemoteIPMasked2 string `bstore:"index RemoteIPMasked2+Received"` // For IPv4 /26, for IPv6 /48.
|
|
RemoteIPMasked3 string `bstore:"index RemoteIPMasked3+Received"` // For IPv4 /21, for IPv6 /32.
|
|
|
|
// Only set if present and not an IP address. Unicode string. Empty for forwarded
|
|
// messages.
|
|
EHLODomain string `bstore:"index EHLODomain+Received"`
|
|
MailFrom string // With localpart and domain. Can be empty.
|
|
MailFromLocalpart smtp.Localpart // SMTP "MAIL FROM", can be empty.
|
|
// Only set if it is a domain, not an IP. Unicode string. Empty for forwarded
|
|
// messages, but see OrigMailFromDomain.
|
|
MailFromDomain string `bstore:"index MailFromDomain+Received"`
|
|
RcptToLocalpart smtp.Localpart // SMTP "RCPT TO", can be empty.
|
|
RcptToDomain string // Unicode string.
|
|
|
|
// Parsed "From" message header, used for reputation along with domain validation.
|
|
MsgFromLocalpart smtp.Localpart
|
|
MsgFromDomain string `bstore:"index MsgFromDomain+Received"` // Unicode string.
|
|
MsgFromOrgDomain string `bstore:"index MsgFromOrgDomain+Received"` // Unicode string.
|
|
|
|
// Simplified statements of the Validation fields below, used for incoming messages
|
|
// to check reputation.
|
|
EHLOValidated bool
|
|
MailFromValidated bool
|
|
MsgFromValidated bool
|
|
|
|
EHLOValidation Validation // Validation can also take reverse IP lookup into account, not only SPF.
|
|
MailFromValidation Validation // Can have SPF-specific validations like ValidationSoftfail.
|
|
MsgFromValidation Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.
|
|
|
|
// Domains with verified DKIM signatures. Unicode string. For forwarded messages, a
|
|
// DKIM domain that matched a ruleset's verified domain is left out, but included
|
|
// in OrigDKIMDomains.
|
|
DKIMDomains []string `bstore:"index DKIMDomains+Received"`
|
|
|
|
// For forwarded messages,
|
|
OrigEHLODomain string
|
|
OrigDKIMDomains []string
|
|
|
|
// Canonicalized Message-Id, always lower-case and normalized quoting, without
|
|
// <>'s. Empty if missing. Used for matching message threads, and to prevent
|
|
// duplicate reject delivery.
|
|
MessageID string `bstore:"index"`
|
|
// lower-case: ../rfc/5256:495
|
|
|
|
// For matching threads in case there is no References/In-Reply-To header. It is
|
|
// lower-cased, white-space collapsed, mailing list tags and re/fwd tags removed.
|
|
SubjectBase string `bstore:"index"`
|
|
// ../rfc/5256:90
|
|
|
|
// Hash of message. For rejects delivery in case there is no Message-ID, only set
|
|
// when delivered as reject.
|
|
MessageHash []byte
|
|
|
|
// ID of message starting this thread.
|
|
ThreadID int64 `bstore:"index"`
|
|
// IDs of parent messages, from closest parent to the root message. Parent messages
|
|
// may be in a different mailbox, or may no longer exist. ThreadParentIDs must
|
|
// never contain the message id itself (a cycle), and parent messages must
|
|
// reference the same ancestors.
|
|
ThreadParentIDs []int64
|
|
// ThreadMissingLink is true if there is no match with a direct parent. E.g. first
|
|
// ID in ThreadParentIDs is not the direct ancestor (an intermediate message may
|
|
// have been deleted), or subject-based matching was done.
|
|
ThreadMissingLink bool
|
|
// If set, newly delivered child messages are automatically marked as read. This
|
|
// field is copied to new child messages. Changes are propagated to the webmail
|
|
// client.
|
|
ThreadMuted bool
|
|
// If set, this (sub)thread is collapsed in the webmail client, for threading mode
|
|
// "on" (mode "unread" ignores it). This field is copied to new child message.
|
|
// Changes are propagated to the webmail client.
|
|
ThreadCollapsed bool
|
|
|
|
// If received message was known to match a mailing list rule (with modified junk
|
|
// filtering).
|
|
IsMailingList bool
|
|
|
|
ReceivedTLSVersion uint16 // 0 if unknown, 1 if plaintext/no TLS, otherwise TLS cipher suite.
|
|
ReceivedTLSCipherSuite uint16
|
|
ReceivedRequireTLS bool // Whether RequireTLS was known to be used for incoming delivery.
|
|
|
|
Flags
|
|
// For keywords other than system flags or the basic well-known $-flags. Only in
|
|
// "atom" syntax (IMAP), they are case-insensitive, always stored in lower-case
|
|
// (for JMAP), sorted.
|
|
Keywords []string `bstore:"index"`
|
|
Size int64
|
|
TrainedJunk *bool // If nil, no training done yet. Otherwise, true is trained as junk, false trained as nonjunk.
|
|
MsgPrefix []byte // Typically holds received headers and/or header separator.
|
|
|
|
// ParsedBuf message structure. Currently saved as JSON of message.Part because bstore
|
|
// cannot yet store recursive types. Created when first needed, and saved in the
|
|
// database.
|
|
// todo: once replaced with non-json storage, remove date fixup in ../message/part.go.
|
|
ParsedBuf []byte
|
|
}
|
|
|
|
// MailboxCounts returns the delta to counts this message means for its
|
|
// mailbox.
|
|
func (m Message) MailboxCounts() (mc MailboxCounts) {
|
|
if m.Expunged {
|
|
return
|
|
}
|
|
if m.Deleted {
|
|
mc.Deleted++
|
|
} else {
|
|
mc.Total++
|
|
}
|
|
if !m.Seen {
|
|
mc.Unseen++
|
|
if !m.Deleted {
|
|
mc.Unread++
|
|
}
|
|
}
|
|
mc.Size += m.Size
|
|
return
|
|
}
|
|
|
|
func (m Message) ChangeAddUID() ChangeAddUID {
|
|
return ChangeAddUID{m.MailboxID, m.UID, m.ModSeq, m.Flags, m.Keywords}
|
|
}
|
|
|
|
func (m Message) ChangeFlags(orig Flags) ChangeFlags {
|
|
mask := m.Flags.Changed(orig)
|
|
return ChangeFlags{MailboxID: m.MailboxID, UID: m.UID, ModSeq: m.ModSeq, Mask: mask, Flags: m.Flags, Keywords: m.Keywords}
|
|
}
|
|
|
|
func (m Message) ChangeThread() ChangeThread {
|
|
return ChangeThread{[]int64{m.ID}, m.ThreadMuted, m.ThreadCollapsed}
|
|
}
|
|
|
|
// ModSeq represents a modseq as stored in the database. ModSeq 0 in the
|
|
// database is sent to the client as 1, because modseq 0 is special in IMAP.
|
|
// ModSeq coming from the client are of type int64.
|
|
type ModSeq int64
|
|
|
|
func (ms ModSeq) Client() int64 {
|
|
if ms == 0 {
|
|
return 1
|
|
}
|
|
return int64(ms)
|
|
}
|
|
|
|
// ModSeqFromClient converts a modseq from a client to a modseq for internal
|
|
// use, e.g. in a database query.
|
|
// ModSeq 1 is turned into 0 (the Go zero value for ModSeq).
|
|
func ModSeqFromClient(modseq int64) ModSeq {
|
|
if modseq == 1 {
|
|
return 0
|
|
}
|
|
return ModSeq(modseq)
|
|
}
|
|
|
|
// PrepareExpunge clears fields that are no longer needed after an expunge, so
|
|
// almost all fields. Does not change ModSeq, but does set Expunged.
|
|
func (m *Message) PrepareExpunge() {
|
|
*m = Message{
|
|
ID: m.ID,
|
|
UID: m.UID,
|
|
MailboxID: m.MailboxID,
|
|
CreateSeq: m.CreateSeq,
|
|
ModSeq: m.ModSeq,
|
|
Expunged: true,
|
|
ThreadID: m.ThreadID,
|
|
}
|
|
}
|
|
|
|
// PrepareThreading sets MessageID and SubjectBase (used in threading) based on the
|
|
// envelope in part.
|
|
func (m *Message) PrepareThreading(log mlog.Log, part *message.Part) {
|
|
if part.Envelope == nil {
|
|
return
|
|
}
|
|
messageID, raw, err := message.MessageIDCanonical(part.Envelope.MessageID)
|
|
if err != nil {
|
|
log.Debugx("parsing message-id, ignoring", err, slog.String("messageid", part.Envelope.MessageID))
|
|
} else if raw {
|
|
log.Debug("could not parse message-id as address, continuing with raw value", slog.String("messageid", part.Envelope.MessageID))
|
|
}
|
|
m.MessageID = messageID
|
|
m.SubjectBase, _ = message.ThreadSubject(part.Envelope.Subject, false)
|
|
}
|
|
|
|
// LoadPart returns a message.Part by reading from m.ParsedBuf.
|
|
func (m Message) LoadPart(r io.ReaderAt) (message.Part, error) {
|
|
if m.ParsedBuf == nil {
|
|
return message.Part{}, fmt.Errorf("message not parsed")
|
|
}
|
|
var p message.Part
|
|
err := json.Unmarshal(m.ParsedBuf, &p)
|
|
if err != nil {
|
|
return p, fmt.Errorf("unmarshal message part")
|
|
}
|
|
p.SetReaderAt(r)
|
|
return p, nil
|
|
}
|
|
|
|
// NeedsTraining returns whether message needs a training update, based on
|
|
// TrainedJunk (current training status) and new Junk/Notjunk flags.
|
|
func (m Message) NeedsTraining() bool {
|
|
untrain := m.TrainedJunk != nil
|
|
untrainJunk := untrain && *m.TrainedJunk
|
|
train := m.Junk || m.Notjunk && !(m.Junk && m.Notjunk)
|
|
trainJunk := m.Junk
|
|
return untrain != train || untrain && train && untrainJunk != trainJunk
|
|
}
|
|
|
|
// JunkFlagsForMailbox sets Junk and Notjunk flags based on mailbox name if configured. Often
|
|
// used when delivering/moving/copying messages to a mailbox. Mail clients are not
|
|
// very helpful with setting junk/notjunk flags. But clients can move/copy messages
|
|
// to other mailboxes. So we set flags when clients move a message.
|
|
func (m *Message) JunkFlagsForMailbox(mb Mailbox, conf config.Account) {
|
|
if mb.Junk {
|
|
m.Junk = true
|
|
m.Notjunk = false
|
|
return
|
|
}
|
|
|
|
if !conf.AutomaticJunkFlags.Enabled {
|
|
return
|
|
}
|
|
|
|
lmailbox := strings.ToLower(mb.Name)
|
|
|
|
if conf.JunkMailbox != nil && conf.JunkMailbox.MatchString(lmailbox) {
|
|
m.Junk = true
|
|
m.Notjunk = false
|
|
} else if conf.NeutralMailbox != nil && conf.NeutralMailbox.MatchString(lmailbox) {
|
|
m.Junk = false
|
|
m.Notjunk = false
|
|
} else if conf.NotJunkMailbox != nil && conf.NotJunkMailbox.MatchString(lmailbox) {
|
|
m.Junk = false
|
|
m.Notjunk = true
|
|
} else if conf.JunkMailbox == nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox != nil {
|
|
m.Junk = true
|
|
m.Notjunk = false
|
|
} else if conf.JunkMailbox != nil && conf.NeutralMailbox == nil && conf.NotJunkMailbox != nil {
|
|
m.Junk = false
|
|
m.Notjunk = false
|
|
} else if conf.JunkMailbox != nil && conf.NeutralMailbox != nil && conf.NotJunkMailbox == nil {
|
|
m.Junk = false
|
|
m.Notjunk = true
|
|
}
|
|
}
|
|
|
|
// Recipient represents the recipient of a message. It is tracked to allow
|
|
// first-time incoming replies from users this account has sent messages to. When a
|
|
// mailbox is added to the Sent mailbox the message is parsed and recipients are
|
|
// inserted as recipient. Recipients are never removed other than for removing the
|
|
// message. On move/copy of a message, recipients aren't modified either. For IMAP,
|
|
// this assumes a client simply appends messages to the Sent mailbox (as opposed to
|
|
// copying messages from some place).
|
|
type Recipient struct {
|
|
ID int64
|
|
MessageID int64 `bstore:"nonzero,ref Message"` // Ref gives it its own index, useful for fast removal as well.
|
|
Localpart smtp.Localpart `bstore:"nonzero"`
|
|
Domain string `bstore:"nonzero,index Domain+Localpart"` // Unicode string.
|
|
OrgDomain string `bstore:"nonzero,index"` // Unicode string.
|
|
Sent time.Time `bstore:"nonzero"`
|
|
}
|
|
|
|
// Outgoing is a message submitted for delivery from the queue. Used to enforce
|
|
// maximum outgoing messages.
|
|
type Outgoing struct {
|
|
ID int64
|
|
Recipient string `bstore:"nonzero,index"` // Canonical international address with utf8 domain.
|
|
Submitted time.Time `bstore:"nonzero,default now"`
|
|
}
|
|
|
|
// RecipientDomainTLS stores TLS capabilities of a recipient domain as encountered
|
|
// during most recent connection (delivery attempt).
|
|
type RecipientDomainTLS struct {
|
|
Domain string // Unicode.
|
|
Updated time.Time `bstore:"default now"`
|
|
STARTTLS bool // Supports STARTTLS.
|
|
RequireTLS bool // Supports RequireTLS SMTP extension.
|
|
}
|
|
|
|
// DiskUsage tracks quota use.
|
|
type DiskUsage struct {
|
|
ID int64 // Always one record with ID 1.
|
|
MessageSize int64 // Sum of all messages, for quota accounting.
|
|
}
|
|
|
|
// SessionToken and CSRFToken are types to prevent mixing them up.
|
|
// Base64 raw url encoded.
|
|
type SessionToken string
|
|
type CSRFToken string
|
|
|
|
// LoginSession represents a login session. We keep a limited number of sessions
|
|
// for a user, removing the oldest session when a new one is created.
|
|
type LoginSession struct {
|
|
ID int64
|
|
Created time.Time `bstore:"nonzero,default now"` // Of original login.
|
|
Expires time.Time `bstore:"nonzero"` // Extended each time it is used.
|
|
SessionTokenBinary [16]byte `bstore:"nonzero"` // Stored in cookie, like "webmailsession" or "webaccountsession".
|
|
CSRFTokenBinary [16]byte // For API requests, in "x-mox-csrf" header.
|
|
AccountName string `bstore:"nonzero"`
|
|
LoginAddress string `bstore:"nonzero"`
|
|
|
|
// Set when loading from database.
|
|
sessionToken SessionToken
|
|
csrfToken CSRFToken
|
|
}
|
|
|
|
// Types stored in DB.
|
|
var DBTypes = []any{NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{}, SyncState{}, Upgrade{}, RecipientDomainTLS{}, DiskUsage{}, LoginSession{}}
|
|
|
|
// Account holds the information about a user, includings mailboxes, messages, imap subscriptions.
|
|
type Account struct {
|
|
Name string // Name, according to configuration.
|
|
Dir string // Directory where account files, including the database, bloom filter, and mail messages, are stored for this account.
|
|
DBPath string // Path to database with mailboxes, messages, etc.
|
|
DB *bstore.DB // Open database connection.
|
|
|
|
// Channel that is closed if/when account has/gets "threads" accounting (see
|
|
// Upgrade.Threads).
|
|
threadsCompleted chan struct{}
|
|
// If threads upgrade completed with error, this is set. Used for warning during
|
|
// delivery, or aborting when importing.
|
|
threadsErr error
|
|
|
|
// Write lock must be held for account/mailbox modifications including message delivery.
|
|
// Read lock for reading mailboxes/messages.
|
|
// When making changes to mailboxes/messages, changes must be broadcasted before
|
|
// releasing the lock to ensure proper UID ordering.
|
|
sync.RWMutex
|
|
|
|
nused int // Reference count, while >0, this account is alive and shared.
|
|
}
|
|
|
|
type Upgrade struct {
|
|
ID byte
|
|
Threads byte // 0: None, 1: Adding MessageID's completed, 2: Adding ThreadID's completed.
|
|
}
|
|
|
|
// InitialUIDValidity returns a UIDValidity used for initializing an account.
|
|
// It can be replaced during tests with a predictable value.
|
|
var InitialUIDValidity = func() uint32 {
|
|
return uint32(time.Now().Unix() >> 1) // A 2-second resolution will get us far enough beyond 2038.
|
|
}
|
|
|
|
var openAccounts = struct {
|
|
names map[string]*Account
|
|
sync.Mutex
|
|
}{
|
|
names: map[string]*Account{},
|
|
}
|
|
|
|
func closeAccount(acc *Account) (rerr error) {
|
|
openAccounts.Lock()
|
|
acc.nused--
|
|
defer openAccounts.Unlock()
|
|
if acc.nused == 0 {
|
|
// threadsCompleted must be closed now because it increased nused.
|
|
rerr = acc.DB.Close()
|
|
acc.DB = nil
|
|
delete(openAccounts.names, acc.Name)
|
|
}
|
|
return
|
|
}
|
|
|
|
// OpenAccount opens an account by name.
|
|
//
|
|
// No additional data path prefix or ".db" suffix should be added to the name.
|
|
// A single shared account exists per name.
|
|
func OpenAccount(log mlog.Log, name string) (*Account, error) {
|
|
openAccounts.Lock()
|
|
defer openAccounts.Unlock()
|
|
if acc, ok := openAccounts.names[name]; ok {
|
|
acc.nused++
|
|
return acc, nil
|
|
}
|
|
|
|
if _, ok := mox.Conf.Account(name); !ok {
|
|
return nil, ErrAccountUnknown
|
|
}
|
|
|
|
acc, err := openAccount(log, name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
openAccounts.names[name] = acc
|
|
return acc, nil
|
|
}
|
|
|
|
// openAccount opens an existing account, or creates it if it is missing.
|
|
func openAccount(log mlog.Log, name string) (a *Account, rerr error) {
|
|
dir := filepath.Join(mox.DataDirPath("accounts"), name)
|
|
return OpenAccountDB(log, dir, name)
|
|
}
|
|
|
|
// OpenAccountDB opens an account database file and returns an initialized account
|
|
// or error. Only exported for use by subcommands that verify the database file.
|
|
// Almost all account opens must go through OpenAccount/OpenEmail/OpenEmailAuth.
|
|
func OpenAccountDB(log mlog.Log, accountDir, accountName string) (a *Account, rerr error) {
|
|
dbpath := filepath.Join(accountDir, "index.db")
|
|
|
|
// Create account if it doesn't exist yet.
|
|
isNew := false
|
|
if _, err := os.Stat(dbpath); err != nil && os.IsNotExist(err) {
|
|
isNew = true
|
|
os.MkdirAll(accountDir, 0770)
|
|
}
|
|
|
|
db, err := bstore.Open(context.TODO(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DBTypes...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
defer func() {
|
|
if rerr != nil {
|
|
db.Close()
|
|
if isNew {
|
|
os.Remove(dbpath)
|
|
}
|
|
}
|
|
}()
|
|
|
|
acc := &Account{
|
|
Name: accountName,
|
|
Dir: accountDir,
|
|
DBPath: dbpath,
|
|
DB: db,
|
|
nused: 1,
|
|
threadsCompleted: make(chan struct{}),
|
|
}
|
|
|
|
if isNew {
|
|
if err := initAccount(db); err != nil {
|
|
return nil, fmt.Errorf("initializing account: %v", err)
|
|
}
|
|
close(acc.threadsCompleted)
|
|
return acc, nil
|
|
}
|
|
|
|
// Ensure mailbox counts and total message size are set.
|
|
var mentioned bool
|
|
err = db.Write(context.TODO(), func(tx *bstore.Tx) error {
|
|
err := bstore.QueryTx[Mailbox](tx).FilterEqual("HaveCounts", false).ForEach(func(mb Mailbox) error {
|
|
if !mentioned {
|
|
mentioned = true
|
|
log.Info("first calculation of mailbox counts for account", slog.String("account", accountName))
|
|
}
|
|
mc, err := mb.CalculateCounts(tx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
mb.HaveCounts = true
|
|
mb.MailboxCounts = mc
|
|
return tx.Update(&mb)
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
du := DiskUsage{ID: 1}
|
|
err = tx.Get(&du)
|
|
if err == nil || !errors.Is(err, bstore.ErrAbsent) {
|
|
return err
|
|
}
|
|
// No DiskUsage record yet, calculate total size and insert.
|
|
err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {
|
|
du.MessageSize += mb.Size
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return tx.Insert(&du)
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("calculating counts for mailbox: %v", err)
|
|
}
|
|
|
|
// Start adding threading if needed.
|
|
up := Upgrade{ID: 1}
|
|
err = db.Write(context.TODO(), func(tx *bstore.Tx) error {
|
|
err := tx.Get(&up)
|
|
if err == bstore.ErrAbsent {
|
|
if err := tx.Insert(&up); err != nil {
|
|
return fmt.Errorf("inserting initial upgrade record: %v", err)
|
|
}
|
|
err = nil
|
|
}
|
|
return err
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("checking message threading: %v", err)
|
|
}
|
|
if up.Threads == 2 {
|
|
close(acc.threadsCompleted)
|
|
return acc, nil
|
|
}
|
|
|
|
// Increase account use before holding on to account in background.
|
|
// Caller holds the lock. The goroutine below decreases nused by calling
|
|
// closeAccount.
|
|
acc.nused++
|
|
|
|
// Ensure all messages have a MessageID and SubjectBase, which are needed when
|
|
// matching threads.
|
|
// Then assign messages to threads, in the same way we do during imports.
|
|
log.Info("upgrading account for threading, in background", slog.String("account", acc.Name))
|
|
go func() {
|
|
defer func() {
|
|
err := closeAccount(acc)
|
|
log.Check(err, "closing use of account after upgrading account storage for threads", slog.String("account", a.Name))
|
|
}()
|
|
|
|
defer func() {
|
|
x := recover() // Should not happen, but don't take program down if it does.
|
|
if x != nil {
|
|
log.Error("upgradeThreads panic", slog.Any("err", x))
|
|
debug.PrintStack()
|
|
metrics.PanicInc(metrics.Upgradethreads)
|
|
acc.threadsErr = fmt.Errorf("panic during upgradeThreads: %v", x)
|
|
}
|
|
|
|
// Mark that upgrade has finished, possibly error is indicated in threadsErr.
|
|
close(acc.threadsCompleted)
|
|
}()
|
|
|
|
err := upgradeThreads(mox.Shutdown, log, acc, &up)
|
|
if err != nil {
|
|
a.threadsErr = err
|
|
log.Errorx("upgrading account for threading, aborted", err, slog.String("account", a.Name))
|
|
} else {
|
|
log.Info("upgrading account for threading, completed", slog.String("account", a.Name))
|
|
}
|
|
}()
|
|
return acc, nil
|
|
}
|
|
|
|
// ThreadingWait blocks until the one-time account threading upgrade for the
|
|
// account has completed, and returns an error if not successful.
|
|
//
|
|
// To be used before starting an import of messages.
|
|
func (a *Account) ThreadingWait(log mlog.Log) error {
|
|
select {
|
|
case <-a.threadsCompleted:
|
|
return a.threadsErr
|
|
default:
|
|
}
|
|
log.Debug("waiting for account upgrade to complete")
|
|
|
|
<-a.threadsCompleted
|
|
return a.threadsErr
|
|
}
|
|
|
|
func initAccount(db *bstore.DB) error {
|
|
return db.Write(context.TODO(), func(tx *bstore.Tx) error {
|
|
uidvalidity := InitialUIDValidity()
|
|
|
|
if err := tx.Insert(&Upgrade{ID: 1, Threads: 2}); err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Insert(&DiskUsage{ID: 1}); err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(mox.Conf.Static.DefaultMailboxes) > 0 {
|
|
// Deprecated in favor of InitialMailboxes.
|
|
defaultMailboxes := mox.Conf.Static.DefaultMailboxes
|
|
mailboxes := []string{"Inbox"}
|
|
for _, name := range defaultMailboxes {
|
|
if strings.EqualFold(name, "Inbox") {
|
|
continue
|
|
}
|
|
mailboxes = append(mailboxes, name)
|
|
}
|
|
for _, name := range mailboxes {
|
|
mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, HaveCounts: true}
|
|
if strings.HasPrefix(name, "Archive") {
|
|
mb.Archive = true
|
|
} else if strings.HasPrefix(name, "Drafts") {
|
|
mb.Draft = true
|
|
} else if strings.HasPrefix(name, "Junk") {
|
|
mb.Junk = true
|
|
} else if strings.HasPrefix(name, "Sent") {
|
|
mb.Sent = true
|
|
} else if strings.HasPrefix(name, "Trash") {
|
|
mb.Trash = true
|
|
}
|
|
if err := tx.Insert(&mb); err != nil {
|
|
return fmt.Errorf("creating mailbox: %w", err)
|
|
}
|
|
if err := tx.Insert(&Subscription{name}); err != nil {
|
|
return fmt.Errorf("adding subscription: %w", err)
|
|
}
|
|
}
|
|
} else {
|
|
mailboxes := mox.Conf.Static.InitialMailboxes
|
|
var zerouse config.SpecialUseMailboxes
|
|
if mailboxes.SpecialUse == zerouse && len(mailboxes.Regular) == 0 {
|
|
mailboxes = DefaultInitialMailboxes
|
|
}
|
|
|
|
add := func(name string, use SpecialUse) error {
|
|
mb := Mailbox{Name: name, UIDValidity: uidvalidity, UIDNext: 1, SpecialUse: use, HaveCounts: true}
|
|
if err := tx.Insert(&mb); err != nil {
|
|
return fmt.Errorf("creating mailbox: %w", err)
|
|
}
|
|
if err := tx.Insert(&Subscription{name}); err != nil {
|
|
return fmt.Errorf("adding subscription: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
addSpecialOpt := func(nameOpt string, use SpecialUse) error {
|
|
if nameOpt == "" {
|
|
return nil
|
|
}
|
|
return add(nameOpt, use)
|
|
}
|
|
l := []struct {
|
|
nameOpt string
|
|
use SpecialUse
|
|
}{
|
|
{"Inbox", SpecialUse{}},
|
|
{mailboxes.SpecialUse.Archive, SpecialUse{Archive: true}},
|
|
{mailboxes.SpecialUse.Draft, SpecialUse{Draft: true}},
|
|
{mailboxes.SpecialUse.Junk, SpecialUse{Junk: true}},
|
|
{mailboxes.SpecialUse.Sent, SpecialUse{Sent: true}},
|
|
{mailboxes.SpecialUse.Trash, SpecialUse{Trash: true}},
|
|
}
|
|
for _, e := range l {
|
|
if err := addSpecialOpt(e.nameOpt, e.use); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for _, name := range mailboxes.Regular {
|
|
if err := add(name, SpecialUse{}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
uidvalidity++
|
|
if err := tx.Insert(&NextUIDValidity{1, uidvalidity}); err != nil {
|
|
return fmt.Errorf("inserting nextuidvalidity: %w", err)
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// Close reduces the reference count, and closes the database connection when
|
|
// it was the last user.
|
|
func (a *Account) Close() error {
|
|
if CheckConsistencyOnClose {
|
|
xerr := a.CheckConsistency()
|
|
err := closeAccount(a)
|
|
if xerr != nil {
|
|
panic(xerr)
|
|
}
|
|
return err
|
|
}
|
|
return closeAccount(a)
|
|
}
|
|
|
|
// CheckConsistency checks the consistency of the database and returns a non-nil
|
|
// error for these cases:
|
|
//
|
|
// - Missing on-disk file for message.
|
|
// - Mismatch between message size and length of MsgPrefix and on-disk file.
|
|
// - Missing HaveCounts.
|
|
// - Incorrect mailbox counts.
|
|
// - Incorrect total message size.
|
|
// - Message with UID >= mailbox uid next.
|
|
// - Mailbox uidvalidity >= account uid validity.
|
|
// - ModSeq > 0, CreateSeq > 0, CreateSeq <= ModSeq.
|
|
// - All messages have a nonzero ThreadID, and no cycles in ThreadParentID, and parent messages the same ThreadParentIDs tail.
|
|
func (a *Account) CheckConsistency() error {
|
|
var uidErrors []string // With a limit, could be many.
|
|
var modseqErrors []string // With limit.
|
|
var fileErrors []string // With limit.
|
|
var threadidErrors []string // With limit.
|
|
var threadParentErrors []string // With limit.
|
|
var threadAncestorErrors []string // With limit.
|
|
var errors []string
|
|
|
|
err := a.DB.Read(context.Background(), func(tx *bstore.Tx) error {
|
|
nuv := NextUIDValidity{ID: 1}
|
|
err := tx.Get(&nuv)
|
|
if err != nil {
|
|
return fmt.Errorf("fetching next uid validity: %v", err)
|
|
}
|
|
|
|
mailboxes := map[int64]Mailbox{}
|
|
err = bstore.QueryTx[Mailbox](tx).ForEach(func(mb Mailbox) error {
|
|
mailboxes[mb.ID] = mb
|
|
|
|
if mb.UIDValidity >= nuv.Next {
|
|
errmsg := fmt.Sprintf("mailbox %q (id %d) has uidvalidity %d >= account next uidvalidity %d", mb.Name, mb.ID, mb.UIDValidity, nuv.Next)
|
|
errors = append(errors, errmsg)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("listing mailboxes: %v", err)
|
|
}
|
|
|
|
counts := map[int64]MailboxCounts{}
|
|
err = bstore.QueryTx[Message](tx).ForEach(func(m Message) error {
|
|
mc := counts[m.MailboxID]
|
|
mc.Add(m.MailboxCounts())
|
|
counts[m.MailboxID] = mc
|
|
|
|
mb := mailboxes[m.MailboxID]
|
|
|
|
if (m.ModSeq == 0 || m.CreateSeq == 0 || m.CreateSeq > m.ModSeq) && len(modseqErrors) < 20 {
|
|
modseqerr := fmt.Sprintf("message %d in mailbox %q (id %d) has invalid modseq %d or createseq %d, both must be > 0 and createseq <= modseq", m.ID, mb.Name, mb.ID, m.ModSeq, m.CreateSeq)
|
|
modseqErrors = append(modseqErrors, modseqerr)
|
|
}
|
|
if m.UID >= mb.UIDNext && len(uidErrors) < 20 {
|
|
uiderr := fmt.Sprintf("message %d in mailbox %q (id %d) has uid %d >= mailbox uidnext %d", m.ID, mb.Name, mb.ID, m.UID, mb.UIDNext)
|
|
uidErrors = append(uidErrors, uiderr)
|
|
}
|
|
if m.Expunged {
|
|
return nil
|
|
}
|
|
p := a.MessagePath(m.ID)
|
|
st, err := os.Stat(p)
|
|
if err != nil {
|
|
existserr := fmt.Sprintf("message %d in mailbox %q (id %d) on-disk file %s: %v", m.ID, mb.Name, mb.ID, p, err)
|
|
fileErrors = append(fileErrors, existserr)
|
|
} else if len(fileErrors) < 20 && m.Size != int64(len(m.MsgPrefix))+st.Size() {
|
|
sizeerr := fmt.Sprintf("message %d in mailbox %q (id %d) has size %d != len msgprefix %d + on-disk file size %d = %d", m.ID, mb.Name, mb.ID, m.Size, len(m.MsgPrefix), st.Size(), int64(len(m.MsgPrefix))+st.Size())
|
|
fileErrors = append(fileErrors, sizeerr)
|
|
}
|
|
|
|
if m.ThreadID <= 0 && len(threadidErrors) < 20 {
|
|
err := fmt.Sprintf("message %d in mailbox %q (id %d) has threadid 0", m.ID, mb.Name, mb.ID)
|
|
threadidErrors = append(threadidErrors, err)
|
|
}
|
|
if slices.Contains(m.ThreadParentIDs, m.ID) && len(threadParentErrors) < 20 {
|
|
err := fmt.Sprintf("message %d in mailbox %q (id %d) references itself in threadparentids", m.ID, mb.Name, mb.ID)
|
|
threadParentErrors = append(threadParentErrors, err)
|
|
}
|
|
for i, pid := range m.ThreadParentIDs {
|
|
am := Message{ID: pid}
|
|
if err := tx.Get(&am); err == bstore.ErrAbsent {
|
|
continue
|
|
} else if err != nil {
|
|
return fmt.Errorf("get ancestor message: %v", err)
|
|
} else if !slices.Equal(m.ThreadParentIDs[i+1:], am.ThreadParentIDs) && len(threadAncestorErrors) < 20 {
|
|
err := fmt.Sprintf("message %d, thread %d has ancestor ids %v, and ancestor at index %d with id %d should have the same tail but has %v\n", m.ID, m.ThreadID, m.ThreadParentIDs, i, am.ID, am.ThreadParentIDs)
|
|
threadAncestorErrors = append(threadAncestorErrors, err)
|
|
} else {
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("reading messages: %v", err)
|
|
}
|
|
|
|
var totalSize int64
|
|
for _, mb := range mailboxes {
|
|
totalSize += mb.Size
|
|
if !mb.HaveCounts {
|
|
errmsg := fmt.Sprintf("mailbox %q (id %d) does not have counts, should be %#v", mb.Name, mb.ID, counts[mb.ID])
|
|
errors = append(errors, errmsg)
|
|
} else if mb.MailboxCounts != counts[mb.ID] {
|
|
mbcounterr := fmt.Sprintf("mailbox %q (id %d) has wrong counts %s, should be %s", mb.Name, mb.ID, mb.MailboxCounts, counts[mb.ID])
|
|
errors = append(errors, mbcounterr)
|
|
}
|
|
}
|
|
|
|
du := DiskUsage{ID: 1}
|
|
if err := tx.Get(&du); err != nil {
|
|
return fmt.Errorf("get diskusage")
|
|
}
|
|
if du.MessageSize != totalSize {
|
|
errmsg := fmt.Sprintf("total message size in database is %d, sum of mailbox message sizes is %d", du.MessageSize, totalSize)
|
|
errors = append(errors, errmsg)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
errors = append(errors, uidErrors...)
|
|
errors = append(errors, modseqErrors...)
|
|
errors = append(errors, fileErrors...)
|
|
errors = append(errors, threadidErrors...)
|
|
errors = append(errors, threadParentErrors...)
|
|
errors = append(errors, threadAncestorErrors...)
|
|
if len(errors) > 0 {
|
|
return fmt.Errorf("%s", strings.Join(errors, "; "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Conf returns the configuration for this account if it still exists. During
|
|
// an SMTP session, a configuration update may drop an account.
|
|
func (a *Account) Conf() (config.Account, bool) {
|
|
return mox.Conf.Account(a.Name)
|
|
}
|
|
|
|
// NextUIDValidity returns the next new/unique uidvalidity to use for this account.
|
|
func (a *Account) NextUIDValidity(tx *bstore.Tx) (uint32, error) {
|
|
nuv := NextUIDValidity{ID: 1}
|
|
if err := tx.Get(&nuv); err != nil {
|
|
return 0, err
|
|
}
|
|
v := nuv.Next
|
|
nuv.Next++
|
|
if err := tx.Update(&nuv); err != nil {
|
|
return 0, err
|
|
}
|
|
return v, nil
|
|
}
|
|
|
|
// NextModSeq returns the next modification sequence, which is global per account,
|
|
// over all types.
|
|
func (a *Account) NextModSeq(tx *bstore.Tx) (ModSeq, error) {
|
|
v := SyncState{ID: 1}
|
|
if err := tx.Get(&v); err == bstore.ErrAbsent {
|
|
// We start assigning from modseq 2. Modseq 0 is not usable, so returned as 1, so
|
|
// already used.
|
|
// HighestDeletedModSeq is -1 so comparison against the default ModSeq zero value
|
|
// makes sense.
|
|
v = SyncState{1, 2, -1}
|
|
return v.LastModSeq, tx.Insert(&v)
|
|
} else if err != nil {
|
|
return 0, err
|
|
}
|
|
v.LastModSeq++
|
|
return v.LastModSeq, tx.Update(&v)
|
|
}
|
|
|
|
func (a *Account) HighestDeletedModSeq(tx *bstore.Tx) (ModSeq, error) {
|
|
v := SyncState{ID: 1}
|
|
err := tx.Get(&v)
|
|
if err == bstore.ErrAbsent {
|
|
return 0, nil
|
|
}
|
|
return v.HighestDeletedModSeq, err
|
|
}
|
|
|
|
// WithWLock runs fn with account writelock held. Necessary for account/mailbox modification. For message delivery, a read lock is required.
|
|
func (a *Account) WithWLock(fn func()) {
|
|
a.Lock()
|
|
defer a.Unlock()
|
|
fn()
|
|
}
|
|
|
|
// WithRLock runs fn with account read lock held. Needed for message delivery.
|
|
func (a *Account) WithRLock(fn func()) {
|
|
a.RLock()
|
|
defer a.RUnlock()
|
|
fn()
|
|
}
|
|
|
|
// DeliverMessage delivers a mail message to the account.
|
|
//
|
|
// The message, with msg.MsgPrefix and msgFile combined, must have a header
|
|
// section. The caller is responsible for adding a header separator to
|
|
// msg.MsgPrefix if missing from an incoming message.
|
|
//
|
|
// If the destination mailbox has the Sent special-use flag, the message is parsed
|
|
// for its recipients (to/cc/bcc). Their domains are added to Recipients for use in
|
|
// dmarc reputation.
|
|
//
|
|
// If sync is true, the message file and its directory are synced. Should be true
|
|
// for regular mail delivery, but can be false when importing many messages.
|
|
//
|
|
// If updateDiskUsage is true, the account total message size (for quota) is
|
|
// updated. Callers must check if a message can be added within quota before
|
|
// calling DeliverMessage.
|
|
//
|
|
// If CreateSeq/ModSeq is not set, it is assigned automatically.
|
|
//
|
|
// Must be called with account rlock or wlock.
|
|
//
|
|
// Caller must broadcast new message.
|
|
//
|
|
// Caller must update mailbox counts.
|
|
func (a *Account) DeliverMessage(log mlog.Log, tx *bstore.Tx, m *Message, msgFile *os.File, sync, notrain, nothreads, updateDiskUsage bool) error {
|
|
if m.Expunged {
|
|
return fmt.Errorf("cannot deliver expunged message")
|
|
}
|
|
|
|
mb := Mailbox{ID: m.MailboxID}
|
|
if err := tx.Get(&mb); err != nil {
|
|
return fmt.Errorf("get mailbox: %w", err)
|
|
}
|
|
m.UID = mb.UIDNext
|
|
mb.UIDNext++
|
|
if err := tx.Update(&mb); err != nil {
|
|
return fmt.Errorf("updating mailbox nextuid: %w", err)
|
|
}
|
|
|
|
if updateDiskUsage {
|
|
du := DiskUsage{ID: 1}
|
|
if err := tx.Get(&du); err != nil {
|
|
return fmt.Errorf("get disk usage: %v", err)
|
|
}
|
|
du.MessageSize += m.Size
|
|
if err := tx.Update(&du); err != nil {
|
|
return fmt.Errorf("update disk usage: %v", err)
|
|
}
|
|
}
|
|
|
|
conf, _ := a.Conf()
|
|
m.JunkFlagsForMailbox(mb, conf)
|
|
|
|
mr := FileMsgReader(m.MsgPrefix, msgFile) // We don't close, it would close the msgFile.
|
|
var part *message.Part
|
|
if m.ParsedBuf == nil {
|
|
p, err := message.EnsurePart(log.Logger, false, mr, m.Size)
|
|
if err != nil {
|
|
log.Infox("parsing delivered message", err, slog.String("parse", ""), slog.Int64("message", m.ID))
|
|
// We continue, p is still valid.
|
|
}
|
|
part = &p
|
|
buf, err := json.Marshal(part)
|
|
if err != nil {
|
|
return fmt.Errorf("marshal parsed message: %w", err)
|
|
}
|
|
m.ParsedBuf = buf
|
|
} else {
|
|
var p message.Part
|
|
if err := json.Unmarshal(m.ParsedBuf, &p); err != nil {
|
|
log.Errorx("unmarshal parsed message, continuing", err, slog.String("parse", ""))
|
|
} else {
|
|
part = &p
|
|
}
|
|
}
|
|
|
|
// If we are delivering to the originally intended mailbox, no need to store the mailbox ID again.
|
|
if m.MailboxDestinedID != 0 && m.MailboxDestinedID == m.MailboxOrigID {
|
|
m.MailboxDestinedID = 0
|
|
}
|
|
if m.CreateSeq == 0 || m.ModSeq == 0 {
|
|
modseq, err := a.NextModSeq(tx)
|
|
if err != nil {
|
|
return fmt.Errorf("assigning next modseq: %w", err)
|
|
}
|
|
m.CreateSeq = modseq
|
|
m.ModSeq = modseq
|
|
}
|
|
|
|
if part != nil && m.MessageID == "" && m.SubjectBase == "" {
|
|
m.PrepareThreading(log, part)
|
|
}
|
|
|
|
// Assign to thread (if upgrade has completed).
|
|
noThreadID := nothreads
|
|
if m.ThreadID == 0 && !nothreads && part != nil {
|
|
select {
|
|
case <-a.threadsCompleted:
|
|
if a.threadsErr != nil {
|
|
log.Info("not assigning threads for new delivery, upgrading to threads failed")
|
|
noThreadID = true
|
|
} else {
|
|
if err := assignThread(log, tx, m, part); err != nil {
|
|
return fmt.Errorf("assigning thread: %w", err)
|
|
}
|
|
}
|
|
default:
|
|
// note: since we have a write transaction to get here, we can't wait for the
|
|
// thread upgrade to finish.
|
|
// If we don't assign a threadid the upgrade process will do it.
|
|
log.Info("not assigning threads for new delivery, upgrading to threads in progress which will assign this message")
|
|
noThreadID = true
|
|
}
|
|
}
|
|
|
|
if err := tx.Insert(m); err != nil {
|
|
return fmt.Errorf("inserting message: %w", err)
|
|
}
|
|
if !noThreadID && m.ThreadID == 0 {
|
|
m.ThreadID = m.ID
|
|
if err := tx.Update(m); err != nil {
|
|
return fmt.Errorf("updating message for its own thread id: %w", err)
|
|
}
|
|
}
|
|
|
|
// todo: perhaps we should match the recipients based on smtp submission and a matching message-id? we now miss the addresses in bcc's. for webmail, we could insert the recipients directly.
|
|
if mb.Sent && part != nil && part.Envelope != nil {
|
|
e := part.Envelope
|
|
sent := e.Date
|
|
if sent.IsZero() {
|
|
sent = m.Received
|
|
}
|
|
if sent.IsZero() {
|
|
sent = time.Now()
|
|
}
|
|
addrs := append(append(e.To, e.CC...), e.BCC...)
|
|
for _, addr := range addrs {
|
|
if addr.User == "" {
|
|
// Would trigger error because Recipient.Localpart must be nonzero. todo: we could allow empty localpart in db, and filter by not using FilterNonzero.
|
|
log.Info("to/cc/bcc address with empty localpart, not inserting as recipient", slog.Any("address", addr))
|
|
continue
|
|
}
|
|
d, err := dns.ParseDomain(addr.Host)
|
|
if err != nil {
|
|
log.Debugx("parsing domain in to/cc/bcc address", err, slog.Any("address", addr))
|
|
continue
|
|
}
|
|
mr := Recipient{
|
|
MessageID: m.ID,
|
|
Localpart: smtp.Localpart(addr.User),
|
|
Domain: d.Name(),
|
|
OrgDomain: publicsuffix.Lookup(context.TODO(), log.Logger, d).Name(),
|
|
Sent: sent,
|
|
}
|
|
if err := tx.Insert(&mr); err != nil {
|
|
return fmt.Errorf("inserting sent message recipients: %w", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
msgPath := a.MessagePath(m.ID)
|
|
msgDir := filepath.Dir(msgPath)
|
|
os.MkdirAll(msgDir, 0770)
|
|
|
|
// Sync file data to disk.
|
|
if sync {
|
|
if err := msgFile.Sync(); err != nil {
|
|
return fmt.Errorf("fsync message file: %w", err)
|
|
}
|
|
}
|
|
|
|
if err := moxio.LinkOrCopy(log, msgPath, msgFile.Name(), &moxio.AtReader{R: msgFile}, true); err != nil {
|
|
return fmt.Errorf("linking/copying message to new file: %w", err)
|
|
}
|
|
|
|
if sync {
|
|
if err := moxio.SyncDir(log, msgDir); err != nil {
|
|
xerr := os.Remove(msgPath)
|
|
log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))
|
|
return fmt.Errorf("sync directory: %w", err)
|
|
}
|
|
}
|
|
|
|
if !notrain && m.NeedsTraining() {
|
|
l := []Message{*m}
|
|
if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil {
|
|
xerr := os.Remove(msgPath)
|
|
log.Check(xerr, "removing message after syncdir error", slog.String("path", msgPath))
|
|
return fmt.Errorf("training junkfilter: %w", err)
|
|
}
|
|
*m = l[0]
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// SetPassword saves a new password for this account. This password is used for
|
|
// IMAP, SMTP (submission) sessions and the HTTP account web page.
|
|
func (a *Account) SetPassword(log mlog.Log, password string) error {
|
|
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
|
if err != nil {
|
|
return fmt.Errorf("generating password hash: %w", err)
|
|
}
|
|
|
|
err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
|
|
if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {
|
|
return fmt.Errorf("deleting existing password: %v", err)
|
|
}
|
|
var pw Password
|
|
pw.Hash = string(hash)
|
|
|
|
// CRAM-MD5 calculates an HMAC-MD5, with the password as key, over a per-attempt
|
|
// unique text that includes a timestamp. HMAC performs two hashes. Both times, the
|
|
// first block is based on the key/password. We hash those first blocks now, and
|
|
// store the hash state in the database. When we actually authenticate, we'll
|
|
// complete the HMAC by hashing only the text. We cannot store crypto/hmac's hash,
|
|
// because it does not expose its internal state and isn't a BinaryMarshaler.
|
|
// ../rfc/2104:121
|
|
pw.CRAMMD5.Ipad = md5.New()
|
|
pw.CRAMMD5.Opad = md5.New()
|
|
key := []byte(password)
|
|
if len(key) > 64 {
|
|
t := md5.Sum(key)
|
|
key = t[:]
|
|
}
|
|
ipad := make([]byte, md5.BlockSize)
|
|
opad := make([]byte, md5.BlockSize)
|
|
copy(ipad, key)
|
|
copy(opad, key)
|
|
for i := range ipad {
|
|
ipad[i] ^= 0x36
|
|
opad[i] ^= 0x5c
|
|
}
|
|
pw.CRAMMD5.Ipad.Write(ipad)
|
|
pw.CRAMMD5.Opad.Write(opad)
|
|
|
|
pw.SCRAMSHA1.Salt = scram.MakeRandom()
|
|
pw.SCRAMSHA1.Iterations = 2 * 4096
|
|
pw.SCRAMSHA1.SaltedPassword = scram.SaltPassword(sha1.New, password, pw.SCRAMSHA1.Salt, pw.SCRAMSHA1.Iterations)
|
|
|
|
pw.SCRAMSHA256.Salt = scram.MakeRandom()
|
|
pw.SCRAMSHA256.Iterations = 4096
|
|
pw.SCRAMSHA256.SaltedPassword = scram.SaltPassword(sha256.New, password, pw.SCRAMSHA256.Salt, pw.SCRAMSHA256.Iterations)
|
|
|
|
if err := tx.Insert(&pw); err != nil {
|
|
return fmt.Errorf("inserting new password: %v", err)
|
|
}
|
|
|
|
return sessionRemoveAll(context.TODO(), log, tx, a.Name)
|
|
})
|
|
if err == nil {
|
|
log.Info("new password set for account", slog.String("account", a.Name))
|
|
}
|
|
return err
|
|
}
|
|
|
|
// Subjectpass returns the signing key for use with subjectpass for the given
|
|
// email address with canonical localpart.
|
|
func (a *Account) Subjectpass(email string) (key string, err error) {
|
|
return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
|
|
v := Subjectpass{Email: email}
|
|
err := tx.Get(&v)
|
|
if err == nil {
|
|
key = v.Key
|
|
return nil
|
|
}
|
|
if !errors.Is(err, bstore.ErrAbsent) {
|
|
return fmt.Errorf("get subjectpass key from accounts database: %w", err)
|
|
}
|
|
key = ""
|
|
const chars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
|
buf := make([]byte, 16)
|
|
if _, err := cryptorand.Read(buf); err != nil {
|
|
return err
|
|
}
|
|
for _, b := range buf {
|
|
key += string(chars[int(b)%len(chars)])
|
|
}
|
|
v.Key = key
|
|
return tx.Insert(&v)
|
|
})
|
|
}
|
|
|
|
// Ensure mailbox is present in database, adding records for the mailbox and its
|
|
// parents if they aren't present.
|
|
//
|
|
// If subscribe is true, any mailboxes that were created will also be subscribed to.
|
|
// Caller must hold account wlock.
|
|
// Caller must propagate changes if any.
|
|
func (a *Account) MailboxEnsure(tx *bstore.Tx, name string, subscribe bool) (mb Mailbox, changes []Change, rerr error) {
|
|
if norm.NFC.String(name) != name {
|
|
return Mailbox{}, nil, fmt.Errorf("mailbox name not normalized")
|
|
}
|
|
|
|
// Quick sanity check.
|
|
if strings.EqualFold(name, "inbox") && name != "Inbox" {
|
|
return Mailbox{}, nil, fmt.Errorf("bad casing for inbox")
|
|
}
|
|
|
|
elems := strings.Split(name, "/")
|
|
q := bstore.QueryTx[Mailbox](tx)
|
|
q.FilterFn(func(mb Mailbox) bool {
|
|
return mb.Name == elems[0] || strings.HasPrefix(mb.Name, elems[0]+"/")
|
|
})
|
|
l, err := q.List()
|
|
if err != nil {
|
|
return Mailbox{}, nil, fmt.Errorf("list mailboxes: %v", err)
|
|
}
|
|
|
|
mailboxes := map[string]Mailbox{}
|
|
for _, xmb := range l {
|
|
mailboxes[xmb.Name] = xmb
|
|
}
|
|
|
|
p := ""
|
|
for _, elem := range elems {
|
|
if p != "" {
|
|
p += "/"
|
|
}
|
|
p += elem
|
|
var ok bool
|
|
mb, ok = mailboxes[p]
|
|
if ok {
|
|
continue
|
|
}
|
|
uidval, err := a.NextUIDValidity(tx)
|
|
if err != nil {
|
|
return Mailbox{}, nil, fmt.Errorf("next uid validity: %v", err)
|
|
}
|
|
mb = Mailbox{
|
|
Name: p,
|
|
UIDValidity: uidval,
|
|
UIDNext: 1,
|
|
HaveCounts: true,
|
|
}
|
|
err = tx.Insert(&mb)
|
|
if err != nil {
|
|
return Mailbox{}, nil, fmt.Errorf("creating new mailbox: %v", err)
|
|
}
|
|
|
|
var flags []string
|
|
if subscribe {
|
|
if tx.Get(&Subscription{p}) != nil {
|
|
err := tx.Insert(&Subscription{p})
|
|
if err != nil {
|
|
return Mailbox{}, nil, fmt.Errorf("subscribing to mailbox: %v", err)
|
|
}
|
|
}
|
|
flags = []string{`\Subscribed`}
|
|
}
|
|
changes = append(changes, ChangeAddMailbox{mb, flags})
|
|
}
|
|
return mb, changes, nil
|
|
}
|
|
|
|
// MailboxExists checks if mailbox exists.
|
|
// Caller must hold account rlock.
|
|
func (a *Account) MailboxExists(tx *bstore.Tx, name string) (bool, error) {
|
|
q := bstore.QueryTx[Mailbox](tx)
|
|
q.FilterEqual("Name", name)
|
|
return q.Exists()
|
|
}
|
|
|
|
// MailboxFind finds a mailbox by name, returning a nil mailbox and nil error if mailbox does not exist.
|
|
func (a *Account) MailboxFind(tx *bstore.Tx, name string) (*Mailbox, error) {
|
|
q := bstore.QueryTx[Mailbox](tx)
|
|
q.FilterEqual("Name", name)
|
|
mb, err := q.Get()
|
|
if err == bstore.ErrAbsent {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("looking up mailbox: %w", err)
|
|
}
|
|
return &mb, nil
|
|
}
|
|
|
|
// SubscriptionEnsure ensures a subscription for name exists. The mailbox does not
|
|
// have to exist. Any parents are not automatically subscribed.
|
|
// Changes are returned and must be broadcasted by the caller.
|
|
func (a *Account) SubscriptionEnsure(tx *bstore.Tx, name string) ([]Change, error) {
|
|
if err := tx.Get(&Subscription{name}); err == nil {
|
|
return nil, nil
|
|
}
|
|
|
|
if err := tx.Insert(&Subscription{name}); err != nil {
|
|
return nil, fmt.Errorf("inserting subscription: %w", err)
|
|
}
|
|
|
|
q := bstore.QueryTx[Mailbox](tx)
|
|
q.FilterEqual("Name", name)
|
|
_, err := q.Get()
|
|
if err == nil {
|
|
return []Change{ChangeAddSubscription{name, nil}}, nil
|
|
} else if err != bstore.ErrAbsent {
|
|
return nil, fmt.Errorf("looking up mailbox for subscription: %w", err)
|
|
}
|
|
return []Change{ChangeAddSubscription{name, []string{`\NonExistent`}}}, nil
|
|
}
|
|
|
|
// MessageRuleset returns the first ruleset (if any) that message the message
|
|
// represented by msgPrefix and msgFile, with smtp and validation fields from m.
|
|
func MessageRuleset(log mlog.Log, dest config.Destination, m *Message, msgPrefix []byte, msgFile *os.File) *config.Ruleset {
|
|
if len(dest.Rulesets) == 0 {
|
|
return nil
|
|
}
|
|
|
|
mr := FileMsgReader(msgPrefix, msgFile) // We don't close, it would close the msgFile.
|
|
p, err := message.Parse(log.Logger, false, mr)
|
|
if err != nil {
|
|
log.Errorx("parsing message for evaluating rulesets, continuing with headers", err, slog.String("parse", ""))
|
|
// note: part is still set.
|
|
}
|
|
// todo optimize: only parse header if needed for rulesets. and probably reuse an earlier parsing.
|
|
header, err := p.Header()
|
|
if err != nil {
|
|
log.Errorx("parsing message headers for evaluating rulesets, delivering to default mailbox", err, slog.String("parse", ""))
|
|
// todo: reject message?
|
|
return nil
|
|
}
|
|
|
|
ruleset:
|
|
for _, rs := range dest.Rulesets {
|
|
if rs.SMTPMailFromRegexpCompiled != nil {
|
|
if !rs.SMTPMailFromRegexpCompiled.MatchString(m.MailFrom) {
|
|
continue ruleset
|
|
}
|
|
}
|
|
|
|
if !rs.VerifiedDNSDomain.IsZero() {
|
|
d := rs.VerifiedDNSDomain.Name()
|
|
suffix := "." + d
|
|
matchDomain := func(s string) bool {
|
|
return s == d || strings.HasSuffix(s, suffix)
|
|
}
|
|
var ok bool
|
|
if m.EHLOValidated && matchDomain(m.EHLODomain) {
|
|
ok = true
|
|
}
|
|
if m.MailFromValidated && matchDomain(m.MailFromDomain) {
|
|
ok = true
|
|
}
|
|
for _, d := range m.DKIMDomains {
|
|
if matchDomain(d) {
|
|
ok = true
|
|
break
|
|
}
|
|
}
|
|
if !ok {
|
|
continue ruleset
|
|
}
|
|
}
|
|
|
|
header:
|
|
for _, t := range rs.HeadersRegexpCompiled {
|
|
for k, vl := range header {
|
|
k = strings.ToLower(k)
|
|
if !t[0].MatchString(k) {
|
|
continue
|
|
}
|
|
for _, v := range vl {
|
|
v = strings.ToLower(strings.TrimSpace(v))
|
|
if t[1].MatchString(v) {
|
|
continue header
|
|
}
|
|
}
|
|
}
|
|
continue ruleset
|
|
}
|
|
return &rs
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// MessagePath returns the file system path of a message.
|
|
func (a *Account) MessagePath(messageID int64) string {
|
|
return strings.Join(append([]string{a.Dir, "msg"}, messagePathElems(messageID)...), string(filepath.Separator))
|
|
}
|
|
|
|
// MessageReader opens a message for reading, transparently combining the
|
|
// message prefix with the original incoming message.
|
|
func (a *Account) MessageReader(m Message) *MsgReader {
|
|
return &MsgReader{prefix: m.MsgPrefix, path: a.MessagePath(m.ID), size: m.Size}
|
|
}
|
|
|
|
// DeliverDestination delivers an email to dest, based on the configured rulesets.
|
|
//
|
|
// Returns ErrOverQuota when account would be over quota after adding message.
|
|
//
|
|
// Caller must hold account wlock (mailbox may be created).
|
|
// Message delivery, possible mailbox creation, and updated mailbox counts are
|
|
// broadcasted.
|
|
func (a *Account) DeliverDestination(log mlog.Log, dest config.Destination, m *Message, msgFile *os.File) error {
|
|
var mailbox string
|
|
rs := MessageRuleset(log, dest, m, m.MsgPrefix, msgFile)
|
|
if rs != nil {
|
|
mailbox = rs.Mailbox
|
|
} else if dest.Mailbox == "" {
|
|
mailbox = "Inbox"
|
|
} else {
|
|
mailbox = dest.Mailbox
|
|
}
|
|
return a.DeliverMailbox(log, mailbox, m, msgFile)
|
|
}
|
|
|
|
// DeliverMailbox delivers an email to the specified mailbox.
|
|
//
|
|
// Returns ErrOverQuota when account would be over quota after adding message.
|
|
//
|
|
// Caller must hold account wlock (mailbox may be created).
|
|
// Message delivery, possible mailbox creation, and updated mailbox counts are
|
|
// broadcasted.
|
|
func (a *Account) DeliverMailbox(log mlog.Log, mailbox string, m *Message, msgFile *os.File) error {
|
|
var changes []Change
|
|
err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
|
|
if ok, _, err := a.CanAddMessageSize(tx, m.Size); err != nil {
|
|
return err
|
|
} else if !ok {
|
|
return ErrOverQuota
|
|
}
|
|
|
|
mb, chl, err := a.MailboxEnsure(tx, mailbox, true)
|
|
if err != nil {
|
|
return fmt.Errorf("ensuring mailbox: %w", err)
|
|
}
|
|
m.MailboxID = mb.ID
|
|
m.MailboxOrigID = mb.ID
|
|
|
|
// Update count early, DeliverMessage will update mb too and we don't want to fetch
|
|
// it again before updating.
|
|
mb.MailboxCounts.Add(m.MailboxCounts())
|
|
if err := tx.Update(&mb); err != nil {
|
|
return fmt.Errorf("updating mailbox for delivery: %w", err)
|
|
}
|
|
|
|
if err := a.DeliverMessage(log, tx, m, msgFile, true, false, false, true); err != nil {
|
|
return err
|
|
}
|
|
|
|
changes = append(changes, chl...)
|
|
changes = append(changes, m.ChangeAddUID(), mb.ChangeCounts())
|
|
return nil
|
|
})
|
|
// todo: if rename succeeded but transaction failed, we should remove the file.
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
BroadcastChanges(a, changes)
|
|
return nil
|
|
}
|
|
|
|
// TidyRejectsMailbox removes old reject emails, and returns whether there is space for a new delivery.
|
|
//
|
|
// Caller most hold account wlock.
|
|
// Changes are broadcasted.
|
|
func (a *Account) TidyRejectsMailbox(log mlog.Log, rejectsMailbox string) (hasSpace bool, rerr error) {
|
|
var changes []Change
|
|
|
|
var remove []Message
|
|
defer func() {
|
|
for _, m := range remove {
|
|
p := a.MessagePath(m.ID)
|
|
err := os.Remove(p)
|
|
log.Check(err, "removing rejects message file", slog.String("path", p))
|
|
}
|
|
}()
|
|
|
|
err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
|
|
mb, err := a.MailboxFind(tx, rejectsMailbox)
|
|
if err != nil {
|
|
return fmt.Errorf("finding mailbox: %w", err)
|
|
}
|
|
if mb == nil {
|
|
// No messages have been delivered yet.
|
|
hasSpace = true
|
|
return nil
|
|
}
|
|
|
|
// Gather old messages to remove.
|
|
old := time.Now().Add(-14 * 24 * time.Hour)
|
|
qdel := bstore.QueryTx[Message](tx)
|
|
qdel.FilterNonzero(Message{MailboxID: mb.ID})
|
|
qdel.FilterEqual("Expunged", false)
|
|
qdel.FilterLess("Received", old)
|
|
remove, err = qdel.List()
|
|
if err != nil {
|
|
return fmt.Errorf("listing old messages: %w", err)
|
|
}
|
|
|
|
changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)
|
|
if err != nil {
|
|
return fmt.Errorf("removing messages: %w", err)
|
|
}
|
|
|
|
// We allow up to n messages.
|
|
qcount := bstore.QueryTx[Message](tx)
|
|
qcount.FilterNonzero(Message{MailboxID: mb.ID})
|
|
qcount.FilterEqual("Expunged", false)
|
|
qcount.Limit(1000)
|
|
n, err := qcount.Count()
|
|
if err != nil {
|
|
return fmt.Errorf("counting rejects: %w", err)
|
|
}
|
|
hasSpace = n < 1000
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
remove = nil // Don't remove files on failure.
|
|
return false, err
|
|
}
|
|
|
|
BroadcastChanges(a, changes)
|
|
|
|
return hasSpace, nil
|
|
}
|
|
|
|
func (a *Account) rejectsRemoveMessages(ctx context.Context, log mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {
|
|
if len(l) == 0 {
|
|
return nil, nil
|
|
}
|
|
ids := make([]int64, len(l))
|
|
anyids := make([]any, len(l))
|
|
for i, m := range l {
|
|
ids[i] = m.ID
|
|
anyids[i] = m.ID
|
|
}
|
|
|
|
// Remove any message recipients. Should not happen, but a user can move messages
|
|
// from a Sent mailbox to the rejects mailbox...
|
|
qdmr := bstore.QueryTx[Recipient](tx)
|
|
qdmr.FilterEqual("MessageID", anyids...)
|
|
if _, err := qdmr.Delete(); err != nil {
|
|
return nil, fmt.Errorf("deleting from message recipient: %w", err)
|
|
}
|
|
|
|
// Assign new modseq.
|
|
modseq, err := a.NextModSeq(tx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("assign next modseq: %w", err)
|
|
}
|
|
|
|
// Expunge the messages.
|
|
qx := bstore.QueryTx[Message](tx)
|
|
qx.FilterIDs(ids)
|
|
var expunged []Message
|
|
qx.Gather(&expunged)
|
|
if _, err := qx.UpdateNonzero(Message{ModSeq: modseq, Expunged: true}); err != nil {
|
|
return nil, fmt.Errorf("expunging messages: %w", err)
|
|
}
|
|
|
|
var totalSize int64
|
|
for _, m := range expunged {
|
|
m.Expunged = false // Was set by update, but would cause wrong count.
|
|
mb.MailboxCounts.Sub(m.MailboxCounts())
|
|
totalSize += m.Size
|
|
}
|
|
if err := tx.Update(mb); err != nil {
|
|
return nil, fmt.Errorf("updating mailbox counts: %w", err)
|
|
}
|
|
if err := a.AddMessageSize(log, tx, -totalSize); err != nil {
|
|
return nil, fmt.Errorf("updating disk usage: %w", err)
|
|
}
|
|
|
|
// Mark as neutral and train so junk filter gets untrained with these (junk) messages.
|
|
for i := range expunged {
|
|
expunged[i].Junk = false
|
|
expunged[i].Notjunk = false
|
|
}
|
|
if err := a.RetrainMessages(ctx, log, tx, expunged, true); err != nil {
|
|
return nil, fmt.Errorf("retraining expunged messages: %w", err)
|
|
}
|
|
|
|
changes := make([]Change, len(l), len(l)+1)
|
|
for i, m := range l {
|
|
changes[i] = ChangeRemoveUIDs{mb.ID, []UID{m.UID}, modseq}
|
|
}
|
|
changes = append(changes, mb.ChangeCounts())
|
|
return changes, nil
|
|
}
|
|
|
|
// RejectsRemove removes a message from the rejects mailbox if present.
|
|
// Caller most hold account wlock.
|
|
// Changes are broadcasted.
|
|
func (a *Account) RejectsRemove(log mlog.Log, rejectsMailbox, messageID string) error {
|
|
var changes []Change
|
|
|
|
var remove []Message
|
|
defer func() {
|
|
for _, m := range remove {
|
|
p := a.MessagePath(m.ID)
|
|
err := os.Remove(p)
|
|
log.Check(err, "removing rejects message file", slog.String("path", p))
|
|
}
|
|
}()
|
|
|
|
err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
|
|
mb, err := a.MailboxFind(tx, rejectsMailbox)
|
|
if err != nil {
|
|
return fmt.Errorf("finding mailbox: %w", err)
|
|
}
|
|
if mb == nil {
|
|
return nil
|
|
}
|
|
|
|
q := bstore.QueryTx[Message](tx)
|
|
q.FilterNonzero(Message{MailboxID: mb.ID, MessageID: messageID})
|
|
q.FilterEqual("Expunged", false)
|
|
remove, err = q.List()
|
|
if err != nil {
|
|
return fmt.Errorf("listing messages to remove: %w", err)
|
|
}
|
|
|
|
changes, err = a.rejectsRemoveMessages(context.TODO(), log, tx, mb, remove)
|
|
if err != nil {
|
|
return fmt.Errorf("removing messages: %w", err)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
remove = nil // Don't remove files on failure.
|
|
return err
|
|
}
|
|
|
|
BroadcastChanges(a, changes)
|
|
|
|
return nil
|
|
}
|
|
|
|
// AddMessageSize adjusts the DiskUsage.MessageSize by size.
|
|
func (a *Account) AddMessageSize(log mlog.Log, tx *bstore.Tx, size int64) error {
|
|
du := DiskUsage{ID: 1}
|
|
if err := tx.Get(&du); err != nil {
|
|
return fmt.Errorf("get diskusage: %v", err)
|
|
}
|
|
du.MessageSize += size
|
|
if du.MessageSize < 0 {
|
|
log.Error("negative total message size", slog.Int64("delta", size), slog.Int64("newtotalsize", du.MessageSize))
|
|
}
|
|
if err := tx.Update(&du); err != nil {
|
|
return fmt.Errorf("update total message size: %v", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// QuotaMessageSize returns the effective maximum total message size for an
|
|
// account. Returns 0 if there is no maximum.
|
|
func (a *Account) QuotaMessageSize() int64 {
|
|
conf, _ := a.Conf()
|
|
size := conf.QuotaMessageSize
|
|
if size == 0 {
|
|
size = mox.Conf.Static.QuotaMessageSize
|
|
}
|
|
if size < 0 {
|
|
size = 0
|
|
}
|
|
return size
|
|
}
|
|
|
|
// CanAddMessageSize checks if a message of size bytes can be added, depending on
|
|
// total message size and configured quota for account.
|
|
func (a *Account) CanAddMessageSize(tx *bstore.Tx, size int64) (ok bool, maxSize int64, err error) {
|
|
maxSize = a.QuotaMessageSize()
|
|
if maxSize <= 0 {
|
|
return true, 0, nil
|
|
}
|
|
|
|
du := DiskUsage{ID: 1}
|
|
if err := tx.Get(&du); err != nil {
|
|
return false, maxSize, fmt.Errorf("get diskusage: %v", err)
|
|
}
|
|
return du.MessageSize+size <= maxSize, maxSize, nil
|
|
}
|
|
|
|
// We keep a cache of recent successful authentications, so we don't have to bcrypt successful calls each time.
|
|
var authCache = struct {
|
|
sync.Mutex
|
|
success map[authKey]string
|
|
}{
|
|
success: map[authKey]string{},
|
|
}
|
|
|
|
type authKey struct {
|
|
email, hash string
|
|
}
|
|
|
|
// StartAuthCache starts a goroutine that regularly clears the auth cache.
|
|
func StartAuthCache() {
|
|
go manageAuthCache()
|
|
}
|
|
|
|
func manageAuthCache() {
|
|
for {
|
|
authCache.Lock()
|
|
authCache.success = map[authKey]string{}
|
|
authCache.Unlock()
|
|
time.Sleep(15 * time.Minute)
|
|
}
|
|
}
|
|
|
|
// OpenEmailAuth opens an account given an email address and password.
|
|
//
|
|
// The email address may contain a catchall separator.
|
|
func OpenEmailAuth(log mlog.Log, email string, password string) (acc *Account, rerr error) {
|
|
acc, _, rerr = OpenEmail(log, email)
|
|
if rerr != nil {
|
|
return
|
|
}
|
|
|
|
defer func() {
|
|
if rerr != nil && acc != nil {
|
|
err := acc.Close()
|
|
log.Check(err, "closing account after open auth failure")
|
|
acc = nil
|
|
}
|
|
}()
|
|
|
|
pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()
|
|
if err != nil {
|
|
if err == bstore.ErrAbsent {
|
|
return acc, ErrUnknownCredentials
|
|
}
|
|
return acc, fmt.Errorf("looking up password: %v", err)
|
|
}
|
|
authCache.Lock()
|
|
ok := len(password) >= 8 && authCache.success[authKey{email, pw.Hash}] == password
|
|
authCache.Unlock()
|
|
if ok {
|
|
return
|
|
}
|
|
if err := bcrypt.CompareHashAndPassword([]byte(pw.Hash), []byte(password)); err != nil {
|
|
rerr = ErrUnknownCredentials
|
|
} else {
|
|
authCache.Lock()
|
|
authCache.success[authKey{email, pw.Hash}] = password
|
|
authCache.Unlock()
|
|
}
|
|
return
|
|
}
|
|
|
|
// OpenEmail opens an account given an email address.
|
|
//
|
|
// The email address may contain a catchall separator.
|
|
func OpenEmail(log mlog.Log, email string) (*Account, config.Destination, error) {
|
|
addr, err := smtp.ParseAddress(email)
|
|
if err != nil {
|
|
return nil, config.Destination{}, fmt.Errorf("%w: %v", ErrUnknownCredentials, err)
|
|
}
|
|
accountName, _, dest, err := mox.FindAccount(addr.Localpart, addr.Domain, false)
|
|
if err != nil && (errors.Is(err, mox.ErrAccountNotFound) || errors.Is(err, mox.ErrDomainNotFound)) {
|
|
return nil, config.Destination{}, ErrUnknownCredentials
|
|
} else if err != nil {
|
|
return nil, config.Destination{}, fmt.Errorf("looking up address: %v", err)
|
|
}
|
|
acc, err := OpenAccount(log, accountName)
|
|
if err != nil {
|
|
return nil, config.Destination{}, err
|
|
}
|
|
return acc, dest, nil
|
|
}
|
|
|
|
// 64 characters, must be power of 2 for MessagePath
|
|
const msgDirChars = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-_"
|
|
|
|
// MessagePath returns the filename of the on-disk filename, relative to the
|
|
// containing directory such as <account>/msg or queue.
|
|
// Returns names like "AB/1".
|
|
func MessagePath(messageID int64) string {
|
|
return strings.Join(messagePathElems(messageID), string(filepath.Separator))
|
|
}
|
|
|
|
// messagePathElems returns the elems, for a single join without intermediate
|
|
// string allocations.
|
|
func messagePathElems(messageID int64) []string {
|
|
v := messageID >> 13 // 8k files per directory.
|
|
dir := ""
|
|
for {
|
|
dir += string(msgDirChars[int(v)&(len(msgDirChars)-1)])
|
|
v >>= 6
|
|
if v == 0 {
|
|
break
|
|
}
|
|
}
|
|
return []string{dir, strconv.FormatInt(messageID, 10)}
|
|
}
|
|
|
|
// Set returns a copy of f, with each flag that is true in mask set to the
|
|
// value from flags.
|
|
func (f Flags) Set(mask, flags Flags) Flags {
|
|
set := func(d *bool, m, v bool) {
|
|
if m {
|
|
*d = v
|
|
}
|
|
}
|
|
r := f
|
|
set(&r.Seen, mask.Seen, flags.Seen)
|
|
set(&r.Answered, mask.Answered, flags.Answered)
|
|
set(&r.Flagged, mask.Flagged, flags.Flagged)
|
|
set(&r.Forwarded, mask.Forwarded, flags.Forwarded)
|
|
set(&r.Junk, mask.Junk, flags.Junk)
|
|
set(&r.Notjunk, mask.Notjunk, flags.Notjunk)
|
|
set(&r.Deleted, mask.Deleted, flags.Deleted)
|
|
set(&r.Draft, mask.Draft, flags.Draft)
|
|
set(&r.Phishing, mask.Phishing, flags.Phishing)
|
|
set(&r.MDNSent, mask.MDNSent, flags.MDNSent)
|
|
return r
|
|
}
|
|
|
|
// Changed returns a mask of flags that have been between f and other.
|
|
func (f Flags) Changed(other Flags) (mask Flags) {
|
|
mask.Seen = f.Seen != other.Seen
|
|
mask.Answered = f.Answered != other.Answered
|
|
mask.Flagged = f.Flagged != other.Flagged
|
|
mask.Forwarded = f.Forwarded != other.Forwarded
|
|
mask.Junk = f.Junk != other.Junk
|
|
mask.Notjunk = f.Notjunk != other.Notjunk
|
|
mask.Deleted = f.Deleted != other.Deleted
|
|
mask.Draft = f.Draft != other.Draft
|
|
mask.Phishing = f.Phishing != other.Phishing
|
|
mask.MDNSent = f.MDNSent != other.MDNSent
|
|
return
|
|
}
|
|
|
|
var systemWellKnownFlags = map[string]bool{
|
|
`\answered`: true,
|
|
`\flagged`: true,
|
|
`\deleted`: true,
|
|
`\seen`: true,
|
|
`\draft`: true,
|
|
`$junk`: true,
|
|
`$notjunk`: true,
|
|
`$forwarded`: true,
|
|
`$phishing`: true,
|
|
`$mdnsent`: true,
|
|
}
|
|
|
|
// ParseFlagsKeywords parses a list of textual flags into system/known flags, and
|
|
// other keywords. Keywords are lower-cased and sorted and check for valid syntax.
|
|
func ParseFlagsKeywords(l []string) (flags Flags, keywords []string, rerr error) {
|
|
fields := map[string]*bool{
|
|
`\answered`: &flags.Answered,
|
|
`\flagged`: &flags.Flagged,
|
|
`\deleted`: &flags.Deleted,
|
|
`\seen`: &flags.Seen,
|
|
`\draft`: &flags.Draft,
|
|
`$junk`: &flags.Junk,
|
|
`$notjunk`: &flags.Notjunk,
|
|
`$forwarded`: &flags.Forwarded,
|
|
`$phishing`: &flags.Phishing,
|
|
`$mdnsent`: &flags.MDNSent,
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, f := range l {
|
|
f = strings.ToLower(f)
|
|
if field, ok := fields[f]; ok {
|
|
*field = true
|
|
} else if seen[f] {
|
|
if mox.Pedantic {
|
|
return Flags{}, nil, fmt.Errorf("duplicate keyword %s", f)
|
|
}
|
|
} else {
|
|
if err := CheckKeyword(f); err != nil {
|
|
return Flags{}, nil, fmt.Errorf("invalid keyword %s", f)
|
|
}
|
|
keywords = append(keywords, f)
|
|
seen[f] = true
|
|
}
|
|
}
|
|
sort.Strings(keywords)
|
|
return flags, keywords, nil
|
|
}
|
|
|
|
// RemoveKeywords removes keywords from l, returning whether any modifications were
|
|
// made, and a slice, a new slice in case of modifications. Keywords must have been
|
|
// validated earlier, e.g. through ParseFlagKeywords or CheckKeyword. Should only
|
|
// be used with valid keywords, not with system flags like \Seen.
|
|
func RemoveKeywords(l, remove []string) ([]string, bool) {
|
|
var copied bool
|
|
var changed bool
|
|
for _, k := range remove {
|
|
if i := slices.Index(l, k); i >= 0 {
|
|
if !copied {
|
|
l = append([]string{}, l...)
|
|
copied = true
|
|
}
|
|
copy(l[i:], l[i+1:])
|
|
l = l[:len(l)-1]
|
|
changed = true
|
|
}
|
|
}
|
|
return l, changed
|
|
}
|
|
|
|
// MergeKeywords adds keywords from add into l, returning whether it added any
|
|
// keyword, and the slice with keywords, a new slice if modifications were made.
|
|
// Keywords are only added if they aren't already present. Should only be used with
|
|
// keywords, not with system flags like \Seen.
|
|
func MergeKeywords(l, add []string) ([]string, bool) {
|
|
var copied bool
|
|
var changed bool
|
|
for _, k := range add {
|
|
if !slices.Contains(l, k) {
|
|
if !copied {
|
|
l = append([]string{}, l...)
|
|
copied = true
|
|
}
|
|
l = append(l, k)
|
|
changed = true
|
|
}
|
|
}
|
|
if changed {
|
|
sort.Strings(l)
|
|
}
|
|
return l, changed
|
|
}
|
|
|
|
// CheckKeyword returns an error if kw is not a valid keyword. Kw should
|
|
// already be in lower-case.
|
|
func CheckKeyword(kw string) error {
|
|
if kw == "" {
|
|
return fmt.Errorf("keyword cannot be empty")
|
|
}
|
|
if systemWellKnownFlags[kw] {
|
|
return fmt.Errorf("cannot use well-known flag as keyword")
|
|
}
|
|
for _, c := range kw {
|
|
// ../rfc/9051:6334
|
|
if c <= ' ' || c > 0x7e || c >= 'A' && c <= 'Z' || strings.ContainsRune(`(){%*"\]`, c) {
|
|
return errors.New(`not a valid keyword, must be lower-case ascii without spaces and without any of these characters: (){%*"\]`)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SendLimitReached checks whether sending a message to recipients would reach
|
|
// the limit of outgoing messages for the account. If so, the message should
|
|
// not be sent. If the returned numbers are >= 0, the limit was reached and the
|
|
// values are the configured limits.
|
|
//
|
|
// To limit damage to the internet and our reputation in case of account
|
|
// compromise, we limit the max number of messages sent in a 24 hour window, both
|
|
// total number of messages and number of first-time recipients.
|
|
func (a *Account) SendLimitReached(tx *bstore.Tx, recipients []smtp.Path) (msglimit, rcptlimit int, rerr error) {
|
|
conf, _ := a.Conf()
|
|
msgmax := conf.MaxOutgoingMessagesPerDay
|
|
if msgmax == 0 {
|
|
// For human senders, 1000 recipients in a day is quite a lot.
|
|
msgmax = 1000
|
|
}
|
|
rcptmax := conf.MaxFirstTimeRecipientsPerDay
|
|
if rcptmax == 0 {
|
|
// Human senders may address a new human-sized list of people once in a while. In
|
|
// case of a compromise, a spammer will probably try to send to many new addresses.
|
|
rcptmax = 200
|
|
}
|
|
|
|
rcpts := map[string]time.Time{}
|
|
n := 0
|
|
err := bstore.QueryTx[Outgoing](tx).FilterGreater("Submitted", time.Now().Add(-24*time.Hour)).ForEach(func(o Outgoing) error {
|
|
n++
|
|
if rcpts[o.Recipient].IsZero() || o.Submitted.Before(rcpts[o.Recipient]) {
|
|
rcpts[o.Recipient] = o.Submitted
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return -1, -1, fmt.Errorf("querying message recipients in past 24h: %w", err)
|
|
}
|
|
if n+len(recipients) > msgmax {
|
|
return msgmax, -1, nil
|
|
}
|
|
|
|
// Only check if max first-time recipients is reached if there are enough messages
|
|
// to trigger the limit.
|
|
if n+len(recipients) < rcptmax {
|
|
return -1, -1, nil
|
|
}
|
|
|
|
isFirstTime := func(rcpt string, before time.Time) (bool, error) {
|
|
exists, err := bstore.QueryTx[Outgoing](tx).FilterNonzero(Outgoing{Recipient: rcpt}).FilterLess("Submitted", before).Exists()
|
|
return !exists, err
|
|
}
|
|
|
|
firsttime := 0
|
|
now := time.Now()
|
|
for _, r := range recipients {
|
|
if first, err := isFirstTime(r.XString(true), now); err != nil {
|
|
return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)
|
|
} else if first {
|
|
firsttime++
|
|
}
|
|
}
|
|
for r, t := range rcpts {
|
|
if first, err := isFirstTime(r, t); err != nil {
|
|
return -1, -1, fmt.Errorf("checking whether recipient is first-time: %v", err)
|
|
} else if first {
|
|
firsttime++
|
|
}
|
|
}
|
|
if firsttime > rcptmax {
|
|
return -1, rcptmax, nil
|
|
}
|
|
return -1, -1, nil
|
|
}
|
|
|
|
// MailboxCreate creates a new mailbox, including any missing parent mailboxes,
|
|
// the total list of created mailboxes is returned in created. On success, if
|
|
// exists is false and rerr nil, the changes must be broadcasted by the caller.
|
|
//
|
|
// Name must be in normalized form.
|
|
func (a *Account) MailboxCreate(tx *bstore.Tx, name string) (changes []Change, created []string, exists bool, rerr error) {
|
|
elems := strings.Split(name, "/")
|
|
var p string
|
|
for i, elem := range elems {
|
|
if i > 0 {
|
|
p += "/"
|
|
}
|
|
p += elem
|
|
exists, err := a.MailboxExists(tx, p)
|
|
if err != nil {
|
|
return nil, nil, false, fmt.Errorf("checking if mailbox exists")
|
|
}
|
|
if exists {
|
|
if i == len(elems)-1 {
|
|
return nil, nil, true, fmt.Errorf("mailbox already exists")
|
|
}
|
|
continue
|
|
}
|
|
_, nchanges, err := a.MailboxEnsure(tx, p, true)
|
|
if err != nil {
|
|
return nil, nil, false, fmt.Errorf("ensuring mailbox exists")
|
|
}
|
|
changes = append(changes, nchanges...)
|
|
created = append(created, p)
|
|
}
|
|
return changes, created, false, nil
|
|
}
|
|
|
|
// MailboxRename renames mailbox mbsrc to dst, and any missing parents for the
|
|
// destination, and any children of mbsrc and the destination.
|
|
//
|
|
// Names must be normalized and cannot be Inbox.
|
|
func (a *Account) MailboxRename(tx *bstore.Tx, mbsrc Mailbox, dst string) (changes []Change, isInbox, notExists, alreadyExists bool, rerr error) {
|
|
if mbsrc.Name == "Inbox" || dst == "Inbox" {
|
|
return nil, true, false, false, fmt.Errorf("inbox cannot be renamed")
|
|
}
|
|
|
|
// We gather existing mailboxes that we need for deciding what to create/delete/update.
|
|
q := bstore.QueryTx[Mailbox](tx)
|
|
srcPrefix := mbsrc.Name + "/"
|
|
dstRoot := strings.SplitN(dst, "/", 2)[0]
|
|
dstRootPrefix := dstRoot + "/"
|
|
q.FilterFn(func(mb Mailbox) bool {
|
|
return mb.Name == mbsrc.Name || strings.HasPrefix(mb.Name, srcPrefix) || mb.Name == dstRoot || strings.HasPrefix(mb.Name, dstRootPrefix)
|
|
})
|
|
q.SortAsc("Name") // We'll rename the parents before children.
|
|
l, err := q.List()
|
|
if err != nil {
|
|
return nil, false, false, false, fmt.Errorf("listing relevant mailboxes: %v", err)
|
|
}
|
|
|
|
mailboxes := map[string]Mailbox{}
|
|
for _, mb := range l {
|
|
mailboxes[mb.Name] = mb
|
|
}
|
|
|
|
if _, ok := mailboxes[mbsrc.Name]; !ok {
|
|
return nil, false, true, false, fmt.Errorf("mailbox does not exist")
|
|
}
|
|
|
|
uidval, err := a.NextUIDValidity(tx)
|
|
if err != nil {
|
|
return nil, false, false, false, fmt.Errorf("next uid validity: %v", err)
|
|
}
|
|
|
|
// Ensure parent mailboxes for the destination paths exist.
|
|
var parent string
|
|
dstElems := strings.Split(dst, "/")
|
|
for i, elem := range dstElems[:len(dstElems)-1] {
|
|
if i > 0 {
|
|
parent += "/"
|
|
}
|
|
parent += elem
|
|
|
|
mb, ok := mailboxes[parent]
|
|
if ok {
|
|
continue
|
|
}
|
|
omb := mb
|
|
mb = Mailbox{
|
|
ID: omb.ID,
|
|
Name: parent,
|
|
UIDValidity: uidval,
|
|
UIDNext: 1,
|
|
HaveCounts: true,
|
|
}
|
|
if err := tx.Insert(&mb); err != nil {
|
|
return nil, false, false, false, fmt.Errorf("creating parent mailbox %q: %v", mb.Name, err)
|
|
}
|
|
if err := tx.Get(&Subscription{Name: parent}); err != nil {
|
|
if err := tx.Insert(&Subscription{Name: parent}); err != nil {
|
|
return nil, false, false, false, fmt.Errorf("creating subscription for %q: %v", parent, err)
|
|
}
|
|
}
|
|
changes = append(changes, ChangeAddMailbox{Mailbox: mb, Flags: []string{`\Subscribed`}})
|
|
}
|
|
|
|
// Process src mailboxes, renaming them to dst.
|
|
for _, srcmb := range l {
|
|
if srcmb.Name != mbsrc.Name && !strings.HasPrefix(srcmb.Name, srcPrefix) {
|
|
continue
|
|
}
|
|
srcName := srcmb.Name
|
|
dstName := dst + srcmb.Name[len(mbsrc.Name):]
|
|
if _, ok := mailboxes[dstName]; ok {
|
|
return nil, false, false, true, fmt.Errorf("destination mailbox %q already exists", dstName)
|
|
}
|
|
|
|
srcmb.Name = dstName
|
|
srcmb.UIDValidity = uidval
|
|
if err := tx.Update(&srcmb); err != nil {
|
|
return nil, false, false, false, fmt.Errorf("renaming mailbox: %v", err)
|
|
}
|
|
|
|
var dstFlags []string
|
|
if tx.Get(&Subscription{Name: dstName}) == nil {
|
|
dstFlags = []string{`\Subscribed`}
|
|
}
|
|
changes = append(changes, ChangeRenameMailbox{MailboxID: srcmb.ID, OldName: srcName, NewName: dstName, Flags: dstFlags})
|
|
}
|
|
|
|
// If we renamed e.g. a/b to a/b/c/d, and a/b/c to a/b/c/d/c, we'll have to recreate a/b and a/b/c.
|
|
srcElems := strings.Split(mbsrc.Name, "/")
|
|
xsrc := mbsrc.Name
|
|
for i := 0; i < len(dstElems) && strings.HasPrefix(dst, xsrc+"/"); i++ {
|
|
mb := Mailbox{
|
|
UIDValidity: uidval,
|
|
UIDNext: 1,
|
|
Name: xsrc,
|
|
HaveCounts: true,
|
|
}
|
|
if err := tx.Insert(&mb); err != nil {
|
|
return nil, false, false, false, fmt.Errorf("creating mailbox at old path %q: %v", mb.Name, err)
|
|
}
|
|
xsrc += "/" + dstElems[len(srcElems)+i]
|
|
}
|
|
return changes, false, false, false, nil
|
|
}
|
|
|
|
// MailboxDelete deletes a mailbox by ID. If it has children, the return value
|
|
// indicates that and an error is returned.
|
|
//
|
|
// Caller should broadcast the changes and remove files for the removed message IDs.
|
|
func (a *Account) MailboxDelete(ctx context.Context, log mlog.Log, tx *bstore.Tx, mailbox Mailbox) (changes []Change, removeMessageIDs []int64, hasChildren bool, rerr error) {
|
|
// Look for existence of child mailboxes. There is a lot of text in the IMAP RFCs about
|
|
// NoInferior and NoSelect. We just require only leaf mailboxes are deleted.
|
|
qmb := bstore.QueryTx[Mailbox](tx)
|
|
mbprefix := mailbox.Name + "/"
|
|
qmb.FilterFn(func(mb Mailbox) bool {
|
|
return strings.HasPrefix(mb.Name, mbprefix)
|
|
})
|
|
if childExists, err := qmb.Exists(); err != nil {
|
|
return nil, nil, false, fmt.Errorf("checking if mailbox has child: %v", err)
|
|
} else if childExists {
|
|
return nil, nil, true, fmt.Errorf("mailbox has a child, only leaf mailboxes can be deleted")
|
|
}
|
|
|
|
// todo jmap: instead of completely deleting a mailbox and its messages, we need to mark them all as expunged.
|
|
|
|
qm := bstore.QueryTx[Message](tx)
|
|
qm.FilterNonzero(Message{MailboxID: mailbox.ID})
|
|
remove, err := qm.List()
|
|
if err != nil {
|
|
return nil, nil, false, fmt.Errorf("listing messages to remove: %v", err)
|
|
}
|
|
|
|
if len(remove) > 0 {
|
|
removeIDs := make([]any, len(remove))
|
|
for i, m := range remove {
|
|
removeIDs[i] = m.ID
|
|
}
|
|
qmr := bstore.QueryTx[Recipient](tx)
|
|
qmr.FilterEqual("MessageID", removeIDs...)
|
|
if _, err = qmr.Delete(); err != nil {
|
|
return nil, nil, false, fmt.Errorf("removing message recipients for messages: %v", err)
|
|
}
|
|
|
|
qm = bstore.QueryTx[Message](tx)
|
|
qm.FilterNonzero(Message{MailboxID: mailbox.ID})
|
|
if _, err := qm.Delete(); err != nil {
|
|
return nil, nil, false, fmt.Errorf("removing messages: %v", err)
|
|
}
|
|
|
|
var totalSize int64
|
|
for _, m := range remove {
|
|
if !m.Expunged {
|
|
removeMessageIDs = append(removeMessageIDs, m.ID)
|
|
totalSize += m.Size
|
|
}
|
|
}
|
|
if err := a.AddMessageSize(log, tx, -totalSize); err != nil {
|
|
return nil, nil, false, fmt.Errorf("updating disk usage: %v", err)
|
|
}
|
|
|
|
// Mark messages as not needing training. Then retrain them, so they are untrained if they were.
|
|
n := 0
|
|
o := 0
|
|
for _, m := range remove {
|
|
if !m.Expunged {
|
|
remove[o] = m
|
|
remove[o].Junk = false
|
|
remove[o].Notjunk = false
|
|
n++
|
|
}
|
|
}
|
|
remove = remove[:n]
|
|
if err := a.RetrainMessages(ctx, log, tx, remove, true); err != nil {
|
|
return nil, nil, false, fmt.Errorf("untraining deleted messages: %v", err)
|
|
}
|
|
}
|
|
|
|
if err := tx.Delete(&Mailbox{ID: mailbox.ID}); err != nil {
|
|
return nil, nil, false, fmt.Errorf("removing mailbox: %v", err)
|
|
}
|
|
return []Change{ChangeRemoveMailbox{MailboxID: mailbox.ID, Name: mailbox.Name}}, removeMessageIDs, false, nil
|
|
}
|
|
|
|
// CheckMailboxName checks if name is valid, returning an INBOX-normalized name.
|
|
// I.e. it changes various casings of INBOX and INBOX/* to Inbox and Inbox/*.
|
|
// Name is invalid if it contains leading/trailing/double slashes, or when it isn't
|
|
// unicode-normalized, or when empty or has special characters.
|
|
//
|
|
// If name is the inbox, and allowInbox is false, this is indicated with the isInbox return parameter.
|
|
// For that case, and for other invalid names, an error is returned.
|
|
func CheckMailboxName(name string, allowInbox bool) (normalizedName string, isInbox bool, rerr error) {
|
|
first := strings.SplitN(name, "/", 2)[0]
|
|
if strings.EqualFold(first, "inbox") {
|
|
if len(name) == len("inbox") && !allowInbox {
|
|
return "", true, fmt.Errorf("special mailbox name Inbox not allowed")
|
|
}
|
|
name = "Inbox" + name[len("Inbox"):]
|
|
}
|
|
|
|
if norm.NFC.String(name) != name {
|
|
return "", false, errors.New("non-unicode-normalized mailbox names not allowed")
|
|
}
|
|
|
|
if name == "" {
|
|
return "", false, errors.New("empty mailbox name")
|
|
}
|
|
if strings.HasPrefix(name, "/") || strings.HasSuffix(name, "/") || strings.Contains(name, "//") {
|
|
return "", false, errors.New("bad slashes in mailbox name")
|
|
}
|
|
|
|
// "%" and "*" are difficult to use with the IMAP LIST command, but we allow mostly
|
|
// allow them. ../rfc/3501:1002 ../rfc/9051:983
|
|
if strings.HasPrefix(name, "#") {
|
|
return "", false, errors.New("mailbox name cannot start with hash due to conflict with imap namespaces")
|
|
}
|
|
|
|
// "#" and "&" are special in IMAP mailbox names. "#" for namespaces, "&" for
|
|
// IMAP-UTF-7 encoding. We do allow them. ../rfc/3501:1018 ../rfc/9051:991
|
|
|
|
for _, c := range name {
|
|
// ../rfc/3501:999 ../rfc/6855:192 ../rfc/9051:979
|
|
if c <= 0x1f || c >= 0x7f && c <= 0x9f || c == 0x2028 || c == 0x2029 {
|
|
return "", false, errors.New("control characters not allowed in mailbox name")
|
|
}
|
|
}
|
|
return name, false, nil
|
|
}
|