update to latest bstore (with support for an index on a []string: Message.DKIMDomains), and cyclic data types (to be used for Message.Part soon); also adds a context.Context to database operations.

ctl.go

@@ -386,7 +386,7 @@ func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, xcmd *string, shu
 		< "ok"
 		< stream
 		*/
-		qmsgs, err := queue.List()
+		qmsgs, err := queue.List(ctx)
 		ctl.xcheck(err, "listing queue")
 		ctl.xwriteok()
 
@@ -425,10 +425,10 @@ func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, xcmd *string, shu
 
 		var count int
 		if cmd == "queuekick" {
-			count, err = queue.Kick(id, todomain, recipient)
+			count, err = queue.Kick(ctx, id, todomain, recipient)
 			ctl.xcheck(err, "kicking queue")
 		} else {
-			count, err = queue.Drop(id, todomain, recipient)
+			count, err = queue.Drop(ctx, id, todomain, recipient)
 			ctl.xcheck(err, "dropping messages from queue")
 		}
 		ctl.xwrite(fmt.Sprintf("%d", count))
@@ -447,7 +447,7 @@ func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, xcmd *string, shu
 		if err != nil {
 			ctl.xcheck(err, "parsing id")
 		}
-		mr, err := queue.OpenMessage(id)
+		mr, err := queue.OpenMessage(ctx, id)
 		ctl.xcheck(err, "opening message")
 		defer func() {
 			err := mr.Close()
@@ -458,7 +458,7 @@ func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, xcmd *string, shu
 
 	case "importmaildir", "importmbox":
 		mbox := cmd == "importmbox"
-		importctl(ctl, mbox)
+		importctl(ctx, ctl, mbox)
 
 	case "domainadd":
 		/* protocol:
@@ -609,7 +609,7 @@ func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, xcmd *string, shu
 			log.Check(err, "removing old junkfilter bloom filter file", mlog.Field("path", bloomPath))
 
 			// Open junk filter, this creates new files.
-			jf, _, err := acc.OpenJunkFilter(ctl.log)
+			jf, _, err := acc.OpenJunkFilter(ctx, ctl.log)
 			ctl.xcheck(err, "open new junk filter")
 			defer func() {
 				if jf == nil {
@@ -621,10 +621,10 @@ func servectlcmd(ctx context.Context, log *mlog.Log, ctl *ctl, xcmd *string, shu
 
 			// Read through messages with junk or nonjunk flag set, and train them.
 			var total, trained int
-			q := bstore.QueryDB[store.Message](acc.DB)
+			q := bstore.QueryDB[store.Message](ctx, acc.DB)
 			err = q.ForEach(func(m store.Message) error {
 				total++
-				ok, err := acc.TrainMessage(ctl.log, jf, m)
+				ok, err := acc.TrainMessage(ctx, ctl.log, jf, m)
 				if ok {
 					trained++
 				}

dmarcdb/db.go

@@ -67,13 +67,13 @@ type DomainFeedback struct {
 	dmarcrpt.Feedback
 }
 
-func database() (rdb *bstore.DB, rerr error) {
+func database(ctx context.Context) (rdb *bstore.DB, rerr error) {
 	mutex.Lock()
 	defer mutex.Unlock()
 	if dmarcDB == nil {
 		p := mox.DataDirPath("dmarcrpt.db")
 		os.MkdirAll(filepath.Dir(p), 0770)
-		db, err := bstore.Open(p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DomainFeedback{})
+		db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, DomainFeedback{})
 		if err != nil {
 			return nil, err
 		}
@@ -84,7 +84,7 @@ func database() (rdb *bstore.DB, rerr error) {
 
 // Init opens the database.
 func Init() error {
-	_, err := database()
+	_, err := database(mox.Shutdown)
 	return err
 }
 
@@ -93,7 +93,7 @@ func Init() error {
 //
 // fromDomain is the domain in the report message From header.
 func AddReport(ctx context.Context, f *dmarcrpt.Feedback, fromDomain dns.Domain) error {
-	db, err := database()
+	db, err := database(ctx)
 	if err != nil {
 		return err
 	}
@@ -104,7 +104,7 @@ func AddReport(ctx context.Context, f *dmarcrpt.Feedback, fromDomain dns.Domain)
 	}
 
 	df := DomainFeedback{0, d.Name(), fromDomain.Name(), *f}
-	if err := db.Insert(&df); err != nil {
+	if err := db.Insert(ctx, &df); err != nil {
 		return err
 	}
 
@@ -143,30 +143,30 @@ func AddReport(ctx context.Context, f *dmarcrpt.Feedback, fromDomain dns.Domain)
 
 // Records returns all reports in the database.
 func Records(ctx context.Context) ([]DomainFeedback, error) {
-	db, err := database()
+	db, err := database(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	return bstore.QueryDB[DomainFeedback](db).List()
+	return bstore.QueryDB[DomainFeedback](ctx, db).List()
 }
 
 // RecordID returns the report for the ID.
 func RecordID(ctx context.Context, id int64) (DomainFeedback, error) {
-	db, err := database()
+	db, err := database(ctx)
 	if err != nil {
 		return DomainFeedback{}, err
 	}
 
 	e := DomainFeedback{ID: id}
-	err = db.Get(&e)
+	err = db.Get(ctx, &e)
 	return e, err
 }
 
 // RecordsPeriodDomain returns the reports overlapping start and end, for the given
 // domain. If domain is empty, all records match for domain.
 func RecordsPeriodDomain(ctx context.Context, start, end time.Time, domain string) ([]DomainFeedback, error) {
-	db, err := database()
+	db, err := database(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -174,7 +174,7 @@ func RecordsPeriodDomain(ctx context.Context, start, end time.Time, domain strin
 	s := start.Unix()
 	e := end.Unix()
 
-	q := bstore.QueryDB[DomainFeedback](db)
+	q := bstore.QueryDB[DomainFeedback](ctx, db)
 	if domain != "" {
 		q.FilterNonzero(DomainFeedback{Domain: domain})
 	}

dmarcdb/db_test.go

@@ -13,7 +13,10 @@ import (
 	"github.com/mjl-/mox/mox-"
 )
 
+var ctxbg = context.Background()
+
 func TestDMARCDB(t *testing.T) {
+	mox.Shutdown = ctxbg
 	mox.ConfigStaticPath = "../testdata/dmarcdb/fake.conf"
 	mox.Conf.Static.DataDir = "."
 
@@ -76,32 +79,32 @@ func TestDMARCDB(t *testing.T) {
 			},
 		},
 	}
-	if err := AddReport(context.Background(), feedback, dns.Domain{ASCII: "google.com"}); err != nil {
+	if err := AddReport(ctxbg, feedback, dns.Domain{ASCII: "google.com"}); err != nil {
 		t.Fatalf("adding report: %s", err)
 	}
 
-	records, err := Records(context.Background())
+	records, err := Records(ctxbg)
 	if err != nil || len(records) != 1 || !reflect.DeepEqual(&records[0].Feedback, feedback) {
 		t.Fatalf("records: got err %v, records %#v, expected no error, single record with feedback %#v", err, records, feedback)
 	}
 
-	record, err := RecordID(context.Background(), records[0].ID)
+	record, err := RecordID(ctxbg, records[0].ID)
 	if err != nil || !reflect.DeepEqual(&record.Feedback, feedback) {
 		t.Fatalf("record id: got err %v, record %#v, expected feedback %#v", err, record, feedback)
 	}
 
 	start := time.Unix(1596412800, 0)
 	end := time.Unix(1596499199, 0)
-	records, err = RecordsPeriodDomain(context.Background(), start, end, "example.org")
+	records, err = RecordsPeriodDomain(ctxbg, start, end, "example.org")
 	if err != nil || len(records) != 1 || !reflect.DeepEqual(&records[0].Feedback, feedback) {
 		t.Fatalf("records: got err %v, records %#v, expected no error, single record with feedback %#v", err, records, feedback)
 	}
 
-	records, err = RecordsPeriodDomain(context.Background(), end, end, "example.org")
+	records, err = RecordsPeriodDomain(ctxbg, end, end, "example.org")
 	if err != nil || len(records) != 0 {
 		t.Fatalf("records: got err %v, records %#v, expected no error and no records", err, records)
 	}
-	records, err = RecordsPeriodDomain(context.Background(), start, end, "other.example")
+	records, err = RecordsPeriodDomain(ctxbg, start, end, "other.example")
 	if err != nil || len(records) != 0 {
 		t.Fatalf("records: got err %v, records %#v, expected no error and no records", err, records)
 	}

export.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"log"
 	"path/filepath"
 	"time"
@@ -56,7 +57,7 @@ func xcmdExport(mbox bool, args []string, c *cmd) {
 	}
 
 	dbpath := filepath.Join(accountDir, "index.db")
-	db, err := bstore.Open(dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, store.Message{}, store.Recipient{}, store.Mailbox{})
+	db, err := bstore.Open(context.Background(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, store.Message{}, store.Recipient{}, store.Mailbox{})
 	xcheckf(err, "open database %q", dbpath)
 	defer func() {
 		if err := db.Close(); err != nil {
@@ -65,7 +66,7 @@ func xcmdExport(mbox bool, args []string, c *cmd) {
 	}()
 
 	a := store.DirArchiver{Dir: dst}
-	err = store.ExportMessages(mlog.New("export"), db, accountDir, a, !mbox, mailbox)
+	err = store.ExportMessages(context.Background(), mlog.New("export"), db, accountDir, a, !mbox, mailbox)
 	xcheckf(err, "exporting messages")
 	err = a.Close()
 	xcheckf(err, "closing archiver")

go.mod

@@ -3,7 +3,7 @@ module github.com/mjl-/mox
 go 1.18
 
 require (
-	github.com/mjl-/bstore v0.0.0-20230211204415-a9899ef6e782
+	github.com/mjl-/bstore v0.0.1
 	github.com/mjl-/sconf v0.0.4
 	github.com/mjl-/sherpa v0.6.5
 	github.com/mjl-/sherpadoc v0.0.10

go.sum

@@ -145,8 +145,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/mjl-/bstore v0.0.0-20230211204415-a9899ef6e782 h1:dVwJA/wXzXXUROM9oM3Stg3cmqixiFh4Zi1Xumvtj74=
-github.com/mjl-/bstore v0.0.0-20230211204415-a9899ef6e782/go.mod h1:/cD25FNBaDfvL/plFRxI3Ba3E+wcB0XVOS8nJDqndg0=
+github.com/mjl-/bstore v0.0.1 h1:OzQfYgpMCvNjNIj9FFJ3HidYzG6eSlLSYzCTzw9sptY=
+github.com/mjl-/bstore v0.0.1/go.mod h1:/cD25FNBaDfvL/plFRxI3Ba3E+wcB0XVOS8nJDqndg0=
 github.com/mjl-/sconf v0.0.4 h1:uyfn4vv5qOULSgiwQsPbbgkiONKnMFMsSOhsHfAiYwI=
 github.com/mjl-/sconf v0.0.4/go.mod h1:ezf7YOn7gtClo8y71SqgZKaEkyMQ5Te7vkv4PmTTfwM=
 github.com/mjl-/sherpa v0.6.5 h1:d90uG/j8fw+2M+ohCTAcVwTSUURGm8ktYDScJO1nKog=

http/account.go

@@ -222,7 +222,7 @@ func accountHandle(w http.ResponseWriter, r *http.Request) {
 			err := archiver.Close()
 			log.Check(err, "exporting mail close")
 		}()
-		if err := store.ExportMessages(log, acc.DB, acc.Dir, archiver, maildir, ""); err != nil {
+		if err := store.ExportMessages(r.Context(), log, acc.DB, acc.Dir, archiver, maildir, ""); err != nil {
 			log.Errorx("exporting mail", err)
 		}
 

http/admin.go

@@ -1500,21 +1500,21 @@ func (Admin) ClientConfigDomain(ctx context.Context, domain string) mox.ClientCo
 
 // QueueList returns the messages currently in the outgoing queue.
 func (Admin) QueueList(ctx context.Context) []queue.Msg {
-	l, err := queue.List()
+	l, err := queue.List(ctx)
 	xcheckf(ctx, err, "listing messages in queue")
 	return l
 }
 
 // QueueSize returns the number of messages currently in the outgoing queue.
 func (Admin) QueueSize(ctx context.Context) int {
-	n, err := queue.Count()
+	n, err := queue.Count(ctx)
 	xcheckf(ctx, err, "listing messages in queue")
 	return n
 }
 
 // QueueKick initiates delivery of a message from the queue.
 func (Admin) QueueKick(ctx context.Context, id int64) {
-	n, err := queue.Kick(id, "", "")
+	n, err := queue.Kick(ctx, id, "", "")
 	if err == nil && n == 0 {
 		err = errors.New("message not found")
 	}
@@ -1523,7 +1523,7 @@ func (Admin) QueueKick(ctx context.Context, id int64) {
 
 // QueueDrop removes a message from the queue.
 func (Admin) QueueDrop(ctx context.Context, id int64) {
-	n, err := queue.Drop(id, "", "")
+	n, err := queue.Drop(ctx, id, "", "")
 	if err == nil && n == 0 {
 		err = errors.New("message not found")
 	}

http/import.go

@@ -249,7 +249,7 @@ func importStart(log *mlog.Log, accName string, f *os.File, skipMailboxPrefix st
 	}
 	acc.Lock() // Not using WithWLock because importMessage is responsible for unlocking.
 
-	tx, err := acc.DB.Begin(true)
+	tx, err := acc.DB.Begin(context.Background(), true)
 	if err != nil {
 		acc.Unlock()
 		xerr := acc.Close()
@@ -346,7 +346,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 
 	conf, _ := acc.Conf()
 
-	jf, _, err := acc.OpenJunkFilter(log)
+	jf, _, err := acc.OpenJunkFilter(ctx, log)
 	if err != nil && !errors.Is(err, store.ErrNoJunkFilter) {
 		ximportcheckf(err, "open junk filter")
 	}
@@ -376,7 +376,7 @@ func importMessages(ctx context.Context, log *mlog.Log, token string, acc *store
 			problemf("parsing message %s for updating junk filter: %v (continuing)", pos, err)
 			return
 		}
-		err = jf.Train(!m.Junk, words)
+		err = jf.Train(ctx, !m.Junk, words)
 		if err != nil {
 			problemf("training junk filter for message %s: %v (continuing)", pos, err)
 			return

imapserver/server.go

@@ -371,7 +371,7 @@ func (c *conn) utf8strings() bool {
 }
 
 func (c *conn) xdbwrite(fn func(tx *bstore.Tx)) {
-	err := c.account.DB.Write(func(tx *bstore.Tx) error {
+	err := c.account.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
 		fn(tx)
 		return nil
 	})
@@ -379,7 +379,7 @@ func (c *conn) xdbwrite(fn func(tx *bstore.Tx)) {
 }
 
 func (c *conn) xdbread(fn func(tx *bstore.Tx)) {
-	err := c.account.DB.Read(func(tx *bstore.Tx) error {
+	err := c.account.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
 		fn(tx)
 		return nil
 	})
@@ -1574,7 +1574,7 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
 		}()
 		var ipadhash, opadhash hash.Hash
 		acc.WithRLock(func() {
-			err := acc.DB.Read(func(tx *bstore.Tx) error {
+			err := acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
 				password, err := bstore.QueryTx[store.Password](tx).Get()
 				if err == bstore.ErrAbsent {
 					xusercodeErrorf("AUTHENTICATIONFAILED", "bad credentials")
@@ -1644,7 +1644,7 @@ func (c *conn) cmdAuthenticate(tag, cmd string, p *parser) {
 		}
 		var xscram store.SCRAM
 		acc.WithRLock(func() {
-			err := acc.DB.Read(func(tx *bstore.Tx) error {
+			err := acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
 				password, err := bstore.QueryTx[store.Password](tx).Get()
 				if authVariant == "scram-sha-1" {
 					xscram = password.SCRAMSHA1
@@ -1998,7 +1998,7 @@ func (c *conn) cmdDelete(tag, cmd string, p *parser) {
 					remove[i].Junk = false
 					remove[i].Notjunk = false
 				}
-				err = c.account.RetrainMessages(c.log, tx, remove, true)
+				err = c.account.RetrainMessages(context.TODO(), c.log, tx, remove, true)
 				xcheckf(err, "untraining deleted messages")
 			}
 
@@ -2743,7 +2743,7 @@ func (c *conn) xexpunge(uidSet *numSet, missingMailboxOK bool) []store.Message {
 				remove[i].Junk = false
 				remove[i].Notjunk = false
 			}
-			err = c.account.RetrainMessages(c.log, tx, remove, true)
+			err = c.account.RetrainMessages(context.TODO(), c.log, tx, remove, true)
 			xcheckf(err, "untraining deleted messages")
 		})
 
@@ -3030,7 +3030,7 @@ func (c *conn) cmdxCopy(isUID bool, tag, cmd string, p *parser) {
 				createdIDs = append(createdIDs, newMsgIDs[i])
 			}
 
-			err = c.account.RetrainMessages(c.log, tx, nmsgs, false)
+			err = c.account.RetrainMessages(context.TODO(), c.log, tx, nmsgs, false)
 			xcheckf(err, "train copied messages")
 		})
 
@@ -3169,7 +3169,7 @@ func (c *conn) cmdxMove(isUID bool, tag, cmd string, p *parser) {
 				xcheckf(err, "updating moved message in database")
 			}
 
-			err = c.account.RetrainMessages(c.log, tx, msgs, false)
+			err = c.account.RetrainMessages(context.TODO(), c.log, tx, msgs, false)
 			xcheckf(err, "retraining messages after move")
 
 			// Prepare broadcast changes to other connections.
@@ -3269,7 +3269,7 @@ func (c *conn) cmdxStore(isUID bool, tag, cmd string, p *parser) {
 				xcheckf(err, "updating flags")
 			}
 
-			err := c.account.RetrainMessages(c.log, tx, updated, false)
+			err := c.account.RetrainMessages(context.TODO(), c.log, tx, updated, false)
 			xcheckf(err, "training messages")
 		})
 

import.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -107,7 +108,7 @@ func xcmdImport(mbox bool, args []string, c *cmd) {
 	fmt.Fprintf(os.Stderr, "%s imported\n", count)
 }
 
-func importctl(ctl *ctl, mbox bool) {
+func importctl(ctx context.Context, ctl *ctl, mbox bool) {
 	/* protocol:
 	> "importmaildir" or "importmbox"
 	> account
@@ -177,7 +178,7 @@ func importctl(ctl *ctl, mbox bool) {
 		msgreader = store.NewMaildirReader(store.CreateMessageTemp, mdnewf, mdcurf, ctl.log)
 	}
 
-	tx, err := a.DB.Begin(true)
+	tx, err := a.DB.Begin(ctx, true)
 	ctl.xcheck(err, "begin transaction")
 	defer func() {
 		if tx != nil {
@@ -239,7 +240,7 @@ func importctl(ctl *ctl, mbox bool) {
 		mb, changes, err = a.MailboxEnsure(tx, mailbox, true)
 		ctl.xcheck(err, "ensuring mailbox exists")
 
-		jf, _, err := a.OpenJunkFilter(ctl.log)
+		jf, _, err := a.OpenJunkFilter(ctx, ctl.log)
 		if err != nil && !errors.Is(err, store.ErrNoJunkFilter) {
 			ctl.xcheck(err, "open junk filter")
 		}
@@ -287,7 +288,7 @@ func importctl(ctl *ctl, mbox bool) {
 				if words, err := jf.ParseMessage(p); err != nil {
 					ctl.log.Infox("parsing message for updating junk filter", err, mlog.Field("parse", ""), mlog.Field("path", origPath))
 				} else {
-					err = jf.Train(!m.Junk, words)
+					err = jf.Train(ctx, !m.Junk, words)
 					ctl.xcheck(err, "training junk filter")
 					m.TrainedJunk = &m.Junk
 				}

integration_test.go

@@ -28,6 +28,8 @@ import (
 	"github.com/mjl-/mox/store"
 )
 
+var ctxbg = context.Background()
+
 func tcheck(t *testing.T, err error, msg string) {
 	t.Helper()
 	if err != nil {
@@ -51,7 +53,7 @@ func TestDeliver(t *testing.T) {
 	// Load mox config.
 	mox.ConfigStaticPath = "testdata/integration/config/mox.conf"
 	filepath.Join(filepath.Dir(mox.ConfigStaticPath), "domains.conf")
-	if errs := mox.LoadConfig(context.Background(), false); len(errs) > 0 {
+	if errs := mox.LoadConfig(ctxbg, false); len(errs) > 0 {
 		t.Fatalf("loading mox config: %v", errs)
 	}
 
@@ -80,7 +82,7 @@ func TestDeliver(t *testing.T) {
 	latestMsgID := func(username string) int64 {
 		// We open the account index database created by mox for the test user. And we keep looking for the email we sent.
 		dbpath := fmt.Sprintf("testdata/integration/data/accounts/%s/index.db", username)
-		db, err := bstore.Open(dbpath, &bstore.Options{Timeout: 3 * time.Second}, store.Message{}, store.Recipient{}, store.Mailbox{}, store.Password{})
+		db, err := bstore.Open(ctxbg, dbpath, &bstore.Options{Timeout: 3 * time.Second}, store.Message{}, store.Recipient{}, store.Mailbox{}, store.Password{})
 		if err != nil && errors.Is(err, bolt.ErrTimeout) {
 			log.Printf("db open timeout (normal delay for new sender with account and db file kept open)")
 			return 0
@@ -88,7 +90,7 @@ func TestDeliver(t *testing.T) {
 		tcheck(t, err, "open test account database")
 		defer db.Close()
 
-		q := bstore.QueryDB[store.Mailbox](db)
+		q := bstore.QueryDB[store.Mailbox](ctxbg, db)
 		q.FilterNonzero(store.Mailbox{Name: "Inbox"})
 		inbox, err := q.Get()
 		if err != nil {
@@ -96,7 +98,7 @@ func TestDeliver(t *testing.T) {
 			return 0
 		}
 
-		qm := bstore.QueryDB[store.Message](db)
+		qm := bstore.QueryDB[store.Message](ctxbg, db)
 		qm.FilterNonzero(store.Message{MailboxID: inbox.ID})
 		qm.SortDesc("ID")
 		qm.Limit(1)

junk.go

@@ -14,6 +14,7 @@ own ham/spam emails.
 */
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"log"
@@ -134,7 +135,7 @@ func cmdJunkTrain(c *cmd) {
 	defer a.Profile()()
 	a.SetLogLevel()
 
-	f := must(junk.NewFilter(mlog.New("junktrain"), a.params, a.databasePath, a.bloomfilterPath))
+	f := must(junk.NewFilter(context.Background(), mlog.New("junktrain"), a.params, a.databasePath, a.bloomfilterPath))
 	defer func() {
 		if err := f.Close(); err != nil {
 			log.Printf("closing junk filter: %v", err)
@@ -164,14 +165,14 @@ func cmdJunkCheck(c *cmd) {
 	defer a.Profile()()
 	a.SetLogLevel()
 
-	f := must(junk.OpenFilter(mlog.New("junkcheck"), a.params, a.databasePath, a.bloomfilterPath, false))
+	f := must(junk.OpenFilter(context.Background(), mlog.New("junkcheck"), a.params, a.databasePath, a.bloomfilterPath, false))
 	defer func() {
 		if err := f.Close(); err != nil {
 			log.Printf("closing junk filter: %v", err)
 		}
 	}()
 
-	prob, _, _, _, err := f.ClassifyMessagePath(args[0])
+	prob, _, _, _, err := f.ClassifyMessagePath(context.Background(), args[0])
 	xcheckf(err, "testing mail")
 
 	fmt.Printf("%.6f\n", prob)
@@ -189,7 +190,7 @@ func cmdJunkTest(c *cmd) {
 	defer a.Profile()()
 	a.SetLogLevel()
 
-	f := must(junk.OpenFilter(mlog.New("junktest"), a.params, a.databasePath, a.bloomfilterPath, false))
+	f := must(junk.OpenFilter(context.Background(), mlog.New("junktest"), a.params, a.databasePath, a.bloomfilterPath, false))
 	defer func() {
 		if err := f.Close(); err != nil {
 			log.Printf("closing junk filter: %v", err)
@@ -202,7 +203,7 @@ func cmdJunkTest(c *cmd) {
 		xcheckf(err, "readdir %q", dir)
 		for _, fi := range files {
 			path := dir + "/" + fi.Name()
-			prob, _, _, _, err := f.ClassifyMessagePath(path)
+			prob, _, _, _, err := f.ClassifyMessagePath(context.Background(), path)
 			if err != nil {
 				log.Printf("classify message %q: %s", path, err)
 				continue
@@ -246,7 +247,7 @@ messages are shuffled, with optional random seed.`
 	defer a.Profile()()
 	a.SetLogLevel()
 
-	f := must(junk.NewFilter(mlog.New("junkanalyze"), a.params, a.databasePath, a.bloomfilterPath))
+	f := must(junk.NewFilter(context.Background(), mlog.New("junkanalyze"), a.params, a.databasePath, a.bloomfilterPath))
 	defer func() {
 		if err := f.Close(); err != nil {
 			log.Printf("closing junk filter: %v", err)
@@ -295,7 +296,7 @@ messages are shuffled, with optional random seed.`
 	testDir := func(dir string, files []string, ham bool) (ok, bad, malformed int) {
 		for _, name := range files {
 			path := dir + "/" + name
-			prob, _, _, _, err := f.ClassifyMessagePath(path)
+			prob, _, _, _, err := f.ClassifyMessagePath(context.Background(), path)
 			if err != nil {
 				// log.Infof("%s: %s", path, err)
 				malformed++
@@ -338,7 +339,7 @@ func cmdJunkPlay(c *cmd) {
 	defer a.Profile()()
 	a.SetLogLevel()
 
-	f := must(junk.NewFilter(mlog.New("junkplay"), a.params, a.databasePath, a.bloomfilterPath))
+	f := must(junk.NewFilter(context.Background(), mlog.New("junkplay"), a.params, a.databasePath, a.bloomfilterPath))
 	defer func() {
 		if err := f.Close(); err != nil {
 			log.Printf("closing junk filter: %v", err)
@@ -414,7 +415,7 @@ func cmdJunkPlay(c *cmd) {
 		if !msg.sent {
 			var prob float64
 			var err error
-			prob, words, _, _, err = f.ClassifyMessagePath(path)
+			prob, words, _, _, err = f.ClassifyMessagePath(context.Background(), path)
 			if err != nil {
 				nbad++
 				return
@@ -455,7 +456,7 @@ func cmdJunkPlay(c *cmd) {
 			}
 		}
 
-		if err := f.Train(msg.ham, words); err != nil {
+		if err := f.Train(context.Background(), msg.ham, words); err != nil {
 			log.Printf("train: %s", err)
 		}
 	}

junk/filter.go

@@ -11,6 +11,7 @@ package junk
 // todo: perhaps: whether anchor text in links in html are different from the url
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -108,7 +109,7 @@ func (f *Filter) Close() error {
 	return err
 }
 
-func OpenFilter(log *mlog.Log, params Params, dbPath, bloomPath string, loadBloom bool) (*Filter, error) {
+func OpenFilter(ctx context.Context, log *mlog.Log, params Params, dbPath, bloomPath string, loadBloom bool) (*Filter, error) {
 	var bloom *Bloom
 	if loadBloom {
 		var err error
@@ -122,7 +123,7 @@ func OpenFilter(log *mlog.Log, params Params, dbPath, bloomPath string, loadBloo
 		}
 	}
 
-	db, err := openDB(dbPath)
+	db, err := openDB(ctx, dbPath)
 	if err != nil {
 		return nil, fmt.Errorf("open database: %s", err)
 	}
@@ -137,7 +138,7 @@ func OpenFilter(log *mlog.Log, params Params, dbPath, bloomPath string, loadBloo
 		db:        db,
 		bloom:     bloom,
 	}
-	err = f.db.Read(func(tx *bstore.Tx) error {
+	err = f.db.Read(ctx, func(tx *bstore.Tx) error {
 		wc := wordscore{Word: "-"}
 		err := tx.Get(&wc)
 		f.hams = wc.Ham
@@ -156,7 +157,7 @@ func OpenFilter(log *mlog.Log, params Params, dbPath, bloomPath string, loadBloo
 // filter is marked as new until the first save, will be done automatically if
 // TrainDirs is called. If the bloom and/or database files exist, an error is
 // returned.
-func NewFilter(log *mlog.Log, params Params, dbPath, bloomPath string) (*Filter, error) {
+func NewFilter(ctx context.Context, log *mlog.Log, params Params, dbPath, bloomPath string) (*Filter, error) {
 	var err error
 	if _, err := os.Stat(bloomPath); err == nil {
 		return nil, fmt.Errorf("bloom filter already exists on disk: %s", bloomPath)
@@ -182,7 +183,7 @@ func NewFilter(log *mlog.Log, params Params, dbPath, bloomPath string) (*Filter,
 	err = bf.Close()
 	log.Check(err, "closing bloomfilter file")
 
-	db, err := newDB(log, dbPath)
+	db, err := newDB(ctx, log, dbPath)
 	if err != nil {
 		xerr := os.Remove(bloomPath)
 		log.Check(xerr, "removing bloom filter file after db init error")
@@ -216,7 +217,7 @@ func openBloom(path string) (*Bloom, error) {
 	return NewBloom(buf, bloomK)
 }
 
-func newDB(log *mlog.Log, path string) (db *bstore.DB, rerr error) {
+func newDB(ctx context.Context, log *mlog.Log, path string) (db *bstore.DB, rerr error) {
 	// Remove any existing files.
 	os.Remove(path)
 
@@ -227,18 +228,18 @@ func newDB(log *mlog.Log, path string) (db *bstore.DB, rerr error) {
 		}
 	}()
 
-	db, err := bstore.Open(path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, wordscore{})
+	db, err := bstore.Open(ctx, path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, wordscore{})
 	if err != nil {
 		return nil, fmt.Errorf("open new database: %w", err)
 	}
 	return db, nil
 }
 
-func openDB(path string) (*bstore.DB, error) {
+func openDB(ctx context.Context, path string) (*bstore.DB, error) {
 	if _, err := os.Stat(path); err != nil {
 		return nil, fmt.Errorf("stat db file: %w", err)
 	}
-	return bstore.Open(path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, wordscore{})
+	return bstore.Open(ctx, path, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, wordscore{})
 }
 
 // Save stores modifications, e.g. from training, to the database and bloom
@@ -280,7 +281,7 @@ func (f *Filter) Save() error {
 			}()
 		}
 	}
-	err := f.db.Write(func(tx *bstore.Tx) error {
+	err := f.db.Write(context.Background(), func(tx *bstore.Tx) error {
 		update := func(w string, ham, spam uint32) error {
 			if f.isNew {
 				return tx.Insert(&wordscore{w, ham, spam})
@@ -318,12 +319,12 @@ func (f *Filter) Save() error {
 	return nil
 }
 
-func loadWords(db *bstore.DB, l []string, dst map[string]word) error {
+func loadWords(ctx context.Context, db *bstore.DB, l []string, dst map[string]word) error {
 	sort.Slice(l, func(i, j int) bool {
 		return l[i] < l[j]
 	})
 
-	err := db.Read(func(tx *bstore.Tx) error {
+	err := db.Read(ctx, func(tx *bstore.Tx) error {
 		for _, w := range l {
 			wc := wordscore{Word: w}
 			if err := tx.Get(&wc); err == nil {
@@ -339,7 +340,7 @@ func loadWords(db *bstore.DB, l []string, dst map[string]word) error {
 }
 
 // ClassifyWords returns the spam probability for the given words, and number of recognized ham and spam words.
-func (f *Filter) ClassifyWords(words map[string]struct{}) (probability float64, nham, nspam int, rerr error) {
+func (f *Filter) ClassifyWords(ctx context.Context, words map[string]struct{}) (probability float64, nham, nspam int, rerr error) {
 	if f.closed {
 		return 0, 0, 0, errClosed
 	}
@@ -380,7 +381,7 @@ func (f *Filter) ClassifyWords(words map[string]struct{}) (probability float64,
 	// Fetch words from database.
 	fetched := map[string]word{}
 	if len(lookupWords) > 0 {
-		if err := loadWords(f.db, lookupWords, fetched); err != nil {
+		if err := loadWords(ctx, f.db, lookupWords, fetched); err != nil {
 			return 0, 0, 0, err
 		}
 		for w, c := range fetched {
@@ -477,7 +478,7 @@ func (f *Filter) ClassifyWords(words map[string]struct{}) (probability float64,
 }
 
 // ClassifyMessagePath is a convenience wrapper for calling ClassifyMessage on a file.
-func (f *Filter) ClassifyMessagePath(path string) (probability float64, words map[string]struct{}, nham, nspam int, rerr error) {
+func (f *Filter) ClassifyMessagePath(ctx context.Context, path string) (probability float64, words map[string]struct{}, nham, nspam int, rerr error) {
 	if f.closed {
 		return 0, nil, 0, 0, errClosed
 	}
@@ -494,35 +495,35 @@ func (f *Filter) ClassifyMessagePath(path string) (probability float64, words ma
 	if err != nil {
 		return 0, nil, 0, 0, err
 	}
-	return f.ClassifyMessageReader(mf, fi.Size())
+	return f.ClassifyMessageReader(ctx, mf, fi.Size())
 }
 
-func (f *Filter) ClassifyMessageReader(mf io.ReaderAt, size int64) (probability float64, words map[string]struct{}, nham, nspam int, rerr error) {
+func (f *Filter) ClassifyMessageReader(ctx context.Context, mf io.ReaderAt, size int64) (probability float64, words map[string]struct{}, nham, nspam int, rerr error) {
 	m, err := message.EnsurePart(mf, size)
 	if err != nil && errors.Is(err, message.ErrBadContentType) {
 		// Invalid content-type header is a sure sign of spam.
 		//f.log.Infox("parsing content", err)
 		return 1, nil, 0, 0, nil
 	}
-	return f.ClassifyMessage(m)
+	return f.ClassifyMessage(ctx, m)
 }
 
 // ClassifyMessage parses the mail message in r and returns the spam probability
 // (between 0 and 1), along with the tokenized words found in the message, and the
 // number of recognized ham and spam words.
-func (f *Filter) ClassifyMessage(m message.Part) (probability float64, words map[string]struct{}, nham, nspam int, rerr error) {
+func (f *Filter) ClassifyMessage(ctx context.Context, m message.Part) (probability float64, words map[string]struct{}, nham, nspam int, rerr error) {
 	var err error
 	words, err = f.ParseMessage(m)
 	if err != nil {
 		return 0, nil, 0, 0, err
 	}
 
-	probability, nham, nspam, err = f.ClassifyWords(words)
+	probability, nham, nspam, err = f.ClassifyWords(ctx, words)
 	return probability, words, nham, nspam, err
 }
 
 // Train adds the words of a single message to the filter.
-func (f *Filter) Train(ham bool, words map[string]struct{}) error {
+func (f *Filter) Train(ctx context.Context, ham bool, words map[string]struct{}) error {
 	if err := f.ensureBloom(); err != nil {
 		return err
 	}
@@ -539,7 +540,7 @@ func (f *Filter) Train(ham bool, words map[string]struct{}) error {
 		}
 	}
 
-	if err := f.loadCache(lwords); err != nil {
+	if err := f.loadCache(ctx, lwords); err != nil {
 		return err
 	}
 
@@ -563,34 +564,34 @@ func (f *Filter) Train(ham bool, words map[string]struct{}) error {
 	return nil
 }
 
-func (f *Filter) TrainMessage(r io.ReaderAt, size int64, ham bool) error {
+func (f *Filter) TrainMessage(ctx context.Context, r io.ReaderAt, size int64, ham bool) error {
 	p, _ := message.EnsurePart(r, size)
 	words, err := f.ParseMessage(p)
 	if err != nil {
 		return fmt.Errorf("parsing mail contents: %v", err)
 	}
-	return f.Train(ham, words)
+	return f.Train(ctx, ham, words)
 }
 
-func (f *Filter) UntrainMessage(r io.ReaderAt, size int64, ham bool) error {
+func (f *Filter) UntrainMessage(ctx context.Context, r io.ReaderAt, size int64, ham bool) error {
 	p, _ := message.EnsurePart(r, size)
 	words, err := f.ParseMessage(p)
 	if err != nil {
 		return fmt.Errorf("parsing mail contents: %v", err)
 	}
-	return f.Untrain(ham, words)
+	return f.Untrain(ctx, ham, words)
 }
 
-func (f *Filter) loadCache(lwords []string) error {
+func (f *Filter) loadCache(ctx context.Context, lwords []string) error {
 	if len(lwords) == 0 {
 		return nil
 	}
 
-	return loadWords(f.db, lwords, f.cache)
+	return loadWords(ctx, f.db, lwords, f.cache)
 }
 
 // Untrain adjusts the filter to undo a previous training of the words.
-func (f *Filter) Untrain(ham bool, words map[string]struct{}) error {
+func (f *Filter) Untrain(ctx context.Context, ham bool, words map[string]struct{}) error {
 	if err := f.ensureBloom(); err != nil {
 		return err
 	}
@@ -602,7 +603,7 @@ func (f *Filter) Untrain(ham bool, words map[string]struct{}) error {
 			lwords = append(lwords, w)
 		}
 	}
-	if err := f.loadCache(lwords); err != nil {
+	if err := f.loadCache(ctx, lwords); err != nil {
 		return err
 	}
 

junk/filter_test.go

@@ -1,6 +1,7 @@
 package junk
 
 import (
+	"context"
 	"fmt"
 	"math"
 	"os"
@@ -10,6 +11,8 @@ import (
 	"github.com/mjl-/mox/mlog"
 )
 
+var ctxbg = context.Background()
+
 func tcheck(t *testing.T, err error, msg string) {
 	t.Helper()
 	if err != nil {
@@ -43,12 +46,12 @@ func TestFilter(t *testing.T) {
 	bloomPath := "../testdata/junk/filter.bloom"
 	os.Remove(dbPath)
 	os.Remove(bloomPath)
-	f, err := NewFilter(log, params, dbPath, bloomPath)
+	f, err := NewFilter(ctxbg, log, params, dbPath, bloomPath)
 	tcheck(t, err, "new filter")
 	err = f.Close()
 	tcheck(t, err, "close filter")
 
-	f, err = OpenFilter(log, params, dbPath, bloomPath, true)
+	f, err = OpenFilter(ctxbg, log, params, dbPath, bloomPath, true)
 	tcheck(t, err, "open filter")
 
 	// Ensure these dirs exist. Developers should bring their own ham/spam example
@@ -75,13 +78,13 @@ func TestFilter(t *testing.T) {
 		return
 	}
 
-	prob, _, _, _, err := f.ClassifyMessagePath(filepath.Join(hamdir, hamfiles[0]))
+	prob, _, _, _, err := f.ClassifyMessagePath(ctxbg, filepath.Join(hamdir, hamfiles[0]))
 	tcheck(t, err, "classify ham message")
 	if prob > 0.1 {
 		t.Fatalf("trained ham file has prob %v, expected <= 0.1", prob)
 	}
 
-	prob, _, _, _, err = f.ClassifyMessagePath(filepath.Join(spamdir, spamfiles[0]))
+	prob, _, _, _, err = f.ClassifyMessagePath(ctxbg, filepath.Join(spamdir, spamfiles[0]))
 	tcheck(t, err, "classify spam message")
 	if prob < 0.9 {
 		t.Fatalf("trained spam file has prob %v, expected > 0.9", prob)
@@ -94,7 +97,7 @@ func TestFilter(t *testing.T) {
 	// classified as ham/spam. Then we untrain to see they are no longer classified.
 	os.Remove(dbPath)
 	os.Remove(bloomPath)
-	f, err = NewFilter(log, params, dbPath, bloomPath)
+	f, err = NewFilter(ctxbg, log, params, dbPath, bloomPath)
 	tcheck(t, err, "open filter")
 
 	hamf, err := os.Open(filepath.Join(hamdir, hamfiles[0]))
@@ -112,18 +115,18 @@ func TestFilter(t *testing.T) {
 	spamsize := spamstat.Size()
 
 	// Train each message twice, to prevent single occurrences from being ignored.
-	err = f.TrainMessage(hamf, hamsize, true)
+	err = f.TrainMessage(ctxbg, hamf, hamsize, true)
 	tcheck(t, err, "train ham message")
 	_, err = hamf.Seek(0, 0)
 	tcheck(t, err, "seek ham message")
-	err = f.TrainMessage(hamf, hamsize, true)
+	err = f.TrainMessage(ctxbg, hamf, hamsize, true)
 	tcheck(t, err, "train ham message")
 
-	err = f.TrainMessage(spamf, spamsize, false)
+	err = f.TrainMessage(ctxbg, spamf, spamsize, false)
 	tcheck(t, err, "train spam message")
 	_, err = spamf.Seek(0, 0)
 	tcheck(t, err, "seek spam message")
-	err = f.TrainMessage(spamf, spamsize, true)
+	err = f.TrainMessage(ctxbg, spamf, spamsize, true)
 	tcheck(t, err, "train spam message")
 
 	if !f.modified {
@@ -142,7 +145,7 @@ func TestFilter(t *testing.T) {
 	// Classify and verify.
 	_, err = hamf.Seek(0, 0)
 	tcheck(t, err, "seek ham message")
-	prob, _, _, _, err = f.ClassifyMessageReader(hamf, hamsize)
+	prob, _, _, _, err = f.ClassifyMessageReader(ctxbg, hamf, hamsize)
 	tcheck(t, err, "classify ham")
 	if prob > 0.1 {
 		t.Fatalf("got prob %v, expected <= 0.1", prob)
@@ -150,7 +153,7 @@ func TestFilter(t *testing.T) {
 
 	_, err = spamf.Seek(0, 0)
 	tcheck(t, err, "seek spam message")
-	prob, _, _, _, err = f.ClassifyMessageReader(spamf, spamsize)
+	prob, _, _, _, err = f.ClassifyMessageReader(ctxbg, spamf, spamsize)
 	tcheck(t, err, "classify spam")
 	if prob < 0.9 {
 		t.Fatalf("got prob %v, expected >= 0.9", prob)
@@ -159,20 +162,20 @@ func TestFilter(t *testing.T) {
 	// Untrain ham & spam.
 	_, err = hamf.Seek(0, 0)
 	tcheck(t, err, "seek ham message")
-	err = f.UntrainMessage(hamf, hamsize, true)
+	err = f.UntrainMessage(ctxbg, hamf, hamsize, true)
 	tcheck(t, err, "untrain ham message")
 	_, err = hamf.Seek(0, 0)
 	tcheck(t, err, "seek ham message")
-	err = f.UntrainMessage(hamf, spamsize, true)
+	err = f.UntrainMessage(ctxbg, hamf, spamsize, true)
 	tcheck(t, err, "untrain ham message")
 
 	_, err = spamf.Seek(0, 0)
 	tcheck(t, err, "seek spam message")
-	err = f.UntrainMessage(spamf, spamsize, true)
+	err = f.UntrainMessage(ctxbg, spamf, spamsize, true)
 	tcheck(t, err, "untrain spam message")
 	_, err = spamf.Seek(0, 0)
 	tcheck(t, err, "seek spam message")
-	err = f.UntrainMessage(spamf, spamsize, true)
+	err = f.UntrainMessage(ctxbg, spamf, spamsize, true)
 	tcheck(t, err, "untrain spam message")
 
 	if !f.modified {
@@ -182,7 +185,7 @@ func TestFilter(t *testing.T) {
 	// Classify again, should be unknown.
 	_, err = hamf.Seek(0, 0)
 	tcheck(t, err, "seek ham message")
-	prob, _, _, _, err = f.ClassifyMessageReader(hamf, hamsize)
+	prob, _, _, _, err = f.ClassifyMessageReader(ctxbg, hamf, hamsize)
 	tcheck(t, err, "classify ham")
 	if math.Abs(prob-0.5) > 0.1 {
 		t.Fatalf("got prob %v, expected 0.5 +-0.1", prob)
@@ -190,7 +193,7 @@ func TestFilter(t *testing.T) {
 
 	_, err = spamf.Seek(0, 0)
 	tcheck(t, err, "seek spam message")
-	prob, _, _, _, err = f.ClassifyMessageReader(spamf, spamsize)
+	prob, _, _, _, err = f.ClassifyMessageReader(ctxbg, spamf, spamsize)
 	tcheck(t, err, "classify spam")
 	if math.Abs(prob-0.5) > 0.1 {
 		t.Fatalf("got prob %v, expected 0.5 +-0.1", prob)

junk/parse_test.go

@@ -23,7 +23,7 @@ func FuzzParseMessage(f *testing.F) {
 	os.Remove(dbPath)
 	os.Remove(bloomPath)
 	params := Params{Twograms: true}
-	jf, err := NewFilter(xlog, params, dbPath, bloomPath)
+	jf, err := NewFilter(ctxbg, xlog, params, dbPath, bloomPath)
 	if err != nil {
 		f.Fatalf("new filter: %v", err)
 	}

main.go

@@ -1879,7 +1879,7 @@ func cmdEnsureParsed(c *cmd) {
 	}()
 
 	n := 0
-	err = a.DB.Write(func(tx *bstore.Tx) error {
+	err = a.DB.Write(context.Background(), func(tx *bstore.Tx) error {
 		q := bstore.QueryTx[store.Message](tx)
 		q.FilterFn(func(m store.Message) bool {
 			return all || m.ParsedBuf == nil
@@ -1952,7 +1952,7 @@ func cmdBumpUIDValidity(c *cmd) {
 	}()
 
 	var uidvalidity uint32
-	err = a.DB.Write(func(tx *bstore.Tx) error {
+	err = a.DB.Write(context.Background(), func(tx *bstore.Tx) error {
 		mb, err := bstore.QueryTx[store.Mailbox](tx).FilterEqual("Name", args[1]).Get()
 		if err != nil {
 			return fmt.Errorf("looking up mailbox: %v", err)

mox-/lifecycle.go

@@ -152,14 +152,15 @@ func Listen(network, addr string) (net.Listener, error) {
 }
 
 // Shutdown is canceled when a graceful shutdown is initiated. SMTP, IMAP, periodic
-// processes should check this before starting a new operation. If true, the
-// operation should be aborted, and new connections should receive a message that
-// the service is currently not available.
+// processes should check this before starting a new operation. If this context is
+// canaceled, the operation should not be started, and new connections/commands should
+// receive a message that the service is currently not available.
 var Shutdown context.Context
 var ShutdownCancel func()
 
-// Context should be used as parent by all operations. It is canceled when mox is
-// shutdown, aborting all pending operations.
+// This context should be used as parent by most operations. It is canceled 1
+// second after graceful shutdown was initiated with the cancelation of the
+// Shutdown context. This should abort active operations.
 //
 // Operations typically have context timeouts, 30s for single i/o like DNS queries,
 // and 1 minute for operations with more back and forth. These are set through a
@@ -167,6 +168,7 @@ var ShutdownCancel func()
 // when shutting down.
 //
 // HTTP servers don't get graceful shutdown, their connections are just aborted.
+// todo: should shut down http connections as well, and shut down the listener and/or return 503 for new requests.
 var Context context.Context
 var ContextCancel func()
 

mtastsdb/db.go

@@ -63,13 +63,13 @@ var (
 var mtastsDB *bstore.DB
 var mutex sync.Mutex
 
-func database() (rdb *bstore.DB, rerr error) {
+func database(ctx context.Context) (rdb *bstore.DB, rerr error) {
 	mutex.Lock()
 	defer mutex.Unlock()
 	if mtastsDB == nil {
 		p := mox.DataDirPath("mtasts.db")
 		os.MkdirAll(filepath.Dir(p), 0770)
-		db, err := bstore.Open(p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, PolicyRecord{})
+		db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, PolicyRecord{})
 		if err != nil {
 			return nil, err
 		}
@@ -81,7 +81,7 @@ func database() (rdb *bstore.DB, rerr error) {
 // Init opens the database and starts a goroutine that refreshes policies in
 // the database, and keeps doing so periodically.
 func Init(refresher bool) error {
-	_, err := database()
+	_, err := database(mox.Shutdown)
 	if err != nil {
 		return err
 	}
@@ -110,7 +110,7 @@ func Close() {
 // Only non-expired records are returned.
 func lookup(ctx context.Context, domain dns.Domain) (*PolicyRecord, error) {
 	log := xlog.WithContext(ctx)
-	db, err := database()
+	db, err := database(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -119,7 +119,7 @@ func lookup(ctx context.Context, domain dns.Domain) (*PolicyRecord, error) {
 		return nil, fmt.Errorf("empty domain")
 	}
 	now := timeNow()
-	q := bstore.QueryDB[PolicyRecord](db)
+	q := bstore.QueryDB[PolicyRecord](ctx, db)
 	q.FilterNonzero(PolicyRecord{Domain: domain.Name()})
 	q.FilterGreater("ValidEnd", now)
 	pr, err := q.Get()
@@ -130,7 +130,7 @@ func lookup(ctx context.Context, domain dns.Domain) (*PolicyRecord, error) {
 	}
 
 	pr.LastUse = now
-	if err := db.Update(&pr); err != nil {
+	if err := db.Update(ctx, &pr); err != nil {
 		log.Errorx("marking cached mta-sts policy as used in database", err)
 	}
 	if pr.Backoff {
@@ -141,13 +141,13 @@ func lookup(ctx context.Context, domain dns.Domain) (*PolicyRecord, error) {
 
 // Upsert adds the policy to the database, overwriting an existing policy for the domain.
 // Policy can be nil, indicating a failure to fetch the policy.
-func Upsert(domain dns.Domain, recordID string, policy *mtasts.Policy) error {
-	db, err := database()
+func Upsert(ctx context.Context, domain dns.Domain, recordID string, policy *mtasts.Policy) error {
+	db, err := database(ctx)
 	if err != nil {
 		return err
 	}
 
-	return db.Write(func(tx *bstore.Tx) error {
+	return db.Write(ctx, func(tx *bstore.Tx) error {
 		pr := PolicyRecord{Domain: domain.Name()}
 		err := tx.Get(&pr)
 		if err != nil && err != bstore.ErrAbsent {
@@ -185,11 +185,11 @@ func Upsert(domain dns.Domain, recordID string, policy *mtasts.Policy) error {
 // PolicyRecords returns all policies in the database, sorted descending by last
 // use, domain.
 func PolicyRecords(ctx context.Context) ([]PolicyRecord, error) {
-	db, err := database()
+	db, err := database(ctx)
 	if err != nil {
 		return nil, err
 	}
-	return bstore.QueryDB[PolicyRecord](db).SortDesc("LastUse", "Domain").List()
+	return bstore.QueryDB[PolicyRecord](ctx, db).SortDesc("LastUse", "Domain").List()
 }
 
 // Get retrieves an MTA-STS policy for domain and whether it is fresh.
@@ -244,7 +244,7 @@ func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy
 			if record != nil {
 				recordID = record.ID
 			}
-			if err := Upsert(domain, recordID, p); err != nil {
+			if err := Upsert(ctx, domain, recordID, p); err != nil {
 				log.Errorx("inserting policy into cache, continuing", err)
 			}
 		}
@@ -259,9 +259,9 @@ func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy
 
 	// Policy was found in database. Check in DNS it is still fresh.
 	policy = &cachedPolicy.Policy
-	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	nctx, cancel := context.WithTimeout(ctx, 30*time.Second)
 	defer cancel()
-	record, _, _, err := mtasts.LookupRecord(ctx, resolver, domain)
+	record, _, _, err := mtasts.LookupRecord(nctx, resolver, domain)
 	if err != nil {
 		if !errors.Is(err, mtasts.ErrNoRecord) {
 			// Could be a temporary DNS or configuration error.
@@ -271,15 +271,16 @@ func Get(ctx context.Context, resolver dns.Resolver, domain dns.Domain) (policy
 	} else if record.ID == cachedPolicy.RecordID {
 		return policy, true, nil
 	}
+
 	// New policy should be available.
-	ctx, cancel = context.WithTimeout(ctx, 30*time.Second)
+	nctx, cancel = context.WithTimeout(ctx, 30*time.Second)
 	defer cancel()
-	p, _, err := mtasts.FetchPolicy(ctx, domain)
+	p, _, err := mtasts.FetchPolicy(nctx, domain)
 	if err != nil {
 		log.Errorx("fetching updated policy for domain, continuing with previously cached policy", err)
 		return policy, false, nil
 	}
-	if err := Upsert(domain, record.ID, p); err != nil {
+	if err := Upsert(ctx, domain, record.ID, p); err != nil {
 		log.Errorx("inserting refreshed policy into cache, continuing with fresh policy", err)
 	}
 	return p, true, nil

mtastsdb/db_test.go

@@ -1,7 +1,6 @@
 package mtastsdb
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"net"
@@ -24,6 +23,7 @@ func tcheckf(t *testing.T, err error, format string, args ...any) {
 }
 
 func TestDB(t *testing.T) {
+	mox.Shutdown = ctxbg
 	mox.ConfigStaticPath = "../testdata/mtasts/fake.conf"
 	mox.Conf.Static.DataDir = "."
 
@@ -37,14 +37,12 @@ func TestDB(t *testing.T) {
 	}
 	defer Close()
 
-	ctx := context.Background()
-
 	// Mock time.
 	now := time.Now().Round(0)
 	timeNow = func() time.Time { return now }
 	defer func() { timeNow = time.Now }()
 
-	if p, err := lookup(ctx, dns.Domain{ASCII: "example.com"}); err != ErrNotFound {
+	if p, err := lookup(ctxbg, dns.Domain{ASCII: "example.com"}); err != ErrNotFound {
 		t.Fatalf("expected not found, got %v, %#v", err, p)
 	}
 
@@ -58,10 +56,10 @@ func TestDB(t *testing.T) {
 		},
 		MaxAgeSeconds: 1296000,
 	}
-	if err := Upsert(dns.Domain{ASCII: "example.com"}, "123", &policy1); err != nil {
+	if err := Upsert(ctxbg, dns.Domain{ASCII: "example.com"}, "123", &policy1); err != nil {
 		t.Fatalf("upsert record: %s", err)
 	}
-	if got, err := lookup(ctx, dns.Domain{ASCII: "example.com"}); err != nil {
+	if got, err := lookup(ctxbg, dns.Domain{ASCII: "example.com"}); err != nil {
 		t.Fatalf("lookup after insert: %s", err)
 	} else if !reflect.DeepEqual(got.Policy, policy1) {
 		t.Fatalf("mismatch between inserted and retrieved: got %#v, want %#v", got, policy1)
@@ -75,17 +73,17 @@ func TestDB(t *testing.T) {
 		},
 		MaxAgeSeconds: 360000,
 	}
-	if err := Upsert(dns.Domain{ASCII: "example.com"}, "124", &policy2); err != nil {
+	if err := Upsert(ctxbg, dns.Domain{ASCII: "example.com"}, "124", &policy2); err != nil {
 		t.Fatalf("upsert record: %s", err)
 	}
-	if got, err := lookup(ctx, dns.Domain{ASCII: "example.com"}); err != nil {
+	if got, err := lookup(ctxbg, dns.Domain{ASCII: "example.com"}); err != nil {
 		t.Fatalf("lookup after insert: %s", err)
 	} else if !reflect.DeepEqual(got.Policy, policy2) {
 		t.Fatalf("mismatch between inserted and retrieved: got %v, want %v", got, policy2)
 	}
 
 	// Check if database holds expected record.
-	records, err := PolicyRecords(context.Background())
+	records, err := PolicyRecords(ctxbg)
 	tcheckf(t, err, "policyrecords")
 	expRecords := []PolicyRecord{
 		{"example.com", now, now.Add(time.Duration(policy2.MaxAgeSeconds) * time.Second), now, now, false, "124", policy2},
@@ -96,10 +94,10 @@ func TestDB(t *testing.T) {
 		t.Fatalf("records mismatch, got %#v, expected %#v", records, expRecords)
 	}
 
-	if err := Upsert(dns.Domain{ASCII: "other.example.com"}, "", nil); err != nil {
+	if err := Upsert(ctxbg, dns.Domain{ASCII: "other.example.com"}, "", nil); err != nil {
 		t.Fatalf("upsert record: %s", err)
 	}
-	records, err = PolicyRecords(context.Background())
+	records, err = PolicyRecords(ctxbg)
 	tcheckf(t, err, "policyrecords")
 	expRecords = []PolicyRecord{
 		{"other.example.com", now, now.Add(5 * 60 * time.Second), now, now, true, "", mtasts.Policy{Mode: mtasts.ModeNone, MaxAgeSeconds: 5 * 60}},
@@ -109,7 +107,7 @@ func TestDB(t *testing.T) {
 		t.Fatalf("records mismatch, got %#v, expected %#v", records, expRecords)
 	}
 
-	if _, err := lookup(context.Background(), dns.Domain{ASCII: "other.example.com"}); err != ErrBackoff {
+	if _, err := lookup(ctxbg, dns.Domain{ASCII: "other.example.com"}); err != ErrBackoff {
 		t.Fatalf("got %#v, expected ErrBackoff", err)
 	}
 
@@ -126,7 +124,7 @@ func TestDB(t *testing.T) {
 
 	testGet := func(domain string, expPolicy *mtasts.Policy, expFresh bool, expErr error) {
 		t.Helper()
-		p, fresh, err := Get(context.Background(), resolver, dns.Domain{ASCII: domain})
+		p, fresh, err := Get(ctxbg, resolver, dns.Domain{ASCII: domain})
 		if (err == nil) != (expErr == nil) || err != nil && !errors.Is(err, expErr) {
 			t.Fatalf("got err %v, expected %v", err, expErr)
 		}

mtastsdb/refresh.go

@@ -52,19 +52,19 @@ func refresh() int {
 // jitter to the timing. Each refresh is done in a new goroutine, so a single slow
 // refresh doesn't mess up the timing.
 func refresh1(ctx context.Context, resolver dns.Resolver, sleep func(d time.Duration)) (int, error) {
-	db, err := database()
+	db, err := database(ctx)
 	if err != nil {
 		return 0, err
 	}
 
 	now := timeNow()
-	qdel := bstore.QueryDB[PolicyRecord](db)
+	qdel := bstore.QueryDB[PolicyRecord](ctx, db)
 	qdel.FilterLess("LastUse", now.Add(-180*24*time.Hour))
 	if _, err := qdel.Delete(); err != nil {
 		return 0, fmt.Errorf("deleting old unused policies: %s", err)
 	}
 
-	qup := bstore.QueryDB[PolicyRecord](db)
+	qup := bstore.QueryDB[PolicyRecord](ctx, db)
 	qup.FilterLess("LastUpdate", now.Add(-12*time.Hour))
 	prs, err := qup.List()
 	if err != nil {
@@ -127,7 +127,7 @@ func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr
 	log.Debug("refreshing mta-sts policy for domain", mlog.Field("domain", d))
 	record, _, _, err := mtasts.LookupRecord(ctx, resolver, d)
 	if err == nil && record.ID == pr.RecordID {
-		qup := bstore.QueryDB[PolicyRecord](db)
+		qup := bstore.QueryDB[PolicyRecord](ctx, db)
 		qup.FilterNonzero(PolicyRecord{Domain: pr.Domain, LastUpdate: pr.LastUpdate})
 		now := timeNow()
 		update := PolicyRecord{
@@ -166,7 +166,7 @@ func refreshDomain(ctx context.Context, db *bstore.DB, resolver dns.Resolver, pr
 	if record != nil {
 		update["RecordID"] = record.ID
 	}
-	qup := bstore.QueryDB[PolicyRecord](db)
+	qup := bstore.QueryDB[PolicyRecord](ctx, db)
 	qup.FilterNonzero(PolicyRecord{Domain: pr.Domain, LastUpdate: pr.LastUpdate})
 	if n, err := qup.UpdateFields(update); err != nil {
 		log.Errorx("updating refreshed, modified policy in database", err)

mtastsdb/refresh_test.go

@@ -26,7 +26,10 @@ import (
 	"github.com/mjl-/mox/mtasts"
 )
 
+var ctxbg = context.Background()
+
 func TestRefresh(t *testing.T) {
+	mox.Shutdown = ctxbg
 	mox.ConfigStaticPath = "../testdata/mtasts/fake.conf"
 	mox.Conf.Static.DataDir = "."
 
@@ -40,7 +43,7 @@ func TestRefresh(t *testing.T) {
 	}
 	defer Close()
 
-	db, err := database()
+	db, err := database(ctxbg)
 	if err != nil {
 		t.Fatalf("database: %s", err)
 	}
@@ -66,7 +69,7 @@ func TestRefresh(t *testing.T) {
 		}
 
 		pr := PolicyRecord{domain, time.Time{}, validEnd, lastUpdate, lastUse, backoff, recordID, policy}
-		if err := db.Insert(&pr); err != nil {
+		if err := db.Insert(ctxbg, &pr); err != nil {
 			t.Fatalf("insert policy: %s", err)
 		}
 	}
@@ -132,7 +135,7 @@ func TestRefresh(t *testing.T) {
 			t.Fatalf("bad sleep duration %v", d)
 		}
 	}
-	if n, err := refresh1(context.Background(), resolver, sleep); err != nil || n != 3 {
+	if n, err := refresh1(ctxbg, resolver, sleep); err != nil || n != 3 {
 		t.Fatalf("refresh1: err %s, n %d, expected no error, 3", err, n)
 	}
 	if slept != 2 {
@@ -141,19 +144,19 @@ func TestRefresh(t *testing.T) {
 	time.Sleep(time.Second / 10) // Give goroutine time to write result, before we cleanup the database.
 
 	// Should not do any more refreshes and return immediately.
-	q := bstore.QueryDB[PolicyRecord](db)
+	q := bstore.QueryDB[PolicyRecord](ctxbg, db)
 	q.FilterNonzero(PolicyRecord{Domain: "policybad.mox.example"})
 	if _, err := q.Delete(); err != nil {
 		t.Fatalf("delete record that would be refreshed: %v", err)
 	}
-	mox.Context = context.Background()
-	mox.Shutdown, mox.ShutdownCancel = context.WithCancel(context.Background())
+	mox.Context = ctxbg
+	mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
 	mox.ShutdownCancel()
 	n := refresh()
 	if n != 0 {
 		t.Fatalf("refresh found unexpected work, n %d", n)
 	}
-	mox.Shutdown, mox.ShutdownCancel = context.WithCancel(context.Background())
+	mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
 }
 
 type pipeListener struct {

queue/queue.go

@@ -122,7 +122,7 @@ func Init() error {
 	}
 
 	var err error
-	queueDB, err = bstore.Open(qpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, Msg{})
+	queueDB, err = bstore.Open(mox.Shutdown, qpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, Msg{})
 	if err != nil {
 		if isNew {
 			os.Remove(qpath)
@@ -141,8 +141,8 @@ func Shutdown() {
 
 // List returns all messages in the delivery queue.
 // Ordered by earliest delivery attempt first.
-func List() ([]Msg, error) {
-	qmsgs, err := bstore.QueryDB[Msg](queueDB).List()
+func List(ctx context.Context) ([]Msg, error) {
+	qmsgs, err := bstore.QueryDB[Msg](ctx, queueDB).List()
 	if err != nil {
 		return nil, err
 	}
@@ -165,8 +165,8 @@ func List() ([]Msg, error) {
 }
 
 // Count returns the number of messages in the delivery queue.
-func Count() (int, error) {
-	return bstore.QueryDB[Msg](queueDB).Count()
+func Count(ctx context.Context) (int, error) {
+	return bstore.QueryDB[Msg](ctx, queueDB).Count()
 }
 
 // Add a new message to the queue. The queue is kicked immediately to start a
@@ -179,7 +179,7 @@ func Count() (int, error) {
 // this data is used as the message when delivering the DSN and the remote SMTP
 // server supports SMTPUTF8. If the remote SMTP server does not support SMTPUTF8,
 // the regular non-utf8 message is delivered.
-func Add(log *mlog.Log, senderAccount string, mailFrom, rcptTo smtp.Path, has8bit, smtputf8 bool, size int64, msgPrefix []byte, msgFile *os.File, dsnutf8Opt []byte, consumeFile bool) error {
+func Add(ctx context.Context, log *mlog.Log, senderAccount string, mailFrom, rcptTo smtp.Path, has8bit, smtputf8 bool, size int64, msgPrefix []byte, msgFile *os.File, dsnutf8Opt []byte, consumeFile bool) error {
 	// todo: Add should accept multiple rcptTo if they are for the same domain. so we can queue them for delivery in one (or just a few) session(s), transferring the data only once. ../rfc/5321:3759
 
 	if Localserve {
@@ -187,7 +187,7 @@ func Add(log *mlog.Log, senderAccount string, mailFrom, rcptTo smtp.Path, has8bi
 		return fmt.Errorf("no queuing with localserve")
 	}
 
-	tx, err := queueDB.Begin(true)
+	tx, err := queueDB.Begin(ctx, true)
 	if err != nil {
 		return fmt.Errorf("begin transaction: %w", err)
 	}
@@ -287,8 +287,8 @@ func queuekick() {
 // and kicks the queue, attempting delivery of those messages. If all parameters
 // are zero, all messages are kicked.
 // Returns number of messages queued for immediate delivery.
-func Kick(ID int64, toDomain string, recipient string) (int, error) {
-	q := bstore.QueryDB[Msg](queueDB)
+func Kick(ctx context.Context, ID int64, toDomain string, recipient string) (int, error) {
+	q := bstore.QueryDB[Msg](ctx, queueDB)
 	if ID > 0 {
 		q.FilterID(ID)
 	}
@@ -311,8 +311,8 @@ func Kick(ID int64, toDomain string, recipient string) (int, error) {
 // Drop removes messages from the queue that match all nonzero parameters.
 // If all parameters are zero, all messages are removed.
 // Returns number of messages removed.
-func Drop(ID int64, toDomain string, recipient string) (int, error) {
-	q := bstore.QueryDB[Msg](queueDB)
+func Drop(ctx context.Context, ID int64, toDomain string, recipient string) (int, error) {
+	q := bstore.QueryDB[Msg](ctx, queueDB)
 	if ID > 0 {
 		q.FilterID(ID)
 	}
@@ -337,9 +337,9 @@ type ReadReaderAtCloser interface {
 }
 
 // OpenMessage opens a message present in the queue.
-func OpenMessage(id int64) (ReadReaderAtCloser, error) {
+func OpenMessage(ctx context.Context, id int64) (ReadReaderAtCloser, error) {
 	qm := Msg{ID: id}
-	err := queueDB.Get(&qm)
+	err := queueDB.Get(ctx, &qm)
 	if err != nil {
 		return nil, err
 	}
@@ -382,14 +382,14 @@ func Start(resolver dns.Resolver, done chan struct{}) error {
 			}
 
 			launchWork(resolver, busyDomains)
-			timer.Reset(nextWork(busyDomains))
+			timer.Reset(nextWork(mox.Shutdown, busyDomains))
 		}
 	}()
 	return nil
 }
 
-func nextWork(busyDomains map[string]struct{}) time.Duration {
-	q := bstore.QueryDB[Msg](queueDB)
+func nextWork(ctx context.Context, busyDomains map[string]struct{}) time.Duration {
+	q := bstore.QueryDB[Msg](ctx, queueDB)
 	if len(busyDomains) > 0 {
 		var doms []any
 		for d := range busyDomains {
@@ -410,7 +410,7 @@ func nextWork(busyDomains map[string]struct{}) time.Duration {
 }
 
 func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int {
-	q := bstore.QueryDB[Msg](queueDB)
+	q := bstore.QueryDB[Msg](mox.Shutdown, queueDB)
 	q.FilterLessEqual("NextAttempt", time.Now())
 	q.SortAsc("NextAttempt")
 	q.Limit(maxConcurrentDeliveries)
@@ -424,7 +424,7 @@ func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int {
 	msgs, err := q.List()
 	if err != nil {
 		xlog.Errorx("querying for work in queue", err)
-		mox.Sleep(mox.Context, 1*time.Second)
+		mox.Sleep(mox.Shutdown, 1*time.Second)
 		return -1
 	}
 
@@ -436,8 +436,8 @@ func launchWork(resolver dns.Resolver, busyDomains map[string]struct{}) int {
 }
 
 // Remove message from queue in database and file system.
-func queueDelete(msgID int64) error {
-	if err := queueDB.Delete(&Msg{ID: msgID}); err != nil {
+func queueDelete(ctx context.Context, msgID int64) error {
+	if err := queueDB.Delete(ctx, &Msg{ID: msgID}); err != nil {
 		return err
 	}
 	// If removing from database fails, we'll also leave the file in the file system.
@@ -483,7 +483,7 @@ func deliver(resolver dns.Resolver, m Msg) {
 	now := time.Now()
 	m.LastAttempt = &now
 	m.NextAttempt = now.Add(backoff)
-	qup := bstore.QueryDB[Msg](queueDB)
+	qup := bstore.QueryDB[Msg](mox.Shutdown, queueDB)
 	qup.FilterID(m.ID)
 	update := Msg{Attempts: m.Attempts, NextAttempt: m.NextAttempt, LastAttempt: m.LastAttempt}
 	if _, err := qup.UpdateNonzero(update); err != nil {
@@ -496,13 +496,13 @@ func deliver(resolver dns.Resolver, m Msg) {
 			qlog.Errorx("permanent failure delivering from queue", errors.New(errmsg))
 			queueDSNFailure(qlog, m, remoteMTA, secodeOpt, errmsg)
 
-			if err := queueDelete(m.ID); err != nil {
+			if err := queueDelete(context.Background(), m.ID); err != nil {
 				qlog.Errorx("deleting message from queue after permanent failure", err)
 			}
 			return
 		}
 
-		qup := bstore.QueryDB[Msg](queueDB)
+		qup := bstore.QueryDB[Msg](context.Background(), queueDB)
 		qup.FilterID(m.ID)
 		if _, err := qup.UpdateNonzero(Msg{LastError: errmsg, DialedIPs: m.DialedIPs}); err != nil {
 			qlog.Errorx("storing delivery error", err, mlog.Field("deliveryerror", errmsg))
@@ -534,7 +534,7 @@ func deliver(resolver dns.Resolver, m Msg) {
 	var policy *mtasts.Policy
 	tlsModeDefault := smtpclient.TLSOpportunistic
 	if !effectiveDomain.IsZero() {
-		cidctx := context.WithValue(mox.Context, mlog.CidKey, cid)
+		cidctx := context.WithValue(mox.Shutdown, mlog.CidKey, cid)
 		policy, policyFresh, err = mtastsdb.Get(cidctx, resolver, effectiveDomain)
 		if err != nil {
 			// No need to refuse to deliver if we have some mtasts error.
@@ -586,7 +586,7 @@ func deliver(resolver dns.Resolver, m Msg) {
 		}
 		if ok {
 			nqlog.Info("delivered from queue")
-			if err := queueDelete(m.ID); err != nil {
+			if err := queueDelete(context.Background(), m.ID); err != nil {
 				nqlog.Errorx("deleting message from queue after delivery", err)
 			}
 			return

queue/queue_test.go

@@ -21,6 +21,8 @@ import (
 	"github.com/mjl-/mox/store"
 )
 
+var ctxbg = context.Background()
+
 func tcheck(t *testing.T, err error, msg string) {
 	if err != nil {
 		t.Helper()
@@ -31,7 +33,7 @@ func tcheck(t *testing.T, err error, msg string) {
 func setup(t *testing.T) (*store.Account, func()) {
 	// Prepare config so email can be delivered to mjl@mox.example.
 	os.RemoveAll("../testdata/queue/data")
-	mox.Context = context.Background()
+	mox.Context = ctxbg
 	mox.ConfigStaticPath = "../testdata/queue/mox.conf"
 	mox.MustLoadConfig(false)
 	acc, err := store.OpenAccount("mjl")
@@ -39,11 +41,11 @@ func setup(t *testing.T) (*store.Account, func()) {
 	err = acc.SetPassword("testtest")
 	tcheck(t, err, "set password")
 	switchDone := store.Switchboard()
-	mox.Shutdown, mox.ShutdownCancel = context.WithCancel(context.Background())
+	mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
 	return acc, func() {
 		acc.Close()
 		mox.ShutdownCancel()
-		mox.Shutdown, mox.ShutdownCancel = context.WithCancel(context.Background())
+		mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
 		Shutdown()
 		close(switchDone)
 	}
@@ -71,22 +73,22 @@ func TestQueue(t *testing.T) {
 	err := Init()
 	tcheck(t, err, "queue init")
 
-	msgs, err := List()
+	msgs, err := List(ctxbg)
 	tcheck(t, err, "listing messages in queue")
 	if len(msgs) != 0 {
 		t.Fatalf("got %d messages in queue, expected 0", len(msgs))
 	}
 
 	path := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
-	err = Add(xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, prepareFile(t), nil, true)
+	err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, prepareFile(t), nil, true)
 	tcheck(t, err, "add message to queue for delivery")
 
 	mf2 := prepareFile(t)
-	err = Add(xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, mf2, nil, false)
+	err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, mf2, nil, false)
 	tcheck(t, err, "add message to queue for delivery")
 	os.Remove(mf2.Name())
 
-	msgs, err = List()
+	msgs, err = List(ctxbg)
 	tcheck(t, err, "listing queue")
 	if len(msgs) != 2 {
 		t.Fatalf("got msgs %v, expected 1", msgs)
@@ -95,18 +97,18 @@ func TestQueue(t *testing.T) {
 	if msg.Attempts != 0 {
 		t.Fatalf("msg attempts %d, expected 0", msg.Attempts)
 	}
-	n, err := Drop(msgs[1].ID, "", "")
+	n, err := Drop(ctxbg, msgs[1].ID, "", "")
 	tcheck(t, err, "drop")
 	if n != 1 {
 		t.Fatalf("dropped %d, expected 1", n)
 	}
 
-	next := nextWork(nil)
+	next := nextWork(ctxbg, nil)
 	if next > 0 {
 		t.Fatalf("nextWork in %s, should be now", next)
 	}
 	busy := map[string]struct{}{"mox.example": {}}
-	if x := nextWork(busy); x != 24*time.Hour {
+	if x := nextWork(ctxbg, busy); x != 24*time.Hour {
 		t.Fatalf("nextWork in %s for busy domain, should be in 24 hours", x)
 	}
 	if nn := launchWork(nil, busy); nn != 0 {
@@ -133,7 +135,7 @@ func TestQueue(t *testing.T) {
 	case <-dialed:
 		i := 0
 		for {
-			m, err := bstore.QueryDB[Msg](queueDB).Get()
+			m, err := bstore.QueryDB[Msg](ctxbg, queueDB).Get()
 			tcheck(t, err, "get")
 			if m.Attempts == 1 {
 				break
@@ -149,11 +151,11 @@ func TestQueue(t *testing.T) {
 	}
 	<-deliveryResult // Deliver sends here.
 
-	_, err = OpenMessage(msg.ID + 1)
+	_, err = OpenMessage(ctxbg, msg.ID+1)
 	if err != bstore.ErrAbsent {
 		t.Fatalf("OpenMessage, got %v, expected ErrAbsent", err)
 	}
-	reader, err := OpenMessage(msg.ID)
+	reader, err := OpenMessage(ctxbg, msg.ID)
 	tcheck(t, err, "open message")
 	defer reader.Close()
 	msgbuf, err := io.ReadAll(reader)
@@ -162,12 +164,12 @@ func TestQueue(t *testing.T) {
 		t.Fatalf("message mismatch, got %q, expected %q", string(msgbuf), testmsg)
 	}
 
-	n, err = Kick(msg.ID+1, "", "")
+	n, err = Kick(ctxbg, msg.ID+1, "", "")
 	tcheck(t, err, "kick")
 	if n != 0 {
 		t.Fatalf("kick %d, expected 0", n)
 	}
-	n, err = Kick(msg.ID, "", "")
+	n, err = Kick(ctxbg, msg.ID, "", "")
 	tcheck(t, err, "kick")
 	if n != 1 {
 		t.Fatalf("kicked %d, expected 1", n)
@@ -215,7 +217,7 @@ func TestQueue(t *testing.T) {
 		case <-smtpdone:
 			i := 0
 			for {
-				xmsgs, err := List()
+				xmsgs, err := List(ctxbg)
 				tcheck(t, err, "list queue")
 				if len(xmsgs) == 0 {
 					break
@@ -235,10 +237,10 @@ func TestQueue(t *testing.T) {
 	<-deliveryResult // Deliver sends here.
 
 	// Add another message that we'll fail to deliver entirely.
-	err = Add(xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, prepareFile(t), nil, true)
+	err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, prepareFile(t), nil, true)
 	tcheck(t, err, "add message to queue for delivery")
 
-	msgs, err = List()
+	msgs, err = List(ctxbg)
 	tcheck(t, err, "list queue")
 	if len(msgs) != 1 {
 		t.Fatalf("queue has %d messages, expected 1", len(msgs))
@@ -283,7 +285,7 @@ func TestQueue(t *testing.T) {
 	for i := 1; i < 8; i++ {
 		go func() { <-deliveryResult }() // Deliver sends here.
 		deliver(resolver, msg)
-		err = queueDB.Get(&msg)
+		err = queueDB.Get(ctxbg, &msg)
 		tcheck(t, err, "get msg")
 		if msg.Attempts != i {
 			t.Fatalf("got attempt %d, expected %d", msg.Attempts, i)
@@ -306,7 +308,7 @@ func TestQueue(t *testing.T) {
 	// Trigger final failure.
 	go func() { <-deliveryResult }() // Deliver sends here.
 	deliver(resolver, msg)
-	err = queueDB.Get(&msg)
+	err = queueDB.Get(ctxbg, &msg)
 	if err != bstore.ErrAbsent {
 		t.Fatalf("attempt to fetch delivered and removed message from queue, got err %v, expected ErrAbsent", err)
 	}
@@ -343,7 +345,7 @@ func TestQueueStart(t *testing.T) {
 	defer func() {
 		mox.ShutdownCancel()
 		<-done
-		mox.Shutdown, mox.ShutdownCancel = context.WithCancel(context.Background())
+		mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
 	}()
 	err := Start(resolver, done)
 	tcheck(t, err, "queue start")
@@ -369,7 +371,7 @@ func TestQueueStart(t *testing.T) {
 	}
 
 	path := smtp.Path{Localpart: "mjl", IPDomain: dns.IPDomain{Domain: dns.Domain{ASCII: "mox.example"}}}
-	err = Add(xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, prepareFile(t), nil, true)
+	err = Add(ctxbg, xlog, "mjl", path, path, false, false, int64(len(testmsg)), nil, prepareFile(t), nil, true)
 	tcheck(t, err, "add message to queue for delivery")
 	checkDialed(true)
 
@@ -378,7 +380,7 @@ func TestQueueStart(t *testing.T) {
 	checkDialed(false)
 
 	// Kick for real, should see another attempt.
-	n, err := Kick(0, "mox.example", "")
+	n, err := Kick(ctxbg, 0, "mox.example", "")
 	tcheck(t, err, "kick queue")
 	if n != 1 {
 		t.Fatalf("kick changed %d messages, expected 1", n)
@@ -402,7 +404,7 @@ func TestWriteFile(t *testing.T) {
 }
 
 func TestGatherHosts(t *testing.T) {
-	mox.Context = context.Background()
+	mox.Context = ctxbg
 
 	// Test basic MX lookup case, but also following CNAME, detecting CNAME loops and
 	// having a CNAME limit, connecting directly to a host, and domain that does not
@@ -524,11 +526,11 @@ func TestDialHost(t *testing.T) {
 	}
 
 	m := Msg{DialedIPs: map[string][]net.IP{}}
-	_, ip, dualstack, err := dialHost(context.Background(), xlog, resolver, ipdomain("dualstack.example"), &m)
+	_, ip, dualstack, err := dialHost(ctxbg, xlog, resolver, ipdomain("dualstack.example"), &m)
 	if err != nil || ip.String() != "10.0.0.1" || !dualstack {
 		t.Fatalf("expected err nil, address 10.0.0.1, dualstack true, got %v %v %v", err, ip, dualstack)
 	}
-	_, ip, dualstack, err = dialHost(context.Background(), xlog, resolver, ipdomain("dualstack.example"), &m)
+	_, ip, dualstack, err = dialHost(ctxbg, xlog, resolver, ipdomain("dualstack.example"), &m)
 	if err != nil || ip.String() != "2001:db8::1" || !dualstack {
 		t.Fatalf("expected err nil, address 2001:db8::1, dualstack true, got %v %v %v", err, ip, dualstack)
 	}

smtpserver/analyze.go

@@ -161,7 +161,7 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
 	var reason string
 	var err error
 	d.acc.WithRLock(func() {
-		err = d.acc.DB.Read(func(tx *bstore.Tx) error {
+		err = d.acc.DB.Read(ctx, func(tx *bstore.Tx) error {
 			// Set message MailboxID to which mail will be delivered. Reputation is
 			// per-mailbox. If referenced mailbox is not found (e.g. does not yet exist), we
 			// can still determine a reputation because we also base it on outgoing
@@ -251,13 +251,13 @@ func analyze(ctx context.Context, log *mlog.Log, resolver dns.Resolver, d delive
 	reason = reasonNoBadSignals
 	accept := true
 	var junkSubjectpass bool
-	f, jf, err := d.acc.OpenJunkFilter(log)
+	f, jf, err := d.acc.OpenJunkFilter(ctx, log)
 	if err == nil {
 		defer func() {
 			err := f.Close()
 			log.Check(err, "closing junkfilter")
 		}()
-		contentProb, _, _, _, err := f.ClassifyMessageReader(store.FileMsgReader(d.m.MsgPrefix, d.dataFile), d.m.Size)
+		contentProb, _, _, _, err := f.ClassifyMessageReader(ctx, store.FileMsgReader(d.m.MsgPrefix, d.dataFile), d.m.Size)
 		if err != nil {
 			log.Errorx("testing for spam", err)
 			return reject(smtp.C451LocalErr, smtp.SeSys3Other0, "error processing", err, reasonJunkClassifyError)

smtpserver/dsn.go

@@ -1,6 +1,7 @@
 package smtpserver
 
 import (
+	"context"
 	"fmt"
 	"os"
 
@@ -11,7 +12,7 @@ import (
 )
 
 // compose dsn message and add it to the queue for delivery to rcptTo.
-func queueDSN(c *conn, rcptTo smtp.Path, m dsn.Message) error {
+func queueDSN(ctx context.Context, c *conn, rcptTo smtp.Path, m dsn.Message) error {
 	buf, err := m.Compose(c.log, false)
 	if err != nil {
 		return err
@@ -45,7 +46,7 @@ func queueDSN(c *conn, rcptTo smtp.Path, m dsn.Message) error {
 	// ../rfc/3464:433
 	const has8bit = false
 	const smtputf8 = false
-	if err := queue.Add(c.log, "", smtp.Path{}, rcptTo, has8bit, smtputf8, int64(len(buf)), nil, f, bufUTF8, true); err != nil {
+	if err := queue.Add(ctx, c.log, "", smtp.Path{}, rcptTo, has8bit, smtputf8, int64(len(buf)), nil, f, bufUTF8, true); err != nil {
 		return err
 	}
 	err = f.Close()

smtpserver/fuzz_test.go

@@ -1,7 +1,6 @@
 package smtpserver
 
 import (
-	"context"
 	"fmt"
 	"net"
 	"os"
@@ -30,7 +29,7 @@ func FuzzServer(f *testing.F) {
 	f.Add("NOOP")
 	f.Add("QUIT")
 
-	mox.Context = context.Background()
+	mox.Context = ctxbg
 	mox.ConfigStaticPath = "../testdata/smtp/mox.conf"
 	mox.MustLoadConfig(false)
 	dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir)

smtpserver/rejects.go

@@ -2,6 +2,7 @@ package smtpserver
 
 import (
 	"bytes"
+	"context"
 	"crypto/sha256"
 	"fmt"
 	"io"
@@ -40,7 +41,7 @@ func rejectPresent(log *mlog.Log, acc *store.Account, rejectsMailbox string, m *
 	var exists bool
 	var err error
 	acc.WithRLock(func() {
-		err = acc.DB.Read(func(tx *bstore.Tx) error {
+		err = acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
 			mbq := bstore.QueryTx[store.Mailbox](tx)
 			mbq.FilterNonzero(store.Mailbox{Name: rejectsMailbox})
 			mb, err := mbq.Get()

smtpserver/reputation.go

@@ -272,16 +272,8 @@ func reputation(tx *bstore.Tx, log *mlog.Log, m *store.Message) (rjunk *bool, rc
 	dkimspfsignals := []float64{}
 	dkimspfmsgs := 0
 	for _, dom := range m.DKIMDomains {
-		// todo: should get dkimdomains in an index for faster lookup. bstore does not yet support "in" indexes.
 		q := messageQuery(nil, year/2, 50)
-		q.FilterFn(func(m store.Message) bool {
-			for _, d := range m.DKIMDomains {
-				if d == dom {
-					return true
-				}
-			}
-			return false
-		})
+		q.FilterIn("DKIMDomains", dom)
 		msgs := xmessageList(q, "dkimdomain")
 		if len(msgs) > 0 {
 			nspam := 0

smtpserver/reputation_test.go

@@ -1,7 +1,6 @@
 package smtpserver
 
 import (
-	"context"
 	"fmt"
 	"net"
 	"os"
@@ -77,7 +76,7 @@ func TestReputation(t *testing.T) {
 
 			MsgFromLocalpart: msgFrom.Localpart,
 			MsgFromDomain:    msgFrom.Domain.Name(),
-			MsgFromOrgDomain: publicsuffix.Lookup(context.Background(), msgFrom.Domain).Name(),
+			MsgFromOrgDomain: publicsuffix.Lookup(ctxbg, msgFrom.Domain).Name(),
 
 			MailFromValidated: mailfromValid,
 			EHLOValidated:     ehloValid,
@@ -103,11 +102,11 @@ func TestReputation(t *testing.T) {
 		p := "../testdata/smtpserver-reputation.db"
 		defer os.Remove(p)
 
-		db, err := bstore.Open(p, &bstore.Options{Timeout: 5 * time.Second}, store.Message{}, store.Recipient{}, store.Mailbox{})
+		db, err := bstore.Open(ctxbg, p, &bstore.Options{Timeout: 5 * time.Second}, store.Message{}, store.Recipient{}, store.Mailbox{})
 		tcheck(t, err, "open db")
 		defer db.Close()
 
-		err = db.Write(func(tx *bstore.Tx) error {
+		err = db.Write(ctxbg, func(tx *bstore.Tx) error {
 			err = tx.Insert(&store.Mailbox{ID: 1, Name: "Inbox"})
 			tcheck(t, err, "insert into db")
 
@@ -117,7 +116,7 @@ func TestReputation(t *testing.T) {
 
 				rcptToDomain, err := dns.ParseDomain(hm.RcptToDomain)
 				tcheck(t, err, "parse rcptToDomain")
-				rcptToOrgDomain := publicsuffix.Lookup(context.Background(), rcptToDomain)
+				rcptToOrgDomain := publicsuffix.Lookup(ctxbg, rcptToDomain)
 				r := store.Recipient{MessageID: hm.ID, Localpart: hm.RcptToLocalpart, Domain: hm.RcptToDomain, OrgDomain: rcptToOrgDomain.Name(), Sent: hm.Received}
 				err = tx.Insert(&r)
 				tcheck(t, err, "insert recipient")
@@ -130,7 +129,7 @@ func TestReputation(t *testing.T) {
 		var isjunk *bool
 		var conclusive bool
 		var method reputationMethod
-		err = db.Read(func(tx *bstore.Tx) error {
+		err = db.Read(ctxbg, func(tx *bstore.Tx) error {
 			var err error
 			isjunk, conclusive, method, err = reputation(tx, xlog, &m)
 			return err

smtpserver/server.go

@@ -1028,7 +1028,7 @@ func (c *conn) cmdAuth(p *parser) {
 		}()
 		var ipadhash, opadhash hash.Hash
 		acc.WithRLock(func() {
-			err := acc.DB.Read(func(tx *bstore.Tx) error {
+			err := acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
 				password, err := bstore.QueryTx[store.Password](tx).Get()
 				if err == bstore.ErrAbsent {
 					xsmtpUserErrorf(smtp.C535AuthBadCreds, smtp.SePol7AuthBadCreds8, "bad user/pass")
@@ -1101,7 +1101,7 @@ func (c *conn) cmdAuth(p *parser) {
 		}
 		var xscram store.SCRAM
 		acc.WithRLock(func() {
-			err := acc.DB.Read(func(tx *bstore.Tx) error {
+			err := acc.DB.Read(context.TODO(), func(tx *bstore.Tx) error {
 				password, err := bstore.QueryTx[store.Password](tx).Get()
 				if authVariant == "scram-sha-1" {
 					xscram = password.SCRAMSHA1
@@ -1672,7 +1672,7 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
 	// Limit damage to the internet and our reputation in case of account compromise by
 	// limiting the max number of messages sent in a 24 hour window, both total number
 	// of messages and number of first-time recipients.
-	err = c.account.DB.Read(func(tx *bstore.Tx) error {
+	err = c.account.DB.Read(ctx, func(tx *bstore.Tx) error {
 		conf, _ := c.account.Conf()
 		msgmax := conf.MaxOutgoingMessagesPerDay
 		if msgmax == 0 {
@@ -1828,7 +1828,7 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
 				metricSubmission.WithLabelValues("ok").Inc()
 				c.log.Info("submitted message delivered", mlog.Field("mailfrom", *c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo), mlog.Field("smtputf8", c.smtputf8), mlog.Field("msgsize", msgSize))
 
-				err := c.account.DB.Insert(&store.Outgoing{Recipient: rcptAcc.rcptTo.XString(true)})
+				err := c.account.DB.Insert(ctx, &store.Outgoing{Recipient: rcptAcc.rcptTo.XString(true)})
 				xcheckf(err, "adding outgoing message")
 			}
 		})
@@ -1852,7 +1852,7 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
 			}
 
 			msgSize := int64(len(xmsgPrefix)) + msgWriter.Size
-			if err := queue.Add(c.log, c.account.Name, *c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.smtputf8, msgSize, xmsgPrefix, dataFile, nil, i == len(c.recipients)-1); err != nil {
+			if err := queue.Add(ctx, c.log, c.account.Name, *c.mailFrom, rcptAcc.rcptTo, msgWriter.Has8bit, c.smtputf8, msgSize, xmsgPrefix, dataFile, nil, i == len(c.recipients)-1); err != nil {
 				// Aborting the transaction is not great. But continuing and generating DSNs will
 				// probably result in errors as well...
 				metricSubmission.WithLabelValues("queueerror").Inc()
@@ -1862,7 +1862,7 @@ func (c *conn) submit(ctx context.Context, recvHdrFor func(string) string, msgWr
 			metricSubmission.WithLabelValues("ok").Inc()
 			c.log.Info("message queued for delivery", mlog.Field("mailfrom", *c.mailFrom), mlog.Field("rcptto", rcptAcc.rcptTo), mlog.Field("smtputf8", c.smtputf8), mlog.Field("msgsize", msgSize))
 
-			err := c.account.DB.Insert(&store.Outgoing{Recipient: rcptAcc.rcptTo.XString(true)})
+			err := c.account.DB.Insert(ctx, &store.Outgoing{Recipient: rcptAcc.rcptTo.XString(true)})
 			xcheckf(err, "adding outgoing message")
 		}
 	}
@@ -2267,7 +2267,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 		// account. They may fill up the mailbox, either with messages that have to be
 		// purged, or by filling the disk. We check both cases for IP's and networks.
 		var rateError bool // Whether returned error represents a rate error.
-		err = acc.DB.Read(func(tx *bstore.Tx) (retErr error) {
+		err = acc.DB.Read(ctx, func(tx *bstore.Tx) (retErr error) {
 			now := time.Now()
 			defer func() {
 				log.Debugx("checking message and size delivery rates", retErr, mlog.Field("duration", time.Since(now)))
@@ -2575,7 +2575,7 @@ func (c *conn) deliver(ctx context.Context, recvHdrFor func(string) string, msgW
 
 		if Localserve {
 			c.log.Error("not queueing dsn for incoming delivery due to localserve")
-		} else if err := queueDSN(c, *c.mailFrom, dsnMsg); err != nil {
+		} else if err := queueDSN(context.TODO(), c, *c.mailFrom, dsnMsg); err != nil {
 			metricServerErrors.WithLabelValues("queuedsn").Inc()
 			c.log.Errorx("queuing DSN for incoming delivery, no DSN sent", err)
 		}

smtpserver/server_test.go

@@ -39,6 +39,8 @@ import (
 	"github.com/mjl-/mox/tlsrptdb"
 )
 
+var ctxbg = context.Background()
+
 func init() {
 	// Don't make tests slow.
 	badClientDelay = 0
@@ -88,7 +90,7 @@ func newTestServer(t *testing.T, configPath string, resolver dns.Resolver) *test
 
 	ts := testserver{t: t, cid: 1, resolver: resolver, tlsmode: smtpclient.TLSOpportunistic}
 
-	mox.Context = context.Background()
+	mox.Context = ctxbg
 	mox.ConfigStaticPath = configPath
 	mox.MustLoadConfig(false)
 	dataDir := mox.ConfigDirPath(mox.Conf.Static.DataDir)
@@ -138,7 +140,7 @@ func (ts *testserver) run(fn func(helloErr error, client *smtpclient.Client)) {
 		authLine = fmt.Sprintf("AUTH PLAIN %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("\u0000%s\u0000%s", ts.user, ts.pass))))
 	}
 
-	client, err := smtpclient.New(context.Background(), xlog.WithCid(ts.cid-1), clientConn, ts.tlsmode, "mox.example", authLine)
+	client, err := smtpclient.New(ctxbg, xlog.WithCid(ts.cid-1), clientConn, ts.tlsmode, "mox.example", authLine)
 	if err != nil {
 		clientConn.Close()
 	} else {
@@ -199,7 +201,7 @@ func TestSubmission(t *testing.T) {
 			mailFrom := "mjl@mox.example"
 			rcptTo := "remote@example.org"
 			if err == nil {
-				err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(submitMessage)), strings.NewReader(submitMessage), false, false)
+				err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(submitMessage)), strings.NewReader(submitMessage), false, false)
 			}
 			var cerr smtpclient.Error
 			if expErr == nil && err != nil || expErr != nil && (err == nil || !errors.As(err, &cerr) || cerr.Secode != expErr.Secode) {
@@ -230,7 +232,7 @@ func TestDelivery(t *testing.T) {
 		mailFrom := "remote@example.org"
 		rcptTo := "mjl@127.0.0.10"
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		}
 		var cerr smtpclient.Error
 		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C550MailboxUnavail {
@@ -242,7 +244,7 @@ func TestDelivery(t *testing.T) {
 		mailFrom := "remote@example.org"
 		rcptTo := "mjl@test.example" // Not configured as destination.
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		}
 		var cerr smtpclient.Error
 		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C550MailboxUnavail {
@@ -254,7 +256,7 @@ func TestDelivery(t *testing.T) {
 		mailFrom := "remote@example.org"
 		rcptTo := "unknown@mox.example" // User unknown.
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		}
 		var cerr smtpclient.Error
 		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C550MailboxUnavail {
@@ -266,7 +268,7 @@ func TestDelivery(t *testing.T) {
 		mailFrom := "remote@example.org"
 		rcptTo := "mjl@mox.example"
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		}
 		var cerr smtpclient.Error
 		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
@@ -280,7 +282,7 @@ func TestDelivery(t *testing.T) {
 		mailFrom := "remote@example.org"
 		rcptTo := "mjl@mox.example"
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		}
 		tcheck(t, err, "deliver to remote")
 
@@ -319,12 +321,12 @@ func tretrain(t *testing.T, acc *store.Account) {
 	bloomPath := filepath.Join(basePath, acc.Name, "junkfilter.bloom")
 	os.Remove(dbPath)
 	os.Remove(bloomPath)
-	jf, _, err := acc.OpenJunkFilter(xlog)
+	jf, _, err := acc.OpenJunkFilter(ctxbg, xlog)
 	tcheck(t, err, "open junk filter")
 	defer jf.Close()
 
 	// Fetch messags to retrain on.
-	q := bstore.QueryDB[store.Message](acc.DB)
+	q := bstore.QueryDB[store.Message](ctxbg, acc.DB)
 	q.FilterFn(func(m store.Message) bool {
 		return m.Flags.Junk || m.Flags.Notjunk
 	})
@@ -339,7 +341,7 @@ func tretrain(t *testing.T, acc *store.Account) {
 		tcheck(t, err, "open message")
 		r := store.FileMsgReader(m.MsgPrefix, f)
 
-		jf.TrainMessage(r, m.Size, ham)
+		jf.TrainMessage(ctxbg, r, m.Size, ham)
 
 		err = r.Close()
 		tcheck(t, err, "close message")
@@ -388,11 +390,11 @@ func TestSpam(t *testing.T) {
 
 	checkRejectsCount := func(expect int) {
 		t.Helper()
-		q := bstore.QueryDB[store.Mailbox](ts.acc.DB)
+		q := bstore.QueryDB[store.Mailbox](ctxbg, ts.acc.DB)
 		q.FilterNonzero(store.Mailbox{Name: "Rejects"})
 		mb, err := q.Get()
 		tcheck(t, err, "get rejects mailbox")
-		qm := bstore.QueryDB[store.Message](ts.acc.DB)
+		qm := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB)
 		qm.FilterNonzero(store.Message{MailboxID: mb.ID})
 		n, err := qm.Count()
 		tcheck(t, err, "count messages in rejects mailbox")
@@ -406,7 +408,7 @@ func TestSpam(t *testing.T) {
 		mailFrom := "remote@example.org"
 		rcptTo := "mjl@mox.example"
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		}
 		var cerr smtpclient.Error
 		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
@@ -418,7 +420,7 @@ func TestSpam(t *testing.T) {
 	})
 
 	// Mark the messages as having good reputation.
-	q := bstore.QueryDB[store.Message](ts.acc.DB)
+	q := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB)
 	_, err := q.UpdateFields(map[string]any{"Junk": false, "Notjunk": true})
 	tcheck(t, err, "update junkiness")
 
@@ -427,7 +429,7 @@ func TestSpam(t *testing.T) {
 		mailFrom := "remote@example.org"
 		rcptTo := "mjl@mox.example"
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		}
 		tcheck(t, err, "deliver")
 
@@ -437,7 +439,7 @@ func TestSpam(t *testing.T) {
 
 	// Undo dmarc pass, mark messages as junk, and train the filter.
 	resolver.TXT = nil
-	q = bstore.QueryDB[store.Message](ts.acc.DB)
+	q = bstore.QueryDB[store.Message](ctxbg, ts.acc.DB)
 	_, err = q.UpdateFields(map[string]any{"Junk": true, "Notjunk": false})
 	tcheck(t, err, "update junkiness")
 	tretrain(t, ts.acc)
@@ -447,7 +449,7 @@ func TestSpam(t *testing.T) {
 		mailFrom := "remote@example.org"
 		rcptTo := "mjl@mox.example"
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		}
 		var cerr smtpclient.Error
 		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
@@ -488,7 +490,7 @@ func TestDMARCSent(t *testing.T) {
 		mailFrom := "remote@example.org"
 		rcptTo := "mjl@mox.example"
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		}
 		var cerr smtpclient.Error
 		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
@@ -499,7 +501,7 @@ func TestDMARCSent(t *testing.T) {
 	// Insert a message that we sent to the address that is about to send to us.
 	var sentMsg store.Message
 	tinsertmsg(t, ts.acc, "Sent", &sentMsg, deliverMessage)
-	err := ts.acc.DB.Insert(&store.Recipient{MessageID: sentMsg.ID, Localpart: "remote", Domain: "example.org", OrgDomain: "example.org", Sent: time.Now()})
+	err := ts.acc.DB.Insert(ctxbg, &store.Recipient{MessageID: sentMsg.ID, Localpart: "remote", Domain: "example.org", OrgDomain: "example.org", Sent: time.Now()})
 	tcheck(t, err, "inserting message recipient")
 
 	// We should now be accepting the message because we recently sent a message.
@@ -507,7 +509,7 @@ func TestDMARCSent(t *testing.T) {
 		mailFrom := "remote@example.org"
 		rcptTo := "mjl@mox.example"
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		}
 		tcheck(t, err, "deliver")
 	})
@@ -540,7 +542,7 @@ func TestBlocklistedSubjectpass(t *testing.T) {
 		mailFrom := "remote@example.org"
 		rcptTo := "mjl@mox.example"
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		}
 		var cerr smtpclient.Error
 		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C451LocalErr {
@@ -559,7 +561,7 @@ func TestBlocklistedSubjectpass(t *testing.T) {
 		mailFrom := "remote@example.org"
 		rcptTo := "mjl@mox.example"
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		}
 		var cerr smtpclient.Error
 		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C550MailboxUnavail {
@@ -577,7 +579,7 @@ func TestBlocklistedSubjectpass(t *testing.T) {
 		rcptTo := "mjl@mox.example"
 		passMessage := strings.Replace(deliverMessage, "Subject: test", "Subject: test "+pass, 1)
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(passMessage)), strings.NewReader(passMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(passMessage)), strings.NewReader(passMessage), false, false)
 		}
 		tcheck(t, err, "deliver with subjectpass")
 	})
@@ -619,11 +621,11 @@ func TestDMARCReport(t *testing.T) {
 			msg := msgb.String()
 
 			if err == nil {
-				err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(msg)), strings.NewReader(msg), false, false)
+				err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(msg)), strings.NewReader(msg), false, false)
 			}
 			tcheck(t, err, "deliver")
 
-			records, err := dmarcdb.Records(context.Background())
+			records, err := dmarcdb.Records(ctxbg)
 			tcheck(t, err, "dmarcdb records")
 			if len(records) != n {
 				t.Fatalf("got %d dmarcdb records, expected %d or more", len(records), n)
@@ -736,16 +738,16 @@ func TestTLSReport(t *testing.T) {
 			tcheck(t, xerr, "write msg")
 			msg := msgb.String()
 
-			headers, xerr := dkim.Sign(context.Background(), "remote", dns.Domain{ASCII: "example.org"}, dkimConf, false, strings.NewReader(msg))
+			headers, xerr := dkim.Sign(ctxbg, "remote", dns.Domain{ASCII: "example.org"}, dkimConf, false, strings.NewReader(msg))
 			tcheck(t, xerr, "dkim sign")
 			msg = headers + msg
 
 			if err == nil {
-				err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(msg)), strings.NewReader(msg), false, false)
+				err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(msg)), strings.NewReader(msg), false, false)
 			}
 			tcheck(t, err, "deliver")
 
-			records, err := tlsrptdb.Records(context.Background())
+			records, err := tlsrptdb.Records(ctxbg)
 			tcheck(t, err, "tlsrptdb records")
 			if len(records) != n {
 				t.Fatalf("got %d tlsrptdb records, expected %d", len(records), n)
@@ -840,11 +842,11 @@ func TestRatelimitDelivery(t *testing.T) {
 		mailFrom := "remote@example.org"
 		rcptTo := "mjl@mox.example"
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		}
 		tcheck(t, err, "deliver to remote")
 
-		err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+		err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		var cerr smtpclient.Error
 		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C452StorageFull {
 			t.Fatalf("got err %v, expected smtpclient error with code 452 for storage full", err)
@@ -857,7 +859,7 @@ func TestRatelimitDelivery(t *testing.T) {
 	// Message was already delivered once. We'll do another one. But the 3rd will fail.
 	// We need the actual size with prepended headers, since that is used in the
 	// calculations.
-	msg, err := bstore.QueryDB[store.Message](ts.acc.DB).Get()
+	msg, err := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB).Get()
 	if err != nil {
 		t.Fatalf("getting delivered message for its size: %v", err)
 	}
@@ -869,11 +871,11 @@ func TestRatelimitDelivery(t *testing.T) {
 		mailFrom := "remote@example.org"
 		rcptTo := "mjl@mox.example"
 		if err == nil {
-			err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+			err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		}
 		tcheck(t, err, "deliver to remote")
 
-		err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+		err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 		var cerr smtpclient.Error
 		if err == nil || !errors.As(err, &cerr) || cerr.Code != smtp.C452StorageFull {
 			t.Fatalf("got err %v, expected smtpclient error with code 452 for storage full", err)
@@ -933,7 +935,7 @@ func TestLimitOutgoing(t *testing.T) {
 	ts.pass = "testtest"
 	ts.submission = true
 
-	err := ts.acc.DB.Insert(&store.Outgoing{Recipient: "a@other.example", Submitted: time.Now().Add(-24*time.Hour - time.Minute)})
+	err := ts.acc.DB.Insert(ctxbg, &store.Outgoing{Recipient: "a@other.example", Submitted: time.Now().Add(-24*time.Hour - time.Minute)})
 	tcheck(t, err, "inserting outgoing/recipient past 24h window")
 
 	testSubmit := func(rcptTo string, expErr *smtpclient.Error) {
@@ -942,7 +944,7 @@ func TestLimitOutgoing(t *testing.T) {
 			t.Helper()
 			mailFrom := "mjl@mox.example"
 			if err == nil {
-				err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(submitMessage)), strings.NewReader(submitMessage), false, false)
+				err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(submitMessage)), strings.NewReader(submitMessage), false, false)
 			}
 			var cerr smtpclient.Error
 			if expErr == nil && err != nil || expErr != nil && (err == nil || !errors.As(err, &cerr) || cerr.Secode != expErr.Secode) {
@@ -979,7 +981,7 @@ func TestCatchall(t *testing.T) {
 			t.Helper()
 			mailFrom := "mjl@other.example"
 			if err == nil {
-				err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(submitMessage)), strings.NewReader(submitMessage), false, false)
+				err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(submitMessage)), strings.NewReader(submitMessage), false, false)
 			}
 			var cerr smtpclient.Error
 			if expErr == nil && err != nil || expErr != nil && (err == nil || !errors.As(err, &cerr) || cerr.Secode != expErr.Secode) {
@@ -993,14 +995,14 @@ func TestCatchall(t *testing.T) {
 	testDeliver("MJL+TEST@mox.example", nil) // Again, and case insensitive.
 	testDeliver("unknown@mox.example", nil)  // Catchall address, to account catchall.
 
-	n, err := bstore.QueryDB[store.Message](ts.acc.DB).Count()
+	n, err := bstore.QueryDB[store.Message](ctxbg, ts.acc.DB).Count()
 	tcheck(t, err, "checking delivered messages")
 	tcompare(t, n, 3)
 
 	acc, err := store.OpenAccount("catchall")
 	tcheck(t, err, "open account")
 	defer acc.Close()
-	n, err = bstore.QueryDB[store.Message](acc.DB).Count()
+	n, err = bstore.QueryDB[store.Message](ctxbg, acc.DB).Count()
 	tcheck(t, err, "checking delivered messages to catchall account")
 	tcompare(t, n, 1)
 }
@@ -1072,21 +1074,21 @@ test email
 
 			rcptTo := "remote@example.org"
 			if err == nil {
-				err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(msg)), strings.NewReader(msg), false, false)
+				err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(msg)), strings.NewReader(msg), false, false)
 			}
 			tcheck(t, err, "deliver")
 
-			msgs, err := queue.List()
+			msgs, err := queue.List(ctxbg)
 			tcheck(t, err, "listing queue")
 			n++
 			tcompare(t, len(msgs), n)
 			sort.Slice(msgs, func(i, j int) bool {
 				return msgs[i].ID > msgs[j].ID
 			})
-			f, err := queue.OpenMessage(msgs[0].ID)
+			f, err := queue.OpenMessage(ctxbg, msgs[0].ID)
 			tcheck(t, err, "open message in queue")
 			defer f.Close()
-			results, err := dkim.Verify(context.Background(), resolver, false, dkim.DefaultPolicy, f, false)
+			results, err := dkim.Verify(ctxbg, resolver, false, dkim.DefaultPolicy, f, false)
 			tcheck(t, err, "verifying dkim message")
 			tcompare(t, len(results), 1)
 			tcompare(t, results[0].Status, dkim.StatusPass)
@@ -1117,7 +1119,7 @@ func TestPostmaster(t *testing.T) {
 			t.Helper()
 			mailFrom := "mjl@other.example"
 			if err == nil {
-				err = client.Deliver(context.Background(), mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
+				err = client.Deliver(ctxbg, mailFrom, rcptTo, int64(len(deliverMessage)), strings.NewReader(deliverMessage), false, false)
 			}
 			var cerr smtpclient.Error
 			if expErr == nil && err != nil || expErr != nil && (err == nil || !errors.As(err, &cerr) || cerr.Code != expErr.Code || cerr.Secode != expErr.Secode) {

store/account.go

@@ -278,7 +278,7 @@ type Message struct {
 	MsgFromValidation  Validation // Desirable validations: Strict, DMARC, Relaxed. Will not be just Pass.
 
 	// todo: needs an "in" index, which bstore does not yet support. for performance while checking reputation.
-	DKIMDomains []string // Domains with verified DKIM signatures. Unicode string.
+	DKIMDomains []string `bstore:"index DKIMDomains+Received"` // Domains with verified DKIM signatures. Unicode string.
 
 	// Value of Message-Id header. Only set for messages that were
 	// delivered to the rejects mailbox. For ensuring such messages are
@@ -455,7 +455,7 @@ func openAccount(name string) (a *Account, rerr error) {
 		os.MkdirAll(dir, 0770)
 	}
 
-	db, err := bstore.Open(dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{})
+	db, err := bstore.Open(context.TODO(), dbpath, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, NextUIDValidity{}, Message{}, Recipient{}, Mailbox{}, Subscription{}, Outgoing{}, Password{}, Subjectpass{})
 	if err != nil {
 		return nil, err
 	}
@@ -484,7 +484,7 @@ func openAccount(name string) (a *Account, rerr error) {
 }
 
 func initAccount(db *bstore.DB) error {
-	return db.Write(func(tx *bstore.Tx) error {
+	return db.Write(context.TODO(), func(tx *bstore.Tx) error {
 		uidvalidity := InitialUIDValidity()
 
 		mailboxes := InitialMailboxes
@@ -700,7 +700,7 @@ func (a *Account) DeliverMessage(log *mlog.Log, tx *bstore.Tx, m *Message, msgFi
 
 	if !notrain && m.NeedsTraining() {
 		l := []Message{*m}
-		if err := a.RetrainMessages(log, tx, l, false); err != nil {
+		if err := a.RetrainMessages(context.TODO(), log, tx, l, false); err != nil {
 			return fmt.Errorf("training junkfilter: %w", err)
 		}
 		*m = l[0]
@@ -739,7 +739,7 @@ func (a *Account) SetPassword(password string) error {
 		return fmt.Errorf("generating password hash: %w", err)
 	}
 
-	err = a.DB.Write(func(tx *bstore.Tx) error {
+	err = a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
 		if _, err := bstore.QueryTx[Password](tx).Delete(); err != nil {
 			return fmt.Errorf("deleting existing password: %v", err)
 		}
@@ -793,7 +793,7 @@ func (a *Account) SetPassword(password string) error {
 // Subjectpass returns the signing key for use with subjectpass for the given
 // email address with canonical localpart.
 func (a *Account) Subjectpass(email string) (key string, err error) {
-	return key, a.DB.Write(func(tx *bstore.Tx) error {
+	return key, a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
 		v := Subjectpass{Email: email}
 		err := tx.Get(&v)
 		if err == nil {
@@ -1036,7 +1036,7 @@ func (a *Account) Deliver(log *mlog.Log, dest config.Destination, m *Message, ms
 // Message delivery and possible mailbox creation are broadcasted.
 func (a *Account) DeliverMailbox(log *mlog.Log, mailbox string, m *Message, msgFile *os.File, consumeFile bool) error {
 	var changes []Change
-	err := a.DB.Write(func(tx *bstore.Tx) error {
+	err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
 		mb, chl, err := a.MailboxEnsure(tx, mailbox, true)
 		if err != nil {
 			return fmt.Errorf("ensuring mailbox: %w", err)
@@ -1075,7 +1075,7 @@ func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasS
 		}
 	}()
 
-	err := a.DB.Write(func(tx *bstore.Tx) error {
+	err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
 		mb, err := a.MailboxFind(tx, rejectsMailbox)
 		if err != nil {
 			return fmt.Errorf("finding mailbox: %w", err)
@@ -1096,7 +1096,7 @@ func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasS
 			return fmt.Errorf("listing old messages: %w", err)
 		}
 
-		changes, err = a.removeMessages(log, tx, mb, remove)
+		changes, err = a.removeMessages(context.TODO(), log, tx, mb, remove)
 		if err != nil {
 			return fmt.Errorf("removing messages: %w", err)
 		}
@@ -1125,7 +1125,7 @@ func (a *Account) TidyRejectsMailbox(log *mlog.Log, rejectsMailbox string) (hasS
 	return hasSpace, nil
 }
 
-func (a *Account) removeMessages(log *mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {
+func (a *Account) removeMessages(ctx context.Context, log *mlog.Log, tx *bstore.Tx, mb *Mailbox, l []Message) ([]Change, error) {
 	if len(l) == 0 {
 		return nil, nil
 	}
@@ -1158,7 +1158,7 @@ func (a *Account) removeMessages(log *mlog.Log, tx *bstore.Tx, mb *Mailbox, l []
 		deleted[i].Junk = false
 		deleted[i].Notjunk = false
 	}
-	if err := a.RetrainMessages(log, tx, deleted, true); err != nil {
+	if err := a.RetrainMessages(ctx, log, tx, deleted, true); err != nil {
 		return nil, fmt.Errorf("training deleted messages: %w", err)
 	}
 
@@ -1184,7 +1184,7 @@ func (a *Account) RejectsRemove(log *mlog.Log, rejectsMailbox, messageID string)
 		}
 	}()
 
-	err := a.DB.Write(func(tx *bstore.Tx) error {
+	err := a.DB.Write(context.TODO(), func(tx *bstore.Tx) error {
 		mb, err := a.MailboxFind(tx, rejectsMailbox)
 		if err != nil {
 			return fmt.Errorf("finding mailbox: %w", err)
@@ -1200,7 +1200,7 @@ func (a *Account) RejectsRemove(log *mlog.Log, rejectsMailbox, messageID string)
 			return fmt.Errorf("listing messages to remove: %w", err)
 		}
 
-		changes, err = a.removeMessages(log, tx, mb, remove)
+		changes, err = a.removeMessages(context.TODO(), log, tx, mb, remove)
 		if err != nil {
 			return fmt.Errorf("removing messages: %w", err)
 		}
@@ -1262,7 +1262,7 @@ func OpenEmailAuth(email string, password string) (acc *Account, rerr error) {
 		}
 	}()
 
-	pw, err := bstore.QueryDB[Password](acc.DB).Get()
+	pw, err := bstore.QueryDB[Password](context.TODO(), acc.DB).Get()
 	if err != nil {
 		if err == bstore.ErrAbsent {
 			return acc, ErrUnknownCredentials

store/account_test.go

@@ -1,6 +1,7 @@
 package store
 
 import (
+	"context"
 	"os"
 	"regexp"
 	"strings"
@@ -16,6 +17,8 @@ import (
 	"github.com/mjl-/mox/mox-"
 )
 
+var ctxbg = context.Background()
+
 func tcheck(t *testing.T, err error, msg string) {
 	t.Helper()
 	if err != nil {
@@ -66,7 +69,7 @@ func TestMailbox(t *testing.T) {
 		err := acc.Deliver(xlog, conf.Destinations["mjl"], &m, msgFile, false)
 		tcheck(t, err, "deliver without consume")
 
-		err = acc.DB.Write(func(tx *bstore.Tx) error {
+		err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
 			var err error
 			mbsent, err = bstore.QueryTx[Mailbox](tx).FilterNonzero(Mailbox{Name: "Sent"}).Get()
 			tcheck(t, err, "sent mailbox")
@@ -89,10 +92,10 @@ func TestMailbox(t *testing.T) {
 		err = acc.Deliver(xlog, conf.Destinations["mjl"], &mconsumed, msgFile, true)
 		tcheck(t, err, "deliver with consume")
 
-		err = acc.DB.Write(func(tx *bstore.Tx) error {
+		err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
 			m.Junk = true
 			l := []Message{m}
-			err = acc.RetrainMessages(log, tx, l, false)
+			err = acc.RetrainMessages(ctxbg, log, tx, l, false)
 			tcheck(t, err, "train as junk")
 			m = l[0]
 			return nil
@@ -102,18 +105,18 @@ func TestMailbox(t *testing.T) {
 
 	m.Junk = false
 	m.Notjunk = true
-	jf, _, err := acc.OpenJunkFilter(log)
+	jf, _, err := acc.OpenJunkFilter(ctxbg, log)
 	tcheck(t, err, "open junk filter")
-	err = acc.DB.Write(func(tx *bstore.Tx) error {
-		return acc.RetrainMessage(log, tx, jf, &m, false)
+	err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
+		return acc.RetrainMessage(ctxbg, log, tx, jf, &m, false)
 	})
 	tcheck(t, err, "retraining as non-junk")
 	err = jf.Close()
 	tcheck(t, err, "close junk filter")
 
 	m.Notjunk = false
-	err = acc.DB.Write(func(tx *bstore.Tx) error {
-		return acc.RetrainMessages(log, tx, []Message{m}, false)
+	err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
+		return acc.RetrainMessages(ctxbg, log, tx, []Message{m}, false)
 	})
 	tcheck(t, err, "untraining non-junk")
 
@@ -134,18 +137,18 @@ func TestMailbox(t *testing.T) {
 	}
 
 	acc.WithWLock(func() {
-		err := acc.DB.Write(func(tx *bstore.Tx) error {
+		err := acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
 			_, _, err := acc.MailboxEnsure(tx, "Testbox", true)
 			return err
 		})
 		tcheck(t, err, "ensure mailbox exists")
-		err = acc.DB.Read(func(tx *bstore.Tx) error {
+		err = acc.DB.Read(ctxbg, func(tx *bstore.Tx) error {
 			_, _, err := acc.MailboxEnsure(tx, "Testbox", true)
 			return err
 		})
 		tcheck(t, err, "ensure mailbox exists")
 
-		err = acc.DB.Write(func(tx *bstore.Tx) error {
+		err = acc.DB.Write(ctxbg, func(tx *bstore.Tx) error {
 			_, _, err := acc.MailboxEnsure(tx, "Testbox2", false)
 			tcheck(t, err, "create mailbox")
 

store/export.go

@@ -5,6 +5,7 @@ import (
 	"archive/zip"
 	"bufio"
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"os"
@@ -104,13 +105,13 @@ func (a DirArchiver) Close() error {
 // Some errors are not fatal and result in skipped messages. In that happens, a
 // file "errors.txt" is added to the archive describing the errors. The goal is to
 // let users export (hopefully) most messages even in the face of errors.
-func ExportMessages(log *mlog.Log, db *bstore.DB, accountDir string, archiver Archiver, maildir bool, mailboxOpt string) error {
+func ExportMessages(ctx context.Context, log *mlog.Log, db *bstore.DB, accountDir string, archiver Archiver, maildir bool, mailboxOpt string) error {
 	// todo optimize: should prepare next file to add to archive (can be an mbox with many messages) while writing a file to the archive (which typically compresses, which takes time).
 
 	// Start transaction without closure, we are going to close it early, but don't
 	// want to deal with declaring many variables now to be able to assign them in a
 	// closure and use them afterwards.
-	tx, err := db.Begin(false)
+	tx, err := db.Begin(ctx, false)
 	if err != nil {
 		return fmt.Errorf("transaction: %v", err)
 	}

store/export_test.go

@@ -49,7 +49,7 @@ func TestExport(t *testing.T) {
 
 	archive := func(archiver Archiver, maildir bool) {
 		t.Helper()
-		err = ExportMessages(log, acc.DB, acc.Dir, archiver, maildir, "")
+		err = ExportMessages(ctxbg, log, acc.DB, acc.Dir, archiver, maildir, "")
 		tcheck(t, err, "export messages")
 		err = archiver.Close()
 		tcheck(t, err, "archiver close")

store/train.go

@@ -1,6 +1,7 @@
 package store
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"os"
@@ -21,7 +22,7 @@ var ErrNoJunkFilter = errors.New("junkfilter: not configured")
 // If the account does not have a junk filter enabled, ErrNotConfigured is returned.
 // Do not forget to save the filter after modifying, and to always close the filter when done.
 // An empty filter is initialized on first access of the filter.
-func (a *Account) OpenJunkFilter(log *mlog.Log) (*junk.Filter, *config.JunkFilter, error) {
+func (a *Account) OpenJunkFilter(ctx context.Context, log *mlog.Log) (*junk.Filter, *config.JunkFilter, error) {
 	conf, ok := mox.Conf.Account(a.Name)
 	if !ok {
 		return nil, nil, ErrAccountUnknown
@@ -36,16 +37,16 @@ func (a *Account) OpenJunkFilter(log *mlog.Log) (*junk.Filter, *config.JunkFilte
 	bloomPath := filepath.Join(basePath, a.Name, "junkfilter.bloom")
 
 	if _, xerr := os.Stat(dbPath); xerr != nil && os.IsNotExist(xerr) {
-		f, err := junk.NewFilter(log, jf.Params, dbPath, bloomPath)
+		f, err := junk.NewFilter(ctx, log, jf.Params, dbPath, bloomPath)
 		return f, jf, err
 	}
-	f, err := junk.OpenFilter(log, jf.Params, dbPath, bloomPath, false)
+	f, err := junk.OpenFilter(ctx, log, jf.Params, dbPath, bloomPath, false)
 	return f, jf, err
 }
 
 // RetrainMessages (un)trains messages, if relevant given their flags. Updates
 // m.TrainedJunk after retraining.
-func (a *Account) RetrainMessages(log *mlog.Log, tx *bstore.Tx, msgs []Message, absentOK bool) (rerr error) {
+func (a *Account) RetrainMessages(ctx context.Context, log *mlog.Log, tx *bstore.Tx, msgs []Message, absentOK bool) (rerr error) {
 	if len(msgs) == 0 {
 		return nil
 	}
@@ -60,7 +61,7 @@ func (a *Account) RetrainMessages(log *mlog.Log, tx *bstore.Tx, msgs []Message,
 		// Lazy open the junk filter.
 		if jf == nil {
 			var err error
-			jf, _, err = a.OpenJunkFilter(log)
+			jf, _, err = a.OpenJunkFilter(ctx, log)
 			if err != nil && errors.Is(err, ErrNoJunkFilter) {
 				// No junk filter configured. Nothing more to do.
 				return nil
@@ -76,7 +77,7 @@ func (a *Account) RetrainMessages(log *mlog.Log, tx *bstore.Tx, msgs []Message,
 				}
 			}()
 		}
-		if err := a.RetrainMessage(log, tx, jf, &msgs[i], absentOK); err != nil {
+		if err := a.RetrainMessage(ctx, log, tx, jf, &msgs[i], absentOK); err != nil {
 			return err
 		}
 	}
@@ -85,7 +86,7 @@ func (a *Account) RetrainMessages(log *mlog.Log, tx *bstore.Tx, msgs []Message,
 
 // RetrainMessage untrains and/or trains a message, if relevant given m.TrainedJunk
 // and m.Junk/m.Notjunk. Updates m.TrainedJunk after retraining.
-func (a *Account) RetrainMessage(log *mlog.Log, tx *bstore.Tx, jf *junk.Filter, m *Message, absentOK bool) error {
+func (a *Account) RetrainMessage(ctx context.Context, log *mlog.Log, tx *bstore.Tx, jf *junk.Filter, m *Message, absentOK bool) error {
 	untrain := m.TrainedJunk != nil
 	untrainJunk := untrain && *m.TrainedJunk
 	train := m.Junk || m.Notjunk && !(m.Junk && m.Notjunk)
@@ -116,14 +117,14 @@ func (a *Account) RetrainMessage(log *mlog.Log, tx *bstore.Tx, jf *junk.Filter,
 	}
 
 	if untrain {
-		err := jf.Untrain(!untrainJunk, words)
+		err := jf.Untrain(ctx, !untrainJunk, words)
 		if err != nil {
 			return err
 		}
 		m.TrainedJunk = nil
 	}
 	if train {
-		err := jf.Train(!trainJunk, words)
+		err := jf.Train(ctx, !trainJunk, words)
 		if err != nil {
 			return err
 		}
@@ -137,7 +138,7 @@ func (a *Account) RetrainMessage(log *mlog.Log, tx *bstore.Tx, jf *junk.Filter,
 
 // TrainMessage trains the junk filter based on the current m.Junk/m.Notjunk flags,
 // disregarding m.TrainedJunk and not updating that field.
-func (a *Account) TrainMessage(log *mlog.Log, jf *junk.Filter, m Message) (bool, error) {
+func (a *Account) TrainMessage(ctx context.Context, log *mlog.Log, jf *junk.Filter, m Message) (bool, error) {
 	if !m.Junk && !m.Notjunk || (m.Junk && m.Notjunk) {
 		return false, nil
 	}
@@ -160,5 +161,5 @@ func (a *Account) TrainMessage(log *mlog.Log, jf *junk.Filter, m Message) (bool,
 		return false, nil
 	}
 
-	return true, jf.Train(m.Notjunk, words)
+	return true, jf.Train(ctx, m.Notjunk, words)
 }

tlsrptdb/db.go

@@ -61,13 +61,13 @@ type TLSReportRecord struct {
 	Report     tlsrpt.Report
 }
 
-func database() (rdb *bstore.DB, rerr error) {
+func database(ctx context.Context) (rdb *bstore.DB, rerr error) {
 	mutex.Lock()
 	defer mutex.Unlock()
 	if tlsrptDB == nil {
 		p := mox.DataDirPath("tlsrpt.db")
 		os.MkdirAll(filepath.Dir(p), 0770)
-		db, err := bstore.Open(p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, TLSReportRecord{})
+		db, err := bstore.Open(ctx, p, &bstore.Options{Timeout: 5 * time.Second, Perm: 0660}, TLSReportRecord{})
 		if err != nil {
 			return nil, err
 		}
@@ -78,7 +78,7 @@ func database() (rdb *bstore.DB, rerr error) {
 
 // Init opens and possibly initializes the database.
 func Init() error {
-	_, err := database()
+	_, err := database(mox.Shutdown)
 	return err
 }
 
@@ -106,7 +106,7 @@ func Close() {
 func AddReport(ctx context.Context, verifiedFromDomain dns.Domain, mailFrom string, r *tlsrpt.Report) error {
 	log := xlog.WithContext(ctx)
 
-	db, err := database()
+	db, err := database(ctx)
 	if err != nil {
 		return err
 	}
@@ -149,39 +149,39 @@ func AddReport(ctx context.Context, verifiedFromDomain dns.Domain, mailFrom stri
 		}
 	}
 	record.Domain = reportdom.Name()
-	return db.Insert(&record)
+	return db.Insert(ctx, &record)
 }
 
 // Records returns all TLS reports in the database.
 func Records(ctx context.Context) ([]TLSReportRecord, error) {
-	db, err := database()
+	db, err := database(ctx)
 	if err != nil {
 		return nil, err
 	}
-	return bstore.QueryDB[TLSReportRecord](db).List()
+	return bstore.QueryDB[TLSReportRecord](ctx, db).List()
 }
 
 // RecordID returns the report for the ID.
 func RecordID(ctx context.Context, id int64) (TLSReportRecord, error) {
-	db, err := database()
+	db, err := database(ctx)
 	if err != nil {
 		return TLSReportRecord{}, err
 	}
 
 	e := TLSReportRecord{ID: id}
-	err = db.Get(&e)
+	err = db.Get(ctx, &e)
 	return e, err
 }
 
 // RecordsPeriodDomain returns the reports overlapping start and end, for the given
 // domain. If domain is empty, all records match for domain.
 func RecordsPeriodDomain(ctx context.Context, start, end time.Time, domain string) ([]TLSReportRecord, error) {
-	db, err := database()
+	db, err := database(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	q := bstore.QueryDB[TLSReportRecord](db)
+	q := bstore.QueryDB[TLSReportRecord](ctx, db)
 	if domain != "" {
 		q.FilterNonzero(TLSReportRecord{Domain: domain})
 	}

tlsrptdb/db_test.go

@@ -15,6 +15,8 @@ import (
 	"github.com/mjl-/mox/tlsrpt"
 )
 
+var ctxbg = context.Background()
+
 const reportJSON = `{
      "organization-name": "Company-X",
      "date-range": {
@@ -59,6 +61,8 @@ const reportJSON = `{
    }`
 
 func TestReport(t *testing.T) {
+	mox.Context = ctxbg
+	mox.Shutdown, mox.ShutdownCancel = context.WithCancel(ctxbg)
 	mox.ConfigStaticPath = "../testdata/tlsrpt/fake.conf"
 	mox.Conf.Static.DataDir = "."
 	// Recognize as configured domain.
@@ -89,7 +93,7 @@ func TestReport(t *testing.T) {
 		if err != nil {
 			t.Fatalf("parsing TLSRPT from message %q: %s", file.Name(), err)
 		}
-		if err := AddReport(context.Background(), dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", report); err != nil {
+		if err := AddReport(ctxbg, dns.Domain{ASCII: "mox.example"}, "tlsrpt@mox.example", report); err != nil {
 			t.Fatalf("adding report to database: %s", err)
 		}
 	}
@@ -97,11 +101,11 @@ func TestReport(t *testing.T) {
 	report, err := tlsrpt.Parse(strings.NewReader(reportJSON))
 	if err != nil {
 		t.Fatalf("parsing report: %v", err)
-	} else if err := AddReport(context.Background(), dns.Domain{ASCII: "company-y.example"}, "tlsrpt@company-y.example", report); err != nil {
+	} else if err := AddReport(ctxbg, dns.Domain{ASCII: "company-y.example"}, "tlsrpt@company-y.example", report); err != nil {
 		t.Fatalf("adding report to database: %s", err)
 	}
 
-	records, err := Records(context.Background())
+	records, err := Records(ctxbg)
 	if err != nil {
 		t.Fatalf("fetching records: %s", err)
 	}
@@ -112,14 +116,14 @@ func TestReport(t *testing.T) {
 		if !reflect.DeepEqual(&r.Report, report) {
 			t.Fatalf("report, got %#v, expected %#v", r.Report, report)
 		}
-		if _, err := RecordID(context.Background(), r.ID); err != nil {
+		if _, err := RecordID(ctxbg, r.ID); err != nil {
 			t.Fatalf("get record by id: %v", err)
 		}
 	}
 
 	start, _ := time.Parse(time.RFC3339, "2016-04-01T00:00:00Z")
 	end, _ := time.Parse(time.RFC3339, "2016-04-01T23:59:59Z")
-	records, err = RecordsPeriodDomain(context.Background(), start, end, "test.xmox.nl")
+	records, err = RecordsPeriodDomain(ctxbg, start, end, "test.xmox.nl")
 	if err != nil || len(records) != 1 {
 		t.Fatalf("got err %v, records %#v, expected no error with 1 record", err, records)
 	}

vendor/github.com/mjl-/bstore/.gitignore

@@ -1,3 +1,4 @@
 /cover.out
 /cover.html
-/testdata/*.db
+/testdata/tmp.*.db
+/testdata/mail.db

vendor/github.com/mjl-/bstore/Makefile

@@ -2,7 +2,7 @@ build:
 	go build ./...
 	go vet ./...
 	GOARCH=386 go vet ./...
-	staticcheck ./...
+	staticcheck -checks 'all,-ST1012' ./...
 	./gendoc.sh
 
 fmt:

vendor/github.com/mjl-/bstore/README.md

@@ -1,22 +1,42 @@
-bstore is a database library for storing and quering Go struct data.
+Bstore is a database library for storing and quering Go values.
 
-See https://pkg.go.dev/github.com/mjl-/bstore
+Bstore is designed as a small, pure Go library that still provides most of
+the common data consistency requirements for modest database use cases. Bstore
+aims to make basic use of cgo-based libraries, such as sqlite, unnecessary.
+
+See https://pkg.go.dev/github.com/mjl-/bstore for features, examples and full
+documentation.
 
 MIT-licensed
 
-# Comparison
 
-Bstore is designed as a small, pure Go library that still provides most of the
-common data consistency requirements for modest database use cases. Bstore aims
-to make basic use of cgo-based libraries, such as sqlite, unnecessary.  Sqlite
-is a great library, but Go applications that require cgo are hard to
+# FAQ - Frequently Asked Questions
+
+## Is bstore an ORM?
+
+No. The API for bstore may look like an ORM. But instead of mapping bstore
+"queries" (function calls) to an SQL query string, bstore executes them
+directly without converting to a query language, storing the data itself.
+
+## How does bstore store its data?
+
+A bstore database is a single-file BoltDB database. BoltDB provides ACID
+properties. Bstore uses a BoltDB "bucket" (key/value store) for each Go type
+stored, with multiple subbuckets: one for type definitions, one for the actual
+data, and one bucket per index. BoltDB stores data in a B+tree. See format.md
+for details.
+
+## How does bstore compare to sqlite?
+
+Sqlite is a great library, but Go applications that require cgo are hard to
 cross-compile. With bstore, cross-compiling to most Go-supported platforms
-stays trivial. Although bstore is much more limited in so many aspects than
-sqlite, bstore also offers some advantages as well.
+stays trivial (though not plan9, unfortunately). Although bstore is much more
+limited in so many aspects than sqlite, bstore also offers some advantages as
+well. Some points of comparison:
 
 - Cross-compilation and reproducibility: Trivial with bstore due to pure Go,
   much harder with sqlite because of cgo.
-- Code complexity: low with bstore (6k lines including comments/docs), high
+- Code complexity: low with bstore (7k lines including comments/docs), high
   with sqlite.
 - Query language: mostly-type-checked function calls in bstore, free-form query
   strings only checked at runtime with sqlite.
@@ -33,19 +53,3 @@ sqlite, bstore also offers some advantages as well.
   WAL or journal files).
 - Test coverage: decent coverage but limited real-world for bstore, versus
   extremely thoroughly tested and with enormous real-world use.
-
-# FAQ
-
-Q: Is bstore an ORM?
-
-A: No. The API for bstore may look like an ORM. But instead of mapping bstore
-"queries" (function calls) to an SQL query string, bstore executes them
-directly without converting to a query language.
-
-Q: How does bstore store its data?
-
-A bstore database is a single-file BoltDB database. BoltDB provides ACID
-properties. Bstore uses a BoltDB "bucket" (key/value store) for each Go type
-stored, with multiple subbuckets: one for type definitions, one for the actual
-data, and one bucket per index. BoltDB stores data in a B+tree. See format.md
-for details.

vendor/github.com/mjl-/bstore/default.go

@@ -24,7 +24,7 @@ func (f field) applyDefault(rv reflect.Value) error {
 	case kindBytes, kindBinaryMarshal, kindMap:
 		return nil
 
-	case kindSlice, kindStruct:
+	case kindSlice, kindStruct, kindArray:
 		return f.Type.applyDefault(rv)
 
 	case kindBool, kindInt, kindInt8, kindInt16, kindInt32, kindInt64, kindUint, kindUint8, kindUint16, kindUint32, kindUint64, kindFloat32, kindFloat64, kindString, kindTime:
@@ -53,9 +53,9 @@ func (f field) applyDefault(rv reflect.Value) error {
 }
 
 // only for recursing. we do not support recursing into maps because it would
-// involve more work making values settable. and how sensible it it anyway?
+// involve more work making values settable. and how sensible is it anyway?
 func (ft fieldType) applyDefault(rv reflect.Value) error {
-	if ft.Ptr && (rv.IsZero() || rv.IsNil()) {
+	if ft.Ptr && rv.IsZero() {
 		return nil
 	} else if ft.Ptr {
 		rv = rv.Elem()
@@ -64,12 +64,19 @@ func (ft fieldType) applyDefault(rv reflect.Value) error {
 	case kindSlice:
 		n := rv.Len()
 		for i := 0; i < n; i++ {
-			if err := ft.List.applyDefault(rv.Index(i)); err != nil {
+			if err := ft.ListElem.applyDefault(rv.Index(i)); err != nil {
+				return err
+			}
+		}
+	case kindArray:
+		n := ft.ArrayLength
+		for i := 0; i < n; i++ {
+			if err := ft.ListElem.applyDefault(rv.Index(i)); err != nil {
 				return err
 			}
 		}
 	case kindStruct:
-		for _, nf := range ft.Fields {
+		for _, nf := range ft.structFields {
 			nfv := rv.FieldByIndex(nf.structField.Index)
 			if err := nf.applyDefault(nfv); err != nil {
 				return err

vendor/github.com/mjl-/bstore/doc.go

@@ -1,5 +1,5 @@
 /*
-Package bstore is a database library for storing and querying Go struct data.
+Package bstore is a database library for storing and querying Go values.
 
 Bstore is designed as a small, pure Go library that still provides most of
 the common data consistency requirements for modest database use cases. Bstore
@@ -9,7 +9,7 @@ Bstore implements autoincrementing primary keys, indices, default values,
 enforcement of nonzero, unique and referential integrity constraints, automatic
 schema updates and a query API for combining filters/sorting/limits. Queries
 are planned and executed using indices for fast execution where possible.
-Bstores is designed with the Go type system in mind: you typically don't have to
+Bstore is designed with the Go type system in mind: you typically don't have to
 write any (un)marshal code for your types.
 
 # Field types
@@ -21,7 +21,7 @@ types, but not pointers to pointers:
   - uint (as uint32), uint8, uint16, uint32, uint64
   - bool, float32, float64, string, []byte
   - Maps, with keys and values of any supported type, except keys with pointer types.
-  - Slices, with elements of any supported type.
+  - Slices and arrays, with elements of any supported type.
   - time.Time
   - Types that implement binary.MarshalBinary and binary.UnmarshalBinary, useful
     for struct types with state in private fields. Do not change the
@@ -32,24 +32,27 @@ Note: int and uint are stored as int32 and uint32, for compatibility of database
 files between 32bit and 64bit systems. Where possible, use explicit (u)int32 or
 (u)int64 types.
 
-Embedded structs are handled by storing the individual fields of the embedded
-struct. The named embedded type is not part of the type schema, and can
-currently only be used with UpdateField and UpdateFields, not for filtering.
+Cyclic types are supported, but cyclic data is not. Attempting to store cyclic
+data will likely result in a stack overflow panic.
+
+Anonymous struct fields are handled by taking in each of the anonymous struct's
+fields as a type's own fields.  The named embedded type is not part of the type
+schema, and with a Query it can currently only be used with UpdateField and
+UpdateFields, not for filtering.
 
 Bstore embraces the use of Go zero values. Use zero values, possibly pointers,
 where you would use NULL values in SQL.
 
-Types that have not yet been implemented: interface values, (fixed length) arrays,
-complex numbers.
-
 # Struct tags
 
 The typical Go struct can be stored in the database. The first field of a
-struct type is its primary key, and must always be unique. Additional behaviour
-can be configured through struct tag "bstore". The values are comma-separated.
-Typically one word, but some have multiple space-separated words:
+struct type is its primary key, must always be unique, and in case of an
+integer type the insertion of a zero value automatically changes it to the next
+sequence number by default.  Additional behaviour can be configured through
+struct tag "bstore". The values are comma-separated.  Typically one word, but
+some have multiple space-separated words:
 
-  - "-" ignores the field entirely.
+  - "-" ignores the field entirely, not stored.
   - "name <fieldname>", use "fieldname" instead of the Go type field name.
   - "nonzero", enforces that field values are not the zero value.
   - "noauto", only valid for integer types, and only for the primary key. By
@@ -57,16 +60,19 @@ Typically one word, but some have multiple space-separated words:
     assigned on insert when it is 0. With noauto inserting a 0 value results in an
     error. For primary keys of other types inserting the zero value always results
     in an error.
-  - "index" or "index <field1+field2+...> [<name>]", adds an index. In the first
-    form, the index is on the field on which the tag is specified, and the index
-    name is the same as the field name. In the second form multiple fields can be
-    specified, and an optional name. The first field must be the field on which
-    the tag is specified. The field names are +-separated. The default name for
-    the second form is the same +-separated string but can be set explicitly to
-    the second parameter. An index can only be set for basic integer types, bools,
-    time and strings. Indices are automatically (re)created when registering a
-    type.
-  - "unique" or "unique  <field1+field2+...> [<name>]", adds an index as with
+  - "index" or "index <field1>+<field2>+<...> [<name>]", adds an index. In the
+    first form, the index is on the field on which the tag is specified, and the
+    index name is the same as the field name. In the second form multiple fields can
+    be specified, and an optional name. The first field must be the field on which
+    the tag is specified. The field names are +-separated. The default name for the
+    second form is the same +-separated string but can be set explicitly with the
+    second parameter. An index can only be set for basic integer types, bools, time
+    and strings. A field of slice type can also have an index (but not a unique
+    index, and only one slice field per index), allowing fast lookup of any single
+    value in the slice with Query.FilterIn. Indices are automatically (re)created
+    when registering a type. Fields with a pointer type cannot have an index.
+    String values used in an index cannot contain a \0.
+  - "unique" or "unique  <field1>+<field2>+<...> [<name>]", adds an index as with
     "index" and also enforces a unique constraint. For time.Time the timezone is
     ignored for the uniqueness check.
   - "ref <type>", enforces that the value exists as primary key for "type".
@@ -80,8 +86,8 @@ Typically one word, but some have multiple space-separated words:
     Times are parsed as time.RFC3339 otherwise. Supported types: bool
     ("true"/"false"), integers, floats, strings. Value is not quoted and no escaping
     of special characters, like the comma that separates struct tag words, is
-    possible.  Defaults are also replaced on fields in nested structs and
-    slices, but not in maps.
+    possible.  Defaults are also replaced on fields in nested structs, slices
+    and arrays, but not in maps.
   - "typename <name>", override name of the type. The name of the Go type is
     used by default. Can only be present on the first field (primary key).
     Useful for doing schema updates.
@@ -89,18 +95,14 @@ Typically one word, but some have multiple space-separated words:
 # Schema updates
 
 Before using a Go type, you must register it for use with the open database by
-passing a (zero) value of that type to the Open or Register functions. For each
-type, a type definition is stored in the database. If a type has an updated
-definition since the previous database open, a new type definition is added to
-the database automatically and any required modifications are made: Indexes
-(re)created, fields added/removed, new nonzero/unique/reference constraints
-validated.
+passing a (possibly zero) value of that type to the Open or Register functions.
+For each type, a type definition is stored in the database. If a type has an
+updated definition since the previous database open, a new type definition is
+added to the database automatically and any required modifications are made and
+checked: Indexes (re)created, fields added/removed, new
+nonzero/unique/reference constraints validated.
 
-If data/types cannot be updated automatically (e.g. converting an int field into
-a string field), custom data migration code is needed. You may have to keep
-track of a data/schema version.
-
-As a special case, you can switch field types between pointer and non-pointer
+As a special case, you can change field types between pointer and non-pointer
 types. With one exception: changing from pointer to non-pointer where the type
 has a field that must be nonzero is not allowed. The on-disk encoding will not be
 changed, and nil pointers will turn into zero values, and zero values into nil
@@ -110,33 +112,95 @@ Because named embed structs are not part of the type definition, you can
 wrap/unwrap fields into a embed/anonymous struct field. No new type definition
 is created.
 
-# BoltDB
+Some schema conversions are not allowed. In some cases due to architectural
+limitations. In some cases because the constraint checks haven't been
+implemented yet, or the parsing code does not yet know how to parse the old
+on-disk values into the updated Go types. If you need a conversion that is not
+supported, you will need to write a manual conversion, and you would have to
+keep track whether the update has been executed.
 
-BoltDB is used as underlying storage. Bolt provides ACID transactions, storing
-its data in a B+tree.  Only a single write transaction can be active at a time,
-but otherwise multiple read-only transactions can be active. Do not start a
-blocking read-only transaction while holding a writable transaction or vice
-versa, this will cause deadlock.
+Changes that are allowed:
 
-Bolt uses Go types that are memory mapped  to the database file. This means bolt
-database files cannot be transferred between machines with different endianness.
-Bolt uses explicit widths for its types, so files can be transferred between
-32bit and 64bit machines of same endianness.
+  - From smaller to larger integer types (same signedness).
+  - Removal of "noauto" on primary keys (always integer types). This updates the
+    "next sequence" counter automatically to continue after the current maximum
+    value.
+  - Adding/removing/modifying an index, including a unique index. When a unique
+    index is added, the current records are verified to be unique.
+  - Adding/removing a reference. When a reference is added, the current records
+    are verified to be valid references.
+  - Add/remove a nonzero constraint. Existing records are verified.
+
+Conversions that are not currently allowed, but may be in the future:
+
+  - Signedness of integer types. With a one-time check that old values fit in the new
+    type, this could be allowed in the future.
+  - Conversions between basic types: strings, []byte, integers, floats, boolean.
+    Checks would have to be added for some of these conversions. For example,
+    from string to integer: the on-disk string values would have to be valid
+    integers.
+  - Types of primary keys cannot be changed, also not from one integer type to a
+    wider integer type of same signedness.
+
+# BoltDB and storage
+
+BoltDB is used as underlying storage. BoltDB stores key/values in a single
+file, in multiple/nested buckets (namespaces) in a B+tree and provides ACID
+transactions.  Either a single write transaction or multiple read-only
+transactions can be active at a time.  Do not start a blocking read-only
+transaction while holding a writable transaction or vice versa, this will cause
+deadlock.
+
+BoltDB returns Go values that are memory mapped to the database file.  This
+means BoltDB/bstore database files cannot be transferred between machines with
+different endianness.  BoltDB uses explicit widths for its types, so files can
+be transferred between 32bit and 64bit machines of same endianness. While
+BoltDB returns read-only memory mapped Go values, bstore only ever returns
+parsed/copied regular writable Go values that require no special programmer
+attention.
+
+For each Go type opened for a database file, bstore ensures a BoltDB bucket
+exists with two subbuckets:
+
+  - "types", with type descriptions of the stored records. Each time the database
+    file is opened with a modified Go type (add/removed/modified
+    field/type/bstore struct tag), a new type description is automatically added,
+    identified by sequence number.
+  - "records", containing all data, with the type's primary key as BoltDB key,
+    and the encoded remaining fields as value. The encoding starts with a
+    reference to a type description.
+
+For each index, another subbucket is created, its name starting with "index.".
+The stored keys consist of the index fields followed by the primary key, and an
+empty value.
 
 # Limitations
 
+Bstore has limitations, not all of which are architectural so may be fixed in
+the future.
+
 Bstore does not implement the equivalent of SQL joins, aggregates, and many
 other concepts.
 
-Filtering/comparing/sorting on pointer fields is not currently allowed.  Pointer
-fields cannot have a (unique) index due to the current index format.  Using zero
-values is recommended instead for now.
+Filtering/comparing/sorting on pointer fields is not allowed.  Pointer fields
+cannot have a (unique) index. Use non-pointer values with the zero value as the
+equivalent of a nil pointer.
 
-Integer field types can be expanded to wider types, but not to a different
-signedness or a smaller integer (fewer bits). The primary key of a type cannot
-currently be changed.
+The first field of a stored struct is always the primary key. Autoincrement is
+only available for the primary key.
 
-The first field of a struct is always the primary key. Types requires an
-explicit primary key. Autoincrement is only available for the primary key.
+BoltDB opens the database file with a lock. Only one process can have the
+database open at a time.
+
+An index stored on disk in BoltDB can consume more disk space than other
+database systems would: For each record, the indexed field(s) and primary key
+are stored in full. Because bstore uses BoltDB as key/value store, and doesn't
+manage disk pages itself, it cannot as efficiently pack an index page with many
+records.
+
+Interface values cannot be stored. This would require storing the type along
+with the value. Instead, use a type that is a BinaryMarshaler.
+
+Values of builtin type "complex" cannot be stored.
 */
 package bstore

vendor/github.com/mjl-/bstore/equal.go

@@ -63,7 +63,15 @@ func (ft fieldType) equal(ov, v reflect.Value) (r bool) {
 			return false
 		}
 		for i := 0; i < n; i++ {
-			if !ft.List.equal(ov.Index(i), v.Index(i)) {
+			if !ft.ListElem.equal(ov.Index(i), v.Index(i)) {
+				return false
+			}
+		}
+		return true
+	case kindArray:
+		n := ft.ArrayLength
+		for i := 0; i < n; i++ {
+			if !ft.ListElem.equal(ov.Index(i), v.Index(i)) {
 				return false
 			}
 		}
@@ -78,7 +86,7 @@ func (ft fieldType) equal(ov, v reflect.Value) (r bool) {
 		}
 		return bytes.Equal(obuf, buf)
 	case kindStruct:
-		for _, f := range ft.Fields {
+		for _, f := range ft.structFields {
 			fov := ov.FieldByIndex(f.structField.Index)
 			fv := v.FieldByIndex(f.structField.Index)
 			if !f.Type.equal(fov, fv) {

vendor/github.com/mjl-/bstore/exec.go

@@ -10,6 +10,8 @@ import (
 	bolt "go.etcd.io/bbolt"
 )
 
+// todo optimize: do not fetch full record if we can apply the filters with just the values we glean from the index key.
+
 // exec represents the execution of a query plan.
 type exec[T any] struct {
 	q    *Query[T]
@@ -94,6 +96,13 @@ func (e *exec[T]) nextKey(write, value bool) ([]byte, T, error) {
 
 	q := e.q
 
+	if q.err == nil {
+		select {
+		case <-q.ctxDone:
+			q.error(q.ctx.Err())
+		default:
+		}
+	}
 	if q.err != nil {
 		return nil, zero, q.err
 	}
@@ -424,6 +433,25 @@ func (e *exec[T]) checkFilter(p *pair[T]) (rok bool, rerr error) {
 					return
 				}
 			}
+		case filterInSlice[T]:
+			v, err := p.Value(e)
+			if err != nil {
+				q.error(err)
+				return false, err
+			}
+			rv := reflect.ValueOf(v)
+			frv := rv.FieldByIndex(f.field.structField.Index)
+			n := frv.Len()
+			var have bool
+			for i := 0; i < n; i++ {
+				if f.field.Type.ListElem.equal(frv.Index(i), f.rvalue) {
+					have = true
+					break
+				}
+			}
+			if !have {
+				return
+			}
 		case filterCompare[T]:
 			v, err := p.Value(e)
 			if err != nil {
@@ -531,10 +559,10 @@ func compare(k kind, a, b reflect.Value) int {
 }
 
 func (e *exec[T]) sort() {
-	// todo: We should check whether we actually need to load values. We're just
-	// always it now for the time being because SortStableFunc isn't going to
-	// give us a *pair (even though it could because of the slice) so we
-	// couldn't set/cache the value T during sorting.
+	// todo: We should check whether we actually need to load values. We're
+	// always loading it for the time being because SortStableFunc isn't
+	// going to give us a *pair (even though it could because of the slice)
+	// so we couldn't set/cache the value T during sorting.
 	q := e.q
 
 	for i := range e.data {

vendor/github.com/mjl-/bstore/export.go

@@ -13,16 +13,18 @@ import (
 // Types returns the types present in the database, regardless of whether they
 // are currently registered using Open or Register. Useful for exporting data
 // with Keys and Records.
-func (db *DB) Types() ([]string, error) {
+func (tx *Tx) Types() ([]string, error) {
+	if err := tx.ctx.Err(); err != nil {
+		return nil, err
+	}
+
 	var types []string
-	err := db.Read(func(tx *Tx) error {
-		return tx.btx.ForEach(func(bname []byte, b *bolt.Bucket) error {
+	err := tx.btx.ForEach(func(bname []byte, b *bolt.Bucket) error {
 		// note: we do not track stats for types operations.
 
 		types = append(types, string(bname))
 		return nil
 	})
-	})
 	if err != nil {
 		return nil, err
 	}
@@ -31,9 +33,12 @@ func (db *DB) Types() ([]string, error) {
 
 // prepareType prepares typeName for export/introspection with DB.Keys,
 // DB.Record, DB.Records. It is different in that it does not require a
-// reflect.Type to parse into. It parses to a map, e.g. for export to JSON. The
-// returned typeVersion has no structFields set in its fields.
+// reflect.Type to parse into. It parses to a map, e.g. for export to JSON.
 func (db *DB) prepareType(tx *Tx, typeName string) (map[uint32]*typeVersion, *typeVersion, *bolt.Bucket, []string, error) {
+	if err := tx.ctx.Err(); err != nil {
+		return nil, nil, nil, nil, err
+	}
+
 	rb, err := tx.recordsBucket(typeName, 0.5)
 	if err != nil {
 		return nil, nil, nil, nil, err
@@ -51,6 +56,7 @@ func (db *DB) prepareType(tx *Tx, typeName string) (map[uint32]*typeVersion, *ty
 		if err != nil {
 			return err
 		}
+
 		versions[ntv.Version] = ntv
 		if tv == nil || ntv.Version > tv.Version {
 			tv = ntv
@@ -74,36 +80,39 @@ func (db *DB) prepareType(tx *Tx, typeName string) (map[uint32]*typeVersion, *ty
 // Keys returns the parsed primary keys for the type "typeName". The type does
 // not have to be registered with Open or Register. For use with Record(s) to
 // export data.
-func (db *DB) Keys(typeName string, fn func(pk any) error) error {
-	return db.Read(func(tx *Tx) error {
-		_, tv, rb, _, err := db.prepareType(tx, typeName)
+func (tx *Tx) Keys(typeName string, fn func(pk any) error) error {
+	_, tv, rb, _, err := tx.db.prepareType(tx, typeName)
 	if err != nil {
 		return err
 	}
 
-		// todo: do not pass nil parser?
-		v := reflect.New(reflect.TypeOf(tv.Fields[0].Type.zero(nil))).Elem()
+	ctxDone := tx.ctx.Done()
+
+	v := reflect.New(reflect.TypeOf(tv.Fields[0].Type.zeroKey())).Elem()
 	return rb.ForEach(func(bk, bv []byte) error {
 		tx.stats.Records.Cursor++
 
+		select {
+		case <-ctxDone:
+			return tx.ctx.Err()
+		default:
+		}
+
 		if err := parsePK(v, bk); err != nil {
 			return err
 		}
 		return fn(v.Interface())
 	})
-	})
 }
 
 // Record returns the record with primary "key" for "typeName" parsed as map.
 // "Fields" is set to the fields of the type. The type does not have to be
 // registered with Open or Register.  Record parses the data without the Go
 // type present. BinaryMarshal fields are returned as bytes.
-func (db *DB) Record(typeName, key string, fields *[]string) (map[string]any, error) {
-	var r map[string]any
-	err := db.Read(func(tx *Tx) error {
-		versions, tv, rb, xfields, err := db.prepareType(tx, typeName)
+func (tx *Tx) Record(typeName, key string, fields *[]string) (map[string]any, error) {
+	versions, tv, rb, xfields, err := tx.db.prepareType(tx, typeName)
 	if err != nil {
-			return err
+		return nil, err
 	}
 	*fields = xfields
 
@@ -143,64 +152,67 @@ func (db *DB) Record(typeName, key string, fields *[]string) (map[string]any, er
 	case kindBytes:
 		kv = []byte(key) // todo: or decode from base64?
 	default:
-			return fmt.Errorf("internal error: unknown primary key kind %v", tv.Fields[0].Type.Kind)
+		return nil, fmt.Errorf("internal error: unknown primary key kind %v", tv.Fields[0].Type.Kind)
 	}
 	if err != nil {
-			return err
+		return nil, err
 	}
 	pkv := reflect.ValueOf(kv)
 	kind, err := typeKind(pkv.Type())
 	if err != nil {
-			return err
+		return nil, err
 	}
 	if kind != tv.Fields[0].Type.Kind {
 		// Convert from various int types above to required type. The ParseInt/ParseUint
 		// calls already validated that the values fit.
-			pkt := reflect.TypeOf(tv.Fields[0].Type.zero(nil))
+		pkt := reflect.TypeOf(tv.Fields[0].Type.zeroKey())
 		pkv = pkv.Convert(pkt)
 	}
 	k, err := packPK(pkv)
 	if err != nil {
-			return err
+		return nil, err
 	}
 
 	tx.stats.Records.Get++
 	bv := rb.Get(k)
 	if bv == nil {
-			return ErrAbsent
+		return nil, ErrAbsent
 	}
 	record, err := parseMap(versions, k, bv)
 	if err != nil {
-			return err
+		return nil, err
 	}
-		r = record
-		return nil
-	})
-	return r, err
+	return record, nil
 }
 
 // Records calls "fn" for each record of "typeName". Records sets "fields" to
 // the fields of the type. The type does not have to be registered with Open or
 // Register.  Record parses the data without the Go type present. BinaryMarshal
 // fields are returned as bytes.
-func (db *DB) Records(typeName string, fields *[]string, fn func(map[string]any) error) error {
-	return db.Read(func(tx *Tx) error {
-		versions, _, rb, xfields, err := db.prepareType(tx, typeName)
+func (tx *Tx) Records(typeName string, fields *[]string, fn func(map[string]any) error) error {
+	versions, _, rb, xfields, err := tx.db.prepareType(tx, typeName)
 	if err != nil {
 		return err
 	}
 	*fields = xfields
 
+	ctxDone := tx.ctx.Done()
+
 	return rb.ForEach(func(bk, bv []byte) error {
 		tx.stats.Records.Cursor++
 
+		select {
+		case <-ctxDone:
+			return tx.ctx.Err()
+		default:
+		}
+
 		record, err := parseMap(versions, bk, bv)
 		if err != nil {
 			return err
 		}
 		return fn(record)
 	})
-	})
 }
 
 // parseMap parses a record into a map with the right typeVersion from versions.
@@ -228,7 +240,7 @@ func parseMap(versions map[uint32]*typeVersion, bk, bv []byte) (record map[strin
 
 	r := map[string]any{}
 
-	v := reflect.New(reflect.TypeOf(tv.Fields[0].Type.zero(p))).Elem()
+	v := reflect.New(reflect.TypeOf(tv.Fields[0].Type.zeroKey())).Elem()
 	err := parsePK(v, bk)
 	if err != nil {
 		return nil, err
@@ -243,12 +255,12 @@ func parseMap(versions map[uint32]*typeVersion, bk, bv []byte) (record map[strin
 		if fm.Nonzero(i) {
 			r[f.Name] = f.Type.parseValue(p)
 		} else {
-			r[f.Name] = f.Type.zero(p)
+			r[f.Name] = f.Type.zeroExportValue()
 		}
 	}
 
 	if len(p.buf) != 0 {
-		return nil, fmt.Errorf("%w: leftover data after parsing", ErrStore)
+		return nil, fmt.Errorf("%w: leftover data after parsing (%d %x %q)", ErrStore, len(p.buf), p.buf, p.buf)
 	}
 
 	return r, nil
@@ -315,14 +327,21 @@ func (ft fieldType) parseValue(p *parser) any {
 		var l []any
 		for i := 0; i < n; i++ {
 			if fm.Nonzero(i) {
-				l = append(l, ft.List.parseValue(p))
+				l = append(l, ft.ListElem.parseValue(p))
 			} else {
 				// Always add non-zero elements, or we would
 				// change the number of elements in a list.
-				l = append(l, ft.List.zero(p))
+				l = append(l, ft.ListElem.zeroExportValue())
 			}
 		}
 		return l
+	case kindArray:
+		n := ft.ArrayLength
+		var l []any
+		for i := 0; i < n; i++ {
+			l = append(l, ft.ListElem.parseValue(p))
+		}
+		return l
 	case kindMap:
 		un := p.Uvarint()
 		n := p.checkInt(un)
@@ -338,19 +357,19 @@ func (ft fieldType) parseValue(p *parser) any {
 			if fm.Nonzero(i) {
 				v = ft.MapValue.parseValue(p)
 			} else {
-				v = ft.MapValue.zero(p)
+				v = ft.MapValue.zeroExportValue()
 			}
 			m[k] = v
 		}
 		return m
 	case kindStruct:
-		fm := p.Fieldmap(len(ft.Fields))
+		fm := p.Fieldmap(len(ft.structFields))
 		m := map[string]any{}
-		for i, f := range ft.Fields {
+		for i, f := range ft.structFields {
 			if fm.Nonzero(i) {
 				m[f.Name] = f.Type.parseValue(p)
 			} else {
-				m[f.Name] = f.Type.zero(p)
+				m[f.Name] = f.Type.zeroExportValue()
 			}
 		}
 		return m
@@ -359,7 +378,7 @@ func (ft fieldType) parseValue(p *parser) any {
 	panic("cannot happen")
 }
 
-var zerovalues = map[kind]any{
+var zeroExportValues = map[kind]any{
 	kindBytes:         []byte(nil),
 	kindBinaryMarshal: []byte(nil), // We don't have the actual type available, so we just return binary data.
 	kindBool:          false,
@@ -380,12 +399,53 @@ var zerovalues = map[kind]any{
 	kindSlice:         []any(nil),
 	kindMap:           map[string]any(nil),
 	kindStruct:        map[string]any(nil),
+	// kindArray handled in zeroExportValue()
 }
 
-func (ft fieldType) zero(p *parser) any {
-	v, ok := zerovalues[ft.Kind]
+// zeroExportValue returns the zero value for a fieldType for use with exporting.
+func (ft fieldType) zeroExportValue() any {
+	if ft.Kind == kindArray {
+		ev := ft.ListElem.zeroExportValue()
+		l := make([]any, ft.ArrayLength)
+		for i := 0; i < ft.ArrayLength; i++ {
+			l[i] = ev
+		}
+		return l
+	}
+	v, ok := zeroExportValues[ft.Kind]
 	if !ok {
-		p.Errorf("internal error: unhandled zero value for field type %v", ft.Kind)
+		panic(fmt.Errorf("internal error: unhandled zero value for field type %v", ft.Kind))
+	}
+	return v
+}
+
+var zeroKeys = map[kind]any{
+	kindBytes:  []byte(nil),
+	kindBool:   false,
+	kindInt8:   int8(0),
+	kindInt16:  int16(0),
+	kindInt32:  int32(0),
+	kindInt:    int(0),
+	kindInt64:  int64(0),
+	kindUint8:  uint8(0),
+	kindUint16: uint16(0),
+	kindUint32: uint32(0),
+	kindUint:   uint(0),
+	kindUint64: uint64(0),
+	kindString: "",
+	kindTime:   zerotime,
+	// kindSlice handled in zeroKeyValue()
+}
+
+// zeroKeyValue returns the zero value for a fieldType for use with exporting.
+func (ft fieldType) zeroKey() any {
+	k := ft.Kind
+	if k == kindSlice {
+		k = ft.ListElem.Kind
+	}
+	v, ok := zeroKeys[k]
+	if !ok {
+		panic(fmt.Errorf("internal error: unhandled zero value for field type %v", ft.Kind))
 	}
 	return v
 }

vendor/github.com/mjl-/bstore/format.md

@@ -17,8 +17,8 @@ version is added to the "types" subbucket. Data is always inserted/updated with
 the most recent type version. But the database may still hold data records
 referencing older type versions. Bstore decodes a packed data record with the
 referenced type version. For storage efficiency: the type version is reused for
-many stored records, a self-describing format (like JSON) would duplicate the
-field names in each stored record.
+many stored records, a self-describing format (like JSON) for each stored
+record would duplicate the field names in each stored record.
 
 # Record storage
 
@@ -51,6 +51,8 @@ more space than the single bit and are stored consecutively after the fieldmap:
     the zero value marked in the fieldmap.
   - Slices use a uvarint for the number of elements, followed by a bitmap for
     nonzero values, followed by the encoded nonzero elements.
+  - Arrays (fixed length) start with a bitmap for nonzero values, followed by
+    the encoded nonzero elements.
   - Maps use a uvariant for the number of key/value pairs, followed by a
     fieldmap for the values (the keys are always present), followed by each
     pair: key (always present), value (only if nonzero); key, value; etc.
@@ -71,7 +73,7 @@ unsigned integer, or between string and []byte.
 Indexes are stored in subbuckets, named starting with "index." followed by the
 index name. Keys are a self-delimiting encodings of the fields that make up the
 key, followed by the primary key for the "records" bucket. Values are always
-empty in index buckets. For bool and integer types, the same fixed with
+empty in index buckets. For bool and integer types, the same fixed width
 encoding as for primary keys in the "records" subbucket is used. Strings are
 encoded by their bytes (no \0 allowed) followed by a delimiting \0. Unlike
 primary keys, an index can cover a field with type time.Time. Times are encoded

vendor/github.com/mjl-/bstore/keys.go

@@ -10,7 +10,7 @@ import (
 
 /*
 The records buckets map a primary key to the record data. The primary key is of
-a form that we can scan/range over. So fixed with for integers. For strings and
+a form that we can scan/range over. So fixed width for integers. For strings and
 bytes they are just their byte representation. We do not store the PK in the
 record data. This means we cannot store a time.Time as primary key, because we
 cannot have the timezone encoded for comparison reasons.
@@ -150,7 +150,12 @@ fields:
 		if err != nil {
 			break
 		}
-		switch f.Type.Kind {
+		ft := f.Type
+		if ft.Kind == kindSlice {
+			// For an index on a slice, we store each value in the slice in a separate index key.
+			ft = *ft.ListElem
+		}
+		switch ft.Kind {
 		case kindString:
 			for i, b := range buf {
 				if b == 0 {
@@ -174,6 +179,8 @@ fields:
 			take(8)
 		case kindTime:
 			take(8 + 4)
+		default:
+			err = fmt.Errorf("%w: unhandled kind %v for index key", ErrStore, ft.Kind)
 		}
 	}
 	if err != nil {
@@ -203,9 +210,14 @@ fields:
 	return pk, nil, nil
 }
 
-// packKey returns a key to store in an index: first the prefix without pk, then
-// the prefix including pk.
-func (idx *index) packKey(rv reflect.Value, pk []byte) ([]byte, []byte, error) {
+type indexkey struct {
+	pre  []byte // Packed fields excluding PK, a slice of full.
+	full []byte // Packed fields including PK.
+}
+
+// packKey returns keys to store in an index: first the key prefixes without pk, then
+// the prefixes including pk.
+func (idx *index) packKey(rv reflect.Value, pk []byte) ([]indexkey, error) {
 	var l []reflect.Value
 	for _, f := range idx.Fields {
 		frv := rv.FieldByIndex(f.structField.Index)
@@ -215,14 +227,45 @@ func (idx *index) packKey(rv reflect.Value, pk []byte) ([]byte, []byte, error) {
 }
 
 // packIndexKeys packs values from l, followed by the pk.
-// It returns the key prefix (without pk), and full key with pk.
-func packIndexKeys(l []reflect.Value, pk []byte) ([]byte, []byte, error) {
-	var prek, ik []byte
+// It returns the key prefixes (without pk), and full keys with pk.
+func packIndexKeys(l []reflect.Value, pk []byte) ([]indexkey, error) {
+	ikl := []indexkey{{}}
 	for _, frv := range l {
+		bufs, err := packIndexKey(frv)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(bufs) == 1 {
+			for i := range ikl {
+				ikl[i].full = append(ikl[i].full, bufs[0]...)
+			}
+		} else if len(ikl) == 1 && len(bufs) > 1 {
+			nikl := make([]indexkey, len(bufs))
+			for i, buf := range bufs {
+				nikl[i] = indexkey{full: append(append([]byte{}, ikl[0].full...), buf...)}
+			}
+			ikl = nikl
+		} else if len(bufs) == 0 {
+			return nil, nil
+		} else {
+			return nil, fmt.Errorf("%w: multiple index fields that result in multiple values, or no data for index key, %d keys so far, %d new buffers", ErrStore, len(ikl), len(bufs))
+		}
+	}
+	for i := range ikl {
+		n := len(ikl[i].full)
+		ikl[i].full = append(ikl[i].full, pk...)
+		ikl[i].pre = ikl[i].full[:n]
+	}
+	return ikl, nil
+}
+
+func packIndexKey(frv reflect.Value) ([][]byte, error) {
 	k, err := typeKind(frv.Type())
 	if err != nil {
-			return nil, nil, err
+		return nil, err
 	}
+
 	var buf []byte
 	switch k {
 	case kindBool:
@@ -239,7 +282,7 @@ func packIndexKeys(l []reflect.Value, pk []byte) ([]byte, []byte, error) {
 	case kindInt:
 		i := frv.Int()
 		if i < math.MinInt32 || i > math.MaxInt32 {
-				return nil, nil, fmt.Errorf("%w: int value %d does not fit in int32", ErrParam, i)
+			return nil, fmt.Errorf("%w: int value %d does not fit in int32", ErrParam, i)
 		}
 		buf = binary.BigEndian.AppendUint32(nil, uint32(int32(i)+math.MinInt32))
 	case kindInt64:
@@ -253,7 +296,7 @@ func packIndexKeys(l []reflect.Value, pk []byte) ([]byte, []byte, error) {
 	case kindUint:
 		i := frv.Uint()
 		if i > math.MaxUint32 {
-				return nil, nil, fmt.Errorf("%w: uint value %d does not fit in uint32", ErrParam, i)
+			return nil, fmt.Errorf("%w: uint value %d does not fit in uint32", ErrParam, i)
 		}
 		buf = binary.BigEndian.AppendUint32(nil, uint32(i))
 	case kindUint64:
@@ -262,7 +305,7 @@ func packIndexKeys(l []reflect.Value, pk []byte) ([]byte, []byte, error) {
 		buf = []byte(frv.String())
 		for _, c := range buf {
 			if c == 0 {
-					return nil, nil, fmt.Errorf("%w: string used as index key cannot have \\0", ErrParam)
+				return nil, fmt.Errorf("%w: string used as index key cannot have \\0", ErrParam)
 			}
 		}
 		buf = append(buf, 0)
@@ -270,13 +313,22 @@ func packIndexKeys(l []reflect.Value, pk []byte) ([]byte, []byte, error) {
 		tm := frv.Interface().(time.Time)
 		buf = binary.BigEndian.AppendUint64(nil, uint64(tm.Unix()+math.MinInt64))
 		buf = binary.BigEndian.AppendUint32(buf, uint32(tm.Nanosecond()))
+	case kindSlice:
+		n := frv.Len()
+		bufs := make([][]byte, n)
+		for i := 0; i < n; i++ {
+			nbufs, err := packIndexKey(frv.Index(i))
+			if err != nil {
+				return nil, fmt.Errorf("packing element from slice field: %w", err)
+			}
+			if len(nbufs) != 1 {
+				return nil, fmt.Errorf("packing element from slice field resulted in multiple buffers (%d)", len(bufs))
+			}
+			bufs[i] = nbufs[0]
+		}
+		return bufs, nil
 	default:
-			return nil, nil, fmt.Errorf("internal error: bad type %v for index", frv.Type()) // todo: should be caught when making index type
+		return nil, fmt.Errorf("internal error: bad type %v for index", frv.Type()) // todo: should be caught when making index type
 	}
-		ik = append(ik, buf...)
-	}
-	n := len(ik)
-	ik = append(ik, pk...)
-	prek = ik[:n]
-	return prek, ik, nil
+	return [][]byte{buf}, nil
 }

vendor/github.com/mjl-/bstore/nonzero.go

@@ -12,207 +12,324 @@ func (ft fieldType) isZero(v reflect.Value) bool {
 		return true
 	}
 	if ft.Ptr {
-		return v.IsNil()
+		return v.IsZero()
 	}
 	switch ft.Kind {
 	case kindStruct:
-		for _, f := range ft.Fields {
+		for _, f := range ft.structFields {
 			if !f.Type.isZero(v.FieldByIndex(f.structField.Index)) {
 				return false
 			}
 		}
 		return true
 	}
+
 	// Use standard IsZero otherwise, also for kindBinaryMarshal.
 	return v.IsZero()
 }
 
-// checkNonzero compare ofields and nfields (from previous type schema vs newly
-// created type schema) for nonzero struct tag. If an existing field got a
-// nonzero struct tag added, we verify that there are indeed no nonzero values
-// in the database. If there are, we return ErrZero.
+// We ensure nonzero constraints when opening a database. An updated schema, with
+// added nonzero constraints, can mean all records have to be checked. With cyclic
+// types, we have to take care not to recurse, and for efficiency we want to only
+// check fields/types that are affected. Steps:
+//
+// - Go through each field of the struct, and recurse into the field types,
+//   gathering the types and newly nonzero fields.
+// - Propagate the need for nonzero checks to types that reference the changed
+//   types.
+// - By now, if there was a new nonzero constraint, the top-level type will be
+//   marked as needing a check, so we'll read through all records and check all the
+//   immediate newly nonzero fields of a type, and recurse into fields of types that
+//   are marked as needing a check.
+
+// nonzeroCheckType is tracked per reflect.Type that has been analysed (always the
+// non-pointer type, i.e. a pointer is dereferenced). These types can be cyclic. We
+// gather them for all types involved, including map and slice types and basic
+// types, but "newlyNonzero" and "fields" will only be set for structs.
+type nonzeroCheckType struct {
+	needsCheck bool
+
+	newlyNonzero []field // Fields in this type that have a new nonzero constraint themselves.
+	fields       []field // All fields in a struct type.
+
+	// Types that reference this type. Used to propagate needsCheck to the top.
+	referencedBy map[reflect.Type]struct{}
+}
+
+func (ct *nonzeroCheckType) markRefBy(t reflect.Type) {
+	if t != nil {
+		ct.referencedBy[t] = struct{}{}
+	}
+}
+
+// checkNonzero compares ofields (optional previous type schema) and nfields (new
+// type schema) for nonzero struct tags. If an existing field has a new nonzero
+// constraint, we verify that there are indeed no nonzero values in the existing
+// records. If there are, we return ErrZero. checkNonzero looks at (potentially
+// cyclic) types referenced by fields.
 func (tx *Tx) checkNonzero(st storeType, tv *typeVersion, ofields, nfields []field) error {
-	// First we gather paths that we need to check, so we can later simply
-	// execute those steps on all data we need to read.
-	paths := &follows{}
-next:
+	// Gather all new nonzero constraints on fields.
+	m := map[reflect.Type]*nonzeroCheckType{}
+	nonzeroCheckGather(m, st.Type, nil, ofields, nfields)
+
+	// Propagate the need for a check on all types due to a referenced type having a
+	// new nonzero constraint.
+	// todo: this can probably be done more elegantly, with fewer graph walks...
+	for t, ct := range m {
+		if ct.needsCheck {
+			nonzeroCheckPropagate(m, t, t, ct)
+		}
+	}
+
+	// If needsCheck wasn't propagated to the top-level, there was no new nonzero
+	// constraint, and we're not going to read all the data.  This is the common case
+	// when opening a database.
+	if !m[st.Type].needsCheck {
+		return nil
+	}
+
+	// Read through all data, and check the new nonzero constraint.
+	// todo optimize: if there are only top-level fields to check, and we have indices on those fields, we can use the index to check this without reading all data.
+	return checkNonzeroRecords(tx, st, tv, m)
+}
+
+// Walk down fields, gathering their types (including those they reference), and
+// marking needsCheck if any of a type's immediate field has a new nonzero
+// constraint. The need for a check is not propagated to referencing types by this
+// function.
+func nonzeroCheckGather(m map[reflect.Type]*nonzeroCheckType, t, refBy reflect.Type, ofields, nfields []field) {
+	ct := m[t]
+	if ct != nil {
+		// Already gathered, don't recurse, for cyclic types.
+		ct.markRefBy(refBy)
+		return
+	}
+	ct = &nonzeroCheckType{
+		fields:       nfields,
+		referencedBy: map[reflect.Type]struct{}{},
+	}
+	ct.markRefBy(refBy)
+	m[t] = ct
+
 	for _, f := range nfields {
-		for _, of := range ofields {
-			if f.Name == of.Name {
-				err := f.checkNonzeroGather(&of, paths)
-				if err != nil {
-					return err
+		// Check if this field is newly nonzero.
+		var of *field
+		for i := range ofields {
+			if f.Name == ofields[i].Name {
+				of = &ofields[i]
+				// Compare with existing field.
+				if f.Nonzero && !of.Nonzero {
+					ct.newlyNonzero = append(ct.newlyNonzero, f)
+					ct.needsCheck = true
 				}
-				continue next
+				break
 			}
 		}
-		if err := f.checkNonzeroGather(nil, paths); err != nil {
-			return err
-		}
+		// Check if this is a new field entirely, with nonzero constraint.
+		if of == nil && f.Nonzero {
+			ct.newlyNonzero = append(ct.newlyNonzero, f)
+			ct.needsCheck = true
 		}
 
-	if len(paths.paths) == 0 {
-		// Common case, not reading all data.
-		return nil
-	}
-
-	// Finally actually do the checks.
-	// todo: if there are only top-level fields to check, and we have an index, we can use the index check this without reading all data.
-	return tx.checkNonzeroPaths(st, tv, paths.paths)
-}
-
-type follow struct {
-	mapKey, mapValue bool
-	field            field
-}
-
-type follows struct {
-	current []follow
-	paths   [][]follow
-}
-
-func (f *follows) push(ff follow) {
-	f.current = append(f.current, ff)
-}
-
-func (f *follows) pop() {
-	f.current = f.current[:len(f.current)-1]
-}
-
-func (f *follows) add() {
-	f.paths = append(f.paths, append([]follow{}, f.current...))
-}
-
-func (f field) checkNonzeroGather(of *field, paths *follows) error {
-	paths.push(follow{field: f})
-	defer paths.pop()
-	if f.Nonzero && (of == nil || !of.Nonzero) {
-		paths.add()
-	}
+		// Descend into referenced types, adding references back to this type.
+		var oft *fieldType
 		if of != nil {
-		return f.Type.checkNonzeroGather(of.Type, paths)
+			oft = &of.Type
+		}
+		ft := f.structField.Type
+		nonzeroCheckGatherFieldType(m, ft, t, oft, f.Type)
 	}
-	return nil
 }
 
-func (ft fieldType) checkNonzeroGather(oft fieldType, paths *follows) error {
-	switch ft.Kind {
+// gather new nonzero constraints for type "t", which is referenced by "refBy" (and
+// will be marked as such). type "t" is described by "nft" and optionally
+// previously by "oft".
+func nonzeroCheckGatherFieldType(m map[reflect.Type]*nonzeroCheckType, t, refBy reflect.Type, oft *fieldType, nft fieldType) {
+	// If this is a pointer type, dereference the reflect type.
+	if nft.Ptr {
+		t = t.Elem()
+	}
+
+	if nft.Kind == kindStruct {
+		var fofields []field
+		if oft != nil {
+			fofields = oft.structFields
+		}
+		nonzeroCheckGather(m, t, refBy, fofields, nft.structFields)
+	}
+
+	// Mark this type as gathered, so we don't process it again if we recurse.
+	ct := m[t]
+	if ct != nil {
+		ct.markRefBy(refBy)
+		return
+	}
+	ct = &nonzeroCheckType{
+		fields:       nft.structFields,
+		referencedBy: map[reflect.Type]struct{}{},
+	}
+	ct.markRefBy(refBy)
+	m[t] = ct
+
+	switch nft.Kind {
 	case kindMap:
-		paths.push(follow{mapKey: true})
-		if err := ft.MapKey.checkNonzeroGather(*oft.MapKey, paths); err != nil {
-			return err
+		var koft, voft *fieldType
+		if oft != nil {
+			koft = oft.MapKey
+			voft = oft.MapValue
 		}
-		paths.pop()
-
-		paths.push(follow{mapValue: true})
-		if err := ft.MapValue.checkNonzeroGather(*oft.MapValue, paths); err != nil {
-			return err
-		}
-		paths.pop()
-
+		nonzeroCheckGatherFieldType(m, t.Key(), t, koft, *nft.MapKey)
+		nonzeroCheckGatherFieldType(m, t.Elem(), t, voft, *nft.MapValue)
 	case kindSlice:
-		err := ft.List.checkNonzeroGather(*oft.List, paths)
-		if err != nil {
-			return err
+		var loft *fieldType
+		if oft != nil {
+			loft = oft.ListElem
 		}
-	case kindStruct:
-	next:
-		for _, ff := range ft.Fields {
-			for _, off := range oft.Fields {
-				if ff.Name == off.Name {
-					err := ff.checkNonzeroGather(&off, paths)
-					if err != nil {
-						return err
+		nonzeroCheckGatherFieldType(m, t.Elem(), t, loft, *nft.ListElem)
+	case kindArray:
+		var loft *fieldType
+		if oft != nil {
+			loft = oft.ListElem
 		}
-					continue next
+		nonzeroCheckGatherFieldType(m, t.Elem(), t, loft, *nft.ListElem)
 	}
-			}
-			err := ff.checkNonzeroGather(nil, paths)
-			if err != nil {
-				return err
-			}
-		}
-
-	}
-	return nil
 }
 
-// checkNonzero reads through all records of a type, and checks that the fields
+// Propagate that type "t" is affected by a new nonzero constrained and needs to be
+// checked. The types referencing "t" are in ct.referencedBy. "origt" is the
+// starting type for this propagation.
+func nonzeroCheckPropagate(m map[reflect.Type]*nonzeroCheckType, origt, t reflect.Type, ct *nonzeroCheckType) {
+	for rt := range ct.referencedBy {
+		if rt == origt {
+			continue // End recursion.
+		}
+		m[rt].needsCheck = true
+		nonzeroCheckPropagate(m, origt, rt, m[rt])
+	}
+}
+
+// checkNonzeroPaths reads through all records of a type, and checks that the fields
 // indicated by paths are nonzero. If not, ErrZero is returned.
-func (tx *Tx) checkNonzeroPaths(st storeType, tv *typeVersion, paths [][]follow) error {
+func checkNonzeroRecords(tx *Tx, st storeType, tv *typeVersion, m map[reflect.Type]*nonzeroCheckType) error {
 	rb, err := tx.recordsBucket(st.Current.name, st.Current.fillPercent)
 	if err != nil {
 		return err
 	}
+
+	ctxDone := tx.ctx.Done()
+
 	return rb.ForEach(func(bk, bv []byte) error {
 		tx.stats.Records.Cursor++
 
+		select {
+		case <-ctxDone:
+			return tx.ctx.Err()
+		default:
+		}
+
+		// todo optimize: instead of parsing the full record, use the fieldmap to see if the value is nonzero.
 		rv, err := st.parseNew(bk, bv)
 		if err != nil {
 			return err
 		}
-		// todo optimization: instead of parsing the full record, use the fieldmap to see if the value is nonzero.
-		for _, path := range paths {
-			frv := rv.FieldByIndex(path[0].field.structField.Index)
-			if err := path[0].field.checkNonzero(frv, path[1:]); err != nil {
-				return err
-			}
-		}
-		return nil
+		ct := m[st.Type]
+		return checkNonzeroFields(m, st.Type, ct.newlyNonzero, ct.fields, rv)
 	})
 }
 
-func (f field) checkNonzero(rv reflect.Value, path []follow) error {
-	if len(path) == 0 {
-		if !f.Nonzero {
-			return fmt.Errorf("internal error: checkNonzero: expected field to have Nonzero set")
-		}
-		if f.Type.isZero(rv) {
+// checkNonzeroFields checks that the newly nonzero fields of a struct value are
+// indeed nonzero, and walks down referenced types, checking the constraint.
+func checkNonzeroFields(m map[reflect.Type]*nonzeroCheckType, t reflect.Type, newlyNonzero, fields []field, rv reflect.Value) error {
+	// Check the newly nonzero fields.
+	for _, f := range newlyNonzero {
+		frv := rv.FieldByIndex(f.structField.Index)
+		if f.Type.isZero(frv) {
 			return fmt.Errorf("%w: field %q", ErrZero, f.Name)
 		}
-		return nil
-	}
-	return f.Type.checkNonzero(rv, path)
-}
-
-func (ft fieldType) checkNonzero(rv reflect.Value, path []follow) error {
-	switch ft.Kind {
-	case kindMap:
-		follow := path[0]
-		path = path[1:]
-		key := follow.mapKey
-		if !key && !follow.mapValue {
-			return fmt.Errorf("internal error: following map, expected mapKey or mapValue, got %#v", follow)
 	}
 
-		iter := rv.MapRange()
-		for iter.Next() {
-			var err error
-			if key {
-				err = ft.MapKey.checkNonzero(iter.Key(), path)
-			} else {
-				err = ft.MapValue.checkNonzero(iter.Value(), path)
-			}
-			if err != nil {
+	// Descend into referenced types.
+	for _, f := range fields {
+		switch f.Type.Kind {
+		case kindMap, kindSlice, kindStruct, kindArray:
+			ft := f.structField.Type
+			if err := checkNonzeroFieldType(m, f.Type, ft, rv.FieldByIndex(f.structField.Index)); err != nil {
 				return err
 			}
 		}
+	}
+
+	return nil
+}
+
+// checkNonzeroFieldType walks down a value, and checks that its (struct) types
+// don't violate nonzero constraints.
+// Does not check whether the value itself is nonzero. If required, that has
+// already been checked.
+func checkNonzeroFieldType(m map[reflect.Type]*nonzeroCheckType, ft fieldType, t reflect.Type, rv reflect.Value) error {
+	if ft.Ptr {
+		t = t.Elem()
+	}
+	if !m[t].needsCheck {
+		return nil
+	}
+
+	if ft.Ptr && rv.IsZero() {
+		return nil
+	}
+
+	if ft.Ptr {
+		rv = rv.Elem()
+	}
+
+	unptr := func(t reflect.Type, ptr bool) reflect.Type {
+		if ptr {
+			return t.Elem()
+		}
+		return t
+	}
+
+	switch ft.Kind {
+	case kindMap:
+		kt := t.Key()
+		vt := t.Elem()
+		checkKey := m[unptr(kt, ft.MapKey.Ptr)].needsCheck
+		checkValue := m[unptr(vt, ft.MapValue.Ptr)].needsCheck
+		iter := rv.MapRange()
+		for iter.Next() {
+			if checkKey {
+				if err := checkNonzeroFieldType(m, *ft.MapKey, kt, iter.Key()); err != nil {
+					return err
+				}
+			}
+			if checkValue {
+				if err := checkNonzeroFieldType(m, *ft.MapValue, vt, iter.Value()); err != nil {
+					return err
+				}
+			}
+		}
+
 	case kindSlice:
+		et := t.Elem()
 		n := rv.Len()
 		for i := 0; i < n; i++ {
-			if err := ft.List.checkNonzero(rv.Index(i), path); err != nil {
+			if err := checkNonzeroFieldType(m, *ft.ListElem, et, rv.Index(i)); err != nil {
+				return err
+			}
+		}
+	case kindArray:
+		et := t.Elem()
+		n := ft.ArrayLength
+		for i := 0; i < n; i++ {
+			if err := checkNonzeroFieldType(m, *ft.ListElem, et, rv.Index(i)); err != nil {
 				return err
 			}
 		}
 	case kindStruct:
-		follow := path[0]
-		path = path[1:]
-		frv := rv.FieldByIndex(follow.field.structField.Index)
-		if err := follow.field.checkNonzero(frv, path); err != nil {
+		ct := m[t]
+		if err := checkNonzeroFields(m, t, ct.newlyNonzero, ct.fields, rv); err != nil {
 			return err
 		}
-	default:
-		return fmt.Errorf("internal error: checkNonzero with non-empty path, but kind %v", ft.Kind)
 	}
 	return nil
 }

vendor/github.com/mjl-/bstore/pack.go

@@ -17,14 +17,14 @@ type fieldmap struct {
 	buf    []byte // Bitmap, we write the next 0/1 at bit n.
 	n      int    // Fields seen so far.
 	offset int    // In final output, we write buf back after finish. Only relevant for packing.
-	Errorf func(format string, args ...any)
+	errorf func(format string, args ...any)
 }
 
 // add bit to fieldmap indicating if the field is nonzero.
 func (f *fieldmap) Field(nonzero bool) {
 	o := f.n / 8
 	if f.n >= f.max {
-		f.Errorf("internal error: too many fields, max %d", f.max)
+		f.errorf("internal error: too many fields, max %d", f.max)
 	}
 	if nonzero {
 		f.buf[o] |= 1 << (7 - f.n%8)
@@ -46,7 +46,7 @@ type packer struct {
 	popped    []*fieldmap // Completed fieldmaps, to be written back during finish.
 }
 
-func (p *packer) Errorf(format string, args ...any) {
+func (p *packer) errorf(format string, args ...any) {
 	panic(packErr{fmt.Errorf(format, args...)})
 }
 
@@ -54,7 +54,7 @@ func (p *packer) Errorf(format string, args ...any) {
 func (p *packer) PushFieldmap(n int) {
 	p.fieldmaps = append(p.fieldmaps, p.fieldmap)
 	buf := make([]byte, (n+7)/8)
-	p.fieldmap = &fieldmap{max: n, buf: buf, offset: p.offset, Errorf: p.Errorf}
+	p.fieldmap = &fieldmap{max: n, buf: buf, offset: p.offset, errorf: p.errorf}
 	p.Write(buf) // Updates offset. Write errors cause panic.
 }
 
@@ -62,7 +62,7 @@ func (p *packer) PushFieldmap(n int) {
 // bytes during finish.
 func (p *packer) PopFieldmap() {
 	if p.fieldmap.n != p.fieldmap.max {
-		p.Errorf("internal error: fieldmap n %d != max %d", p.fieldmap.n, p.fieldmap.max)
+		p.errorf("internal error: fieldmap n %d != max %d", p.fieldmap.n, p.fieldmap.max)
 	}
 	p.popped = append(p.popped, p.fieldmap)
 	p.fieldmap = p.fieldmaps[len(p.fieldmaps)-1]
@@ -73,7 +73,7 @@ func (p *packer) PopFieldmap() {
 // returning the final bytes representation of this record.
 func (p *packer) Finish() []byte {
 	if p.fieldmap != nil {
-		p.Errorf("internal error: leftover fieldmap during finish")
+		p.errorf("internal error: leftover fieldmap during finish")
 	}
 	buf := p.b.Bytes()
 	for _, f := range p.popped {
@@ -90,7 +90,7 @@ func (p *packer) Field(nonzero bool) {
 func (p *packer) Write(buf []byte) (int, error) {
 	n, err := p.b.Write(buf)
 	if err != nil {
-		p.Errorf("write: %w", err)
+		p.errorf("write: %w", err)
 	}
 	if n > 0 {
 		p.offset += n
@@ -149,11 +149,12 @@ func (tv typeVersion) pack(p *packer, rv reflect.Value) {
 		nrv := rv.FieldByIndex(f.structField.Index)
 		if f.Type.isZero(nrv) {
 			if f.Nonzero {
-				p.Errorf("%w: %q", ErrZero, f.Name)
+				p.errorf("%w: %q", ErrZero, f.Name)
 			}
 			p.Field(false)
 			// Pretend to pack to get the nonzero checks.
-			if nrv.IsValid() && (nrv.Kind() != reflect.Ptr || !nrv.IsNil()) {
+			// todo: we should be able to do nonzero-check without pretending to pack.
+			if nrv.IsValid() && (nrv.Kind() != reflect.Ptr || !nrv.IsZero()) {
 				f.Type.pack(&packer{b: &bytes.Buffer{}}, nrv)
 			}
 		} else {
@@ -176,7 +177,7 @@ func (ft fieldType) pack(p *packer, rv reflect.Value) {
 		v := rv
 		buf, err := v.Interface().(encoding.BinaryMarshaler).MarshalBinary()
 		if err != nil {
-			p.Errorf("marshalbinary: %w", err)
+			p.errorf("marshalbinary: %w", err)
 		}
 		p.AddBytes(buf)
 	case kindBool:
@@ -192,7 +193,7 @@ func (ft fieldType) pack(p *packer, rv reflect.Value) {
 	case kindInt:
 		v := rv.Int()
 		if v < math.MinInt32 || v > math.MaxInt32 {
-			p.Errorf("%w: int %d does not fit in int32", ErrParam, v)
+			p.errorf("%w: int %d does not fit in int32", ErrParam, v)
 		}
 		p.Varint(v)
 	case kindInt8, kindInt16, kindInt32, kindInt64:
@@ -202,7 +203,7 @@ func (ft fieldType) pack(p *packer, rv reflect.Value) {
 	case kindUint:
 		v := rv.Uint()
 		if v > math.MaxUint32 {
-			p.Errorf("%w: uint %d does not fit in uint32", ErrParam, v)
+			p.errorf("%w: uint %d does not fit in uint32", ErrParam, v)
 		}
 		p.Uvarint(v)
 	case kindFloat32:
@@ -214,7 +215,7 @@ func (ft fieldType) pack(p *packer, rv reflect.Value) {
 	case kindTime:
 		buf, err := rv.Interface().(time.Time).MarshalBinary()
 		if err != nil {
-			p.Errorf("%w: pack time: %s", ErrParam, err)
+			p.errorf("%w: pack time: %s", ErrParam, err)
 		}
 		p.AddBytes(buf)
 	case kindSlice:
@@ -223,15 +224,32 @@ func (ft fieldType) pack(p *packer, rv reflect.Value) {
 		p.PushFieldmap(n)
 		for i := 0; i < n; i++ {
 			nrv := rv.Index(i)
-			if ft.List.isZero(nrv) {
+			if ft.ListElem.isZero(nrv) {
 				p.Field(false)
 				// Pretend to pack to get the nonzero checks of the element.
-				if nrv.IsValid() && (nrv.Kind() != reflect.Ptr || !nrv.IsNil()) {
-					ft.List.pack(&packer{b: &bytes.Buffer{}}, nrv)
+				if nrv.IsValid() && (nrv.Kind() != reflect.Ptr || !nrv.IsZero()) {
+					ft.ListElem.pack(&packer{b: &bytes.Buffer{}}, nrv)
 				}
 			} else {
 				p.Field(true)
-				ft.List.pack(p, nrv)
+				ft.ListElem.pack(p, nrv)
+			}
+		}
+		p.PopFieldmap()
+	case kindArray:
+		n := ft.ArrayLength
+		p.PushFieldmap(n)
+		for i := 0; i < n; i++ {
+			nrv := rv.Index(i)
+			if ft.ListElem.isZero(nrv) {
+				p.Field(false)
+				// Pretend to pack to get the nonzero checks of the element.
+				if nrv.IsValid() && (nrv.Kind() != reflect.Ptr || !nrv.IsZero()) {
+					ft.ListElem.pack(&packer{b: &bytes.Buffer{}}, nrv)
+				}
+			} else {
+				p.Field(true)
+				ft.ListElem.pack(p, nrv)
 			}
 		}
 		p.PopFieldmap()
@@ -249,7 +267,7 @@ func (ft fieldType) pack(p *packer, rv reflect.Value) {
 			if ft.MapValue.isZero(v) {
 				p.Field(false)
 				// Pretend to pack to get the nonzero checks of the key type.
-				if v.IsValid() && (v.Kind() != reflect.Ptr || !v.IsNil()) {
+				if v.IsValid() && (v.Kind() != reflect.Ptr || !v.IsZero()) {
 					ft.MapValue.pack(&packer{b: &bytes.Buffer{}}, v)
 				}
 			} else {
@@ -259,16 +277,16 @@ func (ft fieldType) pack(p *packer, rv reflect.Value) {
 		}
 		p.PopFieldmap()
 	case kindStruct:
-		p.PushFieldmap(len(ft.Fields))
-		for _, f := range ft.Fields {
+		p.PushFieldmap(len(ft.structFields))
+		for _, f := range ft.structFields {
 			nrv := rv.FieldByIndex(f.structField.Index)
 			if f.Type.isZero(nrv) {
 				if f.Nonzero {
-					p.Errorf("%w: %q", ErrZero, f.Name)
+					p.errorf("%w: %q", ErrZero, f.Name)
 				}
 				p.Field(false)
 				// Pretend to pack to get the nonzero checks.
-				if nrv.IsValid() && (nrv.Kind() != reflect.Ptr || !nrv.IsNil()) {
+				if nrv.IsValid() && (nrv.Kind() != reflect.Ptr || !nrv.IsZero()) {
 					f.Type.pack(&packer{b: &bytes.Buffer{}}, nrv)
 				}
 			} else {
@@ -278,6 +296,6 @@ func (ft fieldType) pack(p *packer, rv reflect.Value) {
 		}
 		p.PopFieldmap()
 	default:
-		p.Errorf("internal error: unhandled field type") // should be prevented when registering type
+		p.errorf("internal error: unhandled field type") // should be prevented when registering type
 	}
 }

vendor/github.com/mjl-/bstore/parse.go

@@ -119,7 +119,7 @@ func (st storeType) parse(rv reflect.Value, buf []byte) (rerr error) {
 	tv.parse(p, rv)
 
 	if len(p.buf) != 0 {
-		return fmt.Errorf("%w: leftover data after parsing", ErrStore)
+		return fmt.Errorf("%w: leftover data after parsing (%d, %x %q)", ErrStore, len(p.buf), p.buf, p.buf)
 	}
 
 	return nil
@@ -173,7 +173,8 @@ func (tv typeVersion) parse(p *parser, rv reflect.Value) {
 
 // parse a nonzero fieldType.
 func (ft fieldType) parse(p *parser, rv reflect.Value) {
-	// Because we allow schema changes from ptr to nonptr, rv can be a pointer or direct value regardless of ft.Ptr.
+	// Because we allow schema changes from ptr to nonptr, rv can be a
+	// pointer or direct value regardless of ft.Ptr.
 	if rv.Kind() == reflect.Ptr {
 		nrv := reflect.New(rv.Type().Elem())
 		rv.Set(nrv)
@@ -239,10 +240,18 @@ func (ft fieldType) parse(p *parser, rv reflect.Value) {
 		slc := reflect.MakeSlice(rv.Type(), n, n)
 		for i := 0; i < int(n); i++ {
 			if fm.Nonzero(i) {
-				ft.List.parse(p, slc.Index(i))
+				ft.ListElem.parse(p, slc.Index(i))
 			}
 		}
 		rv.Set(slc)
+	case kindArray:
+		n := ft.ArrayLength
+		fm := p.Fieldmap(n)
+		for i := 0; i < n; i++ {
+			if fm.Nonzero(i) {
+				ft.ListElem.parse(p, rv.Index(i))
+			}
+		}
 	case kindMap:
 		un := p.Uvarint()
 		n := p.checkInt(un)
@@ -259,11 +268,13 @@ func (ft fieldType) parse(p *parser, rv reflect.Value) {
 		}
 		rv.Set(mp)
 	case kindStruct:
-		fm := p.Fieldmap(len(ft.Fields))
+		fm := p.Fieldmap(len(ft.structFields))
 		strct := reflect.New(rv.Type()).Elem()
-		for i, f := range ft.Fields {
+		for i, f := range ft.structFields {
 			if f.structField.Type == nil {
+				if fm.Nonzero(i) {
 					f.Type.skip(p)
+				}
 				continue
 			}
 			if fm.Nonzero(i) {
@@ -303,7 +314,15 @@ func (ft fieldType) skip(p *parser) {
 		fm := p.Fieldmap(n)
 		for i := 0; i < n; i++ {
 			if fm.Nonzero(i) {
-				ft.List.skip(p)
+				ft.ListElem.skip(p)
+			}
+		}
+	case kindArray:
+		n := ft.ArrayLength
+		fm := p.Fieldmap(n)
+		for i := 0; i < n; i++ {
+			if fm.Nonzero(i) {
+				ft.ListElem.skip(p)
 			}
 		}
 	case kindMap:
@@ -317,8 +336,8 @@ func (ft fieldType) skip(p *parser) {
 			}
 		}
 	case kindStruct:
-		fm := p.Fieldmap(len(ft.Fields))
-		for i, f := range ft.Fields {
+		fm := p.Fieldmap(len(ft.structFields))
+		for i, f := range ft.structFields {
 			if fm.Nonzero(i) {
 				f.Type.skip(p)
 			}

vendor/github.com/mjl-/bstore/plan.go

@@ -7,6 +7,12 @@ import (
 	"sort"
 )
 
+// todo: cache query plans? perhaps explicitly through something like a prepared statement. the current plan includes values in keys,start,stop, which would need to be calculated for each execution. should benchmark time spent in planning first.
+// todo optimize: handle multiple sorts with multikey indices if they match
+// todo optimize: combine multiple filter (not)in/equals calls for same field
+// todo optimize: efficiently pack booleans in an index (eg for Message.Flags), and use it to query.
+// todo optimize: do multiple range scans if necessary when we can use an index for an equal check with multiple values.
+
 // Plan represents a plan to execute a query, possibly using a simple/quick
 // bucket "get" or cursor scan (forward/backward) on either the records or an
 // index.
@@ -31,9 +37,9 @@ type plan[T any] struct {
 	startInclusive bool   // If the start and stop values are inclusive or exclusive.
 	stopInclusive  bool
 
-	// Filter we need to apply on after retrieving the record. If all
-	// original filters from a query were handled by "keys" above, or by a
-	// range scan, this field is empty.
+	// Filter we need to apply after retrieving the record. If all original filters
+	// from a query were handled by "keys" above, or by a range scan, this field is
+	// empty.
 	filters []filter[T]
 
 	// Orders we need to apply after first retrieving all records. As with
@@ -73,8 +79,7 @@ func (q *Query[T]) selectPlan() (*plan[T], error) {
 	// filter per field. If there are multiple, we would use the last one.
 	// That's okay, we'll filter records out when we execute the leftover
 	// filters. Probably not common.
-	// This is common for filterEqual and filterIn on
-	// fields that have a unique index.
+	// This is common for filterEqual and filterIn on fields that have a unique index.
 	equalsIn := map[string]*filter[T]{}
 	for i := range q.xfilters {
 		ff := &q.xfilters[i]
@@ -98,8 +103,8 @@ indices:
 			}
 		}
 		// Calculate all keys that we need to retrieve from the index.
-		// todo optimization: if there is a sort involving these fields, we could do the sorting before fetching data.
-		// todo optimization: we can generate the keys on demand, will help when limit is in use: we are not generating all keys.
+		// todo optimize: if there is a sort involving these fields, we could do the sorting before fetching data.
+		// todo optimize: we can generate the keys on demand, will help when limit is in use: we are not generating all keys.
 		var keys [][]byte
 		var skipFilters []*filter[T] // Filters to remove from the full list because they are handled by quering the index.
 		for i, f := range idx.Fields {
@@ -116,12 +121,15 @@ indices:
 			}
 			fekeys := make([][]byte, len(rvalues))
 			for j, fv := range rvalues {
-				key, _, err := packIndexKeys([]reflect.Value{fv}, nil)
+				ikl, err := packIndexKeys([]reflect.Value{fv}, nil)
 				if err != nil {
 					q.error(err)
 					return nil, err
 				}
-				fekeys[j] = key
+				if len(ikl) != 1 {
+					return nil, fmt.Errorf("internal error: multiple index keys for unique index (%d)", len(ikl))
+				}
+				fekeys[j] = ikl[0].pre
 			}
 			if i == 0 {
 				keys = fekeys
@@ -148,22 +156,26 @@ indices:
 	}
 
 	// Try all other indices. We treat them all as non-unique indices now.
-	// We want to use the one with as many "equal" prefix fields as
-	// possible. Then we hope to use a scan on the remaining, either
-	// because of a filterCompare, or for an ordering. If there is a limit,
-	// orderings are preferred over compares.
+	// We want to use the one with as many "equal" or "inslice" field filters as
+	// possible. Then we hope to use a scan on the remaining, either because of a
+	// filterCompare, or for an ordering. If there is a limit, orderings are preferred
+	// over compares.
 	equals := map[string]*filter[T]{}
+	inslices := map[string]*filter[T]{}
 	for i := range q.xfilters {
 		ff := &q.xfilters[i]
 		switch f := (*ff).(type) {
 		case filterEqual[T]:
 			equals[f.field.Name] = ff
+		case filterInSlice[T]:
+			inslices[f.field.Name] = ff
 		}
 	}
 
-	// We are going to generate new plans, and keep the new one if it is better than what we have.
+	// We are going to generate new plans, and keep the new one if it is better than
+	// what we have so far.
 	var p *plan[T]
-	var nequals int
+	var nexact int
 	var nrange int
 	var ordered bool
 
@@ -181,18 +193,27 @@ indices:
 			}
 		} else {
 			packKeys = func(l []reflect.Value) ([]byte, error) {
-				key, _, err := packIndexKeys(l, nil)
-				return key, err
+				ikl, err := packIndexKeys(l, nil)
+				if err != nil {
+					return nil, err
+				}
+				if err == nil && len(ikl) != 1 {
+					return nil, fmt.Errorf("internal error: multiple index keys for exact filters, %v", ikl)
+				}
+				return ikl[0].pre, nil
 			}
 		}
 
-		var neq = 0
+		var nex = 0
 		// log.Printf("idx %v", idx)
 		var skipFilters []*filter[T]
 		for _, f := range idx.Fields {
-			if ff, ok := equals[f.Name]; ok {
-				skipFilters = append(skipFilters, ff)
-				neq++
+			if equals[f.Name] != nil && f.Type.Kind != kindSlice {
+				skipFilters = append(skipFilters, equals[f.Name])
+				nex++
+			} else if inslices[f.Name] != nil && f.Type.Kind == kindSlice {
+				skipFilters = append(skipFilters, inslices[f.Name])
+				nex++
 			} else {
 				break
 			}
@@ -203,8 +224,8 @@ indices:
 		var nrng int
 		var order *order
 		orders := q.xorders
-		if neq < len(idx.Fields) {
-			nf := idx.Fields[neq]
+		if nex < len(idx.Fields) {
+			nf := idx.Fields[nex]
 			for i := range q.xfilters {
 				ff := &q.xfilters[i]
 				switch f := (*ff).(type) {
@@ -230,7 +251,7 @@ indices:
 			}
 
 			// See if it can be used for ordering.
-			// todo optimization: we could use multiple orders
+			// todo optimize: we could use multiple orders
 			if len(orders) > 0 && orders[0].field.Name == nf.Name {
 				order = &orders[0]
 				orders = orders[1:]
@@ -238,23 +259,29 @@ indices:
 		}
 
 		// See if this is better than what we had.
-		if !(neq > nequals || (neq == nequals && (nrng > nrange || order != nil && !ordered && (q.xlimit > 0 || nrng == nrange)))) {
-			// log.Printf("plan not better, neq %d, nrng %d, limit %d, order %v ordered %v", neq, nrng, q.limit, order, ordered)
+		if !(nex > nexact || (nex == nexact && (nrng > nrange || order != nil && !ordered && (q.xlimit > 0 || nrng == nrange)))) {
+			// log.Printf("plan not better, nex %d, nrng %d, limit %d, order %v ordered %v", nex, nrng, q.limit, order, ordered)
 			return nil
 		}
-		nequals = neq
+		nexact = nex
 		nrange = nrng
 		ordered = order != nil
 
 		// Calculate the prefix key.
 		var kvalues []reflect.Value
-		for i := 0; i < neq; i++ {
+		for i := 0; i < nex; i++ {
 			f := idx.Fields[i]
-			kvalues = append(kvalues, (*equals[f.Name]).(filterEqual[T]).rvalue)
+			var v reflect.Value
+			if f.Type.Kind != kindSlice {
+				v = (*equals[f.Name]).(filterEqual[T]).rvalue
+			} else {
+				v = (*inslices[f.Name]).(filterInSlice[T]).rvalue
+			}
+			kvalues = append(kvalues, v)
 		}
 		var key []byte
 		var err error
-		if neq > 0 {
+		if nex > 0 {
 			key, err = packKeys(kvalues)
 			if err != nil {
 				return err

vendor/github.com/mjl-/bstore/query.go

@@ -1,6 +1,8 @@
 package bstore
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"reflect"
 )
@@ -23,6 +25,8 @@ import (
 //
 // A Query is not safe for concurrent use.
 type Query[T any] struct {
+	ctx        context.Context
+	ctxDone    <-chan struct{} // ctx.Done(), kept here for fast access.
 	st         storeType       // Of T.
 	pkType     reflect.Type    // Shortcut for st.Current.Fields[0].
 	xtx        *Tx             // If nil, a new transaction is automatically created from db. Using a tx goes through tx() one exists.
@@ -99,6 +103,14 @@ type filterNotIn[T any] struct {
 
 func (filterNotIn[T]) filter() {}
 
+// For matching one of the values in a field that is a slice of the same type.
+type filterInSlice[T any] struct {
+	field  field // Of field type, a slice.
+	rvalue reflect.Value
+}
+
+func (filterInSlice[T]) filter() {}
+
 type compareOp byte
 
 const (
@@ -158,21 +170,35 @@ func (p *pair[T]) Value(e *exec[T]) (T, error) {
 // QueryDB returns a new Query for type T. When an operation on the query is
 // executed, a read-only/writable transaction is created as appropriate for the
 // operation.
-func QueryDB[T any](db *DB) *Query[T] {
+func QueryDB[T any](ctx context.Context, db *DB) *Query[T] {
 	// We lock db for storeTypes. We keep it locked until Query is done.
 	db.typesMutex.RLock()
 	q := &Query[T]{xdb: db}
-	q.init(db)
+	q.init(ctx, db)
 	return q
 }
 
-// Query returns a new Query that operates on type T using transaction tx.
+// QueryTx returns a new Query that operates on type T using transaction tx.
+// The context of the transaction is used for the query.
 func QueryTx[T any](tx *Tx) *Query[T] {
 	// note: Since we are in a transaction, we already hold an rlock on the
 	// db types.
 	q := &Query[T]{xtx: tx}
-	q.init(tx.db)
+	if tx.err != nil {
+		q.err = tx.err
 		return q
+	}
+	q.init(tx.ctx, tx.db)
+	return q
+}
+
+func (q *Query[T]) ctxErr() error {
+	select {
+	case <-q.ctxDone:
+		return q.ctx.Err()
+	default:
+		return nil
+	}
 }
 
 // Stats returns the current statistics for this query. When a query finishes,
@@ -182,7 +208,7 @@ func (q *Query[T]) Stats() Stats {
 	return q.stats
 }
 
-func (q *Query[T]) init(db *DB) {
+func (q *Query[T]) init(ctx context.Context, db *DB) {
 	var v T
 	t := reflect.TypeOf(v)
 	if t.Kind() != reflect.Struct {
@@ -194,6 +220,11 @@ func (q *Query[T]) init(db *DB) {
 		q.stats.LastType = q.st.Name
 		q.pkType = q.st.Current.Fields[0].structField.Type
 	}
+	q.ctx = ctx
+	q.ctxDone = ctx.Done()
+	if err := q.ctxErr(); q.err == nil && err != nil {
+		q.err = err
+	}
 }
 
 func (q *Query[T]) tx(write bool) (*Tx, error) {
@@ -207,7 +238,7 @@ func (q *Query[T]) tx(write bool) (*Tx, error) {
 			q.error(err)
 			return nil, q.err
 		}
-		q.xtx = &Tx{db: q.xdb, btx: tx}
+		q.xtx = &Tx{ctx: q.ctx, db: q.xdb, btx: tx}
 		if write {
 			q.stats.Writes++
 		} else {
@@ -308,6 +339,11 @@ func (q *Query[T]) checkErr() bool {
 		// Probably the result of using a Query zero value.
 		q.errorf("%w: invalid query, use QueryDB or QueryTx to make a query", ErrParam)
 	}
+	if q.err == nil {
+		if err := q.ctxErr(); err != nil {
+			q.err = err
+		}
+	}
 	return q.err == nil
 }
 
@@ -365,7 +401,10 @@ func (q *Query[T]) foreachKey(write, value bool, fn func(bk []byte, v T) error)
 			return nil
 		} else if err != nil {
 			return err
-		} else if err := fn(bk, v); err != nil {
+		} else if err := fn(bk, v); err == StopForEach {
+			q.error(ErrFinished)
+			return nil
+		} else if err != nil {
 			q.error(err)
 			return err
 		}
@@ -436,14 +475,14 @@ var convertFieldKinds = map[convertKinds]struct{}{
 // Check type of value for field and return a reflect value that can directly be set on the field.
 // If the field is a pointer, we allow non-pointers and convert them.
 // We require value to be of a type that can be converted without loss of precision to the type of field.
-func (q *Query[T]) prepareValue(fname string, ft fieldType, sf reflect.StructField, rv reflect.Value) (reflect.Value, bool) {
+func (q *Query[T]) prepareValue(fname string, ft fieldType, st reflect.Type, rv reflect.Value) (reflect.Value, bool) {
 	if !rv.IsValid() {
 		q.errorf("%w: invalid value", ErrParam)
 		return rv, false
 	}
 	// Quick check first.
 	t := rv.Type()
-	if t == sf.Type {
+	if t == st {
 		return rv, true
 	}
 	if !ft.Ptr && rv.Kind() == reflect.Ptr {
@@ -461,14 +500,14 @@ func (q *Query[T]) prepareValue(fname string, ft fieldType, sf reflect.StructFie
 		return reflect.Value{}, false
 	}
 	if k != ft.Kind {
-		dt := sf.Type
+		dt := st
 		if ft.Ptr {
 			dt = dt.Elem()
 		}
 		rv = rv.Convert(dt)
 	}
 	if ft.Ptr && rv.Kind() != reflect.Ptr {
-		nv := reflect.New(sf.Type.Elem())
+		nv := reflect.New(st.Elem())
 		nv.Elem().Set(rv)
 		rv = nv
 	}
@@ -654,7 +693,7 @@ func (q *Query[T]) filterEqual(fieldName string, values []any, not bool) {
 		return
 	}
 	if len(values) == 1 {
-		rv, ok := q.prepareValue(ff.Name, ff.Type, ff.structField, reflect.ValueOf(values[0]))
+		rv, ok := q.prepareValue(ff.Name, ff.Type, ff.structField.Type, reflect.ValueOf(values[0]))
 		if !ok {
 			return
 		}
@@ -667,7 +706,7 @@ func (q *Query[T]) filterEqual(fieldName string, values []any, not bool) {
 	}
 	rvs := make([]reflect.Value, len(values))
 	for i, value := range values {
-		rv, ok := q.prepareValue(ff.Name, ff.Type, ff.structField, reflect.ValueOf(value))
+		rv, ok := q.prepareValue(ff.Name, ff.Type, ff.structField.Type, reflect.ValueOf(value))
 		if !ok {
 			return
 		}
@@ -680,6 +719,42 @@ func (q *Query[T]) filterEqual(fieldName string, values []any, not bool) {
 	}
 }
 
+// FilterIn selects records that have one of values of the string slice fieldName.
+//
+// If fieldName has an index, it is used to select rows.
+//
+// Note: Value must be a compatible type for comparison with the elements of
+// fieldName. Go constant numbers become ints, which are not compatible with uint
+// or float types.
+func (q *Query[T]) FilterIn(fieldName string, value any) *Query[T] {
+	if !q.checkErr() {
+		return q
+	}
+	ff, ok := q.lookupField(fieldName)
+	if !ok {
+		return q
+	}
+	if ff.Type.Ptr {
+		q.errorf("%w: cannot compare pointer values", ErrParam)
+		return q
+	}
+	if ff.Type.Kind != kindSlice {
+		q.errorf("%w: field for FilterIn must be a slice", ErrParam)
+		return q
+	}
+	et := ff.Type.ListElem
+	if et.Ptr {
+		q.errorf("%w: cannot compare element pointer values", ErrParam)
+		return q
+	}
+	rv, ok := q.prepareValue(ff.Name, *et, ff.structField.Type.Elem(), reflect.ValueOf(value))
+	if !ok {
+		return q
+	}
+	q.addFilter(filterInSlice[T]{ff, rv})
+	return q
+}
+
 // FilterGreater selects records that have fieldName > value.
 //
 // Note: Value must be a compatible type for comparison with fieldName. Go
@@ -716,7 +791,7 @@ func (q *Query[T]) filterCompare(fieldName string, op compareOp, value reflect.V
 		q.errorf("%w: cannot compare %s", ErrParam, ff.Type.Kind)
 		return q
 	}
-	rv, ok := q.prepareValue(ff.Name, ff.Type, ff.structField, value)
+	rv, ok := q.prepareValue(ff.Name, ff.Type, ff.structField.Type, value)
 	if !ok {
 		return q
 	}
@@ -831,7 +906,8 @@ func (q *Query[T]) gather(v T, rv reflect.Value) {
 	}
 }
 
-// Err returns if an error is set on the query. Can happen for invalid filters.
+// Err returns if an error is set on the query. Can happen for invalid filters or
+// canceled contexts.
 // Finished queries return ErrFinished.
 func (q *Query[T]) Err() error {
 	q.checkErr()
@@ -979,7 +1055,7 @@ next:
 			if i == 0 {
 				return 0, fmt.Errorf("%w: cannot update primary key", ErrParam)
 			}
-			rv, ok := q.prepareValue(f.Name, f.Type, f.structField, reflect.ValueOf(value))
+			rv, ok := q.prepareValue(f.Name, f.Type, f.structField.Type, reflect.ValueOf(value))
 			if !ok {
 				return 0, q.err
 			}
@@ -991,7 +1067,7 @@ next:
 			if ef.Name != name {
 				continue
 			}
-			rv, ok := q.prepareValue(ef.Name, ef.Type, ef.structField, reflect.ValueOf(value))
+			rv, ok := q.prepareValue(ef.Name, ef.Type, ef.structField.Type, reflect.ValueOf(value))
 			if !ok {
 				return 0, q.err
 			}
@@ -1051,6 +1127,8 @@ func (q *Query[T]) IDs(idsptr any) (rerr error) {
 	return nil
 }
 
+// todo: should we have an iteration object that we can call Next and NextID on?
+
 // Next fetches the next record, moving the cursor forward.
 //
 // ErrAbsent is returned if no more records match.
@@ -1116,7 +1194,13 @@ func (q *Query[T]) Exists() (exists bool, rerr error) {
 	return err == nil, err
 }
 
+// StopForEach is an error value that, if returned by the function passed to
+// Query.ForEach, stops further iterations.
+var StopForEach error = errors.New("stop foreach")
+
 // ForEach calls fn on each selected record.
+// If fn returns StopForEach, ForEach stops iterating, so no longer calls fn,
+// and returns nil.
 func (q *Query[T]) ForEach(fn func(value T) error) (rerr error) {
 	defer q.finish(&rerr)
 	q.checkNotNext()

vendor/github.com/mjl-/bstore/register.go

@@ -2,9 +2,12 @@ package bstore
 
 import (
 	"bytes"
+	"context"
 	"encoding/binary"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"os"
 	"reflect"
 	"sort"
 	"strconv"
@@ -14,10 +17,22 @@ import (
 	bolt "go.etcd.io/bbolt"
 )
 
+// todo: implement changing PK type, eg to wider int. requires rewriting all values, and removing old typeVersions.
+// todo: allow schema change between []byte and string?
+// todo: allow more schema changes, eg int to string, bool to int or string, int to bool, perhaps even string to int/bool. and between structs and maps. would require rewriting the records.
+
 const (
+	// First version.
 	ondiskVersion1 = 1
+
+	// With support for cyclic types, adding typeField.FieldsTypeSeq to
+	// define/reference types. Only used when a type has a field that references another
+	// struct type.
+	ondiskVersion2 = 2
 )
 
+var errSchemaCheck = errors.New("schema check")
+
 // Register registers the Go types of each value in typeValues for use with the
 // database. Each value must be a struct, not a pointer.
 //
@@ -30,7 +45,11 @@ const (
 //
 // Register can be called multiple times, with different types. But types that
 // reference each other must be registered in the same call to Registers.
-func (db *DB) Register(typeValues ...any) error {
+//
+// To help during development, if environment variable "bstore_schema_check" is set
+// to "changed", an error is returned if there is no schema change. If it is set to
+// "unchanged", an error is returned if there was a schema change.
+func (db *DB) Register(ctx context.Context, typeValues ...any) error {
 	// We will drop/create new indices as needed. For changed indices, we drop
 	// and recreate. E.g. if an index becomes a unique index, or if a field in
 	// an index changes.  These values map type and index name to their index.
@@ -41,7 +60,10 @@ func (db *DB) Register(typeValues ...any) error {
 	ntypeversions := map[string]*typeVersion{} // New typeversions, through new types or updated versions of existing types.
 	registered := map[string]*storeType{}      // Registered in this call.
 
-	return db.Write(func(tx *Tx) error {
+	checkSchemaChanged := os.Getenv("bstore_schema_check") == "changed"
+	checkSchemaUnchanged := os.Getenv("bstore_schema_check") == "unchanged"
+
+	return db.Write(ctx, func(tx *Tx) error {
 		for _, t := range typeValues {
 			rt := reflect.TypeOf(t)
 			if rt.Kind() != reflect.Struct {
@@ -118,6 +140,11 @@ func (db *DB) Register(typeValues ...any) error {
 
 			// Decide if we need to add a new typeVersion to the database. I.e. a new type schema.
 			if st.Current == nil || !st.Current.typeEqual(*tv) {
+				if checkSchemaUnchanged {
+					return fmt.Errorf("%w: schema changed but bstore_schema_check=unchanged is set (type %v)", errSchemaCheck, st.Name)
+				}
+				checkSchemaChanged = false // After registering types, we check that it is false.
+
 				tv.Version = 1
 				if st.Current != nil {
 					tv.Version = st.Current.Version + 1
@@ -127,6 +154,13 @@ func (db *DB) Register(typeValues ...any) error {
 					return fmt.Errorf("internal error: packing schema for type %q", tv.name)
 				}
 
+				// Sanity check: parse the typeVersion again, and check that we think it is equal to the typeVersion just written.
+				if xtv, err := parseSchema(k, v); err != nil {
+					return fmt.Errorf("%w: parsing generated typeVersion: %v", ErrStore, err)
+				} else if !xtv.typeEqual(*tv) {
+					return fmt.Errorf("%w: generated typeVersion not equal to itself after pack and parse", ErrStore)
+				}
+
 				// note: we don't track types bucket operations in stats.
 				if err := tb.Put(k, v); err != nil {
 					return fmt.Errorf("storing new schema: %w", err)
@@ -202,6 +236,10 @@ func (db *DB) Register(typeValues ...any) error {
 			registered[st.Name] = &st
 		}
 
+		if checkSchemaChanged {
+			return fmt.Errorf("%w: schema did not change while bstore_schema_check=changed is set", errSchemaCheck)
+		}
+
 		// Check that referenced types exist, and make links in the referenced types.
 		for _, st := range registered {
 			tv := st.Current
@@ -237,7 +275,7 @@ func (db *DB) Register(typeValues ...any) error {
 		// We cannot just recalculate the ReferencedBy, because the whole point is to
 		// detect types that are missing in this Register.
 		updateReferencedBy := map[string]struct{}{}
-		for _, ntv := range ntypeversions {
+		for ntname, ntv := range ntypeversions {
 			otv := otypeversions[ntv.name] // Can be nil, on first register.
 
 			// Look for references that were added.
@@ -251,6 +289,66 @@ func (db *DB) Register(typeValues ...any) error {
 				if _, ok := registered[name].Current.ReferencedBy[ntv.name]; ok {
 					return fmt.Errorf("%w: type %q introduces reference to %q but is already marked as ReferencedBy in that type", ErrStore, ntv.name, name)
 				}
+
+				// Verify that the new reference does not violate the foreign key constraint.
+				var foundField bool
+				for _, f := range ntv.Fields {
+					for _, rname := range f.References {
+						if rname != name {
+							continue
+						}
+
+						foundField = true
+
+						// For newly added references, check they are valid.
+						b, err := tx.recordsBucket(ntname, ntv.fillPercent)
+						if err != nil {
+							return fmt.Errorf("%w: bucket for type %s with field with new reference: %v", ErrStore, ntname, err)
+						}
+
+						rb, err := tx.recordsBucket(name, registered[name].Current.fillPercent)
+						if err != nil {
+							return fmt.Errorf("%w: bucket for referenced type %s: %v", ErrStore, name, err)
+						}
+
+						nst := registered[ntname]
+						rv := reflect.New(nst.Type).Elem()
+						ctxDone := ctx.Done()
+						err = b.ForEach(func(bk, bv []byte) error {
+							tx.stats.Records.Cursor++
+
+							select {
+							case <-ctxDone:
+								return tx.ctx.Err()
+							default:
+							}
+
+							if err := nst.parse(rv, bv); err != nil {
+								return fmt.Errorf("parsing record for %s: %w", ntname, err)
+							}
+							frv := rv.FieldByIndex(f.structField.Index)
+							if frv.IsZero() {
+								return nil
+							}
+							rpk, err := packPK(frv)
+							if err != nil {
+								return fmt.Errorf("packing pk for referenced type %s: %w", name, err)
+							}
+							tx.stats.Records.Cursor++
+							if rb.Get(rpk) == nil {
+								return fmt.Errorf("%w: value %v not in %s", ErrReference, frv.Interface(), name)
+							}
+							return nil
+						})
+						if err != nil {
+							return fmt.Errorf("%w: ensuring referential integrity for newly added reference of %s.%s", err, ntname, f.Name)
+						}
+					}
+				}
+				if !foundField {
+					return fmt.Errorf("%w: could not find field causing newly referenced type %s in type %s", ErrStore, name, ntname)
+				}
+
 				// note: we are updating the previous tv's ReferencedBy, not tidy but it is safe.
 				registered[name].Current.ReferencedBy[ntv.name] = struct{}{}
 				updateReferencedBy[name] = struct{}{}
@@ -271,8 +369,10 @@ func (db *DB) Register(typeValues ...any) error {
 				if _, ok := ntv.references[name]; ok {
 					continue
 				}
-				if _, ok := registered[name].Current.ReferencedBy[ntv.name]; !ok {
-					return fmt.Errorf("%w: previously referenced type %q not present in %q", ErrStore, ntv.name, name)
+				if rtv, ok := registered[name]; !ok {
+					return fmt.Errorf("%w: type %q formerly referenced by %q not yet registered", ErrStore, name, ntv.name)
+				} else if _, ok := rtv.Current.ReferencedBy[ntv.name]; !ok {
+					return fmt.Errorf("%w: formerly referenced type %q missing from %q", ErrStore, name, ntv.name)
 				}
 				// note: we are updating the previous tv's ReferencedBy, not tidy but it is safe.
 				delete(registered[name].Current.ReferencedBy, ntv.name)
@@ -416,20 +516,29 @@ func (db *DB) Register(typeValues ...any) error {
 			}
 			ibkeys := make([][]key, len(idxs))
 
+			ctxDone := ctx.Done()
 			err = rb.ForEach(func(bk, bv []byte) error {
 				tx.stats.Records.Cursor++
 
+				select {
+				case <-ctxDone:
+					return tx.ctx.Err()
+				default:
+				}
+
 				rv := reflect.New(st.Type).Elem()
 				if err := st.parse(rv, bv); err != nil {
 					return fmt.Errorf("parsing record for index for %s: %w", name, err)
 				}
 
 				for i, idx := range idxs {
-					prek, ik, err := idx.packKey(rv, bk)
+					ikl, err := idx.packKey(rv, bk)
 					if err != nil {
 						return fmt.Errorf("creating key for %s.%s: %w", name, idx.Name, err)
 					}
-					ibkeys[i] = append(ibkeys[i], key{ik, uint16(len(prek))})
+					for _, ik := range ikl {
+						ibkeys[i] = append(ibkeys[i], key{ik.full, uint16(len(ik.pre))})
+					}
 				}
 				return nil
 			})
@@ -447,14 +556,14 @@ func (db *DB) Register(typeValues ...any) error {
 						prev := keys[i-1]
 						if bytes.Equal(prev.buf[:prev.pre], k.buf[:k.pre]) {
 							// Do quite a bit of work to make a helpful error message.
-							a := reflect.New(reflect.TypeOf(idx.tv.Fields[0].Type.zero(nil))).Elem()
-							b := reflect.New(reflect.TypeOf(idx.tv.Fields[0].Type.zero(nil))).Elem()
+							a := reflect.New(reflect.TypeOf(idx.tv.Fields[0].Type.zeroKey())).Elem()
+							b := reflect.New(reflect.TypeOf(idx.tv.Fields[0].Type.zeroKey())).Elem()
 							parsePK(a, prev.buf[prev.pre:]) // Ignore error, nothing to do.
 							parsePK(b, k.buf[k.pre:])       // Ignore error, nothing to do.
 							var dup []any
 							_, values, _ := idx.parseKey(k.buf, true)
 							for i := range values {
-								x := reflect.New(reflect.TypeOf(idx.Fields[i].Type.zero(nil))).Elem()
+								x := reflect.New(reflect.TypeOf(idx.Fields[i].Type.zeroKey())).Elem()
 								parsePK(x, values[i]) // Ignore error, nothing to do.
 								dup = append(dup, x.Interface())
 							}
@@ -502,8 +611,10 @@ func parseSchema(bk, bv []byte) (*typeVersion, error) {
 	if tv.Version != version {
 		return nil, fmt.Errorf("%w: version in schema %d does not match key %d", ErrStore, tv.Version, version)
 	}
-	if tv.OndiskVersion != ondiskVersion1 {
-		return nil, fmt.Errorf("internal error: OndiskVersion %d not supported", tv.OndiskVersion)
+	switch tv.OndiskVersion {
+	case ondiskVersion1, ondiskVersion2:
+	default:
+		return nil, fmt.Errorf("internal error: OndiskVersion %d not recognized/supported", tv.OndiskVersion)
 	}
 
 	// Fill references, used for comparing/checking schema updates.
@@ -514,12 +625,72 @@ func parseSchema(bk, bv []byte) (*typeVersion, error) {
 		}
 	}
 
+	// Resolve fieldType.structFields, for referencing defined types, used for
+	// supporting cyclic types. The type itself always implicitly has sequence 1.
+	seqFields := map[int][]field{1: tv.Fields}
+	origOndiskVersion := tv.OndiskVersion
+	for i := range tv.Fields {
+		if err := tv.resolveStructFields(seqFields, &tv.Fields[i].Type); err != nil {
+			return nil, fmt.Errorf("%w: resolving struct fields for referencing types: %v", ErrStore, err)
+		}
+	}
+	if tv.OndiskVersion != origOndiskVersion {
+		return nil, fmt.Errorf("%w: resolving cyclic types changed ondisk version from %d to %d", ErrStore, origOndiskVersion, tv.OndiskVersion)
+	}
+
 	return &tv, nil
 }
 
+// Resolve structFields in ft (and recursively), either by setting it to Fields
+// (common), or by setting it to the fields of a referenced type in case of a
+// cyclic data type.
+func (tv *typeVersion) resolveStructFields(seqFields map[int][]field, ft *fieldType) error {
+	if ft.Kind == kindStruct {
+		if ft.FieldsTypeSeq < 0 {
+			var ok bool
+			ft.structFields, ok = seqFields[-ft.FieldsTypeSeq]
+			if !ok {
+				return fmt.Errorf("reference to undefined FieldsTypeSeq %d (n %d)", -ft.FieldsTypeSeq, len(seqFields))
+			}
+			if len(ft.DefinitionFields) != 0 {
+				return fmt.Errorf("reference to FieldsTypeSeq while also defining fields")
+			}
+		} else if ft.FieldsTypeSeq > 0 {
+			if _, ok := seqFields[ft.FieldsTypeSeq]; ok {
+				return fmt.Errorf("duplicate definition of FieldsTypeSeq %d (n %d)", ft.FieldsTypeSeq, len(seqFields))
+			}
+			seqFields[ft.FieldsTypeSeq] = ft.DefinitionFields
+			ft.structFields = ft.DefinitionFields
+		}
+		// note: ondiskVersion1 does not have/use this field, so it defaults to 0.
+		if ft.FieldsTypeSeq == 0 {
+			ft.structFields = ft.DefinitionFields
+		}
+
+		for i := range ft.DefinitionFields {
+			if err := tv.resolveStructFields(seqFields, &ft.DefinitionFields[i].Type); err != nil {
+				return err
+			}
+		}
+	}
+
+	xftl := []*fieldType{ft.MapKey, ft.MapValue, ft.ListElem}
+	for _, xft := range xftl {
+		if xft == nil {
+			continue
+		}
+		if err := tv.resolveStructFields(seqFields, xft); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // packSchema returns a key and value to store in the types bucket.
 func packSchema(tv *typeVersion) ([]byte, []byte, error) {
-	if tv.OndiskVersion != ondiskVersion1 {
+	switch tv.OndiskVersion {
+	case ondiskVersion1, ondiskVersion2:
+	default:
 		return nil, nil, fmt.Errorf("internal error: invalid OndiskVersion %d", tv.OndiskVersion)
 	}
 	v, err := json.Marshal(tv)
@@ -540,12 +711,17 @@ func gatherTypeVersion(t reflect.Type) (*typeVersion, error) {
 	}
 	tv := &typeVersion{
 		Version:       0,              // Set by caller.
-		OndiskVersion: ondiskVersion1, // Current on-disk format.
+		OndiskVersion: ondiskVersion2, // When opening a database with ondiskVersion1, we add a new typeVersion.
 		ReferencedBy:  map[string]struct{}{},
 		name:          tname,
 		fillPercent:   0.5,
 	}
-	tv.Fields, tv.embedFields, err = gatherTypeFields(t, true, true, false)
+
+	// The type being parsed implicitly has sequence 1. Next struct types will be
+	// assigned the next value (based on length of typeseqs). FieldTypes referencing
+	// another type are resolved below, after all fields have been gathered.
+	typeSeqs := map[reflect.Type]int{t: 1}
+	tv.Fields, tv.embedFields, err = gatherTypeFields(typeSeqs, t, true, true, false, true)
 	if err != nil {
 		return nil, err
 	}
@@ -562,6 +738,15 @@ func gatherTypeVersion(t reflect.Type) (*typeVersion, error) {
 		}
 	}
 
+	// Resolve structFields for the typeFields that reference an earlier defined type,
+	// using the same function as used when loading a type from disk.
+	seqFields := map[int][]field{1: tv.Fields}
+	for i := range tv.Fields {
+		if err := tv.resolveStructFields(seqFields, &tv.Fields[i].Type); err != nil {
+			return nil, fmt.Errorf("%w: resolving struct fields for referencing types: %v", ErrStore, err)
+		}
+	}
+
 	// Find indices.
 	tv.Indices = map[string]*index{}
 
@@ -572,6 +757,7 @@ func gatherTypeVersion(t reflect.Type) (*typeVersion, error) {
 		}
 		idx = &index{unique, iname, nil, tv}
 		tv.Indices[iname] = idx
+		nslice := 0
 		for _, f := range fields {
 			// todo: can we have a unique index on bytes? seems like this should be possible to have max 1 []byte in an index key, only to be used for unique get plans.
 			if f.Type.Ptr {
@@ -579,6 +765,14 @@ func gatherTypeVersion(t reflect.Type) (*typeVersion, error) {
 			}
 			switch f.Type.Kind {
 			case kindBool, kindInt8, kindInt16, kindInt32, kindInt64, kindInt, kindUint8, kindUint16, kindUint32, kindUint64, kindUint, kindString, kindTime:
+			case kindSlice:
+				nslice++
+				if nslice > 1 {
+					return fmt.Errorf("%w: can only have one slice field in index, for field %q", ErrType, f.Name)
+				}
+				if unique {
+					return fmt.Errorf("%w: can only use slice field %v in field %q as index without unique", ErrType, f.Type.Kind, f.Name)
+				}
 			default:
 				return fmt.Errorf("%w: cannot use type %v in field %q as index/unique", ErrType, f.Type.Kind, f.Name)
 			}
@@ -692,7 +886,7 @@ func gatherTypeVersion(t reflect.Type) (*typeVersion, error) {
 // field must not be ignored and be a valid primary key field (eg no pointer).
 // topLevel must be true only for the top-level struct fields, not for fields of
 // deeper levels. Deeper levels cannot have index/unique constraints.
-func gatherTypeFields(t reflect.Type, needFirst, topLevel, inMap bool) ([]field, []embed, error) {
+func gatherTypeFields(typeSeqs map[reflect.Type]int, t reflect.Type, needFirst, topLevel, inMap, newSeq bool) ([]field, []embed, error) {
 	var fields []field
 	var embedFields []embed
 
@@ -744,7 +938,7 @@ func gatherTypeFields(t reflect.Type, needFirst, topLevel, inMap bool) ([]field,
 		}
 		names[name] = struct{}{}
 
-		ft, err := gatherFieldType(sf.Type, inMap)
+		ft, err := gatherFieldType(typeSeqs, sf.Type, inMap, newSeq && !sf.Anonymous)
 		if err != nil {
 			return nil, nil, fmt.Errorf("field %q: %w", sf.Name, err)
 		}
@@ -817,11 +1011,13 @@ func gatherTypeFields(t reflect.Type, needFirst, topLevel, inMap bool) ([]field,
 			}
 		}
 
-		if sf.Anonymous {
+		// We don't store anonymous/embed fields, unless it is a cyclic type, because then
+		// we wouldn't have included any of its type's fields.
+		if sf.Anonymous && ft.FieldsTypeSeq == 0 {
 			e := embed{name, ft, sf}
 			embedFields = append(embedFields, e)
 		} else {
-			f := field{name, ft, nonzero, tags.List("ref"), defstr, def, sf, nil}
+			f := field{name, ft, nonzero, tags.List("ref"), defstr, def, sf, false, nil}
 			fields = append(fields, f)
 		}
 	}
@@ -842,12 +1038,13 @@ func checkKeyType(t reflect.Type) error {
 	return fmt.Errorf("%w: type %v not valid for primary key", ErrType, t)
 }
 
-func gatherFieldType(t reflect.Type, inMap bool) (fieldType, error) {
+func gatherFieldType(typeSeqs map[reflect.Type]int, t reflect.Type, inMap, newSeq bool) (fieldType, error) {
 	ft := fieldType{}
 	if t.Kind() == reflect.Ptr {
 		t = t.Elem()
 		ft.Ptr = true
 	}
+
 	k, err := typeKind(t)
 	if err != nil {
 		return fieldType{}, err
@@ -855,32 +1052,52 @@ func gatherFieldType(t reflect.Type, inMap bool) (fieldType, error) {
 	ft.Kind = k
 	switch ft.Kind {
 	case kindSlice:
-		l, err := gatherFieldType(t.Elem(), inMap)
+		l, err := gatherFieldType(typeSeqs, t.Elem(), inMap, newSeq)
 		if err != nil {
-			return ft, fmt.Errorf("list: %w", err)
+			return ft, fmt.Errorf("slice: %w", err)
 		}
-		ft.List = &l
+		ft.ListElem = &l
+	case kindArray:
+		l, err := gatherFieldType(typeSeqs, t.Elem(), inMap, newSeq)
+		if err != nil {
+			return ft, fmt.Errorf("array: %w", err)
+		}
+		ft.ListElem = &l
+		ft.ArrayLength = t.Len()
 	case kindMap:
-		kft, err := gatherFieldType(t.Key(), true)
+		kft, err := gatherFieldType(typeSeqs, t.Key(), true, newSeq)
 		if err != nil {
 			return ft, fmt.Errorf("map key: %w", err)
 		}
 		if kft.Ptr {
 			return ft, fmt.Errorf("%w: map key with pointer type not supported", ErrType)
 		}
-		vft, err := gatherFieldType(t.Elem(), true)
+		vft, err := gatherFieldType(typeSeqs, t.Elem(), true, newSeq)
 		if err != nil {
 			return ft, fmt.Errorf("map value: %w", err)
 		}
 		ft.MapKey = &kft
 		ft.MapValue = &vft
 	case kindStruct:
-		// note: we have no reason to gather embed field beyond top-level
-		fields, _, err := gatherTypeFields(t, false, false, inMap)
+		// If this is a known type, track a reference to the earlier defined type. Once the
+		// type with all Fields is fully parsed, the references will be resolved.
+		if seq, ok := typeSeqs[t]; ok {
+			ft.FieldsTypeSeq = -seq
+			return ft, nil
+		}
+
+		// If we are processing an anonymous (embed) field, we don't assign a new seq,
+		// because we won't be walking it when resolving again.
+		seq := len(typeSeqs) + 1
+		if newSeq {
+			typeSeqs[t] = seq
+			ft.FieldsTypeSeq = seq
+		}
+		fields, _, err := gatherTypeFields(typeSeqs, t, false, false, inMap, newSeq)
 		if err != nil {
 			return fieldType{}, fmt.Errorf("struct: %w", err)
 		}
-		ft.Fields = fields
+		ft.DefinitionFields = fields
 	}
 	return ft, nil
 }
@@ -941,6 +1158,10 @@ tv:
 }
 
 func (f *field) prepare(nfields []field, later, mvlater [][]field) {
+	if f.prepared {
+		return
+	}
+	f.prepared = true
 	for _, nf := range nfields {
 		if nf.Name == f.Name {
 			f.structField = nf.structField
@@ -954,26 +1175,26 @@ func (ft fieldType) laterFields() (later, mvlater []field) {
 		later, _ = ft.MapKey.laterFields()
 		mvlater, _ = ft.MapValue.laterFields()
 		return later, mvlater
-	} else if ft.List != nil {
-		return ft.List.laterFields()
+	} else if ft.ListElem != nil {
+		return ft.ListElem.laterFields()
 	}
-	return ft.Fields, nil
+	return ft.structFields, nil
 }
 
 func (ft fieldType) prepare(nft *fieldType, later, mvlater [][]field) {
-	for i, f := range ft.Fields {
+	for i, f := range ft.structFields {
 		nlater, nmvlater, skip := lookupLater(f.Name, later)
 		if skip {
 			continue
 		}
-		ft.Fields[i].prepare(nft.Fields, nlater, nmvlater)
+		ft.structFields[i].prepare(nft.structFields, nlater, nmvlater)
 	}
 	if ft.MapKey != nil {
 		ft.MapKey.prepare(nft.MapKey, later, nil)
 		ft.MapValue.prepare(nft.MapValue, mvlater, nil)
 	}
-	if ft.List != nil {
-		ft.List.prepare(nft.List, later, mvlater)
+	if ft.ListElem != nil {
+		ft.ListElem.prepare(nft.ListElem, later, mvlater)
 	}
 }
 
@@ -1032,18 +1253,24 @@ func (ft fieldType) typeEqual(nft fieldType) bool {
 	if ft.Ptr != nft.Ptr || ft.Kind != nft.Kind {
 		return false
 	}
-	if len(ft.Fields) != len(nft.Fields) {
+	if ft.FieldsTypeSeq != nft.FieldsTypeSeq {
 		return false
 	}
-	for i, f := range ft.Fields {
-		if !f.typeEqual(nft.Fields[i]) {
+	if len(ft.DefinitionFields) != len(nft.DefinitionFields) {
+		return false
+	}
+	for i, f := range ft.DefinitionFields {
+		if !f.typeEqual(nft.DefinitionFields[i]) {
 			return false
 		}
 	}
 	if ft.MapKey != nil && (!ft.MapKey.typeEqual(*nft.MapKey) || !ft.MapValue.typeEqual(*nft.MapValue)) {
 		return false
 	}
-	if ft.List != nil && !ft.List.typeEqual(*nft.List) {
+	if ft.ListElem != nil && !ft.ListElem.typeEqual(*nft.ListElem) {
+		return false
+	}
+	if ft.ArrayLength != nft.ArrayLength {
 		return false
 	}
 	return true
@@ -1069,12 +1296,16 @@ func (idx *index) typeEqual(nidx *index) bool {
 // into an int32. Indices that need to be recreated (for an int width change) are
 // recorded in recreateIndices.
 func (tx *Tx) checkTypes(otv, ntv *typeVersion, recreateIndices map[string]struct{}) error {
+	// Used to track that two nonzero FieldsTypeSeq have been checked, to prevent
+	// recursing while checking.
+	checked := map[[2]int]struct{}{}
+
 	for _, f := range ntv.Fields {
 		for _, of := range otv.Fields {
 			if f.Name != of.Name {
 				continue
 			}
-			increase, err := of.Type.compatible(f.Type)
+			increase, err := of.Type.compatible(f.Type, checked)
 			if err != nil {
 				return fmt.Errorf("%w: field %q: %s", ErrIncompatible, f.Name, err)
 			}
@@ -1099,7 +1330,7 @@ func (tx *Tx) checkTypes(otv, ntv *typeVersion, recreateIndices map[string]struc
 // for maps/slices/structs). If not an error is returned. If they are, the first
 // return value indicates if this is a field that needs it index recreated
 // (currently for ints that are packed with fixed width encoding).
-func (ft fieldType) compatible(nft fieldType) (bool, error) {
+func (ft fieldType) compatible(nft fieldType, checked map[[2]int]struct{}) (bool, error) {
 	need := func(incr bool, l ...kind) (bool, error) {
 		for _, k := range l {
 			if nft.Kind == k {
@@ -1160,10 +1391,10 @@ func (ft fieldType) compatible(nft fieldType) (bool, error) {
 		if nk != k {
 			return false, fmt.Errorf("map to %v: %w", nk, ErrIncompatible)
 		}
-		if _, err := ft.MapKey.compatible(*nft.MapKey); err != nil {
+		if _, err := ft.MapKey.compatible(*nft.MapKey, checked); err != nil {
 			return false, fmt.Errorf("map key: %w", err)
 		}
-		if _, err := ft.MapValue.compatible(*nft.MapValue); err != nil {
+		if _, err := ft.MapValue.compatible(*nft.MapValue, checked); err != nil {
 			return false, fmt.Errorf("map value: %w", err)
 		}
 		return false, nil
@@ -1171,18 +1402,41 @@ func (ft fieldType) compatible(nft fieldType) (bool, error) {
 		if nk != k {
 			return false, fmt.Errorf("slice to %v: %w", nk, ErrIncompatible)
 		}
-		if _, err := ft.List.compatible(*nft.List); err != nil {
-			return false, fmt.Errorf("list: %w", err)
+		if _, err := ft.ListElem.compatible(*nft.ListElem, checked); err != nil {
+			return false, fmt.Errorf("slice: %w", err)
+		}
+		return false, nil
+	case kindArray:
+		if nk != k {
+			return false, fmt.Errorf("array to %v: %w", nk, ErrIncompatible)
+		}
+		if nft.ArrayLength != ft.ArrayLength {
+			return false, fmt.Errorf("array size cannot change (from %d to %d)", ft.ArrayLength, nft.ArrayLength)
+		}
+		if _, err := ft.ListElem.compatible(*nft.ListElem, checked); err != nil {
+			return false, fmt.Errorf("array: %w", err)
 		}
 		return false, nil
 	case kindStruct:
 		if nk != k {
 			return false, fmt.Errorf("struct to %v: %w", nk, ErrIncompatible)
 		}
-		for _, nf := range nft.Fields {
-			for _, f := range ft.Fields {
+
+		// For ondiskVersion2, the seqs are both nonzero, and we must check that we already
+		// did the check to prevent recursion.
+		haveSeq := nft.FieldsTypeSeq != 0 || ft.FieldsTypeSeq != 0
+		if haveSeq {
+			k := [2]int{nft.FieldsTypeSeq, ft.FieldsTypeSeq}
+			if _, ok := checked[k]; ok {
+				return false, nil
+			}
+			checked[k] = struct{}{} // Set early to prevent recursion in call below.
+		}
+
+		for _, nf := range nft.structFields {
+			for _, f := range ft.structFields {
 				if nf.Name == f.Name {
-					_, err := f.Type.compatible(nf.Type)
+					_, err := f.Type.compatible(nf.Type, checked)
 					if err != nil {
 						return false, fmt.Errorf("field %q: %w", nf.Name, err)
 					}
@@ -1201,11 +1455,11 @@ func (ft fieldType) hasNonzeroField(stopAtPtr bool) bool {
 	}
 	switch ft.Kind {
 	case kindMap:
-		return ft.List.hasNonzeroField(true)
-	case kindSlice:
 		return ft.MapValue.hasNonzeroField(true)
+	case kindSlice, kindArray:
+		return ft.ListElem.hasNonzeroField(true)
 	case kindStruct:
-		for _, f := range ft.Fields {
+		for _, f := range ft.structFields {
 			if f.Nonzero || f.Type.hasNonzeroField(true) {
 				return true
 			}

vendor/github.com/mjl-/bstore/store.go

@@ -1,7 +1,9 @@
 package bstore
 
 import (
+	"context"
 	"encoding"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -14,6 +16,18 @@ import (
 	bolt "go.etcd.io/bbolt"
 )
 
+/*
+- todo: should thoroughly review guarantees, where some of the bstore struct tags are allowed (e.g. top-level fields vs deeper struct fields), check that all features work well when combined (cyclic types, embed structs, default values, nonzero checks, type equality, zero values with fieldmap, skipping values (hidden due to later typeversions) and having different type versions), write more extensive tests.
+- todo: write tests for invalid (meta)data inside the boltdb buckets (not for invalid boltdb files). we should detect the error properly, give a reasonable message. we shouldn't panic (nil deref, out of bounds index, consume too much memory). typeVersions, records, indices.
+- todo: add benchmarks. is there a standard dataset databases use for benchmarking?
+- todo optimize: profile and see if we can optimize for some quick wins.
+- todo: should we add a way for ad-hoc data manipulation? e.g. with sql-like queries, e.g. update, delete, insert; and export results of queries to csv.
+- todo: should we have a function that returns records in a map? eg Map() that is like List() but maps a key to T (too bad we cannot have a type for the key!).
+- todo: better error messages (ordering of description & error; mention typename, fields (path), field types and offending value & type more often)
+- todo: should we add types for dates and numerics?
+- todo: struct tag for enums? where we check if the values match.
+*/
+
 var (
 	ErrAbsent       = errors.New("absent") // If a function can return an ErrAbsent, it can be compared directly, without errors.Is.
 	ErrZero         = errors.New("must be nonzero")
@@ -26,6 +40,7 @@ var (
 	ErrFinished     = errors.New("query finished")
 	ErrStore        = errors.New("internal/storage error") // E.g. when buckets disappear, possibly by external users of the underlying BoltDB database.
 	ErrParam        = errors.New("bad parameters")
+	ErrTxBotched    = errors.New("botched transaction") // Set on transactions after failed and aborted write operations.
 
 	errTxClosed    = errors.New("transaction is closed")
 	errNestedIndex = errors.New("struct tags index/unique only allowed at top-level structs")
@@ -42,7 +57,7 @@ type DB struct {
 	// needs a wlock.
 	typesMutex sync.RWMutex
 	types      map[reflect.Type]storeType
-	typeNames  map[string]storeType // Go type name to store type, for checking duplicates.
+	typeNames  map[string]storeType // Type name to store type, for checking duplicates.
 
 	statsMutex sync.Mutex
 	stats      Stats
@@ -52,6 +67,8 @@ type DB struct {
 //
 // A Tx is not safe for concurrent use.
 type Tx struct {
+	ctx context.Context // Check before starting operations, query next calls, and during foreach.
+	err error           // If not nil, operations return this error. Set when write operations fail, e.g. insert with constraint violations.
 	db  *DB             // If nil, this transaction is closed.
 	btx *bolt.Tx
 
@@ -109,9 +126,9 @@ type typeVersion struct {
 type field struct {
 	Name       string
 	Type       fieldType
-	Nonzero    bool
-	References []string // Referenced fields. Only for the top-level struct fields, not for nested structs.
-	Default    string   // As specified in struct tag. Processed version is defaultValue.
+	Nonzero    bool     `json:",omitempty"`
+	References []string `json:",omitempty"` // Referenced fields. Only for the top-level struct fields, not for nested structs.
+	Default    string   `json:",omitempty"` // As specified in struct tag. Processed version is defaultValue.
 
 	// If not the zero reflect.Value, set this value instead of a zero value on insert.
 	// This is always a non-pointer value. Only set for the current typeVersion
@@ -123,6 +140,9 @@ type field struct {
 	// if this field is no longer in the type, or if it has been removed and
 	// added again in later schema versions.
 	structField reflect.StructField
+	// Whether this field has been prepared for parsing into, i.e.
+	// structField set if needed.
+	prepared bool
 
 	indices map[string]*index
 }
@@ -134,79 +154,135 @@ type embed struct {
 	structField reflect.StructField
 }
 
-type kind int
+type kind string
+
+func (k kind) MarshalJSON() ([]byte, error) {
+	return json.Marshal(string(k))
+}
+
+func (k *kind) UnmarshalJSON(buf []byte) error {
+	if string(buf) == "null" {
+		return nil
+	}
+	if len(buf) > 0 && buf[0] == '"' {
+		var s string
+		if err := json.Unmarshal(buf, &s); err != nil {
+			return fmt.Errorf("parsing fieldType.Kind string value %q: %v", buf, err)
+		}
+		nk, ok := kindsMap[s]
+		if !ok {
+			return fmt.Errorf("unknown fieldType.Kind value %q", s)
+		}
+		*k = nk
+		return nil
+	}
+
+	// In ondiskVersion1, the kinds were integers, starting at 1.
+	var i int
+	if err := json.Unmarshal(buf, &i); err != nil {
+		return fmt.Errorf("parsing fieldType.Kind int value %q: %v", buf, err)
+	}
+	if i <= 0 || i-1 >= len(kinds) {
+		return fmt.Errorf("unknown fieldType.Kind value %d", i)
+	}
+	*k = kinds[i-1]
+	return nil
+}
 
 const (
-	kindInvalid kind = iota
-	kindBytes
-	kindBool
-	kindInt
-	kindInt8
-	kindInt16
-	kindInt32
-	kindInt64
-	kindUint
-	kindUint8
-	kindUint16
-	kindUint32
-	kindUint64
-	kindFloat32
-	kindFloat64
-	kindMap
-	kindSlice
-	kindString
-	kindTime
-	kindBinaryMarshal
-	kindStruct
+	kindBytes         kind = "bytes" // 1, etc
+	kindBool          kind = "bool"
+	kindInt           kind = "int"
+	kindInt8          kind = "int8"
+	kindInt16         kind = "int16"
+	kindInt32         kind = "int32"
+	kindInt64         kind = "int64"
+	kindUint          kind = "uint"
+	kindUint8         kind = "uint8"
+	kindUint16        kind = "uint16"
+	kindUint32        kind = "uint32"
+	kindUint64        kind = "uint64"
+	kindFloat32       kind = "float32"
+	kindFloat64       kind = "float64"
+	kindMap           kind = "map"
+	kindSlice         kind = "slice"
+	kindString        kind = "string"
+	kindTime          kind = "time"
+	kindBinaryMarshal kind = "binarymarshal"
+	kindStruct        kind = "struct"
+	kindArray         kind = "array"
 )
 
-var kindStrings = []string{
-	"(invalid)",
-	"bytes",
-	"bool",
-	"int",
-	"int8",
-	"int16",
-	"int32",
-	"int64",
-	"uint",
-	"uint8",
-	"uint16",
-	"uint32",
-	"uint64",
-	"float32",
-	"float64",
-	"map",
-	"slice",
-	"string",
-	"time",
-	"binarymarshal",
-	"struct",
+// In ondiskVersion1, the kinds were integers, starting at 1.
+// Do not change the order. Add new values at the end.
+var kinds = []kind{
+	kindBytes,
+	kindBool,
+	kindInt,
+	kindInt8,
+	kindInt16,
+	kindInt32,
+	kindInt64,
+	kindUint,
+	kindUint8,
+	kindUint16,
+	kindUint32,
+	kindUint64,
+	kindFloat32,
+	kindFloat64,
+	kindMap,
+	kindSlice,
+	kindString,
+	kindTime,
+	kindBinaryMarshal,
+	kindStruct,
+	kindArray,
 }
 
-func (k kind) String() string {
-	return kindStrings[k]
+func makeKindsMap() map[string]kind {
+	m := map[string]kind{}
+	for _, k := range kinds {
+		m[string(k)] = k
+	}
+	return m
 }
 
+var kindsMap = makeKindsMap()
+
 type fieldType struct {
-	Ptr              bool       // If type is a pointer.
+	Ptr  bool `json:",omitempty"` // If type is a pointer.
 	Kind kind // Type with possible Ptr deferenced.
-	Fields           []field    // For kindStruct.
-	MapKey, MapValue *fieldType // For kindMap.
-	List             *fieldType // For kindSlice.
-}
 
-func (ft fieldType) String() string {
-	s := ft.Kind.String()
-	if ft.Ptr {
-		return s + "ptr"
-	}
-	return s
+	MapKey      *fieldType `json:",omitempty"`
+	MapValue    *fieldType `json:",omitempty"`     // For kindMap.
+	ListElem    *fieldType `json:"List,omitempty"` // For kindSlice and kindArray. Named List in JSON for compatibility.
+	ArrayLength int        `json:",omitempty"`     // For kindArray.
+
+	// For kindStruct, the fields of the struct. Only set for the first use of the type
+	// within a registered type. Code dealing with fields should use structFields
+	// (below) most of the time instead, it has the effective fields after resolving
+	// the type reference.
+	// Named "Fields" in JSON to stay compatible with ondiskVersion1, named
+	// DefinitionFields in Go for clarity.
+	DefinitionFields []field `json:"Fields,omitempty"`
+
+	// For struct types, the sequence number of this type (within the registered type).
+	// Needed for supporting cyclic types.  Each struct type is assigned the next
+	// sequence number. The registered type implicitly has sequence 1. If positive,
+	// this defines a type (i.e. when it is first encountered analyzing fields
+	// depth-first). If negative, it references the type with positive seq (when a
+	// field is encountered of a type that was seen before). New since ondiskVersion2,
+	// structs in ondiskVersion1 will have zero value 0.
+	FieldsTypeSeq int `json:",omitempty"`
+
+	// Fields after taking cyclic types into account. Set when registering/loading a
+	// type. Not stored on disk because of potential cyclic data.
+	structFields []field
 }
 
 // Options configure how a database should be opened or initialized.
 type Options struct {
-	Timeout   time.Duration // Abort if opening DB takes longer than Timeout.
+	Timeout   time.Duration // Abort if opening DB takes longer than Timeout. If not set, the deadline from the context is used.
 	Perm      fs.FileMode   // Permissions for new file if created. If zero, 0600 is used.
 	MustExist bool          // Before opening, check that file exists. If not, io/fs.ErrNotExist is returned.
 }
@@ -219,10 +295,19 @@ type Options struct {
 //
 // Only one DB instance can be open for a file at a time. Use opts.Timeout to
 // specify a timeout during open to prevent indefinite blocking.
-func Open(path string, opts *Options, typeValues ...any) (*DB, error) {
+//
+// The context is used for opening and initializing the database, not for further
+// operations. If the context is canceled while waiting on the database file lock,
+// the operation is not aborted other than when the deadline/timeout is reached.
+//
+// See function Register for checks for changed/unchanged schema during open
+// based on environment variable "bstore_schema_check".
+func Open(ctx context.Context, path string, opts *Options, typeValues ...any) (*DB, error) {
 	var bopts *bolt.Options
 	if opts != nil && opts.Timeout > 0 {
 		bopts = &bolt.Options{Timeout: opts.Timeout}
+	} else if end, ok := ctx.Deadline(); ok {
+		bopts = &bolt.Options{Timeout: time.Until(end)}
 	}
 	var mode fs.FileMode = 0600
 	if opts != nil && opts.Perm != 0 {
@@ -241,7 +326,7 @@ func Open(path string, opts *Options, typeValues ...any) (*DB, error) {
 	typeNames := map[string]storeType{}
 	types := map[reflect.Type]storeType{}
 	db := &DB{bdb: bdb, typeNames: typeNames, types: types}
-	if err := db.Register(typeValues...); err != nil {
+	if err := db.Register(ctx, typeValues...); err != nil {
 		bdb.Close()
 		return nil, err
 	}
@@ -272,6 +357,9 @@ func (tx *Tx) Stats() Stats {
 
 // WriteTo writes the entire database to w, not including changes made during this transaction.
 func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) {
+	if err := tx.error(); err != nil {
+		return 0, err
+	}
 	return tx.btx.WriteTo(w)
 }
 
@@ -328,16 +416,28 @@ func (tx *Tx) indexBucket(idx *index) (*bolt.Bucket, error) {
 // If a type is still referenced by another type, eg through a "ref" struct tag,
 // ErrReference is returned.
 // If the type does not exist, ErrAbsent is returned.
-func (db *DB) Drop(name string) error {
-	return db.Write(func(tx *Tx) error {
+func (db *DB) Drop(ctx context.Context, name string) error {
+	var st storeType
+	var ok bool
+	err := db.Write(ctx, func(tx *Tx) error {
 		tx.stats.Bucket.Get++
 		if tx.btx.Bucket([]byte(name)) == nil {
 			return ErrAbsent
 		}
 
-		if st, ok := db.typeNames[name]; ok && len(st.Current.referencedBy) > 0 {
+		st, ok = db.typeNames[name]
+		if ok && len(st.Current.referencedBy) > 0 {
 			return fmt.Errorf("%w: type is still referenced", ErrReference)
-		} else if ok {
+		}
+
+		tx.stats.Bucket.Delete++
+		return tx.btx.DeleteBucket([]byte(name))
+	})
+	if err != nil {
+		return err
+	}
+
+	if ok {
 		for ref := range st.Current.references {
 			var n []*index
 			for _, idx := range db.typeNames[ref].Current.referencedBy {
@@ -350,36 +450,33 @@ func (db *DB) Drop(name string) error {
 		delete(db.typeNames, name)
 		delete(db.types, st.Type)
 	}
-
-		tx.stats.Bucket.Delete++
-		return tx.btx.DeleteBucket([]byte(name))
-	})
+	return nil
 }
 
 // Delete calls Delete on a new writable Tx.
-func (db *DB) Delete(values ...any) error {
-	return db.Write(func(tx *Tx) error {
+func (db *DB) Delete(ctx context.Context, values ...any) error {
+	return db.Write(ctx, func(tx *Tx) error {
 		return tx.Delete(values...)
 	})
 }
 
 // Get calls Get on a new read-only Tx.
-func (db *DB) Get(values ...any) error {
-	return db.Read(func(tx *Tx) error {
+func (db *DB) Get(ctx context.Context, values ...any) error {
+	return db.Read(ctx, func(tx *Tx) error {
 		return tx.Get(values...)
 	})
 }
 
 // Insert calls Insert on a new writable Tx.
-func (db *DB) Insert(values ...any) error {
-	return db.Write(func(tx *Tx) error {
+func (db *DB) Insert(ctx context.Context, values ...any) error {
+	return db.Write(ctx, func(tx *Tx) error {
 		return tx.Insert(values...)
 	})
 }
 
 // Update calls Update on a new writable Tx.
-func (db *DB) Update(values ...any) error {
-	return db.Write(func(tx *Tx) error {
+func (db *DB) Update(ctx context.Context, values ...any) error {
+	return db.Write(ctx, func(tx *Tx) error {
 		return tx.Update(values...)
 	})
 }
@@ -401,6 +498,7 @@ var typeKinds = map[reflect.Kind]kind{
 	reflect.Map:     kindMap,
 	reflect.Slice:   kindSlice,
 	reflect.String:  kindString,
+	reflect.Array:   kindArray,
 }
 
 func typeKind(t reflect.Type) (kind, error) {
@@ -424,7 +522,10 @@ func typeKind(t reflect.Type) (kind, error) {
 	if t.Kind() == reflect.Struct {
 		return kindStruct, nil
 	}
-	return kind(0), fmt.Errorf("%w: unsupported type %v", ErrType, t)
+	if t.Kind() == reflect.Ptr {
+		return "", fmt.Errorf("%w: pointer to pointers not supported: %v", ErrType, t.Elem())
+	}
+	return "", fmt.Errorf("%w: unsupported type %v", ErrType, t)
 }
 
 func typeName(t reflect.Type) (string, error) {
@@ -509,27 +610,39 @@ func (tv typeVersion) keyValue(tx *Tx, rv reflect.Value, insert bool, rb *bolt.B
 }
 
 // Read calls function fn with a new read-only transaction, ensuring transaction rollback.
-func (db *DB) Read(fn func(*Tx) error) error {
+func (db *DB) Read(ctx context.Context, fn func(*Tx) error) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
 	db.typesMutex.RLock()
 	defer db.typesMutex.RUnlock()
 	return db.bdb.View(func(btx *bolt.Tx) error {
-		tx := &Tx{db: db, btx: btx}
+		tx := &Tx{ctx: ctx, db: db, btx: btx}
 		tx.stats.Reads++
 		defer tx.addStats()
-		return fn(tx)
+		if err := fn(tx); err != nil {
+			return err
+		}
+		return tx.err
 	})
 }
 
 // Write calls function fn with a new read-write transaction. If fn returns
 // nil, the transaction is committed. Otherwise the transaction is rolled back.
-func (db *DB) Write(fn func(*Tx) error) error {
+func (db *DB) Write(ctx context.Context, fn func(*Tx) error) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
 	db.typesMutex.RLock()
 	defer db.typesMutex.RUnlock()
 	return db.bdb.Update(func(btx *bolt.Tx) error {
-		tx := &Tx{db: db, btx: btx}
+		tx := &Tx{ctx: ctx, db: db, btx: btx}
 		tx.stats.Writes++
 		defer tx.addStats()
-		return fn(tx)
+		if err := fn(tx); err != nil {
+			return err
+		}
+		return tx.err
 	})
 }
 

vendor/github.com/mjl-/bstore/tx.go

@@ -2,12 +2,39 @@ package bstore
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"reflect"
 
 	bolt "go.etcd.io/bbolt"
 )
 
+// Mark a tx as botched, mentioning last actual error.
+// Used when write operations fail. The transaction can be in inconsistent
+// state, e.g. only some of a type's indicies may have been updated. We never
+// want to commit such transactions.
+func (tx *Tx) markError(err *error) {
+	if *err != nil && tx.err == nil {
+		tx.err = fmt.Errorf("%w (after %v)", ErrTxBotched, *err)
+	}
+}
+
+// Return if an error condition is set on on the transaction. To be called before
+// starting an operation.
+func (tx *Tx) error() error {
+	if tx.err != nil {
+		return tx.err
+	}
+	if tx.db == nil {
+		return errTxClosed
+	}
+	if err := tx.ctx.Err(); err != nil {
+		tx.err = err
+		return err
+	}
+	return nil
+}
+
 func (tx *Tx) structptr(value any) (reflect.Value, error) {
 	rv := reflect.ValueOf(value)
 	if !rv.IsValid() || rv.Kind() != reflect.Ptr || !rv.Elem().IsValid() || rv.Type().Elem().Kind() != reflect.Struct {
@@ -42,10 +69,23 @@ func (tx *Tx) updateIndices(tv *typeVersion, pk []byte, ov, v reflect.Value) err
 
 	changed := func(idx *index) bool {
 		for _, f := range idx.Fields {
-			rofv := ov.FieldByIndex(f.structField.Index)
-			nofv := v.FieldByIndex(f.structField.Index)
-			// note: checking the interface values is enough, we only allow comparable types as index fields.
-			if rofv.Interface() != nofv.Interface() {
+			ofv := ov.FieldByIndex(f.structField.Index)
+			nfv := v.FieldByIndex(f.structField.Index)
+			if f.Type.Kind == kindSlice {
+				// Index field is a slice type, cannot use direct interface comparison.
+				on := ofv.Len()
+				nn := nfv.Len()
+				if on != nn {
+					return true
+				}
+				for i := 0; i < nn; i++ {
+					// Slice elements are comparable.
+					if ofv.Index(i) != nfv.Index(i) {
+						return true
+					}
+				}
+			} else if ofv.Interface() != nfv.Interface() {
+				// note: checking the interface values is enough.
 				return true
 			}
 		}
@@ -69,39 +109,43 @@ func (tx *Tx) updateIndices(tv *typeVersion, pk []byte, ov, v reflect.Value) err
 			return err
 		}
 		if remove {
-			_, ik, err := idx.packKey(ov, pk)
+			ikl, err := idx.packKey(ov, pk)
 			if err != nil {
 				return err
 			}
+			for _, ik := range ikl {
 				tx.stats.Index.Delete++
 				if sanityChecks {
 					tx.stats.Index.Get++
-				if ib.Get(ik) == nil {
-					return fmt.Errorf("internal error: key missing from index")
+					if ib.Get(ik.full) == nil {
+						return fmt.Errorf("%w: key missing from index", ErrStore)
 					}
 				}
-			if err := ib.Delete(ik); err != nil {
+				if err := ib.Delete(ik.full); err != nil {
 					return fmt.Errorf("%w: removing from index: %s", ErrStore, err)
 				}
 			}
+		}
 		if add {
-			prek, ik, err := idx.packKey(v, pk)
+			ikl, err := idx.packKey(v, pk)
 			if err != nil {
 				return err
 			}
+			for _, ik := range ikl {
 				if idx.Unique {
 					tx.stats.Index.Cursor++
-				if xk, _ := ib.Cursor().Seek(prek); xk != nil && bytes.HasPrefix(xk, prek) {
+					if xk, _ := ib.Cursor().Seek(ik.pre); xk != nil && bytes.HasPrefix(xk, ik.pre) {
 						return fmt.Errorf("%w: %q", ErrUnique, idx.Name)
 					}
 				}
 
 				tx.stats.Index.Put++
-			if err := ib.Put(ik, []byte{}); err != nil {
+				if err := ib.Put(ik.full, []byte{}); err != nil {
 					return fmt.Errorf("inserting into index: %w", err)
 				}
 			}
 		}
+	}
 	return nil
 }
 
@@ -124,7 +168,7 @@ func (tx *Tx) checkReferences(tv *typeVersion, pk []byte, ov, rv reflect.Value)
 				return err
 			}
 			if rb.Get(k) == nil {
-				return fmt.Errorf("%w: value %v from field %q to %q", ErrReference, frv.Interface(), f.Name, name)
+				return fmt.Errorf("%w: value %v from %q to %q", ErrReference, frv.Interface(), tv.name+"."+f.Name, name)
 			}
 		}
 	}
@@ -143,8 +187,8 @@ func (tx *Tx) addStats() {
 //
 // ErrAbsent is returned if the record does not exist.
 func (tx *Tx) Get(values ...any) error {
-	if tx.db == nil {
-		return errTxClosed
+	if err := tx.error(); err != nil {
+		return err
 	}
 
 	for _, value := range values {
@@ -184,8 +228,8 @@ func (tx *Tx) Get(values ...any) error {
 // ErrAbsent is returned if the record does not exist.
 // ErrReference is returned if another record still references this record.
 func (tx *Tx) Delete(values ...any) error {
-	if tx.db == nil {
-		return errTxClosed
+	if err := tx.error(); err != nil {
+		return err
 	}
 
 	for _, value := range values {
@@ -222,7 +266,7 @@ func (tx *Tx) Delete(values ...any) error {
 	return nil
 }
 
-func (tx *Tx) delete(rb *bolt.Bucket, st storeType, k []byte, rov reflect.Value) error {
+func (tx *Tx) delete(rb *bolt.Bucket, st storeType, k []byte, rov reflect.Value) (rerr error) {
 	// Check that anyone referencing this type does not reference this record.
 	for _, refBy := range st.Current.referencedBy {
 		if ib, err := tx.indexBucket(refBy); err != nil {
@@ -236,6 +280,7 @@ func (tx *Tx) delete(rb *bolt.Bucket, st storeType, k []byte, rov reflect.Value)
 	}
 
 	// Delete value from indices.
+	defer tx.markError(&rerr)
 	if err := tx.updateIndices(st.Current, k, rov, reflect.Value{}); err != nil {
 		return fmt.Errorf("removing from indices: %w", err)
 	}
@@ -250,8 +295,8 @@ func (tx *Tx) delete(rb *bolt.Bucket, st storeType, k []byte, rov reflect.Value)
 //
 // ErrAbsent is returned if the record does not exist.
 func (tx *Tx) Update(values ...any) error {
-	if tx.db == nil {
-		return errTxClosed
+	if err := tx.error(); err != nil {
+		return err
 	}
 
 	for _, value := range values {
@@ -282,8 +327,8 @@ func (tx *Tx) Update(values ...any) error {
 // ErrZero is returned if a nonzero constraint would be violated.
 // ErrReference is returned if another record is referenced that does not exist.
 func (tx *Tx) Insert(values ...any) error {
-	if tx.db == nil {
-		return errTxClosed
+	if err := tx.error(); err != nil {
+		return err
 	}
 
 	for _, value := range values {
@@ -298,6 +343,7 @@ func (tx *Tx) Insert(values ...any) error {
 			return err
 		}
 
+		// todo optimize: should track per field whether it (or a child) has a default value, and only applyDefault if so.
 		if err := st.Current.applyDefault(rv); err != nil {
 			return err
 		}
@@ -344,7 +390,7 @@ func (tx *Tx) put(st storeType, rv reflect.Value, insert bool) error {
 	}
 }
 
-func (tx *Tx) insert(rb *bolt.Bucket, st storeType, rv, krv reflect.Value, k []byte) error {
+func (tx *Tx) insert(rb *bolt.Bucket, st storeType, rv, krv reflect.Value, k []byte) (rerr error) {
 	v, err := st.pack(rv)
 	if err != nil {
 		return err
@@ -352,6 +398,7 @@ func (tx *Tx) insert(rb *bolt.Bucket, st storeType, rv, krv reflect.Value, k []b
 	if err := tx.checkReferences(st.Current, k, reflect.Value{}, rv); err != nil {
 		return err
 	}
+	defer tx.markError(&rerr)
 	if err := tx.updateIndices(st.Current, k, reflect.Value{}, rv); err != nil {
 		return fmt.Errorf("updating indices for inserted value: %w", err)
 	}
@@ -363,7 +410,7 @@ func (tx *Tx) insert(rb *bolt.Bucket, st storeType, rv, krv reflect.Value, k []b
 	return nil
 }
 
-func (tx *Tx) update(rb *bolt.Bucket, st storeType, rv, rov reflect.Value, k []byte) error {
+func (tx *Tx) update(rb *bolt.Bucket, st storeType, rv, rov reflect.Value, k []byte) (rerr error) {
 	if st.Current.equal(rov, rv) {
 		return nil
 	}
@@ -375,6 +422,7 @@ func (tx *Tx) update(rb *bolt.Bucket, st storeType, rv, rov reflect.Value, k []b
 	if err := tx.checkReferences(st.Current, k, rov, rv); err != nil {
 		return err
 	}
+	defer tx.markError(&rerr)
 	if err := tx.updateIndices(st.Current, k, rov, rv); err != nil {
 		return fmt.Errorf("updating indices for updated record: %w", err)
 	}
@@ -391,13 +439,16 @@ func (tx *Tx) update(rb *bolt.Bucket, st storeType, rv, rov reflect.Value, k []b
 //
 // A writable Tx can be committed or rolled back. A read-only transaction must
 // always be rolled back.
-func (db *DB) Begin(writable bool) (*Tx, error) {
+func (db *DB) Begin(ctx context.Context, writable bool) (*Tx, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
 	btx, err := db.bdb.Begin(writable)
 	if err != nil {
 		return nil, err
 	}
 	db.typesMutex.RLock()
-	tx := &Tx{db: db, btx: btx}
+	tx := &Tx{ctx: ctx, db: db, btx: btx}
 	if writable {
 		tx.stats.Writes++
 	} else {
@@ -422,9 +473,14 @@ func (tx *Tx) Rollback() error {
 
 // Commit commits changes made in the transaction to the database.
 // Statistics are added to its DB.
+// If the commit fails, or the transaction was botched, the transaction is
+// rolled back.
 func (tx *Tx) Commit() error {
 	if tx.db == nil {
 		return errTxClosed
+	} else if tx.err != nil {
+		tx.Rollback()
+		return tx.err
 	}
 
 	tx.addStats()

vendor/modules.txt

@@ -11,7 +11,7 @@ github.com/golang/protobuf/ptypes/timestamp
 # github.com/matttproud/golang_protobuf_extensions v1.0.1
 ## explicit
 github.com/matttproud/golang_protobuf_extensions/pbutil
-# github.com/mjl-/bstore v0.0.0-20230211204415-a9899ef6e782
+# github.com/mjl-/bstore v0.0.1
 ## explicit; go 1.19
 github.com/mjl-/bstore
 # github.com/mjl-/sconf v0.0.4