mirror of
https://github.com/mjl-/mox.git
synced 2024-12-28 17:33:47 +03:00
339 lines
9.2 KiB
Go
339 lines
9.2 KiB
Go
package bstore
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"fmt"
|
|
"math"
|
|
"reflect"
|
|
"time"
|
|
)
|
|
|
|
/*
|
|
The records buckets map a primary key to the record data. The primary key is of
|
|
a form that we can scan/range over. So fixed width for integers. For strings and
|
|
bytes they are just their byte representation. We do not store the PK in the
|
|
record data. This means we cannot store a time.Time as primary key, because we
|
|
cannot have the timezone encoded for comparison reasons.
|
|
|
|
Index keys are similar to PK's. Unique and non-unique indices are encoded the
|
|
same. The stored values are always empty, the key consists of the field values
|
|
the index was created for, followed by the PK. The encoding of a field is nearly
|
|
the same as the encoding of that type as a primary key. The differences: strings
|
|
end with a \0 to make them self-delimiting; byte slices are not allowed because
|
|
they are not self-delimiting; time.Time is allowed because the time is available
|
|
in full (with timezone) in the record data.
|
|
*/
|
|
|
|
// packPK returns the PK bytes representation for the PK value rv.
|
|
func packPK(rv reflect.Value) ([]byte, error) {
|
|
kv := rv.Interface()
|
|
var buf []byte
|
|
switch k := kv.(type) {
|
|
case string:
|
|
buf = []byte(k)
|
|
case []byte:
|
|
buf = k
|
|
case bool:
|
|
var b byte
|
|
if k {
|
|
b = 1
|
|
}
|
|
buf = []byte{b}
|
|
case int8:
|
|
buf = []byte{byte(uint8(k + math.MinInt8))}
|
|
case int16:
|
|
buf = binary.BigEndian.AppendUint16(nil, uint16(k+math.MinInt16))
|
|
case int32:
|
|
buf = binary.BigEndian.AppendUint32(nil, uint32(k+math.MinInt32))
|
|
case int:
|
|
if k < math.MinInt32 || k > math.MaxInt32 {
|
|
return nil, fmt.Errorf("%w: int %d does not fit in int32", ErrParam, k)
|
|
}
|
|
buf = binary.BigEndian.AppendUint32(nil, uint32(k+math.MinInt32))
|
|
case int64:
|
|
buf = binary.BigEndian.AppendUint64(nil, uint64(k+math.MinInt64))
|
|
case uint8:
|
|
buf = []byte{k}
|
|
case uint16:
|
|
buf = binary.BigEndian.AppendUint16(nil, k)
|
|
case uint32:
|
|
buf = binary.BigEndian.AppendUint32(nil, k)
|
|
case uint:
|
|
if k > math.MaxUint32 {
|
|
return nil, fmt.Errorf("%w: uint %d does not fit in uint32", ErrParam, k)
|
|
}
|
|
buf = binary.BigEndian.AppendUint32(nil, uint32(k))
|
|
case uint64:
|
|
buf = binary.BigEndian.AppendUint64(nil, k)
|
|
default:
|
|
return nil, fmt.Errorf("%w: unsupported primary key type %T", ErrType, kv)
|
|
}
|
|
return buf, nil
|
|
}
|
|
|
|
// parsePK parses primary key bk into rv.
|
|
func parsePK(rv reflect.Value, bk []byte) error {
|
|
k, err := typeKind(rv.Type())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
switch k {
|
|
case kindBytes:
|
|
buf := make([]byte, len(bk))
|
|
copy(buf, bk)
|
|
rv.SetBytes(buf)
|
|
return nil
|
|
case kindString:
|
|
rv.SetString(string(bk))
|
|
return nil
|
|
}
|
|
|
|
var need int
|
|
switch k {
|
|
case kindBool, kindInt8, kindUint8:
|
|
need = 1
|
|
case kindInt16, kindUint16:
|
|
need = 2
|
|
case kindInt32, kindUint32, kindInt, kindUint:
|
|
need = 4
|
|
case kindInt64, kindUint64:
|
|
need = 8
|
|
}
|
|
if len(bk) != need {
|
|
return fmt.Errorf("%w: got %d bytes for PK, need %d", ErrStore, len(bk), need)
|
|
}
|
|
|
|
switch k {
|
|
case kindBool:
|
|
rv.SetBool(bk[0] != 0)
|
|
case kindInt8:
|
|
rv.SetInt(int64(int8(bk[0]) - math.MinInt8))
|
|
case kindInt16:
|
|
rv.SetInt(int64(int16(binary.BigEndian.Uint16(bk)) - math.MinInt16))
|
|
case kindInt32, kindInt:
|
|
rv.SetInt(int64(int32(binary.BigEndian.Uint32(bk)) - math.MinInt32))
|
|
case kindInt64:
|
|
rv.SetInt(int64(int64(binary.BigEndian.Uint64(bk)) - math.MinInt64))
|
|
case kindUint8:
|
|
rv.SetUint(uint64(bk[0]))
|
|
case kindUint16:
|
|
rv.SetUint(uint64(binary.BigEndian.Uint16(bk)))
|
|
case kindUint32, kindUint:
|
|
rv.SetUint(uint64(binary.BigEndian.Uint32(bk)))
|
|
case kindUint64:
|
|
rv.SetUint(uint64(binary.BigEndian.Uint64(bk)))
|
|
default:
|
|
// note: we cannot have kindTime as primary key at the moment.
|
|
return fmt.Errorf("%w: unsupported primary key type %v", ErrType, rv.Type())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// parseKey parses the PK (last element) of an index key.
|
|
// If all is set, also gathers the values before and returns them in the second
|
|
// parameter.
|
|
// If witnull is set, string values will get their ending \0 included.
|
|
func (idx *index) parseKey(buf []byte, all bool, withnull bool) ([]byte, [][]byte, error) {
|
|
var err error
|
|
var keys [][]byte
|
|
take := func(n int) {
|
|
if len(buf) < n {
|
|
err = fmt.Errorf("%w: not enough bytes in index key", ErrStore)
|
|
return
|
|
}
|
|
if all {
|
|
keys = append(keys, buf[:n])
|
|
}
|
|
buf = buf[n:]
|
|
}
|
|
fields:
|
|
for _, f := range idx.Fields {
|
|
if err != nil {
|
|
break
|
|
}
|
|
ft := f.Type
|
|
if ft.Kind == kindSlice {
|
|
// For an index on a slice, we store each value in the slice in a separate index key.
|
|
ft = *ft.ListElem
|
|
}
|
|
switch ft.Kind {
|
|
case kindString:
|
|
for i, b := range buf {
|
|
if b == 0 {
|
|
if all {
|
|
o := i
|
|
if withnull {
|
|
o++
|
|
}
|
|
keys = append(keys, buf[:o])
|
|
}
|
|
buf = buf[i+1:]
|
|
continue fields
|
|
}
|
|
}
|
|
err = fmt.Errorf("%w: bad string without 0 in index key", ErrStore)
|
|
case kindBool:
|
|
take(1)
|
|
case kindInt8, kindUint8:
|
|
take(1)
|
|
case kindInt16, kindUint16:
|
|
take(2)
|
|
case kindInt32, kindUint32, kindInt, kindUint:
|
|
take(4)
|
|
case kindInt64, kindUint64:
|
|
take(8)
|
|
case kindTime:
|
|
take(8 + 4)
|
|
default:
|
|
err = fmt.Errorf("%w: unhandled kind %v for index key", ErrStore, ft.Kind)
|
|
}
|
|
}
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
pk := buf
|
|
|
|
switch idx.tv.Fields[0].Type.Kind {
|
|
case kindBool:
|
|
take(1)
|
|
case kindInt8, kindUint8:
|
|
take(1)
|
|
case kindInt16, kindUint16:
|
|
take(2)
|
|
case kindInt32, kindInt, kindUint32, kindUint:
|
|
take(4)
|
|
case kindInt64, kindUint64:
|
|
take(8)
|
|
}
|
|
if len(pk) != len(buf) && len(buf) != 0 {
|
|
return nil, nil, fmt.Errorf("%w: leftover bytes in index key (%x)", ErrStore, buf)
|
|
}
|
|
if all {
|
|
return pk, keys[:len(keys)-1], nil
|
|
}
|
|
return pk, nil, nil
|
|
}
|
|
|
|
type indexkey struct {
|
|
pre []byte // Packed fields excluding PK, a slice of full.
|
|
full []byte // Packed fields including PK.
|
|
}
|
|
|
|
// packKey returns keys to store in an index: first the key prefixes without pk, then
|
|
// the prefixes including pk.
|
|
func (idx *index) packKey(rv reflect.Value, pk []byte) ([]indexkey, error) {
|
|
var l []reflect.Value
|
|
for _, f := range idx.Fields {
|
|
frv := rv.FieldByIndex(f.structField.Index)
|
|
l = append(l, frv)
|
|
}
|
|
return packIndexKeys(l, pk)
|
|
}
|
|
|
|
// packIndexKeys packs values from l, followed by the pk.
|
|
// It returns the key prefixes (without pk), and full keys with pk.
|
|
func packIndexKeys(l []reflect.Value, pk []byte) ([]indexkey, error) {
|
|
ikl := []indexkey{{}}
|
|
for _, frv := range l {
|
|
bufs, err := packIndexKey(frv)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(bufs) == 1 {
|
|
for i := range ikl {
|
|
ikl[i].full = append(ikl[i].full, bufs[0]...)
|
|
}
|
|
} else if len(ikl) == 1 && len(bufs) > 1 {
|
|
nikl := make([]indexkey, len(bufs))
|
|
for i, buf := range bufs {
|
|
nikl[i] = indexkey{full: append(append([]byte{}, ikl[0].full...), buf...)}
|
|
}
|
|
ikl = nikl
|
|
} else if len(bufs) == 0 {
|
|
return nil, nil
|
|
} else {
|
|
return nil, fmt.Errorf("%w: multiple index fields that result in multiple values, or no data for index key, %d keys so far, %d new buffers", ErrStore, len(ikl), len(bufs))
|
|
}
|
|
}
|
|
for i := range ikl {
|
|
n := len(ikl[i].full)
|
|
ikl[i].full = append(ikl[i].full, pk...)
|
|
ikl[i].pre = ikl[i].full[:n]
|
|
}
|
|
return ikl, nil
|
|
}
|
|
|
|
func packIndexKey(frv reflect.Value) ([][]byte, error) {
|
|
k, err := typeKind(frv.Type())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var buf []byte
|
|
switch k {
|
|
case kindBool:
|
|
buf = []byte{0}
|
|
if frv.Bool() {
|
|
buf[0] = 1
|
|
}
|
|
case kindInt8:
|
|
buf = []byte{byte(int8(frv.Int()) + math.MinInt8)}
|
|
case kindInt16:
|
|
buf = binary.BigEndian.AppendUint16(nil, uint16(int16(frv.Int())+math.MinInt16))
|
|
case kindInt32:
|
|
buf = binary.BigEndian.AppendUint32(nil, uint32(int32(frv.Int())+math.MinInt32))
|
|
case kindInt:
|
|
i := frv.Int()
|
|
if i < math.MinInt32 || i > math.MaxInt32 {
|
|
return nil, fmt.Errorf("%w: int value %d does not fit in int32", ErrParam, i)
|
|
}
|
|
buf = binary.BigEndian.AppendUint32(nil, uint32(int32(i)+math.MinInt32))
|
|
case kindInt64:
|
|
buf = binary.BigEndian.AppendUint64(nil, uint64(frv.Int()+math.MinInt64))
|
|
case kindUint8:
|
|
buf = []byte{byte(frv.Uint())}
|
|
case kindUint16:
|
|
buf = binary.BigEndian.AppendUint16(nil, uint16(frv.Uint()))
|
|
case kindUint32:
|
|
buf = binary.BigEndian.AppendUint32(nil, uint32(frv.Uint()))
|
|
case kindUint:
|
|
i := frv.Uint()
|
|
if i > math.MaxUint32 {
|
|
return nil, fmt.Errorf("%w: uint value %d does not fit in uint32", ErrParam, i)
|
|
}
|
|
buf = binary.BigEndian.AppendUint32(nil, uint32(i))
|
|
case kindUint64:
|
|
buf = binary.BigEndian.AppendUint64(nil, uint64(frv.Uint()))
|
|
case kindString:
|
|
buf = []byte(frv.String())
|
|
for _, c := range buf {
|
|
if c == 0 {
|
|
return nil, fmt.Errorf("%w: string used as index key cannot have \\0", ErrParam)
|
|
}
|
|
}
|
|
buf = append(buf, 0)
|
|
case kindTime:
|
|
tm := frv.Interface().(time.Time)
|
|
buf = binary.BigEndian.AppendUint64(nil, uint64(tm.Unix()+math.MinInt64))
|
|
buf = binary.BigEndian.AppendUint32(buf, uint32(tm.Nanosecond()))
|
|
case kindSlice:
|
|
n := frv.Len()
|
|
bufs := make([][]byte, n)
|
|
for i := 0; i < n; i++ {
|
|
nbufs, err := packIndexKey(frv.Index(i))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("packing element from slice field: %w", err)
|
|
}
|
|
if len(nbufs) != 1 {
|
|
return nil, fmt.Errorf("packing element from slice field resulted in multiple buffers (%d)", len(bufs))
|
|
}
|
|
bufs[i] = nbufs[0]
|
|
}
|
|
return bufs, nil
|
|
default:
|
|
return nil, fmt.Errorf("internal error: bad type %v for index", frv.Type()) // todo: should be caught when making index type
|
|
}
|
|
return [][]byte{buf}, nil
|
|
}
|