Wrapper around bbolt for time-ordered, k/v data.
btkv has a niche use-case.
If your data has:
- Identifiers
- Timestamps
And:
- You're looking for an embedded Go database option
- You primarily fetch data as a large stream of time-ordered data
- You need to be able to R/U/D entities by identity independent of where they exist in the stream
- But write speed is secondary to fetching continuous streams of time-ordered data
Then, btkv is a perfect match ;)
package btkv_test
import (
"context"
"fmt"
"time"
"github.com/johnknl/btkv"
)
type exampleRecord struct {
ts time.Time
id string
data []byte
}
type bulkExampleRecords struct {
records []exampleRecord
idx int
}
func (g *bulkExampleRecords) Ok() bool {
return g.idx < len(g.records)
}
func (g *bulkExampleRecords) Values() (string, []byte, time.Time) {
r := g.records[g.idx]
g.idx++
return r.id, r.data, r.ts
}
func Example() {
ctx := context.Background()
store := btkv.NewBoltTKV("example", "/tmp/btkv-example.db", true)
if err := store.Open(time.Second); err != nil {
panic(err)
}
defer store.Close() // nolint:errcheck // just an example
now := time.Date(2020, 1, 1, 0, 3, 0, 0, time.UTC)
// Set the value of entity "a"
existed, err := store.Set(
[]byte(`{"id":"a"}`),
now,
"a",
)
fmt.Println(existed, err)
// Get the value of id "a"
val, err := store.Get("a")
fmt.Printf("%s %v\n", val, err)
// Bulk set some entities
records := &bulkExampleRecords{
records: []exampleRecord{
{data: []byte(`{"id":"b"}`), ts: now.Add(-time.Minute), id: "b"},
{data: []byte(`{"id":"c"}`), ts: now.Add(-2 * time.Minute), id: "c"},
{data: []byte(`{"id":"d"}`), ts: now.Add(-3 * time.Minute), id: "d"},
{data: []byte(`{"id":"e"}`), ts: now.Add(-4 * time.Hour), id: "e"},
},
}
_ = store.BulkSet(ctx, records)
// Get max 2 entities from a range (oldest first)
from := now.Add(-3 * time.Minute)
to := now.Add(-time.Minute)
for data, err := range store.RangeValues(ctx, &from, &to, 0, 2) {
fmt.Println("value: "+string(data), err)
}
// Iterate borrowed values and release them when done.
for value, err := range store.Range(ctx, &from, &to, 0, 2) {
fmt.Println(
"borrowed:",
value.Key.UnmarshalTime().UTC().Format(time.RFC3339),
string(value.Value),
err,
)
value.Release()
}
// Delete entity "a"
err = store.Delete("a")
fmt.Println(err)
// Output:
// false <nil>
// {"id":"a"} <nil>
// value: {"id":"d"} <nil>
// value: {"id":"c"} <nil>
// borrowed: 2020-01-01T00:00:00Z {"id":"d"} <nil>
// borrowed: 2020-01-01T00:01:00Z {"id":"c"} <nil>
// <nil>
}btkv is read optimized, specifically time range optimized. In the below benchmark RangeValues() is used to
yield 990 values. That is ~60 ns per record. If you do not need a copy of every fetched value and are in a position
to put some processed records back into a pool, you can consider the Range() method which returns a BorrowedValue
which has both a key and a value and a Release() method that allows for reuse.
Write operations are dominated by fsync: it doesn't matter much
if you write 1 or 100 records. The latter seems like a common sense batching size.
go test -v -run ^$ -bench . -benchmem -count=3
goos: linux
goarch: amd64
pkg: github.com/johnknl/btkv
cpu: AMD Ryzen 9 5950X 16-Core Processor
BenchmarkBoltTKV_SetN1
BenchmarkBoltTKV_SetN1-32 1287 941683 ns/op 36720 B/op 85 allocs/op
BenchmarkBoltTKV_SetN1-32 1303 920720 ns/op 36739 B/op 85 allocs/op
BenchmarkBoltTKV_SetN1-32 1312 940323 ns/op 36758 B/op 85 allocs/op
BenchmarkBoltTKV_GetN1
BenchmarkBoltTKV_GetN1-32 1000000 1771 ns/op 873 B/op 16 allocs/op
BenchmarkBoltTKV_GetN1-32 1000000 1748 ns/op 873 B/op 16 allocs/op
BenchmarkBoltTKV_GetN1-32 560088 1866 ns/op 873 B/op 16 allocs/op
BenchmarkBoltTKV_RangeN1000
BenchmarkBoltTKV_RangeN1000/No_Release
BenchmarkBoltTKV_RangeN1000/No_Release-32 2481 488690 ns/op 1063493 B/op 1991 allocs/op
BenchmarkBoltTKV_RangeN1000/No_Release-32 2269 468988 ns/op 1063504 B/op 1991 allocs/op
BenchmarkBoltTKV_RangeN1000/No_Release-32 2353 478374 ns/op 1063502 B/op 1991 allocs/op
BenchmarkBoltTKV_RangeN1000/Half_Release
BenchmarkBoltTKV_RangeN1000/Half_Release-32 3830 268831 ns/op 532032 B/op 1001 allocs/op
BenchmarkBoltTKV_RangeN1000/Half_Release-32 4548 275585 ns/op 532032 B/op 1001 allocs/op
BenchmarkBoltTKV_RangeN1000/Half_Release-32 3956 269056 ns/op 532029 B/op 1001 allocs/op
BenchmarkBoltTKV_RangeN1000/Immediate_Release
BenchmarkBoltTKV_RangeN1000/Immediate_Release-32 28564 40420 ns/op 545 B/op 11 allocs/op
BenchmarkBoltTKV_RangeN1000/Immediate_Release-32 31809 34644 ns/op 545 B/op 11 allocs/op
BenchmarkBoltTKV_RangeN1000/Immediate_Release-32 31561 37311 ns/op 545 B/op 11 allocs/op
BenchmarkBoltTKV_RangeValuesN1000
BenchmarkBoltTKV_RangeValuesN1000-32 18147 65175 ns/op 24304 B/op 1001 allocs/op
BenchmarkBoltTKV_RangeValuesN1000-32 18894 62968 ns/op 24304 B/op 1001 allocs/op
BenchmarkBoltTKV_RangeValuesN1000-32 21574 60986 ns/op 24304 B/op 1001 allocs/op
BenchmarkBoltTKV_BulkSetN100
BenchmarkBoltTKV_BulkSetN100-32 1173 956502 ns/op 46266 B/op 1071 allocs/op
BenchmarkBoltTKV_BulkSetN100-32 1274 951373 ns/op 46243 B/op 1071 allocs/op
BenchmarkBoltTKV_BulkSetN100-32 1201 967273 ns/op 46256 B/op 1071 allocs/op
PASS
ok github.com/johnknl/btkv 26.761s
Documentation and usage examples are available on pkg.go.dev.