diff --git a/README.md b/README.md index 96aa24a..f40eac8 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ LMDB.jl exposes the same database through three surfaces: `AbstractDict{K,V}` over a single LMDB file. Standard library machinery (`merge!`, `filter!`, `pairs`, iteration, …) works out of the box. Reach for this when you want a persistent `Dict`. -- Julia wrappers: `Environment`, `Transaction`, `DBI`, `Cursor`. +- Julia wrappers: `Environment`, `Transaction`, `Database`, `Cursor`. Julia-shaped wrappers around handles, transactions, and cursors, with finalizers, `do`-block forms, and so on. Use these when you want explicit transactions. diff --git a/docs/src/index.md b/docs/src/index.md index c7a905d..85024dd 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -18,7 +18,7 @@ LMDB.jl exposes the same database through three surfaces: | Surface | What it offers | When to use | |---------|----------------|-------------| | High-level interface | `LMDBDict <: AbstractDict{K,V}` | When you want a persistent `Dict`. | -| Julia wrappers | `Environment`, `Transaction`, `DBI`, `Cursor` | When you want explicit transactions and cursors with Julia-shaped wrappers. | +| Julia wrappers | `Environment`, `Transaction`, `Database`, `Cursor` | When you want explicit transactions and cursors with Julia-shaped wrappers. | | C API | `LMDB.mdb_*`, `LMDB.MDB_*` | Raw `ccall` bindings and status-code constants, for custom data layouts or when you want to skip allocations on hot paths. | `MDBValue`, `MDBArg`, and the [`MDBValueIO`](@ref LMDB.MDBValueIO) diff --git a/docs/src/lib/cursors.md b/docs/src/lib/cursors.md index 9bb34ee..46a7e86 100644 --- a/docs/src/lib/cursors.md +++ b/docs/src/lib/cursors.md @@ -4,14 +4,14 @@ CurrentModule = LMDB ``` -A `Cursor` is a positioned iterator over the entries in a `DBI`. Cursors +A `Cursor` is a positioned iterator over the entries in a `Database`. Cursors are bound to a transaction. Closing the txn invalidates the cursor. ## Construction ```@docs Cursor -Base.open(::Transaction, ::DBI) +Cursor(::Transaction, ::Database) Base.close(::Cursor) Base.isopen(::Cursor) renew(::Transaction, ::Cursor) diff --git a/docs/src/lib/databases.md b/docs/src/lib/databases.md index d405c20..ca84fc6 100644 --- a/docs/src/lib/databases.md +++ b/docs/src/lib/databases.md @@ -1,32 +1,31 @@ -# [Databases](@id API-DBI) +# [Databases](@id API-Database) ```@meta CurrentModule = LMDB ``` -A `DBI` (database identifier) is a handle to one B-tree inside an +A `Database` (database identifier) is a handle to one B-tree inside an environment. By default an env has a single anonymous database (the -"main DB"); pass `maxdbs > 0` to `Environment` and a name to `open` to -work with multiple named sub-databases. +"main DB"); pass `maxdbs > 0` to `Environment` and a name to the `Database` +constructor to work with multiple named sub-databases. ## Construction ```@docs -DBI -Base.open(::Transaction, ::String) -Base.close(::Environment, ::DBI) -Base.isopen(::DBI) +Database +Database(::Transaction, ::AbstractString) +Base.close(::Environment, ::Database) +Base.isopen(::Database) flags drop -Base.stat(::Transaction, ::DBI) +Base.stat(::Transaction, ::Database) ``` ## Reads ```@docs -Base.get(::Transaction, ::DBI, ::Any, ::Type{T}) where T -Base.get(::Transaction, ::DBI, ::Any, ::Type{T}, ::Any) where T -tryget +Base.get(::Transaction, ::Database, ::Any, ::Type{T}) where T +Base.get(::Transaction, ::Database, ::Any, ::Type{T}, ::Any) where T ``` `get(txn, dbi, key, T, default)` falls back to `default` if `key` is @@ -35,11 +34,11 @@ missing, matching `Base.get(dict, key, default)`. ## Writes ```@docs -Base.put!(::Transaction, ::DBI, ::Any, ::Any) +Base.put!(::Transaction, ::Database, ::Any, ::Any) put_reserved! -Base.delete!(::Transaction, ::DBI, ::Any) -Base.replace!(::Transaction, ::DBI, ::Any, ::Any) -Base.pop!(::Transaction, ::DBI, ::Any, ::Type) +Base.delete!(::Transaction, ::Database, ::Any) +Base.replace!(::Transaction, ::Database, ::Any, ::Any) +Base.pop!(::Transaction, ::Database, ::Any, ::Type) ``` ## Write flags diff --git a/docs/src/lib/dict.md b/docs/src/lib/dict.md index d501dc3..260c68d 100644 --- a/docs/src/lib/dict.md +++ b/docs/src/lib/dict.md @@ -32,7 +32,7 @@ Everything `AbstractDict` derives (`merge!`, `merge`, `mergewith!`, ## Lifecycle -`close(::LMDBDict)` closes the underlying env (and the default DBI). +`close(::LMDBDict)` closes the underlying env (and the default Database). Idempotent, and also called from the finalizer. ## Prefix-scan helpers diff --git a/docs/src/lib/environments.md b/docs/src/lib/environments.md index e8189f2..6ddccd0 100644 --- a/docs/src/lib/environments.md +++ b/docs/src/lib/environments.md @@ -13,14 +13,11 @@ cursor lives inside one env. ```@docs Environment Environment(::AbstractString) -create -environment ``` ## Lifecycle ```@docs -Base.open(::Environment, ::String) Base.close(::Environment) Base.isopen(::Environment) sync diff --git a/docs/src/lib/errors.md b/docs/src/lib/errors.md index 45ce24e..c5477c8 100644 --- a/docs/src/lib/errors.md +++ b/docs/src/lib/errors.md @@ -39,10 +39,10 @@ the `unchecked_*` companion returns the raw `Cint` so the caller can branch on `MDB_NOTFOUND`/`MDB_KEYEXIST` and friends. At the Julia wrappers, handle methods that wrap status-returning -bindings let `LMDBError` propagate. `tryget` and `get(..., default)` -swallow `MDB_NOTFOUND` and return `nothing` or `default`. -`delete!(txn, dbi, key)` likewise swallows `MDB_NOTFOUND` and returns -`false`. +bindings let `LMDBError` propagate. `get(..., default)` swallows +`MDB_NOTFOUND` and returns `default` (use `nothing` for the +`Union{T,Nothing}` shape). `delete!(txn, dbi, key)` likewise swallows +`MDB_NOTFOUND` and returns `false`. At the high-level interface, missing keys produce `KeyError` (matching `Base.Dict`), and `pop!(d)` on an empty dict throws `ArgumentError`. diff --git a/docs/src/lib/lowlevel.md b/docs/src/lib/lowlevel.md index b8dc802..9c8c6fb 100644 --- a/docs/src/lib/lowlevel.md +++ b/docs/src/lib/lowlevel.md @@ -39,7 +39,7 @@ since there is nothing to check. ## Customisation point: `MDBValueIO` -`tryget`, `get`, `key`, `value`, `item`, typed `walk`, `pop!`, and +`get`, `key`, `value`, `item`, typed `walk`, `pop!`, and `replace!` all go through `read(::MDBValueIO, T)` to decode an `MDB_val` into a Julia value. Define a `Base.read` method on `MDBValueIO` to plug in a custom representation. See [Cursors](@ref) @@ -107,7 +107,7 @@ mdb_txn_reset mdb_txn_renew ``` -### Database (DBI) +### Database (Database) ```julia mdb_dbi_open diff --git a/docs/src/lib/transactions.md b/docs/src/lib/transactions.md index ef952b3..bcd5d2c 100644 --- a/docs/src/lib/transactions.md +++ b/docs/src/lib/transactions.md @@ -12,7 +12,7 @@ concurrent readers but only one writer at a time. ```@docs Transaction -start +Transaction(::Environment) ``` ## Lifecycle @@ -36,7 +36,7 @@ renew(::Transaction) ## Sub-transactions -Pass `parent = txn` to [`start`](@ref) to nest a child write transaction -inside an open write transaction. The child sees the parent's uncommitted -state; on `commit` the child's changes are folded into the parent, on -`abort` they are discarded. +Pass `parent = txn` to [`Transaction`](@ref) to nest a child write +transaction inside an open write transaction. The child sees the +parent's uncommitted state; on `commit` the child's changes are folded +into the parent, on `abort` they are discarded. diff --git a/docs/src/man/cursors.md b/docs/src/man/cursors.md index c0c4c08..48ccaf0 100644 --- a/docs/src/man/cursors.md +++ b/docs/src/man/cursors.md @@ -4,16 +4,16 @@ CurrentModule = LMDB ``` -A `Cursor` is a positioned iterator over a `DBI`. Use it for ordered +A `Cursor` is a positioned iterator over a `Database`. Use it for ordered scans, range queries, or to amortise the per-lookup overhead of `mdb_get` across many keys. ## Opening a cursor ```julia -start(env; flags = LMDB.MDB_RDONLY) do txn - open(txn) do dbi - open(txn, dbi) do cur +Transaction(env; flags = LMDB.MDB_RDONLY) do txn + Database(txn) do dbi + Cursor(txn, dbi) do cur # use cur end end @@ -66,9 +66,9 @@ A typical pattern for "all keys with a given prefix": ```julia prefix = "users/" -start(env; flags = LMDB.MDB_RDONLY) do txn - open(txn) do dbi - open(txn, dbi) do cur +Transaction(env; flags = LMDB.MDB_RDONLY) do txn + Database(txn) do dbi + Cursor(txn, dbi) do cur k = seek_range!(cur, prefix, String) while k !== nothing && startswith(k, prefix) v = LMDB.value(cur, String) @@ -113,9 +113,10 @@ Use the untyped form when you want to inspect raw byte sizes, copy slices, or feed a custom decoder. The data pointers are into LMDB's mmap and are valid only inside the callback (and only for the surrounding txn). The typed form is the iteration analogue of -`tryget(..., T)` and works for any `T` for which `Base.read(io::IO, -::Type{T})` (or `Base.read(io::LMDB.MDBValueIO, ::Type{T})`) is -defined. See [Custom value decoding](@ref). +`get(..., T, nothing)` and works for any `T` for which +`Base.read(io::IO, ::Type{T})` (or +`Base.read(io::LMDB.MDBValueIO, ::Type{T})`) is defined. See +[Custom value decoding](@ref). ## Cursor mutation @@ -134,7 +135,7 @@ key (1 in non-DUPSORT databases). ## Custom value decoding -`tryget`, `get`, `key`, `value`, `item`, and typed `walk` all funnel +`get`, `key`, `value`, `item`, and typed `walk` all funnel through `Base.read(io::IO, ::Type{T})` against an [`MDBValueIO`](@ref LMDB.MDBValueIO). The defaults cover Base's primitive numeric types (`Int8`/…/`Float64`, `Bool`, `Char`, `Ptr`), @@ -153,7 +154,7 @@ function Base.read(io::IO, ::Type{PrefixedBlob}) end # now usable everywhere a value-type parameter is accepted: -LMDB.tryget(txn, dbi, key, PrefixedBlob) +LMDB.get(txn, dbi, key, PrefixedBlob, nothing) walk(cur, String, PrefixedBlob) do k, blob handle(k, blob) end @@ -174,8 +175,8 @@ expensive. Park the txn with [`reset`](@ref Base.reset(::LMDB.Transaction)) and refresh both the txn and the cursor with `renew(txn, cur)`: ```julia -txn = start(env; flags = LMDB.MDB_RDONLY) -cur = open(txn, dbi) +txn = Transaction(env; flags = LMDB.MDB_RDONLY) +cur = Cursor(txn, dbi) while running ... # use cur reset(txn) diff --git a/docs/src/man/databases.md b/docs/src/man/databases.md index 16acb53..91c0222 100644 --- a/docs/src/man/databases.md +++ b/docs/src/man/databases.md @@ -4,31 +4,31 @@ CurrentModule = LMDB ``` -A `DBI` is a handle to one B-tree inside an environment. By default an +A `Database` is a handle to one B-tree inside an environment. By default an env has a single anonymous database (the "main DB"); pass `maxdbs > 0` to `Environment` to support multiple named sub-databases. -## Opening a DBI +## Opening a Database ```julia -dbi = open(txn) # main (unnamed) DB -dbi = open(txn, "users") # named sub-DB; needs maxdbs >= 1 -dbi = open(txn, "edges"; flags = LMDB.MDB_CREATE | LMDB.MDB_DUPSORT) +dbi = Database(txn) # main (unnamed) DB +dbi = Database(txn, "users") # named sub-DB; needs maxdbs >= 1 +dbi = Database(txn, "edges"; flags = LMDB.MDB_CREATE | LMDB.MDB_DUPSORT) ``` -The do-block form closes the DBI on the way out: +The do-block form closes the Database on the way out: ```julia -open(txn, "users") do dbi +Database(txn, "users") do dbi put!(txn, dbi, "1", "Ada") end ``` -In practice you'll rarely *want* to close a DBI handle explicitly. The +In practice you'll rarely *want* to close a Database handle explicitly. The env owns it, and `mdb_dbi_close` is documented as rarely useful. The -env's finalizer cascades through any open DBI handles. +env's finalizer cascades through any open Database handles. -## DBI flags +## Database flags `flags` accepts a bitwise-or of: @@ -44,12 +44,11 @@ env's finalizer cascades through any open DBI handles. ## Reads -Every read takes a value-type parameter `T`. The default forms are: +Every read takes a value-type parameter `T`. The two shapes are: ```julia get(txn, dbi, key, T) # throws LMDBError(MDB_NOTFOUND) on miss -tryget(txn, dbi, key, T) # nothing on miss -get(txn, dbi, key, T, default) # default on miss +get(txn, dbi, key, T, default) # returns `default` on miss ``` `T` is anything `read(::LMDB.MDBValueIO, ::Type{T})` knows how to @@ -57,12 +56,12 @@ decode: `String`, `Vector{E}` for any bitstype `E`, or any bitstype scalar. ```julia -tryget(txn, dbi, "name", String) # → Union{String, Nothing} -tryget(txn, dbi, key, Vector{Float32}) # → Union{Vector{Float32}, Nothing} -tryget(txn, dbi, key, UInt64) # → Union{UInt64, Nothing} +get(txn, dbi, "name", String, nothing) # → Union{String, Nothing} +get(txn, dbi, key, Vector{Float32}, nothing) # → Union{Vector{Float32}, Nothing} +get(txn, dbi, key, UInt64, zero(UInt64)) # → UInt64 ``` -`tryget` is the cheap one: it inspects the raw status code and swallows +The default-form `get` inspects the raw status code and swallows `MDB_NOTFOUND` without throwing. ## Writes @@ -86,8 +85,8 @@ Useful write flags: ```julia # Bulk import in sorted order: -start(env) do txn - open(txn) do dbi +Transaction(env) do txn + Database(txn) do dbi for (k, v) in sorted_pairs put!(txn, dbi, k, v; flags = LMDB.MDB_APPEND) end diff --git a/docs/src/man/dict.md b/docs/src/man/dict.md index 2ffaac2..e730b39 100644 --- a/docs/src/man/dict.md +++ b/docs/src/man/dict.md @@ -5,7 +5,7 @@ CurrentModule = LMDB ``` `LMDBDict{K,V}` is a persistent `AbstractDict{K,V}` backed by a single -LMDB environment plus the default DBI. Open it, treat it like a `Dict`, +LMDB environment plus the default Database. Open it, treat it like a `Dict`, close it. ## Construction diff --git a/docs/src/man/dupsort.md b/docs/src/man/dupsort.md index f31a33b..858c566 100644 --- a/docs/src/man/dupsort.md +++ b/docs/src/man/dupsort.md @@ -12,8 +12,8 @@ values" patterns. ```julia env = Environment("/tmp/edges"; mapsize = 1 << 30, maxdbs = 1) -start(env) do txn - dbi = open(txn, "edges"; flags = LMDB.MDB_CREATE | LMDB.MDB_DUPSORT) +Transaction(env) do txn + dbi = Database(txn, "edges"; flags = LMDB.MDB_CREATE | LMDB.MDB_DUPSORT) put!(txn, dbi, "a", "b") put!(txn, dbi, "a", "c") put!(txn, dbi, "a", "d") diff --git a/docs/src/man/environments.md b/docs/src/man/environments.md index 8b15cc4..4c89430 100644 --- a/docs/src/man/environments.md +++ b/docs/src/man/environments.md @@ -10,8 +10,8 @@ database handle, and cursor lives inside one env. ## Creating and opening -The one-call constructor `create`s the handle, applies any configuration, -and `open`s the directory in one go: +`Environment(path; kwargs...)` does the whole open dance — allocate +the handle, set mapsize/maxreaders/maxdbs/flags, open the directory: ```julia env = Environment("/tmp/mydb"; mapsize = 1 << 30, # 1 GiB virtual map @@ -20,33 +20,23 @@ env = Environment("/tmp/mydb"; mapsize = 1 << 30, # 1 GiB virtual map flags = LMDB.MDB_NOTLS) ``` -If anything fails between `create` and a successful `open`, the -partially constructed env is closed before rethrowing. +If anything fails on the way through, the half-open env is closed +before the exception propagates. -The split form mirrors the LMDB C API: - -```julia -env = create() -env[:MapSize] = 1 << 30 -env[:Readers] = 510 -env[:DBs] = 8 -open(env, "/tmp/mydb"; flags = LMDB.MDB_NOTLS) -``` - -The `[:Flags]`/`[:Readers]`/`[:MapSize]`/`[:DBs]` keys map directly to -`mdb_env_set_flags` / `mdb_env_set_maxreaders` / `mdb_env_set_mapsize` -/ `mdb_env_set_maxdbs`. `set!` / `unset!` flip individual flag bits -after the env is open. +After the env is open, `[:Flags]` / `[:Readers]` / `[:MapSize]` / +`[:DBs]` setindex! keys map to `mdb_env_set_flags` / +`mdb_env_set_maxreaders` / `mdb_env_set_mapsize` / `mdb_env_set_maxdbs`, +and `set!` / `unset!` flip individual flag bits. `getindex` exposes a few read-only views: `env[:Flags]`, `env[:Readers]`, and `env[:KeySize]` (the maximum key length, fixed at compile time of the bundled `LMDB_jll`). -The do-block constructor `environment(f, path; flags, mode)` opens the -env, calls `f(env)`, and closes the env on the way out: +The do-block form `Environment(f, path; kwargs...)` opens the env, +calls `f(env)`, and closes on the way out: ```julia -environment("/tmp/mydb"; flags = LMDB.MDB_NOTLS) do env +Environment("/tmp/mydb"; flags = LMDB.MDB_NOTLS) do env # use env end ``` diff --git a/docs/src/man/essentials.md b/docs/src/man/essentials.md index cb6a4da..16a84f2 100644 --- a/docs/src/man/essentials.md +++ b/docs/src/man/essentials.md @@ -38,7 +38,7 @@ close(d) Behind the scenes this opens an `Environment` with `MDB_NOTLS` (so multiple read transactions can coexist on a single thread) and a single -default `DBI`. Type conversion happens automatically for anything the +default `Database`. Type conversion happens automatically for anything the `MDBValue` constructor accepts: `String`, `Vector{T}` of bitstype `T`, or any bitstype scalar. @@ -50,7 +50,7 @@ control: - The high-level interface ([`LMDBDict`](@ref)) is the `AbstractDict{K,V}` surface. Start here unless you need transactional grouping or zero-copy reads. -- The Julia wrappers (`Environment`, `Transaction`, `DBI`, +- The Julia wrappers (`Environment`, `Transaction`, `Database`, `Cursor`) give you explicit lifetimes and fine-grained control with finalizers, parent refs, and `do`-block forms. Drop down to these via [Environments](@ref) → [Transactions](@ref) → @@ -71,7 +71,7 @@ with a finalizer: |--------|-----------|------------| | `Environment` | `close` (`mdb_env_close`) | – | | `Transaction` | `abort` (`mdb_txn_abort`) | `Environment` | -| `Cursor` | `close` (`mdb_cursor_close`) | `Transaction`, `DBI` | +| `Cursor` | `close` (`mdb_cursor_close`) | `Transaction`, `Database` | | `LMDBDict` | `close` env + dbi | – | Parent references pin the lifetime: a `Cursor` keeps its `Transaction` @@ -84,9 +84,9 @@ when GC runs. The do-block constructors are usually what you want: ```julia -environment("/tmp/mydb"; flags = LMDB.MDB_NOTLS) do env - start(env) do txn - open(txn) do dbi +Environment("/tmp/mydb"; flags = LMDB.MDB_NOTLS) do env + Transaction(env) do txn + Database(txn) do dbi put!(txn, dbi, "k", "v") end end # commits on success, aborts on throw @@ -95,17 +95,7 @@ end # closes env ## Errors -Every LMDB-internal error surfaces as an `LMDBError`: - -```julia -try - LMDB.get(txn, dbi, "missing", String) -catch e - e isa LMDBError && is_notfound(e) || rethrow() - # treat as missing -end -``` - -For the usual "missing key" case, prefer the no-throw paths: -[`tryget(txn, dbi, key, T)`](@ref tryget) returns `nothing` on miss, -and `get(txn, dbi, key, T, default)` falls back to `default`. +Every LMDB-internal error surfaces as an `LMDBError`. For the usual +"missing key" case, prefer the no-throw paths: +`get(txn, dbi, key, T, default)` falls back to `default` (use +`nothing` for the `Union{T,Nothing}` shape). diff --git a/docs/src/man/lowlevel.md b/docs/src/man/lowlevel.md index c63cd7d..37b6037 100644 --- a/docs/src/man/lowlevel.md +++ b/docs/src/man/lowlevel.md @@ -35,7 +35,7 @@ ret == 0 || throw(LMDB.LMDBError(ret)) return read(LMDB.MDBValueIO(val_ref[]), T) ``` -This is exactly the pattern [`tryget`](@ref) uses internally. +This is exactly the pattern `get(txn, dbi, key, T, default)` uses internally. Bindings that don't return a status (`mdb_strerror`, `mdb_version`, `mdb_txn_id`, `mdb_cmp`, `mdb_dcmp`, `mdb_env_get_maxkeysize`, @@ -105,7 +105,7 @@ function Base.read(io::IO, ::Type{AtimedBlob}) return read(io, Vector{UInt8}) end -LMDB.tryget(txn, dbi, key, AtimedBlob) # skip 8-byte prefix, copy tail +LMDB.get(txn, dbi, key, AtimedBlob, nothing) # skip 8-byte prefix, copy tail ``` For an `isbitstype` struct `T`, the standard one-liner is enough: @@ -115,8 +115,8 @@ Base.read(io::IO, ::Type{T}) = read!(io, Ref{T}())[] ``` This is the analogue of heed's `BytesDecode<'txn>` trait. Every typed -read in the Julia wrappers (`tryget`, `get`, `key`, `value`, `item`, -typed `walk`, `pop!`, `replace!`) goes through `read(::MDBValueIO, T)`, +read in the Julia wrappers (`get`, `key`, `value`, `item`, typed +`walk`, `pop!`, `replace!`) goes through `read(::MDBValueIO, T)`, so one method definition makes a custom representation usable across the package. Because `MDBValueIO <: IO`, the standard `Base` IO primitives (`position`, `seek`, `skip`, `read(io)`, `read(io, diff --git a/docs/src/man/transactions.md b/docs/src/man/transactions.md index ac9a9f3..c351131 100644 --- a/docs/src/man/transactions.md +++ b/docs/src/man/transactions.md @@ -11,8 +11,8 @@ time per environment). ## Starting a transaction ```julia -txn = start(env) # read-write -txn = start(env; flags = LMDB.MDB_RDONLY) # read-only +txn = Transaction(env) # read-write +txn = Transaction(env; flags = LMDB.MDB_RDONLY) # read-only ``` LMDB can hold one writer plus an unlimited number of readers @@ -21,10 +21,10 @@ concurrently. Read txns do not block writers and vice versa. The do-block form commits on normal return and aborts on throw: ```julia -result = start(env) do txn - open(txn) do dbi +result = Transaction(env) do txn + Database(txn) do dbi put!(txn, dbi, "k", "v") - tryget(txn, dbi, "k", String) + get(txn, dbi, "k", String, nothing) end end # commits if no throw ``` @@ -48,11 +48,11 @@ Read-only txns are cheap to start and stop, but in a tight loop the pair is cheaper still: ```julia -txn = start(env; flags = LMDB.MDB_RDONLY) +txn = Transaction(env; flags = LMDB.MDB_RDONLY) for batch in batches - open(txn) do dbi + Database(txn) do dbi for k in batch - v = tryget(txn, dbi, k, String) + v = get(txn, dbi, k, String, nothing) handle(k, v) end end @@ -73,18 +73,18 @@ uncommitted state. `commit` on the child folds its changes into the parent; `abort` discards them, but the parent continues: ```julia -start(env) do parent - open(parent) do dbi +Transaction(env) do parent + Database(parent) do dbi put!(parent, dbi, "before", "1") try - start(env; parent = parent) do child + Transaction(env; parent = parent) do child put!(child, dbi, "during", "2") error("oops") # abort propagates end catch end # "before" survives; "during" was rolled back - @assert tryget(parent, dbi, "during", String) === nothing + @assert get(parent, dbi, "during", String, nothing) === nothing end end ``` @@ -102,7 +102,7 @@ reap slots left behind by crashed processes. Aggressive `for … break` over an `LMDBDict` without GC pressure can pile up read txns. If that becomes a problem, use [`walk(f, cur)`](@ref API-Cur-walk) inside an explicit -`open(txn) do …` block instead. +`Database(txn) do …` block instead. ## Picking flags @@ -110,13 +110,13 @@ The most common patterns: ```julia # Hot read path: many small lookups, no writes -start(env; flags = LMDB.MDB_RDONLY) do txn ... end +Transaction(env; flags = LMDB.MDB_RDONLY) do txn ... end # Bulk import: single transaction across many writes (atomic, fast) -start(env) do txn ... end +Transaction(env) do txn ... end # Long-running reader (e.g. background scrubber): reset + renew loop -txn = start(env; flags = LMDB.MDB_RDONLY) +txn = Transaction(env; flags = LMDB.MDB_RDONLY) while running ... reset(txn); renew(txn) diff --git a/res/wrap.jl b/res/wrap.jl index fa2738f..b3908c6 100644 --- a/res/wrap.jl +++ b/res/wrap.jl @@ -41,7 +41,11 @@ function postprocess!(path::AbstractString) last = m.offset + length(m.match) end write(out, SubString(src, last, lastindex(src))) - write(path, String(take!(out))) + src = String(take!(out)) + # Clang.Generators always emits `export $jll_pkg_name` next to its + # `using` line; strip it so LMDB_jll stays an implementation detail. + src = replace(src, r"^export LMDB_jll\n"m => "") + write(path, src) end function main() diff --git a/src/common.jl b/src/common.jl index 7c89079..36b338e 100644 --- a/src/common.jl +++ b/src/common.jl @@ -1,5 +1,4 @@ -export isflagset -@public version, MDBValueIO +@public isflagset, version, MDBValueIO # Zero-valued `MDB_val` sentinels, used as out-parameters and for the # "no value" form of `delete!`. Constructing a non-empty `MDB_val` from a @@ -71,8 +70,8 @@ A read-only `IO` view over an LMDB-owned `MDB_val`. Wraps the package's typed-read path is the standard `Base.read(io, T)`. Any `T` for which `Base.read(io::IO, ::Type{T})` is defined can be -passed to `tryget`, `get`, `key`, `value`, `item`, typed `walk`, -`pop!`, and `replace!`. Out of the box this covers everything Base +passed to `get`, `key`, `value`, `item`, typed `walk`, `pop!`, and +`replace!`. Out of the box this covers everything Base ships: the primitive numeric types (`Int8`/…/`Int128`, `Float16`/…/ `Float64`, `Bool`, `Char`, `Ptr{T}`) plus `String`, all zero-allocation thanks to the `@inline` `unsafe_read` override below. The package adds @@ -92,7 +91,7 @@ sources: return read(io, Vector{UInt8}) end - LMDB.tryget(txn, dbi, key, PrefixedBlob) # → Union{Vector{UInt8}, Nothing} + LMDB.get(txn, dbi, key, PrefixedBlob, nothing) # → Union{Vector{UInt8}, Nothing} For an `isbitstype` struct `T`, the one-liner is the standard Base pattern: diff --git a/src/cursor.jl b/src/cursor.jl index bbcb90f..855a812 100644 --- a/src/cursor.jl +++ b/src/cursor.jl @@ -1,25 +1,20 @@ -export Cursor, walk, - seek!, seek_last!, seek_range!, next!, prev!, - seek_first_dup!, seek_last_dup!, - next_dup!, prev_dup!, next_nodup!, prev_nodup! -@public transaction, database, key, value, item +@public Cursor, + seek!, seek_last!, seek_range!, next!, prev!, + seek_first_dup!, seek_last_dup!, + next_dup!, prev_dup!, next_nodup!, prev_nodup!, + walk, transaction, database, key, value, item """ A handle to a cursor structure for navigating through a database. -A `Cursor` keeps references to its parent `Transaction` and `DBI`, both +A `Cursor` keeps references to its parent `Transaction` and `Database`, both to expose them via `transaction(cur)` / `database(cur)` and to keep the txn alive under GC. The cursor's finalizer closes any still-open handle. """ mutable struct Cursor handle::Ptr{MDB_cursor} txn::Transaction - dbi::DBI - function Cursor(txn::Transaction, dbi::DBI, h::Ptr{MDB_cursor}) - c = new(h, txn, dbi) - finalizer(close, c) - return c - end + dbi::Database end Base.unsafe_convert(::Type{Ptr{MDB_cursor}}, c::Cursor) = c.handle @@ -27,16 +22,28 @@ Base.unsafe_convert(::Type{Ptr{MDB_cursor}}, c::Cursor) = c.handle "Check if cursor is open" isopen(cur::Cursor) = cur.handle != C_NULL -"Create a cursor" -function open(txn::Transaction, dbi::DBI) +""" + Cursor(txn::Transaction, dbi::Database) -> Cursor + +Open a cursor over `dbi` inside `txn`. The cursor is freed by its +finalizer if `close` isn't called explicitly. +""" +function Cursor(txn::Transaction, dbi::Database) cur_ptr_ref = Ref{Ptr{MDB_cursor}}(C_NULL) mdb_cursor_open(txn, dbi, cur_ptr_ref) - return Cursor(txn, dbi, cur_ptr_ref[]) + cur = Cursor(cur_ptr_ref[], txn, dbi) + finalizer(close, cur) + return cur end -"Wrapper of Cursor `open` for `do` construct" -function open(f::Function, txn::Transaction, dbi::DBI) - cur = open(txn, dbi) +""" + Cursor(f::Function, txn::Transaction, dbi::Database) -> result + +`do`-block form: open a cursor, run `f(cur)`, close on the way out. +Returns whatever `f` returns. +""" +function Cursor(f::Function, txn::Transaction, dbi::Database) + cur = Cursor(txn, dbi) try f(cur) finally @@ -73,38 +80,27 @@ Base.show(io::IO, cur::Cursor) = print(io, "Cursor(", isopen(cur) ? "open" : "closed", ")") -# Populate `key_ref` with `searchkey`'s data. Returns the heap-rooted argument -# that the caller must keep alive across the surrounding ccall (use -# `GC.@preserve`); the pointer baked into `key_ref` aliases its data. -@inline function setup_key!(key_ref, k::String) - key_ref[] = MDB_val(Csize_t(sizeof(k)), - Ptr{Cvoid}(Base.unsafe_convert(Ptr{UInt8}, k))) - return k -end -@inline function setup_key!(key_ref, k::AbstractArray{T}) where {T} - key_ref[] = MDB_val(Csize_t(sizeof(T) * length(k)), - Ptr{Cvoid}(Base.unsafe_convert(Ptr{T}, k))) - return k -end -@inline function setup_key!(key_ref, k::Base.RefValue{T}) where {T} - key_ref[] = MDB_val(Csize_t(sizeof(T)), - Ptr{Cvoid}(Base.unsafe_convert(Ptr{T}, k))) - return k -end -@inline function setup_key!(key_ref, k::T) where T - isbitstype(T) || throw(MethodError(setup_key!, (key_ref, k))) - return setup_key!(key_ref, Ref(k)) +# Fill `dst` with the MDB_val that the cconvert/unsafe_convert ladder +# in `common.jl` would build for `k`. `k`'s storage must outlive the +# eventual ccall (use `GC.@preserve`); the pointer baked into `dst` +# aliases its data. +@inline function fill_mdbval!(dst::Ref{MDB_val}, k) + arg = Base.cconvert(Ptr{MDB_val}, k) + dst[] = unsafe_load(Base.unsafe_convert(Ptr{MDB_val}, arg)) + return arg end # Position the cursor with `op`. Returns `true` on success, `false` on -# `MDB_NOTFOUND`. Throws on other errors. +# `MDB_NOTFOUND`. Throws on other errors. `key_ref` is used both to feed +# the search key in (for SET_KEY / SET_RANGE) and to receive the matched +# key on the way out. @inline function cursor_seek!(cur::Cursor, key_ref::Ref{MDB_val}, val_ref::Ref{MDB_val}, op::MDB_cursor_op, searchkey) if searchkey === nothing ret = unchecked_mdb_cursor_get(cur, key_ref, val_ref, op) else - held = setup_key!(key_ref, searchkey) + held = fill_mdbval!(key_ref, searchkey) ret = GC.@preserve held unchecked_mdb_cursor_get(cur, key_ref, val_ref, op) end ret == MDB_NOTFOUND && return false @@ -331,7 +327,7 @@ function walk(f, cur::Cursor; from = nothing) if from === nothing ret = unchecked_mdb_cursor_get(cur, key_ref, val_ref, MDB_FIRST) else - held = setup_key!(key_ref, from) + held = fill_mdbval!(key_ref, from) ret = GC.@preserve held unchecked_mdb_cursor_get(cur, key_ref, val_ref, MDB_SET_RANGE) end @@ -348,7 +344,7 @@ end """ walk(f, cur::Cursor, ::Type{K}, ::Type{V}=K; from = nothing) -Typed overload of `walk` mirroring the `tryget(txn, dbi, key, T)` / +Typed overload of `walk` mirroring the `get(txn, dbi, key, T, default)` / `key(cur, T)` / `seek!(cur, key, T)` shape used elsewhere in the Julia wrappers. Decodes each key and value through `read(::MDBValueIO, K)` / `read(::MDBValueIO, V)` before passing them to `f(k::K, v::V)`. Same @@ -357,7 +353,7 @@ stop contract as the raw form: `f` returning `false` halts iteration. Define a custom `Base.read(io::LMDB.MDBValueIO, ::Type{T})` to control what gets decoded (for example, a `(atime, size)` tuple from a framed value, or a zero-copy view). This is the iteration counterpart to -`tryget(..., T)`. +`get(..., T, nothing)`. """ function walk(f, cur::Cursor, ::Type{K}, ::Type{V} = K; from = nothing) where {K, V} diff --git a/src/database.jl b/src/database.jl index 1a28016..88ead24 100644 --- a/src/database.jl +++ b/src/database.jl @@ -1,30 +1,42 @@ -export DBI, tryget, put_reserved! -@public flags +@public Database, put_reserved!, flags """ A handle for an individual database in the DB environment. """ -mutable struct DBI +mutable struct Database handle::MDB_dbi name::String end -Base.cconvert(::Type{MDB_dbi}, d::DBI) = d.handle +Base.cconvert(::Type{MDB_dbi}, d::Database) = d.handle "Check if database is open" -isopen(dbi::DBI) = dbi.handle != zero(Cuint) +isopen(dbi::Database) = dbi.handle != zero(Cuint) -"Open a database in the environment" -function open(txn::Transaction, dbname::String = ""; flags::Integer = zero(Cuint)) - cdbname = length(dbname) > 0 ? dbname : Ptr{Cchar}(C_NULL) +""" + Database(txn::Transaction, dbname::AbstractString = ""; flags=0) -> Database + +Open a named sub-database inside the transaction. An empty `dbname` +opens the environment's default DB. `flags` is forwarded to +`mdb_dbi_open` (e.g. `MDB_CREATE`, `MDB_DUPSORT`). +""" +function Database(txn::Transaction, dbname::AbstractString = ""; + flags::Integer = zero(Cuint)) + cdbname = length(dbname) > 0 ? String(dbname) : Ptr{Cchar}(C_NULL) handle = Ref{MDB_dbi}() mdb_dbi_open(txn, cdbname, Cuint(flags), handle) - return DBI(handle[], dbname) + return Database(handle[], String(dbname)) end -"Wrapper of DBI `open` for `do` construct" -function open(f::Function, txn::Transaction, dbname::String = ""; flags::Integer = zero(Cuint)) - dbi = open(txn, dbname, flags=Cuint(flags)) +""" + Database(f::Function, txn::Transaction, dbname::AbstractString = ""; kwargs...) -> result + +`do`-block form: open `dbname`, run `f(dbi)`, close the handle on the +way out. Returns whatever `f` returns. +""" +function Database(f::Function, txn::Transaction, dbname::AbstractString = ""; + flags::Integer = zero(Cuint)) + dbi = Database(txn, dbname; flags = Cuint(flags)) tenv = env(txn) try f(dbi) @@ -34,7 +46,7 @@ function open(f::Function, txn::Transaction, dbname::String = ""; flags::Integer end "Close a database handle. Idempotent on both env and dbi." -function close(env::Environment, dbi::DBI) +function close(env::Environment, dbi::Database) isopen(env) || return isopen(dbi) || return mdb_dbi_close(env, dbi) @@ -43,14 +55,14 @@ function close(env::Environment, dbi::DBI) end "Retrieve the DB flags for a database handle" -function flags(txn::Transaction, dbi::DBI) +function flags(txn::Transaction, dbi::Database) flags = Ref{Cuint}(0) mdb_dbi_flags(txn, dbi, flags) return flags[] end -function Base.show(io::IO, dbi::DBI) - print(io, "DBI(") +function Base.show(io::IO, dbi::Database) + print(io, "Database(") isempty(dbi.name) ? print(io, "
") : show(io, dbi.name) print(io, ", ", isopen(dbi) ? "open" : "closed", ")") end @@ -60,17 +72,17 @@ end If parameter `delete` is `false` DB will be emptied, otherwise DB will be deleted from the environment and DB handle will be closed """ -function drop(txn::Transaction, dbi::DBI; delete = false) +function drop(txn::Transaction, dbi::Database; delete = false) mdb_drop(txn, dbi, Cint(delete)) end "Store items into a database" -function put!(txn::Transaction, dbi::DBI, key, val; flags::Integer = zero(Cuint)) +function put!(txn::Transaction, dbi::Database, key, val; flags::Integer = zero(Cuint)) mdb_put(txn, dbi, key, val, Cuint(flags)) end """ - put_reserved!(f, txn::Transaction, dbi::DBI, key, size::Integer; flags=0) + put_reserved!(f, txn::Transaction, dbi::Database, key, size::Integer; flags=0) Allocate `size` bytes of value space at `key` directly in LMDB's mmap'd write buffer, then call `f(buf::Vector{UInt8})` so the caller @@ -88,7 +100,7 @@ return; copy what you want to keep. Cannot be combined with `MDB_DUPSORT` or `MDB_DUPFIXED` databases, since LMDB forbids `MDB_RESERVE` there. """ -function put_reserved!(f, txn::Transaction, dbi::DBI, key, size::Integer; +function put_reserved!(f, txn::Transaction, dbi::Database, key, size::Integer; flags::Integer = zero(Cuint)) val_ref = Ref(MDB_val(Csize_t(size), C_NULL)) mdb_put(txn, dbi, key, val_ref, Cuint(flags) | Cuint(MDB_RESERVE)) @@ -98,8 +110,8 @@ function put_reserved!(f, txn::Transaction, dbi::DBI, key, size::Integer; end """ - delete!(txn::Transaction, dbi::DBI, key) -> Bool - delete!(txn::Transaction, dbi::DBI, key, val) -> Bool + delete!(txn::Transaction, dbi::Database, key) -> Bool + delete!(txn::Transaction, dbi::Database, key, val) -> Bool Delete `key` (or, in `MDB_DUPSORT`, the specific `(key, val)` pair) from the database. Returns `true` if an entry was removed, `false` if the @@ -109,9 +121,9 @@ The Bool-return, no-throw-on-miss shape matches `Base.delete!`'s "if any" contract and the LMDB-binding convention shared by heed, py-lmdb, lmdb-js, and lmdbxx. """ -delete!(txn::Transaction, dbi::DBI, key) = _delete!(txn, dbi, key, MDBValue()) -delete!(txn::Transaction, dbi::DBI, key, val) = _delete!(txn, dbi, key, val) -function _delete!(txn::Transaction, dbi::DBI, key, val_arg) +delete!(txn::Transaction, dbi::Database, key) = _delete!(txn, dbi, key, MDBValue()) +delete!(txn::Transaction, dbi::Database, key, val) = _delete!(txn, dbi, key, val) +function _delete!(txn::Transaction, dbi::Database, key, val_arg) ret = unchecked_mdb_del(txn, dbi, key, val_arg) ret == MDB_NOTFOUND && return false iszero(ret) || throw(LMDBError(ret)) @@ -119,7 +131,7 @@ function _delete!(txn::Transaction, dbi::DBI, key, val_arg) end """ - stat(txn::Transaction, dbi::DBI) -> NamedTuple + stat(txn::Transaction, dbi::Database) -> NamedTuple Return statistics for the database referenced by `dbi` within `txn`: @@ -134,59 +146,62 @@ Return statistics for the database referenced by `dbi` within `txn`: Live byte usage = `(branch_pages + leaf_pages + overflow_pages) * psize`. """ -function stat(txn::Transaction, dbi::DBI) +function stat(txn::Transaction, dbi::Database) s_ref = Ref{MDB_stat}() mdb_stat(txn, dbi, s_ref) return stat_namedtuple(s_ref[]) end -"""Get an item from a database. Throws `LMDBError` if `key` is not present.""" -@inline function get(txn::Transaction, dbi::DBI, key, ::Type{T}) where T +""" + get(txn::Transaction, dbi::Database, key, ::Type{T}) -> T + +Read the value at `key`, decoded as `T`. Throws `LMDBError` if `key` +isn't there — the same shape as `d[k]` on a regular `AbstractDict`. +""" +@inline function get(txn::Transaction, dbi::Database, key, ::Type{T}) where T val_ref = Ref(MDBValue()) mdb_get(txn, dbi, key, val_ref) return Base.read(MDBValueIO(val_ref[]), T) end -"""Get an item from a database, returning `nothing` if `key` is not present. -Use this in preference to `get` + try/catch when a missing key is expected.""" -@inline function tryget(txn::Transaction, dbi::DBI, key, ::Type{T}) where T +""" + get(txn::Transaction, dbi::Database, key, ::Type{T}, default) -> Union{T,typeof(default)} + +Read the value at `key`, decoded as `T`; return `default` if the key +isn't there. Same shape as `Base.get(dict, key, default)`. Pass +`nothing` as `default` for the `Union{T,Nothing}` form. +""" +@inline function get(txn::Transaction, dbi::Database, key, ::Type{T}, default) where T val_ref = Ref(MDBValue()) ret = unchecked_mdb_get(txn, dbi, key, val_ref) - ret == MDB_NOTFOUND && return nothing + ret == MDB_NOTFOUND && return default iszero(ret) || throw(LMDBError(ret)) return Base.read(MDBValueIO(val_ref[]), T) end -"""Get an item from a database, returning `default` if `key` is not present. -The signature mirrors `Base.get(dict, key, default)`.""" -function get(txn::Transaction, dbi::DBI, key, ::Type{T}, default) where T - v = tryget(txn, dbi, key, T) - v === nothing ? default : v -end - """ - replace!(txn::Transaction, dbi::DBI, key, val, ::Type{V}=typeof(val)) + replace!(txn::Transaction, dbi::Database, key, val, ::Type{V}=typeof(val)) -> Union{V,Nothing} Atomically write `val` at `key`, returning the previous value (decoded as `V`) or `nothing` if `key` was not present. Read and write share the same transaction. """ -function replace!(txn::Transaction, dbi::DBI, key, val, +function replace!(txn::Transaction, dbi::Database, key, val, ::Type{V}=typeof(val)) where V - old = tryget(txn, dbi, key, V) + old = get(txn, dbi, key, V, nothing) put!(txn, dbi, key, val) return old end """ - pop!(txn::Transaction, dbi::DBI, key, ::Type{T}) -> Union{T,Nothing} + pop!(txn::Transaction, dbi::Database, key, ::Type{T}) -> Union{T,Nothing} Atomically read and delete the value at `key`, returning it (decoded as `T`) or `nothing` if `key` was not present. """ -function pop!(txn::Transaction, dbi::DBI, key, ::Type{T}) where T - v = tryget(txn, dbi, key, T) +function pop!(txn::Transaction, dbi::Database, key, ::Type{T}) where T + v = get(txn, dbi, key, T, nothing) v === nothing && return nothing delete!(txn, dbi, key) return v diff --git a/src/dictionary.jl b/src/dictionary.jl index 59eb521..9ad4cba 100644 --- a/src/dictionary.jl +++ b/src/dictionary.jl @@ -5,7 +5,7 @@ export LMDBDict LMDBDict{K,V}(path; readonly, rdahead, mapsize, readers, dbs) A persistent `AbstractDict{K,V}` backed by a single LMDB environment -plus its default DBI. Keys and values are encoded as raw bytes; +plus its default Database. Keys and values are encoded as raw bytes; `String`, `Vector{T}` (where `T` is bitstype), and any bitstype scalar all work. @@ -14,8 +14,8 @@ see `LMDB.scan`, `LMDB.scan_keys`, `LMDB.scan_values`, and `LMDB.list_dirs`. """ mutable struct LMDBDict{K,V} <: AbstractDict{K,V} env::LMDB.Environment - dbi::LMDB.DBI - function LMDBDict{K,V}(env::LMDB.Environment, dbi::LMDB.DBI) where {K,V} + dbi::LMDB.Database + function LMDBDict{K,V}(env::LMDB.Environment, dbi::LMDB.Database) where {K,V} x = new{K,V}(env, dbi) finalizer(x) do d LMDB.close(d.env, d.dbi) @@ -37,8 +37,8 @@ function LMDBDict{K,V}(path::String; readonly = false, rdahead = false, readonly && (envflags |= Cuint(MDB_RDONLY)) env = LMDB.Environment(path; mapsize, maxreaders = readers, maxdbs = dbs, flags = envflags) - dbi = LMDB.start(env) do txn - LMDB.open(txn) + dbi = Transaction(env) do txn + Database(txn) end LMDBDict{K,V}(env, dbi) end @@ -53,8 +53,8 @@ end function cursor_do(f, d; readonly = false) txnflags = readonly ? Cuint(LMDB.MDB_RDONLY) : Cuint(0) - LMDB.start(d.env, flags = txnflags) do txn - LMDB.open(txn, d.dbi) do cur + Transaction(d.env; flags = txnflags) do txn + Cursor(txn, d.dbi) do cur f(cur) end end @@ -62,7 +62,7 @@ end function txn_dbi_do(f, d; readonly = false) txnflags = readonly ? Cuint(LMDB.MDB_RDONLY) : Cuint(0) - LMDB.start(d.env, flags = txnflags) do txn + Transaction(d.env; flags = txnflags) do txn f(txn, d.dbi) end end @@ -96,8 +96,8 @@ end # committed and the cursor closed; on early break/throw, Cursor's and # Transaction's finalizers reclaim them. function Base.iterate(d::LMDBDict) - txn = LMDB.start(d.env; flags = Cuint(MDB_RDONLY)) - cur = LMDB.open(txn, d.dbi) + txn = Transaction(d.env; flags = Cuint(MDB_RDONLY)) + cur = Cursor(txn, d.dbi) return iter_step(d, txn, cur, MDB_FIRST) end Base.iterate(d::LMDBDict, (txn, cur)::Tuple{Transaction,Cursor}) = @@ -121,8 +121,6 @@ function iter_step(::LMDBDict{K,V}, txn::Transaction, cur::Cursor, Base.read(LMDB.MDBValueIO(v_ref[]), V), (txn, cur)) end -Base.IteratorSize(::Type{<:LMDBDict}) = Base.HasLength() - function Base.length(d::LMDBDict) txn_dbi_do(d, readonly = true) do txn, dbi Int(LMDB.stat(txn, dbi).entries) @@ -133,14 +131,14 @@ Base.isempty(d::LMDBDict) = iszero(length(d)) function Base.getindex(d::LMDBDict{K,V}, k) where {K,V} txn_dbi_do(d, readonly = true) do txn, dbi - v = LMDB.tryget(txn, dbi, convert(K, k), V) + v = LMDB.get(txn, dbi, convert(K, k), V, nothing) v === nothing ? throw(KeyError(k)) : v end end function Base.haskey(d::LMDBDict{K,V}, k) where {K,V} txn_dbi_do(d, readonly = true) do txn, dbi - LMDB.tryget(txn, dbi, convert(K, k), V) !== nothing + LMDB.get(txn, dbi, convert(K, k), V, nothing) !== nothing end end @@ -152,14 +150,14 @@ end function Base.get(f::Base.Callable, d::LMDBDict{K,V}, k) where {K,V} txn_dbi_do(d, readonly = true) do txn, dbi - v = LMDB.tryget(txn, dbi, convert(K, k), V) + v = LMDB.get(txn, dbi, convert(K, k), V, nothing) v === nothing ? f() : v end end function Base.get!(d::LMDBDict{K,V}, k, default) where {K,V} txn_dbi_do(d) do txn, dbi - v = LMDB.tryget(txn, dbi, convert(K, k), V) + v = LMDB.get(txn, dbi, convert(K, k), V, nothing) v !== nothing && return v LMDB.put!(txn, dbi, convert(K, k), convert(V, default)) return default @@ -168,7 +166,7 @@ end function Base.get!(f::Base.Callable, d::LMDBDict{K,V}, k) where {K,V} txn_dbi_do(d) do txn, dbi - v = LMDB.tryget(txn, dbi, convert(K, k), V) + v = LMDB.get(txn, dbi, convert(K, k), V, nothing) v !== nothing && return v default = f() LMDB.put!(txn, dbi, convert(K, k), convert(V, default)) @@ -185,11 +183,7 @@ end function Base.delete!(d::LMDBDict{K}, k) where K txn_dbi_do(d) do txn, dbi - try - LMDB.delete!(txn, dbi, convert(K, k)) - catch e - e isa LMDBError && LMDB.is_notfound(e) || rethrow() - end + LMDB.delete!(txn, dbi, convert(K, k)) end return d end @@ -211,7 +205,7 @@ end # `pop!(d)` without a key: pops the first entry, mirroring `Base.pop!(::Dict)`. function Base.pop!(d::LMDBDict{K,V}) where {K,V} txn_dbi_do(d) do txn, dbi - LMDB.open(txn, dbi) do cur + Cursor(txn, dbi) do cur LMDB.seek!(cur, K) === nothing && throw(ArgumentError("LMDBDict must be non-empty")) pair = LMDB.item(cur, K, V) @@ -232,6 +226,17 @@ end # `==`, `hash`, `in(::Pair, d)` etc. all kick in for free now that # `iterate` and `length` are defined. +# An LMDBDict needs an on-disk directory to live in, so there's no +# meaningful zero-argument `empty(d)` (or `copy(d)`, which `Base` +# implements as `merge!(empty(d), d)`). Refuse with an explicit error +# so users reach for `Dict(d)` for an in-memory snapshot or +# `copy(d.env, path)` for an on-disk clone. +const _NO_EMPTY_LMDBDICT = + "LMDBDict has no path-less empty form; use Dict(d) for an in-memory " * + "snapshot, or copy(d.env, path) for an on-disk clone" +Base.empty(::LMDBDict, ::Type, ::Type) = throw(ArgumentError(_NO_EMPTY_LMDBDICT)) +Base.copy(::LMDBDict) = throw(ArgumentError(_NO_EMPTY_LMDBDICT)) + # Override the bulk-update fallbacks so they land in a single LMDB write # txn. AbstractDict's default `merge!` / `mergewith!` / `filter!` call # `d[k]=v` / `delete!(d,k)` in a loop, and each of those opens its own @@ -249,7 +254,7 @@ function Base.mergewith!(combine, d::LMDBDict{K,V}, others::AbstractDict...) whe txn_dbi_do(d) do txn, dbi for other in others, (k, v) in other kk = convert(K, k) - existing = LMDB.tryget(txn, dbi, kk, V) + existing = LMDB.get(txn, dbi, kk, V, nothing) new = existing === nothing ? convert(V, v) : convert(V, combine(existing, v)) LMDB.put!(txn, dbi, kk, new) @@ -261,7 +266,7 @@ end function Base.filter!(f, d::LMDBDict{K,V}) where {K,V} txn_dbi_do(d) do txn, dbi to_delete = K[] - LMDB.open(txn, dbi) do cur + Cursor(txn, dbi) do cur LMDB.walk(cur, K, V) do k, v f(k => v) || push!(to_delete, k) end diff --git a/src/environment.jl b/src/environment.jl index e8ddcc1..f5e5c9f 100644 --- a/src/environment.jl +++ b/src/environment.jl @@ -1,14 +1,11 @@ -export Environment, create, environment, - sync, set!, unset!, info, - reader_check, reader_list -@public path +@public Environment, sync, set!, unset!, info, path, + reader_check, reader_list """ A DB environment supports multiple databases, all residing in the same shared-memory map. -Wrapping a raw `Ptr{MDB_env}` in `Environment(h)` takes ownership of the -handle. The handle is closed when the wrapper is garbage-collected, unless -`close` was already called explicitly. Closing is idempotent. +The handle is closed when the wrapper is garbage-collected, unless `close` +was already called explicitly. Closing is idempotent. """ mutable struct Environment handle::Ptr{MDB_env} @@ -17,88 +14,45 @@ mutable struct Environment # walks this list to abort any still-open txn before calling # `mdb_env_close`; otherwise LMDB corrupts state shared across envs. txns::Vector{WeakRef} - function Environment(h::Ptr{MDB_env} = C_NULL) - e = new(h, "", WeakRef[]) - finalizer(close, e) - return e - end end Base.unsafe_convert(::Type{Ptr{MDB_env}}, e::Environment) = e.handle -"Return the path that was used in `open`" +"Return the path that was used to open the environment." path(env::Environment) = env.path "Check if environment is open" isopen(env::Environment) = env.handle != C_NULL -"Create an LMDB environment handle" -function create() - env_ref = Ref{Ptr{MDB_env}}() - mdb_env_create(env_ref) - return Environment(env_ref[]) -end - -"Wrapper of `create` for `do` construct" -function create(f::Function) - env = create() - try - f(env) - finally - close(env) - end -end - -"""Open an environment handle - -`open` function accepts following parameters: -* `env` db environment object -* `path` directory in which the database files reside -* `flags` defines special options for the environment -* `mode` UNIX permissions to set on created files - -*Note:* A database directory must exist and be writable. -""" -function open(env::Environment, path::String; flags::Integer=zero(Cuint), - mode::Integer = mode_t(0o755)) - env.path = path - mdb_env_open(env, path, Cuint(flags), mode_t(mode)) -end - -"Wrapper of `open` for `do` construct" -function environment(f::Function, path::String; flags::Integer=zero(Cuint), - mode::Integer = mode_t(0o755)) - env = create() - try - open(env, path; flags = Cuint(flags), mode = mode_t(mode)) - f(env) - finally - close(env) - end -end - """ Environment(path::AbstractString; mapsize=nothing, maxreaders=nothing, maxdbs=nothing, flags=0, mode=0o755) -> Environment -One-call equivalent of `create()`, optional `setindex!` for `MapSize`, -`Readers`, or `DBs`, and `open(env, path)`. Mirrors py-lmdb's -`Environment(path, **kwargs)` and lmdb-rs's `EnvironmentBuilder.open(path)`. +Open the LMDB environment at `path`. The directory must already exist +and be writable. The configuration kwargs go through +`mdb_env_set_mapsize`, `mdb_env_set_maxreaders`, and `mdb_env_set_maxdbs`; +`flags` is forwarded to `mdb_env_open`. If anything fails before the +open completes, the half-open env is closed before the exception +propagates. -If anything fails between `create` and a successful `open`, the partially -constructed environment is closed before rethrowing. +The shape matches py-lmdb's `Environment(path, **kwargs)` and lmdb-rs's +`EnvironmentBuilder.open(path)`. """ function Environment(path::AbstractString; mapsize::Union{Integer,Nothing} = nothing, maxreaders::Union{Integer,Nothing} = nothing, maxdbs::Union{Integer,Nothing} = nothing, flags::Integer = zero(Cuint), mode::Integer = mode_t(0o755)) - env = create() + env_ref = Ref{Ptr{MDB_env}}() + mdb_env_create(env_ref) + env = Environment(env_ref[], "", WeakRef[]) + finalizer(close, env) try mapsize === nothing || (env[:MapSize] = mapsize) maxreaders === nothing || (env[:Readers] = maxreaders) maxdbs === nothing || (env[:DBs] = maxdbs) - open(env, String(path); flags = Cuint(flags), mode = mode_t(mode)) + env.path = String(path) + mdb_env_open(env, env.path, Cuint(flags), mode_t(mode)) catch close(env) rethrow() @@ -106,6 +60,21 @@ function Environment(path::AbstractString; mapsize::Union{Integer,Nothing} = not return env end +""" + Environment(f::Function, path::AbstractString; kwargs...) -> result + +`do`-block form. Opens the env, runs `f(env)`, closes it on the way +out — even if `f` throws. Returns whatever `f` returns. +""" +function Environment(f::Function, path::AbstractString; kwargs...) + env = Environment(path; kwargs...) + try + f(env) + finally + close(env) + end +end + """Close the environment and release the memory map. Idempotent: calling `close` on an already-closed `Environment` is a @@ -113,7 +82,7 @@ silent no-op, matching the convention of `close(::IO)`. That makes finalizers safe to run after an explicit close. """ function close(env::Environment) - env.handle == C_NULL && return zero(Cint) + env.handle == C_NULL && return nothing # LMDB requires all transactions to be closed before `mdb_env_close`; # otherwise it leaves shared lockfile/heap state corrupted and the # next env-open in the process can crash inside `mdb_txn_renew0`. @@ -125,26 +94,26 @@ function close(env::Environment) mdb_env_close(env) env.handle = C_NULL env.path = "" - return zero(Cint) + return nothing end """Flush the data buffers to disk""" function sync(env::Environment, force::Bool = false) fval = force ? 1 : 0 mdb_env_sync(env, fval) - return zero(Cint) + return nothing end -"""Set environment flags""" +"""Set environment flags. Returns `env` to allow chaining.""" function set!(env::Environment, flag::Integer) mdb_env_set_flags(env, Cuint(flag), one(Cint)) - return flag + return env end -"""Unset environment flags""" +"""Unset environment flags. Returns `env` to allow chaining.""" function unset!(env::Environment, flag::Integer) mdb_env_set_flags(env, Cuint(flag), zero(Cint)) - return flag + return env end @@ -324,15 +293,24 @@ function reader_list(env::Environment) return String(take!(io)) end -function show(io::IO, env::Environment) - print(io,"Environment is ", isopen(env) ? (isempty(env.path) ? "created" : "opened") : "closed") - if !isempty(env.path) - print(io,"\nDB path: $(path(env))") - ei = info(env) - print(io,"\nSize of the data memory map: $(ei.mapsize)") - print(io,"\nID of the last used page: $(ei.last_pgno)") - print(io,"\nID of the last committed transaction: $(ei.last_txnid)") - print(io,"\nMax reader slots in the environment: $(ei.maxreaders)") - print(io,"\nMax reader slots used in the environment: $(ei.numreaders)") - end +function Base.show(io::IO, env::Environment) + state = !isopen(env) ? "closed" : + isempty(env.path) ? "created" : "opened" + print(io, "Environment(", state) + isempty(env.path) || print(io, ", ", repr(env.path)) + print(io, ")") +end + +function Base.show(io::IO, ::MIME"text/plain", env::Environment) + state = !isopen(env) ? "closed" : + isempty(env.path) ? "created" : "opened" + print(io, "Environment is ", state) + isempty(env.path) && return + ei = info(env) + print(io, "\nDB path: ", path(env)) + print(io, "\nSize of the data memory map: ", ei.mapsize) + print(io, "\nID of the last used page: ", ei.last_pgno) + print(io, "\nID of the last committed transaction: ", ei.last_txnid) + print(io, "\nMax reader slots in the environment: ", ei.maxreaders) + print(io, "\nMax reader slots used in the environment: ", ei.numreaders) end diff --git a/src/error.jl b/src/error.jl index 238a2cb..ad230b7 100644 --- a/src/error.jl +++ b/src/error.jl @@ -6,7 +6,8 @@ struct LMDBError <: Exception msg::AbstractString LMDBError(code::Integer) = new(Cint(code), unsafe_string(mdb_strerror(code))) end -show(io::IO, err::LMDBError) = print(io, "Code[$(err.code)]: $(err.msg)") +Base.showerror(io::IO, err::LMDBError) = + print(io, "LMDBError(", err.code, "): ", err.msg) "Throw an `LMDBError` if `code` is non-zero. Returns `code` otherwise." @inline check(code) = iszero(code) ? code : throw(LMDBError(code)) diff --git a/src/liblmdb.jl b/src/liblmdb.jl index dd820d1..1706d94 100644 --- a/src/liblmdb.jl +++ b/src/liblmdb.jl @@ -2,7 +2,6 @@ # To re-generate, execute res/wrap.jl using LMDB_jll -export LMDB_jll using CEnum: CEnum, @cenum diff --git a/src/transaction.jl b/src/transaction.jl index fa7097f..aa64992 100644 --- a/src/transaction.jl +++ b/src/transaction.jl @@ -1,5 +1,4 @@ -export Transaction, start, abort, commit, renew -@public env +@public Transaction, env, abort, commit, renew """ A database transaction. Every database operation requires a transaction. @@ -13,13 +12,6 @@ aborted by its finalizer. mutable struct Transaction handle::Ptr{MDB_txn} env::Environment - function Transaction(env::Environment, h::Ptr{MDB_txn}) - t = new(h, env) - # Track on the env so `close(env)` can abort us if the caller forgot. - push!(env.txns, WeakRef(t)) - finalizer(_finalize_txn, t) - return t - end end Base.unsafe_convert(::Type{Ptr{MDB_txn}}, t::Transaction) = t.handle @@ -30,27 +22,44 @@ env(txn::Transaction) = txn.env "Check if transaction is open." isopen(txn::Transaction) = txn.handle != C_NULL -"""Create a transaction for use with the environment +""" + Transaction(env::Environment; flags=0, parent=nothing) -> Transaction -`start` function creates a new transaction and returns `Transaction` object. -It allows to set transaction flags with `flags` option. +Begin a transaction against `env`. `flags` is forwarded to +`mdb_txn_begin` (e.g. `MDB_RDONLY` for a read-only txn). `parent` lets +you nest a write txn inside an existing one. Call `commit` to persist +or `abort` to discard; a dropped transaction is aborted by its +finalizer. """ -function start(env::Environment; flags::Integer=zero(Cuint), - parent::Union{Transaction,Nothing} = nothing) +function Transaction(env::Environment; flags::Integer = zero(Cuint), + parent::Union{Transaction,Nothing} = nothing) txn_ref = Ref{Ptr{MDB_txn}}(C_NULL) p = parent === nothing ? C_NULL : parent mdb_txn_begin(env, p, Cuint(flags), txn_ref) - return Transaction(env, txn_ref[]) + txn = Transaction(txn_ref[], env) + # Track on the env so `close(env)` can abort us if the caller forgot. + push!(env.txns, WeakRef(txn)) + finalizer(_finalize_txn, txn) + return txn end -function start(f::Function, env::Environment; flags::Integer=zero(Cuint)) - txn = start(env, flags=Cuint(flags)) + +""" + Transaction(f::Function, env::Environment; kwargs...) -> result + +`do`-block form: begin a transaction, run `f(txn)`, commit on a normal +return, abort if `f` throws. Returns whatever `f` returns. +""" +function Transaction(f::Function, env::Environment; + flags::Integer = zero(Cuint), + parent::Union{Transaction,Nothing} = nothing) + txn = Transaction(env; flags = Cuint(flags), parent) try r = f(txn) commit(txn) - r - catch e + return r + catch abort(txn) - rethrow(e) + rethrow() end end diff --git a/test/cursor.jl b/test/cursor.jl index f8c62ce..b73f6ee 100644 --- a/test/cursor.jl +++ b/test/cursor.jl @@ -6,15 +6,14 @@ val = "key value is " # Procedural style + block style smoke test, exercising cursor put!/walk # round-trip and the parent accessors. mktempdir() do dbname - env = create() + env = LMDB.Environment(dbname) try - open(env, dbname) - txn = start(env) - dbi = open(txn) - commit(txn) + txn = LMDB.Transaction(env) + dbi = LMDB.Database(txn) + LMDB.commit(txn) - txn = start(env) - cur = open(txn, dbi) + txn = LMDB.Transaction(env) + cur = LMDB.Cursor(txn, dbi) try @test 0 == put!(cur, key+1, val*string(key+1)) @test 0 == put!(cur, key, val*string(key)) @@ -25,7 +24,7 @@ mktempdir() do dbname @test issetequal(ks, [11, 10]) finally close(cur) - commit(txn) + LMDB.commit(txn) end @test !isopen(cur) @test !isopen(txn) @@ -35,10 +34,10 @@ mktempdir() do dbname @test !isopen(env) # Block style: parent accessors return the actual handles, not synthetic ones. - environment(dbname) do env - start(env) do txn - open(txn) do dbi - open(txn, dbi) do cur + LMDB.Environment(dbname) do env + LMDB.Transaction(env) do txn + LMDB.Database(txn) do dbi + LMDB.Cursor(txn, dbi) do cur @test LMDB.transaction(cur) === txn @test LMDB.database(cur) === dbi @test LMDB.seek!(cur, key, typeof(key)) == key @@ -50,16 +49,16 @@ mktempdir() do dbname end end -# Cursor positioning + walk primitives. +# LMDB.Cursor positioning + walk primitives. mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi + LMDB.Environment(dir) do env + LMDB.Transaction(env) do txn + LMDB.Database(txn) do dbi LMDB.put!(txn, dbi, "a", "1") LMDB.put!(txn, dbi, "b", "2") LMDB.put!(txn, dbi, "c", "3") - LMDB.open(txn, dbi) do cur + LMDB.Cursor(txn, dbi) do cur @test LMDB.seek!(cur, String) == "a" @test LMDB.value(cur, String) == "1" @test LMDB.key(cur, String) == "a" @@ -121,10 +120,10 @@ end # seek!/next! on an empty database returns nothing. mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - LMDB.open(txn, dbi) do cur + LMDB.Environment(dir) do env + LMDB.Transaction(env) do txn + LMDB.Database(txn) do dbi + LMDB.Cursor(txn, dbi) do cur @test LMDB.seek!(cur, String) === nothing @test LMDB.seek_last!(cur, String) === nothing @test LMDB.seek!(cur, "x", String) === nothing @@ -141,25 +140,25 @@ mktempdir() do dir end end -# Cursor.delete!: removes the entry the cursor is on; LMDB advances to +# LMDB.Cursor.delete!: removes the entry the cursor is on; LMDB advances to # the next entry. Throws on an unpositioned cursor (EINVAL), unlike the # txn-based `delete!(txn, dbi, key)` which is Bool-returning on # MDB_NOTFOUND. mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi + LMDB.Environment(dir) do env + LMDB.Transaction(env) do txn + LMDB.Database(txn) do dbi LMDB.put!(txn, dbi, "a", "1") LMDB.put!(txn, dbi, "b", "2") - LMDB.open(txn, dbi) do cur + LMDB.Cursor(txn, dbi) do cur @test LMDB.seek!(cur, "a", String) == "a" LMDB.delete!(cur) # removes "a", cursor now on "b" LMDB.delete!(cur) # removes "b" @test_throws LMDBError LMDB.delete!(cur) # no live entry end - @test LMDB.tryget(txn, dbi, "a", String) === nothing - @test LMDB.tryget(txn, dbi, "b", String) === nothing + @test get(txn, dbi, "a", String, nothing) === nothing + @test get(txn, dbi, "b", String, nothing) === nothing end end end diff --git a/test/database.jl b/test/database.jl index 7816b9c..6f1faa1 100644 --- a/test/database.jl +++ b/test/database.jl @@ -29,17 +29,16 @@ val = "key value is " # Procedural style + block style smoke test, exercising String, Int, and # Vector{Int} round-trips through put!/get/delete!. mktempdir() do dbname - env = create() + env = LMDB.Environment(dbname) try - open(env, dbname) - txn = start(env) - dbi = open(txn) + txn = LMDB.Transaction(env) + dbi = LMDB.Database(txn) put!(txn, dbi, key+1, val*string(key+1)) put!(txn, dbi, key, val*string(key)) put!(txn, dbi, key+2, key+2) put!(txn, dbi, key+3, [key, key+1, key+2]) @test isopen(txn) - commit(txn) + LMDB.commit(txn) @test !isopen(txn) close(env, dbi) @test !isopen(dbi) @@ -49,11 +48,9 @@ mktempdir() do dbname @test !isopen(env) # Block style - create() do env - set!(env, LMDB.MDB_NOSYNC) - open(env, dbname) - start(env) do txn - open(txn, flags = Cuint(LMDB.MDB_REVERSEKEY)) do dbi + LMDB.Environment(dbname; flags = LMDB.MDB_NOSYNC) do env + LMDB.Transaction(env) do txn + LMDB.Database(txn; flags = Cuint(LMDB.MDB_REVERSEKEY)) do dbi k = key value = get(txn, dbi, k, String) @test value == val*string(k) @@ -74,17 +71,17 @@ mktempdir() do dbname end end -# tryget / get-with-default / stat(txn, dbi) — fresh env so the entry -# count is deterministic. +# get-with-default / stat(txn, dbi) — fresh env so the entry count is +# deterministic. mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi + LMDB.Environment(dir) do env + LMDB.Transaction(env) do txn + LMDB.Database(txn) do dbi LMDB.put!(txn, dbi, "k1", "v1") LMDB.put!(txn, dbi, "k2", "v2") - @test LMDB.tryget(txn, dbi, "k1", String) == "v1" - @test LMDB.tryget(txn, dbi, "missing", String) === nothing + @test get(txn, dbi, "k1", String, nothing) == "v1" + @test get(txn, dbi, "missing", String, nothing) === nothing @test get(txn, dbi, "k2", String, "fallback") == "v2" @test get(txn, dbi, "missing", String, "fallback") == "fallback" @@ -99,9 +96,9 @@ end # put_reserved!: callback-style MDB_RESERVE write. mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi + LMDB.Environment(dir) do env + LMDB.Transaction(env) do txn + LMDB.Database(txn) do dbi # Write a 16-byte value where bytes 0..7 are a UInt64 # header and bytes 8..15 are payload. The buffer hands # back is the LMDB-allocated mmap page; we fill it @@ -115,7 +112,7 @@ mktempdir() do dir buf[8 + i] = UInt8(i) end end - raw = LMDB.tryget(txn, dbi, "framed", Vector{UInt8}) + raw = get(txn, dbi, "framed", Vector{UInt8}, nothing) @test length(raw) == 16 @test ltoh(reinterpret(UInt64, raw[1:8])[1]) == UInt64(0xdeadbeef) @@ -134,15 +131,15 @@ end # delete!: Bool-returning, idempotent on MDB_NOTFOUND. mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi + LMDB.Environment(dir) do env + LMDB.Transaction(env) do txn + LMDB.Database(txn) do dbi LMDB.put!(txn, dbi, "k1", "v1") LMDB.put!(txn, dbi, "k2", "v2") # Present key → true, returns and entry is gone. @test LMDB.delete!(txn, dbi, "k1") === true - @test LMDB.tryget(txn, dbi, "k1", String) === nothing + @test get(txn, dbi, "k1", String, nothing) === nothing # Missing key → false, no exception. @test LMDB.delete!(txn, dbi, "ghost") === false @@ -158,20 +155,20 @@ end # replace! / pop! mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi + LMDB.Environment(dir) do env + LMDB.Transaction(env) do txn + LMDB.Database(txn) do dbi # replace! on a missing key returns nothing and creates the entry. @test LMDB.replace!(txn, dbi, "k", "v1") === nothing - @test LMDB.tryget(txn, dbi, "k", String) == "v1" + @test get(txn, dbi, "k", String, nothing) == "v1" # replace! on an existing key returns the old value. @test LMDB.replace!(txn, dbi, "k", "v2") == "v1" - @test LMDB.tryget(txn, dbi, "k", String) == "v2" + @test get(txn, dbi, "k", String, nothing) == "v2" # pop! returns the value and deletes. @test LMDB.pop!(txn, dbi, "k", String) == "v2" - @test LMDB.tryget(txn, dbi, "k", String) === nothing + @test get(txn, dbi, "k", String, nothing) === nothing # pop! on a missing key returns nothing. @test LMDB.pop!(txn, dbi, "k", String) === nothing end @@ -179,28 +176,28 @@ mktempdir() do dir end end -# Round-trip a custom bitstype through put!/tryget/walk using only the +# Round-trip a custom bitstype through put!/get/walk using only the # IO-based extension point (`Base.read(io::IO, ::Type{Point2D})`, # defined at module scope above). mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi + LMDB.Environment(dir) do env + LMDB.Transaction(env) do txn + LMDB.Database(txn) do dbi LMDB.put!(txn, dbi, "origin", Point2D(0f0, 0f0)) LMDB.put!(txn, dbi, "p1", Point2D(1.5f0, 2.5f0)) - @test LMDB.tryget(txn, dbi, "p1", Point2D) == Point2D(1.5f0, 2.5f0) - @test LMDB.tryget(txn, dbi, "origin", Point2D) == Point2D(0f0, 0f0) + @test get(txn, dbi, "p1", Point2D, nothing) == Point2D(1.5f0, 2.5f0) + @test get(txn, dbi, "origin", Point2D, nothing) == Point2D(0f0, 0f0) @test get(txn, dbi, "p1", Point2D) == Point2D(1.5f0, 2.5f0) # The Vector{E} overload also works for any bitstype E, # without the user defining anything extra. - @test LMDB.tryget(txn, dbi, "p1", Vector{Point2D}) == + @test get(txn, dbi, "p1", Vector{Point2D}, nothing) == [Point2D(1.5f0, 2.5f0)] # Typed walk decodes both K and V through Base.read. seen = Pair{String,Point2D}[] - LMDB.open(txn, dbi) do cur + LMDB.Cursor(txn, dbi) do cur LMDB.walk(cur, String, Point2D) do k, v push!(seen, k => v) end @@ -218,14 +215,14 @@ end # payload. Exercises the IO contract from inside user code (see # `FramedU64` / `FRAME_MAGIC` at module scope). mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi + LMDB.Environment(dir) do env + LMDB.Transaction(env) do txn + LMDB.Database(txn) do dbi LMDB.put_reserved!(txn, dbi, "framed", 12) do buf unsafe_store!(Ptr{UInt32}(pointer(buf)), FRAME_MAGIC) unsafe_store!(Ptr{UInt64}(pointer(buf) + 4), htol(UInt64(0x1234_5678))) end - @test LMDB.tryget(txn, dbi, "framed", FramedU64) == + @test get(txn, dbi, "framed", FramedU64, nothing) == FramedU64(0x1234_5678) end end @@ -235,23 +232,23 @@ end # Non-Array AbstractArray inputs (e.g. `ReinterpretArray`, contiguous # `SubArray`) flow through `cconvert(Ptr{MDB_val}, ::AbstractArray)`. mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi + LMDB.Environment(dir) do env + LMDB.Transaction(env) do txn + LMDB.Database(txn) do dbi # ReinterpretArray view onto a backing UInt64 vector. ra_key = reinterpret(UInt8, UInt64[0xdeadbeefcafef00d]) @test !(ra_key isa Array) LMDB.put!(txn, dbi, ra_key, "v-reinterpret") - @test LMDB.tryget(txn, dbi, ra_key, String) == "v-reinterpret" - @test LMDB.tryget(txn, dbi, collect(ra_key), String) == "v-reinterpret" + @test get(txn, dbi, ra_key, String, nothing) == "v-reinterpret" + @test get(txn, dbi, collect(ra_key), String, nothing) == "v-reinterpret" # Contiguous SubArray. backing = collect(0x01:0x10) sv_key = view(backing, 4:8) @test !(sv_key isa Array) LMDB.put!(txn, dbi, sv_key, "v-subarray") - @test LMDB.tryget(txn, dbi, sv_key, String) == "v-subarray" - @test LMDB.tryget(txn, dbi, collect(sv_key), String) == "v-subarray" + @test get(txn, dbi, sv_key, String, nothing) == "v-subarray" + @test get(txn, dbi, collect(sv_key), String, nothing) == "v-subarray" end end end diff --git a/test/dictionary.jl b/test/dictionary.jl index 73ac090..5e76326 100644 --- a/test/dictionary.jl +++ b/test/dictionary.jl @@ -37,6 +37,9 @@ mktempdir() do dir # delete! / pop! / KeyError on missing. delete!(d, "z") @test !haskey(d, "z") + # delete! of a missing key is a no-op, matching Base.delete!(::Dict). + @test delete!(d, "z") === d + @test delete!(d, "never-existed") === d @test_throws KeyError d["z"] @test_throws KeyError pop!(d, "z") @test pop!(d, "z", :missing) === :missing @@ -51,6 +54,34 @@ mktempdir() do dir close(d) end +# `empty`/`copy` refuse to fabricate a path-less LMDBDict; users reach +# for `Dict(d)` to get an in-memory snapshot, or `copy(d.env, path)` to +# clone on disk. `filter` goes through `empty(d)` so it throws as well. +# `merge` builds a fresh `Dict{K,V}(d)` via iteration, so it works and +# returns a regular Dict — the same shape the user would get from +# `Dict(d)` themselves. +mktempdir() do dir + d = LMDBDict{String, Int}(dir) + d["a"] = 1; d["b"] = 2; d["c"] = 3 + + @test_throws ArgumentError empty(d) + @test_throws ArgumentError empty(d, Int64, Float32) + @test_throws ArgumentError copy(d) + @test_throws ArgumentError filter(p -> isodd(p.second), d) + + # Canonical in-memory snapshot: the AbstractDict constructor on Dict. + snap = Dict(d) + @test snap isa Dict{String, Int} + @test snap == Dict("a" => 1, "b" => 2, "c" => 3) + + # `merge` builds a Dict via iteration, no empty(d) involved. + merged = merge(d, Dict("b" => 20, "d" => 40)) + @test merged isa Dict{String, Int} + @test merged == Dict("a" => 1, "b" => 20, "c" => 3, "d" => 40) + + close(d) +end + # Int → Int with a numeric key range. mktempdir() do dir d = LMDBDict{Int64, Int16}(dir) diff --git a/test/dupsort.jl b/test/dupsort.jl index 96983d6..d1e4068 100644 --- a/test/dupsort.jl +++ b/test/dupsort.jl @@ -4,16 +4,16 @@ # dup-aware cursor ops navigate within and across keys correctly, and # that delete!(txn, dbi, key, val) removes one specific dup. mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn, flags = LMDB.MDB_DUPSORT) do dbi + LMDB.Environment(dir) do env + LMDB.Transaction(env) do txn + LMDB.Database(txn; flags = LMDB.MDB_DUPSORT) do dbi LMDB.put!(txn, dbi, "k1", "a") LMDB.put!(txn, dbi, "k1", "b") LMDB.put!(txn, dbi, "k1", "c") LMDB.put!(txn, dbi, "k2", "x") LMDB.put!(txn, dbi, "k2", "y") - LMDB.open(txn, dbi) do cur + LMDB.Cursor(txn, dbi) do cur # Position at first entry; walk through k1's dups. @test LMDB.seek!(cur, String) == "k1" @test LMDB.value(cur, String) == "a" @@ -37,7 +37,7 @@ mktempdir() do dir end # Count dups for k1 via cursor count(). - LMDB.open(txn, dbi) do cur + LMDB.Cursor(txn, dbi) do cur @test LMDB.seek!(cur, "k1", String) == "k1" @test count(cur) == 3 end @@ -45,7 +45,7 @@ mktempdir() do dir # Dup-aware delete: delete!(txn, dbi, key, val) removes only # that one duplicate. LMDB.delete!(txn, dbi, "k1", "b") - LMDB.open(txn, dbi) do cur + LMDB.Cursor(txn, dbi) do cur @test LMDB.seek!(cur, "k1", String) == "k1" @test LMDB.value(cur, String) == "a" @test LMDB.next_dup!(cur, String) == "c" # "b" is gone diff --git a/test/environment.jl b/test/environment.jl index 1453109..5e42a9e 100644 --- a/test/environment.jl +++ b/test/environment.jl @@ -1,80 +1,78 @@ @testset "Environment" begin -# Open environment -env = create() -@test env.handle != C_NULL -@test env[:Readers] == 126 -@test env[:KeySize] == 511 -@test env[:Flags] == 0 - -# Manipulate flags -@test !isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) -set!(env, LMDB.MDB_NOSYNC) -@test isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) -unset!(env, LMDB.MDB_NOSYNC) -@test !isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) - -# Parameters -@test (env[:Readers] = 100) == 100 -@test (env[:MapSize] = 1000^2) == 1000^2 -@test (env[:DBs] = 10) == 10 -@test env[:Readers] == 100 - -# MapSize must accept values that don't fit in Cuint (#38, PR #37, #40). -big = Csize_t(8) * 1024^3 # 8 GiB -@test (env[:MapSize] = big) == big - -# Setting :Flags via setindex! used to fall through to a warning (#24). -@test !isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) -env[:Flags] = LMDB.MDB_NOSYNC -@test isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) -unset!(env, LMDB.MDB_NOSYNC) - -# Unknown options error instead of silently warning + returning bogus values. -@test_throws ArgumentError env[:Bogus] = 1 -@test_throws ArgumentError env[:Bogus] - -# Open a DB on the env, then close it. +# Defaults and getindex on a freshly constructed env. mktempdir() do dir - ret = open(env, dir) - @test ret[1] == 0 - - # stat(env) returns the main DB's stats; before any puts, there are - # no entries and a positive page size. - s = stat(env) - @test s isa NamedTuple - @test s.psize > 0 - @test s.entries == 0 - - # Close environment - close(env) - @test !isopen(env) - - # do block - create() do env - set!(env, LMDB.MDB_NOSYNC) - open(env, dir) + env = LMDB.Environment(dir) + try @test isopen(env) + @test env[:Readers] == 126 + @test env[:KeySize] == 511 + @test env[:Flags] == 0 + + # Manipulate flags via LMDB.set!/LMDB.unset! after open. + @test !LMDB.isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) + LMDB.set!(env, LMDB.MDB_NOSYNC) + @test LMDB.isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) + LMDB.unset!(env, LMDB.MDB_NOSYNC) + @test !LMDB.isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) + + # LMDB.set!/LMDB.unset! return env for chaining. + @test LMDB.set!(env, LMDB.MDB_NOSYNC) === env + @test LMDB.unset!(env, LMDB.MDB_NOSYNC) === env + + # env[:Flags] setindex! used to fall through to a warning (#24). + @test !LMDB.isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) + env[:Flags] = LMDB.MDB_NOSYNC + @test LMDB.isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) + LMDB.unset!(env, LMDB.MDB_NOSYNC) + + # Unknown options error instead of silently warning + returning bogus values. + @test_throws ArgumentError env[:Bogus] = 1 + @test_throws ArgumentError env[:Bogus] + + # stat(env) returns the main DB's stats; before any puts, there are + # no entries and a positive page size. + s = stat(env) + @test s isa NamedTuple + @test s.psize > 0 + @test s.entries == 0 + finally + close(env) + @test !isopen(env) end end -# High-level Environment(path; ...) constructor. +# High-level LMDB.Environment(path; ...) constructor. mktempdir() do dir big = Csize_t(8) * 1024^3 - env = Environment(dir; mapsize = big, maxreaders = 42, maxdbs = 4, + env = LMDB.Environment(dir; mapsize = big, maxreaders = 42, maxdbs = 4, flags = LMDB.MDB_NOSYNC | LMDB.MDB_NOTLS) try @test isopen(env) @test env[:Readers] == 42 - @test info(env).mapsize == big - @test isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) - @test isflagset(env[:Flags], Cuint(LMDB.MDB_NOTLS)) + @test LMDB.info(env).mapsize == big + @test LMDB.isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) + @test LMDB.isflagset(env[:Flags], Cuint(LMDB.MDB_NOTLS)) finally close(env) end - # On failure during open, the Environment ctor closes the partial env. - @test_throws LMDBError Environment(joinpath(dir, "definitely_does_not_exist")) + # On failure during open, the LMDB.Environment ctor closes the partial env. + @test_throws LMDBError LMDB.Environment(joinpath(dir, "definitely_does_not_exist")) +end + +# do-block form: env is closed on the way out, even if the body throws. +mktempdir() do dir + closed_env = LMDB.Environment(dir) do env + @test isopen(env) + env + end + @test !isopen(closed_env) + + @test_throws ErrorException LMDB.Environment(dir) do env + @test isopen(env) + error("boom") + end end # Finalizing an abandoned write txn must abort it; otherwise the next @@ -82,32 +80,32 @@ end # directly because on Julia 1.10+ `GC.gc()` may defer finalizers to a # separate task, so it isn't a reliable trigger from a test. mktempdir() do dir - env = Environment(dir) + env = LMDB.Environment(dir) try - txn = start(env) + txn = LMDB.Transaction(env) @test isopen(txn) finalize(txn) - txn2 = start(env) # deadlocks here if the finalizer didn't abort txn + txn2 = LMDB.Transaction(env) # deadlocks here if the finalizer didn't abort txn try - dbi = open(txn2) + dbi = LMDB.Database(txn2) LMDB.put!(txn2, dbi, "k", "v") finally - commit(txn2) + LMDB.commit(txn2) end finally close(env) end end -# Cursor finalizer: an abandoned cursor must be cleaned up so its +# LMDB.Cursor finalizer: an abandoned cursor must be cleaned up so its # parent txn can commit. (LMDB requires cursors on a write txn to be # closed before commit; for read txns it's safer too.) mktempdir() do dir - env = Environment(dir) + env = LMDB.Environment(dir) try - start(env) do txn - dbi = open(txn) - cur = LMDB.open(txn, dbi) + LMDB.Transaction(env) do txn + dbi = LMDB.Database(txn) + cur = LMDB.Cursor(txn, dbi) @test isopen(cur) finalize(cur) # If the finalizer ran, we can still use the txn. @@ -118,19 +116,19 @@ mktempdir() do dir end end -# Cursor finalizer is safe even after its parent txn has been +# LMDB.Cursor finalizer is safe even after its parent txn has been # explicitly committed: write-txn cursors are freed by the txn's # commit per `lmdb.h`, so `mdb_cursor_close` afterwards would be UB. -# The defensive check in `close(::Cursor)` skips the LMDB call once +# The defensive check in `close(::LMDB.Cursor)` skips the LMDB call once # the parent txn handle is gone. mktempdir() do dir - env = Environment(dir) + env = LMDB.Environment(dir) try - txn = start(env) - dbi = open(txn) - cur = LMDB.open(txn, dbi) + txn = LMDB.Transaction(env) + dbi = LMDB.Database(txn) + cur = LMDB.Cursor(txn, dbi) LMDB.put!(txn, dbi, "k", "v") - commit(txn) # invalidates write-txn cursors + LMDB.commit(txn) # invalidates write-txn cursors @test !isopen(txn) finalize(cur) # finalizer should be a safe no-op finally @@ -141,19 +139,19 @@ end # close(env) must abort any still-open child txns before calling # `mdb_env_close`; otherwise LMDB corrupts shared state and a later # env-open in the same process crashes inside `mdb_txn_renew0`. The -# subsequent Environment is the canary. +# subsequent LMDB.Environment is the canary. mktempdir() do dir - env = Environment(dir) - txn = start(env) + env = LMDB.Environment(dir) + txn = LMDB.Transaction(env) @test isopen(txn) close(env) # would corrupt LMDB state without txn tracking @test !isopen(txn) finalize(txn) # finalizer should also be a safe no-op end mktempdir() do dir - env = Environment(dir; mapsize = Csize_t(8) * 1024^3, maxreaders = 42, maxdbs = 4) - start(env) do txn - open(txn) do dbi + env = LMDB.Environment(dir; mapsize = Csize_t(8) * 1024^3, maxreaders = 42, maxdbs = 4) + LMDB.Transaction(env) do txn + LMDB.Database(txn) do dbi LMDB.put!(txn, dbi, "k", "v") end end @@ -162,12 +160,12 @@ end # Parent refs: env(txn) and transaction(cur) return the actual parents. mktempdir() do dir - env = Environment(dir) + env = LMDB.Environment(dir) try - start(env) do txn + LMDB.Transaction(env) do txn @test LMDB.env(txn) === env - dbi = open(txn) - LMDB.open(txn, dbi) do cur + dbi = LMDB.Database(txn) + LMDB.Cursor(txn, dbi) do cur @test LMDB.transaction(cur) === txn end end @@ -176,39 +174,39 @@ mktempdir() do dir end end -# reader_check / reader_list / copy +# LMDB.reader_check / LMDB.reader_list / copy mktempdir() do dir - environment(dir) do env + LMDB.Environment(dir) do env # Fresh env: no stale readers. - @test reader_check(env) == 0 + @test LMDB.reader_check(env) == 0 - # reader_list always emits a header line listing slot fields. - txt = reader_list(env) + # LMDB.reader_list always emits a header line listing slot fields. + txt = LMDB.reader_list(env) @test txt isa String @test !isempty(txt) # Round-trip a copy. - start(env) do txn - open(txn) do dbi + LMDB.Transaction(env) do txn + LMDB.Database(txn) do dbi LMDB.put!(txn, dbi, "k", "v") end end mktempdir() do dst copy(env, dst) - environment(dst) do env2 - start(env2) do txn - open(txn) do dbi - @test LMDB.tryget(txn, dbi, "k", String) == "v" + LMDB.Environment(dst) do env2 + LMDB.Transaction(env2) do txn + LMDB.Database(txn) do dbi + @test get(txn, dbi, "k", String, nothing) == "v" end end end end mktempdir() do dst copy(env, dst; compact=true) - environment(dst) do env2 - start(env2) do txn - open(txn) do dbi - @test LMDB.tryget(txn, dbi, "k", String) == "v" + LMDB.Environment(dst) do env2 + LMDB.Transaction(env2) do txn + LMDB.Database(txn) do dbi + @test get(txn, dbi, "k", String, nothing) == "v" end end end diff --git a/test/integration.jl b/test/integration.jl index 092bc9c..ae09d39 100644 --- a/test/integration.jl +++ b/test/integration.jl @@ -22,25 +22,25 @@ end @testset "Integration" begin -# Power-user pattern: open an env via the Environment kwargs ctor, run +# Power-user pattern: open an env via the LMDB.Environment kwargs ctor, run # a write txn through the Julia wrappers, then a read txn through a cursor # walk using only the Julia wrappers + raw MDB_val refs (the shape # cuTile.DiskCache follows). Regression guard: ensures no future change # breaks the `walk(...) do k_ref, v_ref` zero-copy idiom. mktempdir() do dir - env = Environment(dir; + env = LMDB.Environment(dir; mapsize = 1 << 28, maxreaders = 64, flags = LMDB.MDB_NOTLS | LMDB.MDB_NORDAHEAD) try - dbi, psize = start(env) do txn - d = open(txn) + dbi, psize = LMDB.Transaction(env) do txn + d = LMDB.Database(txn) (d, LMDB.stat(txn, d).psize) end @test psize > 0 # Populate. - start(env) do txn + LMDB.Transaction(env) do txn for i in 1:5 LMDB.put!(txn, dbi, "key$(i)", "value$(i)") end @@ -50,8 +50,8 @@ mktempdir() do dir # cuTile's eviction scan: zero allocations beyond the per-entry # tuple. entries = Tuple{String, Int}[] - start(env; flags = LMDB.MDB_RDONLY) do txn - LMDB.open(txn, dbi) do cur + LMDB.Transaction(env; flags = LMDB.MDB_RDONLY) do txn + LMDB.Cursor(txn, dbi) do cur LMDB.walk(cur) do k_ref, v_ref kv = k_ref[]; vv = v_ref[] k = unsafe_string(Ptr{UInt8}(kv.mv_data), kv.mv_size) @@ -63,39 +63,39 @@ mktempdir() do dir @test first.(entries) == ["key$i" for i in 1:5] @test all(e -> e[2] == sizeof("value1"), entries) - # tryget vs is_notfound — common cuTile-shaped read path. - start(env; flags = LMDB.MDB_RDONLY) do txn - @test LMDB.tryget(txn, dbi, "key3", String) == "value3" - @test LMDB.tryget(txn, dbi, "ghost", String) === nothing + # get-with-nothing on present vs missing — common cuTile-shaped read path. + LMDB.Transaction(env; flags = LMDB.MDB_RDONLY) do txn + @test get(txn, dbi, "key3", String, nothing) == "value3" + @test get(txn, dbi, "ghost", String, nothing) === nothing end # Batch delete: present keys return true, missing return false, # no exception on either path. cuTile's `_delete_batch!` uses # the Bool to count actual evictions. deleted = 0 - start(env) do txn + LMDB.Transaction(env) do txn for k in ["key1", "ghost", "key3"] LMDB.delete!(txn, dbi, k) && (deleted += 1) end end @test deleted == 2 - start(env; flags = LMDB.MDB_RDONLY) do txn - @test LMDB.tryget(txn, dbi, "key1", String) === nothing - @test LMDB.tryget(txn, dbi, "key2", String) == "value2" - @test LMDB.tryget(txn, dbi, "key3", String) === nothing + LMDB.Transaction(env; flags = LMDB.MDB_RDONLY) do txn + @test get(txn, dbi, "key1", String, nothing) === nothing + @test get(txn, dbi, "key2", String, nothing) == "value2" + @test get(txn, dbi, "key3", String, nothing) === nothing end # MDBValueIO extension: write an 8-byte-prefixed framed value - # and read it back via `tryget(..., AtimedBlob)`, getting the + # and read it back via `get(..., AtimedBlob, nothing)`, getting the # payload tail with one alloc + skip + copy, no slicing. payload = Vector{UInt8}("cubin-bytes-here") atime = UInt64(0xdeadbeefcafebabe) - start(env) do txn + LMDB.Transaction(env) do txn LMDB.put!(txn, dbi, "framed", pack_atimed(atime, payload)) end - start(env; flags = LMDB.MDB_RDONLY) do txn - @test LMDB.tryget(txn, dbi, "framed", AtimedBlob) == payload - @test LMDB.tryget(txn, dbi, "ghost", AtimedBlob) === nothing + LMDB.Transaction(env; flags = LMDB.MDB_RDONLY) do txn + @test get(txn, dbi, "framed", AtimedBlob, nothing) == payload + @test get(txn, dbi, "ghost", AtimedBlob, nothing) === nothing end finally close(env)