diff --git a/DESCRIPTION b/DESCRIPTION index 009ba75..2a38519 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: storr.tiledb Title: A TileDB Storage Driver for Storr -Version: 0.0.38 -Date: 2026-05-18 +Version: 0.0.39 +Date: 2026-05-21 Authors@R: person("Constantinos", "Giachalis", , "xx@github.com", role = c("aut", "cre")) Maintainer: Constantinos Giachalis diff --git a/NAMESPACE b/NAMESPACE index 8a5e6df..eb1451d 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -2,8 +2,10 @@ export(CAS) export(StorrFragments) +export(StorrTimeTravel) export(TileDBDriver) export(TileDBStorr) +export(TimeTravelDriver) export(driver_tiledb) export(driver_tiledb_create) export(get_allocation_size_preference) @@ -15,6 +17,7 @@ export(storr_fragments) export(storr_move) export(storr_rename) export(storr_tiledb) +export(storr_timetravel) export(storr_vacuum) export(tiledb_config) export(tiledb_ctx) diff --git a/NEWS.md b/NEWS.md index cb29f4d..b9ac05f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,3 @@ -# storr.tiledb 0.0.38 +# storr.tiledb 0.0.39 * Initial GitHub release. diff --git a/R/StorrTimeTravel.R b/R/StorrTimeTravel.R new file mode 100644 index 0000000..959e366 --- /dev/null +++ b/R/StorrTimeTravel.R @@ -0,0 +1,497 @@ +#' @title Generate a `StorrTimeTravel` Object +#' +#' @description A [TileDBStorr] is a time-travel variant of `TileDBStorr` designed +#' to query data at specific points in time and in read-only mode with no write +#' capabilities. +#' +#' This class is not intended to be used directly and the preferred +#' usage is through [storr_timetravel()]. +#' +#' @returns A `StorrTimeTravel`, `R6` object. +#' +#' @export +#' +#' @keywords internal +#' +StorrTimeTravel <- R6::R6Class( + classname = "StorrTimeTravel", + cloneable = FALSE, + + public = list( + + #' @field default_namespace The default namespace. + #' + default_namespace = NULL, + + #' @field traits Driver traits. + #' + traits = NULL, + + #' @description Initialise `StorrTimeTravel`. + #' + #' @param driver A [TimeTravelDriver] object. + #' @param default_namespace The default namespace. + #' + initialize = function(driver, default_namespace) { + + if (!inherits(driver, "TimeTravelDriver")) { + stop("Not a valid Time-Travel 'driver'. Please use a 'TTDriver' object.", + call. = FALSE) + } + + private$check_input(default_namespace, n = 1, type = "character") + + # We need the member's object to be available + # e.g., driver$members$tbl_keys$object + if (!driver$is_open() ) { + # Ensure the members are instantiated + driver$open(instantiate = TRUE) + } + + if (driver$is_open() && !driver$members_instantiated) { + driver$close() + driver$open(instantiate = TRUE) + } + + private$DRIVER <- driver + + + self$default_namespace <- default_namespace + self$traits <- storr_traits(driver$traits) + + }, + + #' @description Get an object given a key-namespace pair. + #' + #' @param key `r sto_key()` + #' @param namespace `r sto_namespace()` + #' + #' @return The `R` object if available. + #' + get = function(key, namespace = self$default_namespace) { + private$check_input(key, n = 1, type = "character") + private$check_input(namespace, n = 1, type = "character") + hash <- self$get_hash(key, namespace) + self$get_value(hash) + }, + + #' @description Get multiple objects. + #' + #' `r sto_recycle_note` + #' + #' @param key `r sto_key(1)` + #' @param namespace `r sto_namespace(1)` + #' @param missing Value to use for missing elements. + #' + #' @return A list of `R` objects. + #' + mget = function(key, namespace = self$default_namespace, missing = NULL) { + + # NB: storr::join_key_namespace check is performed inside $query_keys0 + hash <- self$mget_hash(key, namespace) + self$mget_value(hash, missing) + }, + + #' @description Get hash value. + #' + #' + #' @param key `r sto_key()` + #' @param namespace `r sto_namespace()` + #' + #' @return The hash value. + #' + get_hash = function(key, namespace = self$default_namespace) { + + private$check_input(key, n = 1, type = "character") + private$check_input(namespace, n = 1, type = "character") + + if (self$traits$throw_missing) { + tryCatch(private$DRIVER$get_hash(key, namespace), error = function(e) { + stop(KeyError(key,namespace)) + }) + } + else { + if (self$exists(key, namespace)) { + private$DRIVER$get_hash(key, namespace) + } + else { + stop(KeyError(key, namespace)) + } + } + }, + + #' @description Get hash values. + #' + #' `r sto_recycle_note` + #' + #' @param key `r sto_key(1)` + #' @param namespace `r sto_namespace(1)` + #' + #' @return A vector of hashes. + #' + mget_hash = function(key, namespace = self$default_namespace) { + + private$DRIVER$mget_hash(key, namespace) + }, + + + #' @description Get an object given its hash. + #' + #' + #' @param hash The hash value of the object. + #' + #' @return The `R` object if available. + #' + get_value = function(hash) { + + # TODO: no need for traits + if (self$traits$throw_missing) { + value <- tryCatch( + private$DRIVER$get_object(hash), + error = function(e) + stop(HashError(hash)) + ) + } else { + if (!private$DRIVER$exists_object(hash)) { + stop(HashError(hash)) + } + value <- private$DRIVER$get_object(hash) + } + + value + }, + + #' @description Get multiple objects given their hashes. + #' + #' + #' @param hash A vector of hash values." + #' @param missing Value to use for missing elements. + #' + #' @return A list of `R` objects. + #' + mget_value = function(hash, missing = NULL) { + + value <- vector("list", length(hash)) + cached <- logical(length(hash)) + is_missing <- is.na(hash) + + cached[is_missing] <- TRUE + value[is_missing] <- list(missing) + + if (any(!cached)) { + value[!cached] <- private$DRIVER$mget_object(hash[!cached]) + } + + if (any(is_missing)) { + attr(value, "missing") <- which(is_missing) + } + value + }, + + #' @description Get key's metadata. + #' + #' + #' @param key The key name to get metadata values from. + #' @param namespace The namespace to look the key within. + #' + #' @return A named list with the key-metadata: `"expires_at"` + #' and `"notes".` + #' + get_keymeta = function(key, namespace = self$default_namespace) { + + private$check_input(key, n = 1, type = "character") + private$check_input(namespace, n = 1, type = "character") + + keyns <- paste(key, namespace, sep = ":") + + value <- private$DRIVER$get_keymeta(key, namespace) + + value + }, + + #' @description Get multiple key metadata. + #' + #' `r sto_recycle_note` + #' + #' @param key A character vector with keys to get metadata values from. + #' @param namespace A character vector of namespaces to look the keys within. + #' @param missing Fill value for missing keys. Default is `NULL`. + #' + #' @return A list with key metadata for each key-namespace + #' pair. For not found pairs will return the `missing` value. + #' + mget_keymeta = function(key, + namespace = self$default_namespace, + missing = NULL) { + + p <- storr::join_key_namespace(key, namespace) + n <- p$n + + key <- p$key + namespace <- p$namespace + keyns <- paste(key, namespace, sep = ":") + + value <- vector("list", n) + cached <- logical(n) + + # Everything is TRUE, so go to find them in DB + not_cached <- !cached + status_not_cached <- TRUE + num_cached <- 0L + + is_missing <- FALSE + + if (status_not_cached) { + + # From not_cached find also which are truly missing + cc <- private$DRIVER$mget_keymeta(key[not_cached], + namespace[not_cached], + nomatch = missing) + + value[not_cached] <- cc + keyns_not_cached <- keyns[not_cached] + + # not_cached and not found + keyns_missing <- keyns_not_cached[attr(cc, "missing")] + + # Truly missing key-namespace pairs + is_missing <- keyns %in% keyns_missing + } + + + if (any(is_missing)) { + attr(value, "missing") <- which(is_missing) #+ num_cached + } + value + }, + + #' @description Check a key-namespace pair exists. + #' + #' `r sto_recycle_note` + #' + #' @param key `r sto_key(1)` + #' @param namespace `r sto_namespace(1)` + #' + #' @return A logical vector indicating which key-namespace pair exists. + #' + exists = function(key, namespace = self$default_namespace) { + private$DRIVER$exists_hash(key, namespace) + }, + + #' @description Check a serialised object exists given a hash. + #' + #' @param hash `r roxy_hash` + #' + #' @return A logical vector indicating which object exists. + #' + exists_object = function(hash) { + private$DRIVER$exists_object(hash) + }, + + #' @description Get the key-namespace pairs with expiration timestamps. + #' + #' @param namespace `r sto_namespaces_or_null` + #' @param datetimes Should the `expires_at` column be returned? + #' Default is `TRUE`. + #' + #' @return An object of class `data.table`. + #' + keys_with_expiration = function(namespace = self$default_namespace, datetimes = TRUE) { + out <- private$DRIVER$keys_with_expiration(namespace, datetimes = datetimes) + data.table::as.data.table(out) + }, + + #' @description Get the expired key-namespace pairs. + #' + #' @param namespace `r sto_namespaces_or_null` + #' @param datetimes Should the `expires_at` column be returned? + #' Default is `TRUE`. + #' + #' @return An object of class `data.table`. + #' + expired_keys = function(namespace = self$default_namespace, datetimes = TRUE) { + out <- private$DRIVER$expired_keys(namespace, datetimes = datetimes) + data.table::as.data.table(out) + }, + + #' @description Check for expired key-namespace pairs. + #' + #' @param namespace `r sto_namespaces_or_null` + #' + #' @return `TRUE` for expired keys, `FALSE` otherwise. + #' + has_expired_keys = function(namespace = self$default_namespace) { + + private$DRIVER$has_expired_keys(namespace) + + }, + + #' @description List all keys stored in a namespace. + #' + #' @param namespace `r sto_namespace()` + #' + #' @return A sorted character vector with keys. + #' + list = function(namespace = self$default_namespace) { + + sort(private$DRIVER$list_keys(namespace)) + }, + + #' @description List all hashes stored in the storr. + #' + #' + #' @return A sorted character vector with hashes. + #' + list_hashes = function() { + + sort(private$DRIVER$list_hashes()) + }, + + #' @description List all namespaces in the storr. + #' + #' + #' @return A sorted character vector with namespaces. + #' + list_namespaces = function() { + + sort(private$DRIVER$list_namespaces()) + }, + + #' @description Export objects from storr. + #' + #' Use list() to export to a brand new list, or use as.list(object) for a shorthand. + #' + #' @param dest A destination to export objects to. It can be a storr, list, or environment. + #' @param list Names of objects to import (or `NULL` for all objects) . If given it must be a character vector. + #' If named, the names of the character vector will be the names of the objects as created in the storr. + #' @param namespace Namespace to get objects from, and to put objects into. If `NULL`, + #' then this will export namespaces from this (source) storr into the destination; + #' if there is more than one namespace, this is only possible if `dest` + #' is a storr (otherwise there will be an error). + #' @param skip_missing Logical, indicating if missing keys (specified in `list`) + #' should be skipped over, rather than being treated as an error (the default). + #' + #' + #' @return `dest` object, invisibly. + #' + export = function(dest, list = NULL, namespace = self$default_namespace, + skip_missing = FALSE) { + + if (is.null(namespace)) { + namespace <- self$list_namespaces() + } + + invisible(.base_export(dest, self, list, namespace, skip_missing)$dest) + }, + + #' @description Generate a `data.table` with an index of objects + #' present in a storr. + #' + #' + #' @param namespace `r sto_namespaces_or_null` + #' + #' @return An object of class `data.table`. + #' + index_export = function(namespace = NULL) { + + out <- private$DRIVER$filter_keys(character(), namespace = namespace)[] + + if (nrow(out) == 0) { + + d <- data.frame( + namespace = character(0), + key = character(0), + hash = character(0), + expires_at = as.POSIXct(double()), + notes = character(0) + ) + + out <- data.table::as.data.table(d) + } + + out + }, + + #' @description Export objects from storr to another TileDB storr. + #' + #' @param key A character vector of source keys. + #' @param namespace `r sto_namespaces_or_null` + #' @param uri_dest The URI path of destination storr. + #' @param context_dest Optional \link[tiledb:tiledb_ctx]{tiledb_ctx} object + #' for destination storr. + #' + #' @return A logical `TRUE` indicating successful export, invisibly. + #' + export_tdb = function(key = character(0), + namespace = self$default_namespace, + uri_dest, + context_dest = NULL) { + + dest_driver <- driver_tiledb(uri_dest, context = context_dest) + + private$DRIVER$export_tdb(key, namespace = namespace, dest_driver = dest_driver) + } + + ), + + active = list( + + #'@field timestamp Set or retrieve a `TileDB` timestamp range that + #' the resource will be opened at. Effective in `"READ"` mode only. + #' + timestamp = function(value) { + + if (!missing(value)) { + private$DRIVER$timestamp <- value + } else { + private$DRIVER$timestamp + } + + } + ), + + private = list( + + # @field driver The TileDB driver. + # + DRIVER = NULL, + + # NOTE: extracted from storr:::check_length + check_length = function(key, namespace) { + + n_key <- length(key) + n_namespace <- length(namespace) + if (n_key == n_namespace || n_namespace == 1) { + n_key + } + else if (n_key == 1) { + n_namespace + } + else { + stop("Incompatible lengths for key and namespace", call. = FALSE) + } + }, + + check_input = function(x, n, type = NULL) { + name <- deparse(substitute(x)) + + switch (type, 'character' = { + if (isFALSE(.is_character(x))) { + stop(sprintf("'%s' should be a character string, not %s", name, class(x)), + call. = FALSE) + } + }, 'datetime' = { + if (isFALSE(inherits(x, "POSIXct"))) { + stop(sprintf("'%s' should be a date-time object, not %s", name, class(x)), + call. = FALSE) + } + }) + + if (length(x) != n) { + stop(sprintf("'%s' must have %d elements (recieved %d)", name, n, length(x)), + call. = FALSE) + } + } + + ) +) diff --git a/R/TileDBDriver.R b/R/TileDBDriver.R index 2f11c46..211f838 100644 --- a/R/TileDBDriver.R +++ b/R/TileDBDriver.R @@ -1107,7 +1107,7 @@ TileDBDriver <- R6::R6Class( # Copy keys to destination storr # NB: We can not do it with arrow because of: # https://github.com/TileDB-Inc/TileDB-R/issues/847 - arr <- dest_driver$get_member("tbl_keys")$tiledb_array() + arr <- dest_driver$get_member("tbl_keys")$tiledb_array() arr[] <- idx } else { # Diff hash algos diff --git a/R/TileDBStorr.R b/R/TileDBStorr.R index 9a6b9ad..e3c72fb 100644 --- a/R/TileDBStorr.R +++ b/R/TileDBStorr.R @@ -941,7 +941,8 @@ TileDBStorr <- R6::R6Class( get = function(key, namespace = self$default_namespace, use_cache = getOption("storr.tiledb.cache", TRUE)) { private$check_input(key, n = 1, type = "character") private$check_input(namespace, n = 1, type = "character") - self$get_value(self$get_hash(key, namespace), use_cache) + hash <- self$get_hash(key, namespace) + self$get_value(hash, use_cache) }, @@ -960,7 +961,8 @@ TileDBStorr <- R6::R6Class( missing = NULL) { # NB: storr::join_key_namespace check is performed inside $query_keys0 - self$mget_value(self$mget_hash(key, namespace), use_cache, missing) + hash <- self$mget_hash(key, namespace) + self$mget_value(hash, use_cache, missing) }, #' @description Get hash value. @@ -977,8 +979,9 @@ TileDBStorr <- R6::R6Class( private$check_input(namespace, n = 1, type = "character") if (self$traits$throw_missing) { - tryCatch(private$DRIVER$get_hash(key, namespace), error = function(e) stop(KeyError(key, - namespace))) + tryCatch(private$DRIVER$get_hash(key, namespace), error = function(e) { + stop(KeyError(key, namespace)) + }) } else { if (self$exists(key, namespace)) { @@ -1050,7 +1053,7 @@ TileDBStorr <- R6::R6Class( #' @description Get multiple objects given their hashes. #' #' - #' @param hash A vector of hash values." + #' @param hash A vector of hash values. #' @param use_cache `r sto_cache` #' @param missing Value to use for missing elements. #' @@ -1929,7 +1932,7 @@ TileDBStorr <- R6::R6Class( #' Use list() to export to a brand new list, or use as.list(object) for a shorthand. #' #' @param dest A destination to export objects to. It can be a storr, list, or environment. - #' @param list Names of objects to import (or `NULL` for all objects) . If given it must be a character vector. + #' @param list Names of objects to export (or `NULL` for all objects) . If given it must be a character vector. #' If named, the names of the character vector will be the names of the objects as created in the storr. #' @param namespace Namespace to get objects from, and to put objects into. If `NULL`, #' then this will export namespaces from this (source) storr into the destination; diff --git a/R/TimeTravelDriver.R b/R/TimeTravelDriver.R new file mode 100644 index 0000000..c8dcba7 --- /dev/null +++ b/R/TimeTravelDriver.R @@ -0,0 +1,1032 @@ +#' @title Generate a `TimeTravelDriver` Object +#' +#' @description A [TileDBDriver] variant with read only class methods and +#' time-travel support. +#' +#' This class is intended for usage into [StorrTimeTravel]. +#' +#' @returns A `TimeTravelDriver`, `R6` object. +#' +#' @export +#' +#' @keywords internal +#' +TimeTravelDriver <- R6::R6Class( + inherit = TileDBGroup, + cloneable = FALSE, + classname = "TimeTravelDriver", + + public = list( + + #' @field traits Driver traits (**immutable**). + #' + traits = NULL, + + #' @description Instantiate a new `TimeTravelDriver` object. + #' + #' @param uri URI path for the `TimeBDriver` object. + #' @param ctx Optional [tiledb::tiledb_ctx()] object. + #' @param timestamp Set a `TileDB` timestamp range that + #' the resource will be opened at. Effective in `"READ"` mode only. + #' Valid options: + #' - A `NULL` value (default) + #' - An `R` object coercible to `POSIXct` with length 1 which is used for end timestamp, + #' or length 2 with start, end timestamps + #' - An object of class `tiledb_timestamp`. See [R6.tiledb::set_tiledb_timestamp()] + #' + initialize = function(uri, ctx = NULL, timestamp = NULL) { + + super$initialize(uri, ctx = ctx, tiledb_timestamp = timestamp) + + if (self$exists()) { + self$open(instantiate = TRUE) + } + + self$traits <- list(accept = "string", + throw_missing = TRUE) + + lockBinding("traits", self) + + }, + + #' @description Driver type. + #' + #' @return A character string. + #' + type = function() { + "tiledb" + }, + + #' @description Open `TimeTravelDriver` object for read or write. + #' + #' Setting`instantiate` argument to `TRUE`, all members will be instantiated + #' and cached on opening. They can be accessed via `members` active field, i.e., using + #' `$object` element. + #' + #' @param mode Mode to open : either `"READ"` or `"WRITE"`. Default is `"READ"`. + #' @param instantiate Should be all members be instantiated at opening? + #' Default is `FALSE`. + #' + #' @return The object, invisibly. + #' + open = function(mode = c("READ", "WRITE"), instantiate = FALSE) { + + super$open(mode = mode) + + type <- self$get_metadata("type") + + if (type != "storr" || is.null(type)) { + + cli::cli_abort("Not a {.arg TileDB Storr} at URI: {.url {self$uri}}", call = NULL) + + } + + algo <- self$get_metadata("hash_algo") + + # Case where 'hash_algo' key is not present + # + if (is.null(algo)) { + + stop("Hash algorithm not found, defaulting to 'md5'") + } + + private$.hash_algo <- algo + + if (instantiate) { + private$instantiate_members() + private$.members_instantiated <- TRUE + } + + invisible(self) + + }, + + #' @description Close the group object. + #' + #' All instantiated group members will be closed if opened, and before + #' closing the group object. + #' + #' @return The object, invisibly. + #' + close = function() { + + super$close() + private$.members_instantiated <- NULL + + invisible(self) + }, + + #' @description Filter `tbl_keys` by key and namespace + #' + #' @param key `r roxy_key` + #' @param namespace `r roxy_namespace` + #' @param attrnames A character vector with tiledb attributes (columns). + #' + #' @return A `data.table. + #' + filter_keys = function(key, namespace, attrnames = character()) { + + arrobj <- private$keys_array() + + sp <- list(namespace = namespace, key = key) + + arr <- arrobj$object + tiledb::attrs(arr) <- attrnames + tiledb::selected_points(arr) <- sp + tiledb::return_as(arr) <- "arrow" + + # arr <- arrobj$tiledb_array(attrs = attrnames, + # selected_points = sp, + # return_as = "arrow", + # ctx = self$ctx) + + dt <- data.table::as.data.table(arr[]) + + # TODO: Remove when TileDB fixes it + if (attrnames == "expires_at" || length(attrnames) == 0) { + expires_at <- NULL + dt[expires_at <= 0 , expires_at := as.POSIXct(NA)] + } + + dt + }, + + #' @description Get hash values. + #' + #' @param key `r roxy_key` + #' @param namespace `r roxy_namespace` + #' + #' @return A vector of hashes. + #' + get_hash = function(key, namespace) { + + result <- private$query_hash(key, namespace) + + if (is.na(result)) { + # 'get_hash' always returns NA if missing as we need + # this for mget_hash. Here, an error is raised to + # support 'throw_missing' trait. Doing so, we're + # avoiding the extra query (e.g., exists_hash) in storr + # layer. + stop(KeyError(key, namespace)) + } + + result + }, + + + #' @description Get hash values. + #' + #' @param key `r roxy_key` + #' @param namespace `r roxy_namespace` + #' + #' @return A vector of hashes. + #' + mget_hash = function(key, namespace) { + + private$query_hash(key, namespace) + + }, + + #' @description Get R object given a hash. + #' + #' @param hash A hash value. + #' + #' @return A de-serialized R object. + #' + get_object = function(hash) { + + private$check_scalar_character(hash) + result <- self$mget_object(hash)[[1]] + + if (is.null(result)) { + # NB: 'get_object' always returns NULL if missing as we need + # this for mget_object. Here, an error is raised to + # support 'throw_missing' trait. Doing so, we're + # avoiding the extra query (e.g., exists_object) in storr + # layer. + stop(HashError(hash)) + } + + result + + }, + + #' @description Get a list R objects given a hash vector. + #' + #' @param hash `r roxy_hash` + #' + #' @return A list with de-serialized R objects. + #' + mget_object = function(hash) { + + if (length(hash) == 0L){ + return(list()) + } + + arrobj <- private$data_array() + + sp <- list(hash = hash) + arr <- arrobj$object + tiledb::selected_points(arr) <- sp + tiledb::return_as(arr) <- "arrow" + # arr <- arrobj$tiledb_array(extended = TRUE, + # selected_points = sp, + # return_as = "arrow") + + x <- arrow::Array$create(hash) + nona_hash <- arrow::call_function("is_in", + x, + options = list( + value_set = arr[]$GetColumnByName("hash"), + skip_nulls = TRUE + )) + + status_nona <- arrow::call_function("all", nona_hash) + status_nona <- status_nona$as_vector() + + if (status_nona) { + result <- lapply(arr[]$value$as_vector(), { + function(.s) unserialize(charToRaw(.s)) } + ) + } else { + result <- vector("list", length(hash)) + + idx <- which(nona_hash$as_vector()) + + vals <- arr[]$value$as_vector() + for (i in seq_along(idx)) { + result[idx[i]] <- unserialize(charToRaw(vals[i])) + } + } + + # NB: The read query gives unordered results, so we + # extract them in order to match the requested order. + names(result) <- arr[]$hash$as_vector() + unname(result[hash]) + + }, + + #' @description Get key-namespace metadata. + #' + #' @param key A single character key. + #' @param namespace A single character namespace. + #' + #' @return A named list with key-metadata, `"expires_at"` + #' and `"notes".` + #' + get_keymeta = function(key, namespace) { + + arrobj <- private$keys_array() + + sp <- list(namespace = namespace, key = key) + + arr <- arrobj$object + tiledb::extended(arr) <- FALSE + tiledb::attrs(arr) <- c("expires_at", "notes") + tiledb::selected_points(arr) <- sp + tiledb::return_as(arr) <- "arrow" + + # arr <- arrobj$tiledb_array(extended = FALSE, + # attrs = c("expires_at", "notes"), + # selected_points = sp, + # return_as = "arrow") + + DT <- data.table::as.data.table(arr[]) + + # TODO: Remove when TileDB fixes it + expires_at <- NULL + DT[expires_at <= 0 , expires_at := as.POSIXct(NA)] + + + if (nrow(DT) == 0) { + stop(KeyError(key, namespace)) + } + + as.list(DT) + }, + + #' @description Get multiple key-namespace metadata. + #' + #' @param key `r roxy_key` + #' @param namespace `r roxy_namespace` + #' @param nomatch Value to fill in case of no match. + #' + #' @return A list with key metadata for each key-namespace + #' pair. For not found pairs will return the nomatch value. + #' + mget_keymeta = function(key, namespace, nomatch = NULL) { + + arrobj <- private$keys_array() + + # Slice array + sp <- list(namespace = namespace, key = key) + arr <- arrobj$object + tiledb::selected_points(arr) <- sp + tiledb::return_as(arr) <- "arrow" + + # arr <- arrobj$tiledb_array(selected_points = sp, return_as = "arrow") + + DT <- data.table::as.data.table(arr[], key = c("namespace", "key")) + + # TODO: Remove when TileDB fixes it + # Sanitise datetime columns + expires_at <- NULL + DT[expires_at <= 0 , expires_at := as.POSIXct(NA)] + + DT <- DT[.(namespace, key), env = list(namespace = I(namespace), key = I(key))] + hash_isna <- is.na(DT[["hash"]]) + + out <- vector("list", nrow(DT)) + + if (is.null(nomatch)) { + nomatch <- list(nomatch) + } + + for (i in seq_along(out)) { + + if (!hash_isna[i]) { + out[[i]] <- as.list(DT[i, c("expires_at", "notes")]) + } else { + out[[i]] <- nomatch + } + + } + + attr(out, "missing") <- which(hash_isna) + + out + + }, + + #' @description Check a key-namespace pair exists. + #' + #' @param key `r roxy_key` + #' @param namespace `r roxy_namespace` + #' + #' @return A logical vector. + #' + exists_hash = function(key, namespace) { + + p <- storr::join_key_namespace(key, namespace) + + arrobj <- private$keys_array() + + sp <- list(namespace = namespace, key = key) + + arr <- arrobj$object + tiledb::attrs(arr) <- "hash" + tiledb::selected_points(arr) <- sp + tiledb::return_as(arr) <- "arrow" + + # arr <- arrobj$tiledb_array(attrs = "hash", + # selected_points = sp, + # return_as = "arrow") + + DT <- data.table::as.data.table(arr[]) + + data.table::setkeyv(DT, c("namespace", "key")) + + key <- p$key + namespace <- p$namespace + i <- DT[.(namespace, key), "hash", with = FALSE, + env = list(namespace = I(namespace), + key = I(key))] + + !is.na(i[["hash"]]) + }, + + #' @description Check a serialised object exists. + #' + #' @param hash `r roxy_hash` + #' + #' @return A logical vector. + #' + exists_object = function(hash) { + + arrobj <- private$data_array() + + arr <- arrobj$object + tiledb::attrs(arr) <- NA_character_ + tiledb::selected_points(arr) <- list(hash = hash) + tiledb::return_as(arr) <- "arrow" + + # arr <- arrobj$tiledb_array(attrs = NA_character_, + # selected_points = list(hash = hash), + # return_as = "arrow") + + hashes <- arr[]$GetColumnByName("hash")$as_vector() + + x <- arrow::Array$create(hash) + out <- arrow::call_function("is_in", + x, + options = list( + value_set = arr[]$GetColumnByName("hash"), + skip_nulls = TRUE + )) + out$as_vector() + + }, + + #' @description List all hash values. + #' + #' + #' @return A vector of hash values. + #' + list_hashes = function() { + + arrobj <- private$data_array() + + arr <- arrobj$object + tiledb::attrs(arr) <- NA_character_ + tiledb::return_as(arr) <- "arrow" + + # arr <- arrobj$tiledb_array(attrs = NA_character_, + # return_as = "arrow") + + ns <- arr[]$GetColumnByName("hash") + ns$as_vector() + + }, + + #' @description List all namespace values. + #' + #' + #' @return A vector of namespace values. + #' + list_namespaces = function() { + + arrobj <- private$keys_array() + + arr <- arrobj$object + tiledb::attrs(arr) <- NA_character_ + tiledb::return_as(arr) <- "arrow" + + # arr <- arrobj$tiledb_array(attrs = NA_character_, + # return_as = "arrow") + + ns <- arr[]$GetColumnByName("namespace") + ns <- arrow::call_function("unique", ns) + ns$as_vector() + + }, + + #' @description List keys given a namespace. + #' + #' @param namespace A single character namespace. + #' + #' @return A vector of key values. + #' + list_keys = function(namespace) { + + private$check_scalar_character(namespace) + + arrobj <- private$keys_array() + + arr <- arrobj$object + tiledb::attrs(arr) <- NA_character_ + tiledb::selected_points(arr) <- list(namespace = namespace) + tiledb::return_as(arr) <- "arrow" + + # arr <- arrobj$tiledb_array(attrs = NA_character_, + # selected_points = list(namespace = namespace), + # return_as = "arrow") + + keys <- arr[]$GetColumnByName("key") + keys$as_vector() + + }, + + #' @description List unused hashes. + #' + #' + #' @return A vector of hash values. + #' + list_unused_hashes = function() { + + # Get unique hash values from 'tbl_keys' + arrobj <- private$keys_array() + + arr <- arrobj$object + tiledb::attrs(arr) <- "hash" + tiledb::return_as(arr) <- "arrow" + + # arr <- arrobj$tiledb_array(extended = TRUE, + # attrs = "hash", + # return_as = "arrow") + + yh <- arr[]$GetColumnByName("hash") + yh <- arrow::call_function("unique", yh) + yh <- yh$as_vector() + + # Get object hash values from 'tbl_data' + xh <- self$list_hashes() + + # Find unused hashes (equiv. setdiff(x, y)) + xh[data.table::chmatch(xh, yh, 0L) == 0L] + + }, + + #' @description Get the key-namespace pairs with expiration timestamps. + #' + #' @param namespace `r sto_namespaces_or_null` + #' @param datetimes Should the `expires_at` column be returned? + #' Default is `TRUE`. + #' + #' @return An `ArrowObject` object. + #' + keys_with_expiration = function(namespace, datetimes = TRUE) { + + check_character_or_null(namespace) + + arrobj <- private$keys_array() + arr <- arrobj$object + + # Ignore NA datetimes + qc <- tiledb::tiledb_query_condition_init(attr = "expires_at", + value = as.POSIXct(NA), + dtype = "DATETIME_MS", + op = "NE") + + sp <- list() + + if (!is.null(namespace)) { + sp <- list(namespace = namespace) + } + + if (datetimes) { + attrs <- "expires_at" + } else { + attrs <- NA_character_ + } + + tiledb::attrs(arr) <- attrs + tiledb::selected_points(arr) <- sp + tiledb::query_condition(arr) <- qc + tiledb::return_as(arr) <- "arrow" + arr[] + # arrobj$tiledb_array(attrs = attrs, + # selected_points = sp, + # query_condition = qc, + # return_as = "arrow")[] + }, + + #' @description Get the key-namespace pairs without expiration timestamps. + #' + #' @param namespace `r sto_namespaces_or_null` + #' @param datetimes Should the `expires_at` column be returned? + #' Default is `TRUE`. + #' + #' @return An `ArrowObject` object. + #' + keys_without_expiration = function(namespace, datetimes = TRUE) { + + check_character_or_null(namespace) + + arrobj <- private$keys_array() + arr <- arrobj$object + + # Ignore NA datetimes + qc <- tiledb::tiledb_query_condition_init(attr = "expires_at", + value = as.POSIXct(NA), + dtype = "DATETIME_MS", + op = "EQ") + + sp <- list() + + if (!is.null(namespace)) { + sp <- list(namespace = namespace) + } + + if (datetimes) { + attrs <- "expires_at" + } else { + attrs <- NA_character_ + } + + tiledb::attrs(arr) <- attrs + tiledb::selected_points(arr) <- sp + tiledb::query_condition(arr) <- qc + tiledb::return_as(arr) <- "arrow" + arr[] + # + # arrobj$tiledb_array(attrs = attrs, + # selected_points = sp, + # query_condition = qc, + # return_as = "arrow")[] + }, + + #' @description Get the expired key-namespace pairs. + #' + #' @param namespace `r sto_namespaces_or_null` + #' @param datetimes Should the `expires_at` column be returned? + #' Default is `TRUE`. + #' + #' @return An `ArrowObject` object. + #' + expired_keys = function(namespace, datetimes = TRUE) { + + check_character_or_null(namespace) + + arrobj <- private$keys_array() + arr <- arrobj$object + + # Ignore NA datetimes + qc_dttm1 <- tiledb::tiledb_query_condition_init(attr = "expires_at", + value = as.POSIXct(NA), + dtype = "DATETIME_MS", + op = "NE") + # Expired datetimes (now > index) + qc_dttm2 <- tiledb::tiledb_query_condition_init(attr = "expires_at", + value = Sys.time(), + dtype = "DATETIME_MS", + op = "LT") + + qc <- tiledb::tiledb_query_condition_combine(qc_dttm1, qc_dttm2, "AND") + + sp <- list() + + if (!is.null(namespace)) { + sp <- list(namespace = namespace) + } + + if (datetimes) { + attrs <- "expires_at" + } else { + attrs <- NA_character_ + } + + tiledb::attrs(arr) <- attrs + tiledb::selected_points(arr) <- sp + tiledb::query_condition(arr) <- qc + tiledb::return_as(arr) <- "arrow" + arr[] + + # arrobj$tiledb_array(attrs = attrs, + # selected_points = sp, + # query_condition = qc, + # return_as = "arrow")[] + }, + + #' @description Get the unexpired key-namespace pairs. + #' + #' @param namespace `r sto_namespaces_or_null` + #' @param datetimes Should the `expires_at` column be returned? + #' Default is `TRUE`. + #' + #' @return An `ArrowObject` object. + #' + unexpired_keys = function(namespace, datetimes = TRUE) { + + check_character_or_null(namespace) + + arrobj <- private$keys_array() + arr <- arrobj$object + + # Ignore NA datetimes + qc_dttm1 <- tiledb::tiledb_query_condition_init(attr = "expires_at", + value = as.POSIXct(NA), + dtype = "DATETIME_MS", + op = "NE") + # Expired datetimes (now > index) + qc_dttm2 <- tiledb::tiledb_query_condition_init(attr = "expires_at", + value = Sys.time(), + dtype = "DATETIME_MS", + op = "GT") + + qc <- tiledb::tiledb_query_condition_combine(qc_dttm1, qc_dttm2, "AND") + + sp <- list() + + if (!is.null(namespace)) { + sp <- list(namespace = namespace) + } + + if (datetimes) { + attrs <- "expires_at" + } else { + attrs <- NA_character_ + } + + tiledb::attrs(arr) <- attrs + tiledb::selected_points(arr) <- sp + tiledb::query_condition(arr) <- qc + tiledb::return_as(arr) <- "arrow" + arr[] + + # arrobj$tiledb_array(attrs = attrs, + # selected_points = sp, + # query_condition = qc, + # return_as = "arrow")[] + }, + + #' @description Get the number of expired key-namespace pairs. + #' + #' @param namespace `r sto_namespaces_or_null` + #' + #' @return A numeric value. + #' + num_expired_keys = function(namespace) { + + arr <- self$expired_keys(namespace, datetimes = FALSE) + arr[]$num_rows + }, + + #' @description Get the number of unexpired key-namespace pairs. + #' + #' @param namespace `r sto_namespaces_or_null` + #' + #' @return A numeric value. + #' + num_unexpired_keys = function(namespace) { + + arr <- self$unexpired_keys(namespace, datetimes = FALSE) + arr[]$num_rows + }, + + #' @description Check for expired key-namespace pairs. + #' + #' @param namespace `r sto_namespaces_or_null` + #' + #' @return `TRUE` for expired keys, `FALSE` otherwise. + #' + has_expired_keys = function(namespace) { + + arr <- self$expired_keys(namespace, datetimes = FALSE) + arr[]$num_rows != 0 + }, + + #' @description Check for unexpired key-namespace pairs. + #' + #' @param namespace `r sto_namespaces_or_null` + #' + #' @return `TRUE` for unexpired keys, `FALSE` otherwise. + #' + has_unexpired_keys = function(namespace) { + + arr <- self$unexpired_keys(namespace, datetimes = FALSE) + arr[]$num_rows != 0 + }, + + #' @description Export objects from storr to another TileDB storr. + #' + #' @param key A character vector of source keys. + #' @param namespace `r sto_namespaces_or_null` + #' @param dest_driver The destination TileDB driver, See [driver_tiledb()]. + #' + #' @return A logical `TRUE` indicating successful export, invisibly. + #' + export_tdb = function(key, + namespace, + dest_driver) { + + + if (self$uri == dest_driver$uri) { + cli::cli_abort("Destination URI can not be the same as source.", + call = NULL) + } + + # Two cases: + # 1. Storrs with identical hash algorithms + # 2. Storrs with different hash algorithms + + # Get index data.frame (tbl_keys) + idx <- self$filter_keys(key, namespace = namespace)[] + + if (nrow(idx) == 0) { + cli::cli_warn("Nothing to export for the selected key-namespace.") + return(invisible(NULL)) + } + + hashes <- unique(idx$hash) + + if (self$hash_algorithm == dest_driver$hash_algorithm) { + + # Check hashes exists in destination + exist_hash_in_dest <- dest_driver$exists_object(hashes) + + # Hashes to be copied over to destination + new_hash <- hashes[!exist_hash_in_dest] + + # Export new data only + if (length(new_hash) != 0) { + + # Get data from source + arrobj <- private$data_array() + sp <- list(hash = new_hash) + # arr_data_src <- arrobj$tiledb_array(selected_points = sp, return_as = "arrow") + arr_data_src <- arrobj$object + tiledb::selected_points(arr_data_src) <- sp + tiledb::return_as(arr_data_src) <- "arrow" + + # Copy data to destination storr + arr_data_dest <- dest_driver$get_member("tbl_data")$tiledb_array() + arr_data_dest[] <- arr_data_src[] + + # TODO: Use arrow the arrow buffer is fixed + # qry <- tiledb::tiledb_query(arr_data_dest, "WRITE") + # + # naHash <- nanoarrow::as_nanoarrow_array(arr_data_src[][["hash"]]) + # naValue <- nanoarrow::as_nanoarrow_array(arr_data_src[][["value"]]) + # + # qry <- tiledb::tiledb_query_import_buffer(qry, "hash", naHash, ctx = dest_driver$ctx) + # qry <- tiledb::tiledb_query_import_buffer(qry, "value", naValue, ctx = dest_driver$ctx) + # + # tiledb::tiledb_query_set_layout(qry, "UNORDERED") + # tiledb::tiledb_query_submit(qry) + # tiledb::tiledb_query_finalize(qry) + + } + + # Copy keys to destination storr + # NB: We can not do it with arrow because of: + # https://github.com/TileDB-Inc/TileDB-R/issues/847 + arr <- dest_driver$get_member("tbl_keys")$tiledb_array() + arr[] <- idx + + } else { # Diff hash algos + + # Fetch objects, re-hash, check exist in dest and then + # save them in dest + + # Inline helpers + .traits <- storr_traits(dest_driver$traits) + .hash_raw <- make_hash_serialized_object(dest_driver$hash_algorithm, !.traits$drop_r_version) + + # Get data from source + arrobj <- private$data_array() + sp <- list(hash = hashes) + # arr_data_src <- arrobj$tiledb_array(selected_points = sp, return_as = "arrow") + arr_data_src <- arrobj$object + tiledb::selected_points(arr_data_src) <- sp + tiledb::return_as(arr_data_src) <- "arrow" + + dta <- data.table::as.data.table(arr_data_src[], key = "hash") + + # Get them in order we requested + dta <- dta[.(hash), env = list(hash = I(hashes))] + + # Re-hash serialised objects + dta[, `:=` (new_hashes = vcapply(value, .hash_raw))] + + # out <<- dta + # print(dta[, "new_hashes"]) + + # Find hashes to send over + upload <- !dest_driver$exists_object(dta$new_hashes) + + # Copy data to destination storr + if (any(upload)) { + arr_data_dest <- dest_driver$get_member("tbl_data")$tiledb_array() + rehashed_objects <- data.frame(hash = dta$new_hashes[upload], value = dta$value[upload]) + arr_data_dest[] <- rehashed_objects + } + + # Replace index with new hashes and copy + new_hashes <- dta$new_hashes + names(new_hashes) <- dta$hash + idx$hash <- new_hashes[idx$hash] # map to new hashes + + # Copy keys to destination storr + # NB: We can not do it with arrow because of: + # https://github.com/TileDB-Inc/TileDB-R/issues/847 + arr <- dest_driver$get_member("tbl_keys")$tiledb_array() + arr[] <- idx + } + + invisible(TRUE) + } + + ), + + active = list( + + #'@field timestamp Set or retrieve a `TileDB` timestamp range that + #' the resource will be opened at. Effective in `"READ"` mode only. + #' + timestamp = function(value) { + + if (!missing(value)) { + super$tiledb_timestamp <- value + + private$instantiate_members() + + } else { + super$tiledb_timestamp + } + + }, + + #' @field hash_algorithm Hash algorithm + #' + hash_algorithm = function(value) { + + private$check_object_exists() + + if (!missing(value)) { + private$check_read_only("hash_algorithm") + } + + private$.hash_algo + }, + + #' @field members_instantiated Have the members been instantiated? + #' + members_instantiated = function(value) { + + private$check_object_exists() + + if (!missing(value)) { + private$check_read_only("members_instantiated") + } + + if (is.null(private$.members_instantiated)) { + private$.members_instantiated <- FALSE + } + + private$.members_instantiated + + } + ), + + private = list( + + # @field Query for instantiated members + # + .members_instantiated = NULL, + + # @field Hash algorithm to be used + # + .hash_algo = NULL, + + # @description Instantiate group members. + # + instantiate_members = function() { + + members <- private$.member_cache + + dev_null <- lapply(members, function(.m) { + + obj <- if (is.null(.m$object)) { + private$log_debug0("instantiate_members", "Constructing member '{}' type {}", .m$name, .m$type) + obj <- private$construct_member(.m$uri, .m$type) + } else { + .m$object + } + + if (!obj$is_open()) { + obj$open(self$mode) + + } else { + obj$reopen(self$mode) + } + + private$log_debug0("instantiate_members", "Adding cached member '{}' type {}", .m$name, .m$type) + + # Explicitly add the new member to member_cache + private$add_cache_member(.m$name, obj) + }) + + invisible(NULL) + }, + + # @description Get cached 'tbl_keys' array object + # + keys_array = function() { + # TODO: should we check for cached first? + self$members$tbl_keys$object + }, + + # @description Get cached 'tbl_data' array object + # + data_array = function() { + self$members$tbl_data$object + }, + + # @description Query 'tbl_keys' array by single attribute + # + query_hash = function(key, namespace) { + + p <- storr::join_key_namespace(key, namespace) + arrobj <- private$keys_array() + + # Slice array + sp <- list(namespace = namespace, key = key) + arr <- arrobj$object + tiledb::attrs(arr) <- "hash" + tiledb::selected_points(arr) <- sp + tiledb::return_as(arr) <- "arrow" + + # arr <- arrobj$tiledb_array(attrs = "hash", + # selected_points = sp, + # return_as = "arrow", + # ctx = self$ctx) + + + dta <- data.table::as.data.table(arr[], key = c("namespace", "key")) + dta[.(namespace, key), "hash", with = FALSE, + env = list(namespace = I(namespace), key = I(key))][[1]] + + } + ) # private +) diff --git a/R/storr_timetravel.R b/R/storr_timetravel.R new file mode 100644 index 0000000..d21cba4 --- /dev/null +++ b/R/storr_timetravel.R @@ -0,0 +1,122 @@ +#' A Storr with Time-Travel +#' +#' Generate an instance of [StorrTimeTravel], a variant of [TileDBStorr] designed +#' to query key-value data at specific points in time and in read-only mode with +#' no write capabilities. +#' +#' # Class Methods Summary +#' +#' For complete definitions, see **Methods** section in [StorrTimeTravel]. +#' +#' **Active Fields** +#' +#' - **`timestamp`** - Get or set a TileDB timestamp range that the 'storr' will be opened at. +#' +#' **Initialisation & Lifecycle** +#' +#' - **`new()`** - Initialise a StorrTimeTravel object with a TileDB driver, default namespace, and optional timestamp +#' +#' +#' **Single Key-Value Operations** +#' +#' - **`get()`** - Retrieve an object by key-namespace pair +#' - **`get_value()`** - Retrieve an object given its hash +#' +#' **Multiple Key-Value Operations** +#' +#' - **`mget()`** - Get multiple objects by keys +#' - **`mget_value()`** - Get multiple objects by their hashes +#' +#' **Metadata Operations** +#' +#' - **`get_keymeta()`** - Retrieve metadata for a key +#' - **`mget_keymeta()`** - Retrieve metadata for multiple keys +#' +#' **Object Hash Management** +#' +#' - **`get_hash()`** - Get hash value for a key-namespace pair +#' - **`mget_hash()`** - Get hash values for multiple keys +#' +#' **Key Management** +#' +#' - **`exists()`** - Check if key-namespace pair(s) exist +#' - **`exists_object()`** - Check if object(s) with given hash(es) exist +#' +#' **Expiration Management** +#' +#' - **`keys_with_expiration()`** - List keys that have expiration timestamps +#' - **`expired_keys()`** - Get keys that have already expired +#' - **`has_expired_keys()`** - Check if any keys are expired +#' +#' **Listing** +#' +#' - **`list()`** - List all keys in a namespace +#' - **`list_hashes()`** - List all stored object hashes +#' - **`list_namespaces()`** - List all namespaces +#' +#' **Storage Management** +#' +#' - **`index_export()`** - Export object index as data.table +#' - **`export()`** - Export objects to another storr/list/environment +#' - **`export_tdb()`** - Export objects to another TileDB storr +#' +#' @inheritParams driver_tiledb +#' @param default_namespace The default namespace: `"objects"`. +#' @param timestamp Set a `TileDB` timestamp range that +#' the resource will be opened at. Effective only for `"READ"` mode. +#' Valid options: +#' - A `NULL` value (default) +#' - An `R` object coercible to `POSIXct` with length 1 which used for end timestamp, +#' or length 2 with start, end timestamps +#' - An object of class `tiledb_timestamp`. See [R6.tiledb::set_tiledb_timestamp()] +#' +#' Set a new timestamp with active field `$timestamp`, see examples. +#' +#' @returns An object of class [StorrTimeTravel], R6. +#' +#' @export +#' +#' @examples +#' \dontrun{ +#' # URI path +#' uri <- tempfile() +#' sto <- storr_tiledb(uri, init = TRUE) +#' +#' # set key-values +#' t0 <- Sys.time() +#' sto$set("a", 1) +#' +#' t1 <- Sys.time() +#' sto$set("b", 2) +#' +#' t2 <- Sys.time() +#' +#' # Open storr with time-travel support at t0 +#' stor <- storr_timetravel(uri, timestamp = t0) +#' +#' # Read at t0 +#' stor$list_hashes() # character(0), no hashes at t0 +#' +#' stor$get("a") # key 'a' ('objects') not found +#' +#' # Read at t1 +#' stor$timestamp <- t1 +#' stor$get("a") # 1 +#' stor$get("b") # key 'b' ('objects') not found +#' +#' # Read at t2 +#' stor$timestamp <- t2 +#' sto$get("b") # 2 +#' +#'} +#' +#' +storr_timetravel <- function(uri, + default_namespace = "objects", + context = NULL, + timestamp = NULL) { + # check scalar namespace + dr <- TimeTravelDriver$new(uri, ctx = context, timestamp = timestamp) + StorrTimeTravel$new(dr, default_namespace = default_namespace) + +} diff --git a/_pkgdown.yml b/_pkgdown.yml index 9311f13..59e9b6c 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -9,7 +9,7 @@ template: google: Roboto Slab primary: '#090D14' link-color: '#1C4BC4ED' - code-color: '#725c30' + code-color: '#5D4311' text-decoration-color: '#151B21' body-color: '#1B222B' home: @@ -64,6 +64,7 @@ reference: Key functions to work with a 'storr' or creating a TileDB driver. contents: - storr_tiledb + - storr_timetravel - driver_tiledb - subtitle: Storr Management desc: | @@ -92,5 +93,6 @@ reference: - CAS - TileDBDriver - TileDBStorr + - TimeTravelDriver - StorrFragments - + - StorrTimeTravel diff --git a/man/StorrTimeTravel.Rd b/man/StorrTimeTravel.Rd new file mode 100644 index 0000000..f5bda4b --- /dev/null +++ b/man/StorrTimeTravel.Rd @@ -0,0 +1,533 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/StorrTimeTravel.R +\name{StorrTimeTravel} +\alias{StorrTimeTravel} +\title{Generate a \code{StorrTimeTravel} Object} +\value{ +A \code{StorrTimeTravel}, \code{R6} object. +} +\description{ +A \link{TileDBStorr} is a time-travel variant of \code{TileDBStorr} designed +to query data at specific points in time and in read-only mode with no write +capabilities. + +This class is not intended to be used directly and the preferred +usage is through \code{\link[=storr_timetravel]{storr_timetravel()}}. +} +\keyword{internal} +\section{Public fields}{ + \if{html}{\out{
}} + \describe{ + \item{\code{default_namespace}}{The default namespace.} + + \item{\code{traits}}{Driver traits.} + } + \if{html}{\out{
}} +} +\section{Active bindings}{ + \if{html}{\out{
}} + \describe{ + \item{\code{timestamp}}{Set or retrieve a \code{TileDB} timestamp range that +the resource will be opened at. Effective in \code{"READ"} mode only.} + } + \if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ + \itemize{ + \item \href{#method-StorrTimeTravel-initialize}{\code{StorrTimeTravel$new()}} + \item \href{#method-StorrTimeTravel-get}{\code{StorrTimeTravel$get()}} + \item \href{#method-StorrTimeTravel-mget}{\code{StorrTimeTravel$mget()}} + \item \href{#method-StorrTimeTravel-get_hash}{\code{StorrTimeTravel$get_hash()}} + \item \href{#method-StorrTimeTravel-mget_hash}{\code{StorrTimeTravel$mget_hash()}} + \item \href{#method-StorrTimeTravel-get_value}{\code{StorrTimeTravel$get_value()}} + \item \href{#method-StorrTimeTravel-mget_value}{\code{StorrTimeTravel$mget_value()}} + \item \href{#method-StorrTimeTravel-get_keymeta}{\code{StorrTimeTravel$get_keymeta()}} + \item \href{#method-StorrTimeTravel-mget_keymeta}{\code{StorrTimeTravel$mget_keymeta()}} + \item \href{#method-StorrTimeTravel-exists}{\code{StorrTimeTravel$exists()}} + \item \href{#method-StorrTimeTravel-exists_object}{\code{StorrTimeTravel$exists_object()}} + \item \href{#method-StorrTimeTravel-keys_with_expiration}{\code{StorrTimeTravel$keys_with_expiration()}} + \item \href{#method-StorrTimeTravel-expired_keys}{\code{StorrTimeTravel$expired_keys()}} + \item \href{#method-StorrTimeTravel-has_expired_keys}{\code{StorrTimeTravel$has_expired_keys()}} + \item \href{#method-StorrTimeTravel-list}{\code{StorrTimeTravel$list()}} + \item \href{#method-StorrTimeTravel-list_hashes}{\code{StorrTimeTravel$list_hashes()}} + \item \href{#method-StorrTimeTravel-list_namespaces}{\code{StorrTimeTravel$list_namespaces()}} + \item \href{#method-StorrTimeTravel-export}{\code{StorrTimeTravel$export()}} + \item \href{#method-StorrTimeTravel-index_export}{\code{StorrTimeTravel$index_export()}} + \item \href{#method-StorrTimeTravel-export_tdb}{\code{StorrTimeTravel$export_tdb()}} + } +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-initialize}{}}} +\subsection{\code{StorrTimeTravel$new()}}{ + Initialise \code{StorrTimeTravel}. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$new(driver, default_namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{driver}}{A \link{TimeTravelDriver} object.} + \item{\code{default_namespace}}{The default namespace.} + } + \if{html}{\out{
}} + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-get}{}}} +\subsection{\code{StorrTimeTravel$get()}}{ + Get an object given a key-namespace pair. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$get(key, namespace = self$default_namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{A scalar character of key name.} + \item{\code{namespace}}{A scalar character of namespace name.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + The \code{R} object if available. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-mget}{}}} +\subsection{\code{StorrTimeTravel$mget()}}{ + Get multiple objects. + +The arguments \code{key} and \code{namespace} can be recycled if any of them is a +scalar character and the other is a vector. No other recycling rule is permitted. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$mget(key, namespace = self$default_namespace, + missing = NULL)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{A character vector of key names.} + \item{\code{namespace}}{A character vector of namespaces.} + \item{\code{missing}}{Value to use for missing elements.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A list of \code{R} objects. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-get_hash}{}}} +\subsection{\code{StorrTimeTravel$get_hash()}}{ + Get hash value. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$get_hash(key, namespace = self$default_namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{A scalar character of key name.} + \item{\code{namespace}}{A scalar character of namespace name.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + The hash value. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-mget_hash}{}}} +\subsection{\code{StorrTimeTravel$mget_hash()}}{ + Get hash values. + +The arguments \code{key} and \code{namespace} can be recycled if any of them is a +scalar character and the other is a vector. No other recycling rule is permitted. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$mget_hash(key, namespace = self$default_namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{A character vector of key names.} + \item{\code{namespace}}{A character vector of namespaces.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A vector of hashes. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-get_value}{}}} +\subsection{\code{StorrTimeTravel$get_value()}}{ + Get an object given its hash. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$get_value(hash)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{hash}}{The hash value of the object.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + The \code{R} object if available. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-mget_value}{}}} +\subsection{\code{StorrTimeTravel$mget_value()}}{ + Get multiple objects given their hashes. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$mget_value(hash, missing = NULL)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{hash}}{A vector of hash values."} + \item{\code{missing}}{Value to use for missing elements.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A list of \code{R} objects. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-get_keymeta}{}}} +\subsection{\code{StorrTimeTravel$get_keymeta()}}{ + Get key's metadata. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$get_keymeta(key, namespace = self$default_namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{The key name to get metadata values from.} + \item{\code{namespace}}{The namespace to look the key within.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A named list with the key-metadata: \code{"expires_at"} +and \verb{"notes".} + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-mget_keymeta}{}}} +\subsection{\code{StorrTimeTravel$mget_keymeta()}}{ + Get multiple key metadata. + +The arguments \code{key} and \code{namespace} can be recycled if any of them is a +scalar character and the other is a vector. No other recycling rule is permitted. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$mget_keymeta(key, namespace = self$default_namespace, + missing = NULL)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{A character vector with keys to get metadata values from.} + \item{\code{namespace}}{A character vector of namespaces to look the keys within.} + \item{\code{missing}}{Fill value for missing keys. Default is \code{NULL}.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A list with key metadata for each key-namespace +pair. For not found pairs will return the \code{missing} value. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-exists}{}}} +\subsection{\code{StorrTimeTravel$exists()}}{ + Check a key-namespace pair exists. + +The arguments \code{key} and \code{namespace} can be recycled if any of them is a +scalar character and the other is a vector. No other recycling rule is permitted. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$exists(key, namespace = self$default_namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{A character vector of key names.} + \item{\code{namespace}}{A character vector of namespaces.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A logical vector indicating which key-namespace pair exists. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-exists_object}{}}} +\subsection{\code{StorrTimeTravel$exists_object()}}{ + Check a serialised object exists given a hash. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$exists_object(hash)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{hash}}{A vector of hash values.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A logical vector indicating which object exists. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-keys_with_expiration}{}}} +\subsection{\code{StorrTimeTravel$keys_with_expiration()}}{ + Get the key-namespace pairs with expiration timestamps. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$keys_with_expiration(namespace = self$default_namespace, + datetimes = TRUE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{namespace}}{A character vector of namespaces or \code{NULL} for all namespaces.} + \item{\code{datetimes}}{Should the \code{expires_at} column be returned? +Default is \code{TRUE}.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + An object of class \code{data.table}. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-expired_keys}{}}} +\subsection{\code{StorrTimeTravel$expired_keys()}}{ + Get the expired key-namespace pairs. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$expired_keys(namespace = self$default_namespace, + datetimes = TRUE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{namespace}}{A character vector of namespaces or \code{NULL} for all namespaces.} + \item{\code{datetimes}}{Should the \code{expires_at} column be returned? +Default is \code{TRUE}.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + An object of class \code{data.table}. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-has_expired_keys}{}}} +\subsection{\code{StorrTimeTravel$has_expired_keys()}}{ + Check for expired key-namespace pairs. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$has_expired_keys(namespace = self$default_namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{namespace}}{A character vector of namespaces or \code{NULL} for all namespaces.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + \code{TRUE} for expired keys, \code{FALSE} otherwise. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-list}{}}} +\subsection{\code{StorrTimeTravel$list()}}{ + List all keys stored in a namespace. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$list(namespace = self$default_namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{namespace}}{A scalar character of namespace name.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A sorted character vector with keys. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-list_hashes}{}}} +\subsection{\code{StorrTimeTravel$list_hashes()}}{ + List all hashes stored in the storr. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$list_hashes()} + \if{html}{\out{
}} + } + \subsection{Returns}{ + A sorted character vector with hashes. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-list_namespaces}{}}} +\subsection{\code{StorrTimeTravel$list_namespaces()}}{ + List all namespaces in the storr. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$list_namespaces()} + \if{html}{\out{
}} + } + \subsection{Returns}{ + A sorted character vector with namespaces. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-export}{}}} +\subsection{\code{StorrTimeTravel$export()}}{ + Export objects from storr. + +Use list() to export to a brand new list, or use as.list(object) for a shorthand. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$export(dest, list = NULL, + namespace = self$default_namespace, skip_missing = FALSE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{dest}}{A destination to export objects to. It can be a storr, list, or environment.} + \item{\code{list}}{Names of objects to import (or \code{NULL} for all objects) . If given it must be a character vector. +If named, the names of the character vector will be the names of the objects as created in the storr.} + \item{\code{namespace}}{Namespace to get objects from, and to put objects into. If \code{NULL}, +then this will export namespaces from this (source) storr into the destination; +if there is more than one namespace, this is only possible if \code{dest} +is a storr (otherwise there will be an error).} + \item{\code{skip_missing}}{Logical, indicating if missing keys (specified in \code{list}) +should be skipped over, rather than being treated as an error (the default).} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + \code{dest} object, invisibly. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-index_export}{}}} +\subsection{\code{StorrTimeTravel$index_export()}}{ + Generate a \code{data.table} with an index of objects +present in a storr. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$index_export(namespace = NULL)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{namespace}}{A character vector of namespaces or \code{NULL} for all namespaces.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + An object of class \code{data.table}. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-StorrTimeTravel-export_tdb}{}}} +\subsection{\code{StorrTimeTravel$export_tdb()}}{ + Export objects from storr to another TileDB storr. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{StorrTimeTravel$export_tdb(key = character(0), + namespace = self$default_namespace, uri_dest, context_dest = NULL)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{A character vector of source keys.} + \item{\code{namespace}}{A character vector of namespaces or \code{NULL} for all namespaces.} + \item{\code{uri_dest}}{The URI path of destination storr.} + \item{\code{context_dest}}{Optional \link[tiledb:tiledb_ctx]{tiledb_ctx} object +for destination storr.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A logical \code{TRUE} indicating successful export, invisibly. + } +} + +} diff --git a/man/TileDBStorr.Rd b/man/TileDBStorr.Rd index 80cb637..ce297a8 100644 --- a/man/TileDBStorr.Rd +++ b/man/TileDBStorr.Rd @@ -638,7 +638,7 @@ requested digest (hash) of the supplied R object. \subsection{Arguments}{ \if{html}{\out{
}} \describe{ - \item{\code{hash}}{A vector of hash values."} + \item{\code{hash}}{A vector of hash values.} \item{\code{use_cache}}{Should the cache be used? Default is \code{TRUE}.} \item{\code{missing}}{Value to use for missing elements.} } @@ -1317,7 +1317,7 @@ Use list() to export to a brand new list, or use as.list(object) for a shorthand \if{html}{\out{
}} \describe{ \item{\code{dest}}{A destination to export objects to. It can be a storr, list, or environment.} - \item{\code{list}}{Names of objects to import (or \code{NULL} for all objects) . If given it must be a character vector. + \item{\code{list}}{Names of objects to export (or \code{NULL} for all objects) . If given it must be a character vector. If named, the names of the character vector will be the names of the objects as created in the storr.} \item{\code{namespace}}{Namespace to get objects from, and to put objects into. If \code{NULL}, then this will export namespaces from this (source) storr into the destination; diff --git a/man/TimeTravelDriver.Rd b/man/TimeTravelDriver.Rd new file mode 100644 index 0000000..e7f69b1 --- /dev/null +++ b/man/TimeTravelDriver.Rd @@ -0,0 +1,663 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/TimeTravelDriver.R +\name{TimeTravelDriver} +\alias{TimeTravelDriver} +\title{Generate a \code{TimeTravelDriver} Object} +\value{ +A \code{TimeTravelDriver}, \code{R6} object. +} +\description{ +A \link{TileDBDriver} variant with read only class methods and +time-travel support. + +This class is intended for usage into \link{StorrTimeTravel}. +} +\keyword{internal} +\section{Super classes}{ +\code{\link[R6.tiledb:TileDBObject]{R6.tiledb::TileDBObject}} -> \code{\link[R6.tiledb:TileDBGroup]{R6.tiledb::TileDBGroup}} -> \code{TimeTravelDriver} +} +\section{Public fields}{ + \if{html}{\out{
}} + \describe{ + \item{\code{traits}}{Driver traits (\strong{immutable}).} + } + \if{html}{\out{
}} +} +\section{Active bindings}{ + \if{html}{\out{
}} + \describe{ + \item{\code{timestamp}}{Set or retrieve a \code{TileDB} timestamp range that +the resource will be opened at. Effective in \code{"READ"} mode only.} + + \item{\code{hash_algorithm}}{Hash algorithm} + + \item{\code{members_instantiated}}{Have the members been instantiated?} + } + \if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ + \itemize{ + \item \href{#method-TimeTravelDriver-initialize}{\code{TimeTravelDriver$new()}} + \item \href{#method-TimeTravelDriver-type}{\code{TimeTravelDriver$type()}} + \item \href{#method-TimeTravelDriver-open}{\code{TimeTravelDriver$open()}} + \item \href{#method-TimeTravelDriver-close}{\code{TimeTravelDriver$close()}} + \item \href{#method-TimeTravelDriver-filter_keys}{\code{TimeTravelDriver$filter_keys()}} + \item \href{#method-TimeTravelDriver-get_hash}{\code{TimeTravelDriver$get_hash()}} + \item \href{#method-TimeTravelDriver-mget_hash}{\code{TimeTravelDriver$mget_hash()}} + \item \href{#method-TimeTravelDriver-get_object}{\code{TimeTravelDriver$get_object()}} + \item \href{#method-TimeTravelDriver-mget_object}{\code{TimeTravelDriver$mget_object()}} + \item \href{#method-TimeTravelDriver-get_keymeta}{\code{TimeTravelDriver$get_keymeta()}} + \item \href{#method-TimeTravelDriver-mget_keymeta}{\code{TimeTravelDriver$mget_keymeta()}} + \item \href{#method-TimeTravelDriver-exists_hash}{\code{TimeTravelDriver$exists_hash()}} + \item \href{#method-TimeTravelDriver-exists_object}{\code{TimeTravelDriver$exists_object()}} + \item \href{#method-TimeTravelDriver-list_hashes}{\code{TimeTravelDriver$list_hashes()}} + \item \href{#method-TimeTravelDriver-list_namespaces}{\code{TimeTravelDriver$list_namespaces()}} + \item \href{#method-TimeTravelDriver-list_keys}{\code{TimeTravelDriver$list_keys()}} + \item \href{#method-TimeTravelDriver-list_unused_hashes}{\code{TimeTravelDriver$list_unused_hashes()}} + \item \href{#method-TimeTravelDriver-keys_with_expiration}{\code{TimeTravelDriver$keys_with_expiration()}} + \item \href{#method-TimeTravelDriver-keys_without_expiration}{\code{TimeTravelDriver$keys_without_expiration()}} + \item \href{#method-TimeTravelDriver-expired_keys}{\code{TimeTravelDriver$expired_keys()}} + \item \href{#method-TimeTravelDriver-unexpired_keys}{\code{TimeTravelDriver$unexpired_keys()}} + \item \href{#method-TimeTravelDriver-num_expired_keys}{\code{TimeTravelDriver$num_expired_keys()}} + \item \href{#method-TimeTravelDriver-num_unexpired_keys}{\code{TimeTravelDriver$num_unexpired_keys()}} + \item \href{#method-TimeTravelDriver-has_expired_keys}{\code{TimeTravelDriver$has_expired_keys()}} + \item \href{#method-TimeTravelDriver-has_unexpired_keys}{\code{TimeTravelDriver$has_unexpired_keys()}} + \item \href{#method-TimeTravelDriver-export_tdb}{\code{TimeTravelDriver$export_tdb()}} + } +} +\if{html}{\out{
Inherited methods + +
}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-initialize}{}}} +\subsection{\code{TimeTravelDriver$new()}}{ + Instantiate a new \code{TimeTravelDriver} object. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$new(uri, ctx = NULL, timestamp = NULL)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{uri}}{URI path for the \code{TimeBDriver} object.} + \item{\code{ctx}}{Optional \code{\link[tiledb:tiledb_ctx]{tiledb::tiledb_ctx()}} object.} + \item{\code{timestamp}}{Set a \code{TileDB} timestamp range that +the resource will be opened at. Effective in \code{"READ"} mode only. +Valid options: +\itemize{ +\item A \code{NULL} value (default) +\item An \code{R} object coercible to \code{POSIXct} with length 1 which is used for end timestamp, +or length 2 with start, end timestamps +\item An object of class \code{tiledb_timestamp}. See \code{\link[R6.tiledb:set_tiledb_timestamp]{R6.tiledb::set_tiledb_timestamp()}} +}} + } + \if{html}{\out{
}} + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-type}{}}} +\subsection{\code{TimeTravelDriver$type()}}{ + Driver type. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$type()} + \if{html}{\out{
}} + } + \subsection{Returns}{ + A character string. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-open}{}}} +\subsection{\code{TimeTravelDriver$open()}}{ + Open \code{TimeTravelDriver} object for read or write. + +Setting\code{instantiate} argument to \code{TRUE}, all members will be instantiated +and cached on opening. They can be accessed via \code{members} active field, i.e., using +\verb{$object} element. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$open(mode = c("READ", "WRITE"), instantiate = FALSE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{mode}}{Mode to open : either \code{"READ"} or \code{"WRITE"}. Default is \code{"READ"}.} + \item{\code{instantiate}}{Should be all members be instantiated at opening? +Default is \code{FALSE}.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + The object, invisibly. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-close}{}}} +\subsection{\code{TimeTravelDriver$close()}}{ + Close the group object. + +All instantiated group members will be closed if opened, and before +closing the group object. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$close()} + \if{html}{\out{
}} + } + \subsection{Returns}{ + The object, invisibly. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-filter_keys}{}}} +\subsection{\code{TimeTravelDriver$filter_keys()}}{ + Filter \code{tbl_keys} by key and namespace + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$filter_keys(key, namespace, attrnames = character())} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{A character vector of keys.} + \item{\code{namespace}}{A character vector of namespaces.} + \item{\code{attrnames}}{A character vector with tiledb attributes (columns).} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A `data.table. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-get_hash}{}}} +\subsection{\code{TimeTravelDriver$get_hash()}}{ + Get hash values. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$get_hash(key, namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{A character vector of keys.} + \item{\code{namespace}}{A character vector of namespaces.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A vector of hashes. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-mget_hash}{}}} +\subsection{\code{TimeTravelDriver$mget_hash()}}{ + Get hash values. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$mget_hash(key, namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{A character vector of keys.} + \item{\code{namespace}}{A character vector of namespaces.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A vector of hashes. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-get_object}{}}} +\subsection{\code{TimeTravelDriver$get_object()}}{ + Get R object given a hash. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$get_object(hash)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{hash}}{A hash value.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A de-serialized R object. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-mget_object}{}}} +\subsection{\code{TimeTravelDriver$mget_object()}}{ + Get a list R objects given a hash vector. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$mget_object(hash)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{hash}}{A vector of hash values.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A list with de-serialized R objects. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-get_keymeta}{}}} +\subsection{\code{TimeTravelDriver$get_keymeta()}}{ + Get key-namespace metadata. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$get_keymeta(key, namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{A single character key.} + \item{\code{namespace}}{A single character namespace.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A named list with key-metadata, \code{"expires_at"} +and \verb{"notes".} + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-mget_keymeta}{}}} +\subsection{\code{TimeTravelDriver$mget_keymeta()}}{ + Get multiple key-namespace metadata. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$mget_keymeta(key, namespace, nomatch = NULL)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{A character vector of keys.} + \item{\code{namespace}}{A character vector of namespaces.} + \item{\code{nomatch}}{Value to fill in case of no match.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A list with key metadata for each key-namespace +pair. For not found pairs will return the nomatch value. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-exists_hash}{}}} +\subsection{\code{TimeTravelDriver$exists_hash()}}{ + Check a key-namespace pair exists. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$exists_hash(key, namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{A character vector of keys.} + \item{\code{namespace}}{A character vector of namespaces.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A logical vector. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-exists_object}{}}} +\subsection{\code{TimeTravelDriver$exists_object()}}{ + Check a serialised object exists. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$exists_object(hash)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{hash}}{A vector of hash values.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A logical vector. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-list_hashes}{}}} +\subsection{\code{TimeTravelDriver$list_hashes()}}{ + List all hash values. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$list_hashes()} + \if{html}{\out{
}} + } + \subsection{Returns}{ + A vector of hash values. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-list_namespaces}{}}} +\subsection{\code{TimeTravelDriver$list_namespaces()}}{ + List all namespace values. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$list_namespaces()} + \if{html}{\out{
}} + } + \subsection{Returns}{ + A vector of namespace values. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-list_keys}{}}} +\subsection{\code{TimeTravelDriver$list_keys()}}{ + List keys given a namespace. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$list_keys(namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{namespace}}{A single character namespace.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A vector of key values. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-list_unused_hashes}{}}} +\subsection{\code{TimeTravelDriver$list_unused_hashes()}}{ + List unused hashes. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$list_unused_hashes()} + \if{html}{\out{
}} + } + \subsection{Returns}{ + A vector of hash values. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-keys_with_expiration}{}}} +\subsection{\code{TimeTravelDriver$keys_with_expiration()}}{ + Get the key-namespace pairs with expiration timestamps. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$keys_with_expiration(namespace, datetimes = TRUE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{namespace}}{A character vector of namespaces or \code{NULL} for all namespaces.} + \item{\code{datetimes}}{Should the \code{expires_at} column be returned? +Default is \code{TRUE}.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + An \code{ArrowObject} object. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-keys_without_expiration}{}}} +\subsection{\code{TimeTravelDriver$keys_without_expiration()}}{ + Get the key-namespace pairs without expiration timestamps. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$keys_without_expiration(namespace, datetimes = TRUE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{namespace}}{A character vector of namespaces or \code{NULL} for all namespaces.} + \item{\code{datetimes}}{Should the \code{expires_at} column be returned? +Default is \code{TRUE}.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + An \code{ArrowObject} object. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-expired_keys}{}}} +\subsection{\code{TimeTravelDriver$expired_keys()}}{ + Get the expired key-namespace pairs. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$expired_keys(namespace, datetimes = TRUE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{namespace}}{A character vector of namespaces or \code{NULL} for all namespaces.} + \item{\code{datetimes}}{Should the \code{expires_at} column be returned? +Default is \code{TRUE}.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + An \code{ArrowObject} object. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-unexpired_keys}{}}} +\subsection{\code{TimeTravelDriver$unexpired_keys()}}{ + Get the unexpired key-namespace pairs. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$unexpired_keys(namespace, datetimes = TRUE)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{namespace}}{A character vector of namespaces or \code{NULL} for all namespaces.} + \item{\code{datetimes}}{Should the \code{expires_at} column be returned? +Default is \code{TRUE}.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + An \code{ArrowObject} object. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-num_expired_keys}{}}} +\subsection{\code{TimeTravelDriver$num_expired_keys()}}{ + Get the number of expired key-namespace pairs. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$num_expired_keys(namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{namespace}}{A character vector of namespaces or \code{NULL} for all namespaces.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A numeric value. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-num_unexpired_keys}{}}} +\subsection{\code{TimeTravelDriver$num_unexpired_keys()}}{ + Get the number of unexpired key-namespace pairs. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$num_unexpired_keys(namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{namespace}}{A character vector of namespaces or \code{NULL} for all namespaces.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A numeric value. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-has_expired_keys}{}}} +\subsection{\code{TimeTravelDriver$has_expired_keys()}}{ + Check for expired key-namespace pairs. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$has_expired_keys(namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{namespace}}{A character vector of namespaces or \code{NULL} for all namespaces.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + \code{TRUE} for expired keys, \code{FALSE} otherwise. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-has_unexpired_keys}{}}} +\subsection{\code{TimeTravelDriver$has_unexpired_keys()}}{ + Check for unexpired key-namespace pairs. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$has_unexpired_keys(namespace)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{namespace}}{A character vector of namespaces or \code{NULL} for all namespaces.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + \code{TRUE} for unexpired keys, \code{FALSE} otherwise. + } +} + +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-TimeTravelDriver-export_tdb}{}}} +\subsection{\code{TimeTravelDriver$export_tdb()}}{ + Export objects from storr to another TileDB storr. + \subsection{Usage}{ + \if{html}{\out{
}} + \preformatted{TimeTravelDriver$export_tdb(key, namespace, dest_driver)} + \if{html}{\out{
}} + } + \subsection{Arguments}{ + \if{html}{\out{
}} + \describe{ + \item{\code{key}}{A character vector of source keys.} + \item{\code{namespace}}{A character vector of namespaces or \code{NULL} for all namespaces.} + \item{\code{dest_driver}}{The destination TileDB driver, See \code{\link[=driver_tiledb]{driver_tiledb()}}.} + } + \if{html}{\out{
}} + } + \subsection{Returns}{ + A logical \code{TRUE} indicating successful export, invisibly. + } +} + +} diff --git a/man/storr_timetravel.Rd b/man/storr_timetravel.Rd new file mode 100644 index 0000000..601c7e8 --- /dev/null +++ b/man/storr_timetravel.Rd @@ -0,0 +1,137 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/storr_timetravel.R +\name{storr_timetravel} +\alias{storr_timetravel} +\title{A Storr with Time-Travel} +\usage{ +storr_timetravel(uri, default_namespace = "objects", context = NULL, + timestamp = NULL) +} +\arguments{ +\item{uri}{The URI path of storr.} + +\item{default_namespace}{The default namespace: \code{"objects"}.} + +\item{context}{Optional \link[tiledb:tiledb_ctx]{tiledb_ctx} object.} + +\item{timestamp}{Set a \code{TileDB} timestamp range that +the resource will be opened at. Effective only for \code{"READ"} mode. +Valid options: +\itemize{ +\item A \code{NULL} value (default) +\item An \code{R} object coercible to \code{POSIXct} with length 1 which used for end timestamp, +or length 2 with start, end timestamps +\item An object of class \code{tiledb_timestamp}. See \code{\link[R6.tiledb:set_tiledb_timestamp]{R6.tiledb::set_tiledb_timestamp()}} +} + +Set a new timestamp with active field \verb{$timestamp}, see examples.} +} +\value{ +An object of class \link{StorrTimeTravel}, R6. +} +\description{ +Generate an instance of \link{StorrTimeTravel}, a variant of \link{TileDBStorr} designed +to query key-value data at specific points in time and in read-only mode with +no write capabilities. +} +\section{Class Methods Summary}{ +For complete definitions, see \strong{Methods} section in \link{StorrTimeTravel}. + +\strong{Active Fields} +\itemize{ +\item \strong{\code{timestamp}} - Get or set a TileDB timestamp range that the 'storr' will be opened at. +} + +\strong{Initialisation & Lifecycle} +\itemize{ +\item \strong{\code{new()}} - Initialise a StorrTimeTravel object with a TileDB driver, default namespace, and optional timestamp +} + +\strong{Single Key-Value Operations} +\itemize{ +\item \strong{\code{get()}} - Retrieve an object by key-namespace pair +\item \strong{\code{get_value()}} - Retrieve an object given its hash +} + +\strong{Multiple Key-Value Operations} +\itemize{ +\item \strong{\code{mget()}} - Get multiple objects by keys +\item \strong{\code{mget_value()}} - Get multiple objects by their hashes +} + +\strong{Metadata Operations} +\itemize{ +\item \strong{\code{get_keymeta()}} - Retrieve metadata for a key +\item \strong{\code{mget_keymeta()}} - Retrieve metadata for multiple keys +} + +\strong{Object Hash Management} +\itemize{ +\item \strong{\code{get_hash()}} - Get hash value for a key-namespace pair +\item \strong{\code{mget_hash()}} - Get hash values for multiple keys +} + +\strong{Key Management} +\itemize{ +\item \strong{\code{exists()}} - Check if key-namespace pair(s) exist +\item \strong{\code{exists_object()}} - Check if object(s) with given hash(es) exist +} + +\strong{Expiration Management} +\itemize{ +\item \strong{\code{keys_with_expiration()}} - List keys that have expiration timestamps +\item \strong{\code{expired_keys()}} - Get keys that have already expired +\item \strong{\code{has_expired_keys()}} - Check if any keys are expired +} + +\strong{Listing} +\itemize{ +\item \strong{\code{list()}} - List all keys in a namespace +\item \strong{\code{list_hashes()}} - List all stored object hashes +\item \strong{\code{list_namespaces()}} - List all namespaces +} + +\strong{Storage Management} +\itemize{ +\item \strong{\code{index_export()}} - Export object index as data.table +\item \strong{\code{export()}} - Export objects to another storr/list/environment +\item \strong{\code{export_tdb()}} - Export objects to another TileDB storr +} +} + +\examples{ +\dontrun{ +# URI path +uri <- tempfile() +sto <- storr_tiledb(uri, init = TRUE) + +# set key-values +t0 <- Sys.time() +sto$set("a", 1) + +t1 <- Sys.time() +sto$set("b", 2) + +t2 <- Sys.time() + +# Open storr with time-travel support at t0 +stor <- storr_timetravel(uri, timestamp = t0) + +# Read at t0 +stor$list_hashes() # character(0), no hashes at t0 + +stor$get("a") # key 'a' ('objects') not found + +# Read at t1 +stor$timestamp <- t1 +stor$get("a") # 1 +stor$get("b") # key 'b' ('objects') not found + +# Read at t2 +stor$timestamp <- t2 +sto$get("b") # 2 + +} + + +} diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R new file mode 100644 index 0000000..0df721c --- /dev/null +++ b/tests/testthat/setup.R @@ -0,0 +1,3 @@ +Sys.setenv(TESTTHAT_PARALLEL = "true") +Sys.setenv(TESTTHAT_CPUS=4) + diff --git a/tests/testthat/test-StorrTimeTravel.R b/tests/testthat/test-StorrTimeTravel.R new file mode 100644 index 0000000..4b55f19 --- /dev/null +++ b/tests/testthat/test-StorrTimeTravel.R @@ -0,0 +1,264 @@ +test_that("'storr_timetravel()' and 'StorrTimeTravel'", { + + uri <- file.path(withr::local_tempdir(), "test-driver") + dr <- TimeTravelDriver$new(uri) + + # R6Class: object does not exist. + expect_error(StorrTimeTravel$new(dr, "ns1")) + + # Set up 'storr' + driver_tiledb_create(uri) + dr <- TimeTravelDriver$new(uri) + + # 'StorrTimeTravel' + expect_no_error(sto <- StorrTimeTravel$new(dr, "ns1")) + expect_r6_class(sto, "StorrTimeTravel") + + # 'storr_timetravel' wrapper + expect_no_error(sto <- storr_timetravel(uri)) + expect_r6_class(sto, "StorrTimeTravel") + + # Test active field 'timestamp' + expect_s3_class(sto$timestamp, "tiledb_timestamp") + + t1 <- Sys.time() + expect_no_error(sto$timestamp <- t1) + expect_equal(sto$timestamp$timestamp_end, t1, ignore_attr = TRUE) + + # Error: 'storr' didn't exist at 1970-01-01 + expect_error(sto$timestamp <- 0) + +}) + +test_that("'get'/'mget' with time-travel", { + + uri <- file.path(withr::local_tempdir(), "test-storr") + sto <- storr_tiledb(uri, init = TRUE, default_namespace = "ns1") + + t0 <- Sys.time() + sto$set("a", 1) + t1 <- Sys.time() + sto$set("a", 2) + sto$set("b", 3, namespace = "ns2") + t2 <- Sys.time() + + hashes <- sto$list_hashes() + + # Open at t0 --- + stott <- storr_timetravel(uri, timestamp = t0, default_namespace = "ns1") + + # Expect nothing at t0 + expect_error(stott$get("a"), class = "error", "key 'a' ('ns1') not found", fixed = TRUE) + expect_equal(stott$mget(c("a", "b")), structure(list(NULL, NULL), missing = 1:2)) + + expect_all_false(stott$exists(c("a", "b"))) + expect_equal(stott$list(), character()) + expect_equal(stott$list_hashes(), character()) + expect_equal(stott$list_namespaces(), character()) + + # Open at t1 + stott$timestamp <- t1 + expect_equal(stott$get("a"), 1) + expect_equal(stott$mget(c("a", "b")), structure(list(1, NULL), missing = 2L)) + + expect_equal(stott$exists(c("a", "b")), c(TRUE, FALSE)) + expect_equal(stott$list(), "a") + expect_equal(stott$list_hashes(), "38e42db36c4414f7bbc19d750f71a721") + expect_equal(stott$list_namespaces(), "ns1") + + # Open at t2 + stott$timestamp <- t2 + expect_equal(stott$get("a"), 2) + expect_equal(stott$mget(c("a", "b"), namespace = c("ns1", "ns2")), list(2, 3)) + + expect_all_true(stott$exists(c("a", "b"), namespace = c("ns1", "ns2"))) + expect_equal(stott$list("ns2"), "b") + expect_equal(stott$list_hashes(), hashes) + expect_equal(stott$list_namespaces(), c("ns1", "ns2")) + +}) + +test_that("'get_keymeta'/'mget_keymeta' and friends with time-travel", { + + uri <- file.path(withr::local_tempdir(), "test-storr") + sto <- storr_tiledb(uri, init = TRUE, default_namespace = "ns1") + + t0 <- Sys.time() + sto$set("a", 1, notes = "a1") + t1 <- Sys.time() + sto$set_keymeta("a", notes = "a2", expires_at = as.POSIXct(t1)) + sto$set("b", 3, namespace = "ns2", notes = "b3") + t2 <- Sys.time() + + hashes <- sto$list_hashes() + + # Open at t0 --- + stott <- storr_timetravel(uri, timestamp = t0, default_namespace = "ns1") + + # Expect nothing at t0 + expect_error(stott$get_keymeta("a"), class = "error", "key 'a' ('ns1') not found", fixed = TRUE) + expect_equal(stott$mget_keymeta(c("a", "b")), structure(list(list(NULL), list(NULL)), missing = 1:2)) + + expect_all_false(stott$exists(c("a", "b"), namespace = c("ns1", "ns2"))) + + df_trg <- structure(list(namespace = character(0), key = character(0), + expires_at = structure(numeric(0), class = c("POSIXct", "POSIXt" + ))), row.names = integer(0), class = c("data.table", "data.frame" + )) + expect_equal(stott$keys_with_expiration(), df_trg) + expect_equal(stott$expired_keys(), df_trg) + expect_false(stott$has_expired_keys()) + + # Open at t1 + stott$timestamp <- t1 + expect_equal(stott$get_keymeta("a"), list(expires_at = as.POSIXct(NA), notes = "a1")) + expect_equal(stott$mget_keymeta(c("a", "b")), structure(list( + list( + expires_at = structure( + NA_real_, + class = c("POSIXct", "POSIXt"), + tzone = "" + ), + notes = "a1" + ), list(NULL) + ), missing = 2L)) + + expect_equal(stott$exists(c("a", "b"), namespace = c("ns1", "ns2")), c(TRUE, FALSE)) + + expect_equal(stott$keys_with_expiration(), df_trg) + expect_equal(stott$expired_keys(), df_trg) + expect_false(stott$has_expired_keys()) + + + # Open at t2 + stott$timestamp <- t2 + expect_equal(stott$get_keymeta("a"), list(expires_at = as.POSIXct(t1), notes = "a2")) + expect_equal(stott$get_keymeta("b", namespace = "ns2"), list(expires_at = as.POSIXct(NA), notes = "b3")) + expect_equal(stott$mget_keymeta(c("a", "b"), namespace = c("ns1", "ns2")), list(list(expires_at = as.POSIXct(t1), notes = "a2"), + list(expires_at = as.POSIXct(NA), notes = "b3")), ignore_attr = TRUE) + + expect_all_true(stott$exists(c("a", "b"), namespace = c("ns1", "ns2"))) + + df <- data.frame(namespace = "ns1", key = "a", expires_at = t1) + df_trg <- data.table::as.data.table(df) + + expect_equal(stott$keys_with_expiration(), df_trg) + expect_equal(stott$expired_keys(), df_trg) + expect_true(stott$has_expired_keys()) +}) + +test_that("'index_export' with time-travel", { + + uri <- file.path(withr::local_tempdir(), "test-storr") + sto <- storr_tiledb(uri, init = TRUE, default_namespace = "ns1") + + t0 <- Sys.time() + sto$set("a", 1) + t1 <- Sys.time() + sto$set("a", 2) + sto$set("b", 3, namespace = "ns2") + t2 <- Sys.time() + + hashes <- sto$list_hashes() + + # Open at t0 --- + stott <- storr_timetravel(uri, timestamp = t0, default_namespace = "ns1") + + df_trg <- structure(list(namespace = character(0), + key = character(0), + hash = character(0), + expires_at = structure(numeric(0), class = c("POSIXct", "POSIXt" + )), + notes = character(0)), row.names = integer(0), class = c("data.table", "data.frame" + )) + # Expect nothing at t0 + expect_equal(stott$index_export(), df_trg, ignore_attr = TRUE) + + # Open at t1 + stott$timestamp <- t1 + expect_equal(nrow(res1 <- stott$index_export()), 1) + expect_equal(res1$hash, "38e42db36c4414f7bbc19d750f71a721") + + # Open at t2 + stott$timestamp <- t2 + expect_equal(nrow(res1 <- stott$index_export()), 2) + expect_equal(res1$hash, c("87494137ffd66807c0c5c877856799cc", "02c87a685a6264c39c65c94a51de14b8" + )) + expect_equal(res1$key, c("a", "b")) + +}) + + +test_that("'export' with time-travel", { + + uri <- file.path(withr::local_tempdir(), "test-storr") + sto <- storr_tiledb(uri, init = TRUE, default_namespace = "ns1") + + t0 <- Sys.time() + sto$set("a", 1) + t1 <- Sys.time() + sto$set("a", 2) + sto$set("b", 3, namespace = "ns2") + t2 <- Sys.time() + + # Open at t0 --- + stott <- storr_timetravel(uri, timestamp = t0, default_namespace = "ns1") + + # Expect nothing at t0 + expect_no_error(dest_t0 <- stott$export(list())) + expect_error(dest_t0$get("a"), class = "error", "key 'a' ('ns1') not found", fixed = TRUE) + expect_equal(dest_t0$list(namespace = "ns1"), character(0)) + + # Open at t1 + stott$timestamp <- t1 + expect_no_error(dest_t1 <- stott$export(list())) + expect_equal(dest_t1$get("a"), 1) + expect_equal(dest_t1$list(namespace = "ns1"), "a") + + # Open at t2 + stott$timestamp <- t2 + expect_no_error(dest_t2 <- stott$export(list())) + expect_equal(dest_t2$mget(c("a", "b"), namespace = c("ns1", "ns2")), list(2, 3)) + expect_equal(dest_t2$list_namespaces(), c("ns1", "ns2")) + +}) + + +test_that("'export_tdb' with time-travel", { + + uri <- file.path(withr::local_tempdir(), "test-storr") + sto <- storr_tiledb(uri, init = TRUE, default_namespace = "ns1") + + uri2 <- file.path(withr::local_tempdir(), "test-storr2") + sto2 <- storr_tiledb(uri2, init = TRUE, default_namespace = "ns1") + + t0 <- Sys.time() + sto$set("a", 1) + t1 <- Sys.time() + sto$set("a", 2) + sto$set("b", 3, namespace = "ns2") + t2 <- Sys.time() + + # Open at t0 --- + stott <- storr_timetravel(uri, timestamp = t0, default_namespace = "ns1") + + # Expect nothing at t0 + expect_warning(stott$export_tdb(uri_dest = uri2), class = "warning", + "Nothing to export for the selected key-namespace.") + + + # Open at t1 + stott$timestamp <- t1 + expect_no_error(stott$export_tdb(uri_dest = uri2)) + expect_equal(sto2$get("a"), 1) + expect_equal(sto2$list(namespace = "ns1"), "a") + + # Open at t2 + stott$timestamp <- t2 + expect_no_error(stott$export_tdb(uri_dest = uri2, namespace = NULL)) # all namespaces + expect_equal(sto2$get("a"), 2) + expect_equal(sto2$mget(c("a", "b"), namespace = c("ns1", "ns2")), list(2, 3)) + expect_equal(sto2$list_namespaces(), c("ns1", "ns2")) + +}) + diff --git a/tests/testthat/test-TimeTravelDriver.R b/tests/testthat/test-TimeTravelDriver.R new file mode 100644 index 0000000..478e465 --- /dev/null +++ b/tests/testthat/test-TimeTravelDriver.R @@ -0,0 +1,102 @@ +test_that("'TimeTravelBDriver'", { + + uri <- file.path(withr::local_tempdir(), "test-driver") + expect_no_error(dr <- TimeTravelDriver$new(uri)) + + expect_r6_class(dr, "TimeTravelDriver") + expect_false(dr$exists()) + expect_equal(dr$mode, "CLOSED") + expect_equal(dr$type(), "tiledb") + + # public fields + expect_equal(dr$traits, list(accept = "string", + throw_missing = TRUE)) + # public fields are locked + expect_error(dr$traits <- "boo") + + # Nothing to retrieve - object does not exist + expect_error(dr$tiledb_timestamp) + expect_error(dr$members_instantiated) + +}) + +# NB: 'TimeTravelBDriver' is a subset (copy) of 'TileDBDriver'. Here, we're +# performing basic testing in order catch / isolate any issue early. +# Time-travel testing will be carried out with 'StorrTimeTravel' class. + +test_that("'get_hash'/'mget_hash'", { + uri <- file.path(withr::local_tempdir(), "test-storr") + sto <- storr_tiledb(uri, init = TRUE, default_namespace = "ns1") + sto$mset(c("a", "b"), c("a", "b")) + hashes <- sto$mget_hash(c("a", "b")) + + dr <- TimeTravelDriver$new(uri) + + expect_equal(dr$mget_hash(c("a", "b"), "ns1"), hashes) + expect_equal(dr$get_hash("a", "ns1"), hashes[1]) + + # exists_hash + expect_true(dr$exists_hash("a", "ns1")) + expect_equal(dr$exists_hash(c("a", "c"), "ns1"), c(TRUE, FALSE)) + + # listing methods + expect_equal(dr$list_hashes(), hashes) + expect_equal(dr$list_keys("ns1"), c("a", "b")) + expect_equal(dr$list_namespaces(), "ns1") + expect_equal(dr$list_unused_hashes(), character(0)) + +}) + +test_that("'get_object'/'mget_object'", { + uri <- file.path(withr::local_tempdir(), "test-storr") + sto <- storr_tiledb(uri, init = TRUE, default_namespace = "ns1") + sto$mset(c("a", "b"), c("a", "b")) + hashes <- sto$mget_hash(c("a", "b")) + + dr <- TimeTravelDriver$new(uri) + + expect_equal(dr$mget_object(hashes), list("a", "b")) + expect_equal(dr$mget_object(c(hashes, "no-hash")), list("a", "b", NULL)) + expect_equal(dr$get_object(hashes[1]), "a") + + # exists_object + expect_all_true(dr$exists_object(hashes)) + expect_equal(dr$exists_object(c(hashes, "no-hash")), c(TRUE, TRUE, FALSE)) + +}) + + +test_that("'get_keymeta'/'mget_keymeta' and friends", { + + uri <- file.path(withr::local_tempdir(), "test-storr") + sto <- storr_tiledb(uri, init = TRUE, default_namespace = "ns1") + sto$mset(c("a", "b"), c("a", "b"), + notes = c("notes-a", "notes-b"), + expires_at = c(as.POSIXct(NA), as.POSIXct("1990-01-01"))) + + dr <- TimeTravelDriver$new(uri) + + trg <- list(list(as.POSIXct(NA), "notes-a"), + list(as.POSIXct("1990-01-01"), "notes-b")) + expect_equal(dr$mget_keymeta(c("a", "b"), "ns1"), trg, ignore_attr = TRUE) + + trg <- list(list(as.POSIXct(NA), "notes-a"), + list(NULL)) + expect_equal(dr$mget_keymeta(c("a", "c"), "ns1"), trg, ignore_attr = TRUE) + + + expect_equal(dr$get_keymeta("a", "ns1"), trg[[1]], ignore_attr = TRUE) + + expect_equal(dr$keys_with_expiration("ns1")[][["key"]]$as_vector(), "b") + expect_equal(dr$keys_without_expiration("ns1")[][["key"]]$as_vector(), "a") + + expect_equal(dr$expired_keys("ns1")[][["key"]]$as_vector(), "b") + expect_equal(dr$unexpired_keys("ns1")[][["key"]]$as_vector(), character(0)) + + expect_equal(dr$num_expired_keys("ns1"), 1) + expect_equal(dr$num_unexpired_keys("ns1"), 0) + + expect_true(dr$has_expired_keys("ns1")) + expect_false(dr$has_unexpired_keys("ns1")) + +}) diff --git a/tests/testthat/test-storr_tiledb-async.R b/tests/testthat/test-storr_tiledb-async.R index c122361..1c4d9a0 100644 --- a/tests/testthat/test-storr_tiledb-async.R +++ b/tests/testthat/test-storr_tiledb-async.R @@ -152,16 +152,17 @@ test_that("set_by_value_async", { list(expires_at = as.POSIXct(NA), notes = NA_character_)) - h <- c(m1$hash, m2$hash) - expect_equal(sto$mget_keymeta(h, c("objects", "ns2")), trg) - # wait mirai elements to be resolved m1$mirai$obj[] m1$mirai$key[] m2$mirai$obj[] m2$mirai$key[] - # Sys.sleep(1) + Sys.sleep(1) + + h <- c(m1$hash, m2$hash) + expect_equal(sto$mget_keymeta(h, c("objects", "ns2")), trg) + # expect_equal(sto$get("a"), 1) # expect_equal(sto$get("b", "ns2"), 2) @@ -358,15 +359,17 @@ test_that("mset_keymeta_async", { as.POSIXct(NA)), notes = c("😀", NA_character_))) + # wait mirai elements to be resolved + miall <- m1$mirai[] + + Sys.sleep(1) + expect_named(m1, c("mirai", "keyns")) expect_true( mirai::is_mirai(m1$mirai)) expect_equal(m1$keyns, c("x:objects", "y:objects")) expect_equal(sto$mget_keymeta(c("x", "y")), trgval) - # wait mirai elements to be resolved - miall <- m1$mirai[] - expect_equal(sto$mget_keymeta(c("x", "y"), use_cache = FALSE), trgval, ignore_attr = TRUE)