From 2a71de55dccf712fb5bae2bda177bd6870ca5d51 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Thu, 1 May 2025 19:39:10 +0200 Subject: [PATCH 01/28] add new functions --- DESCRIPTION | 1 + NAMESPACE | 6 ++ NEWS.md | 15 ++- R/cat.R | 211 +++++++++++++++++++++++++++++++++++++++++++ R/file_info.R | 134 +++++++++++++++++++++++++++ R/get_bbox.R | 155 +++++++++++++++++++++++++++++++ R/set_bbox.R | 131 +++++++++++++++++++++++++++ R/sort.R | 136 ++++++++++++++++++++++++++++ _pkgdown.yml | 5 + man/osm_cat.Rd | 142 +++++++++++++++++++++++++++++ man/osm_file_info.Rd | 82 +++++++++++++++++ man/osm_get_bbox.Rd | 69 ++++++++++++++ man/osm_set_bbox.Rd | 74 +++++++++++++++ man/osm_sort.Rd | 79 ++++++++++++++++ 14 files changed, 1239 insertions(+), 1 deletion(-) create mode 100644 R/cat.R create mode 100644 R/file_info.R create mode 100644 R/get_bbox.R create mode 100644 R/set_bbox.R create mode 100644 R/sort.R create mode 100644 man/osm_cat.Rd create mode 100644 man/osm_file_info.Rd create mode 100644 man/osm_get_bbox.Rd create mode 100644 man/osm_set_bbox.Rd create mode 100644 man/osm_sort.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 412b3e3..8a0991d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -19,6 +19,7 @@ Imports: checkmate, geojsonsf, processx, + RcppSimdJson, sf, utils Suggests: diff --git a/NAMESPACE b/NAMESPACE index e5abcd2..37c2c0c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,6 +1,12 @@ # Generated by roxygen2: do not edit by hand export(extract) +export(osm_cat) +export(osm_file_info) +export(osm_get_bbox) +export(osm_set_bbox) +export(osm_sort) export(show_content) export(tags_filter) export(time_filter) +importFrom(RcppSimdJson,fparse) diff --git a/NEWS.md b/NEWS.md index 39b0c89..f30aa66 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,8 +4,21 @@ ## New features +- `time_filter()` function to filter OSM history files by time. + +- `osm_file_info()` function to get information about an OSM file. + +- `osm_get_bbox()` function to get the bounding box of an OSM file. + +- `osm_set_bbox()` function to set the bounding box of an OSM file. + +- `osm_cat()` function to concatenate OSM files. + +- All functions that are direct wrappers for `osmium` commands now have web links to the osmium tool documentation for the latest stable version. + + ## Notes -# rosmium 0.1.0 +# rosmium 0.1.0 (2023-11-27) - Initial CRAN release. diff --git a/R/cat.R b/R/cat.R new file mode 100644 index 0000000..80ac116 --- /dev/null +++ b/R/cat.R @@ -0,0 +1,211 @@ +#' Concatenate OSM files +#' +#' This is a wrapper for [osmium cat](https://docs.osmcode.org/osmium/latest/osmium-cat.html). Just like the shell \code{osmium cat} command, \code{osm_cat} can take several input OSM files and concatenate them into one output file. As a side-effect, it can also convert between formats, filter by object type, clean metadata, and set output headers. +#' +#' @param input_paths Character vector of paths to one or more existing OSM files. +#' @param output_path String. Path to the file where the result will be written. +#' The format is inferred from the extension unless \code{output_format} is set. +#' @param object_type Character vector of object types to include. +#' One or more of \code{"node"}, \code{"way"}, \code{"relation"}, +#' \code{"changeset"}. Defaults to all types. +#' @param clean Character vector of attributes to clean (set to zero/empty). +#' One or more of \code{"version"}, \code{"timestamp"}, +#' \code{"changeset"}, \code{"uid"}, \code{"user"}. +#' @param buffer_data Logical(1). If \code{TRUE}, read all inputs into memory +#' before writing. Defaults to \code{FALSE}. +#' @param input_format Optional string. Force the input format (e.g. +#' \code{"osm"}, \code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect. +#' @param output_format Optional string. Force the output format (e.g. +#' \code{"osm"}, \code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect. +#' @param generator Optional string. Specify the \code{generator} header +#' (name and version) to add to the output file. +#' @param overwrite Logical(1). If \code{TRUE}, allow overwriting an existing +#' \code{output_path}. Defaults to \code{FALSE}. +#' @param fsync Logical(1). If \code{TRUE}, call \code{fsync} after writing +#' to force flushing buffers. Defaults to \code{FALSE}. +#' @param output_header Named character vector of additional header options. +#' Each name is the header option and each value is the header value. +#' If the value is \code{""}, the option will be written as \code{OPTION!}. +#' Defaults to \code{NULL}. +#' @param echo_cmd A logical. Whether to print the Osmium command generated by the function call to the screen. +#' Defaults to `FALSE`. +#' @param echo A logical. Whether to print the standard output and error generated by the Osmium call to the screen. +#' Defaults to `TRUE`. +#' @param spinner A logical. Whether to show a reassuring spinner while the Osmium call is being executed. +#' Defaults to `TRUE`. +#' @param verbose A logical. Whether to display detailed information on the running command. Defaults to `FALSE`. +#' @param progress A logical. Whether to display a progress bar while running +#' the command. Defaults to `FALSE`. +#' +#' @return Invisibly returns the normalized path to \code{output_path}. +#' +#' @examplesIf identical(tolower(Sys.getenv("NOT_CRAN")), "true") +#' pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") +#' +#' # 1. get header bbox (an sf::st_bbox) +#' orig_bbox <- osm_get_bbox(pbf) +#' +#' # 2. split bbox in two halves by longitude +#' midx <- (orig_bbox["xmin"] + orig_bbox["xmax"]) / 2 +#' +#' bbox1 <- sf::st_bbox( +#' c( +#' xmin = orig_bbox[["xmin"]], +#' ymin = orig_bbox[["ymin"]], +#' xmax = unname(midx), +#' ymax = orig_bbox[["ymax"]] +#' ), +#' crs = sf::st_crs(orig_bbox) +#' ) +#' +#' bbox2 <- sf::st_bbox( +#' c( +#' xmin = unname(midx), +#' ymin = orig_bbox[["ymin"]], +#' xmax = orig_bbox[["xmax"]], +#' ymax = orig_bbox[["ymax"]] +#' ), +#' crs = sf::st_crs(orig_bbox) +#' ) +#' +#' # 3. extract two halves +#' f1 <- extract( +#' pbf, +#' bbox1, +#' tempfile(fileext = ".osm.pbf"), +#' overwrite = TRUE, +#' echo = FALSE, +#' spinner = FALSE +#' ) +#' f2 <- extract( +#' pbf, +#' bbox2, +#' tempfile(fileext = ".osm.pbf"), +#' overwrite = TRUE, +#' echo = FALSE, +#' spinner = FALSE +#' ) +#' +#' # 4. concatenate +#' cat_out <- tempfile(fileext = ".osm.pbf") +#' osm_cat(c(f1, f2), cat_out, overwrite = TRUE, echo = FALSE, spinner = FALSE) +#' +#' # 5. sort, in case sorting was altered during concatenation +#' sorted_out <- tempfile(fileext = ".osm.pbf") +#' osm_sort(cat_out, sorted_out, overwrite = TRUE, echo = FALSE, spinner = FALSE) +#' +#' # 6. compute final bbox +#' final_bbox <- osm_get_bbox(sorted_out, calculate = TRUE) +#' +#' @export +osm_cat <- function( + input_paths, + output_path, + object_type = c("node", "way", "relation", "changeset"), + clean = character(), + buffer_data = FALSE, + input_format = NULL, + output_format = NULL, + generator = NULL, + overwrite = FALSE, + fsync = FALSE, + output_header = NULL, + echo_cmd = FALSE, + echo = TRUE, + spinner = TRUE, + verbose = FALSE, + progress = FALSE +) { + assert_osmium_is_installed() + # validate inputs + checkmate::assert_character(input_paths, any.missing = FALSE, min.len = 1) + for (p in input_paths) { + checkmate::assert_file_exists(p, access = "r") + } + checkmate::assert_string(output_path) + checkmate::assert_character(object_type, any.missing = FALSE) + checkmate::assert_subset( + object_type, + choices = c("node", "way", "relation", "changeset") + ) + checkmate::assert_character(clean) + checkmate::assert_subset( + clean, + choices = c("version", "timestamp", "changeset", "uid", "user"), + empty.ok = TRUE + ) + checkmate::assert_logical(buffer_data, len = 1) + if (!is.null(input_format)) checkmate::assert_string(input_format) + if (!is.null(output_format)) checkmate::assert_string(output_format) + if (!is.null(generator)) checkmate::assert_string(generator) + checkmate::assert_logical(overwrite, len = 1) + checkmate::assert_logical(fsync, len = 1) + if (!is.null(output_header)) { + checkmate::assert_character(output_header) + checkmate::assert_named(output_header, type = "unique") + } + checkmate::assert_logical(echo_cmd, len = 1) + checkmate::assert_logical(echo, len = 1) + checkmate::assert_logical(spinner, len = 1) + checkmate::assert_logical(verbose, len = 1) + checkmate::assert_logical(progress, len = 1) + + # validate output path and extension + assert_output_path_multi_ext(output_path, overwrite) + + # build command-line arguments + clean_flags <- if (length(clean) > 0) paste0("--clean=", clean) else + character() + type_flags <- paste0("--object-type=", object_type) + buffer_flag <- if (buffer_data) "--buffer-data" else character() + input_fmt_flag <- if (!is.null(input_format)) + paste0("--input-format=", input_format) else character() + output_fmt_flag <- if (!is.null(output_format)) + paste0("--output-format=", output_format) else character() + generator_flag <- if (!is.null(generator)) + paste0("--generator=", generator) else character() + overwrite_flag <- if (overwrite) "--overwrite" else character() + fsync_flag <- if (fsync) "--fsync" else character() + header_flags <- if (!is.null(output_header)) { + unname(vapply( + names(output_header), + function(nm) { + val <- output_header[[nm]] + hdr <- if (nzchar(val)) paste0(nm, "=", val) else paste0(nm, "!") + paste0("--output-header=", hdr) + }, + character(1) + )) + } else character() + verbose_flag <- if (verbose) "--verbose" else character() + progress_flag <- if (progress) "--progress" else "--no-progress" + output_flag <- paste0("--output=", output_path) + + args <- c( + "cat", + input_paths, + clean_flags, + type_flags, + buffer_flag, + input_fmt_flag, + output_fmt_flag, + generator_flag, + output_flag, + overwrite_flag, + fsync_flag, + header_flags, + verbose_flag, + progress_flag + ) + + # run osmium + logs <- processx::run( + "osmium", + args, + echo = echo, + spinner = spinner, + echo_cmd = echo_cmd + ) + + invisible(normalizePath(output_path)) +} diff --git a/R/file_info.R b/R/file_info.R new file mode 100644 index 0000000..c556faa --- /dev/null +++ b/R/file_info.R @@ -0,0 +1,134 @@ +#' Show information about an OSM file +#' +#' @description +#' This is a wrapper for [osmium fileinfo](https://docs.osmcode.org/osmium/latest/osmium-fileinfo.html). +#' Returns OSM file & header info in three possible formats: +#' - \code{"text"}: the original plain‐text dump +#' - \code{"json"}: the raw JSON string emitted by \code{--json} +#' - \code{"list"}: the JSON parsed to an R list via \code{RcppSimdJson::fparse()} +#' +#' @param input_path String. Path to the OSM file. +#' @param extended A logical(1). Read the entire file (not just header)? Defaults to `FALSE`. Reading extended info may be slow, as it reads the entire file. This is also required if the file does not have a bounding box in the header, and you need to calculate it from the data. Specifically, to get or calculate the bounding box, you need can use the [osm_get_bbox] function. +#' @param get String or NULL. If non‐NULL, return only this one variable. +#' @param crc A logical. Calculate CRC32? Defaults to `FALSE`. +#' @param object_type Character vector or NULL. Only for \code{extended = TRUE}. Defaults to \code{NULL}. +#' @param output One of \code{"text"}, \code{"json"}, \code{"list"}. +#' Defaults to \code{"text"}. +#' @param echo_cmd A logical. Whether to print the Osmium command generated by the function call to the screen. +#' Defaults to `FALSE`. +#' @param echo A logical. Whether to print the standard output and error generated by the Osmium call to the screen. +#' Defaults to `TRUE`. +#' @param spinner A logical. Whether to show a reassuring spinner while the Osmium call is being executed. +#' Defaults to `TRUE`. +#' @param verbose A logical. Whether to display detailed information on the running command. Defaults to `FALSE`. +#' @param progress A logical. Whether to display a progress bar while running +#' the command. Defaults to `FALSE`. +#' +#' @return Depending on \code{output}: +#' * \code{"text"}: character scalar (the plain‐text dump). +#' * \code{"json"}: character scalar (the JSON string). +#' * \code{"list"}: an R list parsed via \code{RcppSimdJson::fparse()}. +#' +#' @importFrom RcppSimdJson fparse +#' +#' @examplesIf identical(tolower(Sys.getenv("NOT_CRAN")), "true") +#' pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") +#' +#' # 1. plain-text (default) +#' txt <- osm_file_info(pbf) +#' cat(substr(txt, 1, 200), "...", "\n") +#' +#' # 2. raw JSON +#' j <- osm_file_info(pbf, output = "json") +#' cat(substr(j, 1, 200), "...", "\n") +# +#' # 3. parsed list +#' lst <- osm_file_info(pbf, output = "list") +#' str(lst[c("file","header")]) +#' +#' +#' @export +osm_file_info <- function( + input_path, + extended = FALSE, + get = NULL, + crc = FALSE, + object_type = NULL, + output = c("text", "json", "list"), + echo_cmd = FALSE, + echo = FALSE, + spinner = TRUE, + verbose = FALSE, + progress = FALSE +) { + assert_osmium_is_installed() + checkmate::assert_file_exists(input_path) + checkmate::assert_logical(extended, any.missing = FALSE, len = 1) + checkmate::assert_logical(crc, any.missing = FALSE, len = 1) + checkmate::assert_logical(echo_cmd, any.missing = FALSE, len = 1) + checkmate::assert_logical(echo, any.missing = FALSE, len = 1) + checkmate::assert_logical(spinner, any.missing = FALSE, len = 1) + checkmate::assert_logical(verbose, any.missing = FALSE, len = 1) + checkmate::assert_logical(progress, any.missing = FALSE, len = 1) + output <- match.arg(output) + + if (!is.null(get)) { + checkmate::assert_string(get) + if (output == "list") { + warning("`get` overrides `output = 'list'`; returning plain text") + output <- "text" + } + } + if (!is.null(object_type)) { + checkmate::assert_subset( + object_type, + choices = c("node", "way", "relation", "changeset") + ) + if (!extended) { + warning("`object_type` only applies when `extended = TRUE`") + } + } + + # build the common bits of the command + args <- c("fileinfo", input_path) + if (extended) args <- c(args, "--extended") + if (crc) args <- c(args, "--crc") + if (!is.null(get)) args <- c(args, paste0("--get=", get)) + if (verbose) args <- c(args, "--verbose") + args <- c(args, if (progress) "--progress" else "--no-progress") + if (!is.null(object_type) && extended) { + args <- c(args, paste0("--object-type=", object_type)) + } + + # 1) Plain text + if (output == "text") { + logs <- processx::run( + "osmium", + args, + echo = echo, + spinner = spinner, + echo_cmd = echo_cmd + ) + # logs$stdout and logs$stderr are captured in memory; nothing is written to disk + return(logs$stdout) + } + + # 2) JSON or list: add --json + args_json <- c(args, "--json") + logs <- processx::run( + "osmium", + args_json, + echo = echo, + spinner = spinner, + echo_cmd = echo_cmd + ) + raw_json <- logs$stdout + + if (output == "json") { + return(raw_json) + } + + # 3) output == "list": parse in-memory via RcppSimdJson::fparse() + # (no temp files, no disk writes) + return(RcppSimdJson::fparse(raw_json)) +} diff --git a/R/get_bbox.R b/R/get_bbox.R new file mode 100644 index 0000000..fddc693 --- /dev/null +++ b/R/get_bbox.R @@ -0,0 +1,155 @@ +#' Retrieve the bounding box of an OSM file +#' +#' By default, returns a single \code{sf::st_bbox} if exactly one header +#' bounding box is present. If no header box exists and \code{calculate = TRUE}, +#' computes the true data bounding box via \code{osmium fileinfo --extended} +#' (this reads the entire file and may be slow) and returns that. If multiple +#' header boxes are found and \code{return_bbox_list = FALSE}, the function +#' errors and asks you to retry with \code{return_bbox_list = TRUE}. +#' +#' @param input_path String. Path to an existing OSM file. +#' @param calculate Logical(1). If \code{TRUE}, compute from data when no +#' header bbox is present; otherwise error if missing. Defaults to \code{FALSE}. +#' @param return_bbox_list Logical(1). If \code{TRUE}, return a \emph{list} of +#' \code{sf::st_bbox} objects (one per header box), even if only one is found. +#' Defaults to \code{FALSE}. +#' @param echo_cmd A logical. Whether to print the Osmium command generated by the function call to the screen. +#' Defaults to `FALSE`. +#' @param echo A logical. Whether to print the standard output and error generated by the Osmium call to the screen. +#' Defaults to `TRUE`. +#' @param spinner A logical. Whether to show a reassuring spinner while the Osmium call is being executed. +#' Defaults to `TRUE`. +#' @param verbose A logical. Whether to display detailed information on the running command. Defaults to `FALSE`. +#' @param progress A logical. Whether to display a progress bar while running +#' the command. Defaults to `FALSE`. +#' +#' @return +#' - If \code{return_bbox_list = FALSE}: a single \code{sf::st_bbox}. +#' - If \code{return_bbox_list = TRUE}: a \code{list} of \code{sf::st_bbox}. +#' +#' @examplesIf identical(tolower(Sys.getenv("NOT_CRAN")), "true") +#' pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") +#' +#' # header has one box +#' b1 <- osm_get_bbox(pbf) +#' +#' # force computing true bbox (reads entire file) +#' b2 <- osm_get_bbox(pbf, calculate = TRUE) +#' +#' # get a list of header boxes (even though there is only one) +#' L <- osm_get_bbox(pbf, return_bbox_list = TRUE) +#' @export +osm_get_bbox <- function( + input_path, + calculate = FALSE, + return_bbox_list = FALSE, + echo_cmd = FALSE, + echo = FALSE, + spinner = FALSE, + verbose = FALSE, + progress = FALSE +) { + assert_osmium_is_installed() + checkmate::assert_file_exists(input_path) + checkmate::assert_logical(calculate, len = 1, any.missing = FALSE) + checkmate::assert_logical(return_bbox_list, len = 1, any.missing = FALSE) + checkmate::assert_logical(echo_cmd, len = 1, any.missing = FALSE) + checkmate::assert_logical(echo, len = 1, any.missing = FALSE) + checkmate::assert_logical(spinner, len = 1, any.missing = FALSE) + checkmate::assert_logical(verbose, len = 1, any.missing = FALSE) + checkmate::assert_logical(progress, len = 1, any.missing = FALSE) + + # read only header + info <- osm_file_info( + input_path, + extended = FALSE, + output = "list", + echo_cmd = echo_cmd, + echo = echo, + spinner = spinner, + verbose = verbose, + progress = progress + ) + + boxes_raw <- info$header$boxes + + # normalize into a list of numeric[4] + if (is.matrix(boxes_raw) && ncol(boxes_raw) == 4) { + boxes_list <- split(boxes_raw, seq_len(nrow(boxes_raw))) + } else if (is.list(boxes_raw)) { + boxes_list <- boxes_raw + } else { + boxes_list <- list() + } + + # filter only well‐formed vectors + boxes_list <- Filter(function(x) is.numeric(x) && length(x) == 4, boxes_list) + + n_boxes <- length(boxes_list) + + # if user wants the list, return it immediately + if (return_bbox_list) { + return( + lapply(boxes_list, function(x) { + sf::st_bbox( + c(xmin = x[1], ymin = x[2], xmax = x[3], ymax = x[4]), + crs = 4326 + ) + }) + ) + } + + # not returning list: must have exactly one header box or compute + if (n_boxes > 1) { + stop( + sprintf( + "Found %d header bounding boxes; to retrieve them all, retry with return_bbox_list = TRUE.", + n_boxes + ) + ) + } + + # exactly one header box: return it + if (n_boxes == 1) { + x <- boxes_list[[1]] + return(sf::st_bbox( + c(xmin = x[1], ymin = x[2], xmax = x[3], ymax = x[4]), + crs = 4326 + )) + } + + # no header boxes + if (!calculate) { + stop( + sprintf("No bounding‐box header found in '%s'.\n", input_path), + "Use calculate = TRUE to compute the bbox from the data (this may be slow as it reads the entire file)." + ) + } + + # compute via extended fileinfo + info_ext <- osm_file_info( + input_path, + extended = TRUE, + output = "list", + echo_cmd = echo_cmd, + echo = echo, + spinner = spinner, + verbose = verbose, + progress = progress + ) + + data_bbox <- info_ext$data$bbox + if (!is.numeric(data_bbox) || length(data_bbox) != 4) { + stop("Failed to compute data bbox for file: ", input_path) + } + + sf::st_bbox( + c( + xmin = data_bbox[1], + ymin = data_bbox[2], + xmax = data_bbox[3], + ymax = data_bbox[4] + ), + crs = 4326 + ) +} diff --git a/R/set_bbox.R b/R/set_bbox.R new file mode 100644 index 0000000..c107dbb --- /dev/null +++ b/R/set_bbox.R @@ -0,0 +1,131 @@ +#' Apply or Set Bounding‐Box Header on an OSM File +#' +#' Writes a new PBF file (or overwrites it) with its header bounding box set +#' to either a user‐supplied \code{sf::st_bbox} or—by default—the computed data +#' bounding box. Computing may require reading the entire file and can be slow! +#' +#' @param input_path String. Path to the existing OSM file. +#' @param output_path String. Path where the new file will be written. +#' @param bbox An \code{sf::st_bbox} to use. If \code{NULL}, +#' the bounding box is computed via \code{osm_get_bbox()}. +#' @param calculate Logical(1). If \code{TRUE}, allow computing from data +#' when no header is present; otherwise error if missing. +#' Defaults to \code{TRUE}. +#' @param overwrite Logical(1). If \code{TRUE}, allow overwriting +#' \code{output_path}. Defaults to \code{FALSE}. +#' @param echo_cmd A logical. Whether to print the Osmium command generated by the function call to the screen. +#' Defaults to `FALSE`. +#' @param echo A logical. Whether to print the standard output and error generated by the Osmium call to the screen. +#' Defaults to `TRUE`. +#' @param spinner A logical. Whether to show a reassuring spinner while the Osmium call is being executed. +#' Defaults to `TRUE`. +#' @param verbose A logical. Whether to display detailed information on the running command. Defaults to `FALSE`. +#' @param progress A logical. Whether to display a progress bar while running +#' the command. Defaults to `FALSE`. +#' +#' @return Invisibly returns the normalized path to \code{output_path}. +#' +#' @examplesIf identical(tolower(Sys.getenv("NOT_CRAN")), "true") +#' pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") +#' +#' # Compute true bbox and write to a new file +#' out <- tempfile(fileext = ".osm.pbf") +#' osm_set_bbox(pbf, out, calculate = TRUE, overwrite = TRUE) +#' +#' # Supply a custom bbox +#' library(sf) +#' custom <- st_bbox( +#' c(xmin = -49.3, ymin = -25.45, xmax = -49.24, ymax = -25.41), +#' crs = st_crs(4326) +#' ) +#' out2 <- tempfile(fileext = ".osm.pbf") +#' osm_set_bbox(pbf, out2, bbox = custom, overwrite = TRUE) +#' +#' @export +osm_set_bbox <- function( + input_path, + output_path, + bbox = NULL, + calculate = TRUE, + overwrite = FALSE, + echo_cmd = FALSE, + echo = TRUE, + spinner = TRUE, + verbose = FALSE, + progress = FALSE +) { + assert_osmium_is_installed() + checkmate::assert_file_exists(input_path) + checkmate::assert_string(output_path) + checkmate::assert_logical(calculate, len = 1, any.missing = FALSE) + checkmate::assert_logical(overwrite, len = 1, any.missing = FALSE) + checkmate::assert_logical(echo_cmd, len = 1, any.missing = FALSE) + checkmate::assert_logical(echo, len = 1, any.missing = FALSE) + checkmate::assert_logical(spinner, len = 1, any.missing = FALSE) + checkmate::assert_logical(verbose, len = 1, any.missing = FALSE) + checkmate::assert_logical(progress, len = 1, any.missing = FALSE) + + # Ensure output directory exists + out_dir <- dirname(output_path) + if (!dir.exists(out_dir)) { + stop("Directory does not exist: ", out_dir) + } + + # Overwrite guard + if (file.exists(output_path) && !overwrite) { + stop( + "Output file '", + output_path, + "' already exists;", + " to overwrite, set overwrite = TRUE." + ) + } + + # Determine which bbox to use + if (!is.null(bbox)) { + if (!inherits(bbox, "bbox")) { + stop("`bbox` must be an sf::st_bbox object or NULL.") + } + b <- bbox + } else { + b <- osm_get_bbox( + input_path, + calculate = calculate, + echo_cmd = echo_cmd, + echo = echo, + spinner = spinner, + verbose = verbose, + progress = progress + ) + } + + # Format for osmium + bbox_arg <- paste( + c(b["xmin"], b["ymin"], b["xmax"], b["ymax"]), + collapse = "," + ) + + # Build command + args <- c( + "extract", + input_path, + paste0("--bbox=", bbox_arg), + "--set-bounds", + if (verbose) "--verbose" else NULL, + if (progress) "--progress" else "--no-progress", + "--overwrite", + "-o", + output_path + ) + + # Run osmium + processx::run( + "osmium", + args, + echo = echo, + spinner = spinner, + echo_cmd = echo_cmd + ) + + invisible(normalizePath(output_path)) +} diff --git a/R/sort.R b/R/sort.R new file mode 100644 index 0000000..3612be7 --- /dev/null +++ b/R/sort.R @@ -0,0 +1,136 @@ +#' Sort OSM files +#' +#' This is a wrapper for [osmium sort](https://docs.osmcode.org/osmium/latest/osmium-sort.html). Combines and sorts the content of one or more OSM files. Objects are ordered by type (node, way, relation), by ID (negative IDs first, then positive, sorted by absolute value), and by version. Duplicate objects are preserved. Works with data, history, and change files. +#' +#' @param input_paths Character vector of paths to one or more existing OSM files. +#' @param output_path String. Path to the file where the sorted output will be written. +#' The format is inferred from the extension unless \code{output_format} is set. +#' @param strategy Character(1). Sorting strategy, either \code{"simple"} (default, +#' single-pass in-memory) or \code{"multipass"} (three passes, lower memory). +#' @param input_format Optional string. Force the input format (e.g. \code{"osm"}, +#' \code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect. +#' @param output_format Optional string. Force the output format (e.g. \code{"osm"}, +#' \code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect. +#' @param fsync Logical(1). If \code{TRUE}, call \code{fsync} after writing the +#' output file to flush buffers. Defaults to \code{FALSE}. +#' @param generator Optional string. Name/version to write into the \code{generator} +#' header. Default is \code{"osmium/"}. +#' @param overwrite Logical(1). If \code{TRUE}, allow overwriting an existing +#' \code{output_path}. Defaults to \code{FALSE}. +#' @param output_header Named character vector of additional header options. +#' Each name is the header key, each value is the header value. If a value +#' is \code{""}, writes \code{OPTION!}. Defaults to \code{NULL}. +#' @param echo_cmd A logical. Whether to print the Osmium command generated by the function call to the screen. +#' Defaults to `FALSE`. +#' @param echo A logical. Whether to print the standard output and error generated by the Osmium call to the screen. +#' Defaults to `TRUE`. +#' @param spinner A logical. Whether to show a reassuring spinner while the Osmium call is being executed. +#' Defaults to `TRUE`. +#' @param verbose A logical. Whether to display detailed information on the running command. Defaults to `FALSE`. +#' @param progress A logical. Whether to display a progress bar while running +#' the command. Defaults to `FALSE`. +#' +#' @return Invisibly returns the normalized path to \code{output_path}. +#' +#' @examplesIf identical(tolower(Sys.getenv("NOT_CRAN")), "true") +#' # sort a single PBF file +#' pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") +#' out <- tempfile(fileext = ".osm.pbf") +#' osm_sort(pbf, out, overwrite = TRUE) +#' +#' @export +osm_sort <- function( + input_paths, + output_path, + strategy = c("simple", "multipass"), + input_format = NULL, + output_format = NULL, + fsync = FALSE, + generator = NULL, + overwrite = FALSE, + output_header = NULL, + echo_cmd = FALSE, + echo = TRUE, + spinner = TRUE, + verbose = FALSE, + progress = FALSE +) { + assert_osmium_is_installed() + + # validate inputs + checkmate::assert_character(input_paths, any.missing = FALSE, min.len = 1) + for (p in input_paths) { + checkmate::assert_file_exists(p, access = "r") + } + checkmate::assert_string(output_path) + strategy <- match.arg(strategy) + if (!is.null(input_format)) checkmate::assert_string(input_format) + if (!is.null(output_format)) checkmate::assert_string(output_format) + checkmate::assert_logical(fsync, len = 1, any.missing = FALSE) + if (!is.null(generator)) checkmate::assert_string(generator) + checkmate::assert_logical(overwrite, len = 1, any.missing = FALSE) + if (!is.null(output_header)) { + checkmate::assert_named(output_header, type = "unique") + checkmate::assert_character(output_header) + } + checkmate::assert_logical(echo_cmd, len = 1, any.missing = FALSE) + checkmate::assert_logical(echo, len = 1, any.missing = FALSE) + checkmate::assert_logical(spinner, len = 1, any.missing = FALSE) + checkmate::assert_logical(verbose, len = 1, any.missing = FALSE) + checkmate::assert_logical(progress, len = 1, any.missing = FALSE) + + # check output path + assert_output_path_multi_ext(output_path, overwrite) + + # build flags + strat_flag <- paste0("--strategy=", strategy) + input_flag <- if (!is.null(input_format)) + paste0("--input-format=", input_format) else character() + output_fmt_flag <- if (!is.null(output_format)) + paste0("--output-format=", output_format) else character() + fsync_flag <- if (fsync) "--fsync" else character() + gen_flag <- if (!is.null(generator)) paste0("--generator=", generator) else + character() + overwrite_flag <- if (overwrite) "--overwrite" else character() + header_flags <- if (!is.null(output_header)) { + unname(vapply( + names(output_header), + function(nm) { + val <- output_header[[nm]] + hdr <- if (nzchar(val)) paste0(nm, "=", val) else paste0(nm, "!") + paste0("--output-header=", hdr) + }, + character(1) + )) + } else character() + verbose_flag <- if (verbose) "--verbose" else character() + progress_flag <- if (progress) "--progress" else "--no-progress" + output_flag <- paste0("--output=", output_path) + + # assemble args + args <- c( + "sort", + input_paths, + strat_flag, + input_flag, + output_fmt_flag, + fsync_flag, + gen_flag, + output_flag, + overwrite_flag, + header_flags, + verbose_flag, + progress_flag + ) + + # run osmium sort + processx::run( + "osmium", + args, + echo = echo, + spinner = spinner, + echo_cmd = echo_cmd + ) + + invisible(normalizePath(output_path)) +} diff --git a/_pkgdown.yml b/_pkgdown.yml index d25029a..fe6fa07 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -10,10 +10,15 @@ development: reference: - title: "Functions" - contents: + - osm_file_info - extract - tags_filter - show_content - time_filter + - osm_get_bbox + - osm_set_bbox + - osm_sort + - osm_cat - title: "General documentation" - contents: - file_formats diff --git a/man/osm_cat.Rd b/man/osm_cat.Rd new file mode 100644 index 0000000..fe1bcb6 --- /dev/null +++ b/man/osm_cat.Rd @@ -0,0 +1,142 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/cat.R +\name{osm_cat} +\alias{osm_cat} +\title{Concatenate OSM files} +\usage{ +osm_cat( + input_paths, + output_path, + object_type = c("node", "way", "relation", "changeset"), + clean = character(), + buffer_data = FALSE, + input_format = NULL, + output_format = NULL, + generator = NULL, + overwrite = FALSE, + fsync = FALSE, + output_header = NULL, + echo_cmd = FALSE, + echo = TRUE, + spinner = TRUE, + verbose = FALSE, + progress = FALSE +) +} +\arguments{ +\item{input_paths}{Character vector of paths to one or more existing OSM files.} + +\item{output_path}{String. Path to the file where the result will be written. +The format is inferred from the extension unless \code{output_format} is set.} + +\item{object_type}{Character vector of object types to include. +One or more of \code{"node"}, \code{"way"}, \code{"relation"}, +\code{"changeset"}. Defaults to all types.} + +\item{clean}{Character vector of attributes to clean (set to zero/empty). +One or more of \code{"version"}, \code{"timestamp"}, +\code{"changeset"}, \code{"uid"}, \code{"user"}.} + +\item{buffer_data}{Logical(1). If \code{TRUE}, read all inputs into memory +before writing. Defaults to \code{FALSE}.} + +\item{input_format}{Optional string. Force the input format (e.g. +\code{"osm"}, \code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect.} + +\item{output_format}{Optional string. Force the output format (e.g. +\code{"osm"}, \code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect.} + +\item{generator}{Optional string. Specify the \code{generator} header +(name and version) to add to the output file.} + +\item{overwrite}{Logical(1). If \code{TRUE}, allow overwriting an existing +\code{output_path}. Defaults to \code{FALSE}.} + +\item{fsync}{Logical(1). If \code{TRUE}, call \code{fsync} after writing +to force flushing buffers. Defaults to \code{FALSE}.} + +\item{output_header}{Named character vector of additional header options. +Each name is the header option and each value is the header value. +If the value is \code{""}, the option will be written as \code{OPTION!}. +Defaults to \code{NULL}.} + +\item{echo_cmd}{A logical. Whether to print the Osmium command generated by the function call to the screen. +Defaults to \code{FALSE}.} + +\item{echo}{A logical. Whether to print the standard output and error generated by the Osmium call to the screen. +Defaults to \code{TRUE}.} + +\item{spinner}{A logical. Whether to show a reassuring spinner while the Osmium call is being executed. +Defaults to \code{TRUE}.} + +\item{verbose}{A logical. Whether to display detailed information on the running command. Defaults to \code{FALSE}.} + +\item{progress}{A logical. Whether to display a progress bar while running +the command. Defaults to \code{FALSE}.} +} +\value{ +Invisibly returns the normalized path to \code{output_path}. +} +\description{ +This is a wrapper for \href{https://docs.osmcode.org/osmium/latest/osmium-cat.html}{osmium cat}. Just like the shell \code{osmium cat} command, \code{osm_cat} can take several input OSM files and concatenate them into one output file. As a side-effect, it can also convert between formats, filter by object type, clean metadata, and set output headers. +} +\examples{ +\dontshow{if (identical(tolower(Sys.getenv("NOT_CRAN")), "true")) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + +# 1. get header bbox (an sf::st_bbox) +orig_bbox <- osm_get_bbox(pbf) + +# 2. split bbox in two halves by longitude +midx <- (orig_bbox["xmin"] + orig_bbox["xmax"]) / 2 + +bbox1 <- sf::st_bbox( + c( + xmin = orig_bbox[["xmin"]], + ymin = orig_bbox[["ymin"]], + xmax = unname(midx), + ymax = orig_bbox[["ymax"]] + ), + crs = sf::st_crs(orig_bbox) +) + +bbox2 <- sf::st_bbox( + c( + xmin = unname(midx), + ymin = orig_bbox[["ymin"]], + xmax = orig_bbox[["xmax"]], + ymax = orig_bbox[["ymax"]] + ), + crs = sf::st_crs(orig_bbox) +) + +# 3. extract two halves +f1 <- extract( + pbf, + bbox1, + tempfile(fileext = ".osm.pbf"), + overwrite = TRUE, + echo = FALSE, + spinner = FALSE +) +f2 <- extract( + pbf, + bbox2, + tempfile(fileext = ".osm.pbf"), + overwrite = TRUE, + echo = FALSE, + spinner = FALSE +) + +# 4. concatenate +cat_out <- tempfile(fileext = ".osm.pbf") +osm_cat(c(f1, f2), cat_out, overwrite = TRUE, echo = FALSE, spinner = FALSE) + +# 5. sort, in case sorting was altered during concatenation +sorted_out <- tempfile(fileext = ".osm.pbf") +osm_sort(cat_out, sorted_out, overwrite = TRUE, echo = FALSE, spinner = FALSE) + +# 6. compute final bbox +final_bbox <- osm_get_bbox(sorted_out, calculate = TRUE) +\dontshow{\}) # examplesIf} +} diff --git a/man/osm_file_info.Rd b/man/osm_file_info.Rd new file mode 100644 index 0000000..dd2faf4 --- /dev/null +++ b/man/osm_file_info.Rd @@ -0,0 +1,82 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/file_info.R +\name{osm_file_info} +\alias{osm_file_info} +\title{Show information about an OSM file} +\usage{ +osm_file_info( + input_path, + extended = FALSE, + get = NULL, + crc = FALSE, + object_type = NULL, + output = c("text", "json", "list"), + echo_cmd = FALSE, + echo = FALSE, + spinner = TRUE, + verbose = FALSE, + progress = FALSE +) +} +\arguments{ +\item{input_path}{String. Path to the OSM file.} + +\item{extended}{A logical(1). Read the entire file (not just header)? Defaults to \code{FALSE}. Reading extended info may be slow, as it reads the entire file. This is also required if the file does not have a bounding box in the header, and you need to calculate it from the data. Specifically, to get or calculate the bounding box, you need can use the \link{osm_get_bbox} function.} + +\item{get}{String or NULL. If non‐NULL, return only this one variable.} + +\item{crc}{A logical. Calculate CRC32? Defaults to \code{FALSE}.} + +\item{object_type}{Character vector or NULL. Only for \code{extended = TRUE}. Defaults to \code{NULL}.} + +\item{output}{One of \code{"text"}, \code{"json"}, \code{"list"}. +Defaults to \code{"text"}.} + +\item{echo_cmd}{A logical. Whether to print the Osmium command generated by the function call to the screen. +Defaults to \code{FALSE}.} + +\item{echo}{A logical. Whether to print the standard output and error generated by the Osmium call to the screen. +Defaults to \code{TRUE}.} + +\item{spinner}{A logical. Whether to show a reassuring spinner while the Osmium call is being executed. +Defaults to \code{TRUE}.} + +\item{verbose}{A logical. Whether to display detailed information on the running command. Defaults to \code{FALSE}.} + +\item{progress}{A logical. Whether to display a progress bar while running +the command. Defaults to \code{FALSE}.} +} +\value{ +Depending on \code{output}: +\itemize{ +\item \code{"text"}: character scalar (the plain‐text dump). +\item \code{"json"}: character scalar (the JSON string). +\item \code{"list"}: an R list parsed via \code{RcppSimdJson::fparse()}. +} +} +\description{ +This is a wrapper for \href{https://docs.osmcode.org/osmium/latest/osmium-fileinfo.html}{osmium fileinfo}. +Returns OSM file & header info in three possible formats: +\itemize{ +\item \code{"text"}: the original plain‐text dump +\item \code{"json"}: the raw JSON string emitted by \code{--json} +\item \code{"list"}: the JSON parsed to an R list via \code{RcppSimdJson::fparse()} +} +} +\examples{ +\dontshow{if (identical(tolower(Sys.getenv("NOT_CRAN")), "true")) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + +# 1. plain-text (default) +txt <- osm_file_info(pbf) +cat(substr(txt, 1, 200), "...", "\n") + +# 2. raw JSON +j <- osm_file_info(pbf, output = "json") +cat(substr(j, 1, 200), "...", "\n") +# 3. parsed list +lst <- osm_file_info(pbf, output = "list") +str(lst[c("file","header")]) + +\dontshow{\}) # examplesIf} +} diff --git a/man/osm_get_bbox.Rd b/man/osm_get_bbox.Rd new file mode 100644 index 0000000..f40d2b6 --- /dev/null +++ b/man/osm_get_bbox.Rd @@ -0,0 +1,69 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_bbox.R +\name{osm_get_bbox} +\alias{osm_get_bbox} +\title{Retrieve the bounding box of an OSM file} +\usage{ +osm_get_bbox( + input_path, + calculate = FALSE, + return_bbox_list = FALSE, + echo_cmd = FALSE, + echo = FALSE, + spinner = FALSE, + verbose = FALSE, + progress = FALSE +) +} +\arguments{ +\item{input_path}{String. Path to an existing OSM file.} + +\item{calculate}{Logical(1). If \code{TRUE}, compute from data when no +header bbox is present; otherwise error if missing. Defaults to \code{FALSE}.} + +\item{return_bbox_list}{Logical(1). If \code{TRUE}, return a \emph{list} of +\code{sf::st_bbox} objects (one per header box), even if only one is found. +Defaults to \code{FALSE}.} + +\item{echo_cmd}{A logical. Whether to print the Osmium command generated by the function call to the screen. +Defaults to \code{FALSE}.} + +\item{echo}{A logical. Whether to print the standard output and error generated by the Osmium call to the screen. +Defaults to \code{TRUE}.} + +\item{spinner}{A logical. Whether to show a reassuring spinner while the Osmium call is being executed. +Defaults to \code{TRUE}.} + +\item{verbose}{A logical. Whether to display detailed information on the running command. Defaults to \code{FALSE}.} + +\item{progress}{A logical. Whether to display a progress bar while running +the command. Defaults to \code{FALSE}.} +} +\value{ +\itemize{ +\item If \code{return_bbox_list = FALSE}: a single \code{sf::st_bbox}. +\item If \code{return_bbox_list = TRUE}: a \code{list} of \code{sf::st_bbox}. +} +} +\description{ +By default, returns a single \code{sf::st_bbox} if exactly one header +bounding box is present. If no header box exists and \code{calculate = TRUE}, +computes the true data bounding box via \code{osmium fileinfo --extended} +(this reads the entire file and may be slow) and returns that. If multiple +header boxes are found and \code{return_bbox_list = FALSE}, the function +errors and asks you to retry with \code{return_bbox_list = TRUE}. +} +\examples{ +\dontshow{if (identical(tolower(Sys.getenv("NOT_CRAN")), "true")) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + +# header has one box +b1 <- osm_get_bbox(pbf) + +# force computing true bbox (reads entire file) +b2 <- osm_get_bbox(pbf, calculate = TRUE) + +# get a list of header boxes (even though there is only one) +L <- osm_get_bbox(pbf, return_bbox_list = TRUE) +\dontshow{\}) # examplesIf} +} diff --git a/man/osm_set_bbox.Rd b/man/osm_set_bbox.Rd new file mode 100644 index 0000000..f88f2cc --- /dev/null +++ b/man/osm_set_bbox.Rd @@ -0,0 +1,74 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/set_bbox.R +\name{osm_set_bbox} +\alias{osm_set_bbox} +\title{Apply or Set Bounding‐Box Header on an OSM File} +\usage{ +osm_set_bbox( + input_path, + output_path, + bbox = NULL, + calculate = TRUE, + overwrite = FALSE, + echo_cmd = FALSE, + echo = TRUE, + spinner = TRUE, + verbose = FALSE, + progress = FALSE +) +} +\arguments{ +\item{input_path}{String. Path to the existing OSM file.} + +\item{output_path}{String. Path where the new file will be written.} + +\item{bbox}{An \code{sf::st_bbox} to use. If \code{NULL}, +the bounding box is computed via \code{osm_get_bbox()}.} + +\item{calculate}{Logical(1). If \code{TRUE}, allow computing from data +when no header is present; otherwise error if missing. +Defaults to \code{TRUE}.} + +\item{overwrite}{Logical(1). If \code{TRUE}, allow overwriting +\code{output_path}. Defaults to \code{FALSE}.} + +\item{echo_cmd}{A logical. Whether to print the Osmium command generated by the function call to the screen. +Defaults to \code{FALSE}.} + +\item{echo}{A logical. Whether to print the standard output and error generated by the Osmium call to the screen. +Defaults to \code{TRUE}.} + +\item{spinner}{A logical. Whether to show a reassuring spinner while the Osmium call is being executed. +Defaults to \code{TRUE}.} + +\item{verbose}{A logical. Whether to display detailed information on the running command. Defaults to \code{FALSE}.} + +\item{progress}{A logical. Whether to display a progress bar while running +the command. Defaults to \code{FALSE}.} +} +\value{ +Invisibly returns the normalized path to \code{output_path}. +} +\description{ +Writes a new PBF file (or overwrites it) with its header bounding box set +to either a user‐supplied \code{sf::st_bbox} or—by default—the computed data +bounding box. Computing may require reading the entire file and can be slow! +} +\examples{ +\dontshow{if (identical(tolower(Sys.getenv("NOT_CRAN")), "true")) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + +# Compute true bbox and write to a new file +out <- tempfile(fileext = ".osm.pbf") +osm_set_bbox(pbf, out, calculate = TRUE, overwrite = TRUE) + +# Supply a custom bbox +library(sf) +custom <- st_bbox( + c(xmin = -49.3, ymin = -25.45, xmax = -49.24, ymax = -25.41), + crs = st_crs(4326) +) +out2 <- tempfile(fileext = ".osm.pbf") +osm_set_bbox(pbf, out2, bbox = custom, overwrite = TRUE) +\dontshow{\}) # examplesIf} +} diff --git a/man/osm_sort.Rd b/man/osm_sort.Rd new file mode 100644 index 0000000..8587903 --- /dev/null +++ b/man/osm_sort.Rd @@ -0,0 +1,79 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/sort.R +\name{osm_sort} +\alias{osm_sort} +\title{Sort OSM files} +\usage{ +osm_sort( + input_paths, + output_path, + strategy = c("simple", "multipass"), + input_format = NULL, + output_format = NULL, + fsync = FALSE, + generator = NULL, + overwrite = FALSE, + output_header = NULL, + echo_cmd = FALSE, + echo = TRUE, + spinner = TRUE, + verbose = FALSE, + progress = FALSE +) +} +\arguments{ +\item{input_paths}{Character vector of paths to one or more existing OSM files.} + +\item{output_path}{String. Path to the file where the sorted output will be written. +The format is inferred from the extension unless \code{output_format} is set.} + +\item{strategy}{Character(1). Sorting strategy, either \code{"simple"} (default, +single-pass in-memory) or \code{"multipass"} (three passes, lower memory).} + +\item{input_format}{Optional string. Force the input format (e.g. \code{"osm"}, +\code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect.} + +\item{output_format}{Optional string. Force the output format (e.g. \code{"osm"}, +\code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect.} + +\item{fsync}{Logical(1). If \code{TRUE}, call \code{fsync} after writing the +output file to flush buffers. Defaults to \code{FALSE}.} + +\item{generator}{Optional string. Name/version to write into the \code{generator} +header. Default is \code{"osmium/"}.} + +\item{overwrite}{Logical(1). If \code{TRUE}, allow overwriting an existing +\code{output_path}. Defaults to \code{FALSE}.} + +\item{output_header}{Named character vector of additional header options. +Each name is the header key, each value is the header value. If a value +is \code{""}, writes \code{OPTION!}. Defaults to \code{NULL}.} + +\item{echo_cmd}{A logical. Whether to print the Osmium command generated by the function call to the screen. +Defaults to \code{FALSE}.} + +\item{echo}{A logical. Whether to print the standard output and error generated by the Osmium call to the screen. +Defaults to \code{TRUE}.} + +\item{spinner}{A logical. Whether to show a reassuring spinner while the Osmium call is being executed. +Defaults to \code{TRUE}.} + +\item{verbose}{A logical. Whether to display detailed information on the running command. Defaults to \code{FALSE}.} + +\item{progress}{A logical. Whether to display a progress bar while running +the command. Defaults to \code{FALSE}.} +} +\value{ +Invisibly returns the normalized path to \code{output_path}. +} +\description{ +This is a wrapper for \href{https://docs.osmcode.org/osmium/latest/osmium-sort.html}{osmium sort}. Combines and sorts the content of one or more OSM files. Objects are ordered by type (node, way, relation), by ID (negative IDs first, then positive, sorted by absolute value), and by version. Duplicate objects are preserved. Works with data, history, and change files. +} +\examples{ +\dontshow{if (identical(tolower(Sys.getenv("NOT_CRAN")), "true")) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +# sort a single PBF file +pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") +out <- tempfile(fileext = ".osm.pbf") +osm_sort(pbf, out, overwrite = TRUE) +\dontshow{\}) # examplesIf} +} From deda08b81abc36c99381c046bca7902e31ac6608 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Thu, 1 May 2025 19:45:28 +0200 Subject: [PATCH 02/28] fix non ASCII --- R/get_bbox.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/get_bbox.R b/R/get_bbox.R index fddc693..58d1fd9 100644 --- a/R/get_bbox.R +++ b/R/get_bbox.R @@ -82,7 +82,7 @@ osm_get_bbox <- function( boxes_list <- list() } - # filter only well‐formed vectors + # filter only well formed vectors boxes_list <- Filter(function(x) is.numeric(x) && length(x) == 4, boxes_list) n_boxes <- length(boxes_list) @@ -121,7 +121,7 @@ osm_get_bbox <- function( # no header boxes if (!calculate) { stop( - sprintf("No bounding‐box header found in '%s'.\n", input_path), + sprintf("No bounding-box header found in '%s'.\n", input_path), "Use calculate = TRUE to compute the bbox from the data (this may be slow as it reads the entire file)." ) } From 7eecb0dfc0e46e95f198c78ff856d6d26c07ff79 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Thu, 1 May 2025 19:46:42 +0200 Subject: [PATCH 03/28] add tests for cat, file_info, get_bbox --- tests/testthat/test-cat.R | 90 +++++++++++++++++++++++++++++++++ tests/testthat/test-file_info.R | 46 +++++++++++++++++ tests/testthat/test-get_bbox.R | 43 ++++++++++++++++ 3 files changed, 179 insertions(+) create mode 100644 tests/testthat/test-cat.R create mode 100644 tests/testthat/test-file_info.R create mode 100644 tests/testthat/test-get_bbox.R diff --git a/tests/testthat/test-cat.R b/tests/testthat/test-cat.R new file mode 100644 index 0000000..552611d --- /dev/null +++ b/tests/testthat/test-cat.R @@ -0,0 +1,90 @@ +testthat::skip_on_cran() + +test_that("osm_cat + osm_sort yield correct final bbox", { + pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + + # original header bbox + orig_bbox <- osm_get_bbox(pbf) + expect_s3_class(orig_bbox, "bbox") + + # split mid‐longitude + midx <- (orig_bbox[["xmin"]] + orig_bbox[["xmax"]]) / 2 + + bbox1 <- sf::st_bbox( + c( + xmin = orig_bbox[["xmin"]], + ymin = orig_bbox[["ymin"]], + xmax = midx, + ymax = orig_bbox[["ymax"]] + ), + crs = sf::st_crs(orig_bbox) + ) + bbox2 <- sf::st_bbox( + c( + xmin = midx, + ymin = orig_bbox[["ymin"]], + xmax = orig_bbox[["xmax"]], + ymax = orig_bbox[["ymax"]] + ), + crs = sf::st_crs(orig_bbox) + ) + + # two extracts + f1 <- extract( + pbf, + bbox1, + tempfile(fileext = ".osm.pbf"), + overwrite = TRUE, + echo = FALSE, + spinner = FALSE + ) + f2 <- extract( + pbf, + bbox2, + tempfile(fileext = ".osm.pbf"), + overwrite = TRUE, + echo = FALSE, + spinner = FALSE + ) + expect_true(file.exists(f1)) + expect_true(file.exists(f2)) + + # concatenate + cat_file <- tempfile(fileext = ".osm.pbf") + osm_cat(c(f1, f2), cat_file, overwrite = TRUE, echo = FALSE, spinner = FALSE) + expect_true(file.exists(cat_file)) + + # sort + sorted_file <- tempfile(fileext = ".osm.pbf") + osm_sort( + cat_file, + sorted_file, + overwrite = TRUE, + echo = FALSE, + spinner = FALSE + ) + expect_true(file.exists(sorted_file)) + + # recompute final bbox + final_bbox <- osm_get_bbox(sorted_file, calculate = TRUE) + expect_s3_class(final_bbox, "bbox") + + # 1) it should strictly expand the original envelope + expect_lt(final_bbox[["xmin"]], orig_bbox[["xmin"]]) + expect_lt(final_bbox[["ymin"]], orig_bbox[["ymin"]]) + expect_gt(final_bbox[["xmax"]], orig_bbox[["xmax"]]) + expect_gt(final_bbox[["ymax"]], orig_bbox[["ymax"]]) + + # 2) and match the exact values we observed + expected <- c( + -49.30187, # xmin + -25.48331, # ymin + -49.20784, # xmax + -25.39819 # ymax + ) + expect_equal( + as.numeric(final_bbox), + expected, + tolerance = 1e-6 + ) +}) diff --git a/tests/testthat/test-file_info.R b/tests/testthat/test-file_info.R new file mode 100644 index 0000000..c478533 --- /dev/null +++ b/tests/testthat/test-file_info.R @@ -0,0 +1,46 @@ +testthat::skip_on_cran() + +test_that("osm_file_info returns correct structure for cur.osm.pbf", { + pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + expect_true(file.exists(pbf)) + + # ---- output="list" ---- + info <- osm_file_info(pbf, output = "list") + expect_type(info, "list") + expect_named(info, c("file", "header")) + + # file sub-list + f <- info$file + expect_named(f, c("name", "format", "compression", "size")) + expect_true(grepl("cur\\.osm\\.pbf$", f$name)) + expect_equal(f$format, "PBF") + expect_equal(f$compression, "none") + expect_true(is.numeric(f$size) && f$size > 0) + + # header sub-list + h <- info$header + expect_named(h, c("boxes", "with_history", "option")) + # boxes must be a 1×4 matrix + expect_true(is.matrix(h$boxes)) + expect_equal(dim(h$boxes), c(1, 4)) + expect_equal( + as.numeric(h$boxes), + c(-49.2889, -25.4413, -49.2509, -25.4197), + tolerance = 1e-6 + ) + expect_false(h$with_history) + + opts <- h$option + expect_named(opts, c("generator", "pbf_dense_nodes")) + expect_true(nzchar(opts$generator)) + expect_true(opts$pbf_dense_nodes %in% c("true", "false")) + + # ---- output="json" ---- + j <- osm_file_info(pbf, output = "json") + expect_true(jsonlite::validate(j)) + + # ---- output="text" ---- + t <- osm_file_info(pbf, output = "text") + expect_type(t, "character") + expect_true(grepl("Bounding boxes", t)) +}) diff --git a/tests/testthat/test-get_bbox.R b/tests/testthat/test-get_bbox.R new file mode 100644 index 0000000..2c8e7dd --- /dev/null +++ b/tests/testthat/test-get_bbox.R @@ -0,0 +1,43 @@ +testthat::skip_on_cran() +test_that("osm_get_bbox reads header bbox correctly (with tolerance)", { + pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + expect_true(file.exists(pbf)) + + # default: read header bbox + bbox <- osm_get_bbox(pbf) + expect_s3_class(bbox, "bbox") + + # expected values (one decimal precision) + expected_1dec <- c( + -49.3, # xmin + -25.4, # ymin + -49.3, # xmax + -25.4 # ymax + ) + + # compare with tolerance + expect_equal( + as.numeric(bbox), + expected_1dec, + tolerance = 0.1 + ) + + # return_bbox_list = TRUE: a list of length 1 + lst <- osm_get_bbox(pbf, return_bbox_list = TRUE) + expect_type(lst, "list") + expect_length(lst, 1) + expect_s3_class(lst[[1]], "bbox") + expect_equal( + as.numeric(lst[[1]]), + expected_1dec, + tolerance = 0.1 + ) + + # calculate = TRUE on a header-rich file should still return header + bbox2 <- osm_get_bbox(pbf, calculate = TRUE) + expect_equal( + as.numeric(bbox2), + expected_1dec, + tolerance = 0.1 + ) +}) From f074fe16c5a3684fda2b9ceac63fabe51b616a4f Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Thu, 1 May 2025 22:24:16 +0200 Subject: [PATCH 04/28] fix test for file_info --- tests/testthat/test-file_info.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-file_info.R b/tests/testthat/test-file_info.R index c478533..feb432e 100644 --- a/tests/testthat/test-file_info.R +++ b/tests/testthat/test-file_info.R @@ -37,7 +37,7 @@ test_that("osm_file_info returns correct structure for cur.osm.pbf", { # ---- output="json" ---- j <- osm_file_info(pbf, output = "json") - expect_true(jsonlite::validate(j)) + expect_true(RcppSimdJson::is_valid_json(j)) # ---- output="text" ---- t <- osm_file_info(pbf, output = "text") From 85f6b8253711f674b4cc42ef410c188b790d26c1 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Thu, 1 May 2025 22:27:32 +0200 Subject: [PATCH 05/28] add osmium doc link to extract function --- R/extract.R | 30 ++++++++++++++++-------------- man/extract.Rd | 4 +--- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/R/extract.R b/R/extract.R index c7bc00b..9fb6a97 100644 --- a/R/extract.R +++ b/R/extract.R @@ -1,8 +1,6 @@ #' Create geographical extracts from an OSM file #' -#' Creates geographical extracts from an OSM data file or an OSM history file. -#' The geographical extent can be given either as a bounding box or as a -#' (multi)polygon. +#' This is a wrapper for [osmium extract](https://docs.osmcode.org/osmium/latest/osmium-extract.html). Creates geographical extracts from an OSM data file or an OSM history file. The geographical extent can be given either as a bounding box or as a (multi)polygon. #' #' @param input_path A string. The path to the OSM data/history file whose #' extent should be extracted from. Please see [file_formats] for a list of @@ -89,15 +87,17 @@ #' file.size(output_path) #' #' @export -extract <- function(input_path, - extent, - output_path, - strategy = c("complete_ways", "smart", "simple"), - overwrite = FALSE, - echo_cmd = FALSE, - echo = TRUE, - spinner = TRUE, - verbose = FALSE) { +extract <- function( + input_path, + extent, + output_path, + strategy = c("complete_ways", "smart", "simple"), + overwrite = FALSE, + echo_cmd = FALSE, + echo = TRUE, + spinner = TRUE, + verbose = FALSE +) { assert_osmium_is_installed() checkmate::assert_file_exists(input_path) @@ -204,10 +204,12 @@ check_extent <- function(extent) { } geometry_type <- as.character(sf::st_geometry_type(extent)) - if (! geometry_type %in% c("POLYGON", "MULTIPOLYGON")) { + if (!geometry_type %in% c("POLYGON", "MULTIPOLYGON")) { msg <- paste0( "Geometry type of sf object must be either POLYGON or MULTIPOLYGON. ", - "Found ", geometry_type, " instead" + "Found ", + geometry_type, + " instead" ) if (geometry_type == "GEOMETRYCOLLECTION") { diff --git a/man/extract.Rd b/man/extract.Rd index 77ef072..5acc49d 100644 --- a/man/extract.Rd +++ b/man/extract.Rd @@ -51,9 +51,7 @@ running command. Defaults to \code{FALSE}.} The normalized path to the output file. } \description{ -Creates geographical extracts from an OSM data file or an OSM history file. -The geographical extent can be given either as a bounding box or as a -(multi)polygon. +This is a wrapper for \href{https://docs.osmcode.org/osmium/latest/osmium-extract.html}{osmium extract}. Creates geographical extracts from an OSM data file or an OSM history file. The geographical extent can be given either as a bounding box or as a (multi)polygon. } \section{Strategies}{ From 6c3adb4bad72447044295992518002b3c7dae463 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Thu, 1 May 2025 23:59:21 +0200 Subject: [PATCH 06/28] rewrite file info, create a function to get available file info vars --- NAMESPACE | 2 +- R/file_info.R | 154 ++++++++++++++++++++++++-------- _pkgdown.yml | 1 + man/osm_file_info.Rd | 33 ++++--- man/osm_file_info_variables.Rd | 30 +++++++ tests/testthat/test-file_info.R | 67 ++++++++++++++ 6 files changed, 232 insertions(+), 55 deletions(-) create mode 100644 man/osm_file_info_variables.Rd diff --git a/NAMESPACE b/NAMESPACE index 37c2c0c..95dd218 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,10 +3,10 @@ export(extract) export(osm_cat) export(osm_file_info) +export(osm_file_info_variables) export(osm_get_bbox) export(osm_set_bbox) export(osm_sort) export(show_content) export(tags_filter) export(time_filter) -importFrom(RcppSimdJson,fparse) diff --git a/R/file_info.R b/R/file_info.R index c556faa..6fb655f 100644 --- a/R/file_info.R +++ b/R/file_info.R @@ -7,13 +7,15 @@ #' - \code{"json"}: the raw JSON string emitted by \code{--json} #' - \code{"list"}: the JSON parsed to an R list via \code{RcppSimdJson::fparse()} #' -#' @param input_path String. Path to the OSM file. +#' @param input_path String. Path to the OSM file. +#' #' @param extended A logical(1). Read the entire file (not just header)? Defaults to `FALSE`. Reading extended info may be slow, as it reads the entire file. This is also required if the file does not have a bounding box in the header, and you need to calculate it from the data. Specifically, to get or calculate the bounding box, you need can use the [osm_get_bbox] function. -#' @param get String or NULL. If non‐NULL, return only this one variable. -#' @param crc A logical. Calculate CRC32? Defaults to `FALSE`. -#' @param object_type Character vector or NULL. Only for \code{extended = TRUE}. Defaults to \code{NULL}. -#' @param output One of \code{"text"}, \code{"json"}, \code{"list"}. -#' Defaults to \code{"text"}. +#' +#' @param get String or NULL. If non-NULL, returns only this variable's value (using `--get=`). Must be a full CLI variable name (e.g. `file.name`, `header.option.generator`, `data.bbox`, or any other supported variable listed by `osmium fileinfo --show-variables`). A shorthand of `"generator"` is recognized and maps to `header.option.generator`. To get the list of valid variable names, use `osm_file_info_variables()`. +#' @param crc A logical. Calculate CRC32? Defaults to `FALSE`; automatically enabled if `output = "json"`. Ignored when `get` is used. +#' @param object_type Character vector or NULL. Only applies when `extended = TRUE`. +#' One or more of `"node","way","relation","changeset"`. +#' @param output One of `"text"`, `"json"`, or `"list"`. Defaults to `"text"`. #' @param echo_cmd A logical. Whether to print the Osmium command generated by the function call to the screen. #' Defaults to `FALSE`. #' @param echo A logical. Whether to print the standard output and error generated by the Osmium call to the screen. @@ -24,12 +26,12 @@ #' @param progress A logical. Whether to display a progress bar while running #' the command. Defaults to `FALSE`. #' -#' @return Depending on \code{output}: -#' * \code{"text"}: character scalar (the plain‐text dump). -#' * \code{"json"}: character scalar (the JSON string). -#' * \code{"list"}: an R list parsed via \code{RcppSimdJson::fparse()}. +#' @return +#' - If `output = "text"`: character scalar (plain-text dump). +#' - If `output = "json"`: character scalar (raw JSON string). +#' - If `output = "list"`: an R list parsed via `RcppSimdJson::fparse()`. +#' - If `get` is non-NULL: a single string `"="`. #' -#' @importFrom RcppSimdJson fparse #' #' @examplesIf identical(tolower(Sys.getenv("NOT_CRAN")), "true") #' pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") @@ -41,12 +43,11 @@ #' # 2. raw JSON #' j <- osm_file_info(pbf, output = "json") #' cat(substr(j, 1, 200), "...", "\n") -# +#' #' # 3. parsed list #' lst <- osm_file_info(pbf, output = "list") #' str(lst[c("file","header")]) #' -#' #' @export osm_file_info <- function( input_path, @@ -61,46 +62,88 @@ osm_file_info <- function( verbose = FALSE, progress = FALSE ) { + # Preconditions assert_osmium_is_installed() checkmate::assert_file_exists(input_path) checkmate::assert_logical(extended, any.missing = FALSE, len = 1) checkmate::assert_logical(crc, any.missing = FALSE, len = 1) checkmate::assert_logical(echo_cmd, any.missing = FALSE, len = 1) - checkmate::assert_logical(echo, any.missing = FALSE, len = 1) checkmate::assert_logical(spinner, any.missing = FALSE, len = 1) checkmate::assert_logical(verbose, any.missing = FALSE, len = 1) checkmate::assert_logical(progress, any.missing = FALSE, len = 1) + output <- match.arg(output) + # Auto-enable CRC for JSON + if (output == "json") crc <- TRUE + + # Handle get if (!is.null(get)) { checkmate::assert_string(get) - if (output == "list") { - warning("`get` overrides `output = 'list'`; returning plain text") - output <- "text" + vars <- .get_osm_fileinfo_variables( + echo = echo, + spinner = spinner, + echo_cmd = echo_cmd + ) + # full name? + if (grepl("\\.", get)) { + if (!(get %in% vars)) { + stop(sprintf( + "Unknown variable '%s'; run `osm_file_info_variables()` to see valid names.", + get + )) + } + get_full <- get + } else { + # final-component match + endings <- sub(".*\\.", "", vars) + matches <- vars[endings == get] + if (length(matches) == 1) { + get_full <- matches + } else if (length(matches) > 1) { + stop(sprintf( + "Ambiguous shorthand '%s'; matches: %s", + get, + paste(matches, collapse = ", ") + )) + } else { + stop(sprintf( + "Unknown shorthand '%s'; run `osm_file_info_variables()` to discover valid names.", + get + )) + } } + logs <- processx::run( + "osmium", + c("fileinfo", input_path, paste0("--get=", get_full)), + echo = echo, + spinner = spinner, + echo_cmd = echo_cmd + ) + val <- trimws(logs$stdout) + return(paste0(get, "=", val)) } + + # Validate object_type if (!is.null(object_type)) { checkmate::assert_subset( object_type, - choices = c("node", "way", "relation", "changeset") + c("node", "way", "relation", "changeset") ) - if (!extended) { - warning("`object_type` only applies when `extended = TRUE`") - } + if (!extended) + warning("`object_type` only applies when `extended = TRUE`.") } - # build the common bits of the command + # Build base args args <- c("fileinfo", input_path) if (extended) args <- c(args, "--extended") if (crc) args <- c(args, "--crc") - if (!is.null(get)) args <- c(args, paste0("--get=", get)) if (verbose) args <- c(args, "--verbose") args <- c(args, if (progress) "--progress" else "--no-progress") - if (!is.null(object_type) && extended) { + if (!is.null(object_type) && extended) args <- c(args, paste0("--object-type=", object_type)) - } - # 1) Plain text + # Dispatch if (output == "text") { logs <- processx::run( "osmium", @@ -109,26 +152,65 @@ osm_file_info <- function( spinner = spinner, echo_cmd = echo_cmd ) - # logs$stdout and logs$stderr are captured in memory; nothing is written to disk return(logs$stdout) } - # 2) JSON or list: add --json - args_json <- c(args, "--json") logs <- processx::run( "osmium", - args_json, + c(args, "--json"), echo = echo, spinner = spinner, echo_cmd = echo_cmd ) raw_json <- logs$stdout + if (output == "json") return(raw_json) + return(RcppSimdJson::fparse(raw_json)) +} - if (output == "json") { - return(raw_json) - } +#' List available variables for osm_file_info +#' +#' @description +#' Fetches the set of valid variable names supported by the underlying Osmium CLI. +#' +#' @inheritParams osm_file_info +#' @return Character vector of variable names. +#' +#' @examplesIf identical(tolower(Sys.getenv("NOT_CRAN")), "true") +#' vars <- osm_file_info_variables() +#' head(vars) +#' +#' @export +osm_file_info_variables <- function( + echo_cmd = FALSE, + spinner = TRUE, + echo = FALSE +) { + # Delegates to internal cache-aware fetcher + .get_osm_fileinfo_variables( + echo = echo, + spinner = spinner, + echo_cmd = echo_cmd + ) +} - # 3) output == "list": parse in-memory via RcppSimdJson::fparse() - # (no temp files, no disk writes) - return(RcppSimdJson::fparse(raw_json)) +# internal cache for variable names +.osm_fileinfo_cache <- new.env(parent = emptyenv()) +.get_osm_fileinfo_variables <- function( + echo = FALSE, + spinner = FALSE, + echo_cmd = FALSE +) { + if (is.null(.osm_fileinfo_cache$vars)) { + logs <- processx::run( + "osmium", + c("fileinfo", "--show-variables"), + echo = echo, + spinner = spinner, + echo_cmd = echo_cmd + ) + vars <- strsplit(logs$stdout, "\r?\n")[[1]] + vars <- trimws(vars) + .osm_fileinfo_cache$vars <- vars[nzchar(vars)] + } + .osm_fileinfo_cache$vars } diff --git a/_pkgdown.yml b/_pkgdown.yml index fe6fa07..2a751cb 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -11,6 +11,7 @@ reference: - title: "Functions" - contents: - osm_file_info + - osm_file_info_variables - extract - tags_filter - show_content diff --git a/man/osm_file_info.Rd b/man/osm_file_info.Rd index dd2faf4..e31071a 100644 --- a/man/osm_file_info.Rd +++ b/man/osm_file_info.Rd @@ -21,16 +21,17 @@ osm_file_info( \arguments{ \item{input_path}{String. Path to the OSM file.} -\item{extended}{A logical(1). Read the entire file (not just header)? Defaults to \code{FALSE}. Reading extended info may be slow, as it reads the entire file. This is also required if the file does not have a bounding box in the header, and you need to calculate it from the data. Specifically, to get or calculate the bounding box, you need can use the \link{osm_get_bbox} function.} +\item{extended}{A logical. Read the entire file (not just header)? Defaults to \code{FALSE}. +Required for full data and metadata sections.} -\item{get}{String or NULL. If non‐NULL, return only this one variable.} +\item{get}{String or NULL. If non-NULL, returns only this variable's value (using \verb{--get=}). Must be a full CLI variable name (e.g. \code{file.name}, \code{header.option.generator}, \code{data.bbox}, or any other supported variable listed by \verb{osmium fileinfo --show-variables}). A shorthand of \code{"generator"} is recognized and maps to \code{header.option.generator}. To get the list of valid variable names, use \code{osm_file_info_variables()}.} -\item{crc}{A logical. Calculate CRC32? Defaults to \code{FALSE}.} +\item{crc}{A logical. Calculate CRC32? Defaults to \code{FALSE}; automatically enabled if \code{output = "json"}. Ignored when \code{get} is used.} -\item{object_type}{Character vector or NULL. Only for \code{extended = TRUE}. Defaults to \code{NULL}.} +\item{object_type}{Character vector or NULL. Only applies when \code{extended = TRUE}. +One or more of \verb{"node","way","relation","changeset"}.} -\item{output}{One of \code{"text"}, \code{"json"}, \code{"list"}. -Defaults to \code{"text"}.} +\item{output}{One of \code{"text"}, \code{"json"}, or \code{"list"}. Defaults to \code{"text"}.} \item{echo_cmd}{A logical. Whether to print the Osmium command generated by the function call to the screen. Defaults to \code{FALSE}.} @@ -47,21 +48,17 @@ Defaults to \code{TRUE}.} the command. Defaults to \code{FALSE}.} } \value{ -Depending on \code{output}: \itemize{ -\item \code{"text"}: character scalar (the plain‐text dump). -\item \code{"json"}: character scalar (the JSON string). -\item \code{"list"}: an R list parsed via \code{RcppSimdJson::fparse()}. +\item If \code{output = "text"}: character scalar (plain-text dump). +\item If \code{output = "json"}: character scalar (raw JSON string). +\item If \code{output = "list"}: an R list parsed via \code{RcppSimdJson::fparse()}. +\item If \code{get} is non-NULL: a single string \code{"="}. } } \description{ -This is a wrapper for \href{https://docs.osmcode.org/osmium/latest/osmium-fileinfo.html}{osmium fileinfo}. -Returns OSM file & header info in three possible formats: -\itemize{ -\item \code{"text"}: the original plain‐text dump -\item \code{"json"}: the raw JSON string emitted by \code{--json} -\item \code{"list"}: the JSON parsed to an R list via \code{RcppSimdJson::fparse()} -} +Wrapper around \href{https://docs.osmcode.org/osmium/latest/osmium-fileinfo.html}{osmium fileinfo}. +Retrieves metadata from an OSM (or PBF) file in various formats, fetches a single variable, +or lists available variables. } \examples{ \dontshow{if (identical(tolower(Sys.getenv("NOT_CRAN")), "true")) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} @@ -74,9 +71,9 @@ cat(substr(txt, 1, 200), "...", "\n") # 2. raw JSON j <- osm_file_info(pbf, output = "json") cat(substr(j, 1, 200), "...", "\n") + # 3. parsed list lst <- osm_file_info(pbf, output = "list") str(lst[c("file","header")]) - \dontshow{\}) # examplesIf} } diff --git a/man/osm_file_info_variables.Rd b/man/osm_file_info_variables.Rd new file mode 100644 index 0000000..95c70e5 --- /dev/null +++ b/man/osm_file_info_variables.Rd @@ -0,0 +1,30 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/file_info.R +\name{osm_file_info_variables} +\alias{osm_file_info_variables} +\title{List available variables for osm_file_info} +\usage{ +osm_file_info_variables(echo_cmd = FALSE, spinner = TRUE, echo = FALSE) +} +\arguments{ +\item{echo_cmd}{A logical. Whether to print the Osmium command generated by the function call to the screen. +Defaults to \code{FALSE}.} + +\item{spinner}{A logical. Whether to show a reassuring spinner while the Osmium call is being executed. +Defaults to \code{TRUE}.} + +\item{echo}{A logical. Whether to print the standard output and error generated by the Osmium call to the screen. +Defaults to \code{TRUE}.} +} +\value{ +Character vector of variable names. +} +\description{ +Fetches the set of valid variable names supported by the underlying Osmium CLI. +} +\examples{ +\dontshow{if (identical(tolower(Sys.getenv("NOT_CRAN")), "true")) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +vars <- osm_file_info_variables() +head(vars) +\dontshow{\}) # examplesIf} +} diff --git a/tests/testthat/test-file_info.R b/tests/testthat/test-file_info.R index feb432e..715fd50 100644 --- a/tests/testthat/test-file_info.R +++ b/tests/testthat/test-file_info.R @@ -44,3 +44,70 @@ test_that("osm_file_info returns correct structure for cur.osm.pbf", { expect_type(t, "character") expect_true(grepl("Bounding boxes", t)) }) + +test_that("osm_file_info errors on missing file", { + expect_error( + osm_file_info("no_such_file.osm.pbf"), + "File does not exist", + ignore.case = TRUE + ) +}) + + +test_that("osm_file_info rejects bad output arg", { + pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + expect_error( + osm_file_info(pbf, output = "xml"), + "should be one of" + ) +}) + +test_that("osm_file_info get=boxes returns text with only generator", { + pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + # get generator in text mode + out <- osm_file_info(pbf, get = "boxes", output = "text") + expect_true(grepl("boxes=", out)) + expect_false(grepl("generator", out)) +}) + +test_that("osm_file_info crc=TRUE adds crc field", { + pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + j <- osm_file_info(pbf, output = "list", crc = TRUE, extended = TRUE) + expect_true("crc32" %in% names(j$data)) + expect_true(is.character(j$data$crc32)) +}) + + +test_that("osm_file_info extended=TRUE returns data element", { + pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + info_ext <- osm_file_info(pbf, output = "list", extended = TRUE) + expect_true(is.list(info_ext$data)) + expect_true(all(c("bbox", "timestamp", "count") %in% names(info_ext$data))) +}) + +test_that("osm_file_info warns if object_type used without extended", { + pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + expect_warning( + osm_file_info(pbf, output = "list", object_type = c("node", "way")), + "`object_type` only applies when `extended = TRUE`" + ) +}) + +test_that("osm_file_info respects progress argument", { + pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + # capture the command via echo_cmd so we can inspect it + expect_output( + osm_file_info(pbf, output = "text", echo_cmd = TRUE, progress = TRUE), + "--progress" + ) + expect_output( + osm_file_info(pbf, output = "text", echo_cmd = TRUE, progress = FALSE), + "--no-progress" + ) +}) + +test_that("osm_file_info_variables returns a list of variable names", { + vars <- osm_file_info_variables() + expect_type(vars, "character") + expect_true(all(c("file.name", "file.size") %in% vars)) +}) From ae31a82fcabf9673a3e0f196f0b59c7f00cbb4f9 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Fri, 2 May 2025 00:18:08 +0200 Subject: [PATCH 07/28] add more get bbox tests --- tests/testthat/test-get_bbox.R | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/testthat/test-get_bbox.R b/tests/testthat/test-get_bbox.R index 2c8e7dd..49c0efe 100644 --- a/tests/testthat/test-get_bbox.R +++ b/tests/testthat/test-get_bbox.R @@ -41,3 +41,51 @@ test_that("osm_get_bbox reads header bbox correctly (with tolerance)", { tolerance = 0.1 ) }) + + +pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + +test_that("osm_get_bbox errors on missing file", { + expect_error( + osm_get_bbox("no_such_file.osm"), + "File does not exist" # from assert_osmium_is_installed or file exists + ) +}) + +test_that("osm_get_bbox returns single bbox with correct values", { + expect_true(file.exists(pbf)) + + b1 <- osm_get_bbox(pbf) + expect_s3_class(b1, "bbox") + # Known header values from cur.osm.pbf + expect_equal( + as.numeric(b1), + c(-49.28, -25.44, -49.25, -25.41), + tolerance = 1e-3 + ) +}) + +test_that("osm_get_bbox(return_bbox_list = TRUE) returns a list of bboxes", { + L <- osm_get_bbox(pbf, return_bbox_list = TRUE) + expect_type(L, "list") + expect_length(L, 1) + expect_s3_class(L[[1]], "bbox") + expect_equal(unname(L[[1]]), unname(osm_get_bbox(pbf)), tolerance = 1e-6) +}) + +test_that("osm_get_bbox(calculate = TRUE) computes same bbox when header exists", { + b_hdr <- osm_get_bbox(pbf) + b_calc <- osm_get_bbox(pbf, calculate = TRUE) + expect_equal(unname(b_calc), unname(b_hdr), tolerance = 1e-6) +}) + + +# Parameter validation +test_that("osm_get_bbox rejects non-logical inputs", { + expect_error(osm_get_bbox(pbf, calculate = "yes"), "Assertion on") + expect_error(osm_get_bbox(pbf, return_bbox_list = NA), "Assertion on") + expect_error(osm_get_bbox(pbf, echo_cmd = 1), "Assertion on") + expect_error(osm_get_bbox(pbf, spinner = "no"), "Assertion on") + expect_error(osm_get_bbox(pbf, verbose = NULL), "Assertion on") + expect_error(osm_get_bbox(pbf, progress = "maybe"), "Assertion on") +}) From 3866fb04a48d15127acb69ba45927a004a3c4708 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Fri, 2 May 2025 00:21:09 +0200 Subject: [PATCH 08/28] add a test for multiple bboxes --- tests/testthat/test-get_bbox.R | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/testthat/test-get_bbox.R b/tests/testthat/test-get_bbox.R index 49c0efe..6c4764d 100644 --- a/tests/testthat/test-get_bbox.R +++ b/tests/testthat/test-get_bbox.R @@ -89,3 +89,33 @@ test_that("osm_get_bbox rejects non-logical inputs", { expect_error(osm_get_bbox(pbf, verbose = NULL), "Assertion on") expect_error(osm_get_bbox(pbf, progress = "maybe"), "Assertion on") }) + +# Test multiple header bounding boxes +test_that("osm_get_bbox handles multiple header boxes", { + # Create a minimal OSM XML file with two entries + xml_content <- c( + '', + ' ', + ' ', + '' + ) + tmp_osm <- tempfile(fileext = ".osm") + writeLines(xml_content, tmp_osm) + + # Expect an error if return_bbox_list = FALSE (default) + expect_error( + osm_get_bbox(tmp_osm), + "Found 2 header bounding boxes; to retrieve them all, retry with return_bbox_list = TRUE." + ) + + # With return_bbox_list = TRUE, should return two bbox objects + L2 <- osm_get_bbox(tmp_osm, return_bbox_list = TRUE) + expect_type(L2, "list") + expect_length(L2, 2) + b1 <- as.numeric(L2[[1]]) + b2 <- as.numeric(L2[[2]]) + + # Check numeric contents: xmin, ymin, xmax, ymax + expect_equal(b1, c(1, 0, 3, 2), tolerance = 1e-8) + expect_equal(b2, c(5, 4, 7, 6), tolerance = 1e-8) +}) From 119e4b0c58ecfb2733a0c17fa2b38eeb3e75da04 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Fri, 2 May 2025 17:52:38 +0200 Subject: [PATCH 09/28] add draft merge --- R/merge.R | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 R/merge.R diff --git a/R/merge.R b/R/merge.R new file mode 100644 index 0000000..fd0ba5b --- /dev/null +++ b/R/merge.R @@ -0,0 +1,120 @@ +#' Merge several sorted OSM files +#' +#' This is a wrapper for [osmium merge](https://docs.osmcode.org/osmium/latest/osmium-merge.html). +#' Merges multiple sorted OSM files into a single output, removing duplicate objects. +#' +#' @param input_paths Character vector of paths to one or more existing, sorted OSM files. +#' @param output_path String. Path to the file where the merged result will be written. +#' @param with_history A logical. If `TRUE`, suppress warnings about multiple versions +#' of the same object. Defaults to `FALSE`. +#' @param input_format Optional string. Force input format (e.g. `"osm"`, `"pbf"`). If `NULL`, autodetect. +#' @param output_format Optional string. Force output format (e.g. `"osm"`, `"pbf"`). If `NULL`, autodetect. +#' @param fsync A logical. If `TRUE`, call `fsync` after writing to force flushing buffers. Defaults to `FALSE`. +#' @param generator Optional string. Name/version for the `generator` header. Defaults to Osmium’s own. +#' @param overwrite A logical. If `TRUE`, allow overwriting an existing `output_path`. Defaults to `FALSE`. +#' @param output_header Named character vector of additional header options. Each name is the header key, +#' each value is the header value. A zero‐length value (`""`) writes `OPTION!`. Defaults to `NULL`. +#' @param echo_cmd A logical. Whether to print the Osmium command generated by the function call to the screen. +#' Defaults to `FALSE`. +#' @param echo A logical. Whether to print the standard output and error generated by the Osmium call to the screen. +#' Defaults to `TRUE`. +#' @param spinner A logical. Whether to show a reassuring spinner while the Osmium call is being executed. +#' Defaults to `TRUE`. +#' @param verbose A logical. Whether to display detailed information on the running command. Defaults to `FALSE`. +#' @param progress A logical. Whether to display a progress bar while running +#' the command. Defaults to `FALSE`. +#' +#' @return Invisibly returns the normalized path to `output_path`. +#' +#' @export +osm_merge <- function( + input_paths, + output_path, + with_history = FALSE, + input_format = NULL, + output_format = NULL, + fsync = FALSE, + generator = NULL, + overwrite = FALSE, + output_header = NULL, + echo_cmd = FALSE, + echo = TRUE, + spinner = TRUE, + verbose = FALSE, + progress = FALSE +) { + assert_osmium_is_installed() + + ## Validate inputs + checkmate::assert_character(input_paths, any.missing = FALSE, min.len = 1) + for (p in input_paths) { + checkmate::assert_file_exists(p, access = "r") + } + checkmate::assert_string(output_path) + + checkmate::assert_logical(with_history, len = 1) + if (!is.null(input_format)) checkmate::assert_string(input_format) + if (!is.null(output_format)) checkmate::assert_string(output_format) + checkmate::assert_logical(fsync, len = 1) + if (!is.null(generator)) checkmate::assert_string(generator) + checkmate::assert_logical(overwrite, len = 1) + if (!is.null(output_header)) { + checkmate::assert_named(output_header, type = "unique") + checkmate::assert_character(output_header) + } + checkmate::assert_logical(echo_cmd, len = 1) + checkmate::assert_logical(echo, len = 1) + checkmate::assert_logical(spinner, len = 1) + checkmate::assert_logical(verbose, len = 1) + checkmate::assert_logical(progress, len = 1) + + ## Build flags + hist_flag <- if (with_history) "--with-history" else character() + input_fmt_flag <- if (!is.null(input_format)) + paste0("--input-format=", input_format) else character() + output_fmt_flag <- if (!is.null(output_format)) + paste0("--output-format=", output_format) else character() + fsync_flag <- if (fsync) "--fsync" else character() + gen_flag <- if (!is.null(generator)) paste0("--generator=", generator) else + character() + overwrite_flag <- if (overwrite) "--overwrite" else character() + header_flags <- if (!is.null(output_header)) { + unname(vapply( + names(output_header), + function(nm) { + val <- output_header[[nm]] + hdr <- if (nzchar(val)) paste0(nm, "=", val) else paste0(nm, "!") + paste0("--output-header=", hdr) + }, + character(1) + )) + } else character() + verbose_flag <- if (verbose) "--verbose" else character() + progress_flag <- if (progress) "--progress" else "--no-progress" + output_flag <- paste0("--output=", output_path) + + ## Assemble and run + args <- c( + "merge", + input_paths, + hist_flag, + input_fmt_flag, + output_fmt_flag, + fsync_flag, + gen_flag, + output_flag, + overwrite_flag, + header_flags, + verbose_flag, + progress_flag + ) + processx::run( + "osmium", + args, + echo = echo, + spinner = spinner, + echo_cmd = echo_cmd + ) + + invisible(normalizePath(output_path)) +} From f8fb0cc8d5b88f54707435b569936995011306a2 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Fri, 2 May 2025 17:52:46 +0200 Subject: [PATCH 10/28] draft check refs --- R/check-refs.R | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 R/check-refs.R diff --git a/R/check-refs.R b/R/check-refs.R new file mode 100644 index 0000000..85335ef --- /dev/null +++ b/R/check-refs.R @@ -0,0 +1,73 @@ +#' Check referential integrity of an OSM data file +#' +#' This is a wrapper for [osmium check-refs](https://docs.osmcode.org/osmium/latest/osmium-check-refs.html). +#' Checks that all nodes referenced by ways (and optionally relations) are present. +#' +#' @param input_path String. Path to an existing OSM data file (not history or change file). +#' @param show_ids Logical(1). If `TRUE`, print all missing IDs to stdout. Defaults to `FALSE`. +#' @param check_relations Logical(1). If `TRUE`, also check relations’ references. Defaults to `FALSE`. +#' @param input_format Optional string. Force input format (e.g. `"osm"`, `"pbf"`). If `NULL`, autodetect. +#' @param echo_cmd Logical(1). Whether to print the generated Osmium command. Defaults to `FALSE`. +#' @param echo Logical(1). Whether to print Osmium’s stdout/stderr. Defaults to `FALSE`. +#' @param spinner Logical(1). Whether to show a spinner during execution. Defaults to `TRUE`. +#' @param verbose Logical(1). Whether to pass `--verbose` for detailed logging. Defaults to `FALSE`. +#' @param progress Logical(1). Whether to show a progress bar (`--progress` / `--no-progress`). Defaults to `FALSE`. +#' +#' @return A character scalar: the plain-text output from `osmium check-refs`. +#' +#' @export +osm_check_refs <- function( + input_path, + show_ids = FALSE, + check_relations = FALSE, + input_format = NULL, + echo_cmd = FALSE, + echo = FALSE, + spinner = TRUE, + verbose = FALSE, + progress = FALSE +) { + assert_osmium_is_installed() + + ## Validate inputs + checkmate::assert_string(input_path) + checkmate::assert_file_exists(input_path, access = "r") + checkmate::assert_logical(show_ids, len = 1) + checkmate::assert_logical(check_relations, len = 1) + if (!is.null(input_format)) checkmate::assert_string(input_format) + checkmate::assert_logical(echo_cmd, len = 1) + checkmate::assert_logical(echo, len = 1) + checkmate::assert_logical(spinner, len = 1) + checkmate::assert_logical(verbose, len = 1) + checkmate::assert_logical(progress, len = 1) + + ## Build flags + ids_flag <- if (show_ids) "--show-ids" else character() + rels_flag <- if (check_relations) "--check-relations" else character() + input_fmt <- if (!is.null(input_format)) + paste0("--input-format=", input_format) else character() + verbose_flag <- if (verbose) "--verbose" else character() + progress_flag <- if (progress) "--progress" else "--no-progress" + + ## Assemble and run + args <- c( + "check-refs", + ids_flag, + rels_flag, + input_fmt, + verbose_flag, + progress_flag, + input_path + ) + + logs <- processx::run( + "osmium", + args, + echo = echo, + spinner = spinner, + echo_cmd = echo_cmd + ) + + ## Return the output text + logs$stdout +} From bfc044c2e7dc9c5406b59e9c176a2b3cb8253deb Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Sat, 3 May 2025 20:32:42 +0200 Subject: [PATCH 11/28] add check_refs with tests, parser and print class --- NAMESPACE | 3 + R/check-refs.R | 73 ---------- R/check_refs.R | 234 +++++++++++++++++++++++++++++++ man/osm_check_refs.Rd | 44 ++++++ man/parse_osm_check_refs_log.Rd | 41 ++++++ man/print.osm_check_refs_log.Rd | 16 +++ tests/testthat/test-check_refs.R | 94 +++++++++++++ 7 files changed, 432 insertions(+), 73 deletions(-) delete mode 100644 R/check-refs.R create mode 100644 R/check_refs.R create mode 100644 man/osm_check_refs.Rd create mode 100644 man/parse_osm_check_refs_log.Rd create mode 100644 man/print.osm_check_refs_log.Rd create mode 100644 tests/testthat/test-check_refs.R diff --git a/NAMESPACE b/NAMESPACE index 95dd218..33c68a2 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,10 +1,13 @@ # Generated by roxygen2: do not edit by hand +S3method(print,osm_check_refs_log) export(extract) export(osm_cat) +export(osm_check_refs) export(osm_file_info) export(osm_file_info_variables) export(osm_get_bbox) +export(osm_merge) export(osm_set_bbox) export(osm_sort) export(show_content) diff --git a/R/check-refs.R b/R/check-refs.R deleted file mode 100644 index 85335ef..0000000 --- a/R/check-refs.R +++ /dev/null @@ -1,73 +0,0 @@ -#' Check referential integrity of an OSM data file -#' -#' This is a wrapper for [osmium check-refs](https://docs.osmcode.org/osmium/latest/osmium-check-refs.html). -#' Checks that all nodes referenced by ways (and optionally relations) are present. -#' -#' @param input_path String. Path to an existing OSM data file (not history or change file). -#' @param show_ids Logical(1). If `TRUE`, print all missing IDs to stdout. Defaults to `FALSE`. -#' @param check_relations Logical(1). If `TRUE`, also check relations’ references. Defaults to `FALSE`. -#' @param input_format Optional string. Force input format (e.g. `"osm"`, `"pbf"`). If `NULL`, autodetect. -#' @param echo_cmd Logical(1). Whether to print the generated Osmium command. Defaults to `FALSE`. -#' @param echo Logical(1). Whether to print Osmium’s stdout/stderr. Defaults to `FALSE`. -#' @param spinner Logical(1). Whether to show a spinner during execution. Defaults to `TRUE`. -#' @param verbose Logical(1). Whether to pass `--verbose` for detailed logging. Defaults to `FALSE`. -#' @param progress Logical(1). Whether to show a progress bar (`--progress` / `--no-progress`). Defaults to `FALSE`. -#' -#' @return A character scalar: the plain-text output from `osmium check-refs`. -#' -#' @export -osm_check_refs <- function( - input_path, - show_ids = FALSE, - check_relations = FALSE, - input_format = NULL, - echo_cmd = FALSE, - echo = FALSE, - spinner = TRUE, - verbose = FALSE, - progress = FALSE -) { - assert_osmium_is_installed() - - ## Validate inputs - checkmate::assert_string(input_path) - checkmate::assert_file_exists(input_path, access = "r") - checkmate::assert_logical(show_ids, len = 1) - checkmate::assert_logical(check_relations, len = 1) - if (!is.null(input_format)) checkmate::assert_string(input_format) - checkmate::assert_logical(echo_cmd, len = 1) - checkmate::assert_logical(echo, len = 1) - checkmate::assert_logical(spinner, len = 1) - checkmate::assert_logical(verbose, len = 1) - checkmate::assert_logical(progress, len = 1) - - ## Build flags - ids_flag <- if (show_ids) "--show-ids" else character() - rels_flag <- if (check_relations) "--check-relations" else character() - input_fmt <- if (!is.null(input_format)) - paste0("--input-format=", input_format) else character() - verbose_flag <- if (verbose) "--verbose" else character() - progress_flag <- if (progress) "--progress" else "--no-progress" - - ## Assemble and run - args <- c( - "check-refs", - ids_flag, - rels_flag, - input_fmt, - verbose_flag, - progress_flag, - input_path - ) - - logs <- processx::run( - "osmium", - args, - echo = echo, - spinner = spinner, - echo_cmd = echo_cmd - ) - - ## Return the output text - logs$stdout -} diff --git a/R/check_refs.R b/R/check_refs.R new file mode 100644 index 0000000..c5efcf7 --- /dev/null +++ b/R/check_refs.R @@ -0,0 +1,234 @@ +#' Check referential integrity of an OSM data file +#' +#' This is a wrapper for [osmium check-refs](https://docs.osmcode.org/osmium/latest/osmium-check-refs.html). +#' Checks that all nodes referenced by ways (and optionally relations) are present. +#' +#' @param input_path String. Path to an existing OSM data file (not history or change file). +#' @param show_ids A logical. If `TRUE`, print all missing IDs to stdout. Defaults to `FALSE`. +#' @param check_relations A logical. If `TRUE`, also check relations’ references. Defaults to `FALSE`. +#' @param input_format Optional string. Force input format (e.g. "osm", "pbf"). If `NULL`, autodetect. +#' @param echo_cmd A logical. Whether to print the generated Osmium command. Defaults to `FALSE`. +#' @param echo A logical. Whether to print Osmium’s stdout/stderr. Defaults to `TRUE`. +#' @param spinner A logical. Whether to show a spinner during execution. Defaults to `TRUE`. +#' @param verbose A logical. Whether to pass `--verbose` to Osmium for detailed logging. Defaults to `TRUE`. +#' @param progress A logical. Whether to pass `--progress` / `--no-progress`. Defaults to `FALSE`. +#' +#' @return A character scalar: the plain-text output from `osmium check-refs`, invisibly returned if successful. +#' @export +osm_check_refs <- function( + input_path, + show_ids = FALSE, + check_relations = FALSE, + input_format = NULL, + echo_cmd = FALSE, + echo = TRUE, + spinner = TRUE, + verbose = TRUE, + progress = FALSE +) { + assert_osmium_is_installed() + checkmate::assert_string(input_path) + checkmate::assert_file_exists(input_path, access = "r") + checkmate::assert_flag(show_ids) + checkmate::assert_flag(check_relations) + if (!is.null(input_format)) checkmate::assert_string(input_format) + checkmate::assert_flag(echo_cmd) + checkmate::assert_flag(echo) + checkmate::assert_flag(spinner) + checkmate::assert_flag(verbose) + checkmate::assert_flag(progress) + + args <- c( + "check-refs", + if (show_ids) "--show-ids", + if (check_relations) "--check-relations", + if (!is.null(input_format)) paste0("--input-format=", input_format), + if (verbose) "--verbose", + if (progress) "--progress" else "--no-progress", + input_path + ) + + logs <- processx::run( + "osmium", + args, + echo = echo, + spinner = spinner, + echo_cmd = echo_cmd, + error_on_status = FALSE + ) + + parsed <- parse_osm_check_refs_log( + log_text = logs$stderr, + status_code = logs$status, + verbose = verbose, + request_check_rels = check_relations, + request_verbose = verbose, + request_input_file = input_path + ) + invisible(parsed) +} + +#' Parse Osmium `check-refs` command output +#' +#' This function takes the raw stderr text from `osmium check-refs` and an +#' optional status code, and returns a structured object of class +#' `osm_check_refs_log` with: +#' - status_code: integer exit status +#' - raw: list of captured header/footer lines +#' - metadata: version, option details, plus the original `verbose` and `check_relations` settings +#' - summary: a data.frame of counts (present and missing references) +#' +#' @param log_text Character scalar: stderr output from `osmium check-refs` +#' @param status_code Integer or NULL: exit status from the command +#' @param verbose Logical: was `osmium` run with --verbose? Defaults to FALSE. +#' @param request_check_rels Logical: the original `check_relations` setting +#' @param request_verbose Logical: the original `verbose` setting +#' @return An object of class `osm_check_refs_log` +#' @keywords internal +#' +parse_osm_check_refs_log <- function( + log_text, + status_code = NULL, + verbose = FALSE, + request_check_rels = FALSE, + request_verbose = FALSE, + request_input_file = NULL +) { + lines <- strsplit(log_text, "\n", fixed = TRUE)[[1]] + lines <- sub("\r$", "", lines) + lines <- lines[nzchar(lines)] + lines <- grep("^\\[.*%.*\\]", lines, invert = TRUE, value = TRUE) + + clean <- function(x) trimws(sub("^\\[.*?\\]\\s*", "", x)) + cl <- vapply(lines, clean, "") + + raw <- list( + version_lines = cl[grepl("^(osmium version|libosmium version)", cl)], + summary_line = cl[grepl( + "^There are [0-9]+ nodes, [0-9]+ ways, and [0-9]+ relations", + cl + )], + missing_lines = cl[grepl("missing: *[0-9]+", cl)], + memory_line = cl[grepl("^Memory used for indexes:", cl)] + ) + + # totals + m_tot <- regexec( + "([0-9]+) nodes, ([0-9]+) ways, and ([0-9]+) relations", + raw$summary_line + ) + nums <- as.integer(regmatches(raw$summary_line, m_tot)[[1]][2:4]) + totals <- setNames(nums, c("nodes", "ways", "relations")) + + # missing + patterns <- c( + nodes_in_ways = "^Nodes.*in ways.*missing: *([0-9]+)", + nodes_in_relations = "^Nodes.*in relations.*missing: *([0-9]+)", + ways_in_relations = "^Ways.*in relations.*missing: *([0-9]+)", + relations_in_relations = "^Relations.*in relations.*missing: *([0-9]+)" + ) + missing <- vapply( + patterns, + function(p) { + m <- grep(p, raw$missing_lines, value = TRUE) + if (!length(m)) return(0L) + as.integer(sub(p, "\\1", m[1])) + }, + integer(1) + ) + + df <- data.frame( + description = c(names(totals), names(missing)), + count = c(as.integer(totals), missing), + stringsAsFactors = FALSE + ) + + metadata <- list( + request_check_rels = request_check_rels, + request_verbose = request_verbose, + request_input_file = request_input_file + ) + + res <- list( + status_code = status_code, + raw = raw, + metadata = metadata, + summary = df + ) + class(res) <- "osm_check_refs_log" + res +} + +#' Print method for `osm_check_refs_log` objects +#' +#' Displays parsed results in a format resembling `osmium check-refs` native output. +#' @param x An `osm_check_refs_log` object +#' @param ... Additional args (ignored) +#' @export +#' @method print osm_check_refs_log +print.osm_check_refs_log <- function(x, ...) { + md <- x$metadata + df <- x$summary + raw <- x$raw + + # file name + if (!is.null(md$request_input_file)) { + cat(sprintf("File checked: %s\n\n", md$request_input_file)) + } + + # version info + if (length(raw$version_lines)) { + cat(paste(raw$version_lines, collapse = "\n"), "\n") + } + + # summary + tot <- df$count[df$description %in% c("nodes", "ways", "relations")] + cat(sprintf( + "\nThere are %d nodes, %d ways, and %d relations in this file.\n", + tot[1], + tot[2], + tot[3] + )) + + # missing in ways + nw <- df$count[df$description == "nodes_in_ways"] + cat(sprintf("Nodes in ways missing: %d\n", nw)) + + # missing in relations, if requested + if (isTRUE(md$request_check_rels)) { + nr <- df$count[df$description == "nodes_in_relations"] + wr <- df$count[df$description == "ways_in_relations"] + rr <- df$count[df$description == "relations_in_relations"] + cat(sprintf("Nodes in relations missing: %d\n", nr)) + cat(sprintf("Ways in relations missing: %d\n", wr)) + cat(sprintf("Relations in relations missing: %d\n", rr)) + } + + # memory usage + if (length(raw$memory_line)) { + cat(raw$memory_line, "\n") + } + + # note if relations were not checked + if (!isTRUE(md$request_check_rels)) { + cat( + "\nNote: relations were not checked; relation‐level missing counts are omitted.\n You can run `osm_check_refs()` with `check_relations=TRUE` to check relations.\n" + ) + } + + # Osmium exit status (with human-readable meaning) + if (!is.null(x$status_code)) { + code <- x$status_code + meaning <- switch( + as.character(code), + "0" = "OK = All references satisfied.", + "1" = "Error processing data or some references were not satisfied.\nThis is normal for Geofabrik extracts when you run with `check_relations=TRUE`.", + "2" = "Problem with command line arguments.", + "unexpected status code" + ) + cat(sprintf("\nOsmium status code: %d\n", code)) + cat(sprintf("Osmium status meaning: %s\n", meaning)) + } + + invisible(x) +} diff --git a/man/osm_check_refs.Rd b/man/osm_check_refs.Rd new file mode 100644 index 0000000..677bddc --- /dev/null +++ b/man/osm_check_refs.Rd @@ -0,0 +1,44 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/check_refs.R +\name{osm_check_refs} +\alias{osm_check_refs} +\title{Check referential integrity of an OSM data file} +\usage{ +osm_check_refs( + input_path, + show_ids = FALSE, + check_relations = FALSE, + input_format = NULL, + echo_cmd = FALSE, + echo = TRUE, + spinner = TRUE, + verbose = TRUE, + progress = FALSE +) +} +\arguments{ +\item{input_path}{String. Path to an existing OSM data file (not history or change file).} + +\item{show_ids}{A logical. If \code{TRUE}, print all missing IDs to stdout. Defaults to \code{FALSE}.} + +\item{check_relations}{A logical. If \code{TRUE}, also check relations’ references. Defaults to \code{FALSE}.} + +\item{input_format}{Optional string. Force input format (e.g. "osm", "pbf"). If \code{NULL}, autodetect.} + +\item{echo_cmd}{A logical. Whether to print the generated Osmium command. Defaults to \code{FALSE}.} + +\item{echo}{A logical. Whether to print Osmium’s stdout/stderr. Defaults to \code{TRUE}.} + +\item{spinner}{A logical. Whether to show a spinner during execution. Defaults to \code{TRUE}.} + +\item{verbose}{A logical. Whether to pass \code{--verbose} to Osmium for detailed logging. Defaults to \code{TRUE}.} + +\item{progress}{A logical. Whether to pass \code{--progress} / \code{--no-progress}. Defaults to \code{FALSE}.} +} +\value{ +A character scalar: the plain-text output from \verb{osmium check-refs}, invisibly returned if successful. +} +\description{ +This is a wrapper for \href{https://docs.osmcode.org/osmium/latest/osmium-check-refs.html}{osmium check-refs}. +Checks that all nodes referenced by ways (and optionally relations) are present. +} diff --git a/man/parse_osm_check_refs_log.Rd b/man/parse_osm_check_refs_log.Rd new file mode 100644 index 0000000..a4270ae --- /dev/null +++ b/man/parse_osm_check_refs_log.Rd @@ -0,0 +1,41 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/check_refs.R +\name{parse_osm_check_refs_log} +\alias{parse_osm_check_refs_log} +\title{Parse Osmium \code{check-refs} command output} +\usage{ +parse_osm_check_refs_log( + log_text, + status_code = NULL, + verbose = FALSE, + request_check_rels = FALSE, + request_verbose = FALSE, + request_input_file = NULL +) +} +\arguments{ +\item{log_text}{Character scalar: stderr output from \verb{osmium check-refs}} + +\item{status_code}{Integer or NULL: exit status from the command} + +\item{verbose}{Logical: was \code{osmium} run with --verbose? Defaults to FALSE.} + +\item{request_check_rels}{Logical: the original \code{check_relations} setting} + +\item{request_verbose}{Logical: the original \code{verbose} setting} +} +\value{ +An object of class \code{osm_check_refs_log} +} +\description{ +This function takes the raw stderr text from \verb{osmium check-refs} and an +optional status code, and returns a structured object of class +\code{osm_check_refs_log} with: +\itemize{ +\item status_code: integer exit status +\item raw: list of captured header/footer lines +\item metadata: version, option details, plus the original \code{verbose} and \code{check_relations} settings +\item summary: a data.frame of counts (present and missing references) +} +} +\keyword{internal} diff --git a/man/print.osm_check_refs_log.Rd b/man/print.osm_check_refs_log.Rd new file mode 100644 index 0000000..d29ce8f --- /dev/null +++ b/man/print.osm_check_refs_log.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/check_refs.R +\name{print.osm_check_refs_log} +\alias{print.osm_check_refs_log} +\title{Print method for \code{osm_check_refs_log} objects} +\usage{ +\method{print}{osm_check_refs_log}(x, ...) +} +\arguments{ +\item{x}{An \code{osm_check_refs_log} object} + +\item{...}{Additional args (ignored)} +} +\description{ +Displays parsed results in a format resembling \verb{osmium check-refs} native output. +} diff --git a/tests/testthat/test-check_refs.R b/tests/testthat/test-check_refs.R new file mode 100644 index 0000000..01e5e99 --- /dev/null +++ b/tests/testthat/test-check_refs.R @@ -0,0 +1,94 @@ +test_that("osm_check_refs works with check_relations = FALSE", { + pbf_path <- system.file("extdata/cur.osm.pbf", package = "rosmium") + x <- osm_check_refs( + pbf_path, + verbose = TRUE, + echo = FALSE, + check_relations = FALSE + ) + + ## structure + expect_s3_class(x, "osm_check_refs_log") + expect_equal(x$status_code, 0L) + expect_false(x$metadata$request_check_rels) + expect_true(x$metadata$request_verbose) + expect_equal(x$metadata$request_input_file, pbf_path) + + ## summary counts + df <- x$summary + expect_equal(df$count[df$description == "nodes"], 40185L) + expect_equal(df$count[df$description == "ways"], 6884L) + expect_equal(df$count[df$description == "relations"], 430L) + expect_equal(df$count[df$description == "nodes_in_ways"], 0L) + expect_equal(df$count[df$description == "nodes_in_relations"], 0L) + expect_equal(df$count[df$description == "ways_in_relations"], 0L) + expect_equal(df$count[df$description == "relations_in_relations"], 0L) + + ## raw memory line + expect_true(grepl("Memory used for indexes: 1344 MBytes", x$raw$memory_line)) + + ## printed output + out <- capture.output(print(x)) + # file name + expect_true(grepl("^File checked: .*/cur\\.osm\\.pbf$", out[1])) + # version block present + expect_true(any(grepl("^osmium version 1\\.18\\.0$", out))) + # summary line + expect_true(any(grepl( + "^There are 40185 nodes, 6884 ways, and 430 relations in this file\\.$", + out + ))) + # always nodes_in_ways + expect_true(any(grepl("^Nodes in ways missing: 0$", out))) + # no relations lines + expect_false(any(grepl("in relations missing:", out))) + # memory usage printed + expect_true(any(grepl("^Memory used for indexes: 1344 MBytes", out))) + # note about relations not checked + expect_true(any(grepl( + "relations were not checked", + out + ))) + # status code line + expect_true(any(grepl( + "^Osmium status code: 0", + out + ))) +}) + +test_that("osm_check_refs works with check_relations = TRUE", { + pbf_path <- system.file("extdata/cur.osm.pbf", package = "rosmium") + x <- osm_check_refs( + pbf_path, + verbose = TRUE, + echo = FALSE, + check_relations = TRUE + ) + + ## structure + expect_s3_class(x, "osm_check_refs_log") + expect_equal(x$status_code, 1L) + expect_true(x$metadata$request_check_rels) + + ## summary counts + df <- x$summary + expect_equal(df$count[df$description == "nodes"], 40185L) + expect_equal(df$count[df$description == "ways"], 6884L) + expect_equal(df$count[df$description == "relations"], 430L) + expect_equal(df$count[df$description == "nodes_in_ways"], 0L) + expect_equal(df$count[df$description == "nodes_in_relations"], 433L) + expect_equal(df$count[df$description == "ways_in_relations"], 4258L) + expect_equal(df$count[df$description == "relations_in_relations"], 0L) + + ## printed output + out <- capture.output(print(x)) + # nodes_in_relations line + expect_true(any(grepl("^Nodes in relations missing: 433$", out))) + expect_true(any(grepl("^Ways in relations missing: 4258$", out))) + expect_true(any(grepl("^Relations in relations missing: 0$", out))) + # status code 1 meaning + expect_true(any(grepl( + "^Osmium status code: 1", + out + ))) +}) From ff05c36cf860e33e05e5fcab6dc4b6b3b838c88d Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Sat, 3 May 2025 20:35:45 +0200 Subject: [PATCH 12/28] get rid of setNames --- R/check_refs.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/check_refs.R b/R/check_refs.R index c5efcf7..3205b86 100644 --- a/R/check_refs.R +++ b/R/check_refs.R @@ -118,7 +118,8 @@ parse_osm_check_refs_log <- function( raw$summary_line ) nums <- as.integer(regmatches(raw$summary_line, m_tot)[[1]][2:4]) - totals <- setNames(nums, c("nodes", "ways", "relations")) + names(nums) <- c("nodes", "ways", "relations") + totals <- nums # missing patterns <- c( From 07374e141dfd96995c8bee001238db16276f108d Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Sat, 3 May 2025 20:41:14 +0200 Subject: [PATCH 13/28] fixes in check_refs --- R/check_refs.R | 6 +++--- man/osm_check_refs.Rd | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/R/check_refs.R b/R/check_refs.R index 3205b86..f063bdc 100644 --- a/R/check_refs.R +++ b/R/check_refs.R @@ -5,10 +5,10 @@ #' #' @param input_path String. Path to an existing OSM data file (not history or change file). #' @param show_ids A logical. If `TRUE`, print all missing IDs to stdout. Defaults to `FALSE`. -#' @param check_relations A logical. If `TRUE`, also check relations’ references. Defaults to `FALSE`. +#' @param check_relations A logical. If `TRUE`, also check relations references. Defaults to `FALSE`. #' @param input_format Optional string. Force input format (e.g. "osm", "pbf"). If `NULL`, autodetect. #' @param echo_cmd A logical. Whether to print the generated Osmium command. Defaults to `FALSE`. -#' @param echo A logical. Whether to print Osmium’s stdout/stderr. Defaults to `TRUE`. +#' @param echo A logical. Whether to print Osmium's stdout/stderr. Defaults to `TRUE`. #' @param spinner A logical. Whether to show a spinner during execution. Defaults to `TRUE`. #' @param verbose A logical. Whether to pass `--verbose` to Osmium for detailed logging. Defaults to `TRUE`. #' @param progress A logical. Whether to pass `--progress` / `--no-progress`. Defaults to `FALSE`. @@ -213,7 +213,7 @@ print.osm_check_refs_log <- function(x, ...) { # note if relations were not checked if (!isTRUE(md$request_check_rels)) { cat( - "\nNote: relations were not checked; relation‐level missing counts are omitted.\n You can run `osm_check_refs()` with `check_relations=TRUE` to check relations.\n" + "\nNote: relations were not checked. Relation-level missing counts are omitted.\n You can run `osm_check_refs()` with `check_relations=TRUE` to check relations.\n" ) } diff --git a/man/osm_check_refs.Rd b/man/osm_check_refs.Rd index 677bddc..7144834 100644 --- a/man/osm_check_refs.Rd +++ b/man/osm_check_refs.Rd @@ -21,13 +21,13 @@ osm_check_refs( \item{show_ids}{A logical. If \code{TRUE}, print all missing IDs to stdout. Defaults to \code{FALSE}.} -\item{check_relations}{A logical. If \code{TRUE}, also check relations’ references. Defaults to \code{FALSE}.} +\item{check_relations}{A logical. If \code{TRUE}, also check relations references. Defaults to \code{FALSE}.} \item{input_format}{Optional string. Force input format (e.g. "osm", "pbf"). If \code{NULL}, autodetect.} \item{echo_cmd}{A logical. Whether to print the generated Osmium command. Defaults to \code{FALSE}.} -\item{echo}{A logical. Whether to print Osmium’s stdout/stderr. Defaults to \code{TRUE}.} +\item{echo}{A logical. Whether to print Osmium's stdout/stderr. Defaults to \code{TRUE}.} \item{spinner}{A logical. Whether to show a spinner during execution. Defaults to \code{TRUE}.} From de8b429a8e5f2cc9f2233ae9a610ac0518e47116 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Sat, 3 May 2025 20:45:36 +0200 Subject: [PATCH 14/28] add osm_merge draft doc --- man/osm_merge.Rd | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 man/osm_merge.Rd diff --git a/man/osm_merge.Rd b/man/osm_merge.Rd new file mode 100644 index 0000000..a9a94da --- /dev/null +++ b/man/osm_merge.Rd @@ -0,0 +1,64 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/merge.R +\name{osm_merge} +\alias{osm_merge} +\title{Merge several sorted OSM files} +\usage{ +osm_merge( + input_paths, + output_path, + check_integrity = "no", + input_format = NULL, + output_format = NULL, + fsync = FALSE, + generator = NULL, + overwrite = FALSE, + output_header = NULL, + echo_cmd = FALSE, + echo = TRUE, + spinner = TRUE, + verbose = FALSE, + progress = FALSE +) +} +\arguments{ +\item{input_paths}{A character vector of paths to one or more existing, sorted OSM files.} + +\item{output_path}{A string. Path to the file where the merged result will be written.} + +\item{check_integrity}{A string. One of "no", "simple", or "full". "no" (default) skips integrity checks; +"simple" runs \code{\link[=osm_check_refs]{osm_check_refs()}} without relation checks; +"full" runs \code{\link[=osm_check_refs]{osm_check_refs()}} with relation checks enabled.} + +\item{input_format}{A string or \code{NULL}. Force input format (e.g. "osm", "pbf"). If \code{NULL}, autodetect.} + +\item{output_format}{A string or \code{NULL}. Force output format (e.g. "osm", "pbf"). If \code{NULL}, autodetect.} + +\item{fsync}{A logical. If \code{TRUE}, call \code{fsync} after writing to force flushing buffers. Defaults to \code{FALSE}.} + +\item{generator}{A string or \code{NULL}. Name/version for the \code{generator} header. Defaults to Osmium’s own.} + +\item{overwrite}{A logical. If \code{TRUE}, allow overwriting an existing \code{output_path}. Defaults to \code{FALSE}.} + +\item{output_header}{A named character vector of additional header options. Each name is the header key, +each value is the header value. A zero‐length value (\code{""}) writes \verb{OPTION!}. Defaults to \code{NULL}.} + +\item{echo_cmd}{A logical. Whether to print the generated Osmium command. Defaults to \code{FALSE}.} + +\item{echo}{A logical. Whether to print Osmium’s stdout/stderr. Defaults to \code{TRUE}.} + +\item{spinner}{A logical. Whether to show a spinner during execution. Defaults to \code{TRUE}.} + +\item{verbose}{A logical. Whether to pass \code{--verbose} to Osmium for detailed logging. Defaults to \code{FALSE}.} + +\item{progress}{A logical. Whether to pass \code{--progress} / \code{--no-progress}. Defaults to \code{FALSE}.} +} +\value{ +Invisibly returns the normalized path to \code{output_path}. +} +\description{ +This is a wrapper for \href{https://docs.osmcode.org/osmium/latest/osmium-merge.html}{osmium merge}. +Merges multiple sorted OSM files into a single output, removing duplicate objects. +Optionally checks referential integrity of each input before merging. +} +\keyword{internal} From 731b34ca2aff17804d95202c55ea499dfb99a923 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Sat, 3 May 2025 20:46:13 +0200 Subject: [PATCH 15/28] don't check exact osmium version --- tests/testthat/test-check_refs.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-check_refs.R b/tests/testthat/test-check_refs.R index 01e5e99..f38f5ca 100644 --- a/tests/testthat/test-check_refs.R +++ b/tests/testthat/test-check_refs.R @@ -32,7 +32,7 @@ test_that("osm_check_refs works with check_relations = FALSE", { # file name expect_true(grepl("^File checked: .*/cur\\.osm\\.pbf$", out[1])) # version block present - expect_true(any(grepl("^osmium version 1\\.18\\.0$", out))) + expect_true(any(grepl("^osmium version", out))) # summary line expect_true(any(grepl( "^There are 40185 nodes, 6884 ways, and 430 relations in this file\\.$", From 769306c7f8a46f340e279fde573ac21a74592f84 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Sat, 3 May 2025 20:46:50 +0200 Subject: [PATCH 16/28] merge check integrity option draft --- R/merge.R | 77 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/R/merge.R b/R/merge.R index fd0ba5b..adbf73f 100644 --- a/R/merge.R +++ b/R/merge.R @@ -1,36 +1,36 @@ +# File: R/merge.R + #' Merge several sorted OSM files #' #' This is a wrapper for [osmium merge](https://docs.osmcode.org/osmium/latest/osmium-merge.html). #' Merges multiple sorted OSM files into a single output, removing duplicate objects. +#' Optionally checks referential integrity of each input before merging. #' -#' @param input_paths Character vector of paths to one or more existing, sorted OSM files. -#' @param output_path String. Path to the file where the merged result will be written. -#' @param with_history A logical. If `TRUE`, suppress warnings about multiple versions -#' of the same object. Defaults to `FALSE`. -#' @param input_format Optional string. Force input format (e.g. `"osm"`, `"pbf"`). If `NULL`, autodetect. -#' @param output_format Optional string. Force output format (e.g. `"osm"`, `"pbf"`). If `NULL`, autodetect. +#' @param input_paths A character vector of paths to one or more existing, sorted OSM files. +#' @param output_path A string. Path to the file where the merged result will be written. +#' @param check_integrity A string. One of "no", "simple", or "full". "no" (default) skips integrity checks; +#' "simple" runs [`osm_check_refs()`] without relation checks; +#' "full" runs [`osm_check_refs()`] with relation checks enabled. +#' @param input_format A string or `NULL`. Force input format (e.g. "osm", "pbf"). If `NULL`, autodetect. +#' @param output_format A string or `NULL`. Force output format (e.g. "osm", "pbf"). If `NULL`, autodetect. #' @param fsync A logical. If `TRUE`, call `fsync` after writing to force flushing buffers. Defaults to `FALSE`. -#' @param generator Optional string. Name/version for the `generator` header. Defaults to Osmium’s own. +#' @param generator A string or `NULL`. Name/version for the `generator` header. Defaults to Osmium’s own. #' @param overwrite A logical. If `TRUE`, allow overwriting an existing `output_path`. Defaults to `FALSE`. -#' @param output_header Named character vector of additional header options. Each name is the header key, +#' @param output_header A named character vector of additional header options. Each name is the header key, #' each value is the header value. A zero‐length value (`""`) writes `OPTION!`. Defaults to `NULL`. -#' @param echo_cmd A logical. Whether to print the Osmium command generated by the function call to the screen. -#' Defaults to `FALSE`. -#' @param echo A logical. Whether to print the standard output and error generated by the Osmium call to the screen. -#' Defaults to `TRUE`. -#' @param spinner A logical. Whether to show a reassuring spinner while the Osmium call is being executed. -#' Defaults to `TRUE`. -#' @param verbose A logical. Whether to display detailed information on the running command. Defaults to `FALSE`. -#' @param progress A logical. Whether to display a progress bar while running -#' the command. Defaults to `FALSE`. +#' @param echo_cmd A logical. Whether to print the generated Osmium command. Defaults to `FALSE`. +#' @param echo A logical. Whether to print Osmium’s stdout/stderr. Defaults to `TRUE`. +#' @param spinner A logical. Whether to show a spinner during execution. Defaults to `TRUE`. +#' @param verbose A logical. Whether to pass `--verbose` to Osmium for detailed logging. Defaults to `FALSE`. +#' @param progress A logical. Whether to pass `--progress` / `--no-progress`. Defaults to `FALSE`. #' #' @return Invisibly returns the normalized path to `output_path`. -#' +#' @keywords internal #' @export osm_merge <- function( input_paths, output_path, - with_history = FALSE, + check_integrity = "no", input_format = NULL, output_format = NULL, fsync = FALSE, @@ -52,7 +52,11 @@ osm_merge <- function( } checkmate::assert_string(output_path) - checkmate::assert_logical(with_history, len = 1) + ## check_integrity must be one of no/simple/full + check_integrity <- match.arg(check_integrity, c("no", "simple", "full")) + do_check <- check_integrity != "no" + check_rel_flag <- check_integrity == "full" + if (!is.null(input_format)) checkmate::assert_string(input_format) if (!is.null(output_format)) checkmate::assert_string(output_format) checkmate::assert_logical(fsync, len = 1) @@ -68,12 +72,33 @@ osm_merge <- function( checkmate::assert_logical(verbose, len = 1) checkmate::assert_logical(progress, len = 1) - ## Build flags - hist_flag <- if (with_history) "--with-history" else character() + ## Prepare common flags + verbose_flag <- if (verbose) "--verbose" else character() + progress_flag <- if (progress) "--progress" else "--no-progress" input_fmt_flag <- if (!is.null(input_format)) paste0("--input-format=", input_format) else character() output_fmt_flag <- if (!is.null(output_format)) paste0("--output-format=", output_format) else character() + + ## Optionally check referential integrity + if (do_check) { + for (p in input_paths) { + print(paste0("Checking integrity of ", p, "...")) + osm_check_refs( + input_path = p, + show_ids = FALSE, + check_relations = check_rel_flag, + input_format = input_format, + echo_cmd = echo_cmd, + echo = echo, + spinner = spinner, + verbose = verbose, + progress = progress + ) + } + } + + ## Build merge flags fsync_flag <- if (fsync) "--fsync" else character() gen_flag <- if (!is.null(generator)) paste0("--generator=", generator) else character() @@ -89,15 +114,12 @@ osm_merge <- function( character(1) )) } else character() - verbose_flag <- if (verbose) "--verbose" else character() - progress_flag <- if (progress) "--progress" else "--no-progress" output_flag <- paste0("--output=", output_path) - ## Assemble and run + ## run merge args <- c( "merge", input_paths, - hist_flag, input_fmt_flag, output_fmt_flag, fsync_flag, @@ -108,7 +130,8 @@ osm_merge <- function( verbose_flag, progress_flag ) - processx::run( + + logs <- processx::run( "osmium", args, echo = echo, From 091492dbc272e9132ac011ba0d67b60ef78d1570 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Sat, 3 May 2025 20:47:35 +0200 Subject: [PATCH 17/28] file info doc update --- man/osm_file_info.Rd | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/man/osm_file_info.Rd b/man/osm_file_info.Rd index e31071a..bfe7b1e 100644 --- a/man/osm_file_info.Rd +++ b/man/osm_file_info.Rd @@ -21,8 +21,7 @@ osm_file_info( \arguments{ \item{input_path}{String. Path to the OSM file.} -\item{extended}{A logical. Read the entire file (not just header)? Defaults to \code{FALSE}. -Required for full data and metadata sections.} +\item{extended}{A logical(1). Read the entire file (not just header)? Defaults to \code{FALSE}. Reading extended info may be slow, as it reads the entire file. This is also required if the file does not have a bounding box in the header, and you need to calculate it from the data. Specifically, to get or calculate the bounding box, you need can use the \link{osm_get_bbox} function.} \item{get}{String or NULL. If non-NULL, returns only this variable's value (using \verb{--get=}). Must be a full CLI variable name (e.g. \code{file.name}, \code{header.option.generator}, \code{data.bbox}, or any other supported variable listed by \verb{osmium fileinfo --show-variables}). A shorthand of \code{"generator"} is recognized and maps to \code{header.option.generator}. To get the list of valid variable names, use \code{osm_file_info_variables()}.} @@ -56,9 +55,13 @@ the command. Defaults to \code{FALSE}.} } } \description{ -Wrapper around \href{https://docs.osmcode.org/osmium/latest/osmium-fileinfo.html}{osmium fileinfo}. -Retrieves metadata from an OSM (or PBF) file in various formats, fetches a single variable, -or lists available variables. +This is a wrapper for \href{https://docs.osmcode.org/osmium/latest/osmium-fileinfo.html}{osmium fileinfo}. +Returns OSM file & header info in three possible formats: +\itemize{ +\item \code{"text"}: the original plain‐text dump +\item \code{"json"}: the raw JSON string emitted by \code{--json} +\item \code{"list"}: the JSON parsed to an R list via \code{RcppSimdJson::fparse()} +} } \examples{ \dontshow{if (identical(tolower(Sys.getenv("NOT_CRAN")), "true")) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} From 1f17aa7d3ac4054d16ed986d50a2064a2a119d10 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Mon, 5 May 2025 13:24:24 +0200 Subject: [PATCH 18/28] check-refs returns and prints object, no echo by default --- R/check_refs.R | 5 +++-- man/osm_check_refs.Rd | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/R/check_refs.R b/R/check_refs.R index f063bdc..497c730 100644 --- a/R/check_refs.R +++ b/R/check_refs.R @@ -21,7 +21,7 @@ osm_check_refs <- function( check_relations = FALSE, input_format = NULL, echo_cmd = FALSE, - echo = TRUE, + echo = FALSE, spinner = TRUE, verbose = TRUE, progress = FALSE @@ -65,7 +65,8 @@ osm_check_refs <- function( request_verbose = verbose, request_input_file = input_path ) - invisible(parsed) + + return(parsed) } #' Parse Osmium `check-refs` command output diff --git a/man/osm_check_refs.Rd b/man/osm_check_refs.Rd index 7144834..230bf93 100644 --- a/man/osm_check_refs.Rd +++ b/man/osm_check_refs.Rd @@ -10,7 +10,7 @@ osm_check_refs( check_relations = FALSE, input_format = NULL, echo_cmd = FALSE, - echo = TRUE, + echo = FALSE, spinner = TRUE, verbose = TRUE, progress = FALSE From 46f292b06e1667060c976bed013298684085fd73 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Mon, 5 May 2025 14:24:23 +0200 Subject: [PATCH 19/28] improve doc for check refs --- R/check_refs.R | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/R/check_refs.R b/R/check_refs.R index 497c730..b93befa 100644 --- a/R/check_refs.R +++ b/R/check_refs.R @@ -5,15 +5,23 @@ #' #' @param input_path String. Path to an existing OSM data file (not history or change file). #' @param show_ids A logical. If `TRUE`, print all missing IDs to stdout. Defaults to `FALSE`. -#' @param check_relations A logical. If `TRUE`, also check relations references. Defaults to `FALSE`. +#' @param check_relations A logical. If `TRUE`, also check relations references. Defaults to `FALSE`, which only checks if any `nodes` in `ways`` are missing. #' @param input_format Optional string. Force input format (e.g. "osm", "pbf"). If `NULL`, autodetect. #' @param echo_cmd A logical. Whether to print the generated Osmium command. Defaults to `FALSE`. -#' @param echo A logical. Whether to print Osmium's stdout/stderr. Defaults to `TRUE`. +#' @param echo A logical. Whether to print Osmium's stdout/stderr. Defaults to `FALSE` to avoid duplication because the parsed output is printed to R console by default. #' @param spinner A logical. Whether to show a spinner during execution. Defaults to `TRUE`. #' @param verbose A logical. Whether to pass `--verbose` to Osmium for detailed logging. Defaults to `TRUE`. #' @param progress A logical. Whether to pass `--progress` / `--no-progress`. Defaults to `FALSE`. #' -#' @return A character scalar: the plain-text output from `osmium check-refs`, invisibly returned if successful. +#' @return A list of class `osm_check_refs_log`, containing: +#' - `status_code`: `integer`. Exit status from Osmium. +#' - `raw`: list of raw text captured from Osmium's stdout and stderr. +#' - `metadata`: list of the original `verbose` and `check_relations` settings. +#' - `summary`: data.frame of counts (present and missing references). +#' +#' When printed, provides a formatted summary about the check and possible +#' warnings and errors. +#' #' @export osm_check_refs <- function( input_path, From 598cadcfbf0234a86e30fb495e0bd61e6b53c05d Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Mon, 5 May 2025 14:24:41 +0200 Subject: [PATCH 20/28] advanced merge version with new docs --- R/merge.R | 204 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 146 insertions(+), 58 deletions(-) diff --git a/R/merge.R b/R/merge.R index adbf73f..bea53c3 100644 --- a/R/merge.R +++ b/R/merge.R @@ -1,42 +1,83 @@ -# File: R/merge.R - #' Merge several sorted OSM files #' -#' This is a wrapper for [osmium merge](https://docs.osmcode.org/osmium/latest/osmium-merge.html). -#' Merges multiple sorted OSM files into a single output, removing duplicate objects. -#' Optionally checks referential integrity of each input before merging. +#' @description{ +#' This is a wrapper for **[`osmium merge`](https://docs.osmcode.org/osmium/latest/osmium-merge.html)** that can optionally (1) run referential-integrity pre-flight checks with [`osm_check_refs()`], (2) force-sort each input with [`osm_sort()`], and then (3) perform the actual merge. +#' ## 1. Referential-integrity checks +#' Set `check_integrity` to +#' | value | what is checked | implementation | +#' |----------------|-------------------------------------------------------|----------------------------------------| +#' | `"no"` *(def.)*| nothing | - | +#' | `"simple"` | *nodes referenced by ways* are present | `osm_check_refs(check_relations = FALSE)` | +#' | `"full"` | additionally checks *all references inside relations* | `osm_check_refs(check_relations = TRUE)` | +#' +#' If any check returns exit-status `1` (missing refs) **and** +#' `stop_on_integrity_fail = TRUE`, the function **aborts** before merging. +#' +#' ## 2. Optional *force-sort* step +#' If `sort_before_merge = TRUE`, each input file is first piped through +#' [`osm_sort()`] (using the default `"simple"` strategy) into a temporary +#' file. This strategy is fast but can require **about 10x the on-disk size +#' of a `.pbf`/`.osm.bz2` file in memory**. See [`?osm_sort`] for +#' details. Temporary files are deleted automatically at the end of the +#' R session. If you need permanent, sorted copies, call +#' [`osm_sort()`] yourself and set the desired output path. +#' +#' ## 3. Memory usage during merge +#' `osmium merge` itself needs only small buffers, but when merging **many** +#' files those buffers can add up. The optional sorting step generally +#' dominates RAM usage. #' -#' @param input_paths A character vector of paths to one or more existing, sorted OSM files. -#' @param output_path A string. Path to the file where the merged result will be written. -#' @param check_integrity A string. One of "no", "simple", or "full". "no" (default) skips integrity checks; -#' "simple" runs [`osm_check_refs()`] without relation checks; -#' "full" runs [`osm_check_refs()`] with relation checks enabled. -#' @param input_format A string or `NULL`. Force input format (e.g. "osm", "pbf"). If `NULL`, autodetect. -#' @param output_format A string or `NULL`. Force output format (e.g. "osm", "pbf"). If `NULL`, autodetect. -#' @param fsync A logical. If `TRUE`, call `fsync` after writing to force flushing buffers. Defaults to `FALSE`. -#' @param generator A string or `NULL`. Name/version for the `generator` header. Defaults to Osmium’s own. -#' @param overwrite A logical. If `TRUE`, allow overwriting an existing `output_path`. Defaults to `FALSE`. -#' @param output_header A named character vector of additional header options. Each name is the header key, -#' each value is the header value. A zero‐length value (`""`) writes `OPTION!`. Defaults to `NULL`. -#' @param echo_cmd A logical. Whether to print the generated Osmium command. Defaults to `FALSE`. -#' @param echo A logical. Whether to print Osmium’s stdout/stderr. Defaults to `TRUE`. -#' @param spinner A logical. Whether to show a spinner during execution. Defaults to `TRUE`. -#' @param verbose A logical. Whether to pass `--verbose` to Osmium for detailed logging. Defaults to `FALSE`. -#' @param progress A logical. Whether to pass `--progress` / `--no-progress`. Defaults to `FALSE`. +#'} +#' @param input_paths Character vector. Paths to one or more *existing* +#' **(preferably already-sorted)** OSM files. +#' @param output_path String. Where to write the merged result. +#' @param check_integrity One of: +#' - `"no"` *(default)*: no checks are made, +#' - `"simple"`: runs `osm_check_refs(check_relations = FALSE)` to check +#' that *nodes referenced by ways* are present, +#' - `"full"`: runs `osm_check_refs(check_relations = TRUE)` to check that +#' *all references inside relations* are present. +#' @param stop_on_integrity_fail Logical. Abort on failed integrity check? +#' Default `FALSE`. +#' @param sort_before_merge Logical. If `TRUE`, force-sort each input with +#' [`osm_sort()`] before merging. Default `FALSE`. +#' @param input_format,output_format Force I/O formats (`"pbf"`, `"osm"`, ...). +#' `NULL` lets Osmium autodetect. +#' @param fsync Logical. Pass **`--fsync`** to flush buffers. +#' @param generator Optional string for **`--generator`** header tag. +#' @param overwrite Logical. Allow overwriting `output_path`. +#' @param output_header Named character vector of extra header options +#' (`OPTION = "value"` or `OPTION = ""` to replicate input header via +#' the `OPTION!` shorthand). +#' @param with_history Logical. Pass **`--with-history`** to Osmium. +#' @param echo_cmd A logical. Whether to print the Osmium command generated by +#' the function call to the screen. Defaults to `FALSE`. +#' @param echo A logical. Whether to print the standard output and error +#' generated by the Osmium call to the screen. Defaults to `TRUE`. +#' @param spinner A logical. Whether to show a reassuring spinner while the +#' Osmium call is being executed. Defaults to `TRUE`. +#' @param verbose A logical. Whether to display detailed information on the +#' running command. Defaults to `FALSE`. #' -#' @return Invisibly returns the normalized path to `output_path`. +#' @return (Invisibly) the normalised `output_path`. +#' @keywords internal +#' @export +#' @return (Invisibly) the normalised `output_path`. #' @keywords internal #' @export osm_merge <- function( input_paths, output_path, check_integrity = "no", + stop_on_integrity_fail = FALSE, + sort_before_merge = FALSE, input_format = NULL, output_format = NULL, fsync = FALSE, generator = NULL, overwrite = FALSE, output_header = NULL, + with_history = FALSE, echo_cmd = FALSE, echo = TRUE, spinner = TRUE, @@ -45,49 +86,42 @@ osm_merge <- function( ) { assert_osmium_is_installed() - ## Validate inputs + ## ---- argument validation -------------------------------------------------- checkmate::assert_character(input_paths, any.missing = FALSE, min.len = 1) - for (p in input_paths) { - checkmate::assert_file_exists(p, access = "r") - } + for (p in input_paths) checkmate::assert_file_exists(p, access = "r") checkmate::assert_string(output_path) - ## check_integrity must be one of no/simple/full check_integrity <- match.arg(check_integrity, c("no", "simple", "full")) do_check <- check_integrity != "no" - check_rel_flag <- check_integrity == "full" + check_rels <- identical(check_integrity, "full") + + checkmate::assert_flag(stop_on_integrity_fail) + checkmate::assert_flag(sort_before_merge) if (!is.null(input_format)) checkmate::assert_string(input_format) if (!is.null(output_format)) checkmate::assert_string(output_format) - checkmate::assert_logical(fsync, len = 1) + checkmate::assert_flag(fsync) if (!is.null(generator)) checkmate::assert_string(generator) - checkmate::assert_logical(overwrite, len = 1) + checkmate::assert_flag(overwrite) if (!is.null(output_header)) { checkmate::assert_named(output_header, type = "unique") - checkmate::assert_character(output_header) + checkmate::assert_character(output_header, any.missing = FALSE) } - checkmate::assert_logical(echo_cmd, len = 1) - checkmate::assert_logical(echo, len = 1) - checkmate::assert_logical(spinner, len = 1) - checkmate::assert_logical(verbose, len = 1) - checkmate::assert_logical(progress, len = 1) + checkmate::assert_flag(with_history) + checkmate::assert_flag(echo_cmd) + checkmate::assert_flag(echo) + checkmate::assert_flag(spinner) + checkmate::assert_flag(verbose) + checkmate::assert_flag(progress) - ## Prepare common flags - verbose_flag <- if (verbose) "--verbose" else character() - progress_flag <- if (progress) "--progress" else "--no-progress" - input_fmt_flag <- if (!is.null(input_format)) - paste0("--input-format=", input_format) else character() - output_fmt_flag <- if (!is.null(output_format)) - paste0("--output-format=", output_format) else character() - - ## Optionally check referential integrity + ## ---- 1 referential‑integrity checks ------------------------------------ if (do_check) { for (p in input_paths) { - print(paste0("Checking integrity of ", p, "...")) - osm_check_refs( + if (echo) cat("Checking integrity of", p, "...\n") + log <- osm_check_refs( input_path = p, show_ids = FALSE, - check_relations = check_rel_flag, + check_relations = check_rels, input_format = input_format, echo_cmd = echo_cmd, echo = echo, @@ -95,43 +129,97 @@ osm_merge <- function( verbose = verbose, progress = progress ) + + if ( + isTRUE(stop_on_integrity_fail) && + !is.null(log$status_code) && + log$status_code == 1L + ) { + stop( + sprintf( + "Referential integrity check failed for '%s' (missing references).", + p + ), + call. = FALSE + ) + } } } - ## Build merge flags + ## ---- 2 optional force‑sort --------------------------------------------- + if (sort_before_merge) { + if (echo) cat("Sorting input files before merge (strategy = 'simple')...\n") + + sorted_paths <- character(length(input_paths)) + for (i in seq_along(input_paths)) { + in_file <- input_paths[i] + ext <- tools::file_ext(in_file) + tmp_file <- tempfile(fileext = if (nzchar(ext)) paste0(".", ext) else "") + osm_sort( + input_paths = in_file, + output_path = tmp_file, + strategy = "simple", + input_format = input_format, + output_format = output_format, + fsync = fsync, + generator = generator, + overwrite = TRUE, + echo_cmd = echo_cmd, + echo = echo, + spinner = spinner, + verbose = verbose, + progress = progress + ) + sorted_paths[i] <- tmp_file + } + ## clean up temps when R session ends + old <- sorted_paths # capture for on.exit + on.exit(unlink(old, recursive = FALSE, force = TRUE), add = TRUE) + input_paths <- sorted_paths + } + + ## ---- 3 build Osmium flags ---------------------------------------------- + verbose_flag <- if (verbose) "--verbose" else character() + progress_flag <- if (progress) "--progress" else "--no-progress" + input_flag <- if (!is.null(input_format)) + paste0("--input-format=", input_format) else character() + output_flag <- if (!is.null(output_format)) + paste0("--output-format=", output_format) else character() fsync_flag <- if (fsync) "--fsync" else character() gen_flag <- if (!is.null(generator)) paste0("--generator=", generator) else character() overwrite_flag <- if (overwrite) "--overwrite" else character() + history_flag <- if (with_history) "--with-history" else character() header_flags <- if (!is.null(output_header)) { unname(vapply( names(output_header), function(nm) { val <- output_header[[nm]] - hdr <- if (nzchar(val)) paste0(nm, "=", val) else paste0(nm, "!") - paste0("--output-header=", hdr) + tag <- if (nzchar(val)) paste0(nm, "=", val) else paste0(nm, "!") + paste0("--output-header=", tag) }, character(1) )) } else character() - output_flag <- paste0("--output=", output_path) + out_flag <- paste0("--output=", output_path) - ## run merge + ## ---- 4 run Osmium merge ------------------------------------------------- args <- c( "merge", input_paths, - input_fmt_flag, - output_fmt_flag, + input_flag, + output_flag, fsync_flag, gen_flag, - output_flag, + out_flag, overwrite_flag, + history_flag, header_flags, verbose_flag, progress_flag ) - logs <- processx::run( + processx::run( "osmium", args, echo = echo, From 3feb97481f59b517a46b974df8c29f544070f88b Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Mon, 5 May 2025 14:39:24 +0200 Subject: [PATCH 21/28] sort with docs, examples, tests --- R/sort.R | 127 +++++++++++++++++++++++-------------- man/osm_sort.Rd | 98 ++++++++++++++++++---------- tests/testthat/test-sort.R | 44 +++++++++++++ 3 files changed, 190 insertions(+), 79 deletions(-) create mode 100644 tests/testthat/test-sort.R diff --git a/R/sort.R b/R/sort.R index 3612be7..4d5d190 100644 --- a/R/sort.R +++ b/R/sort.R @@ -1,42 +1,74 @@ #' Sort OSM files #' -#' This is a wrapper for [osmium sort](https://docs.osmcode.org/osmium/latest/osmium-sort.html). Combines and sorts the content of one or more OSM files. Objects are ordered by type (node, way, relation), by ID (negative IDs first, then positive, sorted by absolute value), and by version. Duplicate objects are preserved. Works with data, history, and change files. +#' @description{ +#' #' This is a wrapper for **[`osmium sort`](https://docs.osmcode.org/osmium/latest/osmium-sort.html)**. It combines the content of one or more input files and writes a single file whose objects are strictly ordered by #' -#' @param input_paths Character vector of paths to one or more existing OSM files. -#' @param output_path String. Path to the file where the sorted output will be written. -#' The format is inferred from the extension unless \code{output_format} is set. -#' @param strategy Character(1). Sorting strategy, either \code{"simple"} (default, -#' single-pass in-memory) or \code{"multipass"} (three passes, lower memory). -#' @param input_format Optional string. Force the input format (e.g. \code{"osm"}, -#' \code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect. -#' @param output_format Optional string. Force the output format (e.g. \code{"osm"}, -#' \code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect. -#' @param fsync Logical(1). If \code{TRUE}, call \code{fsync} after writing the -#' output file to flush buffers. Defaults to \code{FALSE}. -#' @param generator Optional string. Name/version to write into the \code{generator} -#' header. Default is \code{"osmium/"}. -#' @param overwrite Logical(1). If \code{TRUE}, allow overwriting an existing -#' \code{output_path}. Defaults to \code{FALSE}. +#' * type (node, way, relation), +#' * ID (negative IDs first, then positive, each by absolute value), and +#' * version (ascending). +#' +#' Duplicate objects are **not** removed; they are simply written in +#' sorted order. To remove duplicate objects, use [osm_merge()], but please sort all input files first. +#' +#' ## Sorting strategies +#' +#' | value | passes | memory use | speed | +#' |---------------|--------|----------------------------------|-------| +#' | \code{"simple"} *(default)* | 1 | keeps **all data in RAM** - expect roughly **10 x** the on-disk size of a \code{.pbf} or \code{.osm.bz2} file | fastest | +#' | \code{"multipass"} | 3 | reads nodes, ways, relations in separate passes - uses **much less RAM** | slower | +#' +#' The multipass strategy cannot be used when reading from \code{stdin}. +#' +#' ## Memory usage +#' +#' With the default \code{"simple"} strategy, \code{osmium sort} +#' stores the entire data set in main memory; the rule of thumb from +#' the manual is about 10x times the on-disk size of the input +#' file. For large extracts or limited RAM, switch to +#' \code{strategy = "multipass"}. +#'} +#' @param input_paths Character vector with paths to one or more existing +#' OSM files. +#' @param output_path String. Path for the sorted output file. The format is +#' inferred from the file extension unless \code{output_format} is supplied. +#' @param strategy Character (length 1). Either \code{"simple"} (in-memory, +#' single pass) or \code{"multipass"} (three passes, lower memory). +#' @param input_format Optional string. Force the input format +#' (for example \code{"osm"}, \code{"pbf"}, \code{"osc"}). +#' If \code{NULL}, the format is autodetected. +#' @param output_format Optional string. Force the output format +#' (for example \code{"osm"}, \code{"pbf"}, \code{"osc"}). +#' If \code{NULL}, the format is autodetected. +#' @param fsync Logical. If \code{TRUE}, call \code{fsync} after writing the +#' output file to flush buffers. Default \code{FALSE}. +#' @param generator Optional string. Value for the \code{generator} header +#' field. Default is \code{"osmium/"}. +#' @param overwrite Logical. Allow an existing \code{output_path} to be +#' overwritten. Default \code{FALSE}. #' @param output_header Named character vector of additional header options. -#' Each name is the header key, each value is the header value. If a value -#' is \code{""}, writes \code{OPTION!}. Defaults to \code{NULL}. -#' @param echo_cmd A logical. Whether to print the Osmium command generated by the function call to the screen. -#' Defaults to `FALSE`. -#' @param echo A logical. Whether to print the standard output and error generated by the Osmium call to the screen. -#' Defaults to `TRUE`. -#' @param spinner A logical. Whether to show a reassuring spinner while the Osmium call is being executed. -#' Defaults to `TRUE`. -#' @param verbose A logical. Whether to display detailed information on the running command. Defaults to `FALSE`. -#' @param progress A logical. Whether to display a progress bar while running -#' the command. Defaults to `FALSE`. +#' Each name is a header key, each value the header value. A value of +#' \code{""} writes \code{OPTION!}. Default \code{NULL}. +#' @param echo_cmd A logical. Whether to print the Osmium command generated by +#' the function call to the screen. Defaults to `FALSE`. +#' @param echo A logical. Whether to print the standard output and error +#' generated by the Osmium call to the screen. Defaults to `TRUE`. +#' @param spinner A logical. Whether to show a reassuring spinner while the +#' Osmium call is being executed. Defaults to `TRUE`. +#' @param verbose A logical. Whether to display detailed information on the +#' running command. Defaults to `FALSE`. #' -#' @return Invisibly returns the normalized path to \code{output_path}. +#' @return Invisibly returns the normalised \code{output_path}. #' #' @examplesIf identical(tolower(Sys.getenv("NOT_CRAN")), "true") -#' # sort a single PBF file #' pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") -#' out <- tempfile(fileext = ".osm.pbf") -#' osm_sort(pbf, out, overwrite = TRUE) +#' +#' # Simple one‑pass sort (fastest, high RAM use) +#' out_simple <- tempfile(fileext = ".osm.pbf") +#' osm_sort(pbf, out_simple, overwrite = TRUE) +#' +#' # Multipass sort (lower RAM use) +#' out_multi <- tempfile(fileext = ".osm.pbf") +#' osm_sort(pbf, out_multi, strategy = "multipass", overwrite = TRUE) #' #' @export osm_sort <- function( @@ -57,32 +89,33 @@ osm_sort <- function( ) { assert_osmium_is_installed() - # validate inputs + ## ---- argument validation -------------------------------------------------- checkmate::assert_character(input_paths, any.missing = FALSE, min.len = 1) - for (p in input_paths) { - checkmate::assert_file_exists(p, access = "r") - } + for (p in input_paths) checkmate::assert_file_exists(p, access = "r") checkmate::assert_string(output_path) + strategy <- match.arg(strategy) if (!is.null(input_format)) checkmate::assert_string(input_format) if (!is.null(output_format)) checkmate::assert_string(output_format) - checkmate::assert_logical(fsync, len = 1, any.missing = FALSE) + checkmate::assert_flag(fsync) if (!is.null(generator)) checkmate::assert_string(generator) - checkmate::assert_logical(overwrite, len = 1, any.missing = FALSE) + checkmate::assert_flag(overwrite) + if (!is.null(output_header)) { checkmate::assert_named(output_header, type = "unique") - checkmate::assert_character(output_header) + checkmate::assert_character(output_header, any.missing = FALSE) } - checkmate::assert_logical(echo_cmd, len = 1, any.missing = FALSE) - checkmate::assert_logical(echo, len = 1, any.missing = FALSE) - checkmate::assert_logical(spinner, len = 1, any.missing = FALSE) - checkmate::assert_logical(verbose, len = 1, any.missing = FALSE) - checkmate::assert_logical(progress, len = 1, any.missing = FALSE) - # check output path + checkmate::assert_flag(echo_cmd) + checkmate::assert_flag(echo) + checkmate::assert_flag(spinner) + checkmate::assert_flag(verbose) + checkmate::assert_flag(progress) + + ## ---- output path check ---------------------------------------------------- assert_output_path_multi_ext(output_path, overwrite) - # build flags + ## ---- build Osmium flags --------------------------------------------------- strat_flag <- paste0("--strategy=", strategy) input_flag <- if (!is.null(input_format)) paste0("--input-format=", input_format) else character() @@ -107,7 +140,7 @@ osm_sort <- function( progress_flag <- if (progress) "--progress" else "--no-progress" output_flag <- paste0("--output=", output_path) - # assemble args + ## ---- assemble arguments --------------------------------------------------- args <- c( "sort", input_paths, @@ -123,7 +156,7 @@ osm_sort <- function( progress_flag ) - # run osmium sort + ## ---- run ------------------------------------------------------------------ processx::run( "osmium", args, diff --git a/man/osm_sort.Rd b/man/osm_sort.Rd index 8587903..718730f 100644 --- a/man/osm_sort.Rd +++ b/man/osm_sort.Rd @@ -22,58 +22,92 @@ osm_sort( ) } \arguments{ -\item{input_paths}{Character vector of paths to one or more existing OSM files.} +\item{input_paths}{Character vector with paths to one or more existing +OSM files.} -\item{output_path}{String. Path to the file where the sorted output will be written. -The format is inferred from the extension unless \code{output_format} is set.} +\item{output_path}{String. Path for the sorted output file. The format is +inferred from the file extension unless \code{output_format} is supplied.} -\item{strategy}{Character(1). Sorting strategy, either \code{"simple"} (default, -single-pass in-memory) or \code{"multipass"} (three passes, lower memory).} +\item{strategy}{Character (length 1). Either \code{"simple"} (in-memory, +single pass) or \code{"multipass"} (three passes, lower memory).} -\item{input_format}{Optional string. Force the input format (e.g. \code{"osm"}, -\code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect.} +\item{input_format}{Optional string. Force the input format +(for example \code{"osm"}, \code{"pbf"}, \code{"osc"}). +If \code{NULL}, the format is autodetected.} -\item{output_format}{Optional string. Force the output format (e.g. \code{"osm"}, -\code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect.} +\item{output_format}{Optional string. Force the output format +(for example \code{"osm"}, \code{"pbf"}, \code{"osc"}). +If \code{NULL}, the format is autodetected.} -\item{fsync}{Logical(1). If \code{TRUE}, call \code{fsync} after writing the -output file to flush buffers. Defaults to \code{FALSE}.} +\item{fsync}{Logical. If \code{TRUE}, call \code{fsync} after writing the +output file to flush buffers. Default \code{FALSE}.} -\item{generator}{Optional string. Name/version to write into the \code{generator} -header. Default is \code{"osmium/"}.} +\item{generator}{Optional string. Value for the \code{generator} header +field. Default is \code{"osmium/"}.} -\item{overwrite}{Logical(1). If \code{TRUE}, allow overwriting an existing -\code{output_path}. Defaults to \code{FALSE}.} +\item{overwrite}{Logical. Allow an existing \code{output_path} to be +overwritten. Default \code{FALSE}.} \item{output_header}{Named character vector of additional header options. -Each name is the header key, each value is the header value. If a value -is \code{""}, writes \code{OPTION!}. Defaults to \code{NULL}.} +Each name is a header key, each value the header value. A value of +\code{""} writes \code{OPTION!}. Default \code{NULL}.} -\item{echo_cmd}{A logical. Whether to print the Osmium command generated by the function call to the screen. -Defaults to \code{FALSE}.} +\item{echo_cmd}{A logical. Whether to print the Osmium command generated by +the function call to the screen. Defaults to \code{FALSE}.} -\item{echo}{A logical. Whether to print the standard output and error generated by the Osmium call to the screen. -Defaults to \code{TRUE}.} +\item{echo}{A logical. Whether to print the standard output and error +generated by the Osmium call to the screen. Defaults to \code{TRUE}.} -\item{spinner}{A logical. Whether to show a reassuring spinner while the Osmium call is being executed. -Defaults to \code{TRUE}.} +\item{spinner}{A logical. Whether to show a reassuring spinner while the +Osmium call is being executed. Defaults to \code{TRUE}.} -\item{verbose}{A logical. Whether to display detailed information on the running command. Defaults to \code{FALSE}.} - -\item{progress}{A logical. Whether to display a progress bar while running -the command. Defaults to \code{FALSE}.} +\item{verbose}{A logical. Whether to display detailed information on the +running command. Defaults to \code{FALSE}.} } \value{ -Invisibly returns the normalized path to \code{output_path}. +Invisibly returns the normalised \code{output_path}. } \description{ -This is a wrapper for \href{https://docs.osmcode.org/osmium/latest/osmium-sort.html}{osmium sort}. Combines and sorts the content of one or more OSM files. Objects are ordered by type (node, way, relation), by ID (negative IDs first, then positive, sorted by absolute value), and by version. Duplicate objects are preserved. Works with data, history, and change files. +{ +#' This is a wrapper for \strong{\href{https://docs.osmcode.org/osmium/latest/osmium-sort.html}{\verb{osmium sort}}}. It combines the content of one or more input files and writes a single file whose objects are strictly ordered by +\itemize{ +\item type (node, way, relation), +\item ID (negative IDs first, then positive, each by absolute value), and +\item version (ascending). +} + +Duplicate objects are \strong{not} removed; they are simply written in +sorted order. To remove duplicate objects, use \code{\link[=osm_merge]{osm_merge()}}, but please sort all input files first. +\subsection{Sorting strategies}{\tabular{llll}{ + value \tab passes \tab memory use \tab speed \cr + \code{"simple"} \emph{(default)} \tab 1 \tab keeps \strong{all data in RAM} - expect roughly \strong{10 x} the on-disk size of a \code{.pbf} or \code{.osm.bz2} file \tab fastest \cr + \code{"multipass"} \tab 3 \tab reads nodes, ways, relations in separate passes - uses \strong{much less RAM} \tab slower \cr +} + + +The multipass strategy cannot be used when reading from \code{stdin}. +} + +\subsection{Memory usage}{ + +With the default \code{"simple"} strategy, \code{osmium sort} +stores the entire data set in main memory; the rule of thumb from +the manual is about 10x times the on-disk size of the input +file. For large extracts or limited RAM, switch to +\code{strategy = "multipass"}. +} +} } \examples{ \dontshow{if (identical(tolower(Sys.getenv("NOT_CRAN")), "true")) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} -# sort a single PBF file pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") -out <- tempfile(fileext = ".osm.pbf") -osm_sort(pbf, out, overwrite = TRUE) + +# Simple one‑pass sort (fastest, high RAM use) +out_simple <- tempfile(fileext = ".osm.pbf") +osm_sort(pbf, out_simple, overwrite = TRUE) + +# Multipass sort (lower RAM use) +out_multi <- tempfile(fileext = ".osm.pbf") +osm_sort(pbf, out_multi, strategy = "multipass", overwrite = TRUE) \dontshow{\}) # examplesIf} } diff --git a/tests/testthat/test-sort.R b/tests/testthat/test-sort.R new file mode 100644 index 0000000..aadc6f5 --- /dev/null +++ b/tests/testthat/test-sort.R @@ -0,0 +1,44 @@ +testthat::skip_on_cran() +test_that("osm_sort sorts a small PBF (simple strategy)", { + pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + out <- tempfile(fileext = ".osm.pbf") + + expect_invisible( + res <- osm_sort( + input_paths = pbf, + output_path = out, + overwrite = TRUE, + echo = FALSE, + spinner = FALSE, + progress = FALSE + ) + ) + + expect_true(file.exists(res)) + expect_gt(file.size(res), 0) +}) + +test_that("osm_sort works with multipass strategy", { + pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + out <- tempfile(fileext = ".osm.pbf") + + expect_invisible( + osm_sort( + input_paths = pbf, + output_path = out, + strategy = "multipass", + overwrite = TRUE, + echo = FALSE, + spinner = FALSE, + progress = FALSE + ) + ) + + expect_true(file.exists(out)) +}) + +test_that("osm_sort errors on non-existent input", { + fake <- tempfile(fileext = ".pbf") + out <- tempfile(fileext = ".osm.pbf") + expect_error(osm_sort(fake, out), "File does not exist") +}) From 7c8986d14caaa904f1024bcc18e6ccf9737b2ab1 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Mon, 5 May 2025 14:39:34 +0200 Subject: [PATCH 22/28] check refs docs --- man/osm_check_refs.Rd | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/man/osm_check_refs.Rd b/man/osm_check_refs.Rd index 230bf93..a103811 100644 --- a/man/osm_check_refs.Rd +++ b/man/osm_check_refs.Rd @@ -21,13 +21,13 @@ osm_check_refs( \item{show_ids}{A logical. If \code{TRUE}, print all missing IDs to stdout. Defaults to \code{FALSE}.} -\item{check_relations}{A logical. If \code{TRUE}, also check relations references. Defaults to \code{FALSE}.} +\item{check_relations}{A logical. If \code{TRUE}, also check relations references. Defaults to \code{FALSE}, which only checks if any \code{nodes} in `ways`` are missing.} \item{input_format}{Optional string. Force input format (e.g. "osm", "pbf"). If \code{NULL}, autodetect.} \item{echo_cmd}{A logical. Whether to print the generated Osmium command. Defaults to \code{FALSE}.} -\item{echo}{A logical. Whether to print Osmium's stdout/stderr. Defaults to \code{TRUE}.} +\item{echo}{A logical. Whether to print Osmium's stdout/stderr. Defaults to \code{FALSE} to avoid duplication because the parsed output is printed to R console by default.} \item{spinner}{A logical. Whether to show a spinner during execution. Defaults to \code{TRUE}.} @@ -36,7 +36,16 @@ osm_check_refs( \item{progress}{A logical. Whether to pass \code{--progress} / \code{--no-progress}. Defaults to \code{FALSE}.} } \value{ -A character scalar: the plain-text output from \verb{osmium check-refs}, invisibly returned if successful. +A list of class \code{osm_check_refs_log}, containing: +\itemize{ +\item \code{status_code}: \code{integer}. Exit status from Osmium. +\item \code{raw}: list of raw text captured from Osmium's stdout and stderr. +\item \code{metadata}: list of the original \code{verbose} and \code{check_relations} settings. +\item \code{summary}: data.frame of counts (present and missing references). +} + +When printed, provides a formatted summary about the check and possible +warnings and errors. } \description{ This is a wrapper for \href{https://docs.osmcode.org/osmium/latest/osmium-check-refs.html}{osmium check-refs}. From 583d68a92702182f115761dba933b7dae97c3f13 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Mon, 5 May 2025 14:39:40 +0200 Subject: [PATCH 23/28] osm merge docs --- man/osm_merge.Rd | 95 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 74 insertions(+), 21 deletions(-) diff --git a/man/osm_merge.Rd b/man/osm_merge.Rd index a9a94da..7fe48ab 100644 --- a/man/osm_merge.Rd +++ b/man/osm_merge.Rd @@ -8,12 +8,15 @@ osm_merge( input_paths, output_path, check_integrity = "no", + stop_on_integrity_fail = FALSE, + sort_before_merge = FALSE, input_format = NULL, output_format = NULL, fsync = FALSE, generator = NULL, overwrite = FALSE, output_header = NULL, + with_history = FALSE, echo_cmd = FALSE, echo = TRUE, spinner = TRUE, @@ -22,43 +25,93 @@ osm_merge( ) } \arguments{ -\item{input_paths}{A character vector of paths to one or more existing, sorted OSM files.} +\item{input_paths}{Character vector. Paths to one or more \emph{existing} +\strong{(preferably already-sorted)} OSM files.} -\item{output_path}{A string. Path to the file where the merged result will be written.} +\item{output_path}{String. Where to write the merged result.} -\item{check_integrity}{A string. One of "no", "simple", or "full". "no" (default) skips integrity checks; -"simple" runs \code{\link[=osm_check_refs]{osm_check_refs()}} without relation checks; -"full" runs \code{\link[=osm_check_refs]{osm_check_refs()}} with relation checks enabled.} +\item{check_integrity}{One of: +\itemize{ +\item \code{"no"} \emph{(default)}: no checks are made, +\item \code{"simple"}: runs \code{osm_check_refs(check_relations = FALSE)} to check +that \emph{nodes referenced by ways} are present, +\item \code{"full"}: runs \code{osm_check_refs(check_relations = TRUE)} to check that +\emph{all references inside relations} are present. +}} -\item{input_format}{A string or \code{NULL}. Force input format (e.g. "osm", "pbf"). If \code{NULL}, autodetect.} +\item{stop_on_integrity_fail}{Logical. Abort on failed integrity check? +Default \code{FALSE}.} -\item{output_format}{A string or \code{NULL}. Force output format (e.g. "osm", "pbf"). If \code{NULL}, autodetect.} +\item{sort_before_merge}{Logical. If \code{TRUE}, force-sort each input with +\code{\link[=osm_sort]{osm_sort()}} before merging. Default \code{FALSE}.} -\item{fsync}{A logical. If \code{TRUE}, call \code{fsync} after writing to force flushing buffers. Defaults to \code{FALSE}.} +\item{input_format, output_format}{Force I/O formats (\code{"pbf"}, \code{"osm"}, ...). +\code{NULL} lets Osmium autodetect.} -\item{generator}{A string or \code{NULL}. Name/version for the \code{generator} header. Defaults to Osmium’s own.} +\item{fsync}{Logical. Pass \strong{\code{--fsync}} to flush buffers.} -\item{overwrite}{A logical. If \code{TRUE}, allow overwriting an existing \code{output_path}. Defaults to \code{FALSE}.} +\item{generator}{Optional string for \strong{\code{--generator}} header tag.} -\item{output_header}{A named character vector of additional header options. Each name is the header key, -each value is the header value. A zero‐length value (\code{""}) writes \verb{OPTION!}. Defaults to \code{NULL}.} +\item{overwrite}{Logical. Allow overwriting \code{output_path}.} -\item{echo_cmd}{A logical. Whether to print the generated Osmium command. Defaults to \code{FALSE}.} +\item{output_header}{Named character vector of extra header options +(\code{OPTION = "value"} or \code{OPTION = ""} to replicate input header via +the \verb{OPTION!} shorthand).} -\item{echo}{A logical. Whether to print Osmium’s stdout/stderr. Defaults to \code{TRUE}.} +\item{with_history}{Logical. Pass \strong{\code{--with-history}} to Osmium.} -\item{spinner}{A logical. Whether to show a spinner during execution. Defaults to \code{TRUE}.} +\item{echo_cmd}{A logical. Whether to print the Osmium command generated by +the function call to the screen. Defaults to \code{FALSE}.} -\item{verbose}{A logical. Whether to pass \code{--verbose} to Osmium for detailed logging. Defaults to \code{FALSE}.} +\item{echo}{A logical. Whether to print the standard output and error +generated by the Osmium call to the screen. Defaults to \code{TRUE}.} -\item{progress}{A logical. Whether to pass \code{--progress} / \code{--no-progress}. Defaults to \code{FALSE}.} +\item{spinner}{A logical. Whether to show a reassuring spinner while the +Osmium call is being executed. Defaults to \code{TRUE}.} + +\item{verbose}{A logical. Whether to display detailed information on the +running command. Defaults to \code{FALSE}.} } \value{ -Invisibly returns the normalized path to \code{output_path}. +(Invisibly) the normalised \code{output_path}. + +(Invisibly) the normalised \code{output_path}. } \description{ -This is a wrapper for \href{https://docs.osmcode.org/osmium/latest/osmium-merge.html}{osmium merge}. -Merges multiple sorted OSM files into a single output, removing duplicate objects. -Optionally checks referential integrity of each input before merging. +{ +This is a wrapper for \strong{\href{https://docs.osmcode.org/osmium/latest/osmium-merge.html}{\verb{osmium merge}}} that can optionally (1) run referential-integrity pre-flight checks with \code{\link[=osm_check_refs]{osm_check_refs()}}, (2) force-sort each input with \code{\link[=osm_sort]{osm_sort()}}, and then (3) perform the actual merge. +\subsection{1. Referential-integrity checks}{ + +Set \code{check_integrity} to\tabular{lll}{ + value \tab what is checked \tab implementation \cr + \code{"no"} \emph{(def.)} \tab nothing \tab - \cr + \code{"simple"} \tab \emph{nodes referenced by ways} are present \tab \code{osm_check_refs(check_relations = FALSE)} \cr + \code{"full"} \tab additionally checks \emph{all references inside relations} \tab \code{osm_check_refs(check_relations = TRUE)} \cr +} + + +If any check returns exit-status \code{1} (missing refs) \strong{and} +\code{stop_on_integrity_fail = TRUE}, the function \strong{aborts} before merging. +} + +\subsection{2. Optional \emph{force-sort} step}{ + +If \code{sort_before_merge = TRUE}, each input file is first piped through +\code{\link[=osm_sort]{osm_sort()}} (using the default \code{"simple"} strategy) into a temporary +file. This strategy is fast but can require \strong{about 10x the on-disk size +of a \code{.pbf}/\code{.osm.bz2} file in memory}. See \code{\link{?osm_sort}} for +details. Temporary files are deleted automatically at the end of the +R session. If you need permanent, sorted copies, call +\code{\link[=osm_sort]{osm_sort()}} yourself and set the desired output path. +} + +\subsection{3. Memory usage during merge}{ + +\verb{osmium merge} itself needs only small buffers, but when merging \strong{many} +files those buffers can add up. The optional sorting step generally +dominates RAM usage. + +} +} } \keyword{internal} From fa21374642c8544f5daa80effa6e1d276d5a4a7d Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Mon, 5 May 2025 14:39:48 +0200 Subject: [PATCH 24/28] add new functions to pkgdown --- _pkgdown.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/_pkgdown.yml b/_pkgdown.yml index 2a751cb..5ad6f0c 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -12,6 +12,7 @@ reference: - contents: - osm_file_info - osm_file_info_variables + - osm_check_refs - extract - tags_filter - show_content @@ -19,6 +20,7 @@ reference: - osm_get_bbox - osm_set_bbox - osm_sort + - osm_merge - osm_cat - title: "General documentation" - contents: From 1c121172c19631f24d2ae61e596fd592dca78087 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Mon, 5 May 2025 14:51:27 +0200 Subject: [PATCH 25/28] update merge doc --- R/merge.R | 2 +- man/osm_merge.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/merge.R b/R/merge.R index bea53c3..643d37d 100644 --- a/R/merge.R +++ b/R/merge.R @@ -1,7 +1,7 @@ #' Merge several sorted OSM files #' #' @description{ -#' This is a wrapper for **[`osmium merge`](https://docs.osmcode.org/osmium/latest/osmium-merge.html)** that can optionally (1) run referential-integrity pre-flight checks with [`osm_check_refs()`], (2) force-sort each input with [`osm_sort()`], and then (3) perform the actual merge. +#' Merges the content of all OSM files given on the command line into one large OSM file. Objects in all files must be sorted by type, ID, and version. The results will also be sorted in the same way. Objects that appear in multiple input files will only be in the output once. This is a wrapper for **[`osmium merge`](https://docs.osmcode.org/osmium/latest/osmium-merge.html)** that can optionally (1) run referential-integrity pre-flight checks with [`osm_check_refs()`], (2) force-sort each input with [`osm_sort()`], and then (3) perform the actual merge. #' ## 1. Referential-integrity checks #' Set `check_integrity` to #' | value | what is checked | implementation | diff --git a/man/osm_merge.Rd b/man/osm_merge.Rd index 7fe48ab..447b546 100644 --- a/man/osm_merge.Rd +++ b/man/osm_merge.Rd @@ -79,7 +79,7 @@ running command. Defaults to \code{FALSE}.} } \description{ { -This is a wrapper for \strong{\href{https://docs.osmcode.org/osmium/latest/osmium-merge.html}{\verb{osmium merge}}} that can optionally (1) run referential-integrity pre-flight checks with \code{\link[=osm_check_refs]{osm_check_refs()}}, (2) force-sort each input with \code{\link[=osm_sort]{osm_sort()}}, and then (3) perform the actual merge. +Merges the content of all OSM files given on the command line into one large OSM file. Objects in all files must be sorted by type, ID, and version. The results will also be sorted in the same way. Objects that appear in multiple input files will only be in the output once. This is a wrapper for \strong{\href{https://docs.osmcode.org/osmium/latest/osmium-merge.html}{\verb{osmium merge}}} that can optionally (1) run referential-integrity pre-flight checks with \code{\link[=osm_check_refs]{osm_check_refs()}}, (2) force-sort each input with \code{\link[=osm_sort]{osm_sort()}}, and then (3) perform the actual merge. \subsection{1. Referential-integrity checks}{ Set \code{check_integrity} to\tabular{lll}{ From f03acf0d3b8193549b464e9e55c3bc3a120153d2 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Mon, 5 May 2025 14:51:37 +0200 Subject: [PATCH 26/28] rewrite cat following the merge example --- R/cat.R | 197 ++++++++++++++++++++++++++++++++++++------------- man/osm_cat.Rd | 105 ++++++++++++++++++-------- 2 files changed, 217 insertions(+), 85 deletions(-) diff --git a/R/cat.R b/R/cat.R index 80ac116..1bd229e 100644 --- a/R/cat.R +++ b/R/cat.R @@ -1,43 +1,70 @@ -#' Concatenate OSM files -#' -#' This is a wrapper for [osmium cat](https://docs.osmcode.org/osmium/latest/osmium-cat.html). Just like the shell \code{osmium cat} command, \code{osm_cat} can take several input OSM files and concatenate them into one output file. As a side-effect, it can also convert between formats, filter by object type, clean metadata, and set output headers. -#' -#' @param input_paths Character vector of paths to one or more existing OSM files. -#' @param output_path String. Path to the file where the result will be written. -#' The format is inferred from the extension unless \code{output_format} is set. -#' @param object_type Character vector of object types to include. -#' One or more of \code{"node"}, \code{"way"}, \code{"relation"}, -#' \code{"changeset"}. Defaults to all types. -#' @param clean Character vector of attributes to clean (set to zero/empty). -#' One or more of \code{"version"}, \code{"timestamp"}, -#' \code{"changeset"}, \code{"uid"}, \code{"user"}. -#' @param buffer_data Logical(1). If \code{TRUE}, read all inputs into memory -#' before writing. Defaults to \code{FALSE}. -#' @param input_format Optional string. Force the input format (e.g. -#' \code{"osm"}, \code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect. -#' @param output_format Optional string. Force the output format (e.g. -#' \code{"osm"}, \code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect. -#' @param generator Optional string. Specify the \code{generator} header -#' (name and version) to add to the output file. -#' @param overwrite Logical(1). If \code{TRUE}, allow overwriting an existing -#' \code{output_path}. Defaults to \code{FALSE}. -#' @param fsync Logical(1). If \code{TRUE}, call \code{fsync} after writing -#' to force flushing buffers. Defaults to \code{FALSE}. -#' @param output_header Named character vector of additional header options. -#' Each name is the header option and each value is the header value. -#' If the value is \code{""}, the option will be written as \code{OPTION!}. -#' Defaults to \code{NULL}. -#' @param echo_cmd A logical. Whether to print the Osmium command generated by the function call to the screen. +#' Concatenate OSM files with optional integrity checks and sorting +#' +#' @description{ +#' Concatenates all input files and writes the result to the output file. The data is not sorted in any way but strictly copied from input to output. This is a wrapper for **[`osmium cat`](https://docs.osmcode.org/osmium/latest/osmium-cat.html)** that can optionally (1) run referential-integrity pre-flight checks with [`osm_check_refs()`], (2) force-sort each input with [`osm_sort()`], and then (3) perform the actual concatenation. +#' +#' ## 1. Referential-integrity checks +#' Set `check_integrity` to +#' | value | what is checked | implementation | +#' |----------------|-------------------------------------------|----------------------------------------------| +#' | "no" (def.) | nothing | - | +#' | "simple" | *nodes referenced by ways* are present | `osm_check_refs(check_relations = FALSE)` | +#' | "full" | additionally checks *all refs in relations* | `osm_check_refs(check_relations = TRUE)` | +#' +#' If any check returns exit-status `1` (missing refs) **and** +#' `stop_on_integrity_fail = TRUE`, the function **aborts** before concatenating. +#' +#' ## 2. Optional force-sort step +#' If `sort_before_cat = TRUE`, each input file is first piped through +#' [`osm_sort()`] (strategy = "simple") into a temporary file. Temporary +#' files are cleaned up at the end of the R session. If you need permanent, +#' sorted copies, call [`osm_sort()`] yourself and set the desired output path. +#' +#' ## 3. Memory usage during concatenation +#' `osmium cat` itself uses minimal buffers, but the optional sorting step +#' can dominate RAM usage (up to ~10x on-disk size). +#' } +#' +#' @param input_paths Character vector of one or more existing OSM files. +#' @param output_path String. Path to write the concatenated result. +#' @param object_type Character vector of types to include: one or more of +#' "node", "way", "relation", "changeset". Defaults to all types. +#' @param clean Character vector of attributes to clean (set to zero/empty): +#' one or more of "version", "timestamp", "changeset", +#' "uid", "user". Defaults to none. +#' @param buffer_data Logical(1). If `TRUE`, read all inputs into memory +#' before writing. Defaults to `FALSE`. +#' @param check_integrity One of: +#' - "no" (default): skip integrity checks, +#' - "simple": run `osm_check_refs(check_relations = FALSE)`, +#' - "full": run `osm_check_refs(check_relations = TRUE)`. +#' @param stop_on_integrity_fail Logical. If `TRUE`, abort on failed +#' integrity check. Defaults to `FALSE`. +#' @param sort_before_cat Logical. If `TRUE`, force-sort each input with +#' [`osm_sort()`] before concatenation. Defaults to `FALSE`. +#' @param input_format Optional string to force the input format +#' (e.g. "osm", "pbf"). `NULL` to autodetect. +#' @param output_format Optional string to force the output format +#' (e.g. "osm", "pbf"). `NULL` to autodetect. +#' @param generator Optional string for the **`--generator`** header tag. +#' @param overwrite Logical(1). If `TRUE`, allow overwriting an existing +#' `output_path`. Defaults to `FALSE`. +#' @param fsync Logical(1). If `TRUE`, call **`--fsync`** after writing. +#' Defaults to `FALSE`. +#' @param output_header Named character vector of additional header options: +#' `OPTION = "value"` or `OPTION = ""` to replicate via `OPTION!`. +#' @param echo_cmd Logical. Whether to print the generated Osmium command. #' Defaults to `FALSE`. -#' @param echo A logical. Whether to print the standard output and error generated by the Osmium call to the screen. +#' @param echo Logical. Whether to print stdout and stderr of the Osmium call. #' Defaults to `TRUE`. -#' @param spinner A logical. Whether to show a reassuring spinner while the Osmium call is being executed. +#' @param spinner Logical. Whether to show a spinner while running. #' Defaults to `TRUE`. -#' @param verbose A logical. Whether to display detailed information on the running command. Defaults to `FALSE`. -#' @param progress A logical. Whether to display a progress bar while running -#' the command. Defaults to `FALSE`. +#' @param verbose Logical. Whether to display detailed command info. +#' Defaults to `FALSE`. +#' @param progress Logical. Whether to show a progress bar. +#' Defaults to `FALSE`. #' -#' @return Invisibly returns the normalized path to \code{output_path}. +#' @return Invisibly returns the normalized `output_path`. #' #' @examplesIf identical(tolower(Sys.getenv("NOT_CRAN")), "true") #' pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") @@ -104,6 +131,9 @@ osm_cat <- function( object_type = c("node", "way", "relation", "changeset"), clean = character(), buffer_data = FALSE, + check_integrity = c("no", "simple", "full"), + stop_on_integrity_fail = FALSE, + sort_before_cat = FALSE, input_format = NULL, output_format = NULL, generator = NULL, @@ -117,7 +147,8 @@ osm_cat <- function( progress = FALSE ) { assert_osmium_is_installed() - # validate inputs + + ## ---- argument validation ---- checkmate::assert_character(input_paths, any.missing = FALSE, min.len = 1) for (p in input_paths) { checkmate::assert_file_exists(p, access = "r") @@ -134,26 +165,86 @@ osm_cat <- function( choices = c("version", "timestamp", "changeset", "uid", "user"), empty.ok = TRUE ) - checkmate::assert_logical(buffer_data, len = 1) + checkmate::assert_flag(buffer_data) + check_integrity <- match.arg(check_integrity) + do_check <- check_integrity != "no" + check_rels <- identical(check_integrity, "full") + checkmate::assert_flag(stop_on_integrity_fail) + checkmate::assert_flag(sort_before_cat) if (!is.null(input_format)) checkmate::assert_string(input_format) if (!is.null(output_format)) checkmate::assert_string(output_format) if (!is.null(generator)) checkmate::assert_string(generator) - checkmate::assert_logical(overwrite, len = 1) - checkmate::assert_logical(fsync, len = 1) + checkmate::assert_flag(overwrite) + checkmate::assert_flag(fsync) if (!is.null(output_header)) { - checkmate::assert_character(output_header) checkmate::assert_named(output_header, type = "unique") + checkmate::assert_character(output_header, any.missing = FALSE) } - checkmate::assert_logical(echo_cmd, len = 1) - checkmate::assert_logical(echo, len = 1) - checkmate::assert_logical(spinner, len = 1) - checkmate::assert_logical(verbose, len = 1) - checkmate::assert_logical(progress, len = 1) + checkmate::assert_flag(echo_cmd) + checkmate::assert_flag(echo) + checkmate::assert_flag(spinner) + checkmate::assert_flag(verbose) + checkmate::assert_flag(progress) - # validate output path and extension - assert_output_path_multi_ext(output_path, overwrite) + ## ---- 1 referential-integrity checks ---- + if (do_check) { + for (p in input_paths) { + if (echo) cat("Checking integrity of", p, "...\n") + log <- osm_check_refs( + input_path = p, + show_ids = FALSE, + check_relations = check_rels, + input_format = input_format, + echo_cmd = echo_cmd, + echo = echo, + spinner = spinner, + verbose = verbose, + progress = progress + ) + if ( + isTRUE(stop_on_integrity_fail) && + !is.null(log$status_code) && + log$status_code == 1L + ) { + stop( + sprintf("Referential integrity check failed for '%s'.", p), + call. = FALSE + ) + } + } + } + + ## ---- 2 optional force-sort ---- + if (sort_before_cat) { + if (echo) cat("Sorting input files before cat (strategy = 'simple')...\n") + sorted_paths <- character(length(input_paths)) + for (i in seq_along(input_paths)) { + in_file <- input_paths[i] + ext <- tools::file_ext(in_file) + tmp_file <- tempfile(fileext = if (nzchar(ext)) paste0(".", ext) else "") + osm_sort( + input_paths = in_file, + output_path = tmp_file, + strategy = "simple", + input_format = input_format, + output_format = output_format, + fsync = fsync, + generator = generator, + overwrite = TRUE, + echo_cmd = echo_cmd, + echo = echo, + spinner = spinner, + verbose = verbose, + progress = progress + ) + sorted_paths[i] <- tmp_file + } + old <- sorted_paths + on.exit(unlink(old, recursive = FALSE, force = TRUE), add = TRUE) + input_paths <- sorted_paths + } - # build command-line arguments + ## ---- build command-line flags ---- clean_flags <- if (length(clean) > 0) paste0("--clean=", clean) else character() type_flags <- paste0("--object-type=", object_type) @@ -171,8 +262,8 @@ osm_cat <- function( names(output_header), function(nm) { val <- output_header[[nm]] - hdr <- if (nzchar(val)) paste0(nm, "=", val) else paste0(nm, "!") - paste0("--output-header=", hdr) + tag <- if (nzchar(val)) paste0(nm, "=", val) else paste0(nm, "!") + paste0("--output-header=", tag) }, character(1) )) @@ -198,8 +289,8 @@ osm_cat <- function( progress_flag ) - # run osmium - logs <- processx::run( + ## ---- run osmium cat ---- + processx::run( "osmium", args, echo = echo, diff --git a/man/osm_cat.Rd b/man/osm_cat.Rd index fe1bcb6..879770e 100644 --- a/man/osm_cat.Rd +++ b/man/osm_cat.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/cat.R \name{osm_cat} \alias{osm_cat} -\title{Concatenate OSM files} +\title{Concatenate OSM files with optional integrity checks and sorting} \usage{ osm_cat( input_paths, @@ -10,6 +10,9 @@ osm_cat( object_type = c("node", "way", "relation", "changeset"), clean = character(), buffer_data = FALSE, + check_integrity = c("no", "simple", "full"), + stop_on_integrity_fail = FALSE, + sort_before_cat = FALSE, input_format = NULL, output_format = NULL, generator = NULL, @@ -24,61 +27,99 @@ osm_cat( ) } \arguments{ -\item{input_paths}{Character vector of paths to one or more existing OSM files.} +\item{input_paths}{Character vector of one or more existing OSM files.} -\item{output_path}{String. Path to the file where the result will be written. -The format is inferred from the extension unless \code{output_format} is set.} +\item{output_path}{String. Path to write the concatenated result.} -\item{object_type}{Character vector of object types to include. -One or more of \code{"node"}, \code{"way"}, \code{"relation"}, -\code{"changeset"}. Defaults to all types.} +\item{object_type}{Character vector of types to include: one or more of +"node", "way", "relation", "changeset". Defaults to all types.} -\item{clean}{Character vector of attributes to clean (set to zero/empty). -One or more of \code{"version"}, \code{"timestamp"}, -\code{"changeset"}, \code{"uid"}, \code{"user"}.} +\item{clean}{Character vector of attributes to clean (set to zero/empty): +one or more of "version", "timestamp", "changeset", +"uid", "user". Defaults to none.} -\item{buffer_data}{Logical(1). If \code{TRUE}, read all inputs into memory +\item{buffer_data}{Logical(1). If \code{TRUE}, read all inputs into memory before writing. Defaults to \code{FALSE}.} -\item{input_format}{Optional string. Force the input format (e.g. -\code{"osm"}, \code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect.} +\item{check_integrity}{One of: +\itemize{ +\item "no" (default): skip integrity checks, +\item "simple": run \code{osm_check_refs(check_relations = FALSE)}, +\item "full": run \code{osm_check_refs(check_relations = TRUE)}. +}} -\item{output_format}{Optional string. Force the output format (e.g. -\code{"osm"}, \code{"pbf"}, \code{"osc"}). If \code{NULL}, autodetect.} +\item{stop_on_integrity_fail}{Logical. If \code{TRUE}, abort on failed +integrity check. Defaults to \code{FALSE}.} -\item{generator}{Optional string. Specify the \code{generator} header -(name and version) to add to the output file.} +\item{sort_before_cat}{Logical. If \code{TRUE}, force-sort each input with +\code{\link[=osm_sort]{osm_sort()}} before concatenation. Defaults to \code{FALSE}.} -\item{overwrite}{Logical(1). If \code{TRUE}, allow overwriting an existing +\item{input_format}{Optional string to force the input format +(e.g. "osm", "pbf"). \code{NULL} to autodetect.} + +\item{output_format}{Optional string to force the output format +(e.g. "osm", "pbf"). \code{NULL} to autodetect.} + +\item{generator}{Optional string for the \strong{\code{--generator}} header tag.} + +\item{overwrite}{Logical(1). If \code{TRUE}, allow overwriting an existing \code{output_path}. Defaults to \code{FALSE}.} -\item{fsync}{Logical(1). If \code{TRUE}, call \code{fsync} after writing -to force flushing buffers. Defaults to \code{FALSE}.} +\item{fsync}{Logical(1). If \code{TRUE}, call \strong{\code{--fsync}} after writing. +Defaults to \code{FALSE}.} -\item{output_header}{Named character vector of additional header options. -Each name is the header option and each value is the header value. -If the value is \code{""}, the option will be written as \code{OPTION!}. -Defaults to \code{NULL}.} +\item{output_header}{Named character vector of additional header options: +\code{OPTION = "value"} or \code{OPTION = ""} to replicate via \verb{OPTION!}.} -\item{echo_cmd}{A logical. Whether to print the Osmium command generated by the function call to the screen. +\item{echo_cmd}{Logical. Whether to print the generated Osmium command. Defaults to \code{FALSE}.} -\item{echo}{A logical. Whether to print the standard output and error generated by the Osmium call to the screen. +\item{echo}{Logical. Whether to print stdout and stderr of the Osmium call. Defaults to \code{TRUE}.} -\item{spinner}{A logical. Whether to show a reassuring spinner while the Osmium call is being executed. +\item{spinner}{Logical. Whether to show a spinner while running. Defaults to \code{TRUE}.} -\item{verbose}{A logical. Whether to display detailed information on the running command. Defaults to \code{FALSE}.} +\item{verbose}{Logical. Whether to display detailed command info. +Defaults to \code{FALSE}.} -\item{progress}{A logical. Whether to display a progress bar while running -the command. Defaults to \code{FALSE}.} +\item{progress}{Logical. Whether to show a progress bar. +Defaults to \code{FALSE}.} } \value{ -Invisibly returns the normalized path to \code{output_path}. +Invisibly returns the normalized \code{output_path}. } \description{ -This is a wrapper for \href{https://docs.osmcode.org/osmium/latest/osmium-cat.html}{osmium cat}. Just like the shell \code{osmium cat} command, \code{osm_cat} can take several input OSM files and concatenate them into one output file. As a side-effect, it can also convert between formats, filter by object type, clean metadata, and set output headers. +{ +Concatenates all input files and writes the result to the output file. The data is not sorted in any way but strictly copied from input to output. This is a wrapper for \strong{\href{https://docs.osmcode.org/osmium/latest/osmium-cat.html}{\verb{osmium cat}}} that can optionally (1) run referential-integrity pre-flight checks with \code{\link[=osm_check_refs]{osm_check_refs()}}, (2) force-sort each input with \code{\link[=osm_sort]{osm_sort()}}, and then (3) perform the actual concatenation. +\subsection{1. Referential-integrity checks}{ + +Set \code{check_integrity} to\tabular{lll}{ + value \tab what is checked \tab implementation \cr + "no" (def.) \tab nothing \tab - \cr + "simple" \tab \emph{nodes referenced by ways} are present \tab \code{osm_check_refs(check_relations = FALSE)} \cr + "full" \tab additionally checks \emph{all refs in relations} \tab \code{osm_check_refs(check_relations = TRUE)} \cr +} + + +If any check returns exit-status \code{1} (missing refs) \strong{and} +\code{stop_on_integrity_fail = TRUE}, the function \strong{aborts} before concatenating. +} + +\subsection{2. Optional force-sort step}{ + +If \code{sort_before_cat = TRUE}, each input file is first piped through +\code{\link[=osm_sort]{osm_sort()}} (strategy = "simple") into a temporary file. Temporary +files are cleaned up at the end of the R session. If you need permanent, +sorted copies, call \code{\link[=osm_sort]{osm_sort()}} yourself and set the desired output path. +} + +\subsection{3. Memory usage during concatenation}{ + +\verb{osmium cat} itself uses minimal buffers, but the optional sorting step +can dominate RAM usage (up to ~10x on-disk size). +} +} } \examples{ \dontshow{if (identical(tolower(Sys.getenv("NOT_CRAN")), "true")) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} From 7888a96c9fb17f3a80602097024b96ebee54cab4 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Mon, 5 May 2025 15:22:07 +0200 Subject: [PATCH 27/28] doc fixes for sort and merge --- R/merge.R | 70 +++++++++++++++++++++++++++++++++++++++++++---- R/sort.R | 2 ++ man/osm_merge.Rd | 71 +++++++++++++++++++++++++++++++++++++++++++++--- man/osm_sort.Rd | 3 ++ 4 files changed, 137 insertions(+), 9 deletions(-) diff --git a/R/merge.R b/R/merge.R index 643d37d..1696e39 100644 --- a/R/merge.R +++ b/R/merge.R @@ -17,7 +17,7 @@ #' If `sort_before_merge = TRUE`, each input file is first piped through #' [`osm_sort()`] (using the default `"simple"` strategy) into a temporary #' file. This strategy is fast but can require **about 10x the on-disk size -#' of a `.pbf`/`.osm.bz2` file in memory**. See [`?osm_sort`] for +#' of a `.pbf`/`.osm.bz2` file in memory**. See [`osm_sort()`] for #' details. Temporary files are deleted automatically at the end of the #' R session. If you need permanent, sorted copies, call #' [`osm_sort()`] yourself and set the desired output path. @@ -58,13 +58,73 @@ #' Osmium call is being executed. Defaults to `TRUE`. #' @param verbose A logical. Whether to display detailed information on the #' running command. Defaults to `FALSE`. +#' @param progress A logical. Whether to display a progress bar while running +#' the command. Defaults to `FALSE`. +#' +#' @examplesIf identical(tolower(Sys.getenv("NOT_CRAN")), "true") +#' pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") +#' +#' # 1. get header bbox (an sf::st_bbox) +#' orig_bbox <- osm_get_bbox(pbf) +#' +#' # 2. split bbox in two halves by longitude +#' midx <- (orig_bbox["xmin"] + orig_bbox["xmax"]) / 2 +#' +#' bbox1 <- sf::st_bbox( +#' c( +#' xmin = orig_bbox[["xmin"]], +#' ymin = orig_bbox[["ymin"]], +#' xmax = unname(midx), +#' ymax = orig_bbox[["ymax"]] +#' ), +#' crs = sf::st_crs(orig_bbox) +#' ) +#' +#' bbox2 <- sf::st_bbox( +#' c( +#' xmin = unname(midx), +#' ymin = orig_bbox[["ymin"]], +#' xmax = orig_bbox[["xmax"]], +#' ymax = orig_bbox[["ymax"]] +#' ), +#' crs = sf::st_crs(orig_bbox) +#' ) +#' +#' # 3. extract two halves +#' f1 <- extract( +#' pbf, +#' bbox1, +#' tempfile(fileext = ".osm.pbf"), +#' overwrite = TRUE, +#' echo = FALSE, +#' spinner = FALSE +#' ) +#' f2 <- extract( +#' pbf, +#' bbox2, +#' tempfile(fileext = ".osm.pbf"), +#' overwrite = TRUE, +#' echo = FALSE, +#' spinner = FALSE +#' ) +#' +#' # 4. merge while ensuring sort +#' merge_out <- tempfile(fileext = ".osm.pbf") +#' merged <- osm_merge( +#' input_paths = c(f1, f2), +#' output_path = merge_out, +#' sort_before_merge = TRUE, +#' overwrite = TRUE, +#' echo = FALSE, +#' spinner = FALSE +#' ) +#' +#' # 5. compute bbox +#' final_bbox <- osm_get_bbox(merged, calculate = TRUE) #' #' @return (Invisibly) the normalised `output_path`. -#' @keywords internal -#' @export -#' @return (Invisibly) the normalised `output_path`. -#' @keywords internal #' @export +#' osm_merge <- function( input_paths, output_path, diff --git a/R/sort.R b/R/sort.R index 4d5d190..6d8d082 100644 --- a/R/sort.R +++ b/R/sort.R @@ -56,6 +56,8 @@ #' Osmium call is being executed. Defaults to `TRUE`. #' @param verbose A logical. Whether to display detailed information on the #' running command. Defaults to `FALSE`. +#' @param progress A logical. Whether to display a progress bar while running +#' the command. Defaults to `FALSE`. #' #' @return Invisibly returns the normalised \code{output_path}. #' diff --git a/man/osm_merge.Rd b/man/osm_merge.Rd index 447b546..298fc2c 100644 --- a/man/osm_merge.Rd +++ b/man/osm_merge.Rd @@ -71,10 +71,11 @@ Osmium call is being executed. Defaults to \code{TRUE}.} \item{verbose}{A logical. Whether to display detailed information on the running command. Defaults to \code{FALSE}.} + +\item{progress}{A logical. Whether to display a progress bar while running +the command. Defaults to \code{FALSE}.} } \value{ -(Invisibly) the normalised \code{output_path}. - (Invisibly) the normalised \code{output_path}. } \description{ @@ -99,7 +100,7 @@ If any check returns exit-status \code{1} (missing refs) \strong{and} If \code{sort_before_merge = TRUE}, each input file is first piped through \code{\link[=osm_sort]{osm_sort()}} (using the default \code{"simple"} strategy) into a temporary file. This strategy is fast but can require \strong{about 10x the on-disk size -of a \code{.pbf}/\code{.osm.bz2} file in memory}. See \code{\link{?osm_sort}} for +of a \code{.pbf}/\code{.osm.bz2} file in memory}. See \code{\link[=osm_sort]{osm_sort()}} for details. Temporary files are deleted automatically at the end of the R session. If you need permanent, sorted copies, call \code{\link[=osm_sort]{osm_sort()}} yourself and set the desired output path. @@ -114,4 +115,66 @@ dominates RAM usage. } } } -\keyword{internal} +\examples{ +\dontshow{if (identical(tolower(Sys.getenv("NOT_CRAN")), "true")) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf} +pbf <- system.file("extdata/cur.osm.pbf", package = "rosmium") + +# 1. get header bbox (an sf::st_bbox) +orig_bbox <- osm_get_bbox(pbf) + +# 2. split bbox in two halves by longitude +midx <- (orig_bbox["xmin"] + orig_bbox["xmax"]) / 2 + +bbox1 <- sf::st_bbox( + c( + xmin = orig_bbox[["xmin"]], + ymin = orig_bbox[["ymin"]], + xmax = unname(midx), + ymax = orig_bbox[["ymax"]] + ), + crs = sf::st_crs(orig_bbox) +) + +bbox2 <- sf::st_bbox( + c( + xmin = unname(midx), + ymin = orig_bbox[["ymin"]], + xmax = orig_bbox[["xmax"]], + ymax = orig_bbox[["ymax"]] + ), + crs = sf::st_crs(orig_bbox) +) + +# 3. extract two halves +f1 <- extract( + pbf, + bbox1, + tempfile(fileext = ".osm.pbf"), + overwrite = TRUE, + echo = FALSE, + spinner = FALSE +) +f2 <- extract( + pbf, + bbox2, + tempfile(fileext = ".osm.pbf"), + overwrite = TRUE, + echo = FALSE, + spinner = FALSE +) + +# 4. merge while ensuring sort +merge_out <- tempfile(fileext = ".osm.pbf") +merged <- osm_merge( + input_paths = c(f1, f2), + output_path = merge_out, + sort_before_merge = TRUE, + overwrite = TRUE, + echo = FALSE, + spinner = FALSE +) + +# 5. compute bbox +final_bbox <- osm_get_bbox(merged, calculate = TRUE) +\dontshow{\}) # examplesIf} +} diff --git a/man/osm_sort.Rd b/man/osm_sort.Rd index 718730f..936d2d1 100644 --- a/man/osm_sort.Rd +++ b/man/osm_sort.Rd @@ -63,6 +63,9 @@ Osmium call is being executed. Defaults to \code{TRUE}.} \item{verbose}{A logical. Whether to display detailed information on the running command. Defaults to \code{FALSE}.} + +\item{progress}{A logical. Whether to display a progress bar while running +the command. Defaults to \code{FALSE}.} } \value{ Invisibly returns the normalised \code{output_path}. From f1fe5d46ae78745dec4d8c667a509424993c6c81 Mon Sep 17 00:00:00 2001 From: Egor Kotov Date: Sat, 7 Feb 2026 11:26:33 +0100 Subject: [PATCH 28/28] ignore macos ds_store --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2ceb3d0..2486f8d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ .Ruserdata README.html private +.DS_Store