diff --git a/DESCRIPTION b/DESCRIPTION index a219cd8..1f3436b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,14 +1,14 @@ Package: debiasRdata Title: Empirical Data Companion for debiasR -Version: 0.0.0.9002 +Version: 0.0.0.9003 Authors@R: c( person("Francisco", "Rowe", email = "fcorowe@liverpool.ac.uk", role = c("aut", "cre")), person("Carmen", "Cabrera", email = "c.cabrera@liverpool.ac.uk", role = "aut") ) Description: Documented empirical MSOA and LAD/LTLA travel-to-work - origin-destination data for debiasR examples, vignettes, validation - workflows, and empirical tests. + origin-destination data, area covariates, and supporting geography assets + for debiasR examples, vignettes, validation workflows, and empirical tests. The package supplies data assets only; bias adjustment, modelling, validation, and example-data loading logic remain in debiasR. License: file LICENSE diff --git a/R/data.R b/R/data.R index 5b3fc0a..33263c9 100644 --- a/R/data.R +++ b/R/data.R @@ -132,3 +132,74 @@ #' #' @seealso [lad_OD_travel2work] "lad_centroids" + +#' Census 2021 MSOA covariates +#' +#' Selected Census 2021 topic-summary covariates for Middle Layer Super Output +#' Areas (MSOAs), using the requested label-5 variable names from +#' `dictionary.csv`. +#' +#' @format A data frame with 7,264 rows and 10 variables: +#' \describe{ +#' \item{area}{Character. MSOA21 code.} +#' \item{name}{Character. MSOA name.} +#' \item{year}{Integer. Census year.} +#' \item{per_ukborn}{Numeric. Percentage UK-born.} +#' \item{per_age_20.29}{Numeric. Percentage aged 20-29.} +#' \item{per_age_70plus}{Numeric. Percentage aged 70 or over.} +#' \item{per_level4}{Numeric. Percentage with level 4 qualifications.} +#' \item{per_hh_no_centralheat}{Numeric. Percentage of households without +#' central heating.} +#' \item{per_NS_SeC_L13_routine}{Numeric. Percentage in NS-SeC L13 routine +#' occupations.} +#' \item{rural_pct}{Numeric. Percentage rural.} +#' } +#' +#' @source Office for National Statistics Census 2021 topic-summary inputs, +#' `combined-data-msoa.csv`, with variable names selected from +#' `dictionary.csv`. Source data are available under the Open Government +#' Licence v3.0. +#' +#' @details The packaged table renames `geography.code` to `area`, +#' `geography` to `name`, and `date` to `year`. The source column +#' `per_age_20-29` is renamed to the label-5/R-facing name +#' `per_age_20.29`. All covariates are finite numeric percentages between +#' 0 and 100, and the areas cover the packaged MSOA OD flow assets. +#' +#' @seealso [msoa_OD_travel2work], [census_msoa_OD_travel2work] +"msoa_covariates" + +#' Census 2021 LAD covariates +#' +#' Selected Census 2021 topic-summary covariates for Local Authority Districts +#' (LADs), using the requested label-5 variable names from `dictionary.csv`. +#' +#' @format A data frame with 331 rows and 10 variables: +#' \describe{ +#' \item{area}{Character. LAD/LTLA area code.} +#' \item{name}{Character. LAD/LTLA name.} +#' \item{year}{Integer. Census year.} +#' \item{per_ukborn}{Numeric. Percentage UK-born.} +#' \item{per_age_20.29}{Numeric. Percentage aged 20-29.} +#' \item{per_age_70plus}{Numeric. Percentage aged 70 or over.} +#' \item{per_level4}{Numeric. Percentage with level 4 qualifications.} +#' \item{per_hh_no_centralheat}{Numeric. Percentage of households without +#' central heating.} +#' \item{per_NS_SeC_L13_routine}{Numeric. Percentage in NS-SeC L13 routine +#' occupations.} +#' \item{rural_pct}{Numeric. Percentage rural.} +#' } +#' +#' @source Office for National Statistics Census 2021 topic-summary inputs, +#' `combined-data-lad.csv`, with variable names selected from +#' `dictionary.csv`. Source data are available under the Open Government +#' Licence v3.0. +#' +#' @details The packaged table renames `geography.code` to `area`, +#' `geography` to `name`, and `date` to `year`. The source column +#' `per_age_20-29` is renamed to the label-5/R-facing name +#' `per_age_20.29`. All covariates are finite numeric percentages between +#' 0 and 100, and the areas cover the packaged LAD/LTLA OD flow assets. +#' +#' @seealso [lad_OD_travel2work], [census_lad_OD_travel2work] +"lad_covariates" diff --git a/R/paths.R b/R/paths.R index 4cca7d9..1205d5c 100644 --- a/R/paths.R +++ b/R/paths.R @@ -7,7 +7,7 @@ #' @param name Character scalar naming a known packaged asset. Accepted values #' are `"msoa_OD_travel2work"`, `"census_msoa_OD_travel2work"`, #' `"lad_OD_travel2work"`, `"census_lad_OD_travel2work"`, and -#' `"lad_centroids"`. +#' `"lad_centroids"`, `"msoa_covariates"`, and `"lad_covariates"`. #' #' @return A character scalar containing the installed file path. #' @export @@ -20,7 +20,9 @@ debiasRdata_path <- function(name) { census_msoa_OD_travel2work = "census_msoa_OD_travel2work.csv.gz", lad_OD_travel2work = "lad_OD_travel2work.csv.gz", census_lad_OD_travel2work = "census_lad_OD_travel2work.csv.gz", - lad_centroids = "lad_centroids.csv.gz" + lad_centroids = "lad_centroids.csv.gz", + msoa_covariates = "msoa_covariates.csv.gz", + lad_covariates = "lad_covariates.csv.gz" ) if (!is.character(name) || length(name) != 1L || is.na(name) || !nzchar(name)) { diff --git a/README.md b/README.md index 7a262e8..5fba495 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,9 @@ directly by `ODWP01EW_LTLA.csv`. lets `debiasR` compute selected-area LAD distances without packaging a full OD distance matrix. +`msoa_covariates` and `lad_covariates` contain selected Census 2021 +topic-summary covariates for MSOA and LAD/LTLA areas. + The four OD datasets use the same normalized schema: | Column | Type | Meaning | @@ -53,6 +56,21 @@ The four OD datasets use the same normalized schema: | `destination` | character | Destination MSOA or LAD/LTLA code | | `flow` | numeric | Non-negative OD flow | +The two covariate datasets use this schema: + +| Column | Type | Meaning | +| --- | --- | --- | +| `area` | character | MSOA or LAD/LTLA code | +| `name` | character | Area name | +| `year` | integer | Census year | +| `per_ukborn` | numeric | Percentage UK-born | +| `per_age_20.29` | numeric | Percentage aged 20-29 | +| `per_age_70plus` | numeric | Percentage aged 70 or over | +| `per_level4` | numeric | Percentage with level 4 qualifications | +| `per_hh_no_centralheat` | numeric | Percentage of households without central heating | +| `per_NS_SeC_L13_routine` | numeric | Percentage in NS-SeC L13 routine occupations | +| `rural_pct` | numeric | Percentage rural | + ## Usage ```r @@ -63,6 +81,8 @@ head(census_msoa_OD_travel2work) head(lad_OD_travel2work) head(census_lad_OD_travel2work) head(lad_centroids) +head(msoa_covariates) +head(lad_covariates) ``` `debiasR` consumes these objects conditionally: @@ -83,6 +103,8 @@ debiasRdata_path("census_msoa_OD_travel2work") debiasRdata_path("lad_OD_travel2work") debiasRdata_path("census_lad_OD_travel2work") debiasRdata_path("lad_centroids") +debiasRdata_path("msoa_covariates") +debiasRdata_path("lad_covariates") ``` The helper only locates installed files. It does not download data during @@ -139,6 +161,19 @@ The LAD centroid table is derived from: northing, and longitude / latitude fields were retained and renamed to `area`, `name`, `easting`, `northing`, `longitude`, and `latitude`. +The covariate tables are derived from: + +- Collection: Census 2021 topic-summary inputs +- Source files: `combined-data-msoa.csv`, `combined-data-lad.csv`, and + `dictionary.csv` +- Source institution: Office for National Statistics +- License: Open Government Licence v3.0, + +- Packaged modifications: `geography.code` was renamed to `area`, `geography` + to `name`, and `date` to `year`; the seven requested label-5 covariates were + retained; source column `per_age_20-29` was renamed to the label-5/R-facing + name `per_age_20.29`. + Build metadata, checksums, row counts, and transformation notes are recorded in `inst/metadata/source-metadata.json`. diff --git a/data-raw/build_data.R b/data-raw/build_data.R index 2401d78..f1e8729 100644 --- a/data-raw/build_data.R +++ b/data-raw/build_data.R @@ -24,9 +24,14 @@ zenodo_download_url <- paste0( nomis_source_url <- "https://www.nomisweb.co.uk/sources/census_2021_od" nomis_download_url <- "https://www.nomisweb.co.uk/output/census/2021/odwp01ew.zip" +census_topic_summary_url <- "https://www.ons.gov.uk/census/aboutcensus/censusproducts/topicsummaries" local_census_root <- "/Volumes/DEBIAS/data/inputs/census/odwp01ew" local_lookup_root <- "/Volumes/DEBIAS/data/inputs/geographies/lookup21ew" local_boundary_root <- "/Volumes/DEBIAS/data/inputs/geographies/boundaries" +local_covariate_roots <- c( + "/Volumes/rdm04/DEBIAS/data/inputs/census/census2021-ts", + "/Volumes/DEBIAS/data/inputs/census/census2021-ts" +) oa_msoa_lad_lookup_url <- paste0( "https://open-geography-portalx-ons.hub.arcgis.com/api/download/v1/items/", "b9ca90c10aaa4b8d9791e9859a38ca67/csv?layers=0" @@ -101,6 +106,42 @@ lad_boundary_path <- find_first_existing(c( file.path(repo_root, "data-raw", "source", "LAD_Dec_2021_GB_BFC_2022.gpkg") )) +covariate_root <- find_first_existing(c( + local_covariate_roots, + file.path(repo_root, "data-raw", "source", "census2021-ts") +)) +if (is.null(covariate_root)) { + stop( + "Covariate build requires dictionary.csv, combined-data-lad.csv, ", + "and combined-data-msoa.csv under one of: ", + paste(c(local_covariate_roots, file.path(repo_root, "data-raw", "source", "census2021-ts")), + collapse = ", " + ) + ) +} +covariate_dictionary_path <- file.path(covariate_root, "dictionary.csv") +lad_covariate_source_path <- file.path(covariate_root, "combined-data-lad.csv") +msoa_covariate_source_path <- file.path(covariate_root, "combined-data-msoa.csv") +missing_covariate_sources <- c( + covariate_dictionary_path, + lad_covariate_source_path, + msoa_covariate_source_path +) +missing_covariate_sources <- missing_covariate_sources[!file.exists(missing_covariate_sources)] +if (length(missing_covariate_sources) > 0L) { + stop("Missing covariate source files: ", paste(missing_covariate_sources, collapse = ", ")) +} + +covariate_columns <- c( + "per_ukborn", + "per_age_20.29", + "per_age_70plus", + "per_level4", + "per_hh_no_centralheat", + "per_NS_SeC_L13_routine", + "rural_pct" +) + normalize_od <- function(df, origin_col, destination_col, flow_col) { out <- data.frame( origin = as.character(df[[origin_col]]), @@ -171,6 +212,37 @@ validate_centroids <- function(df, name) { invisible(TRUE) } +validate_covariates <- function(df, name, area_regex) { + required <- c("area", "name", "year", covariate_columns) + if (!identical(names(df), required)) { + stop(name, " must have columns: ", paste(required, collapse = ", ")) + } + if (!is.character(df$area) || !is.character(df$name)) { + stop(name, "$area and $name must be character.") + } + if (!is.integer(df$year)) { + stop(name, "$year must be integer.") + } + if (anyNA(df$area) || anyNA(df$name) || anyNA(df$year)) { + stop(name, " must not contain missing area identifiers, names, or years.") + } + if (any(!grepl(area_regex, df$area))) { + stop(name, "$area contains codes outside the expected geography format.") + } + if (anyDuplicated(df$area) > 0L) { + stop(name, " must not contain duplicate area codes.") + } + for (column in covariate_columns) { + if (!is.numeric(df[[column]])) { + stop(name, "$", column, " must be numeric.") + } + if (any(!is.finite(df[[column]]) | df[[column]] < 0 | df[[column]] > 100)) { + stop(name, "$", column, " must contain finite percentages between 0 and 100.") + } + } + invisible(TRUE) +} + build_lad_od_from_msoa <- function(df, lookup) { od <- data.frame( origin_msoa = as.character(df[["MSOA21CD_home"]]), @@ -230,6 +302,73 @@ build_lad_centroids <- function(path) { out } +read_covariate_dictionary <- function(path) { + dictionary <- utils::read.csv( + path, + check.names = FALSE, + stringsAsFactors = FALSE + ) + dictionary[] <- lapply(dictionary, function(x) { + if (is.character(x)) { + trimws(x) + } else { + x + } + }) + dictionary +} + +resolve_covariate_sources <- function(dictionary, raw_names) { + source_columns <- character(length(covariate_columns)) + names(source_columns) <- covariate_columns + + for (column in covariate_columns) { + row <- dictionary[trimws(dictionary[["var_label5"]]) == column, , drop = FALSE] + if (nrow(row) != 1L) { + stop("Expected one dictionary row with var_label5 == ", column, ".") + } + + candidates <- unique(trimws(unlist( + row[paste0("var_label", 0:5)], + use.names = FALSE + ))) + candidates <- candidates[nzchar(candidates)] + match <- candidates[candidates %in% raw_names] + if (length(match) == 0L) { + stop("Could not resolve source column for covariate ", column, ".") + } + source_columns[[column]] <- match[[1]] + } + + source_columns +} + +build_covariates <- function(path, source_columns) { + raw <- utils::read.csv( + path, + check.names = FALSE, + stringsAsFactors = FALSE + ) + required_raw <- c("date", "geography", "geography.code", unname(source_columns)) + missing_raw <- setdiff(required_raw, names(raw)) + if (length(missing_raw) > 0L) { + stop("Missing covariate source columns: ", paste(missing_raw, collapse = ", ")) + } + + out <- data.frame( + area = as.character(raw[["geography.code"]]), + name = as.character(raw[["geography"]]), + year = as.integer(raw[["date"]]), + stringsAsFactors = FALSE + ) + for (column in covariate_columns) { + out[[column]] <- suppressWarnings(as.numeric(raw[[source_columns[[column]]]])) + } + out <- out[order(out$area), , drop = FALSE] + row.names(out) <- NULL + out +} + write_csv_gz <- function(df, path) { connection <- gzfile(path, open = "wt") on.exit(close(connection), add = TRUE) @@ -302,6 +441,23 @@ packaged_metadata <- function(object_name, df, csv_path) { ) } +covariate_dictionary_metadata <- function(dictionary, source_columns) { + out <- vector("list", length(covariate_columns)) + names(out) <- covariate_columns + + for (column in covariate_columns) { + row <- dictionary[trimws(dictionary[["var_label5"]]) == column, , drop = FALSE] + out[[column]] <- list( + source_column = unname(source_columns[[column]]), + dictionary_group = row[["group"]][[1]], + dictionary_label2 = row[["var_label2"]][[1]], + dictionary_label5 = column + ) + } + + out +} + mpd_raw <- utils::read.csv( mpd_source_path, check.names = FALSE, @@ -409,6 +565,60 @@ census_lad_OD_travel2work <- normalize_od( ) validate_od(census_lad_OD_travel2work, "census_lad_OD_travel2work") +covariate_dictionary <- read_covariate_dictionary(covariate_dictionary_path) +lad_covariate_header <- names(utils::read.csv( + lad_covariate_source_path, + check.names = FALSE, + stringsAsFactors = FALSE, + nrows = 0 +)) +covariate_source_columns <- resolve_covariate_sources( + covariate_dictionary, + lad_covariate_header +) +msoa_covariates <- build_covariates( + msoa_covariate_source_path, + covariate_source_columns +) +validate_covariates(msoa_covariates, "msoa_covariates", "^[EW][0-9]{8}$") +lad_covariates <- build_covariates( + lad_covariate_source_path, + covariate_source_columns +) +validate_covariates(lad_covariates, "lad_covariates", "^[EW][0-9]{8}$") + +missing_msoa_covariates <- setdiff( + unique(c( + msoa_OD_travel2work$origin, + msoa_OD_travel2work$destination, + census_msoa_OD_travel2work$origin, + census_msoa_OD_travel2work$destination + )), + msoa_covariates$area +) +if (length(missing_msoa_covariates) > 0L) { + stop( + "msoa_covariates is missing MSOA codes used by packaged OD assets: ", + paste(missing_msoa_covariates, collapse = ", ") + ) +} + +missing_lad_covariates <- setdiff( + unique(c( + lad_OD_travel2work$origin, + lad_OD_travel2work$destination, + census_lad_OD_travel2work$origin, + census_lad_OD_travel2work$destination + )), + lad_covariates$area +) +if (length(missing_lad_covariates) > 0L) { + stop( + "lad_covariates is missing LAD/LTLA codes used by packaged OD assets: ", + paste(missing_lad_covariates, collapse = ", ") + ) +} + save( msoa_OD_travel2work, file = file.path(repo_root, "data", "msoa_OD_travel2work.rda"), @@ -434,17 +644,31 @@ save( file = file.path(repo_root, "data", "lad_centroids.rda"), compress = "xz" ) +save( + msoa_covariates, + file = file.path(repo_root, "data", "msoa_covariates.rda"), + compress = "xz" +) +save( + lad_covariates, + file = file.path(repo_root, "data", "lad_covariates.rda"), + compress = "xz" +) mpd_extdata_path <- file.path(repo_root, "inst", "extdata", "msoa_OD_travel2work.csv.gz") census_extdata_path <- file.path(repo_root, "inst", "extdata", "census_msoa_OD_travel2work.csv.gz") lad_mpd_extdata_path <- file.path(repo_root, "inst", "extdata", "lad_OD_travel2work.csv.gz") lad_census_extdata_path <- file.path(repo_root, "inst", "extdata", "census_lad_OD_travel2work.csv.gz") lad_centroids_extdata_path <- file.path(repo_root, "inst", "extdata", "lad_centroids.csv.gz") +msoa_covariates_extdata_path <- file.path(repo_root, "inst", "extdata", "msoa_covariates.csv.gz") +lad_covariates_extdata_path <- file.path(repo_root, "inst", "extdata", "lad_covariates.csv.gz") write_csv_gz(msoa_OD_travel2work, mpd_extdata_path) write_csv_gz(census_msoa_OD_travel2work, census_extdata_path) write_csv_gz(lad_OD_travel2work, lad_mpd_extdata_path) write_csv_gz(census_lad_OD_travel2work, lad_census_extdata_path) write_csv_gz(lad_centroids, lad_centroids_extdata_path) +write_csv_gz(msoa_covariates, msoa_covariates_extdata_path) +write_csv_gz(lad_covariates, lad_covariates_extdata_path) metadata <- list( package = "debiasRdata", @@ -544,6 +768,58 @@ metadata <- list( "Aggregated duplicate origin-destination pairs by summed flow." ) ), + msoa_covariates = list( + title = "Census 2021 MSOA covariates", + object_name = "msoa_covariates", + source_institution = "Office for National Statistics", + source_collection = "Census 2021 topic summaries", + source_record_url = census_topic_summary_url, + source_file_name = "combined-data-msoa.csv", + source_file_size_bytes = file_size(msoa_covariate_source_path), + source_sha256 = file_sha256(msoa_covariate_source_path), + dictionary_file_name = "dictionary.csv", + dictionary_file_size_bytes = file_size(covariate_dictionary_path), + dictionary_sha256 = file_sha256(covariate_dictionary_path), + source_license = "Open Government Licence v3.0", + source_license_url = "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/", + covariates = covariate_dictionary_metadata(covariate_dictionary, covariate_source_columns), + packaged_metadata("msoa_covariates", msoa_covariates, msoa_covariates_extdata_path), + transformations = c( + "Read combined-data-msoa.csv from the local Census 2021 topic-summary covariate source.", + "Selected the requested covariates using dictionary.csv var_label5 names.", + "Renamed geography.code to area, geography to name, and date to year.", + "Kept per_ukborn, per_age_20.29, per_age_70plus, per_level4, per_hh_no_centralheat, per_NS_SeC_L13_routine, and rural_pct.", + "Renamed source column per_age_20-29 to label-5/R-facing name per_age_20.29.", + "Coerced year to integer and covariates to numeric percentages.", + "Validated unique MSOA area codes, non-missing identifiers, finite covariates, and coverage of packaged MSOA OD areas." + ) + ), + lad_covariates = list( + title = "Census 2021 LAD covariates", + object_name = "lad_covariates", + source_institution = "Office for National Statistics", + source_collection = "Census 2021 topic summaries", + source_record_url = census_topic_summary_url, + source_file_name = "combined-data-lad.csv", + source_file_size_bytes = file_size(lad_covariate_source_path), + source_sha256 = file_sha256(lad_covariate_source_path), + dictionary_file_name = "dictionary.csv", + dictionary_file_size_bytes = file_size(covariate_dictionary_path), + dictionary_sha256 = file_sha256(covariate_dictionary_path), + source_license = "Open Government Licence v3.0", + source_license_url = "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/", + covariates = covariate_dictionary_metadata(covariate_dictionary, covariate_source_columns), + packaged_metadata("lad_covariates", lad_covariates, lad_covariates_extdata_path), + transformations = c( + "Read combined-data-lad.csv from the local Census 2021 topic-summary covariate source.", + "Selected the requested covariates using dictionary.csv var_label5 names.", + "Renamed geography.code to area, geography to name, and date to year.", + "Kept per_ukborn, per_age_20.29, per_age_70plus, per_level4, per_hh_no_centralheat, per_NS_SeC_L13_routine, and rural_pct.", + "Renamed source column per_age_20-29 to label-5/R-facing name per_age_20.29.", + "Coerced year to integer and covariates to numeric percentages.", + "Validated unique LAD/LTLA area codes, non-missing identifiers, finite covariates, and coverage of packaged LAD/LTLA OD areas." + ) + ), lad_centroids = list( title = "LAD centroid coordinates", object_name = "lad_centroids", diff --git a/data/lad_covariates.rda b/data/lad_covariates.rda new file mode 100644 index 0000000..58e65fc Binary files /dev/null and b/data/lad_covariates.rda differ diff --git a/data/msoa_covariates.rda b/data/msoa_covariates.rda new file mode 100644 index 0000000..81d249b Binary files /dev/null and b/data/msoa_covariates.rda differ diff --git a/inst/CITATION b/inst/CITATION index 5b15720..1b25820 100644 --- a/inst/CITATION +++ b/inst/CITATION @@ -6,7 +6,7 @@ bibentry( person("Carmen", "Cabrera") ), year = "2026", - note = "R package version 0.0.0.9002", + note = "R package version 0.0.0.9003", url = "https://github.com/de-bias/debiasRdata" ) @@ -46,3 +46,12 @@ bibentry( url = "https://www.data.gov.uk/dataset/local-authority-districts-december-2021-gb-bfc", note = "Used to derive lad_centroids; licensed under the Open Government Licence v3.0" ) + +bibentry( + bibtype = "Misc", + title = "Census 2021 topic summaries", + author = "{Office for National Statistics}", + year = "2023", + url = "https://www.ons.gov.uk/census/aboutcensus/censusproducts/topicsummaries", + note = "Used to derive msoa_covariates and lad_covariates; licensed under the Open Government Licence v3.0" +) diff --git a/inst/extdata/lad_covariates.csv.gz b/inst/extdata/lad_covariates.csv.gz new file mode 100644 index 0000000..05395a9 Binary files /dev/null and b/inst/extdata/lad_covariates.csv.gz differ diff --git a/inst/extdata/msoa_covariates.csv.gz b/inst/extdata/msoa_covariates.csv.gz new file mode 100644 index 0000000..df40eb1 Binary files /dev/null and b/inst/extdata/msoa_covariates.csv.gz differ diff --git a/inst/metadata/debiasR-integration.md b/inst/metadata/debiasR-integration.md index 5c841a2..af0fd5c 100644 --- a/inst/metadata/debiasR-integration.md +++ b/inst/metadata/debiasR-integration.md @@ -13,6 +13,8 @@ Recommended `debiasR` contract: - `lad_OD_travel2work` - `census_lad_OD_travel2work` - `lad_centroids` + - `msoa_covariates` + - `lad_covariates` - Expect the stable schema: - `origin`: character - `destination`: character @@ -21,6 +23,13 @@ Recommended `debiasR` contract: - `area`: character LAD code - `name`: character LAD name - `easting`, `northing`, `longitude`, `latitude`: finite numeric coordinates +- For `msoa_covariates` and `lad_covariates`, expect: + - `area`: character MSOA or LAD/LTLA code + - `name`: character area name + - `year`: integer Census year + - `per_ukborn`, `per_age_20.29`, `per_age_70plus`, `per_level4`, + `per_hh_no_centralheat`, `per_NS_SeC_L13_routine`, `rural_pct`: finite + numeric percentages - Continue to keep modelling, adjustment, validation, and example-data loading logic in `debiasR`. - Use `lad_centroids` to compute selected-area LAD distances in `debiasR` diff --git a/inst/metadata/source-metadata.json b/inst/metadata/source-metadata.json index aece466..387b471 100644 --- a/inst/metadata/source-metadata.json +++ b/inst/metadata/source-metadata.json @@ -114,6 +114,164 @@ }, "transformations": ["Read ODWP01EW_LTLA.csv from the local Census source if present, otherwise from the official Nomis ODWP01EW zip.", "Filtered to Place of work indicator (4 categories) code == 3.", "Renamed Lower tier local authorities code to origin.", "Renamed LTLA of workplace code to destination.", "Renamed Count to flow and coerced to numeric.", "Kept only valid England/Wales local authority origin and destination codes.", "Removed missing OD identifiers and non-finite or negative flows.", "Aggregated duplicate origin-destination pairs by summed flow."] }, + "msoa_covariates": { + "title": "Census 2021 MSOA covariates", + "object_name": "msoa_covariates", + "source_institution": "Office for National Statistics", + "source_collection": "Census 2021 topic summaries", + "source_record_url": "https://www.ons.gov.uk/census/aboutcensus/censusproducts/topicsummaries", + "source_file_name": "combined-data-msoa.csv", + "source_file_size_bytes": 4353595, + "source_sha256": "7e4860a87d823be31b195ba388e55e1ad9e44f94f95ab1d42ef6a484be426f6b", + "dictionary_file_name": "dictionary.csv", + "dictionary_file_size_bytes": 6236, + "dictionary_sha256": "b1886feff558b1f56b5b2d9c0868b6d9a84ebe45fa1b13c86690f968793b13eb", + "source_license": "Open Government Licence v3.0", + "source_license_url": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/", + "covariates": { + "per_ukborn": { + "source_column": "per_ukborn", + "dictionary_group": "excluded", + "dictionary_label2": "UK (%)", + "dictionary_label5": "per_ukborn" + }, + "per_age_20.29": { + "source_column": "per_age_20-29", + "dictionary_group": "demographic", + "dictionary_label2": "Age 20-29 (%)", + "dictionary_label5": "per_age_20.29" + }, + "per_age_70plus": { + "source_column": "per_age_70plus", + "dictionary_group": "demographic", + "dictionary_label2": "Age 70+ (%)", + "dictionary_label5": "per_age_70plus" + }, + "per_level4": { + "source_column": "per_level4", + "dictionary_group": "socioeconomic", + "dictionary_label2": "Level 4 qual. (%)", + "dictionary_label5": "per_level4" + }, + "per_hh_no_centralheat": { + "source_column": "per_hh_no_centralheat", + "dictionary_group": "resource_access", + "dictionary_label2": "No central heating (%)", + "dictionary_label5": "per_hh_no_centralheat" + }, + "per_NS_SeC_L13_routine": { + "source_column": "per_NS_SeC_L13_routine", + "dictionary_group": "socioeconomic", + "dictionary_label2": "Routine (%)", + "dictionary_label5": "per_NS_SeC_L13_routine" + }, + "rural_pct": { + "source_column": "rural_pct", + "dictionary_group": "geographic", + "dictionary_label2": "Rural (%)", + "dictionary_label5": "rural_pct" + } + }, + "packaged_csv_file": "inst/extdata/msoa_covariates.csv.gz", + "packaged_csv_size_bytes": 429337, + "packaged_csv_sha256": "581c314cdc15872a454bd7fdc09afd47ff9b08d7fec49886e70a8103325409fb", + "packaged_rda_file": "data/msoa_covariates.rda", + "packaged_rda_size_bytes": 333820, + "packaged_rda_sha256": "dde94c534219a8baae508dddb123da1d2b09295e8624af07c24809b95d6912ca", + "row_count": 7264, + "columns": { + "area": "character", + "name": "character", + "year": "integer", + "per_ukborn": "numeric", + "per_age_20.29": "numeric", + "per_age_70plus": "numeric", + "per_level4": "numeric", + "per_hh_no_centralheat": "numeric", + "per_NS_SeC_L13_routine": "numeric", + "rural_pct": "numeric" + }, + "transformations": ["Read combined-data-msoa.csv from the local Census 2021 topic-summary covariate source.", "Selected the requested covariates using dictionary.csv var_label5 names.", "Renamed geography.code to area, geography to name, and date to year.", "Kept per_ukborn, per_age_20.29, per_age_70plus, per_level4, per_hh_no_centralheat, per_NS_SeC_L13_routine, and rural_pct.", "Renamed source column per_age_20-29 to label-5/R-facing name per_age_20.29.", "Coerced year to integer and covariates to numeric percentages.", "Validated unique MSOA area codes, non-missing identifiers, finite covariates, and coverage of packaged MSOA OD areas."] + }, + "lad_covariates": { + "title": "Census 2021 LAD covariates", + "object_name": "lad_covariates", + "source_institution": "Office for National Statistics", + "source_collection": "Census 2021 topic summaries", + "source_record_url": "https://www.ons.gov.uk/census/aboutcensus/censusproducts/topicsummaries", + "source_file_name": "combined-data-lad.csv", + "source_file_size_bytes": 202623, + "source_sha256": "edfdd055ab68652fa4233272528d384240c22f38a26e3bb3038145e800545677", + "dictionary_file_name": "dictionary.csv", + "dictionary_file_size_bytes": 6236, + "dictionary_sha256": "b1886feff558b1f56b5b2d9c0868b6d9a84ebe45fa1b13c86690f968793b13eb", + "source_license": "Open Government Licence v3.0", + "source_license_url": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/", + "covariates": { + "per_ukborn": { + "source_column": "per_ukborn", + "dictionary_group": "excluded", + "dictionary_label2": "UK (%)", + "dictionary_label5": "per_ukborn" + }, + "per_age_20.29": { + "source_column": "per_age_20-29", + "dictionary_group": "demographic", + "dictionary_label2": "Age 20-29 (%)", + "dictionary_label5": "per_age_20.29" + }, + "per_age_70plus": { + "source_column": "per_age_70plus", + "dictionary_group": "demographic", + "dictionary_label2": "Age 70+ (%)", + "dictionary_label5": "per_age_70plus" + }, + "per_level4": { + "source_column": "per_level4", + "dictionary_group": "socioeconomic", + "dictionary_label2": "Level 4 qual. (%)", + "dictionary_label5": "per_level4" + }, + "per_hh_no_centralheat": { + "source_column": "per_hh_no_centralheat", + "dictionary_group": "resource_access", + "dictionary_label2": "No central heating (%)", + "dictionary_label5": "per_hh_no_centralheat" + }, + "per_NS_SeC_L13_routine": { + "source_column": "per_NS_SeC_L13_routine", + "dictionary_group": "socioeconomic", + "dictionary_label2": "Routine (%)", + "dictionary_label5": "per_NS_SeC_L13_routine" + }, + "rural_pct": { + "source_column": "rural_pct", + "dictionary_group": "geographic", + "dictionary_label2": "Rural (%)", + "dictionary_label5": "rural_pct" + } + }, + "packaged_csv_file": "inst/extdata/lad_covariates.csv.gz", + "packaged_csv_size_bytes": 23717, + "packaged_csv_sha256": "81419c30af253fffae1f3e9ed63ccb8bd73deb6004336951743cee990902144c", + "packaged_rda_file": "data/lad_covariates.rda", + "packaged_rda_size_bytes": 19692, + "packaged_rda_sha256": "680c3d373abb3c8636ca63ee2fbfd9c28cf612bf8add2c0e592b1f961ad65ac9", + "row_count": 331, + "columns": { + "area": "character", + "name": "character", + "year": "integer", + "per_ukborn": "numeric", + "per_age_20.29": "numeric", + "per_age_70plus": "numeric", + "per_level4": "numeric", + "per_hh_no_centralheat": "numeric", + "per_NS_SeC_L13_routine": "numeric", + "rural_pct": "numeric" + }, + "transformations": ["Read combined-data-lad.csv from the local Census 2021 topic-summary covariate source.", "Selected the requested covariates using dictionary.csv var_label5 names.", "Renamed geography.code to area, geography to name, and date to year.", "Kept per_ukborn, per_age_20.29, per_age_70plus, per_level4, per_hh_no_centralheat, per_NS_SeC_L13_routine, and rural_pct.", "Renamed source column per_age_20-29 to label-5/R-facing name per_age_20.29.", "Coerced year to integer and covariates to numeric percentages.", "Validated unique LAD/LTLA area codes, non-missing identifiers, finite covariates, and coverage of packaged LAD/LTLA OD areas."] + }, "lad_centroids": { "title": "LAD centroid coordinates", "object_name": "lad_centroids", diff --git a/man/debiasRdata_path.Rd b/man/debiasRdata_path.Rd index 2468399..c7e04c0 100644 --- a/man/debiasRdata_path.Rd +++ b/man/debiasRdata_path.Rd @@ -10,7 +10,7 @@ debiasRdata_path(name) \item{name}{Character scalar naming a known packaged asset. Accepted values are \code{"msoa_OD_travel2work"}, \code{"census_msoa_OD_travel2work"}, \code{"lad_OD_travel2work"}, \code{"census_lad_OD_travel2work"}, and -\code{"lad_centroids"}.} +\code{"lad_centroids"}, \code{"msoa_covariates"}, and \code{"lad_covariates"}.} } \value{ A character scalar containing the installed file path. diff --git a/man/lad_covariates.Rd b/man/lad_covariates.Rd new file mode 100644 index 0000000..099be9f --- /dev/null +++ b/man/lad_covariates.Rd @@ -0,0 +1,47 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data.R +\docType{data} +\name{lad_covariates} +\alias{lad_covariates} +\title{Census 2021 LAD covariates} +\format{ +A data frame with 331 rows and 10 variables: +\describe{ +\item{area}{Character. LAD/LTLA area code.} +\item{name}{Character. LAD/LTLA name.} +\item{year}{Integer. Census year.} +\item{per_ukborn}{Numeric. Percentage UK-born.} +\item{per_age_20.29}{Numeric. Percentage aged 20-29.} +\item{per_age_70plus}{Numeric. Percentage aged 70 or over.} +\item{per_level4}{Numeric. Percentage with level 4 qualifications.} +\item{per_hh_no_centralheat}{Numeric. Percentage of households without +central heating.} +\item{per_NS_SeC_L13_routine}{Numeric. Percentage in NS-SeC L13 routine +occupations.} +\item{rural_pct}{Numeric. Percentage rural.} +} +} +\source{ +Office for National Statistics Census 2021 topic-summary inputs, +\code{combined-data-lad.csv}, with variable names selected from +\code{dictionary.csv}. Source data are available under the Open Government +Licence v3.0. +} +\usage{ +lad_covariates +} +\description{ +Selected Census 2021 topic-summary covariates for Local Authority Districts +(LADs), using the requested label-5 variable names from \code{dictionary.csv}. +} +\details{ +The packaged table renames \code{geography.code} to \code{area}, +\code{geography} to \code{name}, and \code{date} to \code{year}. The source column +\code{per_age_20-29} is renamed to the label-5/R-facing name +\code{per_age_20.29}. All covariates are finite numeric percentages between +0 and 100, and the areas cover the packaged LAD/LTLA OD flow assets. +} +\seealso{ +\link{lad_OD_travel2work}, \link{census_lad_OD_travel2work} +} +\keyword{datasets} diff --git a/man/msoa_covariates.Rd b/man/msoa_covariates.Rd new file mode 100644 index 0000000..b449baf --- /dev/null +++ b/man/msoa_covariates.Rd @@ -0,0 +1,48 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data.R +\docType{data} +\name{msoa_covariates} +\alias{msoa_covariates} +\title{Census 2021 MSOA covariates} +\format{ +A data frame with 7,264 rows and 10 variables: +\describe{ +\item{area}{Character. MSOA21 code.} +\item{name}{Character. MSOA name.} +\item{year}{Integer. Census year.} +\item{per_ukborn}{Numeric. Percentage UK-born.} +\item{per_age_20.29}{Numeric. Percentage aged 20-29.} +\item{per_age_70plus}{Numeric. Percentage aged 70 or over.} +\item{per_level4}{Numeric. Percentage with level 4 qualifications.} +\item{per_hh_no_centralheat}{Numeric. Percentage of households without +central heating.} +\item{per_NS_SeC_L13_routine}{Numeric. Percentage in NS-SeC L13 routine +occupations.} +\item{rural_pct}{Numeric. Percentage rural.} +} +} +\source{ +Office for National Statistics Census 2021 topic-summary inputs, +\code{combined-data-msoa.csv}, with variable names selected from +\code{dictionary.csv}. Source data are available under the Open Government +Licence v3.0. +} +\usage{ +msoa_covariates +} +\description{ +Selected Census 2021 topic-summary covariates for Middle Layer Super Output +Areas (MSOAs), using the requested label-5 variable names from +\code{dictionary.csv}. +} +\details{ +The packaged table renames \code{geography.code} to \code{area}, +\code{geography} to \code{name}, and \code{date} to \code{year}. The source column +\code{per_age_20-29} is renamed to the label-5/R-facing name +\code{per_age_20.29}. All covariates are finite numeric percentages between +0 and 100, and the areas cover the packaged MSOA OD flow assets. +} +\seealso{ +\link{msoa_OD_travel2work}, \link{census_msoa_OD_travel2work} +} +\keyword{datasets} diff --git a/tests/testthat/test-data-contract.R b/tests/testthat/test-data-contract.R index c09e230..bc209a7 100644 --- a/tests/testthat/test-data-contract.R +++ b/tests/testthat/test-data-contract.R @@ -9,6 +9,16 @@ file_sha256 <- function(path) { unclass(as.character(openssl::sha256(connection))) } +covariate_columns <- c( + "per_ukborn", + "per_age_20.29", + "per_age_70plus", + "per_level4", + "per_hh_no_centralheat", + "per_NS_SeC_L13_routine", + "rural_pct" +) + check_od_contract <- function(df, meta) { expect_s3_class(df, "data.frame") expect_named(df, c("origin", "destination", "flow")) @@ -42,6 +52,26 @@ check_centroid_contract <- function(df, meta) { expect_false(any(duplicated(df$area))) } +check_covariate_contract <- function(df, meta, area_regex) { + expect_s3_class(df, "data.frame") + expect_named(df, c("area", "name", "year", covariate_columns)) + expect_type(df$area, "character") + expect_type(df$name, "character") + expect_type(df$year, "integer") + expect_false(anyNA(df$area)) + expect_false(anyNA(df$name)) + expect_false(anyNA(df$year)) + expect_true(all(grepl(area_regex, df$area))) + expect_false(any(duplicated(df$area))) + for (column in covariate_columns) { + expect_type(df[[column]], "double") + expect_false(anyNA(df[[column]])) + expect_true(all(is.finite(df[[column]]))) + expect_true(all(df[[column]] >= 0 & df[[column]] <= 100)) + } + expect_equal(nrow(df), meta$row_count) +} + test_that("package data objects satisfy the debiasR contract", { check_od_contract( msoa_OD_travel2work, @@ -63,10 +93,32 @@ test_that("package data objects satisfy the debiasR contract", { lad_centroids, metadata$datasets$lad_centroids ) + check_covariate_contract( + msoa_covariates, + metadata$datasets$msoa_covariates, + "^[EW][0-9]{8}$" + ) + check_covariate_contract( + lad_covariates, + metadata$datasets$lad_covariates, + "^[EW][0-9]{8}$" + ) expect_true(all(unique(c( lad_OD_travel2work$origin, lad_OD_travel2work$destination )) %in% lad_centroids$area)) + expect_true(all(unique(c( + msoa_OD_travel2work$origin, + msoa_OD_travel2work$destination, + census_msoa_OD_travel2work$origin, + census_msoa_OD_travel2work$destination + )) %in% msoa_covariates$area)) + expect_true(all(unique(c( + lad_OD_travel2work$origin, + lad_OD_travel2work$destination, + census_lad_OD_travel2work$origin, + census_lad_OD_travel2work$destination + )) %in% lad_covariates$area)) }) test_that("installed normalized CSV assets match recorded metadata", { @@ -75,12 +127,16 @@ test_that("installed normalized CSV assets match recorded metadata", { lad_mpd_path <- debiasRdata_path("lad_OD_travel2work") lad_census_path <- debiasRdata_path("census_lad_OD_travel2work") lad_centroids_path <- debiasRdata_path("lad_centroids") + msoa_covariates_path <- debiasRdata_path("msoa_covariates") + lad_covariates_path <- debiasRdata_path("lad_covariates") expect_true(file.exists(mpd_path)) expect_true(file.exists(census_path)) expect_true(file.exists(lad_mpd_path)) expect_true(file.exists(lad_census_path)) expect_true(file.exists(lad_centroids_path)) + expect_true(file.exists(msoa_covariates_path)) + expect_true(file.exists(lad_covariates_path)) expect_equal( file_sha256(mpd_path), metadata$datasets$msoa_OD_travel2work$packaged_csv_sha256 @@ -101,6 +157,14 @@ test_that("installed normalized CSV assets match recorded metadata", { file_sha256(lad_centroids_path), metadata$datasets$lad_centroids$packaged_csv_sha256 ) + expect_equal( + file_sha256(msoa_covariates_path), + metadata$datasets$msoa_covariates$packaged_csv_sha256 + ) + expect_equal( + file_sha256(lad_covariates_path), + metadata$datasets$lad_covariates$packaged_csv_sha256 + ) }) test_that("debiasRdata_path rejects unknown assets clearly", {