From 53c3107cd5c8be7afcb2e86fc36069ad38bb51cb Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 5 Jun 2026 09:35:52 -0400 Subject: [PATCH 1/3] allow script to create single file with multiframes, additional arguments --- scripts/sentinelDownload/README.md | 34 +++- scripts/sentinelDownload/sentinel2Download.py | 179 +++++++++++++----- 2 files changed, 167 insertions(+), 46 deletions(-) diff --git a/scripts/sentinelDownload/README.md b/scripts/sentinelDownload/README.md index e6033f97..12c758a3 100644 --- a/scripts/sentinelDownload/README.md +++ b/scripts/sentinelDownload/README.md @@ -27,13 +27,41 @@ The script accepts command-line options via `click`: - `--start-date` _(str, default `2025-01-01`)_ - Start date in `YYYY-MM-DD` format. - `--end-date` _(str, default = today)_ - End date in `YYYY-MM-DD` format. - `--max-results` _(int, default `5`)_ - Maximum number of images to download. -- `--output-dir` _(path, default `sequentialTestRasters`)_ - Directory to save clipped files and JSON file. +- `--output-name` _(str, default `sequentialTestRasters`)_ - Base name for output. Rasters are saved under `downloads//`; ingest JSON is written as `.json` next to the script. - `--cloud-cover` _(float, default `30.0`)_ - Maximum allowed cloud cover percentage of files found. - `--size-km` _(float, default `10.0`)_ - Size of square window (in kilometers) to clip around the point. +- `--single-file` _(flag, default off)_ - Combine all downloaded frames into one multiframe GeoTIFF instead of writing separate files per date. The generated ingest JSON uses `frame_property: "frame"` for multiframe ingest. + +### `--single-file` GDAL requirement + +The `--single-file` option shells out to `gdal_translate` to append each clipped frame as a subdataset in one multiframe GeoTIFF. GDAL must be installed and `gdal_translate` must be on your `PATH`. + +If GDAL is not available, run without `--single-file` (the default writes one GeoTIFF per frame). --- ## Outputs -- **GeoTIFF files** - Clipped Sentinel-2 visual images (RGB). -- **`sample.json`** - JSON metadata describing datasets, layers, and frames, useful for ingestion into GeoDatalytics. +Files are written relative to this script directory: + +``` +scripts/sentinelDownload/ + sentinel2Download.py + .json # ingest manifest (sibling to the script) + downloads/ + / + *.tif # clipped GeoTIFF(s) +``` + +- **GeoTIFF files** - Clipped Sentinel-2 visual images (RGB). With `--single-file`, one multiframe GeoTIFF is written instead of separate per-date files. +- **`.json`** - Ingest manifest describing the project, dataset, layers, and frames. + +## Ingesting into GeoDatalytics + +Then ingest the sequential data from the project root: + +```bash +./manage.py ingest .json --replace +``` + +Use `--replace` if you have previously ingested the same project or dataset and need to refresh it. diff --git a/scripts/sentinelDownload/sentinel2Download.py b/scripts/sentinelDownload/sentinel2Download.py index c0d1ca12..4999192d 100644 --- a/scripts/sentinelDownload/sentinel2Download.py +++ b/scripts/sentinelDownload/sentinel2Download.py @@ -14,6 +14,9 @@ from datetime import UTC, datetime import json from pathlib import Path +import shutil +import subprocess +import tempfile import click import numpy as np @@ -26,6 +29,8 @@ # STAC API from AWS Earth Search STAC_API_URL = "https://earth-search.aws.element84.com/v1" +SCRIPT_DIR = Path(__file__).resolve().parent +DOWNLOADS_DIR = SCRIPT_DIR / "downloads" def default_end_date(): @@ -101,6 +106,44 @@ def read_cog_window_rgb(cog_url, lon, lat, size_km=10): return data, meta +def combine_frames_to_multiframe(frame_paths, output_path): + """ + Combine single-frame GeoTIFFs into one multiframe GeoTIFF. + + Each appended page becomes a scrubbable frame when imported with + frame_property: "frame". + """ + if not frame_paths: + return + + gdal_translate = shutil.which("gdal_translate") + if gdal_translate is None: + msg = ( + "gdal_translate is required for --single-file but was not found on PATH. " + "Install GDAL or run without --single-file." + ) + raise RuntimeError(msg) + + output_path = Path(output_path) + creation_options = ["-co", "COMPRESS=LZW"] + subprocess.run( + [gdal_translate, *creation_options, str(frame_paths[0]), str(output_path)], + check=True, + ) + for frame_path in frame_paths[1:]: + subprocess.run( + [ + gdal_translate, + *creation_options, + "-co", + "APPEND_SUBDATASET=YES", + str(frame_path), + str(output_path), + ], + check=True, + ) + + @click.command() @click.option( "--lat", default=43.135763, type=float, required=True, help="Latitude of the location." @@ -130,11 +173,14 @@ def read_cog_window_rgb(cog_url, lon, lat, size_km=10): help="Maximum number of images to download.", ) @click.option( - "--output-dir", - type=click.Path(), + "--output-name", + type=str, default="sequentialTestRasters", show_default=True, - help="Directory to save the downloaded files.", + help=( + "Base name for output: writes rasters to downloads// and " + "a sibling ingest JSON named .json next to this script." + ), ) @click.option( "--cloud-cover", type=float, default=30.0, show_default=True, help="Max cloud cover percentage." @@ -146,11 +192,21 @@ def read_cog_window_rgb(cog_url, lon, lat, size_km=10): show_default=True, help="Size of square window to clip around the point in kilometers.", ) +@click.option( + "--single-file", + is_flag=True, + default=False, + help=( + "Write all frames into one multiframe GeoTIFF instead of separate files. " + "The generated output JSON will use frame_property: \"frame\"." + ), +) def download_stac_sentinel( # noqa: PLR0913, PLR0915 - lat, lon, start_date, end_date, max_results, output_dir, cloud_cover, size_km + lat, lon, start_date, end_date, max_results, output_name, cloud_cover, size_km, single_file ): """Download clipped Sentinel-2 L1C visual images from AWS via STAC API.""" - Path(output_dir).mkdir(parents=True, exist_ok=True) + output_dir = DOWNLOADS_DIR / output_name + output_dir.mkdir(parents=True, exist_ok=True) catalog = Client.open(STAC_API_URL) @@ -166,7 +222,7 @@ def download_stac_sentinel( # noqa: PLR0913, PLR0915 limit=max_results, ) - items = list(search.get_items()) + items = list(search.items()) if not items: click.echo("⚠️ No Sentinel-2 images found.") @@ -187,40 +243,59 @@ def download_stac_sentinel( # noqa: PLR0913, PLR0915 ) downloaded_files = [] - - for i, item in enumerate(items): - if i >= max_results: - break - date_str = item.datetime.strftime("%Y-%m-%d") - item_id = item.id - click.echo(f"[{i + 1}/{len(items)}] {item_id} from {date_str}") - - visual_asset = item.assets.get("visual") - if visual_asset: - url = visual_asset.href - filename = f"{item_id}_visual_clip_{int(size_km)}km.tif" - filepath = Path(output_dir) / filename - - click.echo(f" - Reading {size_km}km x {size_km}km window around point") - try: - data, meta = read_cog_window_rgb(url, lon, lat, size_km=size_km) - with rasterio.open(filepath, "w", **meta) as dst: - dst.write(data) - except (RasterioError, RasterioIOError) as e: - click.echo(f" - ⚠️ Failed to read or save clipped image: {e}") + downloaded_frame_paths = [] + multiframe_filename = None + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + for i, item in enumerate(items): + if i >= max_results: + break + date_str = item.datetime.strftime("%Y-%m-%d") + item_id = item.id + click.echo(f"[{i + 1}/{len(items)}] {item_id} from {date_str}") + + visual_asset = item.assets.get("visual") + if visual_asset: + url = visual_asset.href + filename = f"{item_id}_visual_clip_{int(size_km)}km.tif" + filepath = output_dir / filename + write_path = temp_path / filename if single_file else filepath + + click.echo(f" - Reading {size_km}km x {size_km}km window around point") + try: + data, meta = read_cog_window_rgb(url, lon, lat, size_km=size_km) + with rasterio.open(write_path, "w", **meta) as dst: + dst.write(data) + except (RasterioError, RasterioIOError) as e: + click.echo(f" - ⚠️ Failed to read or save clipped image: {e}") + else: + if single_file: + click.echo(f" - Buffered frame {len(downloaded_frame_paths)}") + downloaded_frame_paths.append(write_path) + else: + click.echo(f" - Saved clipped image to {filename}") + downloaded_files.append(filename) else: - click.echo(f" - Saved clipped image to {filename}") - downloaded_files.append(filename) - else: - click.echo(f" - ⚠️ Visual asset not available in item {item_id}") + click.echo(f" - ⚠️ Visual asset not available in item {item_id}") + + if single_file and downloaded_frame_paths: + multiframe_filename = f"sentinel_visual_clip_{int(size_km)}km_multiframe.tif" + multiframe_path = output_dir / multiframe_filename + click.echo( + f"Combining {len(downloaded_frame_paths)} frames into {multiframe_filename}..." + ) + combine_frames_to_multiframe(downloaded_frame_paths, multiframe_path) + click.echo(f" - Saved multiframe image to {multiframe_filename}") click.echo("✅ Download complete.") - # Generate dataset.json dataset_json = { "type": "Dataset", "name": "Sequential Test Rasters", "description": "Clipped Sentinel-2 images downloaded and clipped around point", "category": "imagery", + "tags": ["sentinel-2", "imagery", "sequential"], "files": [], "layers": [], } @@ -229,22 +304,40 @@ def download_stac_sentinel( # noqa: PLR0913, PLR0915 "type": "Project", "name": "Sentinel-2 Clipped Images", "datasets": ["Sequential Test Rasters"], - "default_map_center": [lat, lon], + "default_map_center": [lon, lat], "default_map_zoom": 11, } - # Add each file as its own layer - layer_frames = [] - for idx, f in enumerate(downloaded_files): - dataset_json["files"].append({"path": f"{output_dir}/{f}", "name": f"Frame {idx}"}) - layer_frames.append({"name": f"Sequential Layer {idx}", "index": idx, "data": f}) - - layer = {"name": "Sequential Test Layers", "frames": layer_frames} - dataset_json["layers"].append(layer) - - json_path = Path(output_dir, "sample.json") + if single_file and multiframe_filename: + dataset_json["files"].append( + {"path": f"{output_name}/{multiframe_filename}", "name": multiframe_filename} + ) + dataset_json["layers"].append( + { + "name": "Sequential Test Layers", + "frame_property": "frame", + "data": multiframe_filename, + } + ) + else: + layer_frames = [] + for idx, f in enumerate(downloaded_files): + dataset_json["files"].append({"path": f"{output_name}/{f}", "name": f"Frame {idx}"}) + layer_frames.append( + { + "name": f"Sequential Layer {idx}", + "index": idx, + "data": f, + } + ) + + dataset_json["layers"].append({"name": "Sequential Test Layers", "frames": layer_frames}) + + json_path = SCRIPT_DIR / f"{output_name}.json" with json_path.open("w") as jf: json.dump([project_json, dataset_json], jf, indent=4) + click.echo(f" - Wrote ingest JSON to {json_path}") + click.echo(f" - Rasters saved under {output_dir}") if __name__ == "__main__": From 4d475421990ea4be4f439cfe82e056dc3ef34d2a Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 5 Jun 2026 09:45:07 -0400 Subject: [PATCH 2/3] don't force a band when the source_filters are empty --- uvdat/core/tasks/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uvdat/core/tasks/dataset.py b/uvdat/core/tasks/dataset.py index 2724079c..43504278 100644 --- a/uvdat/core/tasks/dataset.py +++ b/uvdat/core/tasks/dataset.py @@ -136,7 +136,7 @@ def create_layers_and_frames(dataset, layer_options=None): # noqa: C901, PLR091 index=index, vector=vector, raster=raster, - source_filters=frame_info.get("source_filters", {"band": 1}), + source_filters=frame_info.get("source_filters", {}), ) From aa64e065e7407524c7251b5a16843244706db95b Mon Sep 17 00:00:00 2001 From: Bryon Lewis Date: Fri, 5 Jun 2026 09:50:29 -0400 Subject: [PATCH 3/3] prevent frame or band from being a style property --- scripts/sentinelDownload/sentinel2Download.py | 14 +-- web/src/store/map.ts | 23 ++--- web/src/store/style.ts | 94 ++++++++++++++++--- 3 files changed, 95 insertions(+), 36 deletions(-) diff --git a/scripts/sentinelDownload/sentinel2Download.py b/scripts/sentinelDownload/sentinel2Download.py index 4999192d..dc90d666 100644 --- a/scripts/sentinelDownload/sentinel2Download.py +++ b/scripts/sentinelDownload/sentinel2Download.py @@ -106,6 +106,10 @@ def read_cog_window_rgb(cog_url, lon, lat, size_km=10): return data, meta +def _run_checked_command(cmd: list[str]) -> None: + subprocess.run(cmd, check=True) # noqa: S603 + + def combine_frames_to_multiframe(frame_paths, output_path): """ Combine single-frame GeoTIFFs into one multiframe GeoTIFF. @@ -126,12 +130,11 @@ def combine_frames_to_multiframe(frame_paths, output_path): output_path = Path(output_path) creation_options = ["-co", "COMPRESS=LZW"] - subprocess.run( + _run_checked_command( [gdal_translate, *creation_options, str(frame_paths[0]), str(output_path)], - check=True, ) for frame_path in frame_paths[1:]: - subprocess.run( + _run_checked_command( [ gdal_translate, *creation_options, @@ -140,7 +143,6 @@ def combine_frames_to_multiframe(frame_paths, output_path): str(frame_path), str(output_path), ], - check=True, ) @@ -198,10 +200,10 @@ def combine_frames_to_multiframe(frame_paths, output_path): default=False, help=( "Write all frames into one multiframe GeoTIFF instead of separate files. " - "The generated output JSON will use frame_property: \"frame\"." + 'The generated output JSON will use frame_property: "frame".' ), ) -def download_stac_sentinel( # noqa: PLR0913, PLR0915 +def download_stac_sentinel( # noqa: C901, PLR0912, PLR0913, PLR0915 lat, lon, start_date, end_date, max_results, output_name, cloud_cover, size_km, single_file ): """Download clipped Sentinel-2 L1C visual images from AWS via STAC API.""" diff --git a/web/src/store/map.ts b/web/src/store/map.ts index d22204ac..a59464dd 100644 --- a/web/src/store/map.ts +++ b/web/src/store/map.ts @@ -524,9 +524,6 @@ export const useMapStore = defineStore("map", () => { ): Source | undefined { const map = getMap(); - const queryParams: { projection: string; style?: string } = { - projection: "epsg:3857", - }; const { layerId, layerCopyId } = parseSourceString(sourceId); const styleSpec = styleStore.selectedLayerStyles[`${layerId}.${layerCopyId}`].style_spec; @@ -540,22 +537,14 @@ export const useMapStore = defineStore("map", () => { (f: LayerFrame) => f.index === layer.current_frame_index, ); if (frame?.source_filters) { - filters = Object.entries(frame.source_filters).map(([k, v]) => ({ - filter_by: k, - list: [v], - include: true, - transparency: true, - apply: true, - })); + filters = styleStore.sourceFiltersToStyleFilters(frame.source_filters); } } - if (styleSpec) { - const styleParams = styleStore.getRasterTilesQuery( - { ...styleSpec, filters }, - styleStore.colormaps, - ); - if (styleParams) queryParams.style = JSON.stringify(styleParams); - } + const queryParams = styleStore.buildRasterTileQueryParams( + styleSpec ?? styleStore.getDefaultStyleSpec(raster, layerId), + filters, + styleStore.colormaps, + ); const query = new URLSearchParams(queryParams); rasterSourceTileURLs.value[sourceId] = `${baseURL}rasters/${raster.id}/tiles/{z}/{x}/{y}.png/?${query}`; diff --git a/web/src/store/style.ts b/web/src/store/style.ts index e2e54fc3..bee2dd72 100644 --- a/web/src/store/style.ts +++ b/web/src/store/style.ts @@ -89,6 +89,51 @@ export function colormapMarkersSubsample( return markers; } +// frame/band select which slice to read; they must be flat query params, not in style JSON +const RASTER_SOURCE_FILTER_KEYS = new Set(["frame", "band"]); + +// Ingest defaults missing source_filters to { band: 1 }; that is not a real band selection. +function isDefaultBandSourceFilter( + sourceFilters: Record, +): boolean { + const keys = Object.keys(sourceFilters); + return ( + keys.length === 1 && + keys[0] === "band" && + (sourceFilters.band === 1 || sourceFilters.band === "1") + ); +} + +function sourceFiltersToStyleFilters( + sourceFilters: Record | undefined, +): StyleFilter[] { + if (!sourceFilters || !Object.keys(sourceFilters).length) return []; + if (isDefaultBandSourceFilter(sourceFilters)) return []; + return Object.entries(sourceFilters).map(([k, v]) => ({ + filter_by: k, + list: [v], + include: true, + transparency: true, + apply: true, + })); +} + +function getRasterSourceFilterParams(filters: StyleFilter[]) { + const params: Record = {}; + filters.forEach((f) => { + if ( + f.apply && + f.filter_by && + f.include && + f.list?.length === 1 && + RASTER_SOURCE_FILTER_KEYS.has(f.filter_by) + ) { + params[f.filter_by] = f.list[0]; + } + }); + return params; +} + function getRasterTilesQuery(styleSpec: StyleSpec, colormaps: Colormap[]) { let query: Record = {}; const colorSpecs = styleSpec.colors || []; @@ -126,13 +171,37 @@ function getRasterTilesQuery(styleSpec: StyleSpec, colormaps: Colormap[]) { } }); styleSpec.filters.forEach((f) => { - if (f.apply && f.filter_by && f.include && f.list?.length === 1) { + if ( + f.apply && + f.filter_by && + f.include && + f.list?.length === 1 && + !RASTER_SOURCE_FILTER_KEYS.has(f.filter_by) + ) { query[f.filter_by] = f.list[0]; } }); return query; } +function buildRasterTileQueryParams( + styleSpec: StyleSpec, + filters: StyleFilter[], + colormaps: Colormap[], +) { + const params: Record = { projection: "epsg:3857" }; + Object.entries(getRasterSourceFilterParams(filters)).forEach( + ([key, value]) => { + params[key] = String(value); + }, + ); + const styleQuery = getRasterTilesQuery({ ...styleSpec, filters }, colormaps); + if (Object.keys(styleQuery).length) { + params.style = JSON.stringify(styleQuery); + } + return params; +} + function getVectorColorPaintProperty( styleSpec: StyleSpec, groupName: string, @@ -441,13 +510,7 @@ export const useStyleStore = defineStore("style", () => { if (frame?.source_filters) { filters = [ ...filters, - ...Object.entries(frame.source_filters).map(([k, v]) => ({ - filter_by: k, - list: [v], - include: true, - transparency: true, - apply: true, - })), + ...sourceFiltersToStyleFilters(frame.source_filters), ]; } const mapLayer = map.getLayer(mapLayerId) as @@ -550,11 +613,13 @@ export const useStyleStore = defineStore("style", () => { const source = map.getSource(mapLayer.source) as RasterTileSource; const sourceURL = mapStore.rasterSourceTileURLs[mapLayer.source]; if (source && sourceURL) { - const newQueryParams: { projection: string; style?: string } = { - projection: "epsg:3857", - }; - newQueryParams.style = JSON.stringify(rasterTilesQuery); - const newQuery = new URLSearchParams(newQueryParams); + const newQuery = new URLSearchParams( + buildRasterTileQueryParams( + { ...styleSpec, filters }, + filters, + colormaps.value, + ), + ); tileURL = sourceURL.split("?")[0] + "?" + newQuery.toString(); return { paint, tileURL }; } @@ -605,6 +670,9 @@ export const useStyleStore = defineStore("style", () => { selectedLayerStyles, fetchColormaps, getRasterTilesQuery, + getRasterSourceFilterParams, + buildRasterTileQueryParams, + sourceFiltersToStyleFilters, colormapMarkersSubsample, getDefaultColor, getDefaultStyleSpec,