diff --git a/Cargo.lock b/Cargo.lock index 8f4927800b..0fbe6ce45b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -806,9 +806,9 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c5bcfa8749ac45dd12cb11055aeeb6b27a3895560d60d71e3c23bf979e60514" +checksum = "9a686bbee5efb88a82df0621b236e74d925f470e5445d3220a5648b892ec99c9" dependencies = [ "anstyle", "bstr", @@ -3529,9 +3529,9 @@ dependencies = [ [[package]] name = "image" -version = "0.25.9" +version = "0.25.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6506c6c10786659413faa717ceebcb8f70731c0a60cbae39795fdf114519c1a" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" dependencies = [ "bytemuck", "byteorder-lite", @@ -3547,8 +3547,8 @@ dependencies = [ "rayon", "rgb", "tiff", - "zune-core 0.5.1", - "zune-jpeg 0.5.12", + "zune-core", + "zune-jpeg", ] [[package]] @@ -4219,9 +4219,9 @@ dependencies = [ [[package]] name = "moxcms" -version = "0.7.11" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac9557c559cd6fc9867e122e20d2cbefc9ca29d80d027a8e39310920ed2f0a97" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" dependencies = [ "num-traits", "pxfm", @@ -5748,9 +5748,9 @@ dependencies = [ [[package]] name = "ravif" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef69c1990ceef18a116855938e74793a5f7496ee907562bd0857b6ac734ab285" +checksum = "e52310197d971b0f5be7fe6b57530dcd27beb35c1b013f29d66c1ad73fbbcc45" dependencies = [ "avif-serialize", "imgref", @@ -6260,9 +6260,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -6949,9 +6949,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.26.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", "getrandom 0.4.2", @@ -7017,16 +7017,16 @@ dependencies = [ [[package]] name = "tiff" -version = "0.10.3" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af9605de7fee8d9551863fd692cce7637f548dbd9db9180fcc07ccc6d26c336f" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" dependencies = [ "fax", "flate2", "half", "quick-error", "weezl", - "zune-jpeg 0.4.21", + "zune-jpeg", ] [[package]] @@ -8779,12 +8779,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "zune-core" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" - [[package]] name = "zune-core" version = "0.5.1" @@ -8800,20 +8794,11 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "zune-jpeg" -version = "0.4.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ce2c8a9384ad323cf564b67da86e21d3cfdff87908bc1223ed5c99bc792713" -dependencies = [ - "zune-core 0.4.12", -] - [[package]] name = "zune-jpeg" version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "410e9ecef634c709e3831c2cfdb8d9c32164fae1c67496d5b68fff728eec37fe" dependencies = [ - "zune-core 0.5.1", + "zune-core", ] diff --git a/justfile b/justfile new file mode 100644 index 0000000000..6c000ed6fc --- /dev/null +++ b/justfile @@ -0,0 +1,41 @@ +_default: + @just --list + +# Generate OpenAPI specification as JSON file. The file is generated in the current directory. +[group("build")] +generate-openapi-spec: + @-clear + cargo run --bin geoengine-cli -- openapi > openapi.json + +# Run lints. +[group("lint")] +lint: + @-clear + just _lint-clippy + +_lint-clippy: + cargo clippy --all-features --all-targets + +# Run clippy for all features and all targets. +[group("lint")] +lint-clippy: + @-clear + just _lint-clippy + +# Run the application. +[group("run")] +run: + @-clear + cargo run + +# Run the tests. Optionally, a filter can be provided to run only a subset of the tests. +[group("test")] +test filter="": + @-clear + cargo test -- {{ filter }} --nocapture + +# Run the tests for the geoengine-services package. Optionally, a filter can be provided to run only a subset of the tests. Example: just test-services workflows::workflow::tests +[group("test")] +test-services filter="": + @-clear + cargo test --package geoengine-services -- {{ filter }} --nocapture \ No newline at end of file diff --git a/openapi.json b/openapi.json index 110c5281de..2f34636d8c 100644 --- a/openapi.json +++ b/openapi.json @@ -5551,6 +5551,7 @@ }, "ClassificationMeasurement": { "type": "object", + "description": "A classification measurement.\nIt contains a mapping from class IDs to class names.", "required": [ "type", "measurement", @@ -5641,6 +5642,69 @@ } } }, + "ColumnNames": { + "oneOf": [ + { + "type": "object", + "title": "Default", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "default" + ] + } + } + }, + { + "type": "object", + "title": "Suffix", + "required": [ + "values", + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "suffix" + ] + }, + "values": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + { + "type": "object", + "title": "Names", + "required": [ + "values", + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "names" + ] + }, + "values": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + ] + }, "ComputationQuota": { "type": "object", "required": [ @@ -5671,6 +5735,7 @@ }, "ContinuousMeasurement": { "type": "object", + "description": "A continuous measurement, e.g., \"temperature\".\nIt may have an optional unit, e.g., \"°C\" for degrees Celsius.", "required": [ "type", "measurement" @@ -6436,6 +6501,87 @@ } } }, + "Expression": { + "type": "object", + "title": "Raster Expression", + "description": "The `Expression` operator performs a pixel-wise mathematical expression on one or more bands of a raster source.\nThe expression is specified as a user-defined script in a very simple language.\nThe output is a raster time series with the result of the expression and with time intervals that are the same as for the inputs.\nUsers can specify an output data type.\nInternally, the expression is evaluated using floating-point numbers.\n\nAn example usage scenario is to calculate NDVI for a red and a near-infrared raster channel.\nThe expression uses a raster source with two bands, referred to as A and B, and calculates the formula `(A - B) / (A + B)`.\nWhen the temporal resolution is months, our output NDVI will also be a monthly time series.\n\n## Types\n\nThe following describes the types used in the parameters.\n\n### Expression\n\nExpressions are simple scripts to perform pixel-wise computations.\nOne can refer to the raster inputs as `A` for the first raster band, `B` for the second, and so on.\nFurthermore, expressions can check with `A IS NODATA`, `B IS NODATA`, etc. for NO DATA values.\nThis is important if `mapNoData` is set to true.\nOtherwise, NO DATA values are mapped automatically to the output NO DATA value.\nFinally, the value `NODATA` can be used to output NO DATA.\n\nUsers can think of this implicit function signature for, e.g., two inputs:\n\n```Rust\nfn (A: f64, B: f64) -> f64\n```\n\nAs a start, expressions contain algebraic operations and mathematical functions.\n\n```Rust\n(A + B) / 2\n```\n\nIn addition, branches can be used to check for conditions.\n\n```Rust\nif A IS NODATA {\n B\n} else {\n A\n}\n```\n\nFunction calls can be used to access utility functions.\n\n```Rust\nmax(A, 0)\n```\n\nCurrently, the following functions are available:\n\n- `abs(a)`: absolute value\n- `min(a, b)`, `min(a, b, c)`: minimum value\n- `max(a, b)`, `max(a, b, c)`: maximum value\n- `sqrt(a)`: square root\n- `ln(a)`: natural logarithm\n- `log10(a)`: base 10 logarithm\n- `cos(a)`, `sin(a)`, `tan(a)`, `acos(a)`, `asin(a)`, `atan(a)`: trigonometric functions\n- `pi()`, `e()`: mathematical constants\n- `round(a)`, `ceil(a)`, `floor(a)`: rounding functions\n- `mod(a, b)`: division remainder\n- `to_degrees(a)`, `to_radians(a)`: conversion to degrees or radians\n\nTo generate more complex expressions, it is possible to have variable assignments.\n\n```Rust\nlet mean = (A + B) / 2;\nlet coefficient = 0.357;\nmean * coefficient\n```\n\nNote, that all assignments are separated by semicolons.\nHowever, the last expression must be without a semicolon.", + "required": [ + "type", + "params", + "sources" + ], + "properties": { + "params": { + "$ref": "#/components/schemas/ExpressionParameters" + }, + "sources": { + "$ref": "#/components/schemas/SingleRasterSource" + }, + "type": { + "type": "string", + "enum": [ + "Expression" + ] + } + }, + "examples": [ + { + "type": "Expression", + "params": { + "expression": "(A - B) / (A + B)", + "outputType": "F32", + "outputBand": { + "name": "NDVI", + "measurement": { + "type": "unitless" + } + }, + "mapNoData": true + }, + "sources": { + "raster": { + "type": "GdalSource", + "params": { + "data": "ndvi" + } + } + } + } + ] + }, + "ExpressionParameters": { + "type": "object", + "description": "## Types\n\nThe following describes the types used in the parameters.", + "required": [ + "expression", + "outputType", + "mapNoData" + ], + "properties": { + "expression": { + "type": "string", + "description": "Expression script\n\nExample: `\"(A - B) / (A + B)\"`", + "examples": [ + "(A - B) / (A + B)" + ] + }, + "mapNoData": { + "type": "boolean", + "description": "Should NO DATA values be mapped with the `expression`? Otherwise, they are mapped automatically to NO DATA.", + "examples": [ + true + ] + }, + "outputBand": { + "$ref": "#/components/schemas/RasterBandDescriptor", + "description": "Description about the output" + }, + "outputType": { + "$ref": "#/components/schemas/RasterDataType", + "description": "A raster data type for the output" + } + } + }, "ExternalDataId": { "type": "object", "required": [ @@ -6458,6 +6604,13 @@ } } }, + "FeatureAggregationMethod": { + "type": "string", + "enum": [ + "first", + "mean" + ] + }, "FeatureDataType": { "type": "string", "enum": [ @@ -6829,6 +6982,63 @@ } } }, + "GdalSource": { + "type": "object", + "title": "GDAL Source", + "description": "The [`GdalSource`] is a source operator that reads raster data using GDAL.\nThe counterpart for vector data is the [`OgrSource`].\n\n## Errors\n\nIf the given dataset does not exist or is not readable, an error is thrown.\n", + "required": [ + "type", + "params" + ], + "properties": { + "params": { + "$ref": "#/components/schemas/GdalSourceParameters" + }, + "type": { + "type": "string", + "enum": [ + "GdalSource" + ] + } + }, + "examples": [ + { + "type": "GdalSource", + "params": { + "data": "ndvi", + "overviewLevel": null + } + } + ] + }, + "GdalSourceParameters": { + "type": "object", + "description": "Parameters for the [`GdalSource`] operator.", + "required": [ + "data" + ], + "properties": { + "data": { + "type": "string", + "description": "Dataset name or identifier to be loaded.", + "examples": [ + "ndvi" + ] + }, + "overviewLevel": { + "type": [ + "integer", + "null" + ], + "format": "int32", + "description": "*Optional*: overview level to use.\n\nIf not provided, the data source will determine the resolution, i.e., uses its native resolution.", + "examples": [ + 3 + ], + "minimum": 0 + } + } + }, "GdalSourceTimePlaceholder": { "type": "object", "required": [ @@ -7260,6 +7470,58 @@ } } }, + "LegacyTypedOperator": { + "type": "object", + "description": "An enum to differentiate between `Operator` variants", + "required": [ + "type", + "operator" + ], + "properties": { + "operator": { + "type": "object", + "required": [ + "type" + ], + "properties": { + "params": { + "type": "object" + }, + "sources": { + "type": "object" + }, + "type": { + "type": "string" + } + } + }, + "type": { + "type": "string", + "enum": [ + "Vector", + "Raster", + "Plot" + ] + } + }, + "examples": [ + { + "type": "MockPointSource", + "params": { + "points": [ + { + "x": 0.0, + "y": 0.1 + }, + { + "x": 1.0, + "y": 1.1 + } + ] + } + } + ] + }, "LineSymbology": { "type": "object", "required": [ @@ -7370,6 +7632,7 @@ "$ref": "#/components/schemas/ClassificationMeasurement" } ], + "description": "Measurement information for a raster band.", "discriminator": { "propertyName": "type", "mapping": { @@ -7645,6 +7908,67 @@ } } }, + "MockPointSource": { + "type": "object", + "title": "Mock Point Source", + "description": "The [`MockPointSource`] is a source operator that provides mock vector point data for testing and development purposes.\n", + "required": [ + "type", + "params" + ], + "properties": { + "params": { + "$ref": "#/components/schemas/MockPointSourceParameters" + }, + "type": { + "type": "string", + "enum": [ + "MockPointSource" + ] + } + }, + "examples": [ + { + "type": "MockPointSource", + "params": { + "points": [ + { + "x": 1.0, + "y": 2.0 + }, + { + "x": 3.0, + "y": 4.0 + } + ], + "spatialBounds": { + "type": "derive" + } + } + } + ] + }, + "MockPointSourceParameters": { + "type": "object", + "description": "Parameters for the [`MockPointSource`] operator.", + "required": [ + "points", + "spatialBounds" + ], + "properties": { + "points": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Coordinate2D" + }, + "description": "Points to be output by the mock point source.\n" + }, + "spatialBounds": { + "$ref": "#/components/schemas/SpatialBoundsDerive", + "description": "Defines how the spatial bounds of the source are derived.\n\nDefaults to `None`." + } + } + }, "MultiBandRasterColorizer": { "type": "object", "required": [ @@ -8470,6 +8794,11 @@ } } }, + "PlotOperator": { + "type": "null", + "description": "An operator that produces plot data.", + "default": null + }, "PlotOutputFormat": { "type": "string", "enum": [ @@ -8919,17 +9248,18 @@ }, "RasterDataType": { "type": "string", + "description": "A raster data type.", "enum": [ - "U8", - "U16", - "U32", - "U64", - "I8", - "I16", - "I32", - "I64", - "F32", - "F64" + "u8", + "u16", + "u32", + "u64", + "i8", + "i16", + "i32", + "i64", + "f32", + "f64" ] }, "RasterDatasetFromWorkflow": { @@ -9005,6 +9335,24 @@ } } }, + "RasterOperator": { + "oneOf": [ + { + "$ref": "#/components/schemas/Expression" + }, + { + "$ref": "#/components/schemas/GdalSource" + } + ], + "description": "An operator that produces raster data.", + "discriminator": { + "propertyName": "type", + "mapping": { + "Expression": "#/components/schemas/Expression", + "GdalSource": "#/components/schemas/GdalSource" + } + } + }, "RasterPropertiesEntryType": { "type": "string", "enum": [ @@ -9103,6 +9451,95 @@ } } }, + "RasterVectorJoin": { + "type": "object", + "title": "Raster Vector Join", + "description": "The `RasterVectorJoin` operator allows combining a single vector input and multiple raster inputs.\nFor each raster input, a new column is added to the collection from the vector input.\nThe new column contains the value of the raster at the location of the vector feature.\nFor features covering multiple pixels like `MultiPoints` or `MultiPolygons`, the value is calculated using an aggregation function selected by the user.\nThe same is true if the temporal extent of a vector feature covers multiple raster time steps.\nMore details are described below.\n\n**Example**:\nYou have a collection of agricultural fields (`Polygons`) and a collection of raster images containing each pixel's monthly NDVI value.\nFor your application, you want to know the NDVI value of each field.\nThe `RasterVectorJoin` operator allows you to combine the vector and raster data and offers multiple spatial and temporal aggregation strategies.\nFor example, you can use the `first` aggregation function to get the NDVI value of the first pixel that intersects with each field.\nThis is useful for exploratory analysis since the computation is very fast.\nTo calculate the mean NDVI value of all pixels that intersect with the field you should use the `mean` aggregation function.\nSince the NDVI data is a monthly time series, you have to specify the temporal aggregation function as well.\nThe default is `none` which will create a new feature for each month.\nOther options are `first` and `mean` which will calculate the first or mean NDVI value for each field over time.\n\n## Inputs\n\nThe `RasterVectorJoin` operator expects one _vector_ input and one or more _raster_ inputs.\n\n| Parameter | Type |\n| --------- | ----------------------------------- |\n| `sources` | `SingleVectorMultipleRasterSources` |\n\n## Errors\n\nIf the length of `names` is not equal to the number of raster inputs, an error is thrown.\n", + "required": [ + "type", + "params", + "sources" + ], + "properties": { + "params": { + "$ref": "#/components/schemas/RasterVectorJoinParameters" + }, + "sources": { + "$ref": "#/components/schemas/SingleVectorMultipleRasterSources" + }, + "type": { + "type": "string", + "enum": [ + "RasterVectorJoin" + ] + } + }, + "examples": [ + { + "type": "RasterVectorJoin", + "params": { + "names": [ + "NDVI" + ], + "featureAggregation": "first", + "temporalAggregation": "mean", + "temporalAggregationIgnoreNoData": true + }, + "sources": { + "vector": { + "type": "OgrSource", + "params": { + "data": "places" + } + }, + "rasters": [ + { + "type": "GdalSource", + "params": { + "data": "ndvi" + } + } + ] + } + } + ] + }, + "RasterVectorJoinParameters": { + "type": "object", + "required": [ + "names", + "featureAggregation", + "temporalAggregation" + ], + "properties": { + "featureAggregation": { + "$ref": "#/components/schemas/FeatureAggregationMethod", + "description": "The aggregation function to use for features covering multiple pixels." + }, + "featureAggregationIgnoreNoData": { + "type": "boolean", + "description": "Whether to ignore no data values in the aggregation. Defaults to `false`.", + "examples": [ + true + ] + }, + "names": { + "$ref": "#/components/schemas/ColumnNames", + "description": "Specify how the new column names are derived from the raster band names.\n\nThe `ColumnNames` type is used to specify how the new column names are derived from the raster band names.\n\n- **default**: Appends \" (n)\" to the band name with the smallest `n` that avoids a conflict.\n- **suffix**: Specifies a suffix for each input, to be appended to the band names.\n- **rename**: A list of names for each new column.\n" + }, + "temporalAggregation": { + "$ref": "#/components/schemas/TemporalAggregationMethod", + "description": "The aggregation function to use for features covering multiple (raster) time steps." + }, + "temporalAggregationIgnoreNoData": { + "type": "boolean", + "description": "Whether to ignore no data values in the aggregation. Defaults to `false`.", + "examples": [ + true + ] + } + } + }, "RegularTimeDimension": { "type": "object", "required": [ @@ -9364,6 +9801,106 @@ } } }, + "SingleRasterSource": { + "type": "object", + "required": [ + "raster" + ], + "properties": { + "raster": { + "$ref": "#/components/schemas/RasterOperator" + } + } + }, + "SingleVectorMultipleRasterSources": { + "type": "object", + "required": [ + "vector", + "rasters" + ], + "properties": { + "rasters": { + "type": "array", + "items": { + "$ref": "#/components/schemas/RasterOperator" + } + }, + "vector": { + "$ref": "#/components/schemas/VectorOperator" + } + } + }, + "SpatialBoundsDerive": { + "oneOf": [ + { + "$ref": "#/components/schemas/SpatialBoundsDeriveDerive" + }, + { + "$ref": "#/components/schemas/SpatialBoundsDeriveBounds" + }, + { + "$ref": "#/components/schemas/SpatialBoundsDeriveNone" + } + ], + "description": "Spatial bounds derivation options for the [`MockPointSource`].", + "discriminator": { + "propertyName": "type", + "mapping": { + "bounds": "#/components/schemas/SpatialBoundsDeriveBounds", + "derive": "#/components/schemas/SpatialBoundsDeriveDerive", + "none": "#/components/schemas/SpatialBoundsDeriveNone" + } + } + }, + "SpatialBoundsDeriveBounds": { + "allOf": [ + { + "$ref": "#/components/schemas/BoundingBox2D" + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "bounds" + ] + } + } + } + ] + }, + "SpatialBoundsDeriveDerive": { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "derive" + ] + } + } + }, + "SpatialBoundsDeriveNone": { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "none" + ] + } + } + }, "SpatialGridDefinition": { "type": "object", "required": [ @@ -9829,6 +10366,14 @@ } ] }, + "TemporalAggregationMethod": { + "type": "string", + "enum": [ + "none", + "first", + "mean" + ] + }, "TextSymbology": { "type": "object", "required": [ @@ -10074,56 +10619,63 @@ ] }, "TypedOperator": { - "type": "object", - "description": "An enum to differentiate between `Operator` variants", - "required": [ - "type", - "operator" - ], - "properties": { - "operator": { + "oneOf": [ + { "type": "object", "required": [ + "operator", "type" ], "properties": { - "params": { - "type": "object" + "operator": { + "$ref": "#/components/schemas/VectorOperator" }, - "sources": { - "type": "object" + "type": { + "type": "string", + "enum": [ + "Vector" + ] + } + } + }, + { + "type": "object", + "required": [ + "operator", + "type" + ], + "properties": { + "operator": { + "$ref": "#/components/schemas/RasterOperator" }, "type": { - "type": "string" + "type": "string", + "enum": [ + "Raster" + ] } } }, - "type": { - "type": "string", - "enum": [ - "Vector", - "Raster", - "Plot" - ] - } - }, - "examples": [ { - "type": "MockPointSource", - "params": { - "points": [ - { - "x": 0.0, - "y": 0.1 - }, - { - "x": 1.0, - "y": 1.1 - } - ] + "type": "object", + "required": [ + "operator", + "type" + ], + "properties": { + "operator": { + "$ref": "#/components/schemas/PlotOperator" + }, + "type": { + "type": "string", + "enum": [ + "Plot" + ] + } } } - ] + ], + "description": "Operator outputs are distinguished by their data type.\nThere are `raster`, `vector` and `plot` operators." }, "TypedPlotResultDescriptor": { "allOf": [ @@ -10211,6 +10763,7 @@ }, "UnitlessMeasurement": { "type": "object", + "description": "A measurement without a unit.", "required": [ "type" ], @@ -10658,6 +11211,24 @@ "MultiPolygon" ] }, + "VectorOperator": { + "oneOf": [ + { + "$ref": "#/components/schemas/MockPointSource" + }, + { + "$ref": "#/components/schemas/RasterVectorJoin" + } + ], + "description": "An operator that produces vector data.", + "discriminator": { + "propertyName": "type", + "mapping": { + "MockPointSource": "#/components/schemas/MockPointSource", + "RasterVectorJoin": "#/components/schemas/RasterVectorJoin" + } + } + }, "VectorResultDescriptor": { "type": "object", "required": [ @@ -10884,9 +11455,20 @@ ] }, "Workflow": { - "allOf": [ + "oneOf": [ + { + "allOf": [ + { + "$ref": "#/components/schemas/TypedOperator" + } + ] + }, { - "$ref": "#/components/schemas/TypedOperator" + "allOf": [ + { + "$ref": "#/components/schemas/LegacyTypedOperator" + } + ] } ] }, diff --git a/services/benches/quota_check.rs b/services/benches/quota_check.rs index 4aabc7b223..d497ad264e 100644 --- a/services/benches/quota_check.rs +++ b/services/benches/quota_check.rs @@ -40,7 +40,7 @@ async fn bench() { let (_, dataset) = add_ndvi_to_datasets2(&app_ctx, true, true).await; - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Raster( TemporalRasterAggregation { params: TemporalRasterAggregationParameters { diff --git a/services/src/api/apidoc.rs b/services/src/api/apidoc.rs index 207eebc982..8fa75a5720 100644 --- a/services/src/api/apidoc.rs +++ b/services/src/api/apidoc.rs @@ -23,12 +23,12 @@ use crate::api::model::datatypes::{ use crate::api::model::operators::{ CsvHeader, FileNotFoundHandling, FormatSpecifics, GdalDatasetParameters, GdalLoadingInfoTemporalSlice, GdalMetaDataList, GdalMetaDataRegular, GdalMetaDataStatic, - GdalMetadataMapping, GdalMetadataNetCdfCf, GdalSourceTimePlaceholder, MlModelMetadata, - MockDatasetDataSourceLoadingInfo, MockMetaData, OgrMetaData, OgrSourceColumnSpec, - OgrSourceDataset, OgrSourceDatasetTimeType, OgrSourceDurationSpec, OgrSourceErrorSpec, - OgrSourceTimeFormat, PlotResultDescriptor, RasterBandDescriptor, RasterBandDescriptors, - RasterResultDescriptor, TimeReference, TypedGeometry, TypedOperator, TypedResultDescriptor, - UnixTimeStampType, VectorColumnInfo, VectorResultDescriptor, + GdalMetadataMapping, GdalMetadataNetCdfCf, GdalSourceTimePlaceholder, LegacyTypedOperator, + MlModelMetadata, MockDatasetDataSourceLoadingInfo, MockMetaData, OgrMetaData, + OgrSourceColumnSpec, OgrSourceDataset, OgrSourceDatasetTimeType, OgrSourceDurationSpec, + OgrSourceErrorSpec, OgrSourceTimeFormat, PlotResultDescriptor, RasterBandDescriptor, + RasterBandDescriptors, RasterResultDescriptor, TimeReference, TypedGeometry, + TypedResultDescriptor, UnixTimeStampType, VectorColumnInfo, VectorResultDescriptor, }; use crate::api::model::responses::datasets::DatasetNameResponse; use crate::api::model::responses::ml_models::MlModelNameResponse; @@ -275,7 +275,7 @@ use utoipa::{Modify, OpenApi}; ServerInfo, Workflow, - TypedOperator, + LegacyTypedOperator, TypedResultDescriptor, PlotResultDescriptor, RasterResultDescriptor, @@ -465,6 +465,9 @@ use utoipa::{Modify, OpenApi}; MlTensorShape3D, ), ), + nest( + (path = "/processingGraphs", api = crate::api::model::processing_graphs::OperatorsApi), + ), modifiers(&SecurityAddon, &ApiDocInfo, &OpenApiServerInfo, &DeriveDiscriminatorMapping), external_docs(url = "https://docs.geoengine.io", description = "Geo Engine Docs") )] diff --git a/services/src/api/handlers/datasets.rs b/services/src/api/handlers/datasets.rs index 2224804962..ff391cb067 100755 --- a/services/src/api/handlers/datasets.rs +++ b/services/src/api/handlers/datasets.rs @@ -3544,7 +3544,7 @@ mod tests { assert_eq!(res.status(), 200, "response: {res:?}"); // create workflow - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: geoengine_operators::engine::TypedOperator::Raster( MultiBandGdalSource { params: MultiBandGdalSourceParameters::new(dataset_name.into()), diff --git a/services/src/api/handlers/layers.rs b/services/src/api/handlers/layers.rs index 2bcc90346b..521927ae8b 100644 --- a/services/src/api/handlers/layers.rs +++ b/services/src/api/handlers/layers.rs @@ -1441,7 +1441,7 @@ mod tests { AddLayer { name: "Layer Name".to_string(), description: "Layer Description".to_string(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: MockPointSource { params: MockPointSourceParams::new(vec![ (0.0, 0.1).into(), @@ -1570,7 +1570,7 @@ mod tests { name: "Foo".to_string(), description: "Bar".to_string(), properties: Default::default(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams { @@ -1597,7 +1597,7 @@ mod tests { let update_layer = UpdateLayer { name: "Foo new".to_string(), description: "Bar new".to_string(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams { @@ -1672,7 +1672,7 @@ mod tests { name: "Foo".to_string(), description: "Bar".to_string(), properties: Default::default(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams { @@ -1722,7 +1722,7 @@ mod tests { name: "Foo".to_string(), description: "Bar".to_string(), properties: Default::default(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams { @@ -1840,7 +1840,7 @@ mod tests { AddLayer { name: "Layer Name".to_string(), description: "Layer Description".to_string(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: MockPointSource { params: MockPointSourceParams::new(vec![ (0.0, 0.1).into(), @@ -2618,11 +2618,11 @@ mod tests { .boxed(); let workflow = if time_shift_millis == 0 { - Workflow { + Workflow::Legacy { operator: raster_source.into(), } } else { - Workflow { + Workflow::Legacy { operator: TypedOperator::Raster(Box::new(TimeShift { params: TimeShiftParams::Relative { granularity: TimeGranularity::Millis, @@ -2761,7 +2761,12 @@ mod tests { }; // query the layer - let workflow_operator = mock_source.workflow.operator.get_raster().unwrap(); + let workflow_operator = mock_source + .workflow + .operator() + .unwrap() + .get_raster() + .unwrap(); // query the newly created dataset let dataset_operator = GdalSource { diff --git a/services/src/api/handlers/permissions.rs b/services/src/api/handlers/permissions.rs index b152222b29..3f9da0aefd 100644 --- a/services/src/api/handlers/permissions.rs +++ b/services/src/api/handlers/permissions.rs @@ -671,7 +671,7 @@ mod tests { let layer = AddLayer { name: "layer".to_string(), description: "description".to_string(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams { diff --git a/services/src/api/handlers/plots.rs b/services/src/api/handlers/plots.rs index a118bedb45..ec0a474841 100644 --- a/services/src/api/handlers/plots.rs +++ b/services/src/api/handlers/plots.rs @@ -111,7 +111,7 @@ async fn get_plot_handler( let workflow_id = WorkflowId(id.into_inner()); let workflow = ctx.db().load_workflow(&workflow_id).await?; - let operator = workflow.operator.get_plot()?; + let operator = workflow.operator()?.get_plot()?; let execution_context = ctx.execution_context()?; @@ -287,7 +287,7 @@ mod tests { let session_id = session.id(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: Statistics { params: StatisticsParams { column_names: vec![], @@ -353,7 +353,7 @@ mod tests { let session_id = session.id(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: Histogram { params: HistogramParams { attribute_name: "band".to_string(), @@ -487,7 +487,7 @@ mod tests { let session_id = session.id(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: Statistics { params: StatisticsParams { column_names: vec![], diff --git a/services/src/api/handlers/wcs.rs b/services/src/api/handlers/wcs.rs index 689d273bf8..6c0db1896d 100644 --- a/services/src/api/handlers/wcs.rs +++ b/services/src/api/handlers/wcs.rs @@ -243,7 +243,7 @@ async fn wcs_describe_coverage( let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); let operator = workflow - .operator + .operator()? .get_raster()? .initialize(workflow_operator_path_root, &exe_ctx) .await?; @@ -387,7 +387,7 @@ async fn wcs_get_coverage( let workflow = ctx.db().load_workflow(&identifier).await?; - let operator = workflow.operator.get_raster()?; + let operator = workflow.operator()?.get_raster()?; let execution_context = ctx.execution_context()?; diff --git a/services/src/api/handlers/wfs.rs b/services/src/api/handlers/wfs.rs index 53cff55f73..0675ff3d86 100644 --- a/services/src/api/handlers/wfs.rs +++ b/services/src/api/handlers/wfs.rs @@ -340,7 +340,7 @@ where let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); let operator = workflow - .operator + .operator()? .get_vector()? .initialize(workflow_operator_path_root, &exe_ctx) .await?; @@ -494,7 +494,7 @@ async fn wfs_get_feature( let workflow: Workflow = ctx.db().load_workflow(&type_names).await?; - let operator = workflow.operator.get_vector()?; + let operator = workflow.operator()?.get_vector()?; let execution_context = ctx.execution_context()?; let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); @@ -840,7 +840,7 @@ x;y let session_id = session.id(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Vector(Box::new(CsvSource { params: CsvSourceParameters { file_path: temp_file.path().into(), @@ -910,7 +910,7 @@ x;y let session_id = session.id(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Vector(Box::new(CsvSource { params: CsvSourceParameters { file_path: temp_file.path().into(), @@ -1032,7 +1032,7 @@ x;y let session_id = session.id(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Vector(Box::new(CsvSource { params: CsvSourceParameters { file_path: temp_file.path().into(), diff --git a/services/src/api/handlers/wms.rs b/services/src/api/handlers/wms.rs index 23b7460e4a..5435b810f6 100644 --- a/services/src/api/handlers/wms.rs +++ b/services/src/api/handlers/wms.rs @@ -214,7 +214,7 @@ where let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); let operator = workflow - .operator + .operator()? .get_raster()? .initialize(workflow_operator_path_root, &exe_ctx) .await?; @@ -344,7 +344,7 @@ async fn wms_get_map( let workflow_id = WorkflowId::from_str(&request.layers)?; let workflow = ctx.db().load_workflow(&workflow_id).await?; - let operator = workflow.operator.get_raster()?; + let operator = workflow.operator()?.get_raster()?; let execution_context = ctx.execution_context()?; diff --git a/services/src/api/handlers/workflows.rs b/services/src/api/handlers/workflows.rs index 418e9459cd..b55ced6771 100755 --- a/services/src/api/handlers/workflows.rs +++ b/services/src/api/handlers/workflows.rs @@ -204,7 +204,7 @@ async fn workflow_metadata( let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); let result_descriptor: geoengine_operators::engine::TypedResultDescriptor = call_on_typed_operator!( - workflow.operator, + workflow.operator()?, operator => { let operator = operator .initialize(workflow_operator_path_root, &execution_context).await @@ -262,7 +262,7 @@ async fn workflow_provenance( let db = ctx.db(); let execution_ctx = ctx.execution_context()?; - let data_names = workflow.operator.data_names(); + let data_names = workflow.operator()?.data_names(); let mut datasets = Vec::::with_capacity(data_names.len()); for data_name in data_names { let data_id = execution_ctx.resolve_named_data(&data_name).await?; @@ -519,7 +519,7 @@ async fn raster_stream_websocket( let workflow = ctx.db().load_workflow(&workflow_id).await?; let operator = workflow - .operator + .operator()? .get_raster() .boxed_context(error::WorkflowMustBeOfTypeRaster)?; @@ -645,7 +645,7 @@ async fn vector_stream_websocket( let workflow = ctx.db().load_workflow(&workflow_id).await?; let operator = workflow - .operator + .operator()? .get_vector() .boxed_context(error::WorkflowMustBeOfTypeVector)?; @@ -718,6 +718,8 @@ pub enum WorkflowApiError { WorkflowMustBeOfTypeRaster { source: Box }, #[snafu(display("You can only query a vector stream for a vector workflow"))] WorkflowMustBeOfTypeVector { source: Box }, + #[snafu(display("Unsupported operator type in workflow: {source}"))] + EngineTypeConversion { source: anyhow::Error }, } #[cfg(test)] @@ -790,7 +792,7 @@ mod tests { let session_id = session.id(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: MockPointSource { params: MockPointSourceParams::new(vec![(0.0, 0.1).into(), (1.0, 1.1).into()]), } @@ -828,7 +830,7 @@ mod tests { #[ge_context::test] async fn register_missing_header(app_ctx: PostgresContext) { - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: MockPointSource { params: MockPointSourceParams::new(vec![(0.0, 0.1).into(), (1.0, 1.1).into()]), } @@ -896,7 +898,7 @@ mod tests { res, 400, "BodyDeserializeError", - "Error in user input: missing field `type` at line 1 column 2", + "Error in user input: data did not match any variant of untagged enum Workflow", ) .await; } @@ -993,7 +995,7 @@ mod tests { let session_id = session.id(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: MockFeatureCollectionSource::single( MultiPointCollection::from_data( MultiPoint::many(vec![(0.0, 0.1)]).unwrap(), @@ -1063,7 +1065,7 @@ mod tests { let session_id = session.id(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: MockRasterSource:: { params: MockRasterSourceParams:: { data: vec![], @@ -1156,7 +1158,7 @@ mod tests { let session = app_ctx.create_anonymous_session().await.unwrap(); let ctx = app_ctx.session_context(session.clone()); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: MockFeatureCollectionSource::single( MultiPointCollection::from_data( MultiPoint::many(vec![(0.0, 0.1)]).unwrap(), @@ -1197,7 +1199,7 @@ mod tests { let session_id = session.id(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: Statistics { params: StatisticsParams { column_names: vec![], @@ -1241,7 +1243,7 @@ mod tests { let session_id = session.id(); let (dataset_id, dataset) = add_ndvi_to_datasets(&app_ctx).await; - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Raster( GdalSource { params: GdalSourceParameters::new(dataset), @@ -1344,7 +1346,7 @@ mod tests { let (dataset_id, dataset_name) = add_ndvi_to_datasets(&app_ctx).await; - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Raster( GdalSource { params: GdalSourceParameters::new(dataset_name.clone()), @@ -1473,7 +1475,7 @@ mod tests { } .boxed(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Raster(operator_a.clone()), }; @@ -1569,7 +1571,7 @@ mod tests { let (_, dataset) = add_ndvi_to_datasets(&app_ctx).await; - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Raster( GdalSource { params: GdalSourceParameters { @@ -1648,7 +1650,7 @@ mod tests { let (_, dataset) = add_ports_to_datasets(&app_ctx, true, true).await; - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Vector( OgrSource { params: OgrSourceParameters { diff --git a/services/src/api/model/mod.rs b/services/src/api/model/mod.rs index 71bf052020..6eff2eed21 100644 --- a/services/src/api/model/mod.rs +++ b/services/src/api/model/mod.rs @@ -1,4 +1,5 @@ pub mod datatypes; pub mod operators; +pub mod processing_graphs; pub mod responses; pub mod services; diff --git a/services/src/api/model/operators.rs b/services/src/api/model/operators.rs index ecd71ed2bd..e439d80fa7 100644 --- a/services/src/api/model/operators.rs +++ b/services/src/api/model/operators.rs @@ -245,13 +245,13 @@ impl From for geoengine_operators::engine::RasterResultD /// An enum to differentiate between `Operator` variants #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(tag = "type", content = "operator")] -pub enum TypedOperator { +pub enum LegacyTypedOperator { Vector(Box), Raster(Box), Plot(Box), } -impl PartialSchema for TypedOperator { +impl PartialSchema for LegacyTypedOperator { fn schema() -> utoipa::openapi::RefOr { use utoipa::openapi::schema::{Object, ObjectBuilder, SchemaType, Type}; ObjectBuilder::new() @@ -288,7 +288,7 @@ impl PartialSchema for TypedOperator { } } -impl ToSchema for TypedOperator {} +impl ToSchema for LegacyTypedOperator {} #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] diff --git a/services/src/api/model/processing_graphs/mod.rs b/services/src/api/model/processing_graphs/mod.rs new file mode 100644 index 0000000000..388c09b9a9 --- /dev/null +++ b/services/src/api/model/processing_graphs/mod.rs @@ -0,0 +1,123 @@ +#![allow(clippy::needless_for_each)] // TODO: remove when clippy is fixed for utoipa + +// use crate::api::model::processing_graphs::{ +// processing::{Expression, ExpressionParameters, RasterVectorJoin, RasterVectorJoinParameters}, +// source::{GdalSource, GdalSourceParameters, MockPointSource, MockPointSourceParameters}, +// }; +use geoengine_operators::{ + engine::{ + RasterOperator as OperatorsRasterOperator, TypedOperator as OperatorsTypedOperator, + VectorOperator as OperatorsVectorOperator, + }, + mock::MockPointSource as OperatorsMockPointSource, + processing::{ + Expression as OperatorsExpression, RasterVectorJoin as OperatorsRasterVectorJoin, + }, + source::GdalSource as OperatorsGdalSource, +}; +use serde::{Deserialize, Serialize}; +use utoipa::{OpenApi, ToSchema}; + +mod parameters; +mod processing; +mod source; + +// TODO: avoid exporting them to outside of API module +#[cfg(test)] +pub(crate) use crate::api::model::processing_graphs::parameters::SpatialBoundsDerive; +pub(crate) use crate::api::model::processing_graphs::{ + processing::{Expression, ExpressionParameters, RasterVectorJoin, RasterVectorJoinParameters}, + source::{GdalSource, GdalSourceParameters, MockPointSource, MockPointSourceParameters}, +}; + +/// Operator outputs are distinguished by their data type. +/// There are `raster`, `vector` and `plot` operators. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[serde(tag = "type", content = "operator")] +pub enum TypedOperator { + Vector(VectorOperator), + Raster(RasterOperator), + Plot(PlotOperator), +} + +/// An operator that produces raster data. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase", untagged)] +#[schema(discriminator = "type")] +pub enum RasterOperator { + Expression(Expression), + GdalSource(GdalSource), +} + +/// An operator that produces vector data. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase", untagged)] +#[schema(discriminator = "type")] +pub enum VectorOperator { + MockPointSource(MockPointSource), + RasterVectorJoin(RasterVectorJoin), +} + +/// An operator that produces plot data. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase", untagged)] +// #[schema(discriminator = "type")] +pub enum PlotOperator { + // Currently no plot operators are defined +} + +impl TryFrom for Box { + type Error = anyhow::Error; + fn try_from(operator: RasterOperator) -> Result { + match operator { + RasterOperator::Expression(expression) => { + OperatorsExpression::try_from(expression).map(OperatorsRasterOperator::boxed) + } + RasterOperator::GdalSource(gdal_source) => { + OperatorsGdalSource::try_from(gdal_source).map(OperatorsRasterOperator::boxed) + } + } + } +} + +impl TryFrom for Box { + type Error = anyhow::Error; + fn try_from(operator: VectorOperator) -> Result { + match operator { + VectorOperator::MockPointSource(mock_point_source) => { + OperatorsMockPointSource::try_from(mock_point_source) + .map(OperatorsVectorOperator::boxed) + } + VectorOperator::RasterVectorJoin(rvj) => { + OperatorsRasterVectorJoin::try_from(rvj).map(OperatorsVectorOperator::boxed) + } + } + } +} + +impl TryFrom for OperatorsTypedOperator { + type Error = anyhow::Error; + fn try_from(operator: TypedOperator) -> Result { + match operator { + TypedOperator::Raster(raster_operator) => Ok(Self::Raster(raster_operator.try_into()?)), + TypedOperator::Vector(vector_operator) => Ok(Self::Vector(vector_operator.try_into()?)), + } + } +} + +#[derive(OpenApi)] +#[openapi(components(schemas( + Expression, + ExpressionParameters, + GdalSource, + GdalSourceParameters, + MockPointSource, + MockPointSourceParameters, + RasterOperator, + RasterVectorJoin, + RasterVectorJoinParameters, + TypedOperator, + VectorOperator, + PlotOperator, +)))] +pub struct OperatorsApi; diff --git a/services/src/api/model/processing_graphs/parameters.rs b/services/src/api/model/processing_graphs/parameters.rs new file mode 100644 index 0000000000..aabffbe91f --- /dev/null +++ b/services/src/api/model/processing_graphs/parameters.rs @@ -0,0 +1,535 @@ +use crate::api::model::{ + datatypes::Coordinate2D, + processing_graphs::{RasterOperator, VectorOperator}, +}; +use anyhow::Context; +use geoengine_macros::type_tag; +use serde::{Deserialize, Serialize, Serializer}; +use std::collections::BTreeMap; +use utoipa::ToSchema; + +/// A raster data type. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub enum RasterDataType { + U8, + U16, + U32, + U64, + I8, + I16, + I32, + I64, + F32, + F64, +} + +impl From for RasterDataType { + fn from(value: geoengine_datatypes::raster::RasterDataType) -> Self { + match value { + geoengine_datatypes::raster::RasterDataType::U8 => Self::U8, + geoengine_datatypes::raster::RasterDataType::U16 => Self::U16, + geoengine_datatypes::raster::RasterDataType::U32 => Self::U32, + geoengine_datatypes::raster::RasterDataType::U64 => Self::U64, + geoengine_datatypes::raster::RasterDataType::I8 => Self::I8, + geoengine_datatypes::raster::RasterDataType::I16 => Self::I16, + geoengine_datatypes::raster::RasterDataType::I32 => Self::I32, + geoengine_datatypes::raster::RasterDataType::I64 => Self::I64, + geoengine_datatypes::raster::RasterDataType::F32 => Self::F32, + geoengine_datatypes::raster::RasterDataType::F64 => Self::F64, + } + } +} + +impl From for geoengine_datatypes::raster::RasterDataType { + fn from(value: RasterDataType) -> Self { + match value { + RasterDataType::U8 => Self::U8, + RasterDataType::U16 => Self::U16, + RasterDataType::U32 => Self::U32, + RasterDataType::U64 => Self::U64, + RasterDataType::I8 => Self::I8, + RasterDataType::I16 => Self::I16, + RasterDataType::I32 => Self::I32, + RasterDataType::I64 => Self::I64, + RasterDataType::F32 => Self::F32, + RasterDataType::F64 => Self::F64, + } + } +} + +/// Measurement information for a raster band. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, ToSchema)] +#[serde(rename_all = "camelCase", untagged)] +#[schema(discriminator = "type")] +pub enum Measurement { + Unitless(UnitlessMeasurement), + Continuous(ContinuousMeasurement), + Classification(ClassificationMeasurement), +} + +impl From for Measurement { + fn from(value: geoengine_datatypes::primitives::Measurement) -> Self { + match value { + geoengine_datatypes::primitives::Measurement::Unitless => { + Self::Unitless(UnitlessMeasurement { + r#type: Default::default(), + }) + } + geoengine_datatypes::primitives::Measurement::Continuous(cm) => { + Self::Continuous(cm.into()) + } + geoengine_datatypes::primitives::Measurement::Classification(cm) => { + Self::Classification(cm.into()) + } + } + } +} + +impl From for geoengine_datatypes::primitives::Measurement { + fn from(value: Measurement) -> Self { + match value { + Measurement::Unitless(_) => Self::Unitless, + Measurement::Continuous(cm) => Self::Continuous(cm.into()), + Measurement::Classification(cm) => Self::Classification(cm.into()), + } + } +} + +/// A measurement without a unit. +#[type_tag(value = "unitless")] +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, ToSchema, Default)] +pub struct UnitlessMeasurement {} + +/// A continuous measurement, e.g., "temperature". +/// It may have an optional unit, e.g., "°C" for degrees Celsius. +#[type_tag(value = "continuous")] +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, ToSchema)] +pub struct ContinuousMeasurement { + pub measurement: String, + pub unit: Option, +} + +impl From for ContinuousMeasurement { + fn from(value: geoengine_datatypes::primitives::ContinuousMeasurement) -> Self { + ContinuousMeasurement { + r#type: Default::default(), + measurement: value.measurement, + unit: value.unit, + } + } +} + +impl From for geoengine_datatypes::primitives::ContinuousMeasurement { + fn from(value: ContinuousMeasurement) -> Self { + Self { + measurement: value.measurement, + unit: value.unit, + } + } +} + +impl From + for ClassificationMeasurement +{ + fn from(value: geoengine_datatypes::primitives::ClassificationMeasurement) -> Self { + let mut classes = BTreeMap::new(); + for (k, v) in value.classes { + classes.insert(k, v); + } + ClassificationMeasurement { + r#type: Default::default(), + measurement: value.measurement, + classes, + } + } +} + +impl From + for geoengine_datatypes::primitives::ClassificationMeasurement +{ + fn from(value: ClassificationMeasurement) -> Self { + geoengine_datatypes::primitives::ClassificationMeasurement { + measurement: value.measurement, + classes: value.classes, + } + } +} + +/// A classification measurement. +/// It contains a mapping from class IDs to class names. +#[type_tag(value = "classification")] +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, ToSchema)] +pub struct ClassificationMeasurement { + pub measurement: String, + #[serde(serialize_with = "serialize_classes")] + #[serde(deserialize_with = "deserialize_classes")] + pub classes: BTreeMap, +} + +fn serialize_classes(classes: &BTreeMap, serializer: S) -> Result +where + S: Serializer, +{ + use serde::ser::SerializeMap; + + let mut map = serializer.serialize_map(Some(classes.len()))?; + for (k, v) in classes { + map.serialize_entry(&k.to_string(), v)?; + } + map.end() +} + +fn deserialize_classes<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::de::Deserializer<'de>, +{ + use serde::de::{MapAccess, Visitor}; + use std::fmt; + + struct ClassesVisitor; + + impl<'de> Visitor<'de> for ClassesVisitor { + type Value = BTreeMap; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a map with numeric string keys") + } + + fn visit_map(self, mut access: M) -> Result + where + M: MapAccess<'de>, + { + let mut map = BTreeMap::new(); + while let Some((key, value)) = access.next_entry::()? { + let k = key.parse::().map_err(serde::de::Error::custom)?; + map.insert(k, value); + } + Ok(map) + } + } + + deserializer.deserialize_map(ClassesVisitor) +} + +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct RasterBandDescriptor { + pub name: String, + pub measurement: Measurement, +} + +impl From for RasterBandDescriptor { + fn from(value: geoengine_operators::engine::RasterBandDescriptor) -> Self { + Self { + name: value.name, + measurement: value.measurement.into(), + } + } +} + +impl From for geoengine_operators::engine::RasterBandDescriptor { + fn from(value: RasterBandDescriptor) -> Self { + Self { + name: value.name, + measurement: value.measurement.into(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, ToSchema)] +#[serde(rename_all = "camelCase", tag = "type")] +pub enum ColumnNames { + #[schema(title = "Default")] + Default, + #[schema(title = "Suffix")] + Suffix { values: Vec }, + #[schema(title = "Names")] + Names { values: Vec }, +} + +impl From for ColumnNames { + fn from(value: geoengine_operators::processing::ColumnNames) -> Self { + match value { + geoengine_operators::processing::ColumnNames::Default => ColumnNames::Default, + geoengine_operators::processing::ColumnNames::Suffix(v) => { + ColumnNames::Suffix { values: v } + } + geoengine_operators::processing::ColumnNames::Names(v) => { + ColumnNames::Names { values: v } + } + } + } +} + +impl From for geoengine_operators::processing::ColumnNames { + fn from(value: ColumnNames) -> Self { + match value { + ColumnNames::Default => geoengine_operators::processing::ColumnNames::Default, + ColumnNames::Suffix { values } => { + geoengine_operators::processing::ColumnNames::Suffix(values) + } + ColumnNames::Names { values } => { + geoengine_operators::processing::ColumnNames::Names(values) + } + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub enum FeatureAggregationMethod { + First, + Mean, +} + +impl From for FeatureAggregationMethod { + fn from(value: geoengine_operators::processing::FeatureAggregationMethod) -> Self { + match value { + geoengine_operators::processing::FeatureAggregationMethod::First => { + FeatureAggregationMethod::First + } + geoengine_operators::processing::FeatureAggregationMethod::Mean => { + FeatureAggregationMethod::Mean + } + } + } +} + +impl From for geoengine_operators::processing::FeatureAggregationMethod { + fn from(value: FeatureAggregationMethod) -> Self { + match value { + FeatureAggregationMethod::First => { + geoengine_operators::processing::FeatureAggregationMethod::First + } + FeatureAggregationMethod::Mean => { + geoengine_operators::processing::FeatureAggregationMethod::Mean + } + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub enum TemporalAggregationMethod { + None, + First, + Mean, +} + +impl From + for TemporalAggregationMethod +{ + fn from(value: geoengine_operators::processing::TemporalAggregationMethod) -> Self { + match value { + geoengine_operators::processing::TemporalAggregationMethod::None => { + TemporalAggregationMethod::None + } + geoengine_operators::processing::TemporalAggregationMethod::First => { + TemporalAggregationMethod::First + } + geoengine_operators::processing::TemporalAggregationMethod::Mean => { + TemporalAggregationMethod::Mean + } + } + } +} + +impl From + for geoengine_operators::processing::TemporalAggregationMethod +{ + fn from(value: TemporalAggregationMethod) -> Self { + match value { + TemporalAggregationMethod::None => { + geoengine_operators::processing::TemporalAggregationMethod::None + } + TemporalAggregationMethod::First => { + geoengine_operators::processing::TemporalAggregationMethod::First + } + TemporalAggregationMethod::Mean => { + geoengine_operators::processing::TemporalAggregationMethod::Mean + } + } + } +} + +/// Spatial bounds derivation options for the [`MockPointSource`]. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase", untagged)] +#[schema(discriminator = "type")] +pub enum SpatialBoundsDerive { + Derive(SpatialBoundsDeriveDerive), + Bounds(SpatialBoundsDeriveBounds), + None(SpatialBoundsDeriveNone), +} + +impl Default for SpatialBoundsDerive { + fn default() -> Self { + SpatialBoundsDerive::None(SpatialBoundsDeriveNone::default()) + } +} + +#[type_tag(value = "derive")] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, Default)] +pub struct SpatialBoundsDeriveDerive {} + +#[type_tag(value = "bounds")] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +pub struct SpatialBoundsDeriveBounds { + #[serde(flatten)] + pub bounding_box: BoundingBox2D, +} + +#[type_tag(value = "none")] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema, Default)] +pub struct SpatialBoundsDeriveNone {} + +impl TryFrom for geoengine_operators::mock::SpatialBoundsDerive { + type Error = anyhow::Error; + fn try_from(value: SpatialBoundsDerive) -> Result { + Ok(match value { + SpatialBoundsDerive::Derive(_) => { + geoengine_operators::mock::SpatialBoundsDerive::Derive + } + SpatialBoundsDerive::Bounds(bounds) => { + geoengine_operators::mock::SpatialBoundsDerive::Bounds( + bounds.bounding_box.try_into()?, + ) + } + SpatialBoundsDerive::None(_) => geoengine_operators::mock::SpatialBoundsDerive::None, + }) + } +} + +/// A bounding box that includes all border points. +/// Note: may degenerate to a point! +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct BoundingBox2D { + lower_left_coordinate: Coordinate2D, + upper_right_coordinate: Coordinate2D, +} + +impl TryFrom for geoengine_datatypes::primitives::BoundingBox2D { + type Error = anyhow::Error; + fn try_from(value: BoundingBox2D) -> Result { + geoengine_datatypes::primitives::BoundingBox2D::new( + value.lower_left_coordinate.into(), + value.upper_right_coordinate.into(), + ) + .context("invalid bounding box") + } +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[schema(no_recursion)] +#[serde(rename_all = "camelCase")] +pub struct SingleRasterSource { + pub raster: RasterOperator, +} + +impl TryFrom for geoengine_operators::engine::SingleRasterSource { + type Error = anyhow::Error; + + fn try_from(value: SingleRasterSource) -> Result { + Ok(Self { + raster: value.raster.try_into()?, + }) + } +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[schema(no_recursion)] +#[serde(rename_all = "camelCase")] +pub struct SingleVectorMultipleRasterSources { + pub vector: VectorOperator, + pub rasters: Vec, +} + +impl TryFrom + for geoengine_operators::engine::SingleVectorMultipleRasterSources +{ + type Error = anyhow::Error; + + fn try_from(value: SingleVectorMultipleRasterSources) -> Result { + Ok(Self { + vector: value.vector.try_into()?, + rasters: value + .rasters + .into_iter() + .map(std::convert::TryInto::try_into) + .collect::>()?, + }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::float_cmp)] // ok for tests + + use geoengine_datatypes::primitives::AxisAlignedRectangle; + + use super::*; + + #[test] + fn it_converts_coordinates() { + let dt = geoengine_datatypes::primitives::Coordinate2D { x: 1.5, y: -2.25 }; + + let api: Coordinate2D = dt.into(); + assert_eq!(api.x, 1.5); + assert_eq!(api.y, -2.25); + + let back: geoengine_datatypes::primitives::Coordinate2D = api.into(); + assert_eq!(back.x, 1.5); + assert_eq!(back.y, -2.25); + } + + #[test] + fn it_converts_raster_data_types() { + use geoengine_datatypes::raster::RasterDataType as Dt; + + let dt = Dt::F32; + let api: RasterDataType = dt.into(); + assert_eq!(api, RasterDataType::F32); + + let back: geoengine_datatypes::raster::RasterDataType = api.into(); + assert_eq!(back, Dt::F32); + } + + #[test] + fn it_converts_raster_band_descriptors() { + use geoengine_datatypes::primitives::Measurement; + use geoengine_operators::engine::RasterBandDescriptor as OpsDesc; + + let ops = OpsDesc { + name: "band 0".into(), + measurement: Measurement::Unitless, + }; + + let api: RasterBandDescriptor = ops.clone().into(); + assert_eq!(api.name, "band 0"); + + let back: geoengine_operators::engine::RasterBandDescriptor = api.into(); + assert_eq!(back, ops); + } + + #[test] + fn it_converts_bounding_boxes() { + let api_bbox = BoundingBox2D { + lower_left_coordinate: Coordinate2D { x: 1.0, y: 2.0 }, + upper_right_coordinate: Coordinate2D { x: 3.0, y: 4.0 }, + }; + + let dt_bbox: geoengine_datatypes::primitives::BoundingBox2D = + api_bbox.try_into().expect("it should convert"); + + assert_eq!( + dt_bbox.upper_left(), + geoengine_datatypes::primitives::Coordinate2D { x: 1.0, y: 4.0 } + ); + assert_eq!( + dt_bbox.lower_right(), + geoengine_datatypes::primitives::Coordinate2D { x: 3.0, y: 2.0 } + ); + } +} diff --git a/services/src/api/model/processing_graphs/processing.rs b/services/src/api/model/processing_graphs/processing.rs new file mode 100644 index 0000000000..178b3f9ac2 --- /dev/null +++ b/services/src/api/model/processing_graphs/processing.rs @@ -0,0 +1,377 @@ +use crate::api::model::processing_graphs::parameters::{ + ColumnNames, FeatureAggregationMethod, RasterBandDescriptor, RasterDataType, + SingleRasterSource, SingleVectorMultipleRasterSources, TemporalAggregationMethod, +}; +use geoengine_macros::type_tag; +use geoengine_operators::processing::{ + Expression as OperatorsExpression, ExpressionParams as OperatorsExpressionParameters, + RasterVectorJoin as OperatorsRasterVectorJoin, + RasterVectorJoinParams as OperatorsRasterVectorJoinParameters, +}; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +/// The `Expression` operator performs a pixel-wise mathematical expression on one or more bands of a raster source. +/// The expression is specified as a user-defined script in a very simple language. +/// The output is a raster time series with the result of the expression and with time intervals that are the same as for the inputs. +/// Users can specify an output data type. +/// Internally, the expression is evaluated using floating-point numbers. +/// +/// An example usage scenario is to calculate NDVI for a red and a near-infrared raster channel. +/// The expression uses a raster source with two bands, referred to as A and B, and calculates the formula `(A - B) / (A + B)`. +/// When the temporal resolution is months, our output NDVI will also be a monthly time series. +/// +/// ## Types +/// +/// The following describes the types used in the parameters. +/// +/// ### Expression +/// +/// Expressions are simple scripts to perform pixel-wise computations. +/// One can refer to the raster inputs as `A` for the first raster band, `B` for the second, and so on. +/// Furthermore, expressions can check with `A IS NODATA`, `B IS NODATA`, etc. for NO DATA values. +/// This is important if `mapNoData` is set to true. +/// Otherwise, NO DATA values are mapped automatically to the output NO DATA value. +/// Finally, the value `NODATA` can be used to output NO DATA. +/// +/// Users can think of this implicit function signature for, e.g., two inputs: +/// +/// ```Rust +/// fn (A: f64, B: f64) -> f64 +/// ``` +/// +/// As a start, expressions contain algebraic operations and mathematical functions. +/// +/// ```Rust +/// (A + B) / 2 +/// ``` +/// +/// In addition, branches can be used to check for conditions. +/// +/// ```Rust +/// if A IS NODATA { +/// B +/// } else { +/// A +/// } +/// ``` +/// +/// Function calls can be used to access utility functions. +/// +/// ```Rust +/// max(A, 0) +/// ``` +/// +/// Currently, the following functions are available: +/// +/// - `abs(a)`: absolute value +/// - `min(a, b)`, `min(a, b, c)`: minimum value +/// - `max(a, b)`, `max(a, b, c)`: maximum value +/// - `sqrt(a)`: square root +/// - `ln(a)`: natural logarithm +/// - `log10(a)`: base 10 logarithm +/// - `cos(a)`, `sin(a)`, `tan(a)`, `acos(a)`, `asin(a)`, `atan(a)`: trigonometric functions +/// - `pi()`, `e()`: mathematical constants +/// - `round(a)`, `ceil(a)`, `floor(a)`: rounding functions +/// - `mod(a, b)`: division remainder +/// - `to_degrees(a)`, `to_radians(a)`: conversion to degrees or radians +/// +/// To generate more complex expressions, it is possible to have variable assignments. +/// +/// ```Rust +/// let mean = (A + B) / 2; +/// let coefficient = 0.357; +/// mean * coefficient +/// ``` +/// +/// Note, that all assignments are separated by semicolons. +/// However, the last expression must be without a semicolon. +#[type_tag(value = "Expression")] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase")] +#[schema( + title = "Raster Expression", + examples(json!({ + "type": "Expression", + "params": { + "expression": "(A - B) / (A + B)", + "outputType": "F32", + "outputBand": { + "name": "NDVI", + "measurement": { "type": "unitless" }, + }, + "mapNoData": true + }, + "sources": { + "raster": { + "type": "GdalSource", + "params": { + "data": "ndvi" + } + } + } + })), +)] +pub struct Expression { + pub params: ExpressionParameters, + pub sources: Box, +} + +/// ## Types +/// +/// The following describes the types used in the parameters. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ExpressionParameters { + /// Expression script + /// + /// Example: `"(A - B) / (A + B)"` + #[schema(examples("(A - B) / (A + B)"))] + pub expression: String, + /// A raster data type for the output + #[schema(examples("F32"))] + pub output_type: RasterDataType, + /// Description about the output + #[schema( + nullable = false /* cannot be null, but left out, avoids `Option>` in openapi client */, + examples(json!({ + "name": "NDVI", + "measurement": { "type": "unitless" }, + })) + )] + pub output_band: Option, + /// Should NO DATA values be mapped with the `expression`? Otherwise, they are mapped automatically to NO DATA. + #[schema(examples(true))] + pub map_no_data: bool, +} + +impl TryFrom for OperatorsExpression { + type Error = anyhow::Error; + + fn try_from(value: Expression) -> Result { + Ok(OperatorsExpression { + params: OperatorsExpressionParameters { + expression: value.params.expression, + output_type: value.params.output_type.into(), + output_band: value.params.output_band.map(Into::into), + map_no_data: value.params.map_no_data, + }, + sources: (*value.sources).try_into()?, + }) + } +} + +/// The `RasterVectorJoin` operator allows combining a single vector input and multiple raster inputs. +/// For each raster input, a new column is added to the collection from the vector input. +/// The new column contains the value of the raster at the location of the vector feature. +/// For features covering multiple pixels like `MultiPoints` or `MultiPolygons`, the value is calculated using an aggregation function selected by the user. +/// The same is true if the temporal extent of a vector feature covers multiple raster time steps. +/// More details are described below. +/// +/// **Example**: +/// You have a collection of agricultural fields (`Polygons`) and a collection of raster images containing each pixel's monthly NDVI value. +/// For your application, you want to know the NDVI value of each field. +/// The `RasterVectorJoin` operator allows you to combine the vector and raster data and offers multiple spatial and temporal aggregation strategies. +/// For example, you can use the `first` aggregation function to get the NDVI value of the first pixel that intersects with each field. +/// This is useful for exploratory analysis since the computation is very fast. +/// To calculate the mean NDVI value of all pixels that intersect with the field you should use the `mean` aggregation function. +/// Since the NDVI data is a monthly time series, you have to specify the temporal aggregation function as well. +/// The default is `none` which will create a new feature for each month. +/// Other options are `first` and `mean` which will calculate the first or mean NDVI value for each field over time. +/// +/// ## Inputs +/// +/// The `RasterVectorJoin` operator expects one _vector_ input and one or more _raster_ inputs. +/// +/// | Parameter | Type | +/// | --------- | ----------------------------------- | +/// | `sources` | `SingleVectorMultipleRasterSources` | +/// +/// ## Errors +/// +/// If the length of `names` is not equal to the number of raster inputs, an error is thrown. +/// +#[type_tag(value = "RasterVectorJoin")] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase")] +#[schema( + title = "Raster Vector Join", + examples(json!({ + "type": "RasterVectorJoin", + "params": { + "names": ["NDVI"], + "featureAggregation": "first", + "temporalAggregation": "mean", + "temporalAggregationIgnoreNoData": true + }, + "sources": { + "vector": { + "type": "OgrSource", + "params": { + "data": "places" + } + }, + "rasters": [ + { + "type": "GdalSource", + "params": { + "data": "ndvi" + } + } + ] + } + })) +)] +pub struct RasterVectorJoin { + pub params: RasterVectorJoinParameters, + pub sources: Box, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct RasterVectorJoinParameters { + /// Specify how the new column names are derived from the raster band names. + /// + /// The `ColumnNames` type is used to specify how the new column names are derived from the raster band names. + /// + /// - **default**: Appends " (n)" to the band name with the smallest `n` that avoids a conflict. + /// - **suffix**: Specifies a suffix for each input, to be appended to the band names. + /// - **rename**: A list of names for each new column. + /// + #[schema(examples( + json!({"type": "default"}), + json!({"type": "suffix", "values": ["_sentinel2"]}), + json!({"type": "rename", "values": ["red", "green", "blue"]}), + ))] + pub names: ColumnNames, + /// The aggregation function to use for features covering multiple pixels. + #[schema(examples("first"))] + pub feature_aggregation: FeatureAggregationMethod, + /// Whether to ignore no data values in the aggregation. Defaults to `false`. + #[serde(default)] + #[schema(examples(true))] + pub feature_aggregation_ignore_no_data: bool, + /// The aggregation function to use for features covering multiple (raster) time steps. + #[schema(examples("mean"))] + pub temporal_aggregation: TemporalAggregationMethod, + /// Whether to ignore no data values in the aggregation. Defaults to `false`. + #[serde(default)] + #[schema(examples(true))] + pub temporal_aggregation_ignore_no_data: bool, +} + +impl TryFrom for OperatorsRasterVectorJoin { + type Error = anyhow::Error; + + fn try_from(value: RasterVectorJoin) -> Result { + Ok(OperatorsRasterVectorJoin { + params: OperatorsRasterVectorJoinParameters { + names: value.params.names.into(), + feature_aggregation: value.params.feature_aggregation.into(), + feature_aggregation_ignore_no_data: value.params.feature_aggregation_ignore_no_data, + temporal_aggregation: value.params.temporal_aggregation.into(), + temporal_aggregation_ignore_no_data: value + .params + .temporal_aggregation_ignore_no_data, + }, + sources: (*value.sources).try_into()?, + }) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::api::model::{ + datatypes::Coordinate2D, + processing_graphs::{ + RasterOperator, VectorOperator, + parameters::SpatialBoundsDerive, + source::{ + GdalSource, GdalSourceParameters, MockPointSource, MockPointSourceParameters, + }, + }, + }; + + #[test] + fn it_converts_expressions() { + let api = Expression { + r#type: Default::default(), + params: ExpressionParameters { + expression: "2 * A + B".to_string(), + output_type: RasterDataType::F32, + output_band: None, + map_no_data: true, + }, + sources: Box::new(SingleRasterSource { + raster: RasterOperator::GdalSource(GdalSource { + r#type: Default::default(), + params: GdalSourceParameters { + data: "example_data".to_string(), + overview_level: None, + }, + }), + }), + }; + + let ops = OperatorsExpression::try_from(api).expect("conversion failed"); + + assert_eq!(ops.params.expression, "2 * A + B"); + assert_eq!( + ops.params.output_type, + geoengine_datatypes::raster::RasterDataType::F32 + ); + assert!(ops.params.output_band.is_none()); + assert!(ops.params.map_no_data); + } + + #[test] + fn it_converts_raster_vector_join_params() { + let api = RasterVectorJoin { + r#type: Default::default(), + params: RasterVectorJoinParameters { + names: ColumnNames::Names { + values: vec!["a".to_string(), "b".to_string()], + }, + feature_aggregation: FeatureAggregationMethod::First, + feature_aggregation_ignore_no_data: true, + temporal_aggregation: TemporalAggregationMethod::Mean, + temporal_aggregation_ignore_no_data: false, + }, + sources: Box::new(SingleVectorMultipleRasterSources { + vector: VectorOperator::MockPointSource(MockPointSource { + r#type: Default::default(), + params: MockPointSourceParameters { + points: vec![Coordinate2D { x: 0.0, y: 0.0 }], + spatial_bounds: SpatialBoundsDerive::Derive(Default::default()), + }, + }), + rasters: vec![RasterOperator::GdalSource(GdalSource { + r#type: Default::default(), + params: GdalSourceParameters { + data: "example_data".to_string(), + overview_level: None, + }, + })], + }), + }; + + let ops_params = OperatorsRasterVectorJoin::try_from(api).expect("conversion failed"); + + assert!(matches!( + ops_params.params.names, + geoengine_operators::processing::ColumnNames::Names(_) + )); + assert_eq!( + ops_params.params.feature_aggregation, + geoengine_operators::processing::FeatureAggregationMethod::First + ); + assert!(ops_params.params.feature_aggregation_ignore_no_data); + assert_eq!( + ops_params.params.temporal_aggregation, + geoengine_operators::processing::TemporalAggregationMethod::Mean + ); + assert!(!ops_params.params.temporal_aggregation_ignore_no_data); + } +} diff --git a/services/src/api/model/processing_graphs/source.rs b/services/src/api/model/processing_graphs/source.rs new file mode 100644 index 0000000000..2d2952987a --- /dev/null +++ b/services/src/api/model/processing_graphs/source.rs @@ -0,0 +1,191 @@ +use crate::api::model::{ + datatypes::Coordinate2D, processing_graphs::parameters::SpatialBoundsDerive, +}; +use geoengine_datatypes::dataset::NamedData; +use geoengine_macros::type_tag; +use geoengine_operators::{ + mock::{ + MockPointSource as OperatorsMockPointSource, + MockPointSourceParams as OperatorsMockPointSourceParameters, + }, + source::{ + GdalSource as OperatorsGdalSource, GdalSourceParameters as OperatorsGdalSourceParameters, + }, +}; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +/// The [`GdalSource`] is a source operator that reads raster data using GDAL. +/// The counterpart for vector data is the [`OgrSource`]. +/// +/// ## Errors +/// +/// If the given dataset does not exist or is not readable, an error is thrown. +/// +#[type_tag(value = "GdalSource")] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase")] +#[schema( + title = "GDAL Source", + examples(json!({ + "type": "GdalSource", + "params": { + "data": "ndvi", + "overviewLevel": null + } + })) +)] +pub struct GdalSource { + pub params: GdalSourceParameters, +} + +/// Parameters for the [`GdalSource`] operator. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct GdalSourceParameters { + /// Dataset name or identifier to be loaded. + #[schema(examples("ndvi"))] + pub data: String, + + /// *Optional*: overview level to use. + /// + /// If not provided, the data source will determine the resolution, i.e., uses its native resolution. + #[schema(examples(3))] + pub overview_level: Option, +} + +impl TryFrom for OperatorsGdalSource { + type Error = anyhow::Error; + fn try_from(value: GdalSource) -> Result { + Ok(OperatorsGdalSource { + params: OperatorsGdalSourceParameters { + data: serde_json::from_str::(&serde_json::to_string( + &value.params.data, + )?)?, + overview_level: value.params.overview_level, + }, + }) + } +} + +/// The [`MockPointSource`] is a source operator that provides mock vector point data for testing and development purposes. +/// +#[type_tag(value = "MockPointSource")] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase")] +#[schema( + title = "Mock Point Source", + examples(json!({ + "type": "MockPointSource", + "params": { + "points": [ { "x": 1.0, "y": 2.0 }, { "x": 3.0, "y": 4.0 } ], + "spatialBounds": { "type": "derive" } + } + })) +)] +pub struct MockPointSource { + pub params: MockPointSourceParameters, +} + +/// Parameters for the [`MockPointSource`] operator. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct MockPointSourceParameters { + /// Points to be output by the mock point source. + /// + #[schema(examples(json!([ + { "x": 1.0, "y": 2.0 }, + { "x": 3.0, "y": 4.0 } + ])))] + pub points: Vec, + + /// Defines how the spatial bounds of the source are derived. + /// + /// Defaults to `None`. + #[schema(examples(json!({ "type": "derive" })))] + pub spatial_bounds: SpatialBoundsDerive, +} + +impl TryFrom for OperatorsMockPointSource { + type Error = anyhow::Error; + fn try_from(value: MockPointSource) -> Result { + Ok(OperatorsMockPointSource { + params: OperatorsMockPointSourceParameters { + points: value.params.points.into_iter().map(Into::into).collect(), + spatial_bounds: value.params.spatial_bounds.try_into()?, + }, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::api::model::processing_graphs::{RasterOperator, TypedOperator, VectorOperator}; + use geoengine_operators::engine::TypedOperator as OperatorsTypedOperator; + + #[test] + fn it_converts_into_gdal_source() { + let api_operator = GdalSource { + r#type: Default::default(), + params: GdalSourceParameters { + data: "example_dataset".to_string(), + overview_level: None, + }, + }; + + let operators_operator: OperatorsGdalSource = + api_operator.try_into().expect("it should convert"); + + assert_eq!( + operators_operator.params.data, + NamedData::with_system_name("example_dataset") + ); + + let typed_operator = TypedOperator::Raster(RasterOperator::GdalSource(GdalSource { + r#type: Default::default(), + params: GdalSourceParameters { + data: "example_dataset".to_string(), + overview_level: None, + }, + })); + + OperatorsTypedOperator::try_from(typed_operator).expect("it should convert"); + } + + #[test] + fn it_converts_mock_point_source() { + let api_operator = MockPointSource { + r#type: Default::default(), + params: MockPointSourceParameters { + points: vec![ + Coordinate2D { x: 1.0, y: 2.0 }, + Coordinate2D { x: 3.0, y: 4.0 }, + ], + spatial_bounds: SpatialBoundsDerive::Derive(Default::default()), + }, + }; + + let operators_operator: OperatorsMockPointSource = + api_operator.try_into().expect("it should convert"); + + assert_eq!( + operators_operator.params.points, + vec![ + geoengine_datatypes::primitives::Coordinate2D { x: 1.0, y: 2.0 }, + geoengine_datatypes::primitives::Coordinate2D { x: 3.0, y: 4.0 } + ] + ); + + let typed_operator = + TypedOperator::Vector(VectorOperator::MockPointSource(MockPointSource { + r#type: Default::default(), + params: MockPointSourceParameters { + points: vec![Coordinate2D { x: 1.0, y: 2.0 }], + spatial_bounds: SpatialBoundsDerive::Derive(Default::default()), + }, + })); + + OperatorsTypedOperator::try_from(typed_operator).expect("it should convert"); + } +} diff --git a/services/src/cli/tile_import.rs b/services/src/cli/tile_import.rs index 07a511551b..b1f2cc049c 100644 --- a/services/src/cli/tile_import.rs +++ b/services/src/cli/tile_import.rs @@ -223,7 +223,7 @@ async fn add_dataset_to_collection( let add_layer = AddLayer { name: layer_name.to_string(), description: String::new(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: geoengine_operators::engine::TypedOperator::Raster( MultiBandGdalSource { params: MultiBandGdalSourceParameters { diff --git a/services/src/contexts/postgres.rs b/services/src/contexts/postgres.rs index 92696a0946..62392425a3 100644 --- a/services/src/contexts/postgres.rs +++ b/services/src/contexts/postgres.rs @@ -730,7 +730,7 @@ mod tests { .unwrap(); let layer_workflow_id = db - .register_workflow(Workflow { + .register_workflow(Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), @@ -744,7 +744,7 @@ mod tests { assert!(db.load_workflow(&layer_workflow_id).await.is_ok()); let plot_workflow_id = db - .register_workflow(Workflow { + .register_workflow(Workflow::Legacy { operator: Statistics { params: StatisticsParams { column_names: vec![], @@ -974,7 +974,7 @@ mod tests { #[ge_context::test] async fn it_persists_workflows(app_ctx: PostgresContext) { - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), @@ -1761,7 +1761,7 @@ mod tests { let layer_db = app_ctx.session_context(session).db(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), @@ -1959,7 +1959,7 @@ mod tests { let layer_db = app_ctx.session_context(session).db(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), @@ -2315,7 +2315,7 @@ mod tests { let user_session = app_ctx.create_anonymous_session().await.unwrap(); let user_layer_db = app_ctx.session_context(user_session.clone()).db(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams { @@ -2838,7 +2838,7 @@ mod tests { let layer_db = app_ctx.session_context(session).db(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), @@ -3018,7 +3018,7 @@ mod tests { let user_session = app_ctx.create_anonymous_session().await.unwrap(); let user_layer_db = app_ctx.session_context(user_session.clone()).db(); - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams { @@ -3555,7 +3555,7 @@ mod tests { let layer = AddLayer { name: "layer".to_string(), description: "description".to_string(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), @@ -3751,7 +3751,7 @@ mod tests { AddLayer { name: "layer".to_string(), description: "description".to_string(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams::new(vec![ @@ -3821,7 +3821,7 @@ mod tests { AddLayer { name: "layer 1".to_string(), description: "description".to_string(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams::new(vec![ @@ -3846,7 +3846,7 @@ mod tests { AddLayer { name: "layer 2".to_string(), description: "description".to_string(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Vector( MockPointSource { params: MockPointSourceParams::new(vec![ diff --git a/services/src/datasets/create_from_workflow.rs b/services/src/datasets/create_from_workflow.rs index b4b6f2bca5..f84a2449b2 100644 --- a/services/src/datasets/create_from_workflow.rs +++ b/services/src/datasets/create_from_workflow.rs @@ -120,7 +120,7 @@ impl RasterDatasetFromWorkflowTask { let initialized_operator = workflow .clone() - .operator + .operator()? .get_raster() .expect("must be raster here") .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) diff --git a/services/src/datasets/dataset_listing_provider.rs b/services/src/datasets/dataset_listing_provider.rs index eebb176b90..f3746eb763 100644 --- a/services/src/datasets/dataset_listing_provider.rs +++ b/services/src/datasets/dataset_listing_provider.rs @@ -277,7 +277,7 @@ where }, name: dataset.display_name, description: dataset.description, - workflow: Workflow { operator }, + workflow: Workflow::Legacy { operator }, symbology: dataset.symbology, properties: vec![], metadata: HashMap::new(), diff --git a/services/src/datasets/external/aruna/mod.rs b/services/src/datasets/external/aruna/mod.rs index 60e903ef52..d945509324 100644 --- a/services/src/datasets/external/aruna/mod.rs +++ b/services/src/datasets/external/aruna/mod.rs @@ -918,7 +918,7 @@ impl LayerCollectionProvider for ArunaDataProvider { }, name: dataset.name, description: dataset.description, - workflow: Workflow { operator }, + workflow: Workflow::Legacy { operator }, symbology: None, properties: vec![], metadata: HashMap::new(), @@ -2004,7 +2004,7 @@ mod tests { } } }), - serde_json::to_value(&result.workflow.operator).unwrap() + serde_json::to_value(result.workflow.operator().unwrap()).unwrap() ); } @@ -2041,7 +2041,7 @@ mod tests { } } }), - serde_json::to_value(&result.workflow.operator).unwrap() + serde_json::to_value(result.workflow.operator().unwrap()).unwrap() ); } diff --git a/services/src/datasets/external/copernicus_dataspace/provider.rs b/services/src/datasets/external/copernicus_dataspace/provider.rs index d59f03c4d9..e69c549f1b 100644 --- a/services/src/datasets/external/copernicus_dataspace/provider.rs +++ b/services/src/datasets/external/copernicus_dataspace/provider.rs @@ -300,7 +300,7 @@ impl CopernicusDataspaceDataProvider { id.product_band.band_name() ), description: String::new(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Raster( GdalSource { params: GdalSourceParameters { diff --git a/services/src/datasets/external/edr.rs b/services/src/datasets/external/edr.rs index 25ff48ce87..55f9bfccc7 100644 --- a/services/src/datasets/external/edr.rs +++ b/services/src/datasets/external/edr.rs @@ -1036,7 +1036,7 @@ impl LayerCollectionProvider for EdrDataProvider { }, name: collection.title.unwrap_or(collection.id), description: String::new(), - workflow: Workflow { operator }, + workflow: Workflow::Legacy { operator }, symbology: None, // TODO properties: vec![], metadata: HashMap::new(), diff --git a/services/src/datasets/external/gbif.rs b/services/src/datasets/external/gbif.rs index fdeee22037..6d49549259 100644 --- a/services/src/datasets/external/gbif.rs +++ b/services/src/datasets/external/gbif.rs @@ -787,7 +787,7 @@ impl LayerCollectionProvider for GbifDataProvider { }, name: canonicalname.to_string(), description: format!("All occurrences with a {taxonrank} of {canonicalname}"), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Vector( OgrSource { params: OgrSourceParameters { @@ -3616,7 +3616,7 @@ mod tests { }, name: "Rhipidia willistoniana".to_string(), description: "All occurrences with a species of Rhipidia willistoniana".to_string(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Vector( OgrSource { params: OgrSourceParameters { diff --git a/services/src/datasets/external/gfbio_abcd.rs b/services/src/datasets/external/gfbio_abcd.rs index b35042fd14..9912ff169d 100644 --- a/services/src/datasets/external/gfbio_abcd.rs +++ b/services/src/datasets/external/gfbio_abcd.rs @@ -352,7 +352,7 @@ impl LayerCollectionProvider for GfbioAbcdDataProvider { }, name: row.get(0), description: row.try_get(1).unwrap_or_else(|_| String::new()), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Vector( OgrSource { params: OgrSourceParameters { diff --git a/services/src/datasets/external/gfbio_collections.rs b/services/src/datasets/external/gfbio_collections.rs index fa828cb65b..2d94454dca 100644 --- a/services/src/datasets/external/gfbio_collections.rs +++ b/services/src/datasets/external/gfbio_collections.rs @@ -683,7 +683,7 @@ impl LayerCollectionProvider for GfbioCollectionsDataProvider { }, name: layer.name, description: String::new(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Vector( OgrSource { params: OgrSourceParameters { diff --git a/services/src/datasets/external/netcdfcf/loading.rs b/services/src/datasets/external/netcdfcf/loading.rs index 3af2de9156..439d1c1a5e 100644 --- a/services/src/datasets/external/netcdfcf/loading.rs +++ b/services/src/datasets/external/netcdfcf/loading.rs @@ -145,7 +145,7 @@ pub fn create_layer( id: provider_layer_id, name: netcdf_entity.name.clone(), description: netcdf_entity.name, - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Raster( GdalSource { params: GdalSourceParameters::new(data_id), diff --git a/services/src/datasets/external/sentinel_s2_l2a_cogs/mod.rs b/services/src/datasets/external/sentinel_s2_l2a_cogs/mod.rs index f4d1237c46..3708c6a68f 100644 --- a/services/src/datasets/external/sentinel_s2_l2a_cogs/mod.rs +++ b/services/src/datasets/external/sentinel_s2_l2a_cogs/mod.rs @@ -238,7 +238,7 @@ impl SentinelS2L2aCogsDataProvider { }, name: format!("Sentinel S2 L2A COGS {}:{} ({})", zone, band.long_name(), band.name()), description: String::new(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: source_operator_from_dataset( GdalSource::TYPE_NAME, &NamedData { @@ -373,7 +373,7 @@ impl LayerCollectionProvider for SentinelS2L2aCogsDataProvider { }, name: dataset.listing.name.clone(), description: dataset.listing.description.clone(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: TypedOperator::Raster( GdalSource { params: GdalSourceParameters::new(NamedData { diff --git a/services/src/datasets/external/wildlive/mod.rs b/services/src/datasets/external/wildlive/mod.rs index 5b28543f51..6a36bed18d 100644 --- a/services/src/datasets/external/wildlive/mod.rs +++ b/services/src/datasets/external/wildlive/mod.rs @@ -364,7 +364,7 @@ impl LayerCollectionProvider for WildliveDataConnector { id: self.layer_id(WildliveLayerId::Projects)?, name: "Projects".to_string(), description: "Overview of all projects".to_string(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: VectorExpression { params: VectorExpressionParams { expression: "centroid(geom)".into(), @@ -432,7 +432,7 @@ impl LayerCollectionProvider for WildliveDataConnector { id: self.layer_id(WildliveLayerId::ProjectBounds)?, name: "Project Bounds".to_string(), description: "Overview of all project bounds".to_string(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: OgrSource { params: OgrSourceParameters { data: self.named_data(WildliveLayerId::ProjectBounds)?, @@ -462,7 +462,7 @@ impl LayerCollectionProvider for WildliveDataConnector { })?, name: format!("Stations for project {project_name}"), description: format!("Overview of all stations within project {project_id}"), - workflow: Workflow { + workflow: Workflow::Legacy { operator: OgrSource { params: OgrSourceParameters { data: self.named_data(WildliveLayerId::Stations { project_id })?, @@ -493,7 +493,7 @@ impl LayerCollectionProvider for WildliveDataConnector { })?, name: format!("Captures for project {project_name}"), description: format!("Overview of all captures within project {project_id}"), - workflow: Workflow { + workflow: Workflow::Legacy { operator: OgrSource { params: OgrSourceParameters { data: self.named_data(WildliveLayerId::Captures { project_id })?, @@ -1207,7 +1207,7 @@ mod tests { }, name: "Project Bounds".to_string(), description: "Overview of all project bounds".to_string(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: OgrSource { params: OgrSourceParameters { data: connector @@ -1239,7 +1239,7 @@ mod tests { }, name: "Projects".to_string(), description: "Overview of all projects".to_string(), - workflow: Workflow { + workflow: Workflow::Legacy { operator: VectorExpression { params: VectorExpressionParams { expression: "centroid(geom)".into(), @@ -1578,7 +1578,7 @@ mod tests { }, name: format!("Captures for project {project_name}"), description: format!("Overview of all captures within project {project_id}"), - workflow: Workflow { + workflow: Workflow::Legacy { operator: OgrSource { params: OgrSourceParameters { data: connector diff --git a/services/src/util/openapi_visitor.rs b/services/src/util/openapi_visitor.rs index bb8a73242b..83e3eb601d 100644 --- a/services/src/util/openapi_visitor.rs +++ b/services/src/util/openapi_visitor.rs @@ -1,3 +1,5 @@ +use std::collections::HashSet; + use utoipa::openapi::{ Components, HttpMethod, OpenApi, PathItem, Ref, RefOr, Response, Schema, path::Operation, @@ -26,36 +28,61 @@ pub fn visit_schema( components: &Components, visitor: &mut T, source_location: &str, + already_visited: &mut HashSet, ) { match schema { RefOr::Ref(reference) => { - visit_reference(reference, components, visitor, source_location); + visit_reference( + reference, + components, + visitor, + source_location, + already_visited, + ); } RefOr::T(concrete) => match concrete { Schema::Array(arr) => { if let ArrayItems::RefOrSchema(schema) = &arr.items { - visit_schema(schema, components, visitor, source_location); + visit_schema( + schema, + components, + visitor, + source_location, + already_visited, + ); } } Schema::Object(obj) => { for property in obj.properties.values() { - visit_schema(property, components, visitor, source_location); + visit_schema( + property, + components, + visitor, + source_location, + already_visited, + ); } if let Some(additional_properties) = &obj.additional_properties && let AdditionalProperties::RefOr(properties_schema) = additional_properties.as_ref() { - visit_schema(properties_schema, components, visitor, source_location); + visit_schema( + properties_schema, + components, + visitor, + source_location, + already_visited, + ); } } Schema::OneOf(oo) => { for item in &oo.items { - visit_schema(item, components, visitor, source_location); + visit_schema(item, components, visitor, source_location, already_visited); } } Schema::AllOf(ao) => { for item in &ao.items { - visit_schema(item, components, visitor, source_location); + visit_schema(item, components, visitor, source_location, already_visited); } } _ => panic!("Unknown schema type"), @@ -73,17 +100,30 @@ fn visit_response( components: &Components, visitor: &mut T, source_location: &str, + already_visited: &mut HashSet, ) { match response { RefOr::Ref(reference) => { - visit_reference(reference, components, visitor, source_location); + visit_reference( + reference, + components, + visitor, + source_location, + already_visited, + ); } RefOr::T(concrete) => { for content in concrete.content.values() { let Some(content_schema) = &content.schema else { continue; }; - visit_schema(content_schema, components, visitor, source_location); + visit_schema( + content_schema, + components, + visitor, + source_location, + already_visited, + ); } } } @@ -100,6 +140,7 @@ fn visit_reference( components: &Components, visitor: &mut T, source_location: &str, + already_visited: &mut HashSet, ) { const SCHEMA_REF_PREFIX: &str = "#/components/schemas/"; const RESPONSE_REF_PREFIX: &str = "#/components/responses/"; @@ -111,13 +152,18 @@ fn visit_reference( None => visitor.resolve_failed(ref_location), Some(resolved) => { visitor.visit_schema_component(schema_name, resolved, source_location); - visit_schema(resolved, components, visitor, ref_location); + if !already_visited.insert(ref_location.to_string()) { + return; // prevent infinite recursion + } + visit_schema(resolved, components, visitor, ref_location, already_visited); } } } else if let Some(response_name) = ref_location.strip_prefix(RESPONSE_REF_PREFIX) { match components.responses.get(response_name) { None => visitor.resolve_failed(ref_location), - Some(resolved) => visit_response(resolved, components, visitor, ref_location), + Some(resolved) => { + visit_response(resolved, components, visitor, ref_location, already_visited); + } } } else { visitor.resolve_failed(ref_location); @@ -141,7 +187,13 @@ pub fn visit_api(api: &OpenApi, visitor: &mut T) { if let Some(parameters) = &path_item.parameters { for parameter in parameters { if let Some(schema) = parameter.schema.as_ref() { - visit_schema(schema, components, visitor, source_location); + visit_schema( + schema, + components, + visitor, + source_location, + &mut Default::default(), + ); } } } @@ -152,14 +204,26 @@ pub fn visit_api(api: &OpenApi, visitor: &mut T) { let Some(content_schema) = &content.schema else { continue; }; - visit_schema(content_schema, components, visitor, source_location); + visit_schema( + content_schema, + components, + visitor, + source_location, + &mut Default::default(), + ); } } if let Some(parameters) = operation.parameters.as_ref() { for parameter in parameters { if let Some(schema) = parameter.schema.as_ref() { - visit_schema(schema, components, visitor, source_location); + visit_schema( + schema, + components, + visitor, + source_location, + &mut Default::default(), + ); } } } @@ -167,14 +231,26 @@ pub fn visit_api(api: &OpenApi, visitor: &mut T) { for response in operation.responses.responses.values() { match response { RefOr::Ref(reference) => { - visit_reference(reference, components, visitor, source_location); + visit_reference( + reference, + components, + visitor, + source_location, + &mut Default::default(), + ); } RefOr::T(concrete) => { for content in concrete.content.values() { let Some(content_schema) = &content.schema else { continue; }; - visit_schema(content_schema, components, visitor, source_location); + visit_schema( + content_schema, + components, + visitor, + source_location, + &mut Default::default(), + ); } } } diff --git a/services/src/util/openapi_visitors.rs b/services/src/util/openapi_visitors.rs index 075317cf37..cf506921b8 100644 --- a/services/src/util/openapi_visitors.rs +++ b/services/src/util/openapi_visitors.rs @@ -79,7 +79,13 @@ mod tests { fn try_resolve_schema(schema: &RefOr, components: &Components) { let mut visitor = CanResolveVisitor { unknown_ref: None }; - visit_schema(schema, components, &mut visitor, "root"); + visit_schema( + schema, + components, + &mut visitor, + "root", + &mut Default::default(), + ); if let Some(unknown_ref) = visitor.unknown_ref { panic!("Cannot resolve reference {unknown_ref}"); @@ -93,7 +99,13 @@ mod tests { let mut visitor = SchemaUseCounter { parents: HashMap::new(), }; - visit_schema(schema, components, &mut visitor, "root"); + visit_schema( + schema, + components, + &mut visitor, + "root", + &mut Default::default(), + ); visitor.get_schema_use_counts() } diff --git a/services/src/util/tests.rs b/services/src/util/tests.rs index 42dc4e00e6..ed7510faf8 100644 --- a/services/src/util/tests.rs +++ b/services/src/util/tests.rs @@ -148,7 +148,7 @@ pub async fn register_ndvi_workflow_helper_with_cache_ttl( ) -> (Workflow, WorkflowId) { let (_, dataset) = add_ndvi_to_datasets_with_cache_ttl(app_ctx, cache_ttl).await; - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Raster( GdalSource { params: GdalSourceParameters::new(dataset), @@ -366,7 +366,7 @@ pub async fn register_ne2_multiband_workflow( ) .await; - let workflow = Workflow { + let workflow = Workflow::Legacy { operator: TypedOperator::Raster( RasterStacker { params: RasterStackerParams { diff --git a/services/src/util/workflows.rs b/services/src/util/workflows.rs index 736c34021b..da94e3f27f 100644 --- a/services/src/util/workflows.rs +++ b/services/src/util/workflows.rs @@ -8,7 +8,7 @@ pub async fn validate_workflow( ) -> Result<()> { let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); - match workflow.clone().operator { + match workflow.clone().operator()? { TypedOperator::Vector(o) => { o.initialize(workflow_operator_path_root, execution_context) .await?; diff --git a/services/src/workflows/workflow.rs b/services/src/workflows/workflow.rs index 7f1d09b24c..03b63f2104 100644 --- a/services/src/workflows/workflow.rs +++ b/services/src/workflows/workflow.rs @@ -1,10 +1,11 @@ +use crate::api::{handlers::workflows::WorkflowApiError, model::processing_graphs::TypedOperator}; +use crate::error::Result; +use crate::identifier; +use geoengine_operators::engine::TypedOperator as OperatorsTypedOperator; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use uuid::Uuid; -use crate::identifier; -use geoengine_operators::engine::TypedOperator; - identifier!(WorkflowId); impl WorkflowId { @@ -20,10 +21,18 @@ impl WorkflowId { } #[derive(Clone, Debug, Serialize, Deserialize, ToSchema)] -pub struct Workflow { - #[serde(flatten)] - #[schema(value_type = crate::api::model::operators::TypedOperator)] - pub operator: TypedOperator, +#[serde(untagged)] +pub enum Workflow { + Typed { + #[serde(flatten)] + operator: TypedOperator, + }, + // TODO: remove this variant when all workflows are migrated to typed ones + Legacy { + #[serde(flatten)] + #[schema(value_type = crate::api::model::operators::LegacyTypedOperator)] + operator: OperatorsTypedOperator, + }, } impl PartialEq for Workflow { @@ -35,22 +44,42 @@ impl PartialEq for Workflow { } } +impl Workflow { + pub fn operator(&self) -> Result { + match self { + Workflow::Typed { operator } => { + operator + .clone() + .try_into() + .map_err(|source: anyhow::Error| crate::error::Error::WorkflowApi { + source: WorkflowApiError::EngineTypeConversion { source }, + }) + } + Workflow::Legacy { operator } => Ok(operator.clone()), + } + } +} + #[cfg(test)] mod tests { use super::*; - use geoengine_datatypes::primitives::Coordinate2D; - use geoengine_operators::engine::VectorOperator; - use geoengine_operators::mock::{MockPointSource, MockPointSourceParams}; + use crate::api::model::{ + datatypes::Coordinate2D, + processing_graphs::{ + MockPointSource, MockPointSourceParameters, SpatialBoundsDerive, VectorOperator, + }, + }; #[test] fn serde() { - let workflow = Workflow { - operator: TypedOperator::Vector( - MockPointSource { - params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), - } - .boxed(), - ), + let workflow = Workflow::Typed { + operator: TypedOperator::Vector(VectorOperator::MockPointSource(MockPointSource { + r#type: Default::default(), + params: MockPointSourceParameters { + points: vec![Coordinate2D { x: 1., y: 2. }; 3], + spatial_bounds: SpatialBoundsDerive::None(Default::default()), + }, + })), }; let serialized_workflow = serde_json::to_value(&workflow).unwrap(); @@ -80,6 +109,8 @@ mod tests { }) ); - // TODO: check deserialization + let deserialized_workflow: Workflow = serde_json::from_value(serialized_workflow).unwrap(); + + assert_eq!(workflow, deserialized_workflow); } }