diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index c94189a7c..c8b2a6b48 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,4 +1,4 @@ # These are supported funding model platforms #github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] -custom: ['https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=WT27AS28UFSNW&source=url'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] +custom: ['https://github.com/geopython/pygeoapi/wiki/Sponsorship'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index a59b51153..b09b56899 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -7,6 +7,12 @@ assignees: '' --- + + **Description** A clear and concise description of what the bug is. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 11fc491ef..31138e7df 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -7,6 +7,12 @@ assignees: '' --- + + **Is your feature request related to a problem? Please describe.** A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 023067888..2d61c7ebb 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,15 @@ # Overview -# Related issue / discussion +# Related Issue / discussion + + # Additional information diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 67544fed2..f04834da5 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -12,7 +12,7 @@ on: branches: [master] env: - DOCKER_REPOSITORY: geopython/pygeoapi + DOCKER_REPOSITORY: ${{ secrets.DOCKER_REPOSITORY || 'geopython/pygeoapi' }} # DOCKER_TEST_IMAGE: geopython/pygeoapi:test jobs: @@ -25,7 +25,7 @@ jobs: contents: read steps: - name: Check out the repo - uses: actions/checkout@v3 + uses: actions/checkout@master - name: Set up QEMU uses: docker/setup-qemu-action@v2.1.0 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..9de9ab025 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,36 @@ +name: Build documentation ⚙️ + +on: + push: + branches: + - master + paths: + - 'docs/**' + pull_request: + branches: + - master + paths: + - 'docs/**' + release: + types: + - released + +jobs: + main: + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - python-version: '3.10' + steps: + - uses: actions/checkout@master + - uses: actions/setup-python@v5 + name: Setup Python ${{ matrix.python-version }} + with: + python-version: ${{ matrix.python-version }} + - name: Install requirements 📦 + run: | + pip3 install -r requirements.txt + pip3 install -r docs/requirements.txt + - name: build docs 🏗️ + run: cd docs && make html diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml new file mode 100644 index 000000000..b57ba36c2 --- /dev/null +++ b/.github/workflows/flake8.yml @@ -0,0 +1,24 @@ +name: flake8 + +on: + [ push, pull_request ] + +jobs: + flake8_py3: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@master + - uses: actions/setup-python@v5 + name: setup Python + with: + python-version: '3.10' + - name: Checkout pygeoapi + uses: actions/checkout@master + - name: Install flake8 + run: pip3 install flake8 + - name: Run flake8 + uses: suo/flake8-github-action@releases/v1 + with: + checkName: 'flake8_py3' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6b154baed..069914afb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,51 +4,41 @@ on: push: paths-ignore: - '**.md' + - 'docs/**' pull_request: branches: - master paths-ignore: - '!**.md' + - 'docs/**' release: types: - released jobs: - flake8_py3: - runs-on: ubuntu-22.04 - steps: - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - architecture: x64 - - name: Checkout pygeoapi - uses: actions/checkout@master - - name: Install flake8 - run: pip install flake8 - - name: Run flake8 - uses: suo/flake8-github-action@releases/v1 - with: - checkName: 'flake8_py3' # NOTE: this needs to be the same as the job name - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - main: - needs: [flake8_py3] runs-on: ubuntu-22.04 strategy: matrix: include: - - python-version: 3.8 + - python-version: '3.10' env: PYGEOAPI_CONFIG: "$(pwd)/pygeoapi-config.yml" steps: + - name: Clear up GitHub runner diskspace + run: | + echo "Space before" + df -h / + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet + echo "Space after" + df -h / - name: Chown user run: | sudo chown -R $USER:$USER $GITHUB_WORKSPACE - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@master + - uses: actions/setup-python@v5 name: Setup Python ${{ matrix.python-version }} with: python-version: ${{ matrix.python-version }} @@ -66,28 +56,37 @@ jobs: - name: Install and run Elasticsearch 📦 uses: getong/elasticsearch-action@v1.2 with: - elasticsearch version: '8.3.1' + elasticsearch version: '8.17.0' host port: 9200 container port: 9200 host node port: 9300 node port: 9300 discovery type: 'single-node' + - name: Install and run OpenSearch 📦 + uses: esmarkowski/opensearch-github-action@v1.0.0 + with: + version: 2.18.0 + security-disabled: true + port: 9209 - name: Install and run MongoDB - uses: supercharge/mongodb-github-action@1.5.0 + uses: supercharge/mongodb-github-action@1.12.0 with: - mongodb-version: 4.4 + mongodb-version: '8.0.4' - name: Install and run SensorThingsAPI - uses: cgs-earth/sensorthings-action@v0.0.2 + uses: cgs-earth/sensorthings-action@v0.1.0 - name: Install sqlite and gpkg dependencies - uses: awalsh128/cache-apt-pkgs-action@latest + uses: awalsh128/cache-apt-pkgs-action@v1.4.3 with: packages: libsqlite3-mod-spatialite version: 4.3.0a-6build1 + - name: Use ubuntuGIS unstable ppa + run: sudo add-apt-repository ppa:ubuntugis/ubuntugis-unstable && sudo apt update + shell: bash - name: Install GDAL with Python bindings - uses: awalsh128/cache-apt-pkgs-action@latest + uses: awalsh128/cache-apt-pkgs-action@v1.4.3 with: packages: gdal-bin libgdal-dev - version: 3.0.4 + version: 3.8.4 - name: Install and run Oracle run: | docker run -d --name oracledb -e ORACLE_PWD=oracle -v ${{ github.workspace }}/tests/data/oracle/init-db:/opt/oracle/scripts/startup -p 1521:1521 container-registry.oracle.com/database/express:21.3.0-xe @@ -98,37 +97,38 @@ jobs: pip3 install -r requirements-starlette.txt pip3 install -r requirements-dev.txt pip3 install -r requirements-provider.txt + pip3 install -r requirements-manager.txt pip3 install -r requirements-django.txt - pip3 install -r docs/requirements.txt python3 setup.py install - pip3 install --upgrade numpy elasticsearch - pip3 install --upgrade numpy "sqlalchemy<2" - pip3 install --upgrade flake8 pip3 install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version` #pip3 install --upgrade rasterio==1.1.8 - name: setup test data ⚙️ run: | python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid - python3 tests/load_es_data.py tests/cite/ogcapi-features/canada-hydat-daily-mean-02HC003.geojson IDENTIFIER + python3 tests/load_opensearch_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid python3 tests/load_mongo_data.py tests/data/ne_110m_populated_places_simple.geojson gunzip < tests/data/hotosm_bdi_waterways.sql.gz | psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/dummy_data.sql + psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/dummy_types_data.sql + psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/postgres_manager_full_structure.backup.sql docker ps python3 tests/load_oracle_data.py - name: run unit tests ⚙️ env: POSTGRESQL_PASSWORD: ${{ secrets.DatabasePassword || 'postgres' }} run: | - pytest tests/test_api.py + pytest tests/api pytest tests/test_api_ogr_provider.py pytest tests/test_config.py pytest tests/test_csv__formatter.py pytest tests/test_csv__provider.py pytest tests/test_django.py pytest tests/test_elasticsearch__provider.py + pytest tests/test_opensearch__provider.py pytest tests/test_esri_provider.py pytest tests/test_filesystem_provider.py pytest tests/test_geojson_provider.py + pytest tests/test_linked_data.py pytest tests/test_mongo_provider.py pytest tests/test_ogr_csv_provider.py pytest tests/test_ogr_esrijson_provider.py @@ -136,60 +136,23 @@ jobs: pytest tests/test_ogr_shapefile_provider.py pytest tests/test_ogr_sqlite_provider.py pytest tests/test_ogr_wfs_provider.py + pytest tests/test_postgresql_manager.py + # pytest tests/test_ogr_wfs_provider_live.py # NOTE: these are skipped in the file but listed here for completeness pytest tests/test_openapi.py pytest tests/test_oracle_provider.py + pytest tests/test_parquet_provider.py pytest tests/test_postgresql_provider.py pytest tests/test_rasterio_provider.py + pytest tests/test_sensorthings_edr_provider.py pytest tests/test_sensorthings_provider.py pytest tests/test_socrata_provider.py + # pytest tests/test_socrata_provider_live.py.py # NOTE: these are skipped in the file but listed here for completeness pytest tests/test_sqlite_geopackage_provider.py pytest tests/test_tinydb_catalogue_provider.py pytest tests/test_tinydb_manager_for_parallel_requests.py pytest tests/test_util.py pytest tests/test_xarray_netcdf_provider.py pytest tests/test_xarray_zarr_provider.py - - name: build docs 🏗️ - run: cd docs && make html - - name: failed tests 🚩 - if: ${{ failure() }} - run: | - pip3 list -v - - admin: - needs: [flake8_py3] - runs-on: ubuntu-20.04 - strategy: - matrix: - include: - - python-version: 3.8 - env: - PYGEOAPI_CONFIG: "tests/pygeoapi-test-config-admin.yml" - PYGEOAPI_OPENAPI: "tests/pygeoapi-test-openapi-admin.yml" - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - name: Setup Python ${{ matrix.python-version }} - with: - python-version: ${{ matrix.python-version }} - - uses: awalsh128/cache-apt-pkgs-action@latest - with: - packages: gunicorn python3-gevent - version: 1.0 - - name: Install requirements 📦 - run: | - pip3 install -r requirements.txt - pip3 install -r requirements-dev.txt - pip3 install -r requirements-admin.txt - python3 setup.py install - - name: Run pygeoapi with admin API ⚙️ - run: | - pygeoapi openapi generate ${PYGEOAPI_CONFIG} --output-file ${PYGEOAPI_OPENAPI} - gunicorn --bind 0.0.0.0:5000 \ - --reload \ - --reload-extra-file ${PYGEOAPI_CONFIG} \ - pygeoapi.flask_app:APP & - - name: run integration tests ⚙️ - run: | pytest tests/test_admin_api.py - name: failed tests 🚩 if: ${{ failure() }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 000000000..61629fb5d --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,48 @@ +name: 'Close stale Issues and Pull Requests' + +env: + STALE_AFTER_INACTIVE_DAYS: 90 + CLOSE_AFTER_INACTIVE_DAYS: 7 + +on: + schedule: + - cron: '1 3 * * 0' # runs every Sunday at 03h01 UTC + # - cron: '0 * * * *' # runs every hour, for debugging +jobs: + stale: + permissions: + issues: write + pull-requests: write + if: github.repository == 'geopython/pygeoapi' + runs-on: ubuntu-latest + steps: + - uses: 'actions/stale@v9' + with: + # debug-only: true + operations-per-run: 1000 + enable-statistics: true + stale-issue-label: stale + stale-pr-label: stale + exempt-issue-labels: blocker + exempt-pr-labels: blocker + days-before-stale: ${{ env.STALE_AFTER_INACTIVE_DAYS }} + days-before-close: ${{ env.CLOSE_AFTER_INACTIVE_DAYS }} + remove-stale-when-updated: true + stale-issue-message: > + This Issue has been inactive for ${{env.STALE_AFTER_INACTIVE_DAYS }} + days. As per [RFC4](https://pygeoapi.io/development/rfc/4), in order + to manage maintenance burden, it will be automatically closed + in ${{ env.CLOSE_AFTER_INACTIVE_DAYS }} days. + stale-pr-message: > + This Pull Request has been inactive for ${{env.STALE_AFTER_INACTIVE_DAYS }} + days. As per [RFC4](https://pygeoapi.io/development/rfc/4), in order + to manage maintenance burden, it will be automatically closed + in ${{ env.CLOSE_AFTER_INACTIVE_DAYS }} days. + close-issue-message: > + As per [RFC4](https://pygeoapi.io/development/rfc/4), this Issue has + been closed due to there being no activity for more + than ${{ env.STALE_AFTER_INACTIVE_DAYS }} days. + close-pr-message: > + As per [RFC4](https://pygeoapi.io/development/rfc/4), this Pull Request + has been closed due to there being no activity for more + than ${{ env.STALE_AFTER_INACTIVE_DAYS }} days. diff --git a/.github/workflows/vulnerabilities.yml b/.github/workflows/vulnerabilities.yml index 3252252ed..d8ac5199c 100644 --- a/.github/workflows/vulnerabilities.yml +++ b/.github/workflows/vulnerabilities.yml @@ -22,7 +22,7 @@ jobs: working-directory: . steps: - name: Checkout pygeoapi - uses: actions/checkout@v4 + uses: actions/checkout@master - name: Scan vulnerabilities with trivy uses: aquasecurity/trivy-action@master with: @@ -37,6 +37,9 @@ jobs: docker buildx build -t ${{ github.repository }}:${{ github.sha }} --platform linux/amd64 --no-cache -f Dockerfile . - name: Scan locally built Docker image for vulnerabilities with trivy uses: aquasecurity/trivy-action@master + env: + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2 + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1 with: scan-type: image exit-code: 1 diff --git a/.gitignore b/.gitignore index dfb7c6861..2829d7060 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,10 @@ ENV/ # pygeoapi artifacts *.openapi.yml +# development setup examples +example-config.yml +example-openapi.yml + # misc *.swp .pytest_cache diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 3eab1cefe..350ac42d8 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -16,3 +16,6 @@ python: - requirements: docs/requirements.txt # - requirements: requirements.txt # - requirements: requirements-dev.txt + +formats: + - pdf diff --git a/Dockerfile b/Dockerfile index 1d8d6a296..04f1a20db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ # Copyright (c) 2020 Tom Kralidis # Copyright (c) 2019 Just van den Broecke # Copyright (c) 2020 Francesco Bartoli -# Copyright (c) 2021 Angelos Tzotsos +# Copyright (c) 2024 Angelos Tzotsos # Copyright (c) 2023 Bernhard Mallinger # # Permission is hereby granted, free of charge, to any person @@ -34,7 +34,7 @@ # # ================================================================= -FROM ubuntu:jammy-20231211.1 +FROM ubuntu:jammy-20240911.1 LABEL maintainer="Just van den Broecke " @@ -86,6 +86,7 @@ ENV TZ=${TZ} \ LANG=${LANG} \ DEBIAN_FRONTEND="noninteractive" \ DEB_BUILD_DEPS="\ + software-properties-common \ curl \ unzip" \ DEB_PACKAGES="\ @@ -97,7 +98,6 @@ ENV TZ=${TZ} \ python3-greenlet \ python3-pip \ python3-tz \ - python3-unicodecsv \ python3-yaml \ ${ADD_DEB_PACKAGES}" @@ -106,19 +106,16 @@ WORKDIR /pygeoapi # Install operating system dependencies RUN \ apt-get update -y \ - && apt-get --no-install-recommends install -y ${DEB_PACKAGES} ${DEB_BUILD_DEPS} \ + && apt-get install -y ${DEB_BUILD_DEPS} \ + && add-apt-repository ppa:ubuntugis/ubuntugis-unstable \ + && apt-get --no-install-recommends install -y ${DEB_PACKAGES} \ && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ && echo "For ${TZ} date=$(date)" && echo "Locale=$(locale)" \ - - # temporary remove - # && add-apt-repository ppa:ubuntugis/ubuntugis-unstable \ - # OGC schemas local setup && mkdir /schemas.opengis.net \ && curl -O http://schemas.opengis.net/SCHEMAS_OPENGIS_NET.zip \ && unzip ./SCHEMAS_OPENGIS_NET.zip "ogcapi/*" -d /schemas.opengis.net \ && rm -f ./SCHEMAS_OPENGIS_NET.zip \ - # Cleanup TODO: remove unused Locales and TZs # NOTE: this tries to remove gcc, but the actual package gcc-11 can't be # removed because python3-scipy depends on python3-pythran which diff --git a/LICENSE.md b/LICENSE.md index 2bbcdeb6f..02edb4e5f 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ # The MIT License (MIT) -Copyright © 2018-2024 pygeoapi development team +Copyright © 2018-2025 pygeoapi development team * * * diff --git a/MANIFEST.in b/MANIFEST.in index ae0c43a1c..764a905e6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include README.md LICENSE.md requirements.txt -recursive-include pygeoapi *.html *.json *.yml +recursive-include pygeoapi *.html *.json *.jsonld *.yml recursive-include pygeoapi/static * diff --git a/docker/default.config.yml b/docker/default.config.yml index 2ac3ebae1..d75298f9f 100644 --- a/docker/default.config.yml +++ b/docker/default.config.yml @@ -38,18 +38,21 @@ server: bind: host: 0.0.0.0 port: 80 - url: http://localhost:5000 + url: ${PYGEOAPI_SERVER_URL:-http://localhost:5000} mimetype: application/json; charset=UTF-8 encoding: utf-8 gzip: false language: en-US cors: true pretty_print: true - limit: 10 + admin: ${PYGEOAPI_SERVER_ADMIN:-false} + limits: + default_items: 10 + max_items: 50 # templates: /path/to/templates map: - url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png - attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' ogc_schemas_location: /schemas.opengis.net logging: @@ -200,7 +203,7 @@ resources: hreflang: nl-NL extents: spatial: - bbox: [50.7539, 7.21097, 53.4658, 3.37087] + bbox: [3.37,50.75,7.21,53.47] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 temporal: begin: @@ -301,7 +304,7 @@ resources: hreflang: en-US extents: spatial: - bbox: [36.0, 17.0, 46.0, 18.0] + bbox: [5.0,36.0,20.0,46.0] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 temporal: begin: @@ -342,7 +345,7 @@ resources: keywords: - Portugal - POI - - Point of Interrest + - Point of Interest - Madeira - Azores - OSM @@ -438,7 +441,7 @@ resources: hreflang: nl-NL extents: spatial: - bbox: [50.7539, 7.21097, 53.4658, 3.37087] + bbox: [3.37,50.75,7.21,53.47] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 temporal: begin: @@ -481,7 +484,7 @@ resources: hreflang: nl-NL extents: spatial: - bbox: [50.7539, 7.21097, 53.4658, 3.37087] + bbox: [3.37,50.75,7.21,53.47] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 temporal: begin: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index b81bed89f..baaa67aa8 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -2,8 +2,10 @@ # ================================================================= # # Authors: Just van den Broecke +# Benjamin Webb # # Copyright (c) 2019 Just van den Broecke +# Copyright (c) 2024 Benjamin Webb # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -43,6 +45,7 @@ SCRIPT_NAME=${SCRIPT_NAME:=/} CONTAINER_NAME=${CONTAINER_NAME:=pygeoapi} CONTAINER_HOST=${CONTAINER_HOST:=0.0.0.0} CONTAINER_PORT=${CONTAINER_PORT:=80} +WSGI_APP=${WSGI_APP:=pygeoapi.flask_app:APP} WSGI_WORKERS=${WSGI_WORKERS:=4} WSGI_WORKER_TIMEOUT=${WSGI_WORKER_TIMEOUT:=6000} WSGI_WORKER_CLASS=${WSGI_WORKER_CLASS:=gevent} @@ -66,12 +69,26 @@ pygeoapi openapi generate ${PYGEOAPI_CONFIG} --output-file ${PYGEOAPI_OPENAPI} echo "openapi.yml generated continue to pygeoapi" +start_gunicorn() { + # SCRIPT_NAME should not have value '/' + [[ "${SCRIPT_NAME}" = '/' ]] && export SCRIPT_NAME="" && echo "make SCRIPT_NAME empty from /" + + echo "Starting gunicorn name=${CONTAINER_NAME} on ${CONTAINER_HOST}:${CONTAINER_PORT} with ${WSGI_WORKERS} workers and SCRIPT_NAME=${SCRIPT_NAME}" + exec gunicorn --workers ${WSGI_WORKERS} \ + --worker-class=${WSGI_WORKER_CLASS} \ + --timeout ${WSGI_WORKER_TIMEOUT} \ + --name=${CONTAINER_NAME} \ + --bind ${CONTAINER_HOST}:${CONTAINER_PORT} \ + ${@} \ + ${WSGI_APP} +} + case ${entry_cmd} in # Run Unit tests test) for test_py in $(ls tests/test_*.py) do - # Skip tests requireing backend server or libs installed + # Skip tests requiring backend server or libs installed case ${test_py} in tests/test_elasticsearch__provider.py) ;& @@ -91,19 +108,21 @@ case ${entry_cmd} in # Run pygeoapi server run) - # SCRIPT_NAME should not have value '/' - [[ "${SCRIPT_NAME}" = '/' ]] && export SCRIPT_NAME="" && echo "make SCRIPT_NAME empty from /" - - echo "Start gunicorn name=${CONTAINER_NAME} on ${CONTAINER_HOST}:${CONTAINER_PORT} with ${WSGI_WORKERS} workers and SCRIPT_NAME=${SCRIPT_NAME}" - exec gunicorn --workers ${WSGI_WORKERS} \ - --worker-class=${WSGI_WORKER_CLASS} \ - --timeout ${WSGI_WORKER_TIMEOUT} \ - --name=${CONTAINER_NAME} \ - --bind ${CONTAINER_HOST}:${CONTAINER_PORT} \ - pygeoapi.flask_app:APP - ;; + # Start + start_gunicorn + ;; + + # Run pygeoapi server with hot reload + run-with-hot-reload) + # Lock all Python files (for gunicorn hot reload) + find . -type f -name "*.py" | xargs chmod 0444 + + # Start with hot reload options + start_gunicorn --reload --reload-extra-file ${PYGEOAPI_CONFIG} + ;; + *) - error "unknown command arg: must be run (default) or test" + error "unknown command arg: must be run (default), run-with-hot-reload, or test" ;; esac diff --git a/docs/Makefile b/docs/Makefile index 3bf255e6f..6542e719b 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,4 +17,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -W diff --git a/docs/source/_static/openapi_admin.png b/docs/source/_static/openapi_admin.png new file mode 100644 index 000000000..8fd9af443 Binary files /dev/null and b/docs/source/_static/openapi_admin.png differ diff --git a/docs/source/_templates/indexsidebar.html b/docs/source/_templates/indexsidebar.html index 8000f32dd..a782c3ba6 100644 --- a/docs/source/_templates/indexsidebar.html +++ b/docs/source/_templates/indexsidebar.html @@ -16,11 +16,14 @@ OGC Reference Implementation + + OGC Reference Implementation + OSGeo Project - - FOSS4G Conference + + FOSS4G Conference

diff --git a/docs/source/admin-api.rst b/docs/source/admin-api.rst index 9a4efca56..19ce12b9a 100644 --- a/docs/source/admin-api.rst +++ b/docs/source/admin-api.rst @@ -14,11 +14,20 @@ The API is enabled with the following server configuration: server: admin: true # boolean on whether to enable Admin API. +.. note:: + + If you generate the OpenAPI definition after enabling the admin API, the admin routes will be exposed on ``/openapi`` + + .. image:: /_static/openapi_admin.png + :alt: admin routes + :align: center + + Access control -------------- It should be made clear that authentication and authorization is beyond the responsibility of pygeoapi. This means that -if a pygeoapi user enables the Admin API, they must provide access control explicity via another service. +if a pygeoapi user enables the Admin API, they must provide access control explicitly via another service. pygeoapi hot reloading in gunicorn ---------------------------------- diff --git a/docs/source/administration.rst b/docs/source/administration.rst index f07413b4a..fcebd291b 100644 --- a/docs/source/administration.rst +++ b/docs/source/administration.rst @@ -49,6 +49,10 @@ To generate the OpenAPI document as JSON, run: property names and their data types. Whenever you make changes to your pygeoapi configuration, always refresh the accompanying OpenAPI document. +.. note:: + By default, pygeoapi OpenAPI document generation will fail on any collection whose provider is improperly + configured or when encountering a connection issue. To skip failing collections, use the + --no-fail-on-invalid-collection flag. The OpenAPI document will then be generated without the failing collection(s) .. seealso:: :ref:`openapi` for more information on pygeoapi's OpenAPI support diff --git a/docs/source/api-documentation.rst b/docs/source/api-documentation.rst index d4cc2bffb..5c81602e2 100644 --- a/docs/source/api-documentation.rst +++ b/docs/source/api-documentation.rst @@ -5,200 +5,22 @@ API documentation Top level code documentation. Follow the links in each section for module/class member information. -API ---- - .. automodule:: pygeoapi.api :members: :private-members: :special-members: - -flask_app ---------- - -.. automodule:: pygeoapi.flask_app - :show-inheritance: - :members: - :private-members: - :special-members: - - -Logging -------- - -.. automodule:: pygeoapi.log - :show-inheritance: - :members: - :private-members: - :special-members: - - -OpenAPI -------- - -.. automodule:: pygeoapi.openapi - :show-inheritance: - :members: - :private-members: - :special-members: - - -Plugins -------- - -.. seealso:: - :ref:`plugins` - -.. automodule:: pygeoapi.plugin - :show-inheritance: - :members: - :private-members: - :special-members: - - -Utils ------ - -.. automodule:: pygeoapi.util - :show-inheritance: - :members: - :private-members: - :special-members: - - -Formatter package ------------------ - -.. automodule:: pygeoapi.formatter - :show-inheritance: - :members: - :private-members: - :special-members: - - -Base class -^^^^^^^^^^ - -.. automodule:: pygeoapi.formatter.base - :show-inheritance: - :members: - :private-members: - :special-members: - - -csv -^^^ - -.. automodule:: pygeoapi.formatter.csv_ - :show-inheritance: - :members: - :private-members: - :special-members: - - -Process package ---------------- - -.. automodule:: pygeoapi.process - :show-inheritance: - :members: - :private-members: - :special-members: - - -Base class -^^^^^^^^^^ - .. automodule:: pygeoapi.process.base - :show-inheritance: :members: :private-members: :special-members: - -hello_world -^^^^^^^^^^^ - -Hello world example process - -.. automodule:: pygeoapi.process.hello_world - :show-inheritance: +.. automodule:: pygeoapi.process.manager.base :members: :private-members: :special-members: -.. _data Provider: - -Provider --------- - -.. automodule:: pygeoapi.provider - :show-inheritance: - :members: - :private-members: - :special-members: - - -Base class -^^^^^^^^^^ - -.. automodule:: pygeoapi.provider.base - :show-inheritance: +.. automodule:: pygeoapi.formatter.base :members: :private-members: :special-members: - - -CSV provider -^^^^^^^^^^^^ - -.. automodule:: pygeoapi.provider.csv_ - :show-inheritance: - :members: - :private-members: - - -Elasticsearch provider -^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: pygeoapi.provider.elasticsearch_ - :show-inheritance: - :members: - :private-members: - - -GeoJSON -^^^^^^^ - -.. automodule:: pygeoapi.provider.geojson - :show-inheritance: - :members: - :private-members: - - -OGR -^^^ - -.. automodule:: pygeoapi.provider.ogr - :show-inheritance: - :members: - :private-members: - - -postgresql -^^^^^^^^^^ - -.. automodule:: pygeoapi.provider.postgresql - :show-inheritance: - :members: - :private-members: - - -sqlite/geopackage -^^^^^^^^^^^^^^^^^ - -.. automodule:: pygeoapi.provider.sqlite - :show-inheritance: - :members: - :private-members: diff --git a/docs/source/conf.py b/docs/source/conf.py index ff6633c4b..62af93639 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -2,7 +2,8 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2025 Tom Kralidis +# Copyright (c) 2025 Angelos Tzotsos # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -102,7 +103,7 @@ def __getattr__(cls, name): project = 'pygeoapi' author = 'pygeoapi development team' license = 'This work is licensed under a Creative Commons Attribution 4.0 International License' # noqa -copyright = '2018-2024, ' + author + ' ' + license +copyright = '2018-2025, ' + author + ' ' + license today_fmt = '%Y-%m-%d' @@ -111,7 +112,7 @@ def __getattr__(cls, name): # built documents. # # The short X.Y version. -version = '0.16.dev0' +version = '0.20.0' # The full version, including alpha/beta/rc tags. release = version diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index b9422d828..2ee11b55e 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -49,16 +49,26 @@ For more information related to API design rules (the ``api_rules`` property in gzip: false # default server config to gzip/compress responses to requests with gzip in the Accept-Encoding header cors: true # boolean on whether server should support CORS pretty_print: true # whether JSON responses should be pretty-printed - limit: 10 # server limit on number of items to return + + limits: # server limits on number of items to return. This property can also be defined at the resource level to override global server settings + default_items: 50 + max_items: 1000 + max_distance_x: 25 + max_distance_y: 25 + max_distance_units: m + on_exceed: throttle # throttle or error (default=throttle) + admin: false # whether to enable the Admin API + # optional configuration to specify a different set of templates for HTML pages. Recommend using absolute paths. Omit this to use the default provided templates + # This property can also be defined at the resource level to override global server settings for specific datasets templates: # optional configuration to specify a different set of templates for HTML pages. Recommend using absolute paths. Omit this to use the default provided templates path: /path/to/jinja2/templates/folder # path to templates folder containing the Jinja2 template HTML files static: /path/to/static/folder # path to static folder containing css, js, images and other static files referenced by the template map: # leaflet map setup for HTML pages - url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png - attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + url: https://tile.openstreetmap.org/{z}/{x}/{y}.png + attribution: '© OpenStreetMap contributors' ogc_schemas_location: /opt/schemas.opengis.net # local copy of https://schemas.opengis.net manager: # optional OGC API - Processes asynchronous job management @@ -83,8 +93,8 @@ The ``logging`` section provides directives for logging messages which are usefu logging: level: ERROR # the logging level (see https://docs.python.org/3/library/logging.html#logging-levels) logfile: /path/to/pygeoapi.log # the full file path to the logfile - logformat: # example for miliseconds:'[%(asctime)s.%(msecs)03d] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s' - dateformat: # example for miliseconds:'%Y-%m-%dT%H:%M:%S' + logformat: # example for milliseconds:'[%(asctime)s.%(msecs)03d] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s' + dateformat: # example for milliseconds:'%Y-%m-%dT%H:%M:%S' .. note:: If ``level`` is defined and ``logfile`` is undefined, logging messages are output to the server's ``stdout``. @@ -129,7 +139,7 @@ The ``metadata`` section provides settings for overall service metadata and desc - api keywords_type: theme # keyword type as per the ISO 19115 MD_KeywordTypeCode codelist. Accepted values are discipline, temporal, place, theme, stratum terms_of_service: https://creativecommons.org/licenses/by/4.0/ # terms of service - url: http://example.org # informative URL about the service + url: https://example.org # informative URL about the service license: # licensing details name: CC-BY 4.0 license url: https://creativecommons.org/licenses/by/4.0/ @@ -187,7 +197,6 @@ default. - observations - monitoring linked-data: # linked data configuration (see Linked Data section) - item_template: tests/data/base.jsonld context: - datetime: https://schema.org/DateTime - vocab: https://example.com/vocab# @@ -206,6 +215,7 @@ default. temporal: # optional begin: 2000-10-30T18:24:39Z # start datetime in RFC3339 end: 2007-10-30T08:57:29Z # end datetime in RFC3339 + trs: http://www.opengis.net/def/uom/ISO-8601/0/Gregorian # TRS providers: # list of 1..n required connections information # provider name # see pygeoapi.plugin for supported providers @@ -240,7 +250,7 @@ default. option_name: option_value hello-world: # name of process - type: collection # REQUIRED (collection, process, or stac-collection) + type: process # REQUIRED (collection, process, or stac-collection) processor: name: HelloWorld # Python path of process definition @@ -251,6 +261,41 @@ default. .. seealso:: :ref:`plugins` for more information on plugins +Using environment variables +--------------------------- + +pygeoapi configuration supports using system environment variables, which can be helpful +for deploying into `12 factor `_ environments for example. + +Below is an example of how to integrate system environment variables in pygeoapi. + +.. code-block:: yaml + + server: + bind: + host: ${MY_HOST} + port: ${MY_PORT} + +Multiple environment variables are supported as follows: + +.. code-block:: yaml + + data: ${MY_HOST}:${MY_PORT} + +It is also possible to define a default value for a variable in case it does not exist in +the environment using a syntax like: ``value: ${ENV_VAR:-the default}`` + +.. code-block:: yaml + + server: + bind: + host: ${MY_HOST:-localhost} + port: ${MY_PORT:-5000} + metadata: + identification: + title: + en: This is pygeoapi host ${MY_HOST} and port ${MY_PORT:-5000}, nice to meet you! + Adding links to collections --------------------------- @@ -386,33 +431,6 @@ If omitted, no header will be added. Common names for this header are ``API-Vers Note that pygeoapi already adds a ``X-Powered-By`` header by default that includes the software version number. -Validating the configuration ----------------------------- - -To ensure your configuration is valid, pygeoapi provides a validation -utility that can be run as follows: - -.. code-block:: bash - - pygeoapi config validate -c /path/to/my-pygeoapi-config.yml - - -Using environment variables ---------------------------- - -pygeoapi configuration supports using system environment variables, which can be helpful -for deploying into `12 factor `_ environments for example. - -Below is an example of how to integrate system environment variables in pygeoapi. - -.. code-block:: yaml - - server: - bind: - host: ${MY_HOST} - port: ${MY_PORT} - - Hierarchical collections ------------------------ @@ -484,6 +502,36 @@ Examples: curl https://example.org/collections/lakes/items # only the name attribute is returned in properties curl https://example.org/collections/lakes/items/{item_id} # only the name attribute is returned in properties +Limiting data responses +----------------------- + +pygeoapi defines a ``limits`` configuration parameter that will allow a user to define default and maximum limits for multiple data types. This parameter is defined at the server level (``server.limits``) with the ability to override at resource level (``resources[*].limits``). An example of this setting is shown below: + +.. code-block:: yaml + + limits: + default_items: 10 # applies to vector data + max_items: 500 # applies to vector data + max_distance_x: 123 # applies to all datasets + max_distance_y: 456 # applies to all datasets + max_distance_units: m # as per UCUM https://ucum.org/ucum#section-Tables-of-Terminal-Symbols + on_exceed: error # one of error, throttle + +The ``limits`` setting is applied as follows: + +- can be defined at both the server and resources levels, with resource limits overriding server wide limits settings +- ``on_exceed`` can be set to ``error`` or ``throttle`` (default). If a client specified limit exceeds those set by the server: + - when set to ``error``, an exception is returned + - when set to ``throttle`` the maximum data allowed by the collection/server/provider is returned + +Vector data (features, records) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- when a limit not specified by the client, ``limits.default_items`` can be used to set the result set size +- when a limit is specified by the client, the minimum of the ``limit`` parameter and ``limits.max_items`` is calculated to set the result set size + +Raster data (coverages, environmental data retrieval) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- when a bbox or spatial subset is specified by the client, ``limits.max_distance_x``, ``limits.max_distance_y`` and ``limits.max_distance_units`` are used to determine whether a request has asked for more data than the collection is configured to provide and respond accordingly (via ``on_exceed``) Linked Data ----------- @@ -597,23 +645,30 @@ This relationship can further be maintained in the JSON-LD structured data using ssn: "http://www.w3.org/ns/ssn/" Datastream: sosa:isMemberOf -Sometimes, the JSON-LD desired for an individual feature in a collection is more complicated than can be achieved by -aliasing properties using a context. In this case, it is possible to specify a Jinja2 template. When ``item_template`` -is defined for a feature collection, the json-ld prepared by pygeoapi will be used to render the Jinja2 template -specified by the path. The path specified can be absolute or relative to pygeoapi's template folder. For even more -deployment flexibility, the path can be specified with string interpolation of environment variables. +Sometimes, the JSON-LD desired for an individual feature in a collection is more complicated than can +be achieved by aliasing properties using a context. In this case, it is possible to implement a custom +Jinja2 template. GeoJSON-LD is rendered using the Jinja2 templates defined in ``collections/items/item.jsonld`` +and ``collections/items/index.jsonld``. A pygeoapi collection requiring custom GeoJSON-LD can overwrite these +templates using dataset level templating. To learn more about Jinja2 templates, see :ref:`html-templating`. .. code-block:: yaml linked-data: - item_template: tests/data/base.jsonld context: - datetime: https://schema.org/DateTime -.. note:: - The template ``tests/data/base.jsonld`` renders the unmodified JSON-LD. For more information on the capacities - of Jinja2 templates, see :ref:`html-templating`. + +Validating the configuration +---------------------------- + +To ensure your configuration is valid, pygeoapi provides a validation +utility that can be run as follows: + +.. code-block:: bash + + pygeoapi config validate -c /path/to/my-pygeoapi-config.yml + Summary ------- @@ -625,5 +680,5 @@ At this point, you have the configuration ready to administer the server. .. _`JSON-LD`: https://json-ld.org .. _`Google Structured Data Testing Tool`: https://search.google.com/structured-data/testing-tool#url=https%3A%2F%2Fdemo.pygeoapi.io%2Fmaster .. _`Google Dataset Search`: https://developers.google.com/search/docs/appearance/structured-data/dataset -.. _RotatingFileHandler: http://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler -.. _TimedRotatingFileHandler: http://docs.python.org/3/library/logging.handlers.html#timedrotatingfilehandler +.. _RotatingFileHandler: https://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler +.. _TimedRotatingFileHandler: https://docs.python.org/3/library/logging.handlers.html#timedrotatingfilehandler diff --git a/docs/source/cql.rst b/docs/source/cql.rst index 4ba1c6c14..66a6d6364 100644 --- a/docs/source/cql.rst +++ b/docs/source/cql.rst @@ -3,15 +3,18 @@ CQL support =========== +OGC Common Query Language (`CQL2`_) is a generic language designed to provide enhanced query and subset/filtering to (primarily) feature and record data. + Providers --------- -As of now the available providers supported for CQL filtering are limited to :ref:`Elasticsearch ` and :ref:`PostgreSQL `. - +CQL2 support is implemented in various pygeoapi feature and record providers. See the :ref:`feature ` and :ref:`metadata ` provider sections +for current provider support. + Limitations ----------- -Support of CQL is limited to `Simple CQL filter `_ and thus it allows to query with the +Support of CQL is limited to `Basic CQL2 `_ and thus it allows to query with the following predicates: * comparison predicates @@ -21,20 +24,20 @@ following predicates: Formats ------- -At the moment Elasticsearch supports only the CQL dialect with the JSON encoding `CQL-JSON `_. +Supported providers leverage the CQL2 dialect with the JSON encoding `CQL-JSON `_. -PostgreSQL supports both CQL-JSON and CQL-text dialects, `CQL-JSON `_ and `CQL-TEXT `_ +PostgreSQL supports both `CQL2 JSON `_ and `CQL text `_ dialects. Queries ^^^^^^^ The PostgreSQL provider uses `pygeofilter `_ allowing a range of filter expressions, see examples for: -* `Comparison predicates `_ -* `Spatial predicates `_ -* `Temporal predicates `_ +* `Comparison predicates (`Advanced `_, `Case-insensitive `_) +* `Spatial predicates `_ +* `Temporal predicates `_ -Using Elasticsearch the following type of queries are supported right now: +Using Elasticsearch the following type of queries are supported currently: * ``between`` predicate query * Logical ``and`` query with ``between`` and ``eq`` expression @@ -59,11 +62,11 @@ A ``BETWEEN`` example for a specific property through an HTTP POST request: curl --location --request POST 'http://localhost:5000/collections/nhsl_hazard_threat_all_indicators_s_bc/items?f=json&limit=50&filter-lang=cql-json' \ --header 'Content-Type: application/query-cql-json' \ --data-raw '{ - "between": { - "value": { "property": "properties.MHn_Intensity" }, - "lower": 0.59, - "upper": 0.60 - } + "op": "between", + "args": [ + {"property": "properties.MHn_Intensity"}, + [0.59, 0.60] + ] }' Or @@ -73,11 +76,11 @@ Or curl --location --request POST 'http://localhost:5000/collections/recentearthquakes/items?f=json&limit=10&filter-lang=cql-json' --header 'Content-Type: application/query-cql-json' --data-raw '{ - "between":{ - "value":{"property": "ml"}, - "lower":4, - "upper":4.5 - } + "op": "between", + "args": [ + {"property": "ml"}, + [4, 4.5] + ] }' The same ``BETWEEN`` query using HTTP GET request formatted as CQL text and URL encoded as below: @@ -93,7 +96,11 @@ An ``EQUALS`` example for a specific property: curl --location --request POST 'http://localhost:5000/collections/recentearthquakes/items?f=json&limit=10&filter-lang=cql-json' --header 'Content-Type: application/query-cql-json' --data-raw '{ - "eq":[{"property": "user_entered"},"APBE"] + "op": "=", + "args": [ + {"property": "user_entered"}, + "APBE" + ] }' A ``CROSSES`` example via an HTTP GET request. The CQL text is passed via the ``filter`` parameter. @@ -115,7 +122,6 @@ The same example, but this time providing a geometry in EWKT format: curl "http://localhost:5000/collections/beni/items?filter=DWITHIN(geometry,SRID=3857;POINT(1392921%205145517),100,meters)" - - - Note that the CQL text has been URL encoded. This is required in curl commands but when entering in a browser, plain text can be used e.g. ``CROSSES(foo_geom, LINESTRING(28 -2, 30 -4))``. + +.. _`CQL2`: https://docs.ogc.org/is/21-065r2/21-065r2.html diff --git a/docs/source/data-publishing/ogcapi-coverages.rst b/docs/source/data-publishing/ogcapi-coverages.rst index 0861751cd..8c486ab4e 100644 --- a/docs/source/data-publishing/ogcapi-coverages.rst +++ b/docs/source/data-publishing/ogcapi-coverages.rst @@ -18,7 +18,7 @@ parameters. :header: Provider, properties, subset, bbox, datetime :align: left - :ref:`Rasterio`,✅,✅,✅, + :ref:`Rasterio`,✅,✅,✅,✅ :ref:`Xarray`,✅,✅,✅,✅ @@ -72,9 +72,12 @@ The `Xarray`_ provider plugin reads and extracts `NetCDF`_ and `Zarr`_ data. data: tests/data/coads_sst.nc # optionally specify x/y/time fields, else provider will attempt # to derive automagically - x_field: lat x_field: lon + y_field: lat time_field: time + # optionally specify the coordinate reference system of your dataset + # else pygeoapi assumes it is WGS84 (EPSG:4326). + storage_crs: 4326 format: name: netcdf mimetype: application/x-netcdf @@ -96,37 +99,59 @@ The `Xarray`_ provider plugin reads and extracts `NetCDF`_ and `Zarr`_ data. be sure to provide the full S3 URL. Any parameters required to open the dataset using fsspec can be added to the config file under `options` and `s3`. +.. note:: + When providing a `storage_crs` value in the xarray configuration, specify the + coordinate reference system using any valid input for + `pyproj.CRS.from_user_input`_. + Data access examples -------------------- * list all collections + * http://localhost:5000/collections + * overview of dataset + * http://localhost:5000/collections/foo -* coverage rangetype - * http://localhost:5000/collections/foo/coverage/rangetype -* coverage domainset - * http://localhost:5000/collections/foo/coverage/domainset + +* schema of dataset + + * http://localhost:5000/collections/foo/schema + * coverage access via CoverageJSON (default) + * http://localhost:5000/collections/foo/coverage?f=json + * coverage access via native format (as defined in ``provider.format.name``) + * http://localhost:5000/collections/foo/coverage?f=GRIB + * coverage access with comma-separated properties + * http://localhost:5000/collections/foo/coverage?properties=1,3 + * coverage access with subsetting + * http://localhost:5000/collections/foo/coverage?subset=lat(10:20)&subset=long(10:20) + * coverage with bbox + * http://localhost:5000/collections/foo/coverage?bbox=10,10,20,20 + * coverage with bbox and bbox CRS + * http://localhost:5000/collections/foo/coverage?bbox=-8794239.772668611,5311971.846945471,-8348961.809495518,5621521.486192066&bbox=crs=3857 + .. note:: ``.../coverage`` queries which return an alternative representation to CoverageJSON (which prompt a download) will have the response filename matching the collection name and appropriate file extension (e.g. ``my-dataset.nc``) -.. _`OGC API - Coverages`: https://github.com/opengeospatial/ogcapi-coverages +.. _`OGC API - Coverages`: https://ogcapi.ogc.org/coverages .. _`Rasterio`: https://rasterio.readthedocs.io .. _`Xarray`: https://docs.xarray.dev/en/stable .. _`NetCDF`: https://en.wikipedia.org/wiki/NetCDF .. _`Zarr`: https://zarr.readthedocs.io/en/stable .. _`GDAL raster driver short name`: https://gdal.org/drivers/raster/index.html +.. _`pyproj.CRS.from_user_input`: https://pyproj4.github.io/pyproj/stable/api/crs/coordinate_system.html#pyproj.crs.CoordinateSystem.from_user_input diff --git a/docs/source/data-publishing/ogcapi-edr.rst b/docs/source/data-publishing/ogcapi-edr.rst index 1014b40d3..307e87525 100644 --- a/docs/source/data-publishing/ogcapi-edr.rst +++ b/docs/source/data-publishing/ogcapi-edr.rst @@ -14,13 +14,22 @@ Providers --------- pygeoapi core EDR providers are listed below, along with a matrix of supported query -parameters. +types and query arguments. + +.. csv-table:: + :header: Provider, Position, Radius, Area, Cube, Trajectory, Corridor, Items, Locations, Instances + :align: left + + `xarray-edr`_,✅,❌,❌,✅,❌,❌,❌,❌,❌ + `SensorThingsEDR`_,❌,❌,✅,✅,❌,❌,✅,✅,❌ + .. csv-table:: :header: Provider, coords, parameter-name, datetime :align: left `xarray-edr`_,✅,✅,✅ + `SensorThingsEDR`_,✅,✅,✅ Below are specific connection examples based on supported providers. @@ -44,9 +53,12 @@ The `xarray-edr`_ provider plugin reads and extracts `NetCDF`_ and `Zarr`_ data data: tests/data/coads_sst.nc # optionally specify x/y/time fields, else provider will attempt # to derive automagically - x_field: lat x_field: lon + y_field: lat time_field: time + # optionally specify the coordinate reference system of your dataset + # else pygeoapi assumes it is WGS84 (EPSG:4326). + storage_crs: 4326 format: name: netcdf mimetype: application/x-netcdf @@ -81,25 +93,58 @@ The `xarray-edr`_ provider plugin reads and extracts `NetCDF`_ and `Zarr`_ data S3 URL. Any parameters required to open the dataset using fsspec can be added to the config file under `options` and `s3`, as shown above. +.. note:: + When providing a `storage_crs` value in the EDR configuration, specify the + coordinate reference system using any valid input for + `pyproj.CRS.from_user_input`_. + + +SensorThingsEDR +^^^^^^^^^^^^^^^ + +The SensorThings API EDR Provider for pygeaopi extends the feature provider to +produce CoverageJSON representations from SensorThings responses repsonses. This provider +relies on using the ObservedProperty Entity to create the `parameter-name` set. + +.. code-block:: yaml + + providers: + - type: edr + name: SensorThingsEDR + data: https://emotional.byteroad.net/FROST-Server/v1.1/ + - type: feature + name: SensorThings + data: https://emotional.byteroad.net/FROST-Server/v1.1/Things + title_field: name + + +.. note:: + The `feature` provider must also be configured to service the `.../items` + EDR query type. + Data access examples -------------------- * list all collections + * http://localhost:5000/collections * overview of dataset + * http://localhost:5000/collections/foo * dataset position query + * http://localhost:5000/collections/foo/position?coords=POINT(-75%2045) * dataset position query for a specific parameter + * http://localhost:5000/collections/foo/position?coords=POINT(-75%2045)¶meter-name=SST * dataset position query for a specific parameter and time step + * http://localhost:5000/collections/foo/position?coords=POINT(-75%2045)¶meter-name=SST&datetime=2000-01-16 .. _`xarray`: https://docs.xarray.dev/en/stable/ .. _`NetCDF`: https://en.wikipedia.org/wiki/NetCDF .. _`Zarr`: https://zarr.readthedocs.io/en/stable - - -.. _`OGC Environmental Data Retrieval (EDR) (API)`: https://github.com/opengeospatial/ogcapi-coverages +.. _`pyproj.CRS.from_user_input`: https://pyproj4.github.io/pyproj/stable/api/crs/coordinate_system.html#pyproj.crs.CoordinateSystem.from_user_input +.. _`OGC Environmental Data Retrieval (EDR) (API)`: https://ogcapi.ogc.org/edr diff --git a/docs/source/data-publishing/ogcapi-features.rst b/docs/source/data-publishing/ogcapi-features.rst index dfbe630c2..702e39ad5 100644 --- a/docs/source/data-publishing/ogcapi-features.rst +++ b/docs/source/data-publishing/ogcapi-features.rst @@ -21,16 +21,19 @@ parameters. `CSV`_,✅/✅,results/hits,❌,❌,❌,✅,❌,❌,✅ `Elasticsearch`_,✅/✅,results/hits,✅,✅,✅,✅,✅,✅,✅ - `ERDDAP Tabledap Service`_,❌/❌,results/hits,✅,✅,❌,❌,❌,❌ + `ERDDAP Tabledap Service`_,❌/❌,results/hits,✅,✅,❌,❌,❌,❌,✅ `ESRI Feature Service`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,✅ `GeoJSON`_,✅/✅,results/hits,❌,❌,❌,✅,❌,❌,✅ `MongoDB`_,✅/❌,results,✅,✅,✅,✅,❌,❌,✅ `OGR`_,✅/❌,results/hits,✅,❌,❌,✅,❌,❌,✅ + `OpenSearch`_,✅/✅,results/hits,✅,✅,✅,✅,✅,✅,✅ `Oracle`_,✅/✅,results/hits,✅,❌,✅,✅,❌,❌,✅ - `PostgreSQL`_,✅/✅,results/hits,✅,✅,✅,✅,✅,❌,✅ + `Parquet`_,✅/✅,results/hits,✅,✅,❌,✅,❌,❌,✅ + `PostgreSQL`_,✅/✅,results/hits,✅,✅,✅,✅,✅,✅,✅ `SQLiteGPKG`_,✅/❌,results/hits,✅,❌,❌,✅,❌,❌,✅ - `SensorThings API`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,✅ + `SensorThings API`_,✅/✅,results/hits,✅,✅,✅,✅,❌,✅,✅ `Socrata`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,✅ + `TinyDB`_,✅/✅,results/hits,✅,✅,✅,✅,❌,✅,✅ .. note:: @@ -70,20 +73,6 @@ definition. - http://www.opengis.net/def/crs/EPSG/0/4326 storage_crs: http://www.opengis.net/def/crs/EPSG/0/28992 - -GeoJSON -^^^^^^^ - -To publish a GeoJSON file, the file must be a valid GeoJSON FeatureCollection. - -.. code-block:: yaml - - providers: - - type: feature - name: GeoJSON - data: tests/data/file.json - id_field: id - .. _Elasticsearch: Elasticsearch @@ -125,15 +114,43 @@ The ES provider also has the support for the CQL queries as indicated in the tab .. seealso:: :ref:`cql` for more details on how to use Common Query Language (CQL) to filter the collection with specific queries. +.. _ERDDAP Tabledap Service: + +ERDDAP Tabledap Service +^^^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + Requires Python package `requests`_ + +To publish from an ERDDAP `Tabledap`_ service, the following are required in your index: + +.. code-block:: yaml + + providers: + - type: feature + name: ERDDAPTabledap + data: http://osmc.noaa.gov/erddap/tabledap/OSMC_Points + id_field: PLATFORM_CODE + time_field: time + options: + filters: "¶meter=\"SLP\"&platform!=\"C-MAN%20WEATHER%20STATIONS\"&platform!=\"TIDE GAUGE STATIONS (GENERIC)\"" + max_age_hours: 12 + +.. note:: + If the ``datetime`` parameter is passed by the client, this overrides the ``options.max_age_hours`` setting. ESRI Feature Service ^^^^^^^^^^^^^^^^^^^^ -To publish an `ESRI Feature Service`_ or `ESRI Map Service`_ specify the URL for the service layer in the ``data`` field. +To publish an ESRI `Feature Service`_ or `Map Service`_ specify the URL for the service layer in the ``data`` field. * ``id_field`` will often be ``OBJECTID``, ``objectid``, or ``FID``. * If the map or feature service is not shared publicly, the ``username`` and ``password`` fields can be set in the - configuration to authenticate into the service. + configuration to authenticate to the service. +* If the map or feature service is self-hosted and not shared publicly, the ``token_service`` and optional ``referer`` fields + can be set in the configuration to authenticate to the service. + +To publish from an ArcGIS online hosted service: .. code-block:: yaml @@ -146,7 +163,64 @@ To publish an `ESRI Feature Service`_ or `ESRI Map Service`_ specify the URL for crs: 4326 # Optional crs (default is EPSG:4326) username: username # Optional ArcGIS username password: password # Optional ArcGIS password + token_service: https://your.server.com/arcgis/sharing/rest/generateToken # optional URL to your generateToken service + referer: https://your.server.com # optional referer, defaults to https://www.arcgis.com if not set + +To publish from a self-hosted service that is not publicly accessible, the ``token_service`` field is required: + +.. code-block:: yaml + + providers: + - type: feature + name: ESRI + data: https://your.server.com/arcgis/rest/services/your-layer/MapServer/0 + id_field: objectid + time_field: date_in_your_device_time_zone # Optional time field + crs: 4326 # Optional crs (default is EPSG:4326) + username: username # Optional ArcGIS username + password: password # Optional ArcGIS password + token_service: https://your.server.com/arcgis/sharing/rest/generateToken # Optional url to your generateToken service + referer: https://your.server.com # Optional referer, defaults to https://www.arcgis.com if not set + +GeoJSON +^^^^^^^ + +To publish a GeoJSON file, the file must be a valid GeoJSON FeatureCollection. + +.. code-block:: yaml + + providers: + - type: feature + name: GeoJSON + data: tests/data/file.json + id_field: id + +MongoDB +^^^^^^^ + +.. note:: + Requires Python package pymongo + +.. note:: + Mongo 5 or greater is supported. + +MongoDB (`website `_) is a powerful and versatile NoSQL database that provides numerous advantages, making it a preferred choice for many applications. One of the main reasons to use MongoDB is its ability to handle large volumes of unstructured data, making it ideal for managing diverse data types such as text, geospatial, and multimedia data. Additionally, MongoDB's flexible document model allows for easy schema evolution, enabling developers to iterate quickly and adapt to changing requirements. + +`MongoDB GeoJSON `_ support is available, thus a GeoJSON file can be added to MongoDB using following command +`mongoimport --db test -c points --file "path/to/file.geojson" --jsonArray` + +Here `test` is the name of database , `points` is the target collection name. + +* each document must be a GeoJSON Feature, with a valid geometry. + +.. code-block:: yaml + + providers: + - type: feature + name: MongoDB + data: mongodb://localhost:27017/testdb + collection: testplaces OGR ^^^ @@ -249,33 +323,43 @@ The OGR provider requires a recent (3+) version of GDAL to be installed. The `crs` query parameter is used as follows: e.g. ``http://localhost:5000/collections/foo/items?crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992``. +.. _OpenSearch: -MongoDB -^^^^^^^ +OpenSearch +^^^^^^^^^^ .. note:: - Requires Python package pymongo + Requires Python package opensearch-py -.. note:: - Mongo 5 or greater is supported. +To publish an OpenSearch index, the following are required in your index: -`MongoDB `_ is a powerful and versatile NoSQL database that provides numerous advantages, making it a preferred choice for many applications. One of the main reasons to use MongoDB is its ability to handle large volumes of unstructured data, making it ideal for managing diverse data types such as text, geospatial, and multimedia data. Additionally, MongoDB's flexible document model allows for easy schema evolution, enabling developers to iterate quickly and adapt to changing requirements. +* indexes must be documents of valid GeoJSON Features +* index mappings must define the GeoJSON ``geometry`` as a ``geo_shape`` -`GeoJSON `_ support is available officially by MongoDB , thus a GeoJSON file can be added to MongoDB using following command +.. code-block:: yaml -`mongoimport --db test -c points --file "path/to/file.geojson" --jsonArray` + providers: + - type: feature + name: OpenSearch + editable: true|false # optional, default is false + data: http://localhost:9200/ne_110m_populated_places_simple + id_field: geonameid + time_field: datetimefield -Here `test` is the name of database , `points` is the target collection name. +.. note:: -* each document must be a GeoJSON Feature, with a valid geometry. + For OpenSearch indexes that are password protect, a RFC1738 URL can be used as follows: -.. code-block:: yaml + ``data: http://username:password@localhost:9200/ne_110m_populated_places_simple`` - providers: - - type: feature - name: MongoDB - data: mongodb://localhost:27017/testdb - collection: testplaces + To further conceal authentication credentials, environment variables can be used: + + ``data: http://${MY_USERNAME}:${MY_PASSWORD}@localhost:9200/ne_110m_populated_places_simple`` + +The OpenSearch provider also has the support for the CQL queries as indicated in the table above. + +.. seealso:: + :ref:`cql` for more details on how to use Common Query Language (CQL) to filter the collection with specific queries. .. _Oracle: @@ -357,19 +441,92 @@ Mandatory properties table: lakes geom_field: geometry title_field: name - manadory_properties: + mandatory_properties: - example_group_id On large tables it could be useful to disallow a query on the complete dataset. For this reason it is possible to -configure mandatory properties. When this is activated, the provoder throws an exception when the parameter +configure mandatory properties. When this is activated, the provider throws an exception when the parameter is not in the query uri. +Extra properties +"""""""""""""""" +.. code-block:: yaml + + providers: + - type: feature + name: OracleDB + data: + host: 127.0.0.1 + port: 1521 + service_name: XEPDB1 + user: geo_test + password: geo_test + id_field: id + table: lakes + geom_field: geometry + title_field: name + extra_properties: + - "'Here we have ' || name AS tooltip" + +Extra properties is a list of strings which are added as fields for data retrieval in the SELECT clauses. They +can be used to return expressions computed by the database. + +Session Pooling +""""""""""""""" + +Configured using environment variables. + +.. code-block:: bash + + export ORACLE_POOL_MIN=2 + export ORACLE_POOL_MAX=10 + + +The ``ORACLE_POOL_MIN`` and ``ORACLE_POOL_MAX`` environment variables are used to trigger session pool creation in the Oracle Provider and the ``DatabaseConnection`` class. Supports auth via user + password or wallet. For an example of the configuration see above at Oracle - Connection. See https://python-oracledb.readthedocs.io/en/latest/api_manual/module.html#oracledb.create_pool for documentation of the ``create_pool`` function. + +If none or only one of the environment variables is set, session pooling will not be activated and standalone connections are established at every request. + + Custom SQL Manipulator Plugin """"""""""""""""""""""""""""" The provider supports a SQL-Manipulator-Plugin class. With this, the SQL statement could be manipulated. This is useful e.g. for authorization at row level or manipulation of the explain plan with hints. -An example an more informations about that feature you can find in the test class in tests/test_oracle_provider.py. +An example an more information about that feature you can find in the test class in tests/test_oracle_provider.py. + +.. _Parquet: + +Parquet +^^^^^^^ + +.. note:: + Requires Python package pyarrow + +To publish a GeoParquet file (with a geometry column) the geopandas package is also required. + +.. note:: + Reading data directly from a public s3 bucket is also supported. + +.. code-block:: yaml + + providers: + - type: feature + name: Parquet + data: + source: ./tests/data/parquet/random.parquet + id_field: id + time_field: time + x_field: + - minlon + - maxlon + y_field: + - minlat + - maxlat + +For GeoParquet data, the `x_field` and `y_field` must be specified in the provider definition, +and they must be arrays of two column names that contain the x and y coordinates of the +bounding box of each geometry. If the geometries in the data are all points, the `x_field` and `y_field` +can be strings instead of arrays and refer to a single column each. .. _PostgreSQL: @@ -464,7 +621,7 @@ SQLiteGPKG ^^^^^^^^^^ .. note:: - Requries Spatialite installation + Requires Spatialite installation SQLite file: @@ -510,6 +667,10 @@ If ``intralink`` is true for an adjacent STA provider collection within a pygeoapi instance, the expanded entity is instead represented by an intra-pygeoapi link to the other entity or it's ``uri_field`` if declared. +Additionally there is the optional field ``expand``. This field will overwrite the default +pygeoapi expand behavior and instead implement the configured expand strategy. This is +particularly useful if you have Datastreams with many observations. + .. code-block:: yaml providers: @@ -520,6 +681,7 @@ link to the other entity or it's ``uri_field`` if declared. entity: Datastreams time_field: phenomenonTime intralink: true + expand: Thing/Locations,Observations($select=result,phenomenonTime;$orderby=phenomenonTime desc;$top=1) If all three entities are configured, the STA provider will represent a complete STA endpoint as OGC-API feature collections. The ``Things`` features will include links @@ -554,31 +716,25 @@ To publish a `Socrata Open Data API (SODA)`_ endpoint, pygeoapi heavily relies o token: my_token # Optional app token -.. _ERDDAP Tabledap Service: - -ERDDAP Tabledap Service -^^^^^^^^^^^^^^^^^^^^^^^ +TinyDB +^^^^^^ .. note:: - Requires Python package `requests`_ + Requires Python package tinydb -To publish from an ERDDAP `Tabledap`_ service, the following are required in your index: +To publish a TinyDB (`see website `_) index, the following are required in your index: + +* indexes must be documents of valid GeoJSON Features .. code-block:: yaml providers: - type: feature - name: ERDDAPTabledap - data: http://osmc.noaa.gov/erddap/tabledap/OSMC_Points - id_field: PLATFORM_CODE - time_field: time - options: - filters: "¶meter=\"SLP\"&platform!=\"C-MAN%20WEATHER%20STATIONS\"&platform!=\"TIDE GAUGE STATIONS (GENERIC)\"" - max_age_hours: 12 - - -.. note:: - If the ``datetime`` parameter is passed by the client, this overrides the ``options.max_age_hours`` setting. + editable: true|false # optional, default is false + name: TinyDB + data: /path/to/file.db + id_field: identifier + time_field: datetimefield Controlling the order of properties ----------------------------------- @@ -597,36 +753,52 @@ Data access examples -------------------- * list all collections + * http://localhost:5000/collections * overview of dataset + * http://localhost:5000/collections/foo * queryables + * http://localhost:5000/collections/foo/queryables * browse features + * http://localhost:5000/collections/foo/items * paging + * http://localhost:5000/collections/foo/items?offset=10&limit=10 * CSV outputs + * http://localhost:5000/collections/foo/items?f=csv * query features (spatial) + * http://localhost:5000/collections/foo/items?bbox=-180,-90,180,90 * query features (spatial with bbox-crs) + * http://localhost:5000/collections/foo/items?bbox=120000,450000,130000,460000&bbox-crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992 * query features (attribute) + * http://localhost:5000/collections/foo/items?propertyname=foo * query features (temporal) + * http://localhost:5000/collections/foo/items?datetime=2020-04-10T14:11:00Z * query features (temporal) and sort ascending by a property (if no +/- indicated, + is assumed) + * http://localhost:5000/collections/foo/items?datetime=2020-04-10T14:11:00Z&sortby=+datetime * query features (temporal) and sort descending by a property + * http://localhost:5000/collections/foo/items?datetime=2020-04-10T14:11:00Z&sortby=-datetime * query features in a given (and supported) CRS + * http://localhost:5000/collections/foo/items?crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F32633 * query features in a given bounding BBOX and return in given CRS + * http://localhost:5000/collections/foo/items?bbox=120000,450000,130000,460000&bbox-crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F28992&crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F32633 * fetch a specific feature + * http://localhost:5000/collections/foo/items/123 * fetch a specific feature in a given (and supported) CRS + * http://localhost:5000/collections/foo/items/123?crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F32633 .. note:: @@ -643,10 +815,10 @@ Data access examples provider `id_field` values support slashes (i.e. ``my/cool/identifier``). The client request would then be responsible for encoding the identifier accordingly (i.e. ``http://localhost:5000/collections/foo/items/my%2Fcool%2Fidentifier``) -.. _`ESRI Feature Service`: https://enterprise.arcgis.com/en/server/latest/publish-services/windows/what-is-a-feature-service-.htm -.. _`ESRI Map Service`: https://enterprise.arcgis.com/en/server/latest/publish-services/windows/what-is-a-map-service.htm +.. _`Feature Service`: https://enterprise.arcgis.com/en/server/latest/publish-services/windows/what-is-a-feature-service-.htm +.. _`Map Service`: https://enterprise.arcgis.com/en/server/latest/publish-services/windows/what-is-a-map-service.htm .. _`Google Cloud SQL`: https://cloud.google.com/sql -.. _`OGC API - Features`: https://www.ogc.org/standards/ogcapi-features +.. _`OGC API - Features`: https://ogcapi.ogc.org/features .. _`Socrata Open Data API (SODA)`: https://dev.socrata.com .. _`sodapy`: https://github.com/xmunoz/sodapy .. _`Tabledap`: https://coastwatch.pfeg.noaa.gov/erddap/tabledap/documentation.html diff --git a/docs/source/data-publishing/ogcapi-maps.rst b/docs/source/data-publishing/ogcapi-maps.rst index 08a340f12..679e445c9 100644 --- a/docs/source/data-publishing/ogcapi-maps.rst +++ b/docs/source/data-publishing/ogcapi-maps.rst @@ -18,8 +18,8 @@ parameters. :header: Provider, bbox, width/height :align: left - `MapScript`,,✅,✅ - `WMSFacade`_,,✅,✅ + `MapScript`_,✅,✅ + `WMSFacade`_,✅,✅ Below are specific connection examples based on supported providers. @@ -30,7 +30,7 @@ Connection examples MapScript ^^^^^^^^^ -`MapScript`_ is MapServer's scripting interface to map rendering. +MapScript (`see website`_) is MapServer's scripting interface to map rendering. To publish a map via MapScript, the path to data is required, as well as the layer type (`options.type`). To style the data, set `options.style`. If @@ -51,7 +51,7 @@ Currently supported style files (`options.style`): .. code-block:: yaml providers: - - type: map + - type: map name: MapScript data: /path/to/data.shp options: @@ -59,9 +59,40 @@ Currently supported style files (`options.style`): layer: foo_name style: ./foo.sld format: - name: png + name: png mimetype: image/png +Projections are supported through EPSG codes (`options.projection`): + +.. code-block:: yaml + + options: + type: MS_LAYER_POINT + layer: foo_name + projection: 32631 + +This parameter is optional, defaulting to WGS84 (4236). + +This provider also supports `tile indexing `_, +which lets MapScript create a mosaic on the fly, piecing together a set of files. +In order to enable it, set `options.tileindex` to `True` and set the location of the index file on the `data` path. + +.. code-block:: yaml + + providers: + - type: map + name: MapScript + data: /data/index.shp + options: + type: MS_LAYER_RASTER + tileindex: True + layer: index + format: + name: png + mimetype: image/png + +The `options.tileindex` parameter is optional, defaulting to `False`. + WMSFacade ^^^^^^^^^ @@ -71,14 +102,15 @@ required. An optional style name can be defined via `options.style`. .. code-block:: yaml providers: - - type: map + - type: map name: WMSFacade data: https://demo.mapserver.org/cgi-bin/msautotest options: layer: world_latlong style: default + version: 1.3.0 format: - name: png + name: png mimetype: image/png @@ -86,17 +118,23 @@ Data visualization examples --------------------------- * list all collections + * http://localhost:5000/collections * overview of dataset + * http://localhost:5000/collections/foo * map (default format) + * http://localhost:5000/collections/foo/map * map with bbox subset + * http://localhost:5000/collections/foo/map?bbox=-142,42,-52,84 * map with bbox and temporal subset + * http://localhost:5000/collections/foo/map?bbox=-142,42,-52,84&datetime=2020-04-10T14:11:00Z * map with bbox and bbox-crs + * http://localhost:5000/collections/foo/map?bbox-crs=http%3A%2F%2Fwww.opengis.net%2Fdef%2Fcrs%2FEPSG%2F0%2F3857&bbox=4.022369384765626%2C50.690447870569436%2C4.681549072265626%2C51.00260125274477&width=800&height=600&transparent .. _`OGC API - Maps`: https://ogcapi.ogc.org/maps -.. _`MapScript`: https://mapserver.org/mapscript/index.html +.. _`see website`: https://mapserver.org/mapscript/index.html diff --git a/docs/source/data-publishing/ogcapi-processes.rst b/docs/source/data-publishing/ogcapi-processes.rst index 3480b1518..6cd214c5c 100644 --- a/docs/source/data-publishing/ogcapi-processes.rst +++ b/docs/source/data-publishing/ogcapi-processes.rst @@ -9,18 +9,52 @@ fashion (inputs, outputs). pygeoapi implements OGC API - Processes functionality by providing a plugin architecture, thereby allowing developers to implement custom processing workflows in Python. -A `sample`_ ``hello-world`` process is provided with the pygeoapi default configuration. +The pygeoapi offers two processes: a default ``hello-world`` process which allows you to quickly explore the capabilities of processes, and an optional ``shapely-functions`` process with more advanced features that leverages `Shapely`_ to expose various geometric processing functionality. Configuration ------------- +The below configuration is an example of a process defined within the pygeoapi internal plugin registry: + .. code-block:: yaml processes: + # enabled by default hello-world: processor: name: HelloWorld +The below configuration is an example of a process defined as part of a custom Python process: + +.. code-block:: yaml + + processes: + # enabled by default + hello-world: + processor: + # refer to a process in the standard PYTHONPATH + # e.g. my_package/my_module/my_file.py (class MyProcess) + # the MyProcess class must subclass from pygeoapi.process.base.BaseProcessor + name: my_package.my_module.my_file.MyProcess + +See :ref:`example-custom-pygeoapi-processing-plugin` for processing plugin examples. + +Processing and response handling +-------------------------------- + +pygeoapi processing plugins must return a tuple of media type and native outputs. Multipart +responses are not supported at this time, and it is up to the process plugin implementor to return a single +payload defining multiple artifacts (or references to them). + +By default (or via the OGC API - Processes ``response: raw`` execution parameter), pygeoapi provides +processing responses in their native encoding and media type, as defined by a given +plugin (which needs to set the response content type and payload accordingly). + +pygeoapi also supports a JSON-based response type (via the OGC API - Processes ``response: document`` +execution parameter). When this mode is requested, the response will always be a JSON encoding, embedding +the resulting payload (part of which may be Base64 encoded for binary data, for example). + + Asynchronous support -------------------- @@ -31,15 +65,27 @@ an asynchronous design pattern. This means that when a job is submitted in asyn mode, the server responds immediately with a reference to the job, which allows the client to periodically poll the server for the processing status of a given job. -pygeoapi provides asynchronous support by providing a 'manager' concept which, well, +In keeping with the OGC API - Processes specification, asynchronous process execution +can be requested by including the ``Prefer: respond-async`` HTTP header in the request. + +Job management is required for asynchronous functionality. + +Job management +-------------- + +pygeoapi provides job management by providing a 'manager' concept which, well, manages job execution. The manager concept is implemented as part of the pygeoapi :ref:`plugins` architecture. pygeoapi provides a default manager implementation based on `TinyDB`_ for simplicity. Custom manager plugins can be developed for more advanced job management capabilities (e.g. Kubernetes, databases, etc.). -In keeping with the OGC API - Processes specification, asynchronous process execution -can be requested by including the ``Prefer: respond-async`` HTTP header in the request +Job managers +------------ +TinyDB +^^^^^^ + +TinyDB is the default job manager for pygeoapi when enabled. .. code-block:: yaml @@ -50,11 +96,12 @@ can be requested by including the ``Prefer: respond-async`` HTTP header in the r output_dir: /tmp/ MongoDB --------------------- -As an alternative to the default a manager employing `MongoDB`_ can be used. -The connection to an installed `MongoDB`_ instance must be provided in the configuration. -`MongoDB`_ uses the localhost and port 27017 by default. Jobs are stored in a collection named -job_manager_pygeoapi. +^^^^^^^ + +As an alternative to the default, a manager employing `MongoDB`_ can be used. +The connection to a `MongoDB`_ instance must be provided in the configuration. +`MongoDB`_ uses ``localhost`` and port ``27017`` by default. Jobs are stored in a collection named +``job_manager_pygeoapi``. .. code-block:: yaml @@ -64,11 +111,34 @@ job_manager_pygeoapi. connection: mongodb://host:port output_dir: /tmp/ +PostgreSQL +^^^^^^^^^^ + +As another alternative to the default, a manager employing `PostgreSQL`_ can be used. +The connection to a `PostgreSQL`_ database must be provided in the configuration. +`PostgreSQL`_ uses ``localhost`` and port ``5432`` by default. Jobs are stored in a table named ``jobs``. + +.. code-block:: yaml + + server: + manager: + name: PostgreSQL + connection: + host: localhost + port: 5432 + database: test + user: postgres + password: ${POSTGRESQL_PASSWORD:-postgres} + # Alternative accepted connection definition: + # connection: postgresql://postgres:postgres@localhost:5432/test + # connection: postgresql://postgres:${POSTGRESQL_PASSWORD:-postgres}@localhost:5432/test + output_dir: /tmp + Putting it all together ----------------------- -To summarize how pygeoapi processes and managers work together:: +To summarize how pygeoapi processes and managers work together: * process plugins implement the core processing / workflow functionality * manager plugins control and manage how processes are executed @@ -76,6 +146,9 @@ To summarize how pygeoapi processes and managers work together:: Processing examples ------------------- +Hello World (Default) +^^^^^^^^^^^^^^^^^^^^^ + .. code-block:: sh # list all processes @@ -105,11 +178,87 @@ Processing examples # execute a job for the ``hello-world`` process in asynchronous mode curl -X POST http://localhost:5000/processes/hello-world/execution \ -H "Content-Type: application/json" \ - -H "Prefer: respond-async" + -H "Prefer: respond-async" \ -d "{\"inputs\":{\"name\": \"hi there2\"}}" + # execute a job for the ``hello-world`` process with a success subscriber + curl -X POST http://localhost:5000/processes/hello-world/execution \ + -H "Content-Type: application/json" \ + -d "{\"inputs\":{\"name\": \"hi there2\"}, \ + \"subscriber\": {\"successUri\": \"https://www.example.com/success\"}}" + +Shapely Functions (Optional) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The `shapely-functions` process exposes some selected Shapely_ functions as sample process. The selection cut across different operations in Shapely. To avoid function collision, it uses the name of the function category as the namespace. E.g *union* operation under the *set* module is described as *set:union*. + +The process is configured to accept a list of geometry *inputs* (WKT and/or GeoJSON geometry), *operation* and an optional *output_format*. It performs the specified operation and returns the result in the specified *output_format* (If the operation does not return a geometry, then this is ignored). + + +Configuration +------------- + +.. code-block:: yaml + + processes: + shapely-functions: + processor: + name: ShapelyFunctions + + +**Supported operations** + +* **measurement:bounds** - Computes the bounds (extent) of a geometry. +* **measurement:area** - Computes the area of a (multi)polygon. +* **measurement:distance** - Computes the Cartesian distance between two geometries. +* **predicates:covers** - Returns True if no point in geometry B is outside geometry A. +* **predicates:within** - Returns True if geometry A is completely inside geometry B. +* **set:difference** - Returns the part of geometry A that does not intersect with geometry B. +* **set:union** - Merges geometries into one. +* **constructive:buffer** - Computes the buffer of a geometry for positive and negative buffer distance. +* **constructive:centroid** - Computes the geometric center (center-of-mass) of a geometry. + +**Limitation** + +There is no support for passing optional function arguments yet. E.g when computing buffer on a geometry, no option to pass in the buffer distance. + +.. code-block:: sh + + # describe the ``shapely-functions`` process + curl http://localhost:5000/processes/shapely-functions + + # execute a job for the ``shapely-functions`` process that computes the bounds of a WKT + curl -X POST http://localhost:5000/processes/shapely-functions/execution \ + -H "Content-Type: application/json" \ + -d "{\"inputs\":{\"operation\": \"measurement:bounds\",\"geoms\": [\"POINT(83.27651071580385 22.593553859283745)\"]}}" + + # execute a job for the ``shapely-functions`` process that calculates the area of a WKT Polygon + curl -X POST http://localhost:5000/processes/shapely-functions/execution \ + -H "Content-Type: application/json" \ + -d "{\"inputs\":{\"operation\": \"measurement:area\",\"geoms\": [\"POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))\"]}}" + + # execute a job for the ``shapely-functions`` process that calculates the distance between two WKTs + curl -X POST http://localhost:5000/processes/shapely-functions/execution \ + -H "Content-Type: application/json" \ + -d "{\"inputs\":{\"operation\": \"measurement:distance\",\"geoms\": [\"POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))\",\"POINT(83.27651071580385 22.593553859283745)\"]}}" + + # execute a job for the ``shapely-functions`` process that calculates the predicate difference between two WKTs and returns a GeoJSON feature + curl -X POST http://localhost:5000/processes/shapely-functions/execution \ + -H "Content-Type: application/json" \ + -d "{\"inputs\":{\"operation\": \"set:difference\",\"geoms\": [\"POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))\",\"POINT(83.27651071580385 22.593553859283745)\"],\"output_format\":\"geojson\"}}" + + # execute a job for the ``shapely-functions`` process that calculates the predicate difference between two WKTs and returns a WKT + curl -X POST http://localhost:5000/processes/shapely-functions/execution \ + -H "Content-Type: application/json" \ + -d "{\"inputs\":{\"operation\": \"set:difference\",\"geoms\": [\"POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))\",\"POINT(83.27651071580385 22.593553859283745)\"],\"output_format\":\"wkt\"}}" + + # execute a job for the ``shapely-functions`` process that computes the buffer of a GeoJSON feature and returns a WKT + curl -X POST http://localhost:5000/processes/shapely-functions/execution \ + -H "Content-Type: application/json" \ + -d "{\"inputs\":{\"operation\": \"constructive:buffer\",\"geoms\": [{\"type\": \"LineString\",\"coordinates\": [[102.0,0.0],[103.0, 1.0],[104.0,0.0]]}],\"output_format\":\"wkt\"}}" -.. todo:: add more examples once OAProc implementation is complete .. _`OGC API - Processes`: https://ogcapi.ogc.org/processes .. _`sample`: https://github.com/geopython/pygeoapi/blob/master/pygeoapi/process/hello_world.py +.. _`shapely_functions`: https://github.com/geopython/pygeoapi/blob/master/pygeoapi/process/shapely_functions.py .. _`TinyDB`: https://tinydb.readthedocs.io/en/latest +.. _`Shapely`: https://shapely.readthedocs.io/ diff --git a/docs/source/data-publishing/ogcapi-records.rst b/docs/source/data-publishing/ogcapi-records.rst index c29c082a1..5f550da11 100644 --- a/docs/source/data-publishing/ogcapi-records.rst +++ b/docs/source/data-publishing/ogcapi-records.rst @@ -18,9 +18,9 @@ parameters. :header: Provider, properties (filters), resulttype, q, bbox, datetime, sortby, properties (display), CQL, transactions :align: left - `ElasticsearchCatalogue`_,✅,results/hits,✅,✅,✅,✅,✅,✅ - `TinyDBCatalogue`_,✅,results/hits,✅,✅,✅,✅,❌,✅ - `CSWFacade`_,✅,results/hits,✅,✅,✅,❌,❌,❌ + `ElasticsearchCatalogue`_,✅,results/hits,✅,✅,✅,✅,✅,✅,✅ + `TinyDBCatalogue`_,✅,results/hits,✅,✅,✅,✅,❌,✅,✅ + `CSWFacade`_,✅,results/hits,✅,✅,✅,❌,❌,❌,❌ Below are specific connection examples based on supported providers. @@ -100,27 +100,49 @@ Metadata search examples ------------------------ * overview of record collection + * http://localhost:5000/collections/metadata-records + * queryables + * http://localhost:5000/collections/foo/queryables + * browse records + * http://localhost:5000/collections/foo/items + * paging + * http://localhost:5000/collections/foo/items?offset=10&limit=10 + * CSV outputs + * http://localhost:5000/collections/foo/items?f=csv + * query records (spatial) + * http://localhost:5000/collections/foo/items?bbox=-180,-90,180,90 + * query records (attribute) + * http://localhost:5000/collections/foo/items?propertyname=foo + * query records (temporal) + * http://localhost:5000/collections/my-metadata/items?datetime=2020-04-10T14:11:00Z + * query features (temporal) and sort ascending by a property (if no +/- indicated, + is assumed) + * http://localhost:5000/collections/my-metadata/items?datetime=2020-04-10T14:11:00Z&sortby=datetime + * query features (temporal) and sort descending by a property + * http://localhost:5000/collections/my-metadata/items?datetime=2020-04-10T14:11:00Z&sortby=-datetime + * fetch a specific record + * http://localhost:5000/collections/my-metadata/items/123 + .. note:: provider `id_field` values support slashes (i.e. ``my/cool/identifier``). The client request would then diff --git a/docs/source/data-publishing/ogcapi-tiles.rst b/docs/source/data-publishing/ogcapi-tiles.rst index c3a40b677..3f669061a 100644 --- a/docs/source/data-publishing/ogcapi-tiles.rst +++ b/docs/source/data-publishing/ogcapi-tiles.rst @@ -7,7 +7,7 @@ Publishing tiles to OGC API - Tiles (map, vector, coverage, etc.). pygeoapi can publish tiles from local or remote data sources (including cloud -object storage or a tile service). +object storage or a tile service). Providers --------- @@ -15,25 +15,24 @@ Providers pygeoapi core tile providers are listed below, along with supported features. .. csv-table:: - :header: Provider, rendered on-the-fly, properties + :header: Provider, rendered on-the-fly, properties, WebMercatorQuad, WorldCRS84Quad, raster, vector :align: left - `MVT-tippecanoe`_,❌,✅ - `MVT-elastic`_,✅,❌ + `MVT-tippecanoe`_,❌,✅,✅,❌,❌,✅ + `MVT-elastic`_,✅,✅,✅,❌,❌,✅ + `MVT-proxy`_,❓,❓,❓,❓,❌,✅ + `WMTSFacade`_,✅,❌,✅,✅,✅,❌ Below are specific connection examples based on supported providers. -.. note:: - Currently only `Mapbox Vector Tiles (MVT) `_ are supported in pygeoapi. - Connection examples ------------------- MVT-tippecanoe ^^^^^^^^^^^^^^ -This provider gives support to serving tiles generated using `Mapbox Tippecanoe `_. -The tiles can be integrated from a path on disk, or from a static url (e.g.: from an S3 or MinIO bucket). +This provider gives support to serving tiles generated using `Mapbox Tippecanoe `_. +The tiles can be integrated from a path on disk, or from a static url (e.g.: from an S3 or MinIO bucket). In both cases, they have to be rendered before using pygeoapi. This code block shows how to configure pygeoapi to read Mapbox vector tiles generated with tippecanoe, from disk or a URL. @@ -42,26 +41,26 @@ This code block shows how to configure pygeoapi to read Mapbox vector tiles gene providers: - type: tile - name: MVT-tippecanoe + name: MVT-tippecanoe data: tests/data/tiles/ne_110m_lakes # local directory tree # data: http://localhost:9000/ne_110m_lakes/{z}/{x}/{y}.pbf # tiles stored on a MinIO bucket options: zoom: min: 0 max: 5 - schemes: - - WorldCRS84Quad + # MVT-elastic always uses WebMercatorQuad tiling scheme format: - name: pbf + name: pbf mimetype: application/vnd.mapbox-vector-tile .. tip:: - On `this tutorial `_ you can find detailed instructions on how-to generate tiles using tippecanoe and integrate them into pygeoapi. + + In the diving into pygeoapi workshop `OGC API - Tiles Exercise `_, detailed instructions can be found on how to generate tiles using tippecanoe and integrate them into pygeoapi. MVT-elastic -^^^^^^^^^^^^ +^^^^^^^^^^^ -This provider gives support to serving tiles generated using `Elasticsearch `_. +This provider gives support to serving tiles generated using `Elasticsearch `_. These tiles are rendered on-the-fly using the `Elasticsearch Vector tile search API `_. In order to use it, the only requirement is to have the data stored in an Elasticsearch index. @@ -71,39 +70,120 @@ This code block shows how to configure pygeoapi to read Mapbox vector tiles from providers: - type: tile - name: MVT-elastic + name: MVT-elastic data: http://localhost:9200/ne_110m_populated_places_simple2/_mvt/geometry/{z}/{x}/{y}?grid_precision=0 - # if you don't use precision 0, you will be requesting for aggregations which are not supported in the + # if you don't use precision 0, you will be requesting for aggregations which are not supported in the # free version of elastic options: zoom: min: 0 max: 5 - schemes: - - WorldCRS84Quad + # MVT-elastic always uses WebMercatorQuad tiling scheme format: - name: pbf + name: pbf mimetype: application/vnd.mapbox-vector-tile .. tip:: On `this tutorial `_ you can find detailed instructions on publish tiles stored in an Elasticsearch endpoint. - + +MVT-proxy +^^^^^^^^^ + +This provider gives support to serving tiles from a generic tiles provider `{z}/{x}/{y}`. + +For example, you can get and publish tiles from PostGIS providers like `pg_tileserver `_ +or `martin `_. Both of them render tiles on the fly and provide properties. + +Following block shows how to configure pygeoapi to read Mapbox vector tiles from pg_tileserver endpoint. + +.. code-block:: yaml + + providers: + - type: tile + name: MVT-proxy + data: http://localhost:7800/public.ne_50m_admin_0_countries/{z}/{x}/{y}.mvt + options: + zoom: + min: 0 + max: 15 + schemes: + - WebMercatorQuad # this option is needed in the MVT-proxy provider + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile + +Following code block shows how to configure pygeoapi to read Mapbox vector tiles from martin endpoint. + +.. code-block:: yaml + + providers: + - type: tile + name: MVT-proxy + data: http://localhost:3000/ne_50m_admin_0_countries/{z}/{x}/{y} + options: + zoom: + min: 0 + max: 15 + schemes: + - WebMercatorQuad + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile + + +WMTSFacade +^^^^^^^^^^ + +This provider gives support to serving map tiles generated using a WMTS. + +It is important that the WMTS provides a tile matrix set that matches exactly one of the configured schemes in pygeoapi. +Currently only `WebMercatorQuad` and `WorldCRS84Quad` are available in pygeopi. + +This code block shows how to configure pygeoapi to read map tiles from a WMTS. + +.. code-block:: yaml + + providers: + - type: tile + name: WMTSFacade + data: https://emotional.byteroad.net/geoserver/gwc/service/wmts + format: + name: png # png or jpeg + mimetype: image/png + options: + wmts_layer: camb:hex350_grid_mental_1920 # the layer name of the wmts + wmts_tile_matrix_set: WebMercatorQuad # the name of the tile matrix set of the wmts. + scheme: WebMercatorQuad # the aligning scheme in pygeoapi. + zoom: + min: 0 + max: 20 + Data access examples -------------------- * list all collections + * http://localhost:5000/collections + * overview of dataset + * http://localhost:5000/collections/foo + * overview of dataset tiles + * http://localhost:5000/collections/foo/tiles + * tile matrix metadata + * http://localhost:5000/collections/lakes/tiles/WorldCRS84Quad/metadata + * tiles URI template + * `http://localhost:5000/collections/lakes/tiles/{tileMatrixSetId}/{tileMatrix}/{tileRow}/{tileCol}?f=mvt `_ + -.. _`OGC API - Tiles`: https://github.com/opengeospatial/ogcapi-tiles +.. _`OGC API - Tiles`: https://ogcapi.ogc.org/tiles .. _`tippecanoe`: https://github.com/mapbox/tippecanoe .. _`Elasticsearch`: https://www.elastic.co/ .. _`Mapbox Vector Tiles`: https://docs.mapbox.com/data/tilesets/guides/vector-tiles-introduction/ diff --git a/docs/source/data-publishing/stac.rst b/docs/source/data-publishing/stac.rst index 0fad719f9..97a23c965 100644 --- a/docs/source/data-publishing/stac.rst +++ b/docs/source/data-publishing/stac.rst @@ -49,7 +49,7 @@ Connection examples pygeometa metadata control files -------------------------------- -pygeoapi's STAC filesystem fuctionality supports `pygeometa`_ MCF files residing +pygeoapi's STAC filesystem functionality supports `pygeometa`_ MCF files residing in the same directory as data files. If an MCF file is found, it will be used as part of generating the STAC item metadata (e.g. a file named ``birds.csv`` having an associated ``birds.yml`` file). If no MCF file is found, then @@ -66,6 +66,7 @@ Data access examples -------------------- * STAC root page + * http://localhost:5000/stac From here, browse the filesystem accordingly. @@ -333,7 +334,7 @@ The example below shows the content of a file named *arcticdem-frontiere-0.json* ], "assets": { "image": { - "href": "http://absolute/path/to/the/ressource/arcticdem-frontiere-0.tif", + "href": "https://example.com/path/to/resource/arcticdem-frontiere-0.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": [] } diff --git a/docs/source/development.rst b/docs/source/development.rst index 106db842b..23bd8c6d5 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -20,45 +20,7 @@ Tests can be run locally as part of development workflow. They are also run on `GitHub Actions setup`_ against all commits and pull requests to the code repository. To run all tests, simply run ``pytest`` in the repository. To run a specific test file, -run ``pytest tests/test_api.py``, for example. - - -CQL extension lifecycle ------------------------ - -Limitations -^^^^^^^^^^^ - -This workflow is valid only for the `CQL-JSON` format. - -Schema -^^^^^^ - -The Common Query Language (CQL) is the part 3 of the standard OGC API - Features. This extension has its specification available at -`OGC API - Features - Part 3: Filtering and the Common Query Language (CQL) `_ and the schema exists in development at -`cql.json `_. - -Model generation -^^^^^^^^^^^^^^^^ - -pygeoapi uses a class-based Python model interface to translate the schema into Python objects defined by `pydantic `_ models. -The model is generated with the pre-processing of the schema through the utility ``datamodel-codegen``, which is part -of the `datamodel-code-generator `_ package: - - -.. code-block:: bash - - # Generate from local downloaded json schema file - datamodel-codegen --input ~/Download/cql-schema.json --input-file-type jsonschema --output ./pygeoapi/models/cql_update.py --class-name CQLModel - -Note that datamodel-code-generator must be explicitly installed, as it is not a pygeoapi runtime dependency - -How to merge -^^^^^^^^^^^^ - -Once the new pydantic models have been generated then the content of the Python file ``cql_update.py`` can be used to replace the old classes within the ``cql.py`` file. -Update everything above the function ``get_next_node`` and then verify if the tests for the CQL are still passing, for example ``test_post_cql_json_between_query`` -in ``tests/test_elasticsearch__provider.py``. +run ``pytest tests/api/test_itemtypes.py``, for example. Working with Spatialite on OSX ------------------------------ @@ -82,7 +44,7 @@ Install Python with the option to enable SQLite extensions: .. code-block:: bash - LDFLAGS="-L/usr/local/opt/sqlite/lib -L/usr/local/opt/zlib/lib" CPPFLAGS="-I/usr/local/opt/sqlite/include -I/usr/local/opt/zlib/include" PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" pyenv install 3.7.6 + LDFLAGS="-L/usr/local/opt/sqlite/lib -L/usr/local/opt/zlib/lib" CPPFLAGS="-I/usr/local/opt/sqlite/include -I/usr/local/opt/zlib/include" PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" pyenv install 3.10.12 Configure SQLite from Homebrew over that one shipped with the OS: diff --git a/docs/source/html-templating.rst b/docs/source/html-templating.rst index 14b0b63b9..f1f5ce29b 100644 --- a/docs/source/html-templating.rst +++ b/docs/source/html-templating.rst @@ -5,7 +5,7 @@ HTML Templating pygeoapi uses `Jinja`_ as its templating engine to render HTML and `Flask`_ to provide route paths of the API that returns HTTP responses. For complete details on how to use these modules, refer to the `Jinja documentation`_ and the `Flask documentation`_. -The default pygeoapi configuration has ``server.templates`` commented out and defaults to the pygeoapi ``pygeoapi/templates`` and ``pygeoapi/static`` folder. To point to a different set of template configuration, you can edit your configuration: +The default pygeoapi configuration has ``server.templates`` commented out and defaults to the pygeoapi ``pygeoapi/templates`` and ``pygeoapi/static`` folder. To point to a different set of template configuration, you can edit your configuration as follows: .. code-block:: yaml @@ -20,9 +20,9 @@ Your templates folder should mimic the same file names and structure of the defa Note that you need only copy and edit the templates you are interested in updating. For example, if you are only interested in updating the ``landing_page.html`` template, then create your own version -of the only that same file. When pygeoapi detects that a custom HTML template is being used, -it will look for the custom template in ``server.templates.path``. If it does not exists, pygeoapi -will render the default HTML template for the given endpoint/requuest. +of only that same file. When pygeoapi detects that a custom HTML template is being used, +it will look for the custom template in ``server.templates.path``. If it does not exist, pygeoapi +will render the default HTML template for the given endpoint/request. Linking to a static file in your HTML templates can be done using Jinja syntax and the exposed ``config['server']['url']``: @@ -35,22 +35,44 @@ Linking to a static file in your HTML templates can be done using Jinja syntax a +Dataset level templates +----------------------- -Featured templates ------------------- +The ``templates`` configuration directive is applied to the entire server by default. It can also be used for a dataset specific look and feel. As example use case is defining a template for a specific dataset to be able to add custom UI/UX functionality (e.g. search/filter widget). -The following themes provide useful examples of pygeoapi templates implemented -by downstream applications. +.. note:: -.. csv-table:: - :header: "Plugin(s)", "Organization/Project","Description" - :align: left + Dataset level templates apply to ``/collections/{collectionId}`` and below. - `pygeoapi-skin-dashboard`_,GeoCat bv,skin for pygeoapi based on a typical dashboard interface +Example +^^^^^^^ + +The below is an example dataset specific template using pygeoapi's default theme: + + +.. code-block:: html + + {% extends "_base.html" %} + + {% block body %} + +

My cool dataset

+ + {% endblock %} + +.. note:: + + You can choose to use pygeoapi's default base theme, or your own as desired. + + +Featured themes +--------------- + +Community based themes can be found on the `pygeoapi Community Plugins and Themes wiki page`_. .. _`Jinja`: https://palletsprojects.com/p/jinja/ .. _`Jinja documentation`: https://jinja.palletsprojects.com .. _`Flask`: https://palletsprojects.com/p/flask/ .. _`Flask documentation`: https://flask.palletsprojects.com -.. _`pygeoapi-skin-dashboard`: https://github.com/GeoCat/pygeoapi-skin-dashboard +.. _`pygeoapi Community Plugins and Themes wiki page`: https://github.com/geopython/pygeoapi/wiki/CommunityPluginsThemes diff --git a/docs/source/index.rst b/docs/source/index.rst index 1b6ab82c4..49191a7c1 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -40,6 +40,7 @@ reference documentation on all aspects of the project. data-publishing/index transactions admin-api + security plugins html-templating crs @@ -51,7 +52,6 @@ reference documentation on all aspects of the project. support further-reading license - api-documentation Indices and tables diff --git a/docs/source/installation.rst b/docs/source/installation.rst index c3ff21b7b..de43fdfc7 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -11,6 +11,13 @@ Requirements and dependencies pygeoapi runs on Python 3. +.. note:: + + The exact Python version requirements are aligned with the version of Python on the pygeoapi supported Ubuntu + operating system version. For example, as of 2024-07, the supported version of Python is bound to Ubuntu 22.04 + (Jammy) which supports Python 3.10. Ensure you have a Python version that is compatible with the current Ubuntu + version that is specified in pygeoapi's `Dockerfile`_. + Core dependencies are included as part of a given pygeoapi installation procedure. More specific requirements details are described below depending on the platform. @@ -32,7 +39,7 @@ For developers and the truly impatient vi example-config.yml # edit as required export PYGEOAPI_CONFIG=example-config.yml export PYGEOAPI_OPENAPI=example-openapi.yml - pygeoapi openapi generate $PYGEOAPI_CONFIG > $PYGEOAPI_OPENAPI + pygeoapi openapi generate $PYGEOAPI_CONFIG --output-file $PYGEOAPI_OPENAPI pygeoapi serve curl http://localhost:5000 @@ -52,7 +59,7 @@ Docker Using DockerHub ^^^^^^^^^^^^^^^ -`Docker image`_ +`DockerHub image`_ .. code-block:: bash @@ -61,7 +68,7 @@ Using DockerHub Using GitHub Container Registry ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -`Docker image`_ +`GCHR image`_ .. code-block:: bash @@ -141,4 +148,6 @@ Congratulations! Whichever of the abovementioned methods you chose, you have su onto your system. -.. _`Docker image`: https://github.com/geopython/pygeoapi/pkgs/container/pygeoapi +.. _`DockerHub image`: https://hub.docker.com/r/geopython/pygeoapi +.. _`GCHR image`: https://github.com/geopython/pygeoapi/pkgs/container/pygeoapi +.. _`Dockerfile`: https://github.com/geopython/pygeoapi/blob/master/Dockerfile diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index fb78f6a27..d41ff33f4 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -10,15 +10,19 @@ Features * out of the box modern OGC API server * certified OGC Compliant and Reference Implementation + * OGC API - Features * OGC API - Environmental Data Retrieval + * OGC API - Tiles + * OGC API - Processes + * additionally implements + * OGC API - Coverages * OGC API - Maps - * OGC API - Tiles - * OGC API - Processes * OGC API - Records * SpatioTemporal Asset Library + * out of the box data provider plugins for rasterio, GDAL/OGR, Elasticsearch, PostgreSQL/PostGIS * easy to use OpenAPI / Swagger documentation for developers * supports JSON, GeoJSON, HTML and CSV output @@ -48,7 +52,7 @@ Standards are at the core of pygeoapi. Below is the project's standards support `OGC API - Coverages`_,Implementing `OGC API - Maps`_,Implementing `OGC API - Tiles`_,Reference Implementation - `OGC API - Processes`_,Implementing + `OGC API - Processes`_,Compliant `OGC API - Records`_,Implementing `OGC API - Environmental Data Retrieval`_,Reference Implementation `SpatioTemporal Asset Catalog`_,Implementing diff --git a/docs/source/language.rst b/docs/source/language.rst index 3aa3f4566..26f0740ec 100644 --- a/docs/source/language.rst +++ b/docs/source/language.rst @@ -15,9 +15,9 @@ End user guide There are 2 ways to affect the language of the results returned by pygeoapi, both for the HTML and JSON(-LD) formats: 1. After the requested pygeoapi URL, append a ``lang=`` query parameter, where ```` should be replaced by a well-known language code. - This can be an ISO 639-1 code (e.g. `de` for German), optionally accompanied by an ISO 3166-1 alpha-2 country code (e.g. `de-CH` for Swiss-German). - Please refer to this `W3C article `_ for more information or - this `list of language codes `_ for more examples. + This can be an `ISO 639-1 code `_ (e.g. `de` for German), optionally accompanied by an `ISO 3166-1 alpha-2 country code `_ (e.g. `de-CH` for Swiss-German). + + Please refer to this `W3C article `_ for more information. Another option is to send a complex definition with quality weights (e.g. `de-CH, de;q=0.9, en;q=0.8, fr;q=0.7, \*;q=0.5`). pygeoapi will then figure out the best match for the requested language. @@ -176,7 +176,7 @@ Translator guide Hardcoded strings in pygeoapi templates are translated using the Babel translation system. By default, pygeoapi stores translation files in the `locale` directory at the root of the -source code repository. This value can be overriden in the pygeoapi configuration with +source code repository. This value can be overridden in the pygeoapi configuration with the `server.locale_dir` directive. Translators can follow these steps to prepare their environment for translations. diff --git a/docs/source/openapi.rst b/docs/source/openapi.rst index 1d3593316..71383c61c 100644 --- a/docs/source/openapi.rst +++ b/docs/source/openapi.rst @@ -13,7 +13,7 @@ The official OpenAPI specification can be found `on GitHub {self.name}' + + +The example above handles a dictionary of the JSON payload passed from the client, calculates the square root of a float or integer, and returns the result in an output JSON payload. The plugin is responsible for defining the expected inputs and outputs in ``PROCESS_METADATA`` and to return the output in any format along with the corresponding media type. + +.. note:: + + Additional processing plugins can also be found in ``pygeoapi/process``. Example: custom pygeoapi formatter ---------------------------------- @@ -295,47 +366,20 @@ The below template provides a minimal example (let's call the file ``mycooljsonf def write(self, options={}, data=None): """custom writer""" - out_data {'rows': []} + out_data = {'rows': []} for feature in data['features']: - out_data.append(feature['properties']) + out_data['rows'].append(feature['properties']) return out_data -Processing plugins ------------------- - -Processing plugins are following the OGC API - Processes development. Given that the specification is -under development, the implementation in ``pygeoapi/process/hello_world.py`` provides a suitable example -for the time being. - - Featured plugins ---------------- -The following plugins provide useful examples of pygeoapi plugins implemented -by downstream applications. - -.. csv-table:: - :header: "Plugin(s)", "Organization/Project","Description" - :align: left - - `msc-pygeoapi`_,Meteorological Service of Canada,processes for weather/climate/water data workflows - `pygeoapi-kubernetes-papermill`_,Euro Data Cube,processes for executing Jupyter notebooks via Kubernetes - `local-outlier-factor-plugin`_,Manaaki Whenua – Landcare Research,processes for local outlier detection - `ogc-edc`_,Euro Data Cube,coverage provider atop the EDC API - `nldi_xstool`_,United States Geological Survey,Water data processing - `pygeometa-plugin`_,pygeometa project,pygeometa as a service - `cgs-plugins`_,Center for Geospatial Solutions,feature and processes plugins +Community based plugins can be found on the `pygeoapi Community Plugins and Themes wiki page`_. -.. _`cgs-plugins`: https://github.com/cgs-earth/pygeoapi-plugins +.. _`pygeoapi Community Plugins and Themes wiki page`: https://github.com/geopython/pygeoapi/wiki/CommunityPluginsThemes .. _`Cookiecutter`: https://github.com/audreyfeldroy/cookiecutter-pypackage -.. _`msc-pygeoapi`: https://github.com/ECCC-MSC/msc-pygeoapi -.. _`pygeoapi-kubernetes-papermill`: https://github.com/eurodatacube/pygeoapi-kubernetes-papermill -.. _`local-outlier-factor-plugin`: https://github.com/manaakiwhenua/local-outlier-factor-plugin -.. _`ogc-edc`: https://github.com/eurodatacube/ogc-edc/tree/oapi/edc_ogc/pygeoapi -.. _`nldi_xstool`: https://code.usgs.gov/wma/nhgf/toolsteam/nldi-xstool .. _`pygeoapi-plugin-cookiecutter`: https://code.usgs.gov/wma/nhgf/pygeoapi-plugin-cookiecutter -.. _`pygeometa-plugin`: https://geopython.github.io/pygeometa/pygeoapi-plugin diff --git a/docs/source/running-with-docker.rst b/docs/source/running-with-docker.rst index afd9533f3..a1094fef4 100644 --- a/docs/source/running-with-docker.rst +++ b/docs/source/running-with-docker.rst @@ -29,6 +29,12 @@ To run with the default built-in configuration and data: ...then browse to http://localhost:5000 +You can also run pygeoapi with run-with-hot-reload of the configuration enabled + +.. code-block:: bash + + docker run -p 5000:80 -it geopython/pygeoapi run-with-hot-reload + You can also run all unit tests to verify: .. code-block:: bash @@ -71,6 +77,32 @@ Or you can create a ``Dockerfile`` extending the base image and **copy** in your A corresponding example can be found in https://github.com/geopython/demo.pygeoapi.io/tree/master/services/pygeoapi_master +Environment Variables for Configuration +--------------------------------------- + +The base Docker image supports two additional environment variables for configuring the `pygeoapi` server behavior: + +1. **`PYGEOAPI_SERVER_URL`**: + This variable sets the `pygeoapi` server URL in the configuration. It is useful for dynamically configuring the server URL during container deployment. For example: + + .. code-block:: bash + + docker run -p 2018:80 -e PYGEOAPI_SERVER_URL='http://localhost:2018' -it geopython/pygeoapi + + This ensures the service URLs in the configuration file are automatically updated to reflect the specified URL. + +2. **`PYGEOAPI_SERVER_ADMIN`**: + This boolean environment variable enables or disables the `pygeoapi` Admin API. By default, the Admin API is disabled. To enable it: + + .. code-block:: bash + + docker run -p 5000:80 -e PYGEOAPI_SERVER_ADMIN=true -it geopython/pygeoapi + # run with hot reload + docker run -p 5000:80 -e PYGEOAPI_SERVER_ADMIN=true -it geopython/pygeoapi run-with-hot-reload + + To learn more about the Admin API see :ref:`admin-api`. + + Deploying on a sub-path ----------------------- diff --git a/docs/source/running.rst b/docs/source/running.rst index 8ea173f01..c92bbee80 100644 --- a/docs/source/running.rst +++ b/docs/source/running.rst @@ -100,7 +100,7 @@ To integrate pygeoapi as part of another Starlette application: from starlette.applications import Starlette from starlette.responses import PlainTextResponse from starlette.routing import Route - from pygeoapi.starlette_app import app as pygeoapi_app + from pygeoapi.starlette_app import APP as pygeoapi_app async def homepage(request): @@ -158,8 +158,12 @@ main Django application urls: ] -This integration can be seen in the provided example Django project. Refer to `examples/django/sample_project/README.md` -for the integration of pygeoapi with an already exising Django application. +This integration can be seen in the provided example Django project. Refer to the `Django example in the pygeoapi-examples repository`_ +for the integration of pygeoapi with an already existing Django application. + + +.. note:: + To enable HTTP POST/PUT/PATCH/DELETE functionality, `django.middleware.csrf.CsrfViewMiddleware` must not be set. Note that this enables create/replace/update/delete functionality against resources in your application. Hot-reloading ^^^^^^^^^^^^^ @@ -172,7 +176,7 @@ By hot-reloading we mean to be able to directly see changes reflected in the app This is useful for development, as the changes made by developers are easily and rapidly reflected and they can take advantage of the hot-reloading capabilities that offer each of the web servers available. -For enabling hot-reloading, install the pygeoapi package using pip (instead of the setup.py script) with the following command: +To enable hot-reloading, install the pygeoapi package using pip (instead of `setup.py`) with the following command: .. code-block:: bash @@ -181,8 +185,7 @@ For enabling hot-reloading, install the pygeoapi package using pip (instead of t .. note:: This command must be run from the root directory of pygeoapi. -After the local package is built, you can use the ``pygeoapi serve`` -again and the changes on the codebase will be directly reflected on the running instance. +After the local package is built, run ``pygeoapi serve`` again and the changes to the codebase will be reflected live on the running instance. Running in production @@ -294,3 +297,4 @@ and modify accordingly. .. _`Uvicorn`: https://www.uvicorn.org .. _`mod_wsgi`: https://modwsgi.readthedocs.io/en/master .. _`Django`: https://www.djangoproject.com +.. _`Django example in the pygeoapi-examples repository`: https://github.com/geopython/pygeoapi-examples/blob/main/django/sample_project/README.md diff --git a/docs/source/security.rst b/docs/source/security.rst new file mode 100644 index 000000000..df81f8a18 --- /dev/null +++ b/docs/source/security.rst @@ -0,0 +1,18 @@ +.. _security: + +Security +======== + +There exist use cases which require authentication and authorization against an API at various granularities +(collections, processes, etc.), restricting access to a given user, group or role. Implementing security +can be as simple as HTTP basic authentication, or as complex as fine-grained access control against a specific +collection item. + +By design, pygeoapi does not have built-in support for access control. It is up to the user to secure pygeoapi +as required. + +The following projects provide security frameworks atop pygeoapi: + +* `fastgeoapi `_ +* `pygeoapi-auth-deployment `_ +* `pygeoapi-auth `_ (Python package for use along with pygeoapi-auth-deployment) diff --git a/docs/source/tour.rst b/docs/source/tour.rst index 06a6e363d..4fd4e51da 100644 --- a/docs/source/tour.rst +++ b/docs/source/tour.rst @@ -117,19 +117,12 @@ Delete an item from a collection: Raster data ----------- -Collection coverage domainset -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Collection coverage schema +^^^^^^^^^^^^^^^^^^^^^^^^^^ -This page provides information on a collection coverage spatial properties and axis information. +This page provides information on a collection coverage information. -http://localhost:5000/collections/gdps-temperature/coverage/domainset - -Collection coverage rangetype -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This page provides information on a collection coverage rangetype (bands) information. - -http://localhost:5000/collections/gdps-temperature/coverage/rangetype +http://localhost:5000/collections/gdps-temperature/schema Collection coverage data ^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/transactions.rst b/docs/source/transactions.rst index ac846d0f2..4c6327174 100644 --- a/docs/source/transactions.rst +++ b/docs/source/transactions.rst @@ -14,6 +14,6 @@ Access control ^^^^^^^^^^^^^^ It should be made clear that authentication and authorization is beyond the responsibility of pygeoapi. This means that -if a pygeoapi user enables transactions, they must provide access control explicity via another service. +if a pygeoapi user enables transactions, they must provide access control explicitly via another service. .. _`OGC API - Features - Part 4: Create, Replace, Update and Delete`: https://docs.ogc.org/DRAFTS/20-002.html diff --git a/locale/README.md b/locale/README.md index ee3c40741..49f8bdc19 100644 --- a/locale/README.md +++ b/locale/README.md @@ -1,4 +1,4 @@ -# Howto set up language files +# How to set up language files Inspired by https://phrase.com/blog/posts/i18n-advantages-babel-python/#Integration_with_Jinja2_templates @@ -22,6 +22,21 @@ Then extract the base messages from templates: This file is not persisted on github. +If the above command gives this ouput: +``` +AttributeError: module 'jinja2.ext' has no attribute 'autoescape' +``` + +Just comment out the extensions line in the `babel-mapping.ini` file: +``` +[python: **.py] +[jinja2: **/templates/**.html] +;extensions=jinja2.ext.i18n,jinja2.ext.autoescape,jinja2.ext.with_ +``` + +Then rerun the command for extracting base messages. + + Now setup a new language (french) using the init command: > pybabel init -d locale -l fr -i locale/messages.pot diff --git a/locale/ar/LC_MESSAGES/messages.po b/locale/ar/LC_MESSAGES/messages.po new file mode 100644 index 000000000..87e035ba5 --- /dev/null +++ b/locale/ar/LC_MESSAGES/messages.po @@ -0,0 +1,748 @@ +# Arabic translations for PROJECT. +# Copyright (C) 2024 OSGeo +# This file is distributed under the same license as the pygeoapi project. +# FIRST AUTHOR Youssef Harby, 2024. +# +msgid "" +msgstr "" +"Project-Id-Version: 0.0.19\n" +"Report-Msgid-Bugs-To: pygeoapi@lists.osgeo.org\n" +"POT-Creation-Date: 2024-11-19 23:22+0200\n" +"PO-Revision-Date: 2024-11-19 23:22+0200\n" +"Last-Translator: Youssef Harby \n" +"Language: ar\n" +"Language-Team: ar \n" +"Plural-Forms: nplurals=6; plural=(n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : " +"n%100>=3 && n%100<=10 ? 3 : n%100>=0 && n%100<=2 ? 4 : 5);\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.13.0\n" + +#: build/lib/pygeoapi/templates/_base.html:62 +#: build/lib/pygeoapi/templates/landing_page.html:2 +#: pygeoapi/templates/_base.html:67 pygeoapi/templates/landing_page.html:2 +msgid "Home" +msgstr "الصفحة الرئيسية" + +#: build/lib/pygeoapi/templates/_base.html:70 +#: build/lib/pygeoapi/templates/_base.html:78 pygeoapi/templates/_base.html:75 +#: pygeoapi/templates/_base.html:83 +msgid "json" +msgstr "json" + +#: build/lib/pygeoapi/templates/_base.html:73 +#: build/lib/pygeoapi/templates/_base.html:81 pygeoapi/templates/_base.html:78 +#: pygeoapi/templates/_base.html:86 +msgid "jsonld" +msgstr "jsonld" + +#: build/lib/pygeoapi/templates/_base.html:100 +#: pygeoapi/templates/_base.html:107 +msgid "Powered by " +msgstr "مدعوم بواسطة " + +#: build/lib/pygeoapi/templates/conformance.html:2 +#: build/lib/pygeoapi/templates/conformance.html:4 +#: build/lib/pygeoapi/templates/conformance.html:8 +#: build/lib/pygeoapi/templates/landing_page.html:86 +#: pygeoapi/templates/conformance.html:2 pygeoapi/templates/conformance.html:4 +#: pygeoapi/templates/conformance.html:8 +#: pygeoapi/templates/landing_page.html:95 +msgid "Conformance" +msgstr "التوافق" + +#: build/lib/pygeoapi/templates/exception.html:2 +#: build/lib/pygeoapi/templates/exception.html:5 +#: pygeoapi/templates/exception.html:2 pygeoapi/templates/exception.html:5 +msgid "Exception" +msgstr "استثناء" + +#: build/lib/pygeoapi/templates/landing_page.html:25 +#: pygeoapi/templates/landing_page.html:25 +msgid "Terms of service" +msgstr "شروط الخدمة" + +#: build/lib/pygeoapi/templates/collections/collection.html:38 +#: build/lib/pygeoapi/templates/landing_page.html:35 +#: pygeoapi/templates/collections/collection.html:38 +#: pygeoapi/templates/landing_page.html:35 +msgid "License" +msgstr "الرخصة" + +#: build/lib/pygeoapi/templates/collections/collection.html:6 +#: build/lib/pygeoapi/templates/collections/coverage/domainset.html:4 +#: build/lib/pygeoapi/templates/collections/coverage/rangetype.html:4 +#: build/lib/pygeoapi/templates/collections/edr/query.html:4 +#: build/lib/pygeoapi/templates/collections/index.html:2 +#: build/lib/pygeoapi/templates/collections/index.html:4 +#: build/lib/pygeoapi/templates/collections/items/index.html:4 +#: build/lib/pygeoapi/templates/collections/items/item.html:27 +#: build/lib/pygeoapi/templates/collections/queryables.html:4 +#: build/lib/pygeoapi/templates/collections/tiles/index.html:4 +#: build/lib/pygeoapi/templates/collections/tiles/metadata.html:4 +#: build/lib/pygeoapi/templates/landing_page.html:48 +#: pygeoapi/templates/collections/collection.html:6 +#: pygeoapi/templates/collections/edr/query.html:4 +#: pygeoapi/templates/collections/index.html:2 +#: pygeoapi/templates/collections/index.html:4 +#: pygeoapi/templates/collections/items/index.html:4 +#: pygeoapi/templates/collections/items/item.html:27 +#: pygeoapi/templates/collections/queryables.html:4 +#: pygeoapi/templates/collections/schema.html:4 +#: pygeoapi/templates/collections/tiles/index.html:4 +#: pygeoapi/templates/collections/tiles/metadata.html:4 +#: pygeoapi/templates/landing_page.html:57 +#: pygeoapi/templates/stac/collection_base.html:19 +msgid "Collections" +msgstr "المجموعات" + +#: build/lib/pygeoapi/templates/landing_page.html:50 +#: pygeoapi/templates/landing_page.html:59 +msgid "View the collections in this service" +msgstr "عرض المجموعات في هذه الخدمة" + +#: build/lib/pygeoapi/templates/landing_page.html:56 +#: pygeoapi/templates/landing_page.html:65 +msgid "SpatioTemporal Assets" +msgstr "الأصول الزمانية والمكانية (STAC)" + +#: build/lib/pygeoapi/templates/landing_page.html:58 +#: pygeoapi/templates/landing_page.html:67 +msgid "View the SpatioTemporal Assets in this service" +msgstr "عرض الأصول الزمانية والمكانية في هذه الخدمة" + +#: build/lib/pygeoapi/templates/landing_page.html:64 +#: build/lib/pygeoapi/templates/processes/index.html:2 +#: build/lib/pygeoapi/templates/processes/index.html:4 +#: build/lib/pygeoapi/templates/processes/process.html:4 +#: pygeoapi/templates/landing_page.html:73 +#: pygeoapi/templates/processes/index.html:2 +#: pygeoapi/templates/processes/index.html:4 +#: pygeoapi/templates/processes/process.html:4 +msgid "Processes" +msgstr "العمليات" + +#: build/lib/pygeoapi/templates/landing_page.html:66 +#: pygeoapi/templates/landing_page.html:75 +msgid "View the processes in this service" +msgstr "عرض العمليات في هذه الخدمة" + +#: build/lib/pygeoapi/templates/jobs/index.html:2 +#: build/lib/pygeoapi/templates/jobs/index.html:4 +#: build/lib/pygeoapi/templates/jobs/index.html:11 +#: build/lib/pygeoapi/templates/jobs/job.html:4 +#: build/lib/pygeoapi/templates/jobs/results/index.html:4 +#: build/lib/pygeoapi/templates/landing_page.html:70 +#: build/lib/pygeoapi/templates/processes/process.html:76 +#: pygeoapi/templates/jobs/index.html:2 pygeoapi/templates/jobs/index.html:4 +#: pygeoapi/templates/jobs/index.html:11 pygeoapi/templates/jobs/job.html:4 +#: pygeoapi/templates/jobs/results/index.html:4 +#: pygeoapi/templates/landing_page.html:79 +#: pygeoapi/templates/processes/process.html:76 +msgid "Jobs" +msgstr "المهام" + +#: build/lib/pygeoapi/templates/landing_page.html:72 +#: build/lib/pygeoapi/templates/processes/process.html:77 +#: pygeoapi/templates/landing_page.html:81 +#: pygeoapi/templates/processes/process.html:77 +msgid "Browse jobs" +msgstr "تصفح المهام" + +#: build/lib/pygeoapi/templates/landing_page.html:77 +#: pygeoapi/templates/landing_page.html:86 +msgid "API Definition" +msgstr "تعريف API" + +#: build/lib/pygeoapi/templates/landing_page.html:79 +#: pygeoapi/templates/landing_page.html:88 +msgid "Documentation" +msgstr "التوثيق" + +#: build/lib/pygeoapi/templates/landing_page.html:79 +#: pygeoapi/templates/landing_page.html:88 +msgid "Swagger UI" +msgstr "واجهة Swagger" + +#: build/lib/pygeoapi/templates/landing_page.html:79 +#: pygeoapi/templates/landing_page.html:88 +msgid "ReDoc" +msgstr "ReDoc" + +#: build/lib/pygeoapi/templates/landing_page.html:82 +#: pygeoapi/templates/landing_page.html:91 +msgid "OpenAPI Document" +msgstr "وثيقة OpenAPI" + +#: build/lib/pygeoapi/templates/landing_page.html:88 +#: pygeoapi/templates/landing_page.html:97 +msgid "View the conformance classes of this service" +msgstr "عرض فئات التوافق لهذه الخدمة" + +#: build/lib/pygeoapi/templates/landing_page.html:95 +#: pygeoapi/templates/landing_page.html:110 +msgid "Provider" +msgstr "المزود" + +#: build/lib/pygeoapi/templates/landing_page.html:104 +#: pygeoapi/templates/landing_page.html:119 +msgid "Contact point" +msgstr "نقطة الاتصال" + +#: build/lib/pygeoapi/templates/landing_page.html:107 +#: pygeoapi/templates/landing_page.html:122 +msgid "Address" +msgstr "العنوان" + +#: build/lib/pygeoapi/templates/landing_page.html:116 +#: pygeoapi/templates/landing_page.html:131 +msgid "Email" +msgstr "البريد الإلكتروني" + +#: build/lib/pygeoapi/templates/landing_page.html:119 +#: pygeoapi/templates/landing_page.html:134 +msgid "Telephone" +msgstr "الهاتف" + +#: build/lib/pygeoapi/templates/landing_page.html:123 +#: pygeoapi/templates/landing_page.html:138 +msgid "Fax" +msgstr "الفاكس" + +#: build/lib/pygeoapi/templates/landing_page.html:127 +#: pygeoapi/templates/landing_page.html:142 +msgid "Contact URL" +msgstr "رابط الاتصال" + +#: build/lib/pygeoapi/templates/landing_page.html:131 +#: pygeoapi/templates/landing_page.html:146 +msgid "Hours" +msgstr "ساعات العمل" + +#: build/lib/pygeoapi/templates/landing_page.html:135 +#: pygeoapi/templates/landing_page.html:150 +msgid "Contact instructions" +msgstr "تعليمات الاتصال" + +#: build/lib/pygeoapi/templates/collections/collection.html:51 +#: pygeoapi/templates/collections/collection.html:51 +msgid "Browse" +msgstr "تصفح" + +#: build/lib/pygeoapi/templates/collections/collection.html:55 +#: pygeoapi/templates/collections/collection.html:55 +msgid "Browse Items" +msgstr "تصفح العناصر" + +#: build/lib/pygeoapi/templates/collections/collection.html:56 +#: pygeoapi/templates/collections/collection.html:56 +msgid "Browse through the items of" +msgstr "تصفح عناصر" + +#: build/lib/pygeoapi/templates/collections/collection.html:59 +#: build/lib/pygeoapi/templates/collections/queryables.html:6 +#: build/lib/pygeoapi/templates/collections/queryables.html:17 +#: pygeoapi/templates/collections/collection.html:59 +#: pygeoapi/templates/collections/queryables.html:6 +#: pygeoapi/templates/collections/queryables.html:17 +msgid "Queryables" +msgstr "قابليات الاستعلام" + +#: build/lib/pygeoapi/templates/collections/collection.html:63 +#: pygeoapi/templates/collections/collection.html:63 +msgid "Display Queryables" +msgstr "عرض قابليات الاستعلام" + +#: build/lib/pygeoapi/templates/collections/collection.html:64 +#: pygeoapi/templates/collections/collection.html:64 +msgid "Display Queryables of" +msgstr "عرض قابليات الاستعلام لـ" + +#: build/lib/pygeoapi/templates/collections/collection.html:69 +#: build/lib/pygeoapi/templates/collections/tiles/index.html:6 +#: build/lib/pygeoapi/templates/collections/tiles/metadata.html:6 +#: pygeoapi/templates/collections/collection.html:77 +#: pygeoapi/templates/collections/tiles/index.html:6 +#: pygeoapi/templates/collections/tiles/metadata.html:6 +msgid "Tiles" +msgstr "البلاطات" + +#: build/lib/pygeoapi/templates/collections/collection.html:73 +#: pygeoapi/templates/collections/collection.html:81 +msgid "Display Tiles" +msgstr "عرض البلاطات" + +#: build/lib/pygeoapi/templates/collections/collection.html:73 +#: pygeoapi/templates/collections/collection.html:81 +msgid "Display Tiles of" +msgstr "عرض البلاطات لـ" + +#: build/lib/pygeoapi/templates/collections/collection.html:80 +#: build/lib/pygeoapi/templates/jobs/job.html:50 +#: build/lib/pygeoapi/templates/processes/process.html:78 +#: pygeoapi/templates/collections/collection.html:107 +#: pygeoapi/templates/collections/items/item.html:101 +#: pygeoapi/templates/jobs/job.html:50 +#: pygeoapi/templates/processes/process.html:78 +msgid "Links" +msgstr "الروابط" + +#: build/lib/pygeoapi/templates/collections/collection.html:90 +#: pygeoapi/templates/collections/collection.html:117 +msgid "Reference Systems" +msgstr "أنظمة الإسناد المرجعي" + +#: build/lib/pygeoapi/templates/collections/collection.html:98 +#: pygeoapi/templates/collections/collection.html:125 +msgid "Storage CRS" +msgstr "نظام الإحداثيات المرجعي للتخزين" + +#: build/lib/pygeoapi/templates/collections/index.html:12 +#: build/lib/pygeoapi/templates/processes/index.html:14 +#: build/lib/pygeoapi/templates/stac/catalog.html:17 +#: build/lib/pygeoapi/templates/stac/collection.html:17 +#: pygeoapi/templates/collections/index.html:12 +#: pygeoapi/templates/collections/index.html:39 +#: pygeoapi/templates/processes/index.html:14 +#: pygeoapi/templates/stac/catalog.html:17 +#: pygeoapi/templates/stac/collection.html:17 +msgid "Name" +msgstr "الاسم" + +#: build/lib/pygeoapi/templates/collections/index.html:13 +#: build/lib/pygeoapi/templates/stac/catalog.html:18 +#: pygeoapi/templates/stac/catalog.html:18 +msgid "Type" +msgstr "النوع" + +#: build/lib/pygeoapi/templates/collections/index.html:14 +#: build/lib/pygeoapi/templates/processes/index.html:15 +#: build/lib/pygeoapi/templates/processes/process.html:26 +#: build/lib/pygeoapi/templates/processes/process.html:56 +#: build/lib/pygeoapi/templates/stac/collection.html:18 +#: pygeoapi/templates/collections/index.html:13 +#: pygeoapi/templates/collections/index.html:40 +#: pygeoapi/templates/processes/index.html:15 +#: pygeoapi/templates/processes/process.html:26 +#: pygeoapi/templates/processes/process.html:56 +#: pygeoapi/templates/stac/collection.html:18 +#: pygeoapi/templates/tilematrixsets/index.html:16 +msgid "Description" +msgstr "الوصف" + +#: build/lib/pygeoapi/templates/collections/coverage/domainset.html:11 +msgid "Coverage domain set" +msgstr "مجموعة مجال التغطية" + +#: build/lib/pygeoapi/templates/collections/coverage/domainset.html:12 +msgid "Axis labels" +msgstr "تسميات المحاور" + +#: build/lib/pygeoapi/templates/collections/coverage/domainset.html:18 +msgid "Extent" +msgstr "النطاق" + +#: build/lib/pygeoapi/templates/collections/coverage/domainset.html:24 +msgid "Coordinate reference system" +msgstr "نظام الإحداثيات المرجعي" + +#: build/lib/pygeoapi/templates/collections/coverage/domainset.html:26 +#: build/lib/pygeoapi/templates/stac/catalog.html:20 +#: build/lib/pygeoapi/templates/stac/item.html:34 +#: pygeoapi/templates/stac/catalog.html:20 +#: pygeoapi/templates/stac/collection_base.html:34 +#: pygeoapi/templates/stac/item.html:34 +msgid "Size" +msgstr "الحجم" + +#: build/lib/pygeoapi/templates/collections/coverage/domainset.html:28 +msgid "width" +msgstr "العرض" + +#: build/lib/pygeoapi/templates/collections/coverage/domainset.html:29 +msgid "height" +msgstr "الارتفاع" + +#: build/lib/pygeoapi/templates/collections/coverage/domainset.html:31 +msgid "Resolution" +msgstr "الدقة" + +#: build/lib/pygeoapi/templates/collections/coverage/domainset.html:33 +msgid "x" +msgstr "x" + +#: build/lib/pygeoapi/templates/collections/coverage/domainset.html:34 +msgid "y" +msgstr "y" + +#: build/lib/pygeoapi/templates/collections/coverage/rangetype.html:11 +msgid "Coverage range type" +msgstr "نوع نطاق التغطية" + +#: build/lib/pygeoapi/templates/collections/coverage/rangetype.html:12 +msgid "Fields" +msgstr "الحقول" + +#: build/lib/pygeoapi/templates/collections/edr/query.html:11 +#: build/lib/pygeoapi/templates/collections/items/index.html:11 +#: build/lib/pygeoapi/templates/collections/items/item.html:33 +#: pygeoapi/templates/collections/items/index.html:11 +#: pygeoapi/templates/collections/items/item.html:33 +msgid "Items" +msgstr "العناصر" + +#: build/lib/pygeoapi/templates/collections/items/index.html:25 +#: pygeoapi/templates/collections/items/index.html:38 +#: pygeoapi/templates/collections/items/index.html:142 +msgid "Items in this collection" +msgstr "العناصر في هذه المجموعة" + +#: build/lib/pygeoapi/templates/collections/items/index.html:38 +#: pygeoapi/templates/collections/items/index.html:51 +msgid "Warning: Higher limits not recommended!" +msgstr "تحذير: لا يُنصح بالحدود الأعلى!" + +#: build/lib/pygeoapi/templates/collections/items/index.html:43 +#: pygeoapi/templates/collections/items/index.html:44 +#: pygeoapi/templates/jobs/index.html:53 +msgid "Limit" +msgstr "الحد" + +#: build/lib/pygeoapi/templates/collections/items/index.html:45 +#: pygeoapi/templates/collections/items/index.html:46 +#: pygeoapi/templates/jobs/index.html:55 +msgid "default" +msgstr "افتراضي" + +#: build/lib/pygeoapi/templates/collections/items/index.html:66 +#: build/lib/pygeoapi/templates/collections/items/item.html:62 +#: pygeoapi/templates/collections/items/index.html:68 +#: pygeoapi/templates/collections/items/item.html:62 +#: pygeoapi/templates/jobs/index.html:76 +msgid "Prev" +msgstr "السابق" + +#: build/lib/pygeoapi/templates/collections/items/index.html:68 +#: build/lib/pygeoapi/templates/collections/items/item.html:64 +#: pygeoapi/templates/collections/items/index.html:70 +#: pygeoapi/templates/collections/items/item.html:64 +#: pygeoapi/templates/jobs/index.html:78 +msgid "Next" +msgstr "التالي" + +#: build/lib/pygeoapi/templates/collections/items/index.html:139 +#: pygeoapi/templates/collections/edr/query.html:37 +#: pygeoapi/templates/collections/items/index.html:147 +msgid "No items" +msgstr "لا توجد عناصر" + +#: build/lib/pygeoapi/templates/collections/items/item.html:77 +#: build/lib/pygeoapi/templates/stac/item.html:58 +#: pygeoapi/templates/collections/items/item.html:77 +#: pygeoapi/templates/stac/collection_base.html:58 +#: pygeoapi/templates/stac/item.html:58 +msgid "Property" +msgstr "الخاصية" + +#: build/lib/pygeoapi/templates/collections/items/item.html:78 +#: build/lib/pygeoapi/templates/stac/item.html:59 +#: pygeoapi/templates/collections/items/item.html:78 +#: pygeoapi/templates/stac/collection_base.html:59 +#: pygeoapi/templates/stac/item.html:59 +msgid "Value" +msgstr "القيمة" + +#: build/lib/pygeoapi/templates/collections/tiles/index.html:31 +#: pygeoapi/templates/collections/tiles/index.html:31 +msgid "Tile Matrix Set" +msgstr "مجموعة مصفوفة البلاطات" + +#: build/lib/pygeoapi/templates/collections/tiles/index.html:43 +#: pygeoapi/templates/collections/tiles/index.html:42 +msgid "Metadata" +msgstr "البيانات الوصفية" + +#: build/lib/pygeoapi/templates/collections/tiles/metadata.html:18 +msgid "Tiles metadata" +msgstr "بيانات وصفية للبلاطات" + +#: build/lib/pygeoapi/templates/collections/tiles/metadata.html:18 +msgid "format" +msgstr "التنسيق" + +#: build/lib/pygeoapi/templates/collections/tiles/metadata.html:19 +msgid "Tileset" +msgstr "مجموعة البلاطات" + +#: build/lib/pygeoapi/templates/jobs/index.html:14 +#: pygeoapi/templates/jobs/index.html:14 +msgid "Job ID" +msgstr "معرف المهمة" + +#: build/lib/pygeoapi/templates/jobs/index.html:15 +#: pygeoapi/templates/jobs/index.html:15 +msgid "Process ID" +msgstr "معرف العملية" + +#: build/lib/pygeoapi/templates/jobs/index.html:16 +#: pygeoapi/templates/jobs/index.html:16 +msgid "Start" +msgstr "بداية" + +#: build/lib/pygeoapi/templates/jobs/index.html:17 +#: build/lib/pygeoapi/templates/jobs/job.html:37 +#: pygeoapi/templates/jobs/index.html:17 pygeoapi/templates/jobs/job.html:37 +msgid "Duration" +msgstr "المدة" + +#: build/lib/pygeoapi/templates/jobs/index.html:18 +#: build/lib/pygeoapi/templates/jobs/job.html:17 +#: build/lib/pygeoapi/templates/jobs/job.html:35 +#: pygeoapi/templates/jobs/index.html:18 pygeoapi/templates/jobs/job.html:17 +#: pygeoapi/templates/jobs/job.html:35 +msgid "Progress" +msgstr "التقدم" + +#: build/lib/pygeoapi/templates/jobs/index.html:19 +#: build/lib/pygeoapi/templates/jobs/job.html:16 +#: pygeoapi/templates/jobs/index.html:19 pygeoapi/templates/jobs/job.html:16 +msgid "Status" +msgstr "الحالة" + +#: build/lib/pygeoapi/templates/jobs/index.html:20 +#: build/lib/pygeoapi/templates/jobs/job.html:21 +#: pygeoapi/templates/jobs/index.html:20 pygeoapi/templates/jobs/job.html:21 +msgid "Message" +msgstr "الرسالة" + +#: build/lib/pygeoapi/templates/jobs/job.html:2 +#: build/lib/pygeoapi/templates/jobs/job.html:10 +#: pygeoapi/templates/jobs/job.html:2 pygeoapi/templates/jobs/job.html:10 +msgid "Job status" +msgstr "حالة المهمة" + +#: build/lib/pygeoapi/templates/jobs/job.html:26 +#: pygeoapi/templates/jobs/job.html:26 +msgid "Parameters" +msgstr "المعلمات" + +#: build/lib/pygeoapi/templates/jobs/job.html:45 +#: pygeoapi/templates/jobs/job.html:45 +msgid "Started processing" +msgstr "بدأت المعالجة" + +#: build/lib/pygeoapi/templates/jobs/job.html:47 +#: pygeoapi/templates/jobs/job.html:47 +msgid "Finished processing" +msgstr "انتهت المعالجة" + +#: build/lib/pygeoapi/templates/jobs/results/index.html:2 +#: pygeoapi/templates/jobs/results/index.html:2 +msgid "Job result" +msgstr "نتيجة المهمة" + +#: build/lib/pygeoapi/templates/jobs/results/index.html:6 +#: pygeoapi/templates/jobs/results/index.html:6 +msgid "Results" +msgstr "النتائج" + +#: build/lib/pygeoapi/templates/jobs/results/index.html:10 +#: pygeoapi/templates/jobs/results/index.html:10 +msgid "Results of job" +msgstr "نتائج المهمة" + +#: build/lib/pygeoapi/templates/processes/index.html:8 +#: pygeoapi/templates/processes/index.html:8 +msgid "Processes in this service" +msgstr "العمليات في هذه الخدمة" + +#: build/lib/pygeoapi/templates/processes/process.html:20 +#: pygeoapi/templates/processes/process.html:20 +msgid "Inputs" +msgstr "المدخلات" + +#: build/lib/pygeoapi/templates/processes/process.html:23 +#: build/lib/pygeoapi/templates/processes/process.html:54 +#: pygeoapi/templates/processes/process.html:23 +#: pygeoapi/templates/processes/process.html:54 +msgid "Id" +msgstr "معرف" + +#: build/lib/pygeoapi/templates/processes/process.html:24 +#: build/lib/pygeoapi/templates/processes/process.html:55 +#: pygeoapi/templates/processes/process.html:24 +#: pygeoapi/templates/processes/process.html:55 +#: pygeoapi/templates/tilematrixsets/index.html:15 +msgid "Title" +msgstr "العنوان" + +#: build/lib/pygeoapi/templates/processes/process.html:25 +#: pygeoapi/templates/processes/process.html:25 +msgid "Data Type" +msgstr "نوع البيانات" + +#: build/lib/pygeoapi/templates/processes/process.html:51 +#: pygeoapi/templates/processes/process.html:51 +msgid "Outputs" +msgstr "المخرجات" + +#: build/lib/pygeoapi/templates/processes/process.html:71 +#: pygeoapi/templates/processes/process.html:71 +msgid "Execution modes" +msgstr "أنماط التنفيذ" + +#: build/lib/pygeoapi/templates/processes/process.html:73 +#: pygeoapi/templates/processes/process.html:73 +msgid "Synchronous" +msgstr "متزامن" + +#: build/lib/pygeoapi/templates/processes/process.html:74 +#: pygeoapi/templates/processes/process.html:74 +msgid "Asynchronous" +msgstr "غير متزامن" + +#: build/lib/pygeoapi/templates/stac/catalog.html:4 +#: build/lib/pygeoapi/templates/stac/collection.html:2 +#: build/lib/pygeoapi/templates/stac/collection.html:4 +#: build/lib/pygeoapi/templates/stac/item.html:4 +#: pygeoapi/templates/stac/catalog.html:4 +#: pygeoapi/templates/stac/collection.html:2 +#: pygeoapi/templates/stac/collection.html:4 +#: pygeoapi/templates/stac/collection_base.html:4 +#: pygeoapi/templates/stac/item.html:4 +msgid "SpatioTemporal Asset Catalog" +msgstr "كتالوج الأصول الزمانية والمكانية (STAC)" + +#: build/lib/pygeoapi/templates/stac/catalog.html:19 +#: pygeoapi/templates/stac/catalog.html:19 +msgid "Last modified" +msgstr "آخر تعديل" + +#: build/lib/pygeoapi/templates/stac/collection.html:9 +#: pygeoapi/templates/stac/collection.html:9 +msgid "STAC Version" +msgstr "إصدار STAC" + +#: build/lib/pygeoapi/templates/stac/item.html:19 +#: pygeoapi/templates/stac/item.html:19 +msgid "Item" +msgstr "العنصر" + +#: build/lib/pygeoapi/templates/stac/item.html:28 +#: pygeoapi/templates/stac/collection_base.html:28 +#: pygeoapi/templates/stac/item.html:28 +msgid "Assets" +msgstr "الأصول" + +#: build/lib/pygeoapi/templates/stac/item.html:32 +#: pygeoapi/templates/landing_page.html:45 +#: pygeoapi/templates/stac/collection_base.html:32 +#: pygeoapi/templates/stac/item.html:32 +msgid "URL" +msgstr "الرابط" + +#: build/lib/pygeoapi/templates/stac/item.html:33 +#: pygeoapi/templates/stac/collection_base.html:33 +#: pygeoapi/templates/stac/item.html:33 +msgid "Last Modified" +msgstr "آخر تعديل" + +#: build/lib/pygeoapi/templates/stac/item.html:64 +#: pygeoapi/templates/stac/collection_base.html:64 +#: pygeoapi/templates/stac/item.html:64 +msgid "id" +msgstr "معرف" + +#: pygeoapi/templates/_base.html:2 +msgid "text_direction" +msgstr "rtl" + +#: pygeoapi/templates/_base.html:47 +msgid "Contact" +msgstr "الاتصال" + +#: pygeoapi/templates/_base.html:51 +msgid "Admin" +msgstr "الإدارة" + +#: pygeoapi/templates/landing_page.html:101 +msgid "Tile Matrix Sets" +msgstr "مجموعات مصفوفة البلاطات" + +#: pygeoapi/templates/landing_page.html:103 +msgid "View the Tile Matrix Sets available on this service" +msgstr "عرض مجموعات مصفوفة البلاطات المتاحة في هذه الخدمة" + +#: pygeoapi/templates/collections/collection.html:67 +#: pygeoapi/templates/collections/schema.html:6 +#: pygeoapi/templates/collections/schema.html:17 +msgid "Schema" +msgstr "المخطط" + +#: pygeoapi/templates/collections/collection.html:71 +msgid "Display Schema" +msgstr "عرض المخطط" + +#: pygeoapi/templates/collections/collection.html:72 +msgid "Display Schema of" +msgstr "عرض مخطط لـ" + +#: pygeoapi/templates/collections/collection.html:128 +msgid "CRS" +msgstr "CRS" + +#: pygeoapi/templates/collections/collection.html:131 +msgid "Epoch" +msgstr "العصر" + +#: pygeoapi/templates/collections/index.html:8 +msgid "Data collections in this service" +msgstr "مجموعات البيانات في هذه الخدمة" + +#: pygeoapi/templates/collections/index.html:35 +msgid "Record collections in this service" +msgstr "مجموعات السجلات في هذه الخدمة" + +#: pygeoapi/templates/collections/edr/query.html:11 +#, python-format +msgid "%(query_type)s" +msgstr "%(query_type)s" + +#: pygeoapi/templates/collections/tiles/metadata.html:15 +msgid "TileJSON" +msgstr "TileJSON" + +#: pygeoapi/templates/collections/tiles/metadata.html:17 +msgid "JSON" +msgstr "JSON" + +#: pygeoapi/templates/stac/collection_base.html:68 +msgid "description" +msgstr "الوصف" + +#: pygeoapi/templates/stac/collection_base.html:72 +msgid "extent" +msgstr "النطاق" + +#: pygeoapi/templates/stac/collection_base.html:77 +msgid "cube:dimensions" +msgstr "cube:dimensions" + +#: pygeoapi/templates/stac/collection_base.html:83 +msgid "cube:variables" +msgstr "cube:variables" + +#: pygeoapi/templates/tilematrixsets/index.html:2 +#: pygeoapi/templates/tilematrixsets/index.html:4 +#: pygeoapi/templates/tilematrixsets/tilematrixset.html:4 +msgid "TileMatrixSets" +msgstr "مجموعات مصفوفة البلاطات" + +#: pygeoapi/templates/tilematrixsets/index.html:9 +msgid "Tile matrix sets available in this service" +msgstr "مجموعات مصفوفة البلاطات المتاحة في هذه الخدمة" + +#: pygeoapi/templates/tilematrixsets/tilematrixset.html:2 +msgid "TileMatrixSet" +msgstr "مجموعة مصفوفة البلاطات" diff --git a/locale/bs/LC_MESSAGES/messages.po b/locale/bs/LC_MESSAGES/messages.po new file mode 100644 index 000000000..15d33b9f8 --- /dev/null +++ b/locale/bs/LC_MESSAGES/messages.po @@ -0,0 +1,686 @@ +# Bosnian translations for PROJECT. +# Copyright (C) 2024 ORGANIZATION +# This file is distributed under the same license as the PROJECT project. +# FIRST AUTHOR , 2024. +# +msgid "" +msgstr "" +"Project-Id-Version: PROJECT VERSION\n" +"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" +"POT-Creation-Date: 2024-02-27 14:39+0100\n" +"PO-Revision-Date: 2024-02-27 14:40+0100\n" +"Last-Translator: Bojan Kuzmanović \n" +"Language: bs\n" +"Language-Team: bs \n" +"Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && " +"n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.14.0\n" + +#: pygeoapi/templates/_base.html:2 +msgid "text_direction" +msgstr "ltr" + +#: pygeoapi/templates/_base.html:51 +msgid "Admin" +msgstr "Admin" + +#: pygeoapi/templates/_base.html:67 pygeoapi/templates/landing_page.html:2 +msgid "Home" +msgstr "Početna" + +#: pygeoapi/templates/_base.html:75 pygeoapi/templates/_base.html:83 +msgid "json" +msgstr "json" + +#: pygeoapi/templates/_base.html:78 pygeoapi/templates/_base.html:86 +msgid "jsonld" +msgstr "jsonld" + +#: pygeoapi/templates/_base.html:105 +msgid "Powered by " +msgstr "Pokreće ga" + +#: pygeoapi/templates/conformance.html:2 pygeoapi/templates/conformance.html:4 +#: pygeoapi/templates/conformance.html:8 +#: pygeoapi/templates/landing_page.html:86 +msgid "Conformance" +msgstr "Usklađenost" + +#: pygeoapi/templates/exception.html:2 pygeoapi/templates/exception.html:5 +msgid "Exception" +msgstr "Izuzetak" + +#: pygeoapi/templates/landing_page.html:25 +msgid "Terms of service" +msgstr "Uslovi korišćenja" + +#: pygeoapi/templates/collections/collection.html:38 +#: pygeoapi/templates/landing_page.html:35 +msgid "License" +msgstr "Licensa" + +#: pygeoapi/templates/collections/collection.html:6 +#: pygeoapi/templates/collections/coverage/domainset.html:4 +#: pygeoapi/templates/collections/coverage/rangetype.html:4 +#: pygeoapi/templates/collections/edr/query.html:4 +#: pygeoapi/templates/collections/index.html:2 +#: pygeoapi/templates/collections/index.html:4 +#: pygeoapi/templates/collections/items/index.html:4 +#: pygeoapi/templates/collections/items/item.html:27 +#: pygeoapi/templates/collections/queryables.html:4 +#: pygeoapi/templates/collections/tiles/index.html:4 +#: pygeoapi/templates/collections/tiles/metadata.html:4 +#: pygeoapi/templates/landing_page.html:48 +#: pygeoapi/templates/stac/collection_base.html:19 +msgid "Collections" +msgstr "Kolekcije" + +#: pygeoapi/templates/landing_page.html:50 +msgid "View the collections in this service" +msgstr "Pogledajte kolekcije u ovoj usluzi" + +#: pygeoapi/templates/landing_page.html:56 +msgid "SpatioTemporal Assets" +msgstr "Prostorno-vremenski resursi" + +#: pygeoapi/templates/landing_page.html:58 +msgid "View the SpatioTemporal Assets in this service" +msgstr "Pretražite prostorno-vremenske resurse u ovoj usluzi" + +#: pygeoapi/templates/landing_page.html:64 +#: pygeoapi/templates/processes/index.html:2 +#: pygeoapi/templates/processes/index.html:4 +#: pygeoapi/templates/processes/process.html:4 +msgid "Processes" +msgstr "Procesi" + +#: pygeoapi/templates/landing_page.html:66 +msgid "View the processes in this service" +msgstr "Pogledajte procese u ovoj usluzi" + +#: pygeoapi/templates/jobs/index.html:2 pygeoapi/templates/jobs/index.html:4 +#: pygeoapi/templates/jobs/index.html:11 pygeoapi/templates/jobs/job.html:4 +#: pygeoapi/templates/jobs/results/index.html:4 +#: pygeoapi/templates/landing_page.html:70 +#: pygeoapi/templates/processes/process.html:76 +msgid "Jobs" +msgstr "Zadaci" + +#: pygeoapi/templates/landing_page.html:72 +#: pygeoapi/templates/processes/process.html:77 +msgid "Browse jobs" +msgstr "Pregledajte zadatke" + +#: pygeoapi/templates/landing_page.html:77 +msgid "API Definition" +msgstr "API definicija" + +#: pygeoapi/templates/landing_page.html:79 +msgid "Documentation" +msgstr "Dokumentacija" + +#: pygeoapi/templates/landing_page.html:79 +msgid "Swagger UI" +msgstr "Swagger UI" + +#: pygeoapi/templates/landing_page.html:79 +msgid "ReDoc" +msgstr "ReDoc" + +#: pygeoapi/templates/landing_page.html:82 +msgid "OpenAPI Document" +msgstr "OpenAPI dokument" + +#: pygeoapi/templates/landing_page.html:88 +msgid "View the conformance classes of this service" +msgstr "Poglejte klase usklađenosti ove usluge" + +#: pygeoapi/templates/landing_page.html:92 +msgid "Tile Matrix Sets" +msgstr "Matrica tile-ova" + +#: pygeoapi/templates/landing_page.html:94 +msgid "View the Tile Matrix Sets available on this service" +msgstr "Pogledajte matrice tile-ova dostupne u ovoj usluzi" + +#: pygeoapi/templates/landing_page.html:101 +msgid "Provider" +msgstr "Provajder" + +#: pygeoapi/templates/landing_page.html:110 +msgid "Contact point" +msgstr "Kontakt tačka" + +#: pygeoapi/templates/landing_page.html:113 +msgid "Address" +msgstr "Adresa" + +#: pygeoapi/templates/landing_page.html:122 +msgid "Email" +msgstr "Email" + +#: pygeoapi/templates/landing_page.html:125 +msgid "Telephone" +msgstr "Telefon" + +#: pygeoapi/templates/landing_page.html:129 +msgid "Fax" +msgstr "Fax" + +#: pygeoapi/templates/landing_page.html:133 +msgid "Contact URL" +msgstr "Konktakt URL" + +#: pygeoapi/templates/landing_page.html:137 +msgid "Hours" +msgstr "Sati" + +#: pygeoapi/templates/landing_page.html:141 +msgid "Contact instructions" +msgstr "Konktakt instrukcije" + +#: pygeoapi/templates/collections/collection.html:51 +msgid "Browse" +msgstr "Pretražite" + +#: pygeoapi/templates/collections/collection.html:55 +msgid "Browse Items" +msgstr "Pretražite stavke" + +#: pygeoapi/templates/collections/collection.html:56 +msgid "Browse through the items of" +msgstr "Pretražite kroz stavke od" + +#: pygeoapi/templates/collections/collection.html:59 +#: pygeoapi/templates/collections/queryables.html:6 +#: pygeoapi/templates/collections/queryables.html:17 +msgid "Queryables" +msgstr "Upitni" + +#: pygeoapi/templates/collections/collection.html:63 +msgid "Display Queryables" +msgstr "Prikaži upitne" + +#: pygeoapi/templates/collections/collection.html:64 +msgid "Display Queryables of" +msgstr "Prikaži upitne od" + +#: pygeoapi/templates/collections/collection.html:69 +#: pygeoapi/templates/collections/tiles/index.html:6 +#: pygeoapi/templates/collections/tiles/metadata.html:6 +msgid "Tiles" +msgstr "Tile-ovi" + +#: pygeoapi/templates/collections/collection.html:73 +msgid "Display Tiles" +msgstr "Prikaži tile-ove" + +#: pygeoapi/templates/collections/collection.html:73 +msgid "Display Tiles of" +msgstr "Prikaži tile-ove od" + +#: pygeoapi/templates/collections/collection.html:80 +#: pygeoapi/templates/jobs/job.html:50 +#: pygeoapi/templates/processes/process.html:78 +msgid "Links" +msgstr "Linkovi" + +#: pygeoapi/templates/collections/collection.html:90 +msgid "Reference Systems" +msgstr "Referentni sistemi" + +#: pygeoapi/templates/collections/collection.html:98 +msgid "Storage CRS" +msgstr "Skladišni CRS" + +#: pygeoapi/templates/collections/index.html:12 +#: pygeoapi/templates/processes/index.html:14 +#: pygeoapi/templates/stac/catalog.html:17 +#: pygeoapi/templates/stac/collection.html:17 +msgid "Name" +msgstr "Naziv" + +#: pygeoapi/templates/collections/index.html:13 +#: pygeoapi/templates/stac/catalog.html:18 +msgid "Type" +msgstr "Tip" + +#: pygeoapi/templates/collections/index.html:14 +#: pygeoapi/templates/processes/index.html:15 +#: pygeoapi/templates/processes/process.html:26 +#: pygeoapi/templates/processes/process.html:56 +#: pygeoapi/templates/stac/collection.html:18 +#: pygeoapi/templates/tilematrixsets/index.html:16 +msgid "Description" +msgstr "Opis" + +#: pygeoapi/templates/collections/coverage/domainset.html:11 +msgid "Coverage domain set" +msgstr "Pokrivenost domenskog skupa" + +#: pygeoapi/templates/collections/coverage/domainset.html:12 +msgid "Axis labels" +msgstr "Labele osa" + +#: pygeoapi/templates/collections/coverage/domainset.html:18 +msgid "Extent" +msgstr "Extent" + +#: pygeoapi/templates/collections/coverage/domainset.html:24 +msgid "Coordinate reference system" +msgstr "Koordinate referentnog sistema" + +#: pygeoapi/templates/collections/coverage/domainset.html:26 +#: pygeoapi/templates/stac/catalog.html:20 +#: pygeoapi/templates/stac/collection_base.html:34 +#: pygeoapi/templates/stac/item.html:34 +msgid "Size" +msgstr "Veličina" + +#: pygeoapi/templates/collections/coverage/domainset.html:28 +msgid "width" +msgstr "širina" + +#: pygeoapi/templates/collections/coverage/domainset.html:29 +msgid "height" +msgstr "visina" + +#: pygeoapi/templates/collections/coverage/domainset.html:31 +msgid "Resolution" +msgstr "Rezolucija" + +#: pygeoapi/templates/collections/coverage/domainset.html:33 +msgid "x" +msgstr "x" + +#: pygeoapi/templates/collections/coverage/domainset.html:34 +msgid "y" +msgstr "y" + +#: pygeoapi/templates/collections/coverage/rangetype.html:11 +msgid "Coverage range type" +msgstr "Pokrivenost tipa opsega" + +#: pygeoapi/templates/collections/coverage/rangetype.html:12 +msgid "Fields" +msgstr "Polja" + +#: pygeoapi/templates/collections/edr/query.html:11 +#: pygeoapi/templates/collections/items/index.html:11 +#: pygeoapi/templates/collections/items/item.html:33 +msgid "Items" +msgstr "Stavke" + +#: pygeoapi/templates/collections/items/index.html:25 +msgid "Items in this collection" +msgstr "Stavke u ovoj kolekciji" + +#: pygeoapi/templates/collections/items/index.html:38 +msgid "Warning: Higher limits not recommended!" +msgstr "Upozorenje: Veće granice nisu preporučljivi!" + +#: pygeoapi/templates/collections/items/index.html:43 +msgid "Limit" +msgstr "Granica" + +#: pygeoapi/templates/collections/items/index.html:45 +msgid "default" +msgstr "podrazumijevano" + +#: pygeoapi/templates/collections/items/index.html:66 +#: pygeoapi/templates/collections/items/item.html:62 +msgid "Prev" +msgstr "Prethodno" + +#: pygeoapi/templates/collections/items/index.html:68 +#: pygeoapi/templates/collections/items/item.html:64 +msgid "Next" +msgstr "Sljedeće" + +#: pygeoapi/templates/collections/items/index.html:139 +msgid "No items" +msgstr "Nema stavki" + +#: pygeoapi/templates/collections/items/item.html:77 +#: pygeoapi/templates/stac/collection_base.html:58 +#: pygeoapi/templates/stac/item.html:58 +msgid "Property" +msgstr "Svojstvo" + +#: pygeoapi/templates/collections/items/item.html:78 +#: pygeoapi/templates/stac/collection_base.html:59 +#: pygeoapi/templates/stac/item.html:59 +msgid "Value" +msgstr "Vrijednost" + +#: pygeoapi/templates/collections/tiles/index.html:31 +msgid "Tile Matrix Set" +msgstr "Matrica tile-ova" + +#: pygeoapi/templates/collections/tiles/index.html:42 +msgid "Metadata" +msgstr "Meta podaci" + +#: pygeoapi/templates/collections/tiles/metadata.html:15 +msgid "TileJSON" +msgstr "TileJSON" + +#: pygeoapi/templates/collections/tiles/metadata.html:17 +msgid "JSON" +msgstr "JSON" + +#: pygeoapi/templates/jobs/index.html:14 +msgid "Job ID" +msgstr "ID zadatka" + +#: pygeoapi/templates/jobs/index.html:15 +msgid "Process ID" +msgstr "ID procesa" + +#: pygeoapi/templates/jobs/index.html:16 +msgid "Start" +msgstr "Početak" + +#: pygeoapi/templates/jobs/index.html:17 pygeoapi/templates/jobs/job.html:37 +msgid "Duration" +msgstr "Trajanje" + +#: pygeoapi/templates/jobs/index.html:18 pygeoapi/templates/jobs/job.html:17 +#: pygeoapi/templates/jobs/job.html:35 +msgid "Progress" +msgstr "Napredak" + +#: pygeoapi/templates/jobs/index.html:19 pygeoapi/templates/jobs/job.html:16 +msgid "Status" +msgstr "Status" + +#: pygeoapi/templates/jobs/index.html:20 pygeoapi/templates/jobs/job.html:21 +msgid "Message" +msgstr "Poruka" + +#: pygeoapi/templates/jobs/job.html:2 pygeoapi/templates/jobs/job.html:10 +msgid "Job status" +msgstr "Status zadatka" + +#: pygeoapi/templates/jobs/job.html:26 +msgid "Parameters" +msgstr "Parametri" + +#: pygeoapi/templates/jobs/job.html:45 +msgid "Started processing" +msgstr "Započeto procesiranje" + +#: pygeoapi/templates/jobs/job.html:47 +msgid "Finished processing" +msgstr "Završeno procesiranje" + +#: pygeoapi/templates/jobs/results/index.html:2 +msgid "Job result" +msgstr "Rezuultat zadatka" + +#: pygeoapi/templates/jobs/results/index.html:6 +msgid "Results" +msgstr "Rezuultati" + +#: pygeoapi/templates/jobs/results/index.html:10 +msgid "Results of job" +msgstr "Rezultati zadatka" + +#: pygeoapi/templates/processes/index.html:8 +msgid "Processes in this service" +msgstr "Procesi u ovoj usluzi" + +#: pygeoapi/templates/processes/process.html:20 +msgid "Inputs" +msgstr "Ulazi" + +#: pygeoapi/templates/processes/process.html:23 +#: pygeoapi/templates/processes/process.html:54 +msgid "Id" +msgstr "Id" + +#: pygeoapi/templates/processes/process.html:24 +#: pygeoapi/templates/processes/process.html:55 +#: pygeoapi/templates/tilematrixsets/index.html:15 +msgid "Title" +msgstr "Naslov" + +#: pygeoapi/templates/processes/process.html:25 +msgid "Data Type" +msgstr "Vrst podataka" + +#: pygeoapi/templates/processes/process.html:51 +msgid "Outputs" +msgstr "Izlazi" + +#: pygeoapi/templates/processes/process.html:71 +msgid "Execution modes" +msgstr "Modovi izvršavanja" + +#: pygeoapi/templates/processes/process.html:73 +msgid "Synchronous" +msgstr "Sinhroni" + +#: pygeoapi/templates/processes/process.html:74 +msgid "Asynchronous" +msgstr "Asinhroni" + +#: pygeoapi/templates/stac/catalog.html:4 +#: pygeoapi/templates/stac/collection.html:2 +#: pygeoapi/templates/stac/collection.html:4 +#: pygeoapi/templates/stac/collection_base.html:4 +#: pygeoapi/templates/stac/item.html:4 +msgid "SpatioTemporal Asset Catalog" +msgstr "Prostorno-vremenski katalog resursa" + +#: pygeoapi/templates/stac/catalog.html:19 +msgid "Last modified" +msgstr "Zadnji put izmijenjeno" + +#: pygeoapi/templates/stac/collection.html:9 +msgid "STAC Version" +msgstr "STAC verzija" + +#: pygeoapi/templates/stac/collection_base.html:28 +#: pygeoapi/templates/stac/item.html:28 +msgid "Assets" +msgstr "Resursi" + +#: pygeoapi/templates/stac/collection_base.html:32 +#: pygeoapi/templates/stac/item.html:32 +msgid "URL" +msgstr "URL" + +#: pygeoapi/templates/stac/collection_base.html:33 +#: pygeoapi/templates/stac/item.html:33 +msgid "Last Modified" +msgstr "Zadnji put izmijenjeno" + +#: pygeoapi/templates/stac/collection_base.html:64 +#: pygeoapi/templates/stac/item.html:64 +msgid "id" +msgstr "id" + +#: pygeoapi/templates/stac/collection_base.html:68 +msgid "description" +msgstr "opis" + +#: pygeoapi/templates/stac/collection_base.html:72 +msgid "extent" +msgstr "extent" + +#: pygeoapi/templates/stac/collection_base.html:77 +msgid "cube:dimensions" +msgstr "kocka:dimenzije" + +#: pygeoapi/templates/stac/collection_base.html:83 +msgid "cube:variables" +msgstr "kocka:varijable" + +#: pygeoapi/templates/stac/item.html:19 +msgid "Item" +msgstr "Stavka" + +#: pygeoapi/templates/tilematrixsets/index.html:2 +#: pygeoapi/templates/tilematrixsets/index.html:4 +msgid "TileMatrixSets" +msgstr "" + +#: pygeoapi/templates/tilematrixsets/index.html:9 +msgid "Tile matrix sets available in this service" +msgstr "Matrica tile-ova dostupna u ovoj usluzi" + +#: pygeoapi/templates/tilematrixsets/tilematrixset.html:2 +#: pygeoapi/templates/tilematrixsets/tilematrixset.html:4 +msgid "TileMatrixSet" +msgstr "Matrica tile-ova" + +msgid "This document as JSON" +msgstr "" + +msgid "This document as RDF (JSON-LD)" +msgstr "" + +msgid "This document as HTML" +msgstr "" + +msgid "The OpenAPI definition as JSON" +msgstr "" + +msgid "The list of supported tiling schemes as JSON" +msgstr "" + +msgid "The list of supported tiling schemes as HTML" +msgstr "" + +msgid "The landing page of this server as JSON" +msgstr "" + +msgid "The landing page of this server as HTML" +msgstr "" + +msgid "Schema of collection in JSON" +msgstr "" + +msgid "Schema of collection in HTML" +msgstr "" + +msgid "Queryables for this collection as JSON" +msgstr "" + +msgid "Queryables for this collection as HTML" +msgstr "" + +msgid "Items as GeoJSON" +msgstr "" + +msgid "Items as HTML" +msgstr "" + +msgid "Items as RDF (GeoJSON-LD)" +msgstr "" + +msgid "Coverage data" +msgstr "" + +msgid "Coverage data as" +msgstr "" + +msgid "Tiles as HTML" +msgstr "" + +msgid "Tiles as JSON" +msgstr "" + +msgid "query for this collection as HTML" +msgstr "" + +msgid "query for this collection as JSON" +msgstr "" + +msgid "Items (prev)" +msgstr "" + +msgid "Items (next)" +msgstr "" + +msgid "Process description as JSON" +msgstr "" + +msgid "Process description as HTML" +msgstr "" + +msgid "query for this collection as HTML" +msgstr "" + +msgid "Execution for process as JSON" +msgstr "" + +msgid "Jobs list as JSON" +msgstr "" + +msgid "Jobs list as HTML" +msgstr "" + +msgid "Results of job as JSON" +msgstr "" + +msgid "Results of job as HTML" +msgstr "" + +msgid "The job list for the current process" +msgstr "" + +msgid "TileMatrixSet definition in JSON" +msgstr "" + +msgid "Data collections in this service" +msgstr "" + +msgid "Record collections in this service" +msgstr "" + +msgid "Display Schema" +msgstr "" + +msgid "Display Schema of" +msgstr "" + +msgid "Contact" +msgstr "" + +msgid "CRS" +msgstr "" + +msgid "Epoch" +msgstr "" + +msgid "not specified" +msgstr "" + +msgid "Position" +msgstr "" + +msgid "Cube" +msgstr "" + +msgid "Area" +msgstr "" + +msgid "Corridor" +msgstr "" + +msgid "Trajectory" +msgstr "" + +msgid "Radius" +msgstr "" + +msgid "Locations" +msgstr "" + +msgid "Instances" +msgstr "" diff --git a/locale/de/LC_MESSAGES/messages.po b/locale/de/LC_MESSAGES/messages.po index a47e6a6d6..3648a2767 100644 --- a/locale/de/LC_MESSAGES/messages.po +++ b/locale/de/LC_MESSAGES/messages.po @@ -18,6 +18,10 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.9.1\n" +#: pygeoapi/templates/_base.html:2 +msgid "text_direction" +msgstr "ltr" + #: build/lib/pygeoapi/templates/_base.html:40 #: build/lib/pygeoapi/templates/landing_page.html:2 #: pygeoapi/templates/_base.html:40 pygeoapi/templates/landing_page.html:2 @@ -584,3 +588,149 @@ msgstr "Ergebnis" msgid "Results of job" msgstr "Ergebnis des Auftrags" +msgid "This document as JSON" +msgstr "" + +msgid "This document as RDF (JSON-LD)" +msgstr "" + +msgid "This document as HTML" +msgstr "" + +msgid "The OpenAPI definition as JSON" +msgstr "" + +msgid "The list of supported tiling schemes as JSON" +msgstr "" + +msgid "The list of supported tiling schemes as HTML" +msgstr "" + +msgid "The landing page of this server as JSON" +msgstr "" + +msgid "The landing page of this server as HTML" +msgstr "" + +msgid "Schema of collection in JSON" +msgstr "" + +msgid "Schema of collection in HTML" +msgstr "" + +msgid "Queryables for this collection as JSON" +msgstr "" + +msgid "Queryables for this collection as HTML" +msgstr "" + +msgid "Items as GeoJSON" +msgstr "" + +msgid "Items as HTML" +msgstr "" + +msgid "Items as RDF (GeoJSON-LD)" +msgstr "" + +msgid "Coverage data" +msgstr "" + +msgid "Coverage data as" +msgstr "" + +msgid "Tiles as HTML" +msgstr "" + +msgid "Tiles as JSON" +msgstr "" + +msgid "query for this collection as HTML" +msgstr "" + +msgid "query for this collection as JSON" +msgstr "" + +msgid "Items (prev)" +msgstr "" + +msgid "Items (next)" +msgstr "" + +msgid "Process description as JSON" +msgstr "" + +msgid "Process description as HTML" +msgstr "" + +msgid "query for this collection as HTML" +msgstr "" + +msgid "Execution for process as JSON" +msgstr "" + +msgid "Jobs list as JSON" +msgstr "" + +msgid "Jobs list as HTML" +msgstr "" + +msgid "Results of job as JSON" +msgstr "" + +msgid "Results of job as HTML" +msgstr "" + +msgid "The job list for the current process" +msgstr "" + +msgid "TileMatrixSet definition in JSON" +msgstr "" + +msgid "Data collections in this service" +msgstr "" + +msgid "Record collections in this service" +msgstr "" + +msgid "Display Schema" +msgstr "" + +msgid "Display Schema of" +msgstr "" + +msgid "Contact" +msgstr "" + +msgid "CRS" +msgstr "" + +msgid "Epoch" +msgstr "" + +msgid "not specified" +msgstr "" + +msgid "Position" +msgstr "" + +msgid "Cube" +msgstr "" + +msgid "Area" +msgstr "" + +msgid "Corridor" +msgstr "" + +msgid "Trajectory" +msgstr "" + +msgid "Radius" +msgstr "" + +msgid "Locations" +msgstr "" + +msgid "Instances" +msgstr "" diff --git a/locale/en/LC_MESSAGES/messages.po b/locale/en/LC_MESSAGES/messages.po index e3ed1ce45..17a2698e9 100644 --- a/locale/en/LC_MESSAGES/messages.po +++ b/locale/en/LC_MESSAGES/messages.po @@ -18,6 +18,10 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.9.1\n" +#: pygeoapi/templates/_base.html:2 +msgid "text_direction" +msgstr "ltr" + #: build/lib/pygeoapi/templates/_base.html:40 #: build/lib/pygeoapi/templates/landing_page.html:2 #: pygeoapi/templates/_base.html:40 pygeoapi/templates/landing_page.html:2 @@ -586,3 +590,149 @@ msgstr "" #~ msgid "Powered by" #~ msgstr "" +msgid "This document as JSON" +msgstr "" + +msgid "This document as RDF (JSON-LD)" +msgstr "" + +msgid "This document as HTML" +msgstr "" + +msgid "The OpenAPI definition as JSON" +msgstr "" + +msgid "The list of supported tiling schemes as JSON" +msgstr "" + +msgid "The list of supported tiling schemes as HTML" +msgstr "" + +msgid "The landing page of this server as JSON" +msgstr "" + +msgid "The landing page of this server as HTML" +msgstr "" + +msgid "Schema of collection in JSON" +msgstr "" + +msgid "Schema of collection in HTML" +msgstr "" + +msgid "Queryables for this collection as JSON" +msgstr "" + +msgid "Queryables for this collection as HTML" +msgstr "" + +msgid "Items as GeoJSON" +msgstr "" + +msgid "Items as HTML" +msgstr "" + +msgid "Items as RDF (GeoJSON-LD)" +msgstr "" + +msgid "Coverage data" +msgstr "" + +msgid "Coverage data as" +msgstr "" + +msgid "Tiles as HTML" +msgstr "" + +msgid "Tiles as JSON" +msgstr "" + +msgid "query for this collection as HTML" +msgstr "" + +msgid "query for this collection as JSON" +msgstr "" + +msgid "Items (prev)" +msgstr "" + +msgid "Items (next)" +msgstr "" + +msgid "Process description as JSON" +msgstr "" + +msgid "Process description as HTML" +msgstr "" + +msgid "query for this collection as HTML" +msgstr "" + +msgid "Execution for process as JSON" +msgstr "" + +msgid "Jobs list as JSON" +msgstr "" + +msgid "Jobs list as HTML" +msgstr "" + +msgid "Results of job as JSON" +msgstr "" + +msgid "Results of job as HTML" +msgstr "" + +msgid "The job list for the current process" +msgstr "" + +msgid "TileMatrixSet definition in JSON" +msgstr "" + +msgid "Data collections in this service" +msgstr "" + +msgid "Record collections in this service" +msgstr "" + +msgid "Display Schema" +msgstr "" + +msgid "Display Schema of" +msgstr "" + +msgid "Contact" +msgstr "" + +msgid "CRS" +msgstr "" + +msgid "Epoch" +msgstr "" + +msgid "not specified" +msgstr "" + +msgid "Position" +msgstr "" + +msgid "Cube" +msgstr "" + +msgid "Area" +msgstr "" + +msgid "Corridor" +msgstr "" + +msgid "Trajectory" +msgstr "" + +msgid "Radius" +msgstr "" + +msgid "Locations" +msgstr "" + +msgid "Instances" +msgstr "" diff --git a/locale/es/LC_MESSAGES/messages.po b/locale/es/LC_MESSAGES/messages.po new file mode 100644 index 000000000..36bce5c22 --- /dev/null +++ b/locale/es/LC_MESSAGES/messages.po @@ -0,0 +1,551 @@ +# Spanish translations for PROJECT. +# Copyright (C) 2023 ORGANIZATION +# This file is distributed under the same license as the PROJECT project. +# FIRST AUTHOR , 2023. +# +msgid "" +msgstr "" +"Project-Id-Version: PROJECT VERSION\n" +"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" +"POT-Creation-Date: 2024-06-28 13:12+0200\n" +"PO-Revision-Date: 2024-07-01 13:15+0200\n" +"Last-Translator: IDEE \n" +"Language: de\n" +"Language-Team: de \n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.11.0\n" + +#: pygeoapi/templates/_base.html:2 +msgid "text_direction" +msgstr "ltr" + +#: pygeoapi/templates/_base.html:51 +msgid "Admin" +msgstr "Admin" + +#: pygeoapi/templates/_base.html:67 pygeoapi/templates/landing_page.html:2 +msgid "Home" +msgstr "Inicio" + +#: pygeoapi/templates/_base.html:75 pygeoapi/templates/_base.html:83 +msgid "json" +msgstr "json" + +#: pygeoapi/templates/_base.html:78 pygeoapi/templates/_base.html:86 +msgid "jsonld" +msgstr "jsonld" + +#: pygeoapi/templates/_base.html:105 +msgid "Powered by " +msgstr "Desarrollado por " + +#: pygeoapi/templates/conformance.html:2 pygeoapi/templates/conformance.html:4 +#: pygeoapi/templates/conformance.html:8 +#: pygeoapi/templates/landing_page.html:86 +msgid "Conformance" +msgstr "Conformidad" + +#: pygeoapi/templates/exception.html:2 pygeoapi/templates/exception.html:5 +msgid "Exception" +msgstr "Excepción" + +#: pygeoapi/templates/landing_page.html:25 +msgid "Terms of service" +msgstr "Términos del servicio" + +#: pygeoapi/templates/collections/collection.html:38 +#: pygeoapi/templates/landing_page.html:35 +msgid "License" +msgstr "Licencia" + +#: pygeoapi/templates/collections/collection.html:6 +#: pygeoapi/templates/collections/edr/query.html:4 +#: pygeoapi/templates/collections/index.html:2 +#: pygeoapi/templates/collections/index.html:4 +#: pygeoapi/templates/collections/items/index.html:4 +#: pygeoapi/templates/collections/items/item.html:27 +#: pygeoapi/templates/collections/queryables.html:4 +#: pygeoapi/templates/collections/schema.html:4 +#: pygeoapi/templates/collections/tiles/index.html:4 +#: pygeoapi/templates/collections/tiles/metadata.html:4 +#: pygeoapi/templates/landing_page.html:48 +#: pygeoapi/templates/stac/collection_base.html:19 +msgid "Collections" +msgstr "Colecciones" + +#: pygeoapi/templates/landing_page.html:50 +msgid "View the collections in this service" +msgstr "Ver las colecciones de este servicio" + +#: pygeoapi/templates/landing_page.html:56 +msgid "SpatioTemporal Assets" +msgstr "SpatioTemporal Assets" + +#: pygeoapi/templates/landing_page.html:58 +msgid "View the SpatioTemporal Assets in this service" +msgstr "Ver los SpatioTemporal Assets de este servicio" + +#: pygeoapi/templates/landing_page.html:64 +#: pygeoapi/templates/processes/index.html:2 +#: pygeoapi/templates/processes/index.html:4 +#: pygeoapi/templates/processes/process.html:4 +msgid "Processes" +msgstr "Procesos" + +#: pygeoapi/templates/landing_page.html:66 +msgid "View the processes in this service" +msgstr "Ver los procesos de este servicio" + +#: pygeoapi/templates/jobs/index.html:2 pygeoapi/templates/jobs/index.html:4 +#: pygeoapi/templates/jobs/index.html:11 pygeoapi/templates/jobs/job.html:4 +#: pygeoapi/templates/jobs/results/index.html:4 +#: pygeoapi/templates/landing_page.html:70 +#: pygeoapi/templates/processes/process.html:76 +msgid "Jobs" +msgstr "Tareas" + +#: pygeoapi/templates/landing_page.html:72 +#: pygeoapi/templates/processes/process.html:77 +msgid "Browse jobs" +msgstr "Ver tareas" + +#: pygeoapi/templates/landing_page.html:77 +msgid "API Definition" +msgstr "Definición de API" + +#: pygeoapi/templates/landing_page.html:79 +msgid "Documentation" +msgstr "Documentación de peticiones" + +#: pygeoapi/templates/landing_page.html:79 +msgid "Swagger UI" +msgstr "Swagger UI" + +#: pygeoapi/templates/landing_page.html:79 +msgid "ReDoc" +msgstr "ReDoc" + +#: pygeoapi/templates/landing_page.html:82 +msgid "OpenAPI Document" +msgstr "Documento OpenAPI" + +#: pygeoapi/templates/landing_page.html:88 +msgid "View the conformance classes of this service" +msgstr "Ver las clases de conformidad de este servicio" + +#: pygeoapi/templates/landing_page.html:92 +msgid "Tile Matrix Sets" +msgstr "Tile Matrix Sets" + +#: pygeoapi/templates/landing_page.html:94 +msgid "View the Tile Matrix Sets available on this service" +msgstr "Ver los Tile Matrix Sets disponibles en este servicio" + +#: pygeoapi/templates/landing_page.html:101 +msgid "Provider" +msgstr "Proveedor" + +#: pygeoapi/templates/landing_page.html:110 +msgid "Contact point" +msgstr "Punto de contacto" + +#: pygeoapi/templates/landing_page.html:113 +msgid "Address" +msgstr "Dirección" + +#: pygeoapi/templates/landing_page.html:122 +msgid "Email" +msgstr "Correo electrónico" + +#: pygeoapi/templates/landing_page.html:125 +msgid "Telephone" +msgstr "Teléfono" + +#: pygeoapi/templates/landing_page.html:129 +msgid "Fax" +msgstr "Fax" + +#: pygeoapi/templates/landing_page.html:133 +msgid "Contact URL" +msgstr "URL de contacto" + +#: pygeoapi/templates/landing_page.html:137 +msgid "Hours" +msgstr "Horario" + +#: pygeoapi/templates/landing_page.html:141 +msgid "Contact instructions" +msgstr "Instrucciones de contacto" + +#: pygeoapi/templates/collections/collection.html:51 +msgid "Browse" +msgstr "Explorar" + +#: pygeoapi/templates/collections/collection.html:55 +msgid "Browse Items" +msgstr "Ver los objetos" + +#: pygeoapi/templates/collections/collection.html:56 +msgid "Browse through the items of" +msgstr "Ver los objetos de" + +#: pygeoapi/templates/collections/collection.html:59 +#: pygeoapi/templates/collections/queryables.html:6 +#: pygeoapi/templates/collections/queryables.html:17 +msgid "Queryables" +msgstr "Atributos consultables" + +#: pygeoapi/templates/collections/collection.html:63 +msgid "Display Queryables" +msgstr "Mostrar los atributos consultables" + +#: pygeoapi/templates/collections/collection.html:64 +msgid "Display Queryables of" +msgstr "Mostrar los atributos consultables de" + +#: pygeoapi/templates/collections/collection.html:67 +#: pygeoapi/templates/collections/schema.html:6 +#: pygeoapi/templates/collections/schema.html:17 +msgid "Schema" +msgstr "Esquema" + +#: pygeoapi/templates/collections/collection.html:71 +msgid "Display Schema" +msgstr "Mostrar esquema" + +#: pygeoapi/templates/collections/collection.html:72 +msgid "Display Schema of" +msgstr "Mostrar el esquema de" + +#: pygeoapi/templates/collections/collection.html:77 +#: pygeoapi/templates/collections/tiles/index.html:6 +#: pygeoapi/templates/collections/tiles/metadata.html:6 +msgid "Tiles" +msgstr "Teselas" + +#: pygeoapi/templates/collections/collection.html:81 +msgid "Display Tiles" +msgstr "Mostrar las teselas" + +#: pygeoapi/templates/collections/collection.html:81 +msgid "Display Tiles of" +msgstr "Mostrar las teselas de" + +#: pygeoapi/templates/collections/collection.html:88 +#: pygeoapi/templates/jobs/job.html:50 +#: pygeoapi/templates/processes/process.html:78 +msgid "Links" +msgstr "Enlaces" + +#: pygeoapi/templates/collections/collection.html:98 +msgid "Reference Systems" +msgstr "Sistemas de referencia" + +#: pygeoapi/templates/collections/collection.html:106 +msgid "Storage CRS" +msgstr "SRC de almacenamiento" + +#: pygeoapi/templates/collections/index.html:8 +msgid "Data collections in this service" +msgstr "Colecciones de datos en este servicio" + +#: pygeoapi/templates/collections/index.html:12 +#: pygeoapi/templates/collections/index.html:39 +#: pygeoapi/templates/processes/index.html:14 +#: pygeoapi/templates/stac/catalog.html:17 +#: pygeoapi/templates/stac/collection.html:17 +msgid "Name" +msgstr "Nombre" + +#: pygeoapi/templates/collections/index.html:13 +#: pygeoapi/templates/collections/index.html:40 +#: pygeoapi/templates/processes/index.html:15 +#: pygeoapi/templates/processes/process.html:26 +#: pygeoapi/templates/processes/process.html:56 +#: pygeoapi/templates/stac/collection.html:18 +#: pygeoapi/templates/tilematrixsets/index.html:16 +msgid "Description" +msgstr "Descripción" + +#: pygeoapi/templates/collections/index.html:35 +msgid "Record collections in this service" +msgstr "Colecciones de registros en este servicio" + +#: pygeoapi/templates/collections/edr/query.html:11 +#: pygeoapi/templates/collections/items/index.html:11 +#: pygeoapi/templates/collections/items/item.html:33 +msgid "Items" +msgstr "Objetos" + +#: pygeoapi/templates/collections/items/index.html:25 +msgid "Items in this collection" +msgstr "Objetos en esta colección" + +#: pygeoapi/templates/collections/items/index.html:38 +msgid "Warning: Higher limits not recommended!" +msgstr "Atención: No se recomiendan límites superiores" + +#: pygeoapi/templates/collections/items/index.html:43 +msgid "Limit" +msgstr "Límite" + +#: pygeoapi/templates/collections/items/index.html:45 +msgid "default" +msgstr "por defecto" + +#: pygeoapi/templates/collections/items/index.html:66 +#: pygeoapi/templates/collections/items/item.html:62 +msgid "Prev" +msgstr "Anterior" + +#: pygeoapi/templates/collections/items/index.html:68 +#: pygeoapi/templates/collections/items/item.html:64 +msgid "Next" +msgstr "Siguiente" + +#: pygeoapi/templates/collections/items/index.html:139 +msgid "No items" +msgstr "No hay objetos" + +#: pygeoapi/templates/collections/items/item.html:77 +#: pygeoapi/templates/stac/collection_base.html:58 +#: pygeoapi/templates/stac/item.html:58 +msgid "Property" +msgstr "Propiedad" + +#: pygeoapi/templates/collections/items/item.html:78 +#: pygeoapi/templates/stac/collection_base.html:59 +#: pygeoapi/templates/stac/item.html:59 +msgid "Value" +msgstr "Valor" + +#: pygeoapi/templates/collections/tiles/index.html:31 +msgid "Tile Matrix Set" +msgstr "Tile Matrix Set" + +#: pygeoapi/templates/collections/tiles/index.html:42 +msgid "Metadata" +msgstr "Metadatos" + +#: pygeoapi/templates/collections/tiles/metadata.html:15 +msgid "TileJSON" +msgstr "TileJSON" + +#: pygeoapi/templates/collections/tiles/metadata.html:17 +msgid "JSON" +msgstr "JSON" + +#: pygeoapi/templates/jobs/index.html:14 +msgid "Job ID" +msgstr "ID tarea" + +#: pygeoapi/templates/jobs/index.html:15 +msgid "Process ID" +msgstr "ID proceso" + +#: pygeoapi/templates/jobs/index.html:16 +msgid "Start" +msgstr "Inicio" + +#: pygeoapi/templates/jobs/index.html:17 pygeoapi/templates/jobs/job.html:37 +msgid "Duration" +msgstr "Duración" + +#: pygeoapi/templates/jobs/index.html:18 pygeoapi/templates/jobs/job.html:17 +#: pygeoapi/templates/jobs/job.html:35 +msgid "Progress" +msgstr "Progreso" + +#: pygeoapi/templates/jobs/index.html:19 pygeoapi/templates/jobs/job.html:16 +msgid "Status" +msgstr "Estado" + +#: pygeoapi/templates/jobs/index.html:20 pygeoapi/templates/jobs/job.html:21 +msgid "Message" +msgstr "Mensaje" + +#: pygeoapi/templates/jobs/job.html:2 pygeoapi/templates/jobs/job.html:10 +msgid "Job status" +msgstr "Estado de la tarea" + +#: pygeoapi/templates/jobs/job.html:26 +msgid "Parameters" +msgstr "Parámetros" + +#: pygeoapi/templates/jobs/job.html:45 +msgid "Started processing" +msgstr "Proceso iniciado" + +#: pygeoapi/templates/jobs/job.html:47 +msgid "Finished processing" +msgstr "Proceso finalizado" + +#: pygeoapi/templates/jobs/results/index.html:2 +msgid "Job result" +msgstr "Resultado de la tarea" + +#: pygeoapi/templates/jobs/results/index.html:6 +msgid "Results" +msgstr "Resultados" + +#: pygeoapi/templates/jobs/results/index.html:10 +msgid "Results of job" +msgstr "Resultados de la tarea" + +#: pygeoapi/templates/processes/index.html:8 +msgid "Processes in this service" +msgstr "Procesos en este servicio" + +#: pygeoapi/templates/processes/process.html:20 +msgid "Inputs" +msgstr "Datos de entrada" + +#: pygeoapi/templates/processes/process.html:23 +#: pygeoapi/templates/processes/process.html:54 +msgid "Id" +msgstr "ID" + +#: pygeoapi/templates/processes/process.html:24 +#: pygeoapi/templates/processes/process.html:55 +#: pygeoapi/templates/tilematrixsets/index.html:15 +msgid "Title" +msgstr "Título" + +#: pygeoapi/templates/processes/process.html:25 +msgid "Data Type" +msgstr "Tipo de dato" + +#: pygeoapi/templates/processes/process.html:51 +msgid "Outputs" +msgstr "Datos de salida" + +#: pygeoapi/templates/processes/process.html:71 +msgid "Execution modes" +msgstr "Modos de ejecución" + +#: pygeoapi/templates/processes/process.html:73 +msgid "Synchronous" +msgstr "Síncrono" + +#: pygeoapi/templates/processes/process.html:74 +msgid "Asynchronous" +msgstr "Asíncrono" + +#: pygeoapi/templates/stac/catalog.html:4 +#: pygeoapi/templates/stac/collection.html:2 +#: pygeoapi/templates/stac/collection.html:4 +#: pygeoapi/templates/stac/collection_base.html:4 +#: pygeoapi/templates/stac/item.html:4 +msgid "SpatioTemporal Asset Catalog" +msgstr "Catálogo de SpatioTemporal Asset" + +#: pygeoapi/templates/stac/catalog.html:18 +msgid "Type" +msgstr "Tipo" + +#: pygeoapi/templates/stac/catalog.html:19 +msgid "Last modified" +msgstr "Última modificación" + +#: pygeoapi/templates/stac/catalog.html:20 +#: pygeoapi/templates/stac/collection_base.html:34 +#: pygeoapi/templates/stac/item.html:34 +msgid "Size" +msgstr "Tamaño" + +#: pygeoapi/templates/stac/collection.html:9 +msgid "STAC Version" +msgstr "Versión STAC" + +#: pygeoapi/templates/stac/collection_base.html:28 +#: pygeoapi/templates/stac/item.html:28 +msgid "Assets" +msgstr "Assets" + +#: pygeoapi/templates/stac/collection_base.html:32 +#: pygeoapi/templates/stac/item.html:32 +msgid "URL" +msgstr "URL" + +#: pygeoapi/templates/stac/collection_base.html:33 +#: pygeoapi/templates/stac/item.html:33 +msgid "Last Modified" +msgstr "Última modificación" + +#: pygeoapi/templates/stac/collection_base.html:64 +#: pygeoapi/templates/stac/item.html:64 +msgid "id" +msgstr "ID" + +#: pygeoapi/templates/stac/collection_base.html:68 +msgid "description" +msgstr "Descripción" + +#: pygeoapi/templates/stac/collection_base.html:72 +msgid "extent" +msgstr "Extensión" + +#: pygeoapi/templates/stac/collection_base.html:77 +msgid "cube:dimensions" +msgstr "cubo:dimensiones" + +#: pygeoapi/templates/stac/collection_base.html:83 +msgid "cube:variables" +msgstr "cubo:variables" + +#: pygeoapi/templates/stac/item.html:19 +msgid "Item" +msgstr "Objeto" + +#: pygeoapi/templates/tilematrixsets/index.html:2 +#: pygeoapi/templates/tilematrixsets/index.html:4 +#: pygeoapi/templates/tilematrixsets/tilematrixset.html:4 +msgid "TileMatrixSets" +msgstr "Tile Matrix Sets" + +#: pygeoapi/templates/tilematrixsets/index.html:9 +msgid "Tile matrix sets available in this service" +msgstr "Tile Matrix Sets disponibles en este servicio" + +#: pygeoapi/templates/tilematrixsets/tilematrixset.html:2 +msgid "TileMatrixSet" +msgstr "Tile Matrix Set" + +msgid "Contact" +msgstr "" + +msgid "CRS" +msgstr "" + +msgid "Epoch" +msgstr "" + +msgid "not specified" +msgstr "" + +msgid "Position" +msgstr "" + +msgid "Cube" +msgstr "" + +msgid "Area" +msgstr "" + +msgid "Corridor" +msgstr "" + +msgid "Trajectory" +msgstr "" + +msgid "Radius" +msgstr "" + +msgid "Locations" +msgstr "" + +msgid "Instances" +msgstr "" diff --git a/locale/fr/LC_MESSAGES/messages.po b/locale/fr/LC_MESSAGES/messages.po index 364e441e6..f8d62bc3f 100644 --- a/locale/fr/LC_MESSAGES/messages.po +++ b/locale/fr/LC_MESSAGES/messages.po @@ -18,6 +18,10 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.9.1\n" +#: pygeoapi/templates/_base.html:2 +msgid "text_direction" +msgstr "ltr" + #: build/lib/pygeoapi/templates/_base.html:40 #: build/lib/pygeoapi/templates/landing_page.html:2 #: pygeoapi/templates/_base.html:40 pygeoapi/templates/landing_page.html:2 @@ -593,3 +597,149 @@ msgstr "" msgid "Results of job" msgstr "" +msgid "This document as JSON" +msgstr "" + +msgid "This document as RDF (JSON-LD)" +msgstr "" + +msgid "This document as HTML" +msgstr "" + +msgid "The OpenAPI definition as JSON" +msgstr "" + +msgid "The list of supported tiling schemes as JSON" +msgstr "" + +msgid "The list of supported tiling schemes as HTML" +msgstr "" + +msgid "The landing page of this server as JSON" +msgstr "" + +msgid "The landing page of this server as HTML" +msgstr "" + +msgid "Schema of collection in JSON" +msgstr "" + +msgid "Schema of collection in HTML" +msgstr "" + +msgid "Queryables for this collection as JSON" +msgstr "" + +msgid "Queryables for this collection as HTML" +msgstr "" + +msgid "Items as GeoJSON" +msgstr "" + +msgid "Items as HTML" +msgstr "" + +msgid "Items as RDF (GeoJSON-LD)" +msgstr "" + +msgid "Coverage data" +msgstr "" + +msgid "Coverage data as" +msgstr "" + +msgid "Tiles as HTML" +msgstr "" + +msgid "Tiles as JSON" +msgstr "" + +msgid "query for this collection as HTML" +msgstr "" + +msgid "query for this collection as JSON" +msgstr "" + +msgid "Items (prev)" +msgstr "" + +msgid "Items (next)" +msgstr "" + +msgid "Process description as JSON" +msgstr "" + +msgid "Process description as HTML" +msgstr "" + +msgid "query for this collection as HTML" +msgstr "" + +msgid "Execution for process as JSON" +msgstr "" + +msgid "Jobs list as JSON" +msgstr "" + +msgid "Jobs list as HTML" +msgstr "" + +msgid "Results of job as JSON" +msgstr "" + +msgid "Results of job as HTML" +msgstr "" + +msgid "The job list for the current process" +msgstr "" + +msgid "TileMatrixSet definition in JSON" +msgstr "" + +msgid "Data collections in this service" +msgstr "" + +msgid "Record collections in this service" +msgstr "" + +msgid "Display Schema" +msgstr "" + +msgid "Display Schema of" +msgstr "" + +msgid "Contact" +msgstr "" + +msgid "CRS" +msgstr "" + +msgid "Epoch" +msgstr "" + +msgid "not specified" +msgstr "" + +msgid "Position" +msgstr "" + +msgid "Cube" +msgstr "" + +msgid "Area" +msgstr "" + +msgid "Corridor" +msgstr "" + +msgid "Trajectory" +msgstr "" + +msgid "Radius" +msgstr "" + +msgid "Locations" +msgstr "" + +msgid "Instances" +msgstr "" diff --git a/locale/sr/LC_MESSAGES/messages.po b/locale/sr/LC_MESSAGES/messages.po new file mode 100644 index 000000000..878ca5e5a --- /dev/null +++ b/locale/sr/LC_MESSAGES/messages.po @@ -0,0 +1,686 @@ +# Serbian translations for PROJECT. +# Copyright (C) 2024 ORGANIZATION +# This file is distributed under the same license as the PROJECT project. +# FIRST AUTHOR , 2024. +# +msgid "" +msgstr "" +"Project-Id-Version: PROJECT VERSION\n" +"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" +"POT-Creation-Date: 2024-02-27 14:39+0100\n" +"PO-Revision-Date: 2024-02-27 14:40+0100\n" +"Last-Translator: Bojan Kuzmanović \n" +"Language: sr\n" +"Language-Team: sr \n" +"Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && " +"n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.14.0\n" + +#: pygeoapi/templates/_base.html:2 +msgid "text_direction" +msgstr "ltr" + +#: pygeoapi/templates/_base.html:51 +msgid "Admin" +msgstr "Admin" + +#: pygeoapi/templates/_base.html:67 pygeoapi/templates/landing_page.html:2 +msgid "Home" +msgstr "Početna" + +#: pygeoapi/templates/_base.html:75 pygeoapi/templates/_base.html:83 +msgid "json" +msgstr "json" + +#: pygeoapi/templates/_base.html:78 pygeoapi/templates/_base.html:86 +msgid "jsonld" +msgstr "jsonld" + +#: pygeoapi/templates/_base.html:105 +msgid "Powered by " +msgstr "Pokreće ga " + +#: pygeoapi/templates/conformance.html:2 pygeoapi/templates/conformance.html:4 +#: pygeoapi/templates/conformance.html:8 +#: pygeoapi/templates/landing_page.html:86 +msgid "Conformance" +msgstr "Usaglašenost" + +#: pygeoapi/templates/exception.html:2 pygeoapi/templates/exception.html:5 +msgid "Exception" +msgstr "Izuzetak" + +#: pygeoapi/templates/landing_page.html:25 +msgid "Terms of service" +msgstr "Uslovi korišćenja" + +#: pygeoapi/templates/collections/collection.html:38 +#: pygeoapi/templates/landing_page.html:35 +msgid "License" +msgstr "Licensa" + +#: pygeoapi/templates/collections/collection.html:6 +#: pygeoapi/templates/collections/coverage/domainset.html:4 +#: pygeoapi/templates/collections/coverage/rangetype.html:4 +#: pygeoapi/templates/collections/edr/query.html:4 +#: pygeoapi/templates/collections/index.html:2 +#: pygeoapi/templates/collections/index.html:4 +#: pygeoapi/templates/collections/items/index.html:4 +#: pygeoapi/templates/collections/items/item.html:27 +#: pygeoapi/templates/collections/queryables.html:4 +#: pygeoapi/templates/collections/tiles/index.html:4 +#: pygeoapi/templates/collections/tiles/metadata.html:4 +#: pygeoapi/templates/landing_page.html:48 +#: pygeoapi/templates/stac/collection_base.html:19 +msgid "Collections" +msgstr "Kolekcije" + +#: pygeoapi/templates/landing_page.html:50 +msgid "View the collections in this service" +msgstr "Pogledajte kolekcije u ovom servisu" + +#: pygeoapi/templates/landing_page.html:56 +msgid "SpatioTemporal Assets" +msgstr "Prostorno-vremenski resursi" + +#: pygeoapi/templates/landing_page.html:58 +msgid "View the SpatioTemporal Assets in this service" +msgstr "Pogledajte prostorno-vremenske resurse u ovom servisu" + +#: pygeoapi/templates/landing_page.html:64 +#: pygeoapi/templates/processes/index.html:2 +#: pygeoapi/templates/processes/index.html:4 +#: pygeoapi/templates/processes/process.html:4 +msgid "Processes" +msgstr "Procesi" + +#: pygeoapi/templates/landing_page.html:66 +msgid "View the processes in this service" +msgstr "Poglejte procese u ovom servisu" + +#: pygeoapi/templates/jobs/index.html:2 pygeoapi/templates/jobs/index.html:4 +#: pygeoapi/templates/jobs/index.html:11 pygeoapi/templates/jobs/job.html:4 +#: pygeoapi/templates/jobs/results/index.html:4 +#: pygeoapi/templates/landing_page.html:70 +#: pygeoapi/templates/processes/process.html:76 +msgid "Jobs" +msgstr "Zadaci" + +#: pygeoapi/templates/landing_page.html:72 +#: pygeoapi/templates/processes/process.html:77 +msgid "Browse jobs" +msgstr "Pogledajte zadatke" + +#: pygeoapi/templates/landing_page.html:77 +msgid "API Definition" +msgstr "API definicija" + +#: pygeoapi/templates/landing_page.html:79 +msgid "Documentation" +msgstr "Dokuemntacija" + +#: pygeoapi/templates/landing_page.html:79 +msgid "Swagger UI" +msgstr "Swagger UI" + +#: pygeoapi/templates/landing_page.html:79 +msgid "ReDoc" +msgstr "ReDoc" + +#: pygeoapi/templates/landing_page.html:82 +msgid "OpenAPI Document" +msgstr "OpenAPI dokument" + +#: pygeoapi/templates/landing_page.html:88 +msgid "View the conformance classes of this service" +msgstr "Pogledajte usaglašenost ovog servisa" + +#: pygeoapi/templates/landing_page.html:92 +msgid "Tile Matrix Sets" +msgstr "Matrica skupova tile-ova" + +#: pygeoapi/templates/landing_page.html:94 +msgid "View the Tile Matrix Sets available on this service" +msgstr "Pogledajte matrice skupova tile-ova dostupne u ovom servisu" + +#: pygeoapi/templates/landing_page.html:101 +msgid "Provider" +msgstr "Provajder" + +#: pygeoapi/templates/landing_page.html:110 +msgid "Contact point" +msgstr "Kontakt tačka" + +#: pygeoapi/templates/landing_page.html:113 +msgid "Address" +msgstr "Adresa" + +#: pygeoapi/templates/landing_page.html:122 +msgid "Email" +msgstr "Email" + +#: pygeoapi/templates/landing_page.html:125 +msgid "Telephone" +msgstr "Telefon" + +#: pygeoapi/templates/landing_page.html:129 +msgid "Fax" +msgstr "Fax" + +#: pygeoapi/templates/landing_page.html:133 +msgid "Contact URL" +msgstr "Kontakt URL" + +#: pygeoapi/templates/landing_page.html:137 +msgid "Hours" +msgstr "Sati" + +#: pygeoapi/templates/landing_page.html:141 +msgid "Contact instructions" +msgstr "Kontakt instrukcije" + +#: pygeoapi/templates/collections/collection.html:51 +msgid "Browse" +msgstr "Pretraži" + +#: pygeoapi/templates/collections/collection.html:55 +msgid "Browse Items" +msgstr "Pretraži stavke" + +#: pygeoapi/templates/collections/collection.html:56 +msgid "Browse through the items of" +msgstr "Pretrazite kroz stavke od" + +#: pygeoapi/templates/collections/collection.html:59 +#: pygeoapi/templates/collections/queryables.html:6 +#: pygeoapi/templates/collections/queryables.html:17 +msgid "Queryables" +msgstr "Upitni" + +#: pygeoapi/templates/collections/collection.html:63 +msgid "Display Queryables" +msgstr "Prikaži upitne" + +#: pygeoapi/templates/collections/collection.html:64 +msgid "Display Queryables of" +msgstr "Prikaži upitne od" + +#: pygeoapi/templates/collections/collection.html:69 +#: pygeoapi/templates/collections/tiles/index.html:6 +#: pygeoapi/templates/collections/tiles/metadata.html:6 +msgid "Tiles" +msgstr "Tile-ovi" + +#: pygeoapi/templates/collections/collection.html:73 +msgid "Display Tiles" +msgstr "Prikaži tile-ove" + +#: pygeoapi/templates/collections/collection.html:73 +msgid "Display Tiles of" +msgstr "Prikaži tile-ove od" + +#: pygeoapi/templates/collections/collection.html:80 +#: pygeoapi/templates/jobs/job.html:50 +#: pygeoapi/templates/processes/process.html:78 +msgid "Links" +msgstr "Linkovi" + +#: pygeoapi/templates/collections/collection.html:90 +msgid "Reference Systems" +msgstr "Referentni sistemi" + +#: pygeoapi/templates/collections/collection.html:98 +msgid "Storage CRS" +msgstr "Skaldišni CRS" + +#: pygeoapi/templates/collections/index.html:12 +#: pygeoapi/templates/processes/index.html:14 +#: pygeoapi/templates/stac/catalog.html:17 +#: pygeoapi/templates/stac/collection.html:17 +msgid "Name" +msgstr "Naziv" + +#: pygeoapi/templates/collections/index.html:13 +#: pygeoapi/templates/stac/catalog.html:18 +msgid "Type" +msgstr "Tip" + +#: pygeoapi/templates/collections/index.html:14 +#: pygeoapi/templates/processes/index.html:15 +#: pygeoapi/templates/processes/process.html:26 +#: pygeoapi/templates/processes/process.html:56 +#: pygeoapi/templates/stac/collection.html:18 +#: pygeoapi/templates/tilematrixsets/index.html:16 +msgid "Description" +msgstr "Opis" + +#: pygeoapi/templates/collections/coverage/domainset.html:11 +msgid "Coverage domain set" +msgstr "Pokrivenost domenskog skupa" + +#: pygeoapi/templates/collections/coverage/domainset.html:12 +msgid "Axis labels" +msgstr "Labele osa" + +#: pygeoapi/templates/collections/coverage/domainset.html:18 +msgid "Extent" +msgstr "Extent" + +#: pygeoapi/templates/collections/coverage/domainset.html:24 +msgid "Coordinate reference system" +msgstr "Koordiantni referentni sistem" + +#: pygeoapi/templates/collections/coverage/domainset.html:26 +#: pygeoapi/templates/stac/catalog.html:20 +#: pygeoapi/templates/stac/collection_base.html:34 +#: pygeoapi/templates/stac/item.html:34 +msgid "Size" +msgstr "Veličina" + +#: pygeoapi/templates/collections/coverage/domainset.html:28 +msgid "width" +msgstr "širina" + +#: pygeoapi/templates/collections/coverage/domainset.html:29 +msgid "height" +msgstr "visina" + +#: pygeoapi/templates/collections/coverage/domainset.html:31 +msgid "Resolution" +msgstr "rezolucija" + +#: pygeoapi/templates/collections/coverage/domainset.html:33 +msgid "x" +msgstr "x" + +#: pygeoapi/templates/collections/coverage/domainset.html:34 +msgid "y" +msgstr "y" + +#: pygeoapi/templates/collections/coverage/rangetype.html:11 +msgid "Coverage range type" +msgstr "Pokrivenost tipa opsega" + +#: pygeoapi/templates/collections/coverage/rangetype.html:12 +msgid "Fields" +msgstr "Polja" + +#: pygeoapi/templates/collections/edr/query.html:11 +#: pygeoapi/templates/collections/items/index.html:11 +#: pygeoapi/templates/collections/items/item.html:33 +msgid "Items" +msgstr "Stavke" + +#: pygeoapi/templates/collections/items/index.html:25 +msgid "Items in this collection" +msgstr "Stavke u ovoj kolekciji" + +#: pygeoapi/templates/collections/items/index.html:38 +msgid "Warning: Higher limits not recommended!" +msgstr "Upozorenje: Veća granica nije preporučljiva!" + +#: pygeoapi/templates/collections/items/index.html:43 +msgid "Limit" +msgstr "Granica" + +#: pygeoapi/templates/collections/items/index.html:45 +msgid "default" +msgstr "podrazumevano" + +#: pygeoapi/templates/collections/items/index.html:66 +#: pygeoapi/templates/collections/items/item.html:62 +msgid "Prev" +msgstr "Prethodno" + +#: pygeoapi/templates/collections/items/index.html:68 +#: pygeoapi/templates/collections/items/item.html:64 +msgid "Next" +msgstr "Sljedeće" + +#: pygeoapi/templates/collections/items/index.html:139 +msgid "No items" +msgstr "Nema stavki" + +#: pygeoapi/templates/collections/items/item.html:77 +#: pygeoapi/templates/stac/collection_base.html:58 +#: pygeoapi/templates/stac/item.html:58 +msgid "Property" +msgstr "Svojstvo" + +#: pygeoapi/templates/collections/items/item.html:78 +#: pygeoapi/templates/stac/collection_base.html:59 +#: pygeoapi/templates/stac/item.html:59 +msgid "Value" +msgstr "Vrednost" + +#: pygeoapi/templates/collections/tiles/index.html:31 +msgid "Tile Matrix Set" +msgstr "Matrica skupa tile-ova" + +#: pygeoapi/templates/collections/tiles/index.html:42 +msgid "Metadata" +msgstr "Metapodaci" + +#: pygeoapi/templates/collections/tiles/metadata.html:15 +msgid "TileJSON" +msgstr "TileJSON" + +#: pygeoapi/templates/collections/tiles/metadata.html:17 +msgid "JSON" +msgstr "JSON" + +#: pygeoapi/templates/jobs/index.html:14 +msgid "Job ID" +msgstr "ID zadatka" + +#: pygeoapi/templates/jobs/index.html:15 +msgid "Process ID" +msgstr "ID procesa" + +#: pygeoapi/templates/jobs/index.html:16 +msgid "Start" +msgstr "Započni" + +#: pygeoapi/templates/jobs/index.html:17 pygeoapi/templates/jobs/job.html:37 +msgid "Duration" +msgstr "Trajanje" + +#: pygeoapi/templates/jobs/index.html:18 pygeoapi/templates/jobs/job.html:17 +#: pygeoapi/templates/jobs/job.html:35 +msgid "Progress" +msgstr "Napredak" + +#: pygeoapi/templates/jobs/index.html:19 pygeoapi/templates/jobs/job.html:16 +msgid "Status" +msgstr "Status" + +#: pygeoapi/templates/jobs/index.html:20 pygeoapi/templates/jobs/job.html:21 +msgid "Message" +msgstr "Poruka" + +#: pygeoapi/templates/jobs/job.html:2 pygeoapi/templates/jobs/job.html:10 +msgid "Job status" +msgstr "Status zadatka" + +#: pygeoapi/templates/jobs/job.html:26 +msgid "Parameters" +msgstr "Parametri" + +#: pygeoapi/templates/jobs/job.html:45 +msgid "Started processing" +msgstr "Započeto procesiranje" + +#: pygeoapi/templates/jobs/job.html:47 +msgid "Finished processing" +msgstr "Završeno procesiranje" + +#: pygeoapi/templates/jobs/results/index.html:2 +msgid "Job result" +msgstr "Rezultat zadatka" + +#: pygeoapi/templates/jobs/results/index.html:6 +msgid "Results" +msgstr "Rezultati" + +#: pygeoapi/templates/jobs/results/index.html:10 +msgid "Results of job" +msgstr "Rezultati zadatka" + +#: pygeoapi/templates/processes/index.html:8 +msgid "Processes in this service" +msgstr "Processi u ovom servisu" + +#: pygeoapi/templates/processes/process.html:20 +msgid "Inputs" +msgstr "Unosi" + +#: pygeoapi/templates/processes/process.html:23 +#: pygeoapi/templates/processes/process.html:54 +msgid "Id" +msgstr "Id" + +#: pygeoapi/templates/processes/process.html:24 +#: pygeoapi/templates/processes/process.html:55 +#: pygeoapi/templates/tilematrixsets/index.html:15 +msgid "Title" +msgstr "Naslov" + +#: pygeoapi/templates/processes/process.html:25 +msgid "Data Type" +msgstr "Tip podatka" + +#: pygeoapi/templates/processes/process.html:51 +msgid "Outputs" +msgstr "Izlazi" + +#: pygeoapi/templates/processes/process.html:71 +msgid "Execution modes" +msgstr "Načini izvršavanja" + +#: pygeoapi/templates/processes/process.html:73 +msgid "Synchronous" +msgstr "Sinhrono" + +#: pygeoapi/templates/processes/process.html:74 +msgid "Asynchronous" +msgstr "Asinhrono" + +#: pygeoapi/templates/stac/catalog.html:4 +#: pygeoapi/templates/stac/collection.html:2 +#: pygeoapi/templates/stac/collection.html:4 +#: pygeoapi/templates/stac/collection_base.html:4 +#: pygeoapi/templates/stac/item.html:4 +msgid "SpatioTemporal Asset Catalog" +msgstr "Prostorno-vremenski katalog resursa" + +#: pygeoapi/templates/stac/catalog.html:19 +msgid "Last modified" +msgstr "Zadnja izmjena" + +#: pygeoapi/templates/stac/collection.html:9 +msgid "STAC Version" +msgstr "STAC verzija" + +#: pygeoapi/templates/stac/collection_base.html:28 +#: pygeoapi/templates/stac/item.html:28 +msgid "Assets" +msgstr "Resursi" + +#: pygeoapi/templates/stac/collection_base.html:32 +#: pygeoapi/templates/stac/item.html:32 +msgid "URL" +msgstr "URL" + +#: pygeoapi/templates/stac/collection_base.html:33 +#: pygeoapi/templates/stac/item.html:33 +msgid "Last Modified" +msgstr "Zadnja izmjena" + +#: pygeoapi/templates/stac/collection_base.html:64 +#: pygeoapi/templates/stac/item.html:64 +msgid "id" +msgstr "id" + +#: pygeoapi/templates/stac/collection_base.html:68 +msgid "description" +msgstr "opis" + +#: pygeoapi/templates/stac/collection_base.html:72 +msgid "extent" +msgstr "extent" + +#: pygeoapi/templates/stac/collection_base.html:77 +msgid "cube:dimensions" +msgstr "kocka:dimenzije" + +#: pygeoapi/templates/stac/collection_base.html:83 +msgid "cube:variables" +msgstr "kocka:varijable" + +#: pygeoapi/templates/stac/item.html:19 +msgid "Item" +msgstr "Stavka" + +#: pygeoapi/templates/tilematrixsets/index.html:2 +#: pygeoapi/templates/tilematrixsets/index.html:4 +msgid "TileMatrixSets" +msgstr "Matrica skupova tile-ova" + +#: pygeoapi/templates/tilematrixsets/index.html:9 +msgid "Tile matrix sets available in this service" +msgstr "Matrice skupova tile-ova dostupne u ovom servisu" + +#: pygeoapi/templates/tilematrixsets/tilematrixset.html:2 +#: pygeoapi/templates/tilematrixsets/tilematrixset.html:4 +msgid "TileMatrixSet" +msgstr "Matrica skupa tile-ova" + +msgid "This document as JSON" +msgstr "" + +msgid "This document as RDF (JSON-LD)" +msgstr "" + +msgid "This document as HTML" +msgstr "" + +msgid "The OpenAPI definition as JSON" +msgstr "" + +msgid "The list of supported tiling schemes as JSON" +msgstr "" + +msgid "The list of supported tiling schemes as HTML" +msgstr "" + +msgid "The landing page of this server as JSON" +msgstr "" + +msgid "The landing page of this server as HTML" +msgstr "" + +msgid "Schema of collection in JSON" +msgstr "" + +msgid "Schema of collection in HTML" +msgstr "" + +msgid "Queryables for this collection as JSON" +msgstr "" + +msgid "Queryables for this collection as HTML" +msgstr "" + +msgid "Items as GeoJSON" +msgstr "" + +msgid "Items as HTML" +msgstr "" + +msgid "Items as RDF (GeoJSON-LD)" +msgstr "" + +msgid "Coverage data" +msgstr "" + +msgid "Coverage data as" +msgstr "" + +msgid "Tiles as HTML" +msgstr "" + +msgid "Tiles as JSON" +msgstr "" + +msgid "query for this collection as HTML" +msgstr "" + +msgid "query for this collection as JSON" +msgstr "" + +msgid "Items (prev)" +msgstr "" + +msgid "Items (next)" +msgstr "" + +msgid "Process description as JSON" +msgstr "" + +msgid "Process description as HTML" +msgstr "" + +msgid "query for this collection as HTML" +msgstr "" + +msgid "Execution for process as JSON" +msgstr "" + +msgid "Jobs list as JSON" +msgstr "" + +msgid "Jobs list as HTML" +msgstr "" + +msgid "Results of job as JSON" +msgstr "" + +msgid "Results of job as HTML" +msgstr "" + +msgid "The job list for the current process" +msgstr "" + +msgid "TileMatrixSet definition in JSON" +msgstr "" + +msgid "Data collections in this service" +msgstr "" + +msgid "Record collections in this service" +msgstr "" + +msgid "Display Schema" +msgstr "" + +msgid "Display Schema of" +msgstr "" + +msgid "Contact" +msgstr "" + +msgid "CRS" +msgstr "" + +msgid "Epoch" +msgstr "" + +msgid "not specified" +msgstr "" + +msgid "Position" +msgstr "" + +msgid "Cube" +msgstr "" + +msgid "Area" +msgstr "" + +msgid "Corridor" +msgstr "" + +msgid "Trajectory" +msgstr "" + +msgid "Radius" +msgstr "" + +msgid "Locations" +msgstr "" + +msgid "Instances" +msgstr "" diff --git a/pygeoapi-config.yml b/pygeoapi-config.yml index a889062e3..b260c76d2 100644 --- a/pygeoapi-config.yml +++ b/pygeoapi-config.yml @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2020 Tom Kralidis +# Copyright (c) 2025 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -41,7 +41,9 @@ server: - fr-CA # cors: true pretty_print: true - limit: 10 + limits: + default_items: 20 + max_items: 50 # templates: # path: /path/to/Jinja2/templates # static: /path/to/static/folder # css/js/img @@ -53,7 +55,8 @@ server: # connection: /tmp/pygeoapi-process-manager.db # output_dir: /tmp/ # ogc_schemas_location: /opt/schemas.opengis.net - + admin: false # enable admin api + logging: level: ERROR #logfile: /tmp/pygeoapi.log diff --git a/pygeoapi/__init__.py b/pygeoapi/__init__.py index 0f76addea..802b0cb32 100644 --- a/pygeoapi/__init__.py +++ b/pygeoapi/__init__.py @@ -5,6 +5,7 @@ # # Copyright (c) 2021 Tom Kralidis # Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2025 Angelos Tzotsos # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -29,7 +30,7 @@ # # ================================================================= -__version__ = '0.16.dev0' +__version__ = '0.20.0' import click try: @@ -96,11 +97,9 @@ def serve(ctx, server): if server == "flask": from pygeoapi.flask_app import serve as serve_flask - ctx.forward(serve_flask) ctx.invoke(serve_flask) elif server == "starlette": from pygeoapi.starlette_app import serve as serve_starlette - ctx.forward(serve_starlette) ctx.invoke(serve_starlette) elif server == "django": from pygeoapi.django_app import main as serve_django diff --git a/pygeoapi/admin.py b/pygeoapi/admin.py index ff98b611f..860e18bb0 100644 --- a/pygeoapi/admin.py +++ b/pygeoapi/admin.py @@ -3,7 +3,7 @@ # Authors: Tom Kralidis # Benjamin Webb # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # Copyright (c) 2023 Benjamin Webb # # Permission is hereby granted, free of charge, to any person @@ -32,16 +32,16 @@ from copy import deepcopy import os import json -from jsonpatch import make_patch -from jsonschema.exceptions import ValidationError import logging -from typing import Any, Tuple, Union +from typing import Tuple -from pygeoapi.api import API, APIRequest, F_HTML, pre_process +from dateutil.parser import parse as parse_date +from jsonpatch import make_patch +from jsonschema.exceptions import ValidationError +from pygeoapi.api import API, APIRequest, F_HTML from pygeoapi.config import get_config, validate_config from pygeoapi.openapi import get_oas -# from pygeoapi.openapi import validate_openapi_document from pygeoapi.util import to_json, render_j2_template, yaml_dump @@ -128,15 +128,15 @@ def write_config(self, config): # Preserve env variables LOGGER.debug('Reading env variables in configuration') - raw_conf = get_config(raw=True) - conf = get_config() + raw_conf = json.loads(to_json(get_config(raw=True))) + conf = json.loads(to_json(get_config())) patch = make_patch(conf, raw_conf) LOGGER.debug('Merging env variables') config = patch.apply(config) # write pygeoapi configuration - LOGGER.debug('Writing pygeoapi configutation') + LOGGER.debug('Writing pygeoapi configuration') yaml_dump(config, self.PYGEOAPI_CONFIG) LOGGER.debug('Finished writing pygeoapi configuration') @@ -157,467 +157,464 @@ def write_oas(self, config): yaml_dump(oas, self.PYGEOAPI_OPENAPI) LOGGER.debug('Finished writing OpenAPI document') - @pre_process - def get_config( - self, - request: Union[APIRequest, Any] - ) -> Tuple[dict, int, str]: - """ - Provide admin configuration document - - :param request: request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers() - - cfg = get_config(raw=True) - - if request.format == F_HTML: - content = render_j2_template( - self.config, 'admin/index.html', cfg, request.locale - ) - else: - content = to_json(cfg, self.pretty_print) - - return headers, 200, content - @pre_process - def put_config( - self, - request: Union[APIRequest, Any] - ) -> Tuple[dict, int, str]: - """ - Update complete pygeoapi configuration +def get_config_( + admin: Admin, + request: APIRequest, +) -> Tuple[dict, int, str]: + """ + Provide admin configuration document - :param request: request object + :param request: request object - :returns: tuple of headers, status code, content - """ + :returns: tuple of headers, status code, content + """ - LOGGER.debug('Updating configuration') - - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers() - - data = request.data - if not data: - msg = 'missing request data' - return self.get_exception( - 400, headers, request.format, 'MissingParameterValue', msg - ) - - try: - # Parse data - data = data.decode() - except (UnicodeDecodeError, AttributeError): - pass - - try: - data = json.loads(data) - except (json.decoder.JSONDecodeError, TypeError) as err: - # Input is not valid JSON - LOGGER.error(err) - msg = 'invalid request data' - return self.get_exception( - 400, headers, request.format, 'InvalidParameterValue', msg - ) - - LOGGER.debug('Updating configuration') - try: - self.validate(data) - except ValidationError as err: - LOGGER.error(err) - msg = 'Schema validation error' - return self.get_exception( - 400, headers, request.format, 'ValidationError', msg - ) - - self.write(data) - - return headers, 204, {} - - @pre_process - def patch_config( - self, request: Union[APIRequest, Any] - ) -> Tuple[dict, int, str]: - """ - Update partial pygeoapi configuration + headers = request.get_response_headers() + + cfg = get_config(raw=True) + + if request.format == F_HTML: + content = render_j2_template( + admin.tpl_config, admin.config['server']['templates'], + 'admin/index.html', cfg, request.locale + ) + else: + content = to_json(cfg, admin.pretty_print) + + return headers, 200, content + + +def put_config( + admin: Admin, + request: APIRequest, +) -> Tuple[dict, int, str]: + """ + Update complete pygeoapi configuration + + :param request: request object + + :returns: tuple of headers, status code, content + """ + + LOGGER.debug('Updating configuration') + + headers = request.get_response_headers() + + data = request.data + if not data: + msg = 'missing request data' + return admin.get_exception( + 400, headers, request.format, 'MissingParameterValue', msg + ) + + try: + # Parse data + data = data.decode() + except (UnicodeDecodeError, AttributeError): + pass - :param request: request object - :param resource_id: resource identifier + try: + data = json.loads(data) + for key, value in data.get('resources', {}).items(): + temporal_extents_str2datetime(value.get('extents', {})) + except (json.decoder.JSONDecodeError, TypeError) as err: + # Input is not valid JSON + LOGGER.error(err) + msg = 'invalid request data' + return admin.get_exception( + 400, headers, request.format, 'InvalidParameterValue', msg + ) - :returns: tuple of headers, status code, content - """ + LOGGER.debug('Updating configuration') + try: + admin.validate(data) + except ValidationError as err: + LOGGER.error(err) + msg = 'Schema validation error' + return admin.get_exception( + 400, headers, request.format, 'ValidationError', msg + ) - if not request.is_valid(): - return self.get_format_exception(request) - - config = deepcopy(self.config) - headers = request.get_response_headers() - - data = request.data - if not data: - msg = 'missing request data' - return self.get_exception( - 400, headers, request.format, 'MissingParameterValue', msg - ) - - try: - # Parse data - data = data.decode() - except (UnicodeDecodeError, AttributeError): - pass - - try: - data = json.loads(data) - except (json.decoder.JSONDecodeError, TypeError) as err: - # Input is not valid JSON - LOGGER.error(err) - msg = 'invalid request data' - return self.get_exception( - 400, headers, request.format, 'InvalidParameterValue', msg - ) - - LOGGER.debug('Merging configuration') - config = self.merge(config, data) - - try: - self.validate(config) - except ValidationError as err: - LOGGER.error(err) - msg = 'Schema validation error' - return self.get_exception( - 400, headers, request.format, 'ValidationError', msg - ) - - self.write(config) - - content = to_json(config, self.pretty_print) - - return headers, 204, content - - @pre_process - def get_resources( - self, request: Union[APIRequest, Any] - ) -> Tuple[dict, int, str]: - """ - Provide admin document + admin.write(data) - :param request: request object + return headers, 204, {} - :returns: tuple of headers, status code, content - """ - if not request.is_valid(): - return self.get_format_exception(request) +def patch_config( + admin: Admin, request: APIRequest, +) -> Tuple[dict, int, str]: + """ + Update partial pygeoapi configuration - headers = request.get_response_headers() + :param request: request object + :param resource_id: resource identifier - cfg = get_config(raw=True) + :returns: tuple of headers, status code, content + """ - if request.format == F_HTML: - content = render_j2_template( - self.config, - 'admin/index.html', - cfg['resources'], - request.locale, - ) - else: - content = to_json(cfg['resources'], self.pretty_print) - - return headers, 200, content - - @pre_process - def post_resource( - self, request: Union[APIRequest, Any] - ) -> Tuple[dict, int, str]: - """ - Add resource configuration + config = deepcopy(admin.config) + headers = request.get_response_headers() + + data = request.data + if not data: + msg = 'missing request data' + return admin.get_exception( + 400, headers, request.format, 'MissingParameterValue', msg + ) + + try: + # Parse data + data = data.decode() + except (UnicodeDecodeError, AttributeError): + pass + + try: + data = json.loads(data) + for key, value in data.get('resources', {}).items(): + temporal_extents_str2datetime(value.get('extents', {})) + except (json.decoder.JSONDecodeError, TypeError) as err: + # Input is not valid JSON + LOGGER.error(err) + msg = 'invalid request data' + return admin.get_exception( + 400, headers, request.format, 'InvalidParameterValue', msg + ) + + LOGGER.debug('Merging configuration') + config = admin.merge(config, data) + + try: + admin.validate(config) + except ValidationError as err: + LOGGER.error(err) + msg = 'Schema validation error' + return admin.get_exception( + 400, headers, request.format, 'ValidationError', msg + ) + + admin.write(config) + + content = to_json(config, admin.pretty_print) + + return headers, 204, content + + +def get_resources( + admin: Admin, request: APIRequest, +) -> Tuple[dict, int, str]: + """ + Provide admin document + + :param request: request object + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers() + + cfg = get_config(raw=True) + + if request.format == F_HTML: + content = render_j2_template( + admin.tpl_config, admin.config['server']['templates'], + 'admin/index.html', cfg['resources'], request.locale + ) + else: + content = to_json(cfg['resources'], admin.pretty_print) + + return headers, 200, content + + +def post_resource( + admin: Admin, request: APIRequest, +) -> Tuple[dict, int, str]: + """ + Add resource configuration + + :param request: request object + + :returns: tuple of headers, status code, content + """ + + config = deepcopy(admin.config) + headers = request.get_response_headers() + + data = request.data + if not data: + msg = 'missing request data' + return admin.get_exception( + 400, headers, request.format, 'MissingParameterValue', msg + ) + + try: + # Parse data + data = data.decode() + except (UnicodeDecodeError, AttributeError): + pass + + try: + data = json.loads(data) + res = list(data.keys())[0] + temporal_extents_str2datetime(data[res].get('extents', {})) + except (json.decoder.JSONDecodeError, TypeError) as err: + # Input is not valid JSON + LOGGER.error(err) + msg = 'invalid request data' + return admin.get_exception( + 400, headers, request.format, 'InvalidParameterValue', msg + ) + + resource_id = next(iter(data.keys())) + + if config['resources'].get(resource_id) is not None: + # Resource already exists + msg = f'Resource exists: {resource_id}' + LOGGER.error(msg) + return admin.get_exception( + 400, headers, request.format, 'NoApplicableCode', msg + ) + + LOGGER.debug(f'Adding resource: {resource_id}') + config['resources'].update(data) + + try: + admin.validate(config) + except ValidationError as err: + LOGGER.error(err) + msg = 'Schema validation error' + return admin.get_exception( + 400, headers, request.format, 'ValidationError', msg + ) - :param request: request object + admin.write(config) - :returns: tuple of headers, status code, content - """ + content = f'Location: /{request.path_info}/{resource_id}' + LOGGER.debug(f'Success at {content}') + + return headers, 201, content - if not request.is_valid(): - return self.get_format_exception(request) - - config = deepcopy(self.config) - headers = request.get_response_headers() - - data = request.data - if not data: - msg = 'missing request data' - return self.get_exception( - 400, headers, request.format, 'MissingParameterValue', msg - ) - - try: - # Parse data - data = data.decode() - except (UnicodeDecodeError, AttributeError): - pass - - try: - data = json.loads(data) - except (json.decoder.JSONDecodeError, TypeError) as err: - # Input is not valid JSON - LOGGER.error(err) - msg = 'invalid request data' - return self.get_exception( - 400, headers, request.format, 'InvalidParameterValue', msg - ) - - resource_id = next(iter(data.keys())) - - if config['resources'].get(resource_id) is not None: - # Resource already exists - msg = f'Resource exists: {resource_id}' - LOGGER.error(msg) - return self.get_exception( - 400, headers, request.format, 'NoApplicableCode', msg - ) - - LOGGER.debug(f'Adding resource: {resource_id}') - config['resources'].update(data) - - try: - self.validate(config) - except ValidationError as err: - LOGGER.error(err) - msg = 'Schema validation error' - return self.get_exception( - 400, headers, request.format, 'ValidationError', msg - ) - - self.write(config) - - content = f'Location: /{request.path_info}/{resource_id}' - LOGGER.debug(f'Success at {content}') - - return headers, 201, content - - @pre_process - def get_resource( - self, request: Union[APIRequest, Any], resource_id: str - ) -> Tuple[dict, int, str]: - """ - Get resource configuration - - :param request: request object - :param resource_id: - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers() - - cfg = get_config(raw=True) - - try: - resource = cfg['resources'][resource_id] - except KeyError: - msg = f'Resource not found: {resource_id}' - return self.get_exception( - 400, headers, request.format, 'ResourceNotFound', msg - ) - - if request.format == F_HTML: - content = render_j2_template( - self.config, 'admin/index.html', resource, request.locale - ) - else: - content = to_json(resource, self.pretty_print) - - return headers, 200, content - - @pre_process - def delete_resource( - self, request: Union[APIRequest, Any], resource_id: str - ) -> Tuple[dict, int, str]: - """ - Delete resource configuration - - :param request: request object - :param resource_id: resource identifier - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - config = deepcopy(self.config) - headers = request.get_response_headers() - - try: - LOGGER.debug(f'Removing resource configuration for: {resource_id}') - config['resources'].pop(resource_id) - except KeyError: - msg = f'Resource not found: {resource_id}' - return self.get_exception( - 400, headers, request.format, 'ResourceNotFound', msg - ) - - LOGGER.debug('Resource removed, validating and saving configuration') - try: - self.validate(config) - except ValidationError as err: - LOGGER.error(err) - msg = 'Schema validation error' - return self.get_exception( - 400, headers, request.format, 'ValidationError', msg - ) - - self.write(config) - - return headers, 204, {} - - @pre_process - def put_resource( - self, - request: Union[APIRequest, Any], - resource_id: str, - ) -> Tuple[dict, int, str]: - """ - Update complete resource configuration - - :param request: request object - :param resource_id: resource identifier - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - config = deepcopy(self.config) - headers = request.get_response_headers() - - try: - LOGGER.debug('Verifying resource exists') - config['resources'][resource_id] - except KeyError: - msg = f'Resource not found: {resource_id}' - return self.get_exception( - 400, headers, request.format, 'ResourceNotFound', msg - ) - - data = request.data - if not data: - msg = 'missing request data' - return self.get_exception( - 400, headers, request.format, 'MissingParameterValue', msg - ) - - try: - # Parse data - data = data.decode() - except (UnicodeDecodeError, AttributeError): - pass - - try: - data = json.loads(data) - except (json.decoder.JSONDecodeError, TypeError) as err: - # Input is not valid JSON - LOGGER.error(err) - msg = 'invalid request data' - return self.get_exception( - 400, headers, request.format, 'InvalidParameterValue', msg - ) - - LOGGER.debug(f'Updating resource: {resource_id}') - config['resources'].update({resource_id: data}) - try: - self.validate(config) - except ValidationError as err: - LOGGER.error(err) - msg = 'Schema validation error' - return self.get_exception( - 400, headers, request.format, 'ValidationError', msg - ) - - self.write(config) - - return headers, 204, {} - - @pre_process - def patch_resource( - self, request: Union[APIRequest, Any], resource_id: str - ) -> Tuple[dict, int, str]: - """ - Update partial resource configuration - - :param request: request object - :param resource_id: resource identifier - - :returns: tuple of headers, status code, content - """ - if not request.is_valid(): - return self.get_format_exception(request) - - config = deepcopy(self.config) - headers = request.get_response_headers() - - try: - LOGGER.debug('Verifying resource exists') - resource = config['resources'][resource_id] - except KeyError: - msg = f'Resource not found: {resource_id}' - return self.get_exception( - 400, headers, request.format, 'ResourceNotFound', msg - ) - - data = request.data - if not data: - msg = 'missing request data' - return self.get_exception( - 400, headers, request.format, 'MissingParameterValue', msg - ) - - try: - # Parse data - data = data.decode() - except (UnicodeDecodeError, AttributeError): - pass - - try: - data = json.loads(data) - except (json.decoder.JSONDecodeError, TypeError) as err: - # Input is not valid JSON - LOGGER.error(err) - msg = 'invalid request data' - return self.get_exception( - 400, headers, request.format, 'InvalidParameterValue', msg - ) - - LOGGER.debug('Merging resource block') - data = self.merge(resource, data) - LOGGER.debug('Updating resource') - config['resources'].update({resource_id: data}) - - try: - self.validate(config) - except ValidationError as err: - LOGGER.error(err) - msg = 'Schema validation error' - return self.get_exception( - 400, headers, request.format, 'ValidationError', msg - ) - - self.write(config) - - content = to_json(resource, self.pretty_print) - - return headers, 204, content +def get_resource( + admin: Admin, request: APIRequest, resource_id: str +) -> Tuple[dict, int, str]: + """ + Get resource configuration + + :param request: request object + :param resource_id: + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers() + + cfg = get_config(raw=True) + + try: + resource = cfg['resources'][resource_id] + except KeyError: + msg = f'Resource not found: {resource_id}' + return admin.get_exception( + 400, headers, request.format, 'ResourceNotFound', msg + ) + + if request.format == F_HTML: + content = render_j2_template( + admin.tpl_config, admin.config['server']['templates'], + 'admin/index.html', resource, request.locale + ) + else: + content = to_json(resource, admin.pretty_print) + + return headers, 200, content + + +def delete_resource( + admin: Admin, request: APIRequest, resource_id: str +) -> Tuple[dict, int, str]: + """ + Delete resource configuration + + :param request: request object + :param resource_id: resource identifier + + :returns: tuple of headers, status code, content + """ + + config = deepcopy(admin.config) + headers = request.get_response_headers() + + try: + LOGGER.debug(f'Removing resource configuration for: {resource_id}') + config['resources'].pop(resource_id) + except KeyError: + msg = f'Resource not found: {resource_id}' + return admin.get_exception( + 400, headers, request.format, 'ResourceNotFound', msg + ) + + LOGGER.debug('Resource removed, validating and saving configuration') + try: + admin.validate(config) + except ValidationError as err: + LOGGER.error(err) + msg = 'Schema validation error' + return admin.get_exception( + 400, headers, request.format, 'ValidationError', msg + ) + + admin.write(config) + + return headers, 204, {} + + +def put_resource( + admin: Admin, + request: APIRequest, + resource_id: str, +) -> Tuple[dict, int, str]: + """ + Update complete resource configuration + + :param request: request object + :param resource_id: resource identifier + + :returns: tuple of headers, status code, content + """ + + config = deepcopy(admin.config) + headers = request.get_response_headers() + + try: + LOGGER.debug('Verifying resource exists') + config['resources'][resource_id] + except KeyError: + msg = f'Resource not found: {resource_id}' + return admin.get_exception( + 400, headers, request.format, 'ResourceNotFound', msg + ) + + data = request.data + if not data: + msg = 'missing request data' + return admin.get_exception( + 400, headers, request.format, 'MissingParameterValue', msg + ) + + try: + # Parse data + data = data.decode() + except (UnicodeDecodeError, AttributeError): + pass + + try: + data = json.loads(data) + temporal_extents_str2datetime(data.get('extents', {})) + except (json.decoder.JSONDecodeError, TypeError) as err: + # Input is not valid JSON + LOGGER.error(err) + msg = 'invalid request data' + return admin.get_exception( + 400, headers, request.format, 'InvalidParameterValue', msg + ) + + LOGGER.debug(f'Updating resource: {resource_id}') + config['resources'].update({resource_id: data}) + try: + admin.validate(config) + except ValidationError as err: + LOGGER.error(err) + msg = 'Schema validation error' + return admin.get_exception( + 400, headers, request.format, 'ValidationError', msg + ) + + admin.write(config) + + return headers, 204, {} + + +def patch_resource( + admin: Admin, request: APIRequest, resource_id: str +) -> Tuple[dict, int, str]: + """ + Update partial resource configuration + + :param request: request object + :param resource_id: resource identifier + + :returns: tuple of headers, status code, content + """ + + config = deepcopy(admin.config) + headers = request.get_response_headers() + + try: + LOGGER.debug('Verifying resource exists') + resource = config['resources'][resource_id] + except KeyError: + msg = f'Resource not found: {resource_id}' + return admin.get_exception( + 400, headers, request.format, 'ResourceNotFound', msg + ) + + data = request.data + if not data: + msg = 'missing request data' + return admin.get_exception( + 400, headers, request.format, 'MissingParameterValue', msg + ) + + try: + # Parse data + data = data.decode() + except (UnicodeDecodeError, AttributeError): + pass + + try: + data = json.loads(data) + temporal_extents_str2datetime(data.get('extents', {})) + except (json.decoder.JSONDecodeError, TypeError) as err: + # Input is not valid JSON + LOGGER.error(err) + msg = 'invalid request data' + return admin.get_exception( + 400, headers, request.format, 'InvalidParameterValue', msg + ) + + LOGGER.debug('Merging resource block') + data = admin.merge(resource, data) + LOGGER.debug('Updating resource') + config['resources'].update({resource_id: data}) + + try: + admin.validate(config) + except ValidationError as err: + LOGGER.error(err) + msg = 'Schema validation error' + return admin.get_exception( + 400, headers, request.format, 'ValidationError', msg + ) + + admin.write(config) + + content = to_json(resource, admin.pretty_print) + + return headers, 204, content + + +def temporal_extents_str2datetime(extents: dict) -> None: + """ + Helper function to coerce datetime strings into datetime objects + + :extents: `dict` of pygeoapi resource extents object + + :returns: `None` (changes made directly) + """ + + try: + extents['temporal']['begin'] = parse_date(extents['temporal']['begin']) + extents['temporal']['end'] = parse_date(extents['temporal']['end']) + except (KeyError, TypeError): + LOGGER.debug('No temporal extents found') diff --git a/pygeoapi/api.py b/pygeoapi/api.py deleted file mode 100644 index fe14409d7..000000000 --- a/pygeoapi/api.py +++ /dev/null @@ -1,4305 +0,0 @@ -# ================================================================= -# -# Authors: Tom Kralidis -# Francesco Bartoli -# Sander Schaminee -# John A Stevenson -# Colin Blackburn -# Ricardo Garcia Silva -# -# Copyright (c) 2023 Tom Kralidis -# Copyright (c) 2022 Francesco Bartoli -# Copyright (c) 2022 John A Stevenson and Colin Blackburn -# Copyright (c) 2023 Ricardo Garcia Silva -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# ================================================================= -""" Root level code of pygeoapi, parsing content provided by web framework. -Returns content from plugins and sets responses. -""" - -import asyncio -from collections import OrderedDict -from copy import deepcopy -from datetime import datetime, timezone -from functools import partial -from gzip import compress -from http import HTTPStatus -import json -import logging -import re -from typing import Any, Tuple, Union, Optional -import urllib.parse - -from dateutil.parser import parse as dateparse -from pygeofilter.parsers.ecql import parse as parse_ecql_text -from pygeofilter.parsers.cql_json import parse as parse_cql_json -from pyproj.exceptions import CRSError -import pytz -from shapely.errors import WKTReadingError -from shapely.wkt import loads as shapely_loads - -from pygeoapi import __version__, l10n -from pygeoapi.formatter.base import FormatterSerializationError -from pygeoapi.linked_data import (geojson2jsonld, jsonldify, - jsonldify_collection) -from pygeoapi.log import setup_logger -from pygeoapi.process.base import ( - JobNotFoundError, - JobResultNotFoundError, - ProcessorExecuteError, -) -from pygeoapi.process.manager.base import get_manager -from pygeoapi.plugin import load_plugin, PLUGINS -from pygeoapi.provider.base import ( - ProviderGenericError, ProviderConnectionError, ProviderNotFoundError, - ProviderTypeError) -from pygeoapi.models.provider.base import (TilesMetadataFormat, - TileMatrixSetEnum) - -from pygeoapi.models.cql import CQLModel -from pygeoapi.util import (dategetter, RequestedProcessExecutionMode, - DATETIME_FORMAT, UrlPrefetcher, - filter_dict_by_key_value, get_provider_by_type, - get_provider_default, get_typed_value, JobStatus, - json_serial, render_j2_template, str2bool, - TEMPLATES, to_json, get_api_rules, get_base_url, - get_crs_from_uri, get_supported_crs_list, - modify_pygeofilter, CrsTransformSpec, - transform_bbox) - -LOGGER = logging.getLogger(__name__) - -#: Return headers for requests (e.g:X-Powered-By) -HEADERS = { - 'Content-Type': 'application/json', - 'X-Powered-By': f'pygeoapi {__version__}' -} - -CHARSET = ['utf-8'] -F_JSON = 'json' -F_HTML = 'html' -F_JSONLD = 'jsonld' -F_GZIP = 'gzip' -F_PNG = 'png' -F_MVT = 'mvt' -F_NETCDF = 'NetCDF' - -#: Formats allowed for ?f= requests (order matters for complex MIME types) -FORMAT_TYPES = OrderedDict(( - (F_HTML, 'text/html'), - (F_JSONLD, 'application/ld+json'), - (F_JSON, 'application/json'), - (F_PNG, 'image/png'), - (F_MVT, 'application/vnd.mapbox-vector-tile'), - (F_NETCDF, 'application/x-netcdf'), -)) - -#: Locale used for system responses (e.g. exceptions) -SYSTEM_LOCALE = l10n.Locale('en', 'US') - -CONFORMANCE = { - 'common': [ - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/landing-page', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/json', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/html', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/oas30' - ], - 'feature': [ - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/req/oas30', - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/html', - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson', - 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs', - 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables', - 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables-query-parameters', # noqa - 'http://www.opengis.net/spec/ogcapi-features-4/1.0/conf/create-replace-delete' # noqa - ], - 'coverage': [ - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/oas30', - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/html', - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/geodata-coverage', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-subset', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-rangesubset', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-bbox', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-datetime' # noqa - ], - 'map': [ - 'http://www.opengis.net/spec/ogcapi-maps-1/1.0/conf/core' - ], - 'tile': [ - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/mvt', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tileset', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tilesets-list', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/oas30', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/geodata-tilesets' - ], - 'record': [ - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/sorting', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/opensearch', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/json', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/html' - ], - 'process': [ - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/ogc-process-description', # noqa - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/json', - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/oas30' - ], - 'edr': [ - 'http://www.opengis.net/spec/ogcapi-edr-1/1.0/conf/core' - ] -} - -OGC_RELTYPES_BASE = 'http://www.opengis.net/def/rel/ogc/1.0' - -DEFAULT_CRS_LIST = [ - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', -] - -DEFAULT_CRS = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' -DEFAULT_STORAGE_CRS = DEFAULT_CRS - - -def pre_process(func): - """ - Decorator that transforms an incoming Request instance specific to the - web framework (i.e. Flask, Starlette or Django) into a generic - :class:`APIRequest` instance. - - :param func: decorated function - - :returns: `func` - """ - - def inner(*args): - cls, req_in = args[:2] - req_out = APIRequest.with_data(req_in, getattr(cls, 'locales', set())) - if len(args) > 2: - return func(cls, req_out, *args[2:]) - else: - return func(cls, req_out) - - return inner - - -def gzip(func): - """ - Decorator that compresses the content of an outgoing API result - instance if the Content-Encoding response header was set to gzip. - - :param func: decorated function - - :returns: `func` - """ - - def inner(*args, **kwargs): - headers, status, content = func(*args, **kwargs) - charset = CHARSET[0] - if F_GZIP in headers.get('Content-Encoding', []): - try: - if isinstance(content, bytes): - # bytes means Content-Type needs to be set upstream - content = compress(content) - else: - headers['Content-Type'] = \ - f"{headers['Content-Type']}; charset={charset}" - content = compress(content.encode(charset)) - except TypeError as err: - headers.pop('Content-Encoding') - LOGGER.error(f'Error in compression: {err}') - - return headers, status, content - - return inner - - -class APIRequest: - """ - Transforms an incoming server-specific Request into an object - with some generic helper methods and properties. - - .. note:: Typically, this instance is created automatically by the - :func:`pre_process` decorator. **Every** API method that has - been routed to a REST endpoint should be decorated by the - :func:`pre_process` function. - Therefore, **all** routed API methods should at least have 1 - argument that holds the (transformed) request. - - The following example API method will: - - - transform the incoming Flask/Starlette/Django `Request` into an - `APIRequest`using the :func:`pre_process` decorator; - - call :meth:`is_valid` to check if the incoming request was valid, i.e. - that the user requested a valid output format or no format at all - (which means the default format); - - call :meth:`API.get_format_exception` if the requested format was - invalid; - - create a `dict` with the appropriate `Content-Type` header for the - requested format and a `Content-Language` header if any specific language - was requested. - - .. code-block:: python - - @pre_process - def example_method(self, request: Union[APIRequest, Any], custom_arg): - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers() - - # generate response_body here - - return headers, HTTPStatus.OK, response_body - - - The following example API method is similar as the one above, but will also - allow the user to request a non-standard format (e.g. ``f=xml``). - If `xml` was requested, we set the `Content-Type` ourselves. For the - standard formats, the `APIRequest` object sets the `Content-Type`. - - .. code-block:: python - - @pre_process - def example_method(self, request: Union[APIRequest, Any], custom_arg): - if not request.is_valid(['xml']): - return self.get_format_exception(request) - - content_type = 'application/xml' if request.format == 'xml' else None - headers = request.get_response_headers(content_type) - - # generate response_body here - - return headers, HTTPStatus.OK, response_body - - Note that you don't *have* to call :meth:`is_valid`, but that you can also - perform a custom check on the requested output format by looking at the - :attr:`format` property. - Other query parameters are available through the :attr:`params` property as - a `dict`. The request body is available through the :attr:`data` property. - - .. note:: If the request data (body) is important, **always** create a - new `APIRequest` instance using the :meth:`with_data` factory - method. - The :func:`pre_process` decorator will use this automatically. - - :param request: The web platform specific Request instance. - :param supported_locales: List or set of supported Locale instances. - """ - def __init__(self, request, supported_locales): - # Set default request data - self._data = b'' - - # Copy request query parameters - self._args = self._get_params(request) - - # Get path info - if hasattr(request, 'scope'): - self._path_info = request.scope['path'].strip('/') - elif hasattr(request.headers, 'environ'): - self._path_info = request.headers.environ['PATH_INFO'].strip('/') - elif hasattr(request, 'path_info'): - self._path_info = request.path_info - - # Extract locale from params or headers - self._raw_locale, self._locale = self._get_locale(request.headers, - supported_locales) - - # Determine format - self._format = self._get_format(request.headers) - - # Get received headers - self._headers = self.get_request_headers(request.headers) - - @classmethod - def with_data(cls, request, supported_locales) -> 'APIRequest': - """ - Factory class method to create an `APIRequest` instance with data. - - If the request body is required, an `APIRequest` should always be - instantiated using this class method. The reason for this is, that the - Starlette request body needs to be awaited (async), which cannot be - achieved in the :meth:`__init__` method of the `APIRequest`. - However, `APIRequest` can still be initialized using :meth:`__init__`, - but then the :attr:`data` property value will always be empty. - - :param request: The web platform specific Request instance. - :param supported_locales: List or set of supported Locale instances. - :returns: An `APIRequest` instance with data. - """ - - api_req = cls(request, supported_locales) - if hasattr(request, 'data'): - # Set data from Flask request - api_req._data = request.data - elif hasattr(request, 'body'): - if 'django' in str(request.__class__): - # Set data from Django request - api_req._data = request.body - else: - # Set data from Starlette request after async - # coroutine completion - # TODO: - # this now blocks, but once Flask v2 with async support - # has been implemented, with_data() can become async too - loop = asyncio.get_event_loop() - api_req._data = asyncio.run_coroutine_threadsafe( - request.body(), loop).result(1) - return api_req - - @staticmethod - def _get_params(request): - """ - Extracts the query parameters from the `Request` object. - - :param request: A Flask or Starlette Request instance - :returns: `ImmutableMultiDict` or empty `dict` - """ - - if hasattr(request, 'args'): - # Return ImmutableMultiDict from Flask request - return request.args - elif hasattr(request, 'query_params'): - # Return ImmutableMultiDict from Starlette request - return request.query_params - elif hasattr(request, 'GET'): - # Return QueryDict from Django GET request - return request.GET - elif hasattr(request, 'POST'): - # Return QueryDict from Django GET request - return request.POST - LOGGER.debug('No query parameters found') - return {} - - def _get_locale(self, headers, supported_locales): - """ - Detects locale from "lang=" param or `Accept-Language` - header. Returns a tuple of (raw, locale) if found in params or headers. - Returns a tuple of (raw default, default locale) if not found. - - :param headers: A dict with Request headers - :param supported_locales: List or set of supported Locale instances - :returns: A tuple of (str, Locale) - """ - - raw = None - try: - default_locale = l10n.str2locale(supported_locales[0]) - except (TypeError, IndexError, l10n.LocaleError) as err: - # This should normally not happen, since the API class already - # loads the supported languages from the config, which raises - # a LocaleError if any of these languages are invalid. - LOGGER.error(err) - raise ValueError(f"{self.__class__.__name__} must be initialized" - f"with a list of valid supported locales") - - for func, mapping in ((l10n.locale_from_params, self._args), - (l10n.locale_from_headers, headers)): - loc_str = func(mapping) - if loc_str: - if not raw: - # This is the first-found locale string: set as raw - raw = loc_str - # Check if locale string is a good match for the UI - loc = l10n.best_match(loc_str, supported_locales) - is_override = func is l10n.locale_from_params - if loc != default_locale or is_override: - return raw, loc - - return raw, default_locale - - def _get_format(self, headers) -> Union[str, None]: - """ - Get `Request` format type from query parameters or headers. - - :param headers: Dict of Request headers - :returns: format value or None if not found/specified - """ - - # Optional f=html or f=json query param - # Overrides Accept header and might differ from FORMAT_TYPES - format_ = (self._args.get('f') or '').strip() - if format_: - return format_ - - # Format not specified: get from Accept headers (MIME types) - # e.g. format_ = 'text/html' - h = headers.get('accept', headers.get('Accept', '')).strip() # noqa - (fmts, mimes) = zip(*FORMAT_TYPES.items()) - # basic support for complex types (i.e. with "q=0.x") - for type_ in (t.split(';')[0].strip() for t in h.split(',') if t): - if type_ in mimes: - idx_ = mimes.index(type_) - format_ = fmts[idx_] - break - - return format_ or None - - @property - def data(self) -> bytes: - """Returns the additional data send with the Request (bytes)""" - return self._data - - @property - def params(self) -> dict: - """Returns the Request query parameters dict""" - return self._args - - @property - def path_info(self) -> str: - """Returns the web server request path info part""" - return self._path_info - - @property - def locale(self) -> l10n.Locale: - """ - Returns the user-defined locale from the request object. - If no locale has been defined or if it is invalid, - the default server locale is returned. - - .. note:: The locale here determines the language in which pygeoapi - should return its responses. This may not be the language - that the user requested. It may also not be the language - that is supported by a collection provider, for example. - For this reason, you should pass the `raw_locale` property - to the :func:`l10n.get_plugin_locale` function, so that - the best match for the provider can be determined. - - :returns: babel.core.Locale - """ - - return self._locale - - @property - def raw_locale(self) -> Union[str, None]: - """ - Returns the raw locale string from the `Request` object. - If no "lang" query parameter or `Accept-Language` header was found, - `None` is returned. - Pass this value to the :func:`l10n.get_plugin_locale` function to let - the provider determine a best match for the locale, which may be - different from the locale used by pygeoapi's UI. - - :returns: a locale string or None - """ - - return self._raw_locale - - @property - def format(self) -> Union[str, None]: - """ - Returns the content type format from the - request query parameters or headers. - - :returns: Format name or None - """ - - return self._format - - @property - def headers(self) -> dict: - """ - Returns the dictionary of the headers from - the request. - - :returns: Request headers dictionary - """ - - return self._headers - - def get_linkrel(self, format_: str) -> str: - """ - Returns the hyperlink relationship (rel) attribute value for - the given API format string. - - The string is compared against the request format and if it matches, - the value 'self' is returned. Otherwise, 'alternate' is returned. - However, if `format_` is 'json' and *no* request format was found, - the relationship 'self' is returned as well (JSON is the default). - - :param format_: The format to compare the request format against. - :returns: A string 'self' or 'alternate'. - """ - - fmt = format_.lower() - if fmt == self._format or (fmt == F_JSON and not self._format): - return 'self' - return 'alternate' - - def is_valid(self, additional_formats=None) -> bool: - """ - Returns True if: - - the format is not set (None) - - the requested format is supported - - the requested format exists in a list if additional formats - - .. note:: Format names are matched in a case-insensitive manner. - - :param additional_formats: Optional additional supported formats list - - :returns: bool - """ - - if not self._format: - return True - if self._format in FORMAT_TYPES.keys(): - return True - if self._format in (f.lower() for f in (additional_formats or ())): - return True - return False - - def get_response_headers(self, force_lang: l10n.Locale = None, - force_type: str = None, - force_encoding: str = None, - **custom_headers) -> dict: - """ - Prepares and returns a dictionary with Response object headers. - - This method always adds a 'Content-Language' header, where the value - is determined by the 'lang' query parameter or 'Accept-Language' - header from the request. - If no language was requested, the default pygeoapi language is used, - unless a `force_lang` override was specified (see notes below). - - A 'Content-Type' header is also always added to the response. - If the user does not specify `force_type`, the header is based on - the `format` APIRequest property. If that is invalid, the default MIME - type `application/json` is used. - - ..note:: If a `force_lang` override is applied, that language - is always set as the 'Content-Language', regardless of - a 'lang' query parameter or 'Accept-Language' header. - If an API response always needs to be in the same - language, 'force_lang' should be set to that language. - - :param force_lang: An optional Content-Language header override. - :param force_type: An optional Content-Type header override. - :param force_encoding: An optional Content-Encoding header override. - :returns: A header dict - """ - - headers = HEADERS.copy() - headers.update(**custom_headers) - l10n.set_response_language(headers, force_lang or self._locale) - if force_type: - # Set custom MIME type if specified - headers['Content-Type'] = force_type - elif self.is_valid() and self._format: - # Set MIME type for valid formats - headers['Content-Type'] = FORMAT_TYPES[self._format] - - if F_GZIP in FORMAT_TYPES: - if force_encoding: - headers['Content-Encoding'] = force_encoding - elif F_GZIP in self._headers.get('Accept-Encoding', ''): - headers['Content-Encoding'] = F_GZIP - - return headers - - def get_request_headers(self, headers) -> dict: - """ - Obtains and returns a dictionary with Request object headers. - - This method adds the headers of the original request and - makes them available to the API object. - - :returns: A header dict - """ - - headers_ = {item[0]: item[1] for item in headers.items()} - return headers_ - - -class API: - """API object""" - - def __init__(self, config, openapi): - """ - constructor - - :param config: configuration dict - :param openapi: openapi dict - - :returns: `pygeoapi.API` instance - """ - - self.config = config - self.openapi = openapi - self.api_headers = get_api_rules(self.config).response_headers - self.base_url = get_base_url(self.config) - self.prefetcher = UrlPrefetcher() - - CHARSET[0] = config['server'].get('encoding', 'utf-8') - if config['server'].get('gzip'): - FORMAT_TYPES[F_GZIP] = 'application/gzip' - FORMAT_TYPES.move_to_end(F_JSON) - - # Process language settings (first locale is default!) - self.locales = l10n.get_locales(config) - self.default_locale = self.locales[0] - - if 'templates' not in self.config['server']: - self.config['server']['templates'] = {'path': TEMPLATES} - - if 'pretty_print' not in self.config['server']: - self.config['server']['pretty_print'] = False - - self.pretty_print = self.config['server']['pretty_print'] - - setup_logger(self.config['logging']) - - # Create config clone for HTML templating with modified base URL - self.tpl_config = deepcopy(self.config) - self.tpl_config['server']['url'] = self.base_url - - self.manager = get_manager(self.config) - LOGGER.info('Process manager plugin loaded') - - @gzip - @pre_process - @jsonldify - def landing_page(self, - request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: - """ - Provide API landing page - - :param request: A request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - fcm = { - 'links': [], - 'title': l10n.translate( - self.config['metadata']['identification']['title'], - request.locale), - 'description': - l10n.translate( - self.config['metadata']['identification']['description'], - request.locale) - } - - LOGGER.debug('Creating links') - # TODO: put title text in config or translatable files? - fcm['links'] = [{ - 'rel': request.get_linkrel(F_JSON), - 'type': FORMAT_TYPES[F_JSON], - 'title': 'This document as JSON', - 'href': f"{self.base_url}?f={F_JSON}" - }, { - 'rel': request.get_linkrel(F_JSONLD), - 'type': FORMAT_TYPES[F_JSONLD], - 'title': 'This document as RDF (JSON-LD)', - 'href': f"{self.base_url}?f={F_JSONLD}" - }, { - 'rel': request.get_linkrel(F_HTML), - 'type': FORMAT_TYPES[F_HTML], - 'title': 'This document as HTML', - 'href': f"{self.base_url}?f={F_HTML}", - 'hreflang': self.default_locale - }, { - 'rel': 'service-desc', - 'type': 'application/vnd.oai.openapi+json;version=3.0', - 'title': 'The OpenAPI definition as JSON', - 'href': f"{self.base_url}/openapi" - }, { - 'rel': 'service-doc', - 'type': FORMAT_TYPES[F_HTML], - 'title': 'The OpenAPI definition as HTML', - 'href': f"{self.base_url}/openapi?f={F_HTML}", - 'hreflang': self.default_locale - }, { - 'rel': 'conformance', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Conformance', - 'href': f"{self.base_url}/conformance" - }, { - 'rel': 'data', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Collections', - 'href': self.get_collections_url() - }, { - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/processes', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Processes', - 'href': f"{self.base_url}/processes" - }, { - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Jobs', - 'href': f"{self.base_url}/jobs" - }, { - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-schemes', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'The list of supported tiling schemes (as JSON)', - 'href': f"{self.base_url}/TileMatrixSets?f=json" - }, { - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-schemes', - 'type': FORMAT_TYPES[F_HTML], - 'title': 'The list of supported tiling schemes (as HTML)', - 'href': f"{self.base_url}/TileMatrixSets?f=html" - }] - - headers = request.get_response_headers(**self.api_headers) - if request.format == F_HTML: # render - - fcm['processes'] = False - fcm['stac'] = False - fcm['collection'] = False - - if filter_dict_by_key_value(self.config['resources'], - 'type', 'process'): - fcm['processes'] = True - - if filter_dict_by_key_value(self.config['resources'], - 'type', 'stac-collection'): - fcm['stac'] = True - - if filter_dict_by_key_value(self.config['resources'], - 'type', 'collection'): - fcm['collection'] = True - - content = render_j2_template(self.tpl_config, 'landing_page.html', - fcm, request.locale) - return headers, HTTPStatus.OK, content - - if request.format == F_JSONLD: - return headers, HTTPStatus.OK, to_json( - self.fcmld, self.pretty_print) - - return headers, HTTPStatus.OK, to_json(fcm, self.pretty_print) - - @gzip - @pre_process - def openapi_(self, request: Union[APIRequest, Any]) -> Tuple[ - dict, int, str]: - """ - Provide OpenAPI document - - :param request: A request object - :param openapi: dict of OpenAPI definition - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers(**self.api_headers) - - if request.format == F_HTML: - template = 'openapi/swagger.html' - if request._args.get('ui') == 'redoc': - template = 'openapi/redoc.html' - - path = f'{self.base_url}/openapi' - data = { - 'openapi-document-path': path - } - content = render_j2_template(self.tpl_config, template, data, - request.locale) - return headers, HTTPStatus.OK, content - - headers['Content-Type'] = 'application/vnd.oai.openapi+json;version=3.0' # noqa - - if isinstance(self.openapi, dict): - return headers, HTTPStatus.OK, to_json(self.openapi, - self.pretty_print) - else: - return headers, HTTPStatus.OK, self.openapi - - @gzip - @pre_process - def conformance(self, - request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: - """ - Provide conformance definition - - :param request: A request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - conformance_list = CONFORMANCE['common'] - - for key, value in self.config['resources'].items(): - if value['type'] == 'process': - conformance_list.extend(CONFORMANCE[value['type']]) - else: - for provider in value['providers']: - if provider['type'] in CONFORMANCE: - conformance_list.extend(CONFORMANCE[provider['type']]) - - conformance = { - 'conformsTo': list(set(conformance_list)) - } - - headers = request.get_response_headers(**self.api_headers) - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, 'conformance.html', - conformance, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(conformance, self.pretty_print) - - @gzip - @pre_process - def tilematrixsets(self, - request: Union[APIRequest, Any]) -> Tuple[dict, int, - str]: - """ - Provide tileMatrixSets definition - - :param request: A request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers(**self.api_headers) - - # Retrieve available TileMatrixSets - enums = [e.value for e in TileMatrixSetEnum] - - tms = {"tileMatrixSets": []} - - for e in enums: - tms['tileMatrixSets'].append({ - "title": e.title, - "id": e.tileMatrixSet, - "uri": e.tileMatrixSetURI, - "links": [ - { - "rel": "self", - "type": "text/html", - "title": f"The HTML representation of the {e.tileMatrixSet} tile matrix set", # noqa - "href": f"{self.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=html" # noqa - }, - { - "rel": "self", - "type": "application/json", - "title": f"The JSON representation of the {e.tileMatrixSet} tile matrix set", # noqa - "href": f"{self.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=json" # noqa - } - ] - }) - - tms['links'] = [{ - "rel": "alternate", - "type": "text/html", - "title": "This document as HTML", - "href": f"{self.base_url}/tileMatrixSets?f=html" - }, { - "rel": "self", - "type": "application/json", - "title": "This document", - "href": f"{self.base_url}/tileMatrixSets?f=json" - }] - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'tilematrixsets/index.html', - tms, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(tms, self.pretty_print) - - @gzip - @pre_process - def tilematrixset(self, - request: Union[APIRequest, Any], - tileMatrixSetId) -> Tuple[dict, - int, str]: - """ - Provide tile matrix definition - - :param request: A request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers(**self.api_headers) - - # Retrieve relevant TileMatrixSet - enums = [e.value for e in TileMatrixSetEnum] - enum = None - - try: - for e in enums: - if tileMatrixSetId == e.tileMatrixSet: - enum = e - if not enum: - raise ValueError('could not find this tilematrixset') - except ValueError as err: - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', str(err)) - - tms = { - "title": enum.tileMatrixSet, - "crs": enum.crs, - "id": enum.tileMatrixSet, - "uri": enum.tileMatrixSetURI, - "orderedAxes": enum.orderedAxes, - "wellKnownScaleSet": enum.wellKnownScaleSet, - "tileMatrices": enum.tileMatrices - } - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'tilematrixsets/tilematrixset.html', - tms, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(tms, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def describe_collections(self, request: Union[APIRequest, Any], - dataset=None) -> Tuple[dict, int, str]: - """ - Provide collection metadata - - :param request: A request object - :param dataset: name of collection - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - fcm = { - 'collections': [], - 'links': [] - } - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if all([dataset is not None, dataset not in collections.keys()]): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - if dataset is not None: - collections_dict = { - k: v for k, v in collections.items() if k == dataset - } - else: - collections_dict = collections - - LOGGER.debug('Creating collections') - for k, v in collections_dict.items(): - if v.get('visibility', 'default') == 'hidden': - LOGGER.debug(f'Skipping hidden layer: {k}') - continue - collection_data = get_provider_default(v['providers']) - collection_data_type = collection_data['type'] - - collection_data_format = None - - if 'format' in collection_data: - collection_data_format = collection_data['format'] - - collection = { - 'id': k, - 'title': l10n.translate(v['title'], request.locale), - 'description': l10n.translate(v['description'], request.locale), # noqa - 'keywords': l10n.translate(v['keywords'], request.locale), - 'links': [] - } - - bbox = v['extents']['spatial']['bbox'] - # The output should be an array of bbox, so if the user only - # provided a single bbox, wrap it in a array. - if not isinstance(bbox[0], list): - bbox = [bbox] - collection['extent'] = { - 'spatial': { - 'bbox': bbox - } - } - if 'crs' in v['extents']['spatial']: - collection['extent']['spatial']['crs'] = \ - v['extents']['spatial']['crs'] - - t_ext = v.get('extents', {}).get('temporal', {}) - if t_ext: - begins = dategetter('begin', t_ext) - ends = dategetter('end', t_ext) - collection['extent']['temporal'] = { - 'interval': [[begins, ends]] - } - if 'trs' in t_ext: - collection['extent']['temporal']['trs'] = t_ext['trs'] - - LOGGER.debug('Processing configured collection links') - for link in l10n.translate(v.get('links', []), request.locale): - lnk = { - 'type': link['type'], - 'rel': link['rel'], - 'title': l10n.translate(link['title'], request.locale), - 'href': l10n.translate(link['href'], request.locale), - } - if 'hreflang' in link: - lnk['hreflang'] = l10n.translate( - link['hreflang'], request.locale) - content_length = link.get('length', 0) - - if lnk['rel'] == 'enclosure' and content_length == 0: - # Issue HEAD request for enclosure links without length - lnk_headers = self.prefetcher.get_headers(lnk['href']) - content_length = int(lnk_headers.get('content-length', 0)) - content_type = lnk_headers.get('content-type', lnk['type']) - if content_length == 0: - # Skip this (broken) link - LOGGER.debug(f"Enclosure {lnk['href']} is invalid") - continue - if content_type != lnk['type']: - # Update content type if different from specified - lnk['type'] = content_type - LOGGER.debug( - f"Fixed media type for enclosure {lnk['href']}") - - if content_length > 0: - lnk['length'] = content_length - - collection['links'].append(lnk) - - # TODO: provide translations - LOGGER.debug('Adding JSON and HTML link relations') - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'root', - 'title': 'The landing page of this server as JSON', - 'href': f"{self.base_url}?f={F_JSON}" - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'root', - 'title': 'The landing page of this server as HTML', - 'href': f"{self.base_url}?f={F_HTML}" - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as JSON', - 'href': f'{self.get_collections_url()}/{k}?f={F_JSON}' - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': request.get_linkrel(F_JSONLD), - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{self.get_collections_url()}/{k}?f={F_JSONLD}' - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{self.get_collections_url()}/{k}?f={F_HTML}' - }) - - if collection_data_type in ['feature', 'record', 'tile']: - # TODO: translate - collection['itemType'] = collection_data_type - LOGGER.debug('Adding feature/record based links') - collection['links'].append({ - 'type': 'application/schema+json', - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/queryables', - 'title': 'Queryables for this collection as JSON', - 'href': f'{self.get_collections_url()}/{k}/queryables?f={F_JSON}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/queryables', - 'title': 'Queryables for this collection as HTML', - 'href': f'{self.get_collections_url()}/{k}/queryables?f={F_HTML}' # noqa - }) - collection['links'].append({ - 'type': 'application/geo+json', - 'rel': 'items', - 'title': 'items as GeoJSON', - 'href': f'{self.get_collections_url()}/{k}/items?f={F_JSON}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': 'items', - 'title': 'items as RDF (GeoJSON-LD)', - 'href': f'{self.get_collections_url()}/{k}/items?f={F_JSONLD}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'items', - 'title': 'Items as HTML', - 'href': f'{self.get_collections_url()}/{k}/items?f={F_HTML}' # noqa - }) - - # OAPIF Part 2 - list supported CRSs and StorageCRS - if collection_data_type == 'feature': - collection['crs'] = get_supported_crs_list(collection_data, DEFAULT_CRS_LIST) # noqa - collection['storageCRS'] = collection_data.get('storage_crs', DEFAULT_STORAGE_CRS) # noqa - if 'storage_crs_coordinate_epoch' in collection_data: - collection['storageCrsCoordinateEpoch'] = collection_data.get('storage_crs_coordinate_epoch') # noqa - - elif collection_data_type == 'coverage': - # TODO: translate - LOGGER.debug('Adding coverage based links') - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'collection', - 'title': 'Detailed Coverage metadata in JSON', - 'href': f'{self.get_collections_url()}/{k}?f={F_JSON}' - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'collection', - 'title': 'Detailed Coverage metadata in HTML', - 'href': f'{self.get_collections_url()}/{k}?f={F_HTML}' - }) - coverage_url = f'{self.get_collections_url()}/{k}/coverage' - - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': f'{OGC_RELTYPES_BASE}/coverage-domainset', - 'title': 'Coverage domain set of collection in JSON', - 'href': f'{coverage_url}/domainset?f={F_JSON}' - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': f'{OGC_RELTYPES_BASE}/coverage-domainset', - 'title': 'Coverage domain set of collection in HTML', - 'href': f'{coverage_url}/domainset?f={F_HTML}' - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': f'{OGC_RELTYPES_BASE}/coverage-rangetype', - 'title': 'Coverage range type of collection in JSON', - 'href': f'{coverage_url}/rangetype?f={F_JSON}' - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': f'{OGC_RELTYPES_BASE}/coverage-rangetype', - 'title': 'Coverage range type of collection in HTML', - 'href': f'{coverage_url}/rangetype?f={F_HTML}' - }) - collection['links'].append({ - 'type': 'application/prs.coverage+json', - 'rel': f'{OGC_RELTYPES_BASE}/coverage', - 'title': 'Coverage data', - 'href': f'{self.get_collections_url()}/{k}/coverage?f={F_JSON}' # noqa - }) - if collection_data_format is not None: - collection['links'].append({ - 'type': collection_data_format['mimetype'], - 'rel': f'{OGC_RELTYPES_BASE}/coverage', - 'title': f"Coverage data as {collection_data_format['name']}", # noqa - 'href': f"{self.get_collections_url()}/{k}/coverage?f={collection_data_format['name']}" # noqa - }) - if dataset is not None: - LOGGER.debug('Creating extended coverage metadata') - try: - provider_def = get_provider_by_type( - self.config['resources'][k]['providers'], - 'coverage') - p = load_plugin('provider', provider_def) - except ProviderConnectionError: - msg = 'connection error (check logs)' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, - headers, request.format, - 'NoApplicableCode', msg) - except ProviderTypeError: - pass - else: - collection['crs'] = [p.crs] - collection['domainset'] = p.get_coverage_domainset() - collection['rangetype'] = p.get_coverage_rangetype() - - try: - tile = get_provider_by_type(v['providers'], 'tile') - p = load_plugin('provider', tile) - except ProviderConnectionError: - msg = 'connection error (check logs)' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, - headers, request.format, - 'NoApplicableCode', msg) - except ProviderTypeError: - tile = None - - if tile: - # TODO: translate - - LOGGER.debug('Adding tile links') - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': f'http://www.opengis.net/def/rel/ogc/1.0/tilesets-{p.tile_type}', # noqa - 'title': 'Tiles as JSON', - 'href': f'{self.get_collections_url()}/{k}/tiles?f={F_JSON}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': f'http://www.opengis.net/def/rel/ogc/1.0/tilesets-{p.tile_type}', # noqa - 'title': 'Tiles as HTML', - 'href': f'{self.get_collections_url()}/{k}/tiles?f={F_HTML}' # noqa - }) - - try: - map_ = get_provider_by_type(v['providers'], 'map') - except ProviderTypeError: - map_ = None - - if map_: - LOGGER.debug('Adding map links') - - map_mimetype = map_['format']['mimetype'] - map_format = map_['format']['name'] - - collection['links'].append({ - 'type': map_mimetype, - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/map', - 'title': f'Map as {map_format}', - 'href': f"{self.get_collections_url()}/{k}/map?f={map_format}" # noqa - }) - - try: - edr = get_provider_by_type(v['providers'], 'edr') - p = load_plugin('provider', edr) - except ProviderConnectionError: - msg = 'connection error (check logs)' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'NoApplicableCode', msg) - except ProviderTypeError: - edr = None - - if edr: - # TODO: translate - LOGGER.debug('Adding EDR links') - parameters = p.get_fields() - if parameters: - collection['parameter_names'] = {} - for f in parameters['field']: - collection['parameter_names'][f['id']] = f - - for qt in p.get_query_types(): - collection['links'].append({ - 'type': 'application/json', - 'rel': 'data', - 'title': f'{qt} query for this collection as JSON', - 'href': f'{self.get_collections_url()}/{k}/{qt}?f={F_JSON}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'data', - 'title': f'{qt} query for this collection as HTML', - 'href': f'{self.get_collections_url()}/{k}/{qt}?f={F_HTML}' # noqa - }) - - if dataset is not None and k == dataset: - fcm = collection - break - - fcm['collections'].append(collection) - - if dataset is None: - # TODO: translate - fcm['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as JSON', - 'href': f'{self.get_collections_url()}?f={F_JSON}' - }) - fcm['links'].append({ - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': request.get_linkrel(F_JSONLD), - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{self.get_collections_url()}?f={F_JSONLD}' - }) - fcm['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{self.get_collections_url()}?f={F_HTML}' - }) - - if request.format == F_HTML: # render - fcm['collections_path'] = self.get_collections_url() - if dataset is not None: - content = render_j2_template(self.tpl_config, - 'collections/collection.html', - fcm, request.locale) - else: - content = render_j2_template(self.tpl_config, - 'collections/index.html', fcm, - request.locale) - - return headers, HTTPStatus.OK, content - - if request.format == F_JSONLD: - jsonld = self.fcmld.copy() - if dataset is not None: - jsonld['dataset'] = jsonldify_collection(self, fcm, - request.locale) - else: - jsonld['dataset'] = [ - jsonldify_collection(self, c, request.locale) - for c in fcm.get('collections', []) - ] - return headers, HTTPStatus.OK, to_json(jsonld, self.pretty_print) - - return headers, HTTPStatus.OK, to_json(fcm, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def get_collection_queryables(self, request: Union[APIRequest, Any], - dataset=None) -> Tuple[dict, int, str]: - """ - Provide collection queryables - - :param request: A request object - :param dataset: name of collection - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection queryables') - try: - LOGGER.debug('Loading feature provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'feature')) - except ProviderTypeError: - LOGGER.debug('Loading record provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'record')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - queryables = { - 'type': 'object', - 'title': l10n.translate( - self.config['resources'][dataset]['title'], request.locale), - 'properties': {}, - '$schema': 'http://json-schema.org/draft/2019-09/schema', - '$id': f'{self.get_collections_url()}/{dataset}/queryables' - } - - if p.fields: - queryables['properties']['geometry'] = { - '$ref': 'https://geojson.org/schema/Geometry.json' - } - - for k, v in p.fields.items(): - show_field = False - if p.properties: - if k in p.properties: - show_field = True - else: - show_field = True - - if show_field: - queryables['properties'][k] = { - 'title': k, - 'type': v['type'] - } - if 'values' in v: - queryables['properties'][k]['enum'] = v['values'] - - if request.format == F_HTML: # render - queryables['title'] = l10n.translate( - self.config['resources'][dataset]['title'], request.locale) - - queryables['collections_path'] = self.get_collections_url() - - content = render_j2_template(self.tpl_config, - 'collections/queryables.html', - queryables, request.locale) - - return headers, HTTPStatus.OK, content - - headers['Content-Type'] = 'application/schema+json' - - return headers, HTTPStatus.OK, to_json(queryables, self.pretty_print) - - @gzip - @pre_process - def get_collection_items( - self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Queries collection - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - properties = [] - reserved_fieldnames = ['bbox', 'bbox-crs', 'crs', 'f', 'lang', 'limit', - 'offset', 'resulttype', 'datetime', 'sortby', - 'properties', 'skipGeometry', 'q', - 'filter', 'filter-lang', 'filter-crs'] - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Processing offset parameter') - try: - offset = int(request.params.get('offset')) - if offset < 0: - msg = 'offset value should be positive or zero' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - offset = 0 - except ValueError: - msg = 'offset value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing limit parameter') - try: - limit = int(request.params.get('limit')) - # TODO: We should do more validation, against the min and max - # allowed by the server configuration - if limit <= 0: - msg = 'limit value should be strictly positive' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - limit = int(self.config['server']['limit']) - except ValueError: - msg = 'limit value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - resulttype = request.params.get('resulttype') or 'results' - - LOGGER.debug('Processing bbox parameter') - - bbox = request.params.get('bbox') - - if bbox is None: - bbox = [] - else: - try: - bbox = validate_bbox(bbox) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - datetime_ = validate_datetime(collections[dataset]['extents'], - datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('processing q parameter') - q = request.params.get('q') or None - - LOGGER.debug('Loading provider') - - provider_def = None - try: - provider_type = 'feature' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - try: - provider_type = 'record' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - msg = 'Invalid provider type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - crs_transform_spec = None - if provider_type == 'feature': - # crs query parameter is only available for OGC API - Features - # right now, not for OGC API - Records. - LOGGER.debug('Processing crs parameter') - query_crs_uri = request.params.get('crs') - try: - crs_transform_spec = self._create_crs_transform_spec( - provider_def, query_crs_uri, - ) - except (ValueError, CRSError) as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - self._set_content_crs_header(headers, provider_def, query_crs_uri) - - LOGGER.debug('Processing bbox-crs parameter') - bbox_crs = request.params.get('bbox-crs') - if bbox_crs is not None: - # Validate bbox-crs parameter - if len(bbox) == 0: - msg = 'bbox-crs specified without bbox parameter' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - - if len(bbox_crs) == 0: - msg = 'bbox-crs specified but is empty' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - - supported_crs_list = get_supported_crs_list(provider_def, DEFAULT_CRS_LIST) # noqa - if bbox_crs not in supported_crs_list: - msg = f'bbox-crs {bbox_crs} not supported for this collection' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - elif len(bbox) > 0: - # bbox but no bbox-crs parm: assume bbox is in default CRS - bbox_crs = DEFAULT_CRS - - # Transform bbox to storageCRS - # when bbox-crs different from storageCRS. - if len(bbox) > 0: - try: - # Get a pyproj CRS instance for the Collection's Storage CRS - storage_crs = provider_def.get('storage_crs', DEFAULT_STORAGE_CRS) # noqa - - # Do the (optional) Transform to the Storage CRS - bbox = transform_bbox(bbox, bbox_crs, storage_crs) - except CRSError as e: - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', str(e)) - - LOGGER.debug('processing property parameters') - for k, v in request.params.items(): - if k not in reserved_fieldnames and k in list(p.fields.keys()): - LOGGER.debug(f'Adding property filter {k}={v}') - properties.append((k, v)) - - LOGGER.debug('processing sort parameter') - val = request.params.get('sortby') - - if val is not None: - sortby = [] - sorts = val.split(',') - for s in sorts: - prop = s - order = '+' - if s[0] in ['+', '-']: - order = s[0] - prop = s[1:] - - if prop not in p.fields.keys(): - msg = 'bad sort property' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - sortby.append({'property': prop, 'order': order}) - else: - sortby = [] - - LOGGER.debug('processing properties parameter') - val = request.params.get('properties') - - if val is not None: - select_properties = val.split(',') - properties_to_check = set(p.properties) | set(p.fields.keys()) - - if (len(list(set(select_properties) - - set(properties_to_check))) > 0): - msg = 'unknown properties specified' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - select_properties = [] - - LOGGER.debug('processing skipGeometry parameter') - val = request.params.get('skipGeometry') - if val is not None: - skip_geometry = str2bool(val) - else: - skip_geometry = False - - LOGGER.debug('Processing filter-crs parameter') - filter_crs_uri = request.params.get('filter-crs', DEFAULT_CRS) - LOGGER.debug('processing filter parameter') - cql_text = request.params.get('filter') - if cql_text is not None: - try: - filter_ = parse_ecql_text(cql_text) - filter_ = modify_pygeofilter( - filter_, - filter_crs_uri=filter_crs_uri, - storage_crs_uri=provider_def.get('storage_crs'), - geometry_column_name=provider_def.get('geom_field'), - ) - except Exception as err: - LOGGER.error(err) - msg = f'Bad CQL string : {cql_text}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - filter_ = None - - LOGGER.debug('Processing filter-lang parameter') - filter_lang = request.params.get('filter-lang') - # Currently only cql-text is handled, but it is optional - if filter_lang not in [None, 'cql-text']: - msg = 'Invalid filter language' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - # Get provider locale (if any) - prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) - - LOGGER.debug('Querying provider') - LOGGER.debug(f'offset: {offset}') - LOGGER.debug(f'limit: {limit}') - LOGGER.debug(f'resulttype: {resulttype}') - LOGGER.debug(f'sortby: {sortby}') - LOGGER.debug(f'bbox: {bbox}') - if provider_type == 'feature': - LOGGER.debug(f'crs: {query_crs_uri}') - LOGGER.debug(f'datetime: {datetime_}') - LOGGER.debug(f'properties: {properties}') - LOGGER.debug(f'select properties: {select_properties}') - LOGGER.debug(f'skipGeometry: {skip_geometry}') - LOGGER.debug(f'language: {prv_locale}') - LOGGER.debug(f'q: {q}') - LOGGER.debug(f'cql_text: {cql_text}') - LOGGER.debug(f'filter_: {filter_}') - LOGGER.debug(f'filter-lang: {filter_lang}') - LOGGER.debug(f'filter-crs: {filter_crs_uri}') - - try: - content = p.query(offset=offset, limit=limit, - resulttype=resulttype, bbox=bbox, - datetime_=datetime_, properties=properties, - sortby=sortby, skip_geometry=skip_geometry, - select_properties=select_properties, - crs_transform_spec=crs_transform_spec, - q=q, language=prv_locale, filterq=filter_) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - serialized_query_params = '' - for k, v in request.params.items(): - if k not in ('f', 'offset'): - serialized_query_params += '&' - serialized_query_params += urllib.parse.quote(k, safe='') - serialized_query_params += '=' - serialized_query_params += urllib.parse.quote(str(v), safe=',') - - # TODO: translate titles - uri = f'{self.get_collections_url()}/{dataset}/items' - content['links'] = [{ - 'type': 'application/geo+json', - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as GeoJSON', - 'href': f'{uri}?f={F_JSON}{serialized_query_params}' - }, { - 'rel': request.get_linkrel(F_JSONLD), - 'type': FORMAT_TYPES[F_JSONLD], - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{uri}?f={F_JSONLD}{serialized_query_params}' - }, { - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{uri}?f={F_HTML}{serialized_query_params}' - }] - - if offset > 0: - prev = max(0, offset - limit) - content['links'].append( - { - 'type': 'application/geo+json', - 'rel': 'prev', - 'title': 'items (prev)', - 'href': f'{uri}?offset={prev}{serialized_query_params}' - }) - - if 'numberMatched' in content: - if content['numberMatched'] > (limit + offset): - next_ = offset + limit - next_href = f'{uri}?offset={next_}{serialized_query_params}' - content['links'].append( - { - 'type': 'application/geo+json', - 'rel': 'next', - 'title': 'items (next)', - 'href': next_href - }) - - content['links'].append( - { - 'type': FORMAT_TYPES[F_JSON], - 'title': l10n.translate( - collections[dataset]['title'], request.locale), - 'rel': 'collection', - 'href': uri - }) - - content['timeStamp'] = datetime.utcnow().strftime( - '%Y-%m-%dT%H:%M:%S.%fZ') - - # Set response language to requested provider locale - # (if it supports language) and/or otherwise the requested pygeoapi - # locale (or fallback default locale) - l10n.set_response_language(headers, prv_locale, request.locale) - - if request.format == F_HTML: # render - # For constructing proper URIs to items - - content['items_path'] = uri - content['dataset_path'] = '/'.join(uri.split('/')[:-1]) - content['collections_path'] = self.get_collections_url() - - content['offset'] = offset - - content['id_field'] = p.id_field - if p.uri_field is not None: - content['uri_field'] = p.uri_field - if p.title_field is not None: - content['title_field'] = l10n.translate(p.title_field, - request.locale) - # If title exists, use it as id in html templates - content['id_field'] = content['title_field'] - content = render_j2_template(self.tpl_config, - 'collections/items/index.html', - content, request.locale) - return headers, HTTPStatus.OK, content - elif request.format == 'csv': # render - formatter = load_plugin('formatter', - {'name': 'CSV', 'geom': True}) - - try: - content = formatter.write( - data=content, - options={ - 'provider_def': get_provider_by_type( - collections[dataset]['providers'], - 'feature') - } - ) - except FormatterSerializationError as err: - LOGGER.error(err) - msg = 'Error serializing output' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, - 'NoApplicableCode', msg) - - headers['Content-Type'] = formatter.mimetype - - if p.filename is None: - filename = f'{dataset}.csv' - else: - filename = f'{p.filename}' - - cd = f'attachment; filename="{filename}"' - headers['Content-Disposition'] = cd - - return headers, HTTPStatus.OK, content - - elif request.format == F_JSONLD: - content = geojson2jsonld( - self, content, dataset, id_field=(p.uri_field or 'id') - ) - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @gzip - @pre_process - def post_collection_items( - self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Queries collection or filter an item - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - request_headers = request.headers - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - properties = [] - reserved_fieldnames = ['bbox', 'f', 'limit', 'offset', - 'resulttype', 'datetime', 'sortby', - 'properties', 'skipGeometry', 'q', - 'filter-lang', 'filter-crs'] - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Invalid collection' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Processing offset parameter') - try: - offset = int(request.params.get('offset')) - if offset < 0: - msg = 'offset value should be positive or zero' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - offset = 0 - except ValueError: - msg = 'offset value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing limit parameter') - try: - limit = int(request.params.get('limit')) - # TODO: We should do more validation, against the min and max - # allowed by the server configuration - if limit <= 0: - msg = 'limit value should be strictly positive' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - limit = int(self.config['server']['limit']) - except ValueError: - msg = 'limit value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - resulttype = request.params.get('resulttype') or 'results' - - LOGGER.debug('Processing bbox parameter') - - bbox = request.params.get('bbox') - - if bbox is None: - bbox = [] - else: - try: - bbox = validate_bbox(bbox) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - datetime_ = validate_datetime(collections[dataset]['extents'], - datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('processing q parameter') - val = request.params.get('q') - - q = None - if val is not None: - q = val - - LOGGER.debug('Loading provider') - - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'feature') - except ProviderTypeError: - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'record') - except ProviderTypeError: - msg = 'Invalid provider type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - - try: - p = load_plugin('provider', provider_def) - except ProviderGenericError as err: - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - LOGGER.debug('processing property parameters') - for k, v in request.params.items(): - if k not in reserved_fieldnames and k not in p.fields.keys(): - msg = f'unknown query parameter: {k}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - elif k not in reserved_fieldnames and k in p.fields.keys(): - LOGGER.debug(f'Add property filter {k}={v}') - properties.append((k, v)) - - LOGGER.debug('processing sort parameter') - val = request.params.get('sortby') - - if val is not None: - sortby = [] - sorts = val.split(',') - for s in sorts: - prop = s - order = '+' - if s[0] in ['+', '-']: - order = s[0] - prop = s[1:] - - if prop not in p.fields.keys(): - msg = 'bad sort property' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - sortby.append({'property': prop, 'order': order}) - else: - sortby = [] - - LOGGER.debug('processing properties parameter') - val = request.params.get('properties') - - if val is not None: - select_properties = val.split(',') - properties_to_check = set(p.properties) | set(p.fields.keys()) - - if (len(list(set(select_properties) - - set(properties_to_check))) > 0): - msg = 'unknown properties specified' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - select_properties = [] - - LOGGER.debug('processing skipGeometry parameter') - val = request.params.get('skipGeometry') - if val is not None: - skip_geometry = str2bool(val) - else: - skip_geometry = False - - LOGGER.debug('Processing filter-crs parameter') - filter_crs = request.params.get('filter-crs', DEFAULT_CRS) - LOGGER.debug('Processing filter-lang parameter') - filter_lang = request.params.get('filter-lang') - if filter_lang != 'cql-json': # @TODO add check from the configuration - msg = 'Invalid filter language' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Querying provider') - LOGGER.debug(f'offset: {offset}') - LOGGER.debug(f'limit: {limit}') - LOGGER.debug(f'resulttype: {resulttype}') - LOGGER.debug(f'sortby: {sortby}') - LOGGER.debug(f'bbox: {bbox}') - LOGGER.debug(f'datetime: {datetime_}') - LOGGER.debug(f'properties: {select_properties}') - LOGGER.debug(f'skipGeometry: {skip_geometry}') - LOGGER.debug(f'q: {q}') - LOGGER.debug(f'filter-lang: {filter_lang}') - LOGGER.debug(f'filter-crs: {filter_crs}') - - LOGGER.debug('Processing headers') - - LOGGER.debug('Processing request content-type header') - if (request_headers.get( - 'Content-Type') or request_headers.get( - 'content-type')) != 'application/query-cql-json': - msg = ('Invalid body content-type') - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidHeaderValue', msg) - - LOGGER.debug('Processing body') - - if not request.data: - msg = 'missing request data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'MissingParameterValue', msg) - - filter_ = None - try: - # Parse bytes data, if applicable - data = request.data.decode() - LOGGER.debug(data) - except UnicodeDecodeError as err: - LOGGER.error(err) - msg = 'Unicode error in data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - # FIXME: remove testing backend in use once CQL support is normalized - if p.name == 'PostgreSQL': - LOGGER.debug('processing PostgreSQL CQL_JSON data') - try: - filter_ = parse_cql_json(data) - filter_ = modify_pygeofilter( - filter_, - filter_crs_uri=filter_crs, - storage_crs_uri=provider_def.get('storage_crs'), - geometry_column_name=provider_def.get('geom_field') - ) - except Exception as err: - LOGGER.error(err) - msg = f'Bad CQL string : {data}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - LOGGER.debug('processing Elasticsearch CQL_JSON data') - try: - filter_ = CQLModel.model_validate_json(data) - except Exception as err: - LOGGER.error(err) - msg = f'Bad CQL string : {data}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - try: - content = p.query(offset=offset, limit=limit, - resulttype=resulttype, bbox=bbox, - datetime_=datetime_, properties=properties, - sortby=sortby, - select_properties=select_properties, - skip_geometry=skip_geometry, - q=q, - filterq=filter_) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @gzip - @pre_process - def manage_collection_item( - self, request: Union[APIRequest, Any], - action, dataset, identifier=None) -> Tuple[dict, int, str]: - """ - Adds an item to a collection - - :param request: A request object - :param action: an action among 'create', 'update', 'delete', 'options' - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Loading provider') - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'feature') - p = load_plugin('provider', provider_def) - except ProviderTypeError: - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'record') - p = load_plugin('provider', provider_def) - except ProviderTypeError: - msg = 'Invalid provider type' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if action == 'options': - headers['Allow'] = 'HEAD, GET' - if p.editable: - if identifier is None: - headers['Allow'] += ', POST' - else: - headers['Allow'] += ', PUT, DELETE' - return headers, HTTPStatus.OK, '' - - if not p.editable: - msg = 'Collection is not editable' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if action in ['create', 'update'] and not request.data: - msg = 'No data found' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if action == 'create': - LOGGER.debug('Creating item') - try: - identifier = p.create(request.data) - except TypeError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - headers['Location'] = f'{self.get_collections_url()}/{dataset}/items/{identifier}' # noqa - - return headers, HTTPStatus.CREATED, '' - - if action == 'update': - LOGGER.debug('Updating item') - try: - _ = p.update(identifier, request.data) - except TypeError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - return headers, HTTPStatus.NO_CONTENT, '' - - if action == 'delete': - LOGGER.debug('Deleting item') - try: - _ = p.delete(identifier) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - return headers, HTTPStatus.OK, '' - - @gzip - @pre_process - def get_collection_item(self, request: Union[APIRequest, Any], - dataset, identifier) -> Tuple[dict, int, str]: - """ - Get a single collection item - - :param request: A request object - :param dataset: dataset name - :param identifier: item identifier - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - LOGGER.debug('Processing query parameters') - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Loading provider') - - try: - provider_type = 'feature' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - try: - provider_type = 'record' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - msg = 'Invalid provider type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - crs_transform_spec = None - if provider_type == 'feature': - # crs query parameter is only available for OGC API - Features - # right now, not for OGC API - Records. - LOGGER.debug('Processing crs parameter') - query_crs_uri = request.params.get('crs') - try: - crs_transform_spec = self._create_crs_transform_spec( - provider_def, query_crs_uri, - ) - except (ValueError, CRSError) as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - self._set_content_crs_header(headers, provider_def, query_crs_uri) - - # Get provider language (if any) - prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) - - try: - LOGGER.debug(f'Fetching id {identifier}') - content = p.get( - identifier, - language=prv_locale, - crs_transform_spec=crs_transform_spec, - ) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - if content is None: - msg = 'identifier not found' - return self.get_exception(HTTPStatus.BAD_REQUEST, headers, - request.format, 'NotFound', msg) - - uri = content['properties'].get(p.uri_field) if p.uri_field else \ - f'{self.get_collections_url()}/{dataset}/items/{identifier}' - - if 'links' not in content: - content['links'] = [] - - content['links'].extend([{ - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'root', - 'title': 'The landing page of this server as JSON', - 'href': f"{self.base_url}?f={F_JSON}" - }, { - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'root', - 'title': 'The landing page of this server as HTML', - 'href': f"{self.base_url}?f={F_HTML}" - }, { - 'rel': request.get_linkrel(F_JSON), - 'type': 'application/geo+json', - 'title': 'This document as GeoJSON', - 'href': f'{uri}?f={F_JSON}' - }, { - 'rel': request.get_linkrel(F_JSONLD), - 'type': FORMAT_TYPES[F_JSONLD], - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{uri}?f={F_JSONLD}' - }, { - 'rel': request.get_linkrel(F_HTML), - 'type': FORMAT_TYPES[F_HTML], - 'title': 'This document as HTML', - 'href': f'{uri}?f={F_HTML}' - }, { - 'rel': 'collection', - 'type': FORMAT_TYPES[F_JSON], - 'title': l10n.translate(collections[dataset]['title'], - request.locale), - 'href': f'{self.get_collections_url()}/{dataset}' - }]) - - link_request_format = ( - request.format if request.format is not None else F_JSON - ) - if 'prev' in content: - content['links'].append({ - 'rel': 'prev', - 'type': FORMAT_TYPES[link_request_format], - 'href': f"{self.get_collections_url()}/{dataset}/items/{content['prev']}?f={link_request_format}" # noqa - }) - if 'next' in content: - content['links'].append({ - 'rel': 'next', - 'type': FORMAT_TYPES[link_request_format], - 'href': f"{self.get_collections_url()}/{dataset}/items/{content['next']}?f={link_request_format}" # noqa - }) - - # Set response language to requested provider locale - # (if it supports language) and/or otherwise the requested pygeoapi - # locale (or fallback default locale) - l10n.set_response_language(headers, prv_locale, request.locale) - - # GA customisation - Add PID Link - if collections[dataset]['other']: - content['pid'] = f'{collections[dataset]["other"]["href"]}{identifier}' - # End GA customisation - - if request.format == F_HTML: # render - content['title'] = l10n.translate(collections[dataset]['title'], - request.locale) - content['id_field'] = p.id_field - if p.uri_field is not None: - content['uri_field'] = p.uri_field - if p.title_field is not None: - content['title_field'] = l10n.translate(p.title_field, - request.locale) - content['collections_path'] = self.get_collections_url() - - content = render_j2_template(self.tpl_config, - 'collections/items/item.html', - content, request.locale) - return headers, HTTPStatus.OK, content - - elif request.format == F_JSONLD: - content = geojson2jsonld( - self, content, dataset, uri, (p.uri_field or 'id') - ) - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @pre_process - @jsonldify - def get_collection_coverage(self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Returns a subset of a collection coverage - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - query_args = {} - format_ = request.format or F_JSON - - # Force response content type and language (en-US only) headers - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - LOGGER.debug('Loading provider') - try: - collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'coverage') - - p = load_plugin('provider', collection_def) - except KeyError: - msg = 'collection does not exist' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, format_, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - LOGGER.debug('Processing bbox parameter') - - bbox = request.params.get('bbox') - - if bbox is None: - bbox = [] - else: - try: - bbox = validate_bbox(bbox) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, format_, - 'InvalidParameterValue', msg) - - query_args['bbox'] = bbox - - LOGGER.debug('Processing bbox-crs parameter') - - bbox_crs = request.params.get('bbox-crs') - if bbox_crs is not None: - query_args['bbox_crs'] = bbox_crs - - LOGGER.debug('Processing datetime parameter') - - datetime_ = request.params.get('datetime') - - try: - datetime_ = validate_datetime( - self.config['resources'][dataset]['extents'], datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - query_args['datetime_'] = datetime_ - query_args['format_'] = format_ - - properties = request.params.get('properties') - if properties: - LOGGER.debug('Processing properties parameter') - query_args['properties'] = [rs for - rs in properties.split(',') if rs] - LOGGER.debug(f"Fields: {query_args['properties']}") - - for a in query_args['properties']: - if a not in p.fields: - msg = 'Invalid field specified' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - if 'subset' in request.params: - LOGGER.debug('Processing subset parameter') - try: - subsets = validate_subset(request.params['subset'] or '') - except (AttributeError, ValueError) as err: - msg = f'Invalid subset: {err}' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - if not set(subsets.keys()).issubset(p.axes): - msg = 'Invalid axis name' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - query_args['subsets'] = subsets - LOGGER.debug(f"Subsets: {query_args['subsets']}") - - LOGGER.debug('Querying coverage') - try: - data = p.query(**query_args) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - mt = collection_def['format']['name'] - if format_ == mt: # native format - if p.filename is not None: - cd = f'attachment; filename="{p.filename}"' - headers['Content-Disposition'] = cd - - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - elif format_ == F_JSON: - headers['Content-Type'] = 'application/prs.coverage+json' - return headers, HTTPStatus.OK, to_json(data, self.pretty_print) - else: - return self.get_format_exception(request) - - @gzip - @pre_process - @jsonldify - def get_collection_coverage_domainset( - self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Returns a collection coverage domainset - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - format_ = request.format or F_JSON - headers = request.get_response_headers(**self.api_headers) - - LOGGER.debug('Loading provider') - try: - collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'coverage') - - p = load_plugin('provider', collection_def) - - data = p.get_coverage_domainset() - except KeyError: - msg = 'collection does not exist' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, format_, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - if format_ == F_JSON: - return headers, HTTPStatus.OK, to_json(data, self.pretty_print) - - elif format_ == F_HTML: - data['id'] = dataset - data['title'] = l10n.translate( - self.config['resources'][dataset]['title'], - self.default_locale) - data['collections_path'] = self.get_collections_url() - content = render_j2_template(self.tpl_config, - 'collections/coverage/domainset.html', - data, self.default_locale) - return headers, HTTPStatus.OK, content - else: - return self.get_format_exception(request) - - @gzip - @pre_process - @jsonldify - def get_collection_coverage_rangetype( - self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Returns a collection coverage rangetype - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - format_ = request.format or F_JSON - headers = request.get_response_headers(self.default_locale, - **self.api_headers) - LOGGER.debug('Loading provider') - try: - collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'coverage') - - p = load_plugin('provider', collection_def) - - data = p.get_coverage_rangetype() - except KeyError: - msg = 'collection does not exist' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, format_, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - if format_ == F_JSON: - return headers, HTTPStatus.OK, to_json(data, self.pretty_print) - - elif format_ == F_HTML: - data['id'] = dataset - data['title'] = l10n.translate( - self.config['resources'][dataset]['title'], - self.default_locale) - data['collections_path'] = self.get_collections_url() - content = render_j2_template(self.tpl_config, - 'collections/coverage/rangetype.html', - data, self.default_locale) - return headers, HTTPStatus.OK, content - else: - return self.get_format_exception(request) - - @gzip - @pre_process - @jsonldify - def get_collection_tiles(self, request: Union[APIRequest, Any], - dataset=None) -> Tuple[dict, int, str]: - """ - Provide collection tiles - - :param request: A request object - :param dataset: name of collection - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection tiles') - LOGGER.debug('Loading provider') - try: - t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') - p = load_plugin('provider', t) - except (KeyError, ProviderTypeError): - msg = 'Invalid collection tiles' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - tiles = { - 'links': [], - 'tilesets': [] - } - - tiles['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as JSON', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_JSON}' - }) - tiles['links'].append({ - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': request.get_linkrel(F_JSONLD), - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_JSONLD}' # noqa - }) - tiles['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_HTML}' - }) - - tile_services = p.get_tiles_service( - baseurl=self.base_url, - servicepath=f'{self.get_collections_url()}/{dataset}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f=mvt' # noqa - ) - - for service in tile_services['links']: - tiles['links'].append(service) - - tiling_schemes = p.get_tiling_schemes() - - for matrix in tiling_schemes: - tile_matrix = { - 'title': dataset, - 'tileMatrixSetURI': matrix.tileMatrixSetURI, - 'crs': matrix.crs, - 'dataType': 'vector', - 'links': [] - } - tile_matrix['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme', - 'title': f'{matrix.tileMatrixSet} TileMatrixSet definition (as {F_JSON})', # noqa - 'href': f'{self.base_url}/TileMatrixSets/{matrix.tileMatrixSet}?f={F_JSON}' # noqa - }) - tile_matrix['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_JSON}', - 'href': f'{self.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_JSON}' # noqa - }) - tile_matrix['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_HTML}', - 'href': f'{self.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_HTML}' # noqa - }) - - tiles['tilesets'].append(tile_matrix) - - if request.format == F_HTML: # render - tiles['id'] = dataset - tiles['title'] = l10n.translate( - self.config['resources'][dataset]['title'], SYSTEM_LOCALE) - tiles['tilesets'] = [ - scheme.tileMatrixSet for scheme in p.get_tiling_schemes()] - tiles['bounds'] = \ - self.config['resources'][dataset]['extents']['spatial']['bbox'] - tiles['minzoom'] = p.options['zoom']['min'] - tiles['maxzoom'] = p.options['zoom']['max'] - tiles['collections_path'] = self.get_collections_url() - - content = render_j2_template(self.tpl_config, - 'collections/tiles/index.html', tiles, - request.locale) - - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(tiles, self.pretty_print) - - @pre_process - def get_collection_tiles_data( - self, request: Union[APIRequest, Any], - dataset=None, matrix_id=None, - z_idx=None, y_idx=None, x_idx=None) -> Tuple[dict, int, str]: - """ - Get collection items tiles - - :param request: A request object - :param dataset: dataset name - :param matrix_id: matrix identifier - :param z_idx: z index - :param y_idx: y index - :param x_idx: x index - - :returns: tuple of headers, status code, content - """ - - format_ = request.format - if not format_: - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - LOGGER.debug('Processing tiles') - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Loading tile provider') - try: - t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') - p = load_plugin('provider', t) - - format_ = p.format_type - headers['Content-Type'] = format_ - - LOGGER.debug(f'Fetching tileset id {matrix_id} and tile {z_idx}/{y_idx}/{x_idx}') # noqa - content = p.get_tiles(layer=p.get_layer(), tileset=matrix_id, - z=z_idx, y=y_idx, x=x_idx, format_=format_) - if content is None: - msg = 'identifier not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, format_, 'NotFound', msg) - else: - return headers, HTTPStatus.OK, content - - # @TODO: figure out if the spec requires to return json errors - except KeyError: - msg = 'Invalid collection tiles' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - @gzip - @pre_process - @jsonldify - def get_collection_tiles_metadata( - self, request: Union[APIRequest, Any], - dataset=None, matrix_id=None) -> Tuple[dict, int, str]: - """ - Get collection items tiles - - :param request: A request object - :param dataset: dataset name - :param matrix_id: matrix identifier - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid([TilesMetadataFormat.TILEJSON]): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection tiles') - LOGGER.debug('Loading provider') - try: - t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') - p = load_plugin('provider', t) - except KeyError: - msg = 'Invalid collection tiles' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - # Get provider language (if any) - prv_locale = l10n.get_plugin_locale(t, request.raw_locale) - - if matrix_id not in p.options['schemes']: - msg = 'tileset not found' - return self.get_exception(HTTPStatus.NOT_FOUND, headers, - request.format, 'NotFound', msg) - - # Set response language to requested provider locale - # (if it supports language) and/or otherwise the requested pygeoapi - # locale (or fallback default locale) - l10n.set_response_language(headers, prv_locale, request.locale) - - tiles_metadata = p.get_metadata( - dataset=dataset, server_url=self.base_url, - layer=p.get_layer(), tileset=matrix_id, - metadata_format=request._format, title=l10n.translate( - self.config['resources'][dataset]['title'], - request.locale), - description=l10n.translate( - self.config['resources'][dataset]['description'], - request.locale), - language=prv_locale) - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'collections/tiles/metadata.html', - tiles_metadata, request.locale) - - return headers, HTTPStatus.OK, content - else: - return headers, HTTPStatus.OK, tiles_metadata - - @gzip - @pre_process - def get_collection_map(self, request: Union[APIRequest, Any], - dataset, style=None) -> Tuple[dict, int, str]: - """ - Returns a subset of a collection map - - :param request: A request object - :param dataset: dataset name - :param style: style name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - query_args = { - 'crs': 'CRS84' - } - - format_ = request.format or 'png' - headers = request.get_response_headers(**self.api_headers) - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Loading provider') - try: - collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'map') - - p = load_plugin('provider', collection_def) - except KeyError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'collection does not exist' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.NOT_FOUND, to_json( - exception, self.pretty_print) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - query_args['format_'] = request.params.get('f', 'png') - query_args['style'] = style - query_args['crs'] = request.params.get('bbox-crs', 4326) - query_args['transparent'] = request.params.get('transparent', True) - - try: - query_args['width'] = int(request.params.get('width', 500)) - query_args['height'] = int(request.params.get('height', 300)) - except ValueError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'invalid width/height' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) - - LOGGER.debug('Processing bbox parameter') - try: - bbox = request.params.get('bbox').split(',') - if len(bbox) != 4: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'bbox values should be minx,miny,maxx,maxy' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) - except AttributeError: - bbox = self.config['resources'][dataset]['extents']['spatial']['bbox'] # noqa - try: - query_args['bbox'] = [float(c) for c in bbox] - except ValueError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'bbox values must be numbers' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - query_args['datetime_'] = validate_datetime( - self.config['resources'][dataset]['extents'], datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Generating map') - try: - data = p.query(**query_args) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - mt = collection_def['format']['name'] - - if format_ == mt: - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - elif format_ in [None, 'html']: - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - else: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'invalid format parameter' - } - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - data, self.pretty_print) - - @gzip - def get_collection_map_legend( - self, request: Union[APIRequest, Any], - dataset, style=None) -> Tuple[dict, int, str]: - """ - Returns a subset of a collection map legend - - :param request: A request object - :param dataset: dataset name - :param style: style name - - :returns: tuple of headers, status code, content - """ - - format_ = 'png' - headers = request.get_response_headers(**self.api_headers) - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Loading provider') - try: - collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'map') - - p = load_plugin('provider', collection_def) - except KeyError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'collection does not exist' - } - LOGGER.error(exception) - return headers, HTTPStatus.NOT_FOUND, to_json( - exception, self.pretty_print) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - LOGGER.debug('Generating legend') - try: - data = p.get_legend(style, request.params.get('f', 'png')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - mt = collection_def['format']['name'] - - if format_ == mt: - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - else: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'invalid format parameter' - } - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - data, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def describe_processes(self, request: Union[APIRequest, Any], - process=None) -> Tuple[dict, int, str]: - """ - Provide processes metadata - - :param request: A request object - :param process: process identifier, defaults to None to obtain - information about all processes - - :returns: tuple of headers, status code, content - """ - - processes = [] - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if process is not None: - if process not in self.manager.processes.keys(): - msg = 'Identifier not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'NoSuchProcess', msg) - - if len(self.manager.processes) > 0: - if process is not None: - relevant_processes = [process] - else: - LOGGER.debug('Processing limit parameter') - try: - limit = int(request.params.get('limit')) - - if limit <= 0: - msg = 'limit value should be strictly positive' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - relevant_processes = list(self.manager.processes)[:limit] - except TypeError: - LOGGER.debug('returning all processes') - relevant_processes = self.manager.processes.keys() - except ValueError: - msg = 'limit value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - for key in relevant_processes: - p = self.manager.get_processor(key) - p2 = l10n.translate_struct(deepcopy(p.metadata), - request.locale) - p2['id'] = key - - if process is None: - p2.pop('inputs') - p2.pop('outputs') - p2.pop('example', None) - - p2['jobControlOptions'] = ['sync-execute'] - if self.manager.is_async: - p2['jobControlOptions'].append('async-execute') - - p2['outputTransmission'] = ['value'] - p2['links'] = p2.get('links', []) - - jobs_url = f"{self.base_url}/jobs" - process_url = f"{self.base_url}/processes/{key}" - - # TODO translation support - link = { - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'href': f'{process_url}?f={F_JSON}', - 'title': 'Process description as JSON', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'href': f'{process_url}?f={F_HTML}', - 'title': 'Process description as HTML', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', - 'href': f'{jobs_url}?f={F_HTML}', - 'title': 'jobs for this process as HTML', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', - 'href': f'{jobs_url}?f={F_JSON}', - 'title': 'jobs for this process as JSON', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/execute', - 'href': f'{process_url}/execution?f={F_JSON}', - 'title': 'Execution for this process as JSON', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - processes.append(p2) - - if process is not None: - response = processes[0] - else: - process_url = f"{self.base_url}/processes" - response = { - 'processes': processes, - 'links': [{ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as JSON', - 'href': f'{process_url}?f={F_JSON}' - }, { - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': request.get_linkrel(F_JSONLD), - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{process_url}?f={F_JSONLD}' - }, { - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{process_url}?f={F_HTML}' - }] - } - - if request.format == F_HTML: # render - if process is not None: - response = render_j2_template(self.tpl_config, - 'processes/process.html', - response, request.locale) - else: - response = render_j2_template(self.tpl_config, - 'processes/index.html', response, - request.locale) - - return headers, HTTPStatus.OK, response - - return headers, HTTPStatus.OK, to_json(response, self.pretty_print) - - @gzip - @pre_process - def get_jobs(self, request: Union[APIRequest, Any], - job_id=None) -> Tuple[dict, int, str]: - """ - Get process jobs - - :param request: A request object - :param job_id: id of job - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - if job_id is None: - jobs = sorted(self.manager.get_jobs(), - key=lambda k: k['job_start_datetime'], - reverse=True) - else: - try: - jobs = [self.manager.get_job(job_id)] - except JobNotFoundError: - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, - 'InvalidParameterValue', job_id) - - serialized_jobs = { - 'jobs': [], - 'links': [{ - 'href': f"{self.base_url}/jobs?f={F_HTML}", - 'rel': request.get_linkrel(F_HTML), - 'type': FORMAT_TYPES[F_HTML], - 'title': 'Jobs list as HTML' - }, { - 'href': f"{self.base_url}/jobs?f={F_JSON}", - 'rel': request.get_linkrel(F_JSON), - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Jobs list as JSON' - }] - } - for job_ in jobs: - job2 = { - 'processID': job_['process_id'], - 'jobID': job_['identifier'], - 'status': job_['status'], - 'message': job_['message'], - 'progress': job_['progress'], - 'parameters': job_.get('parameters'), - 'job_start_datetime': job_['job_start_datetime'], - 'job_end_datetime': job_['job_end_datetime'] - } - - # TODO: translate - if JobStatus[job_['status']] in ( - JobStatus.successful, JobStatus.running, JobStatus.accepted): - - job_result_url = f"{self.base_url}/jobs/{job_['identifier']}/results" # noqa - - job2['links'] = [{ - 'href': f'{job_result_url}?f={F_HTML}', - 'rel': 'about', - 'type': FORMAT_TYPES[F_HTML], - 'title': f'results of job {job_id} as HTML' - }, { - 'href': f'{job_result_url}?f={F_JSON}', - 'rel': 'about', - 'type': FORMAT_TYPES[F_JSON], - 'title': f'results of job {job_id} as JSON' - }] - - if job_['mimetype'] not in (FORMAT_TYPES[F_JSON], - FORMAT_TYPES[F_HTML]): - job2['links'].append({ - 'href': job_result_url, - 'rel': 'about', - 'type': job_['mimetype'], - 'title': f"results of job {job_id} as {job_['mimetype']}" # noqa - }) - - serialized_jobs['jobs'].append(job2) - - if job_id is None: - j2_template = 'jobs/index.html' - else: - serialized_jobs = serialized_jobs['jobs'][0] - j2_template = 'jobs/job.html' - - if request.format == F_HTML: - data = { - 'jobs': serialized_jobs, - 'now': datetime.now(timezone.utc).strftime(DATETIME_FORMAT) - } - response = render_j2_template(self.tpl_config, j2_template, data, - request.locale) - return headers, HTTPStatus.OK, response - - return headers, HTTPStatus.OK, to_json(serialized_jobs, - self.pretty_print) - - @gzip - @pre_process - def execute_process(self, request: Union[APIRequest, Any], - process_id) -> Tuple[dict, int, str]: - """ - Execute process - - :param request: A request object - :param process_id: id of process - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - # Responses are always in US English only - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - if process_id not in self.manager.processes: - msg = 'identifier not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'NoSuchProcess', msg) - - data = request.data - if not data: - # TODO not all processes require input, e.g. time-dependent or - # random value generators - msg = 'missing request data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'MissingParameterValue', msg) - - try: - # Parse bytes data, if applicable - data = data.decode() - LOGGER.debug(data) - except (UnicodeDecodeError, AttributeError): - pass - - try: - data = json.loads(data) - except (json.decoder.JSONDecodeError, TypeError) as err: - # Input does not appear to be valid JSON - LOGGER.error(err) - msg = 'invalid request data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - data_dict = data.get('inputs', {}) - LOGGER.debug(data_dict) - - try: - execution_mode = RequestedProcessExecutionMode( - request.headers.get('Prefer', request.headers.get('prefer')) - ) - except ValueError: - execution_mode = None - try: - LOGGER.debug('Executing process') - result = self.manager.execute_process( - process_id, data_dict, execution_mode=execution_mode) - job_id, mime_type, outputs, status, additional_headers = result - headers.update(additional_headers or {}) - headers['Location'] = f'{self.base_url}/jobs/{job_id}' - except ProcessorExecuteError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, - request.format, err.ogc_exception_code, err.message) - - response = {} - if status == JobStatus.failed: - response = outputs - - if data.get('response', 'raw') == 'raw': - headers['Content-Type'] = mime_type - response = outputs - elif status not in (JobStatus.failed, JobStatus.accepted): - response['outputs'] = [outputs] - - if status == JobStatus.accepted: - http_status = HTTPStatus.CREATED - else: - http_status = HTTPStatus.OK - - if mime_type == 'application/json': - response2 = to_json(response, self.pretty_print) - else: - response2 = response - - return headers, http_status, response2 - - @gzip - @pre_process - def get_job_result(self, request: Union[APIRequest, Any], - job_id) -> Tuple[dict, int, str]: - """ - Get result of job (instance of a process) - - :param request: A request object - :param job_id: ID of job - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - try: - job = self.manager.get_job(job_id) - except JobNotFoundError: - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'NoSuchJob', job_id - ) - - status = JobStatus[job['status']] - - if status == JobStatus.running: - msg = 'job still running' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'ResultNotReady', msg) - - elif status == JobStatus.accepted: - # NOTE: this case is not mentioned in the specification - msg = 'job accepted but not yet running' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'ResultNotReady', msg) - - elif status == JobStatus.failed: - msg = 'job failed' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - try: - mimetype, job_output = self.manager.get_job_result(job_id) - except JobResultNotFoundError: - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'JobResultNotFound', job_id - ) - - if mimetype not in (None, FORMAT_TYPES[F_JSON]): - headers['Content-Type'] = mimetype - content = job_output - else: - if request.format == F_JSON: - content = json.dumps(job_output, sort_keys=True, indent=4, - default=json_serial) - else: - # HTML - headers['Content-Type'] = "text/html" - data = { - 'job': {'id': job_id}, - 'result': job_output - } - content = render_j2_template( - self.config, 'jobs/results/index.html', - data, request.locale) - - return headers, HTTPStatus.OK, content - - @pre_process - def delete_job( - self, request: Union[APIRequest, Any], job_id - ) -> Tuple[dict, int, str]: - """ - Delete a process job - - :param job_id: job identifier - - :returns: tuple of headers, status code, content - """ - response_headers = request.get_response_headers( - SYSTEM_LOCALE, **self.api_headers) - try: - success = self.manager.delete_job(job_id) - except JobNotFoundError: - return self.get_exception( - HTTPStatus.NOT_FOUND, response_headers, request.format, - 'NoSuchJob', job_id - ) - else: - if success: - http_status = HTTPStatus.OK - jobs_url = f"{self.base_url}/jobs" - - response = { - 'jobID': job_id, - 'status': JobStatus.dismissed.value, - 'message': 'Job dismissed', - 'progress': 100, - 'links': [{ - 'href': jobs_url, - 'rel': 'up', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'The job list for the current process' - }] - } - else: - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, response_headers, - request.format, 'InternalError', job_id - ) - LOGGER.info(response) - # TODO: this response does not have any headers - return {}, http_status, response - - @gzip - @pre_process - def get_collection_edr_query( - self, request: Union[APIRequest, Any], - dataset, instance, query_type) -> Tuple[dict, int, str]: - """ - Queries collection EDR - - :param request: APIRequest instance with query params - :param dataset: dataset name - :param instance: instance name - :param query_type: EDR query type - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - headers = request.get_response_headers(self.default_locale, - **self.api_headers) - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - datetime_ = validate_datetime(collections[dataset]['extents'], - datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing parameter_names parameter') - parameternames = request.params.get('parameter_names') or [] - if isinstance(parameternames, str): - parameternames = parameternames.split(',') - - bbox = None - if query_type == 'cube': - LOGGER.debug('Processing cube bbox') - try: - bbox = validate_bbox(request.params.get('bbox')) - if not bbox: - raise ValueError('bbox parameter required by cube queries') - except ValueError as err: - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', str(err)) - - LOGGER.debug('Processing coords parameter') - wkt = request.params.get('coords') - - if wkt: - try: - wkt = shapely_loads(wkt) - except WKTReadingError: - msg = 'invalid coords parameter' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - elif query_type != 'cube': - msg = 'missing coords parameter' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - within = within_units = None - if query_type == 'radius': - LOGGER.debug('Processing within / within-units parameters') - within = request.params.get('within') - within_units = request.params.get('within-units') - - LOGGER.debug('Processing z parameter') - z = request.params.get('z') - - LOGGER.debug('Loading provider') - try: - p = load_plugin('provider', get_provider_by_type( - collections[dataset]['providers'], 'edr')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - if instance is not None and not p.get_instance(instance): - msg = 'Invalid instance identifier' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, - request.format, 'InvalidParameterValue', msg) - - if query_type not in p.get_query_types(): - msg = 'Unsupported query type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if parameternames and not any((fld['id'] in parameternames) - for fld in p.get_fields()['field']): - msg = 'Invalid parameter_names' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - query_args = dict( - query_type=query_type, - instance=instance, - format_=request.format, - datetime_=datetime_, - select_properties=parameternames, - wkt=wkt, - z=z, - bbox=bbox, - within=within, - within_units=within_units, - limit=int(self.config['server']['limit']) - ) - - try: - data = p.query(**query_args) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'collections/edr/query.html', data, - self.default_locale) - else: - content = to_json(data, self.pretty_print) - - return headers, HTTPStatus.OK, content - - @gzip - @pre_process - @jsonldify - def get_stac_root( - self, request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: - """ - Provide STAC root page - - :param request: APIRequest instance with query params - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - id_ = 'pygeoapi-stac' - stac_version = '1.0.0-rc.2' - stac_url = f'{self.base_url}/stac' - - content = { - 'id': id_, - 'type': 'Catalog', - 'stac_version': stac_version, - 'title': l10n.translate( - self.config['metadata']['identification']['title'], - request.locale), - 'description': l10n.translate( - self.config['metadata']['identification']['description'], - request.locale), - 'links': [] - } - - stac_collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'stac-collection') - - for key, value in stac_collections.items(): - content['links'].append({ - 'rel': 'child', - 'href': f'{stac_url}/{key}?f={F_JSON}', - 'type': FORMAT_TYPES[F_JSON] - }) - content['links'].append({ - 'rel': 'child', - 'href': f'{stac_url}/{key}', - 'type': FORMAT_TYPES[F_HTML] - }) - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'stac/collection.html', - content, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def get_stac_path(self, request: Union[APIRequest, Any], - path) -> Tuple[dict, int, str]: - """ - Provide STAC resource path - - :param request: APIRequest instance with query params - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - dataset = None - LOGGER.debug(f'Path: {path}') - dir_tokens = path.split('/') - if dir_tokens: - dataset = dir_tokens[0] - - stac_collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'stac-collection') - - if dataset not in stac_collections: - msg = 'Collection not found' - return self.get_exception(HTTPStatus.NOT_FOUND, headers, - request.format, 'NotFound', msg) - - LOGGER.debug('Loading provider') - try: - p = load_plugin('provider', get_provider_by_type( - stac_collections[dataset]['providers'], 'stac')) - except ProviderConnectionError as err: - LOGGER.error(err) - msg = 'connection error (check logs)' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'NoApplicableCode', msg) - - id_ = f'{dataset}-stac' - stac_version = '1.0.0-rc.2' - - content = { - 'id': id_, - 'type': 'Catalog', - 'stac_version': stac_version, - 'description': l10n.translate( - stac_collections[dataset]['description'], request.locale), - 'links': [] - } - try: - stac_data = p.get_data_path( - f'{self.base_url}/stac', - path, - path.replace(dataset, '', 1) - ) - except ProviderNotFoundError as err: - LOGGER.error(err) - msg = 'resource not found' - return self.get_exception(HTTPStatus.NOT_FOUND, headers, - request.format, 'NotFound', msg) - except Exception as err: - LOGGER.error(err) - msg = 'data query error' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'NoApplicableCode', msg) - - if isinstance(stac_data, dict): - content.update(stac_data) - content['links'].extend(stac_collections[dataset]['links']) - - if request.format == F_HTML: # render - content['path'] = path - if 'assets' in content: # item view - if content['type'] == 'Collection': - content = render_j2_template( - self.tpl_config, - 'stac/collection_base.html', - content, - request.locale - ) - elif content['type'] == 'Feature': - content = render_j2_template( - self.tpl_config, - 'stac/item.html', - content, - request.locale - ) - else: - msg = f'Unknown STAC type {content.type}' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, - headers, - request.format, - 'NoApplicableCode', - msg) - else: - content = render_j2_template(self.tpl_config, - 'stac/catalog.html', - content, request.locale) - - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - else: # send back file - headers.pop('Content-Type', None) - return headers, HTTPStatus.OK, stac_data - - def get_exception(self, status, headers, format_, code, - description) -> Tuple[dict, int, str]: - """ - Exception handler - - :param status: HTTP status code - :param headers: dict of HTTP response headers - :param format_: format string - :param code: OGC API exception code - :param description: OGC API exception code - - :returns: tuple of headers, status, and message - """ - - LOGGER.error(description) - exception = { - 'code': code, - 'description': description - } - - if format_ == F_HTML: - headers['Content-Type'] = FORMAT_TYPES[F_HTML] - content = render_j2_template( - self.config, 'exception.html', exception, SYSTEM_LOCALE) - else: - content = to_json(exception, self.pretty_print) - - return headers, status, content - - def get_format_exception(self, request) -> Tuple[dict, int, str]: - """ - Returns a format exception. - - :param request: An APIRequest instance. - - :returns: tuple of (headers, status, message) - """ - - # Content-Language is in the system locale (ignore language settings) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - msg = f'Invalid format: {request.format}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, - request.format, 'InvalidParameterValue', msg) - - def get_collections_url(self): - return f"{self.base_url}/collections" - - @staticmethod - def _create_crs_transform_spec( - config: dict, - query_crs_uri: Optional[str] = None, - ) -> Union[None, CrsTransformSpec]: - """Create a `CrsTransformSpec` instance based on provider config and - *crs* query parameter. - - :param config: Provider config dictionary. - :type config: dict - :param query_crs_uri: Uniform resource identifier of the coordinate - reference system (CRS) specified in query parameter (if specified). - :type query_crs_uri: str, optional - - :raises ValueError: Error raised if the CRS specified in the query - parameter is not in the list of supported CRSs of the provider. - :raises `CRSError`: Error raised if no CRS could be identified from the - query *crs* parameter (URI). - - :returns: `CrsTransformSpec` instance if the CRS specified in query - parameter differs from the storage CRS, else `None`. - :rtype: Union[None, CrsTransformSpec] - """ - # Get storage/default CRS for Collection. - storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) - - if not query_crs_uri: - if storage_crs_uri in DEFAULT_CRS_LIST: - # Could be that storageCRS is - # http://www.opengis.net/def/crs/OGC/1.3/CRS84h - query_crs_uri = storage_crs_uri - else: - query_crs_uri = DEFAULT_CRS - LOGGER.debug(f'no crs parameter, using default: {query_crs_uri}') - - supported_crs_list = get_supported_crs_list(config, DEFAULT_CRS_LIST) - # Check that the crs specified by the query parameter is supported. - if query_crs_uri not in supported_crs_list: - raise ValueError( - f'CRS {query_crs_uri!r} not supported for this ' - 'collection. List of supported CRSs: ' - f'{", ".join(supported_crs_list)}.' - ) - crs_out = get_crs_from_uri(query_crs_uri) - - storage_crs = get_crs_from_uri(storage_crs_uri) - # Check if the crs specified in query parameter differs from the - # storage crs. - if str(storage_crs) != str(crs_out): - LOGGER.debug( - f'CRS transformation: {storage_crs} -> {crs_out}' - ) - return CrsTransformSpec( - source_crs_uri=storage_crs_uri, - source_crs_wkt=storage_crs.to_wkt(), - target_crs_uri=query_crs_uri, - target_crs_wkt=crs_out.to_wkt(), - ) - else: - LOGGER.debug('No CRS transformation') - return None - - @staticmethod - def _set_content_crs_header( - headers: dict, - config: dict, - query_crs_uri: Optional[str] = None, - ): - """Set the *Content-Crs* header in responses from providers of Feature - type. - - :param headers: Response headers dictionary. - :type headers: dict - :param config: Provider config dictionary. - :type config: dict - :param query_crs_uri: Uniform resource identifier of the coordinate - reference system specified in query parameter (if specified). - :type query_crs_uri: str, optional - """ - if query_crs_uri: - content_crs_uri = query_crs_uri - else: - # If empty use default CRS - storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) - if storage_crs_uri in DEFAULT_CRS_LIST: - # Could be that storageCRS is one of the defaults like - # http://www.opengis.net/def/crs/OGC/1.3/CRS84h - content_crs_uri = storage_crs_uri - else: - content_crs_uri = DEFAULT_CRS - - headers['Content-Crs'] = f'<{content_crs_uri}>' - - -def validate_bbox(value=None) -> list: - """ - Helper function to validate bbox parameter - - :param value: `list` of minx, miny, maxx, maxy - - :returns: bbox as `list` of `float` values - """ - - if value is None: - LOGGER.debug('bbox is empty') - return [] - - bbox = value.split(',') - - if len(bbox) not in [4, 6]: - msg = 'bbox should be either 4 values (minx,miny,maxx,maxy) ' \ - 'or 6 values (minx,miny,minz,maxx,maxy,maxz)' - LOGGER.debug(msg) - raise ValueError(msg) - - try: - bbox = [float(c) for c in bbox] - except ValueError as err: - msg = 'bbox values must be numbers' - err.args = (msg,) - LOGGER.debug(msg) - raise - - if (len(bbox) == 4 and bbox[1] > bbox[3]) \ - or (len(bbox) == 6 and bbox[1] > bbox[4]): - msg = 'miny should be less than maxy' - LOGGER.debug(msg) - raise ValueError(msg) - - if (len(bbox) == 4 and bbox[0] > bbox[2]) \ - or (len(bbox) == 6 and bbox[0] > bbox[3]): - msg = 'minx is greater than maxx (possibly antimeridian bbox)' - LOGGER.debug(msg) - - if len(bbox) == 6 and bbox[2] > bbox[5]: - msg = 'minz should be less than maxz' - LOGGER.debug(msg) - raise ValueError(msg) - - return bbox - - -def validate_datetime(resource_def, datetime_=None) -> str: - """ - Helper function to validate temporal parameter - - :param resource_def: `dict` of configuration resource definition - :param datetime_: `str` of datetime parameter - - :returns: `str` of datetime input, if valid - """ - - # TODO: pass datetime to query as a `datetime` object - # we would need to ensure partial dates work accordingly - # as well as setting '..' values to `None` so that underlying - # providers can just assume a `datetime.datetime` object - # - # NOTE: needs testing when passing partials from API to backend - - datetime_invalid = False - - if datetime_ is not None and 'temporal' in resource_def: - - dateparse_begin = partial(dateparse, default=datetime.min) - dateparse_end = partial(dateparse, default=datetime.max) - unix_epoch = datetime(1970, 1, 1, 0, 0, 0) - dateparse_ = partial(dateparse, default=unix_epoch) - - te = resource_def['temporal'] - - try: - if te['begin'] is not None and te['begin'].tzinfo is None: - te['begin'] = te['begin'].replace(tzinfo=pytz.UTC) - if te['end'] is not None and te['end'].tzinfo is None: - te['end'] = te['end'].replace(tzinfo=pytz.UTC) - except AttributeError: - msg = 'Configured times should be RFC3339' - LOGGER.error(msg) - raise ValueError(msg) - - if '/' in datetime_: # envelope - LOGGER.debug('detected time range') - LOGGER.debug('Validating time windows') - - # normalize "" to ".." (actually changes datetime_) - datetime_ = re.sub(r'^/', '../', datetime_) - datetime_ = re.sub(r'/$', '/..', datetime_) - - datetime_begin, datetime_end = datetime_.split('/') - if datetime_begin != '..': - datetime_begin = dateparse_begin(datetime_begin) - if datetime_begin.tzinfo is None: - datetime_begin = datetime_begin.replace( - tzinfo=pytz.UTC) - - if datetime_end != '..': - datetime_end = dateparse_end(datetime_end) - if datetime_end.tzinfo is None: - datetime_end = datetime_end.replace(tzinfo=pytz.UTC) - - datetime_invalid = any([ - (te['end'] is not None and datetime_begin != '..' and - datetime_begin > te['end']), - (te['begin'] is not None and datetime_end != '..' and - datetime_end < te['begin']) - ]) - - else: # time instant - LOGGER.debug('detected time instant') - datetime__ = dateparse_(datetime_) - if datetime__ != '..': - if datetime__.tzinfo is None: - datetime__ = datetime__.replace(tzinfo=pytz.UTC) - datetime_invalid = any([ - (te['begin'] is not None and datetime__ != '..' and - datetime__ < te['begin']), - (te['end'] is not None and datetime__ != '..' and - datetime__ > te['end']) - ]) - - if datetime_invalid: - msg = 'datetime parameter out of range' - LOGGER.debug(msg) - raise ValueError(msg) - - return datetime_ - - -def validate_subset(value: str) -> dict: - """ - Helper function to validate subset parameter - - :param value: `subset` parameter - - :returns: dict of axis/values - """ - - subsets = {} - - for s in value.split(','): - LOGGER.debug(f'Processing subset {s}') - m = re.search(r'(.*)\((.*)\)', s) - subset_name, values = m.group(1, 2) - - if '"' in values: - LOGGER.debug('Values are strings') - if values.count('"') % 2 != 0: - msg = 'Invalid format: subset should be like axis("min"[:"max"])' # noqa - LOGGER.error(msg) - raise ValueError(msg) - try: - LOGGER.debug('Value is an interval') - m = re.search(r'"(\S+)":"(\S+)"', values) - values = list(m.group(1, 2)) - except AttributeError: - LOGGER.debug('Value is point') - m = re.search(r'"(.*)"', values) - values = [m.group(1)] - else: - LOGGER.debug('Values are numbers') - try: - LOGGER.debug('Value is an interval') - m = re.search(r'(\S+):(\S+)', values) - values = list(m.group(1, 2)) - except AttributeError: - LOGGER.debug('Value is point') - values = [values] - - subsets[subset_name] = list(map(get_typed_value, values)) - - return subsets diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py new file mode 100644 index 000000000..c10ba82b7 --- /dev/null +++ b/pygeoapi/api/__init__.py @@ -0,0 +1,1673 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# +# Copyright (c) 2025 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +""" +Root level code of pygeoapi, parsing content provided by web framework. +Returns content from plugins and sets responses. +""" + +from collections import ChainMap, OrderedDict +from copy import deepcopy +from datetime import datetime +from functools import partial +from gzip import compress +from http import HTTPStatus +import logging +import re +import sys +from typing import Any, Tuple, Union, Optional + +from dateutil.parser import parse as dateparse +import pytz + +from pygeoapi import __version__, l10n +from pygeoapi.linked_data import jsonldify, jsonldify_collection +from pygeoapi.log import setup_logger +from pygeoapi.plugin import load_plugin +from pygeoapi.process.manager.base import get_manager +from pygeoapi.provider.base import ( + ProviderConnectionError, ProviderGenericError, ProviderTypeError) + +from pygeoapi.util import ( + CrsTransformSpec, TEMPLATES, UrlPrefetcher, dategetter, + filter_dict_by_key_value, filter_providers_by_type, get_api_rules, + get_base_url, get_provider_by_type, get_provider_default, get_typed_value, + get_crs_from_uri, get_supported_crs_list, render_j2_template, to_json +) + +LOGGER = logging.getLogger(__name__) + +#: Return headers for requests (e.g:X-Powered-By) +HEADERS = { + 'Content-Type': 'application/json', + 'X-Powered-By': f'pygeoapi {__version__}' +} + +CHARSET = ['utf-8'] +F_JSON = 'json' +F_COVERAGEJSON = 'json' +F_HTML = 'html' +F_JSONLD = 'jsonld' +F_GZIP = 'gzip' +F_PNG = 'png' +F_JPEG = 'jpeg' +F_MVT = 'mvt' +F_NETCDF = 'NetCDF' + +#: Formats allowed for ?f= requests (order matters for complex MIME types) +FORMAT_TYPES = OrderedDict(( + (F_HTML, 'text/html'), + (F_JSONLD, 'application/ld+json'), + (F_JSON, 'application/json'), + (F_PNG, 'image/png'), + (F_JPEG, 'image/jpeg'), + (F_MVT, 'application/vnd.mapbox-vector-tile'), + (F_NETCDF, 'application/x-netcdf'), +)) + +#: Locale used for system responses (e.g. exceptions) +SYSTEM_LOCALE = l10n.Locale('en', 'US') + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/landing-page', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/json', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/html', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/oas30' +] + +OGC_RELTYPES_BASE = 'http://www.opengis.net/def/rel/ogc/1.0' + +DEFAULT_CRS_LIST = [ + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', +] + +DEFAULT_CRS = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' +DEFAULT_STORAGE_CRS = DEFAULT_CRS + + +def all_apis() -> dict: + """ + Return all supported API modules + + NOTE: this is a function and not a constant to avoid import loops + + :returns: `dict` of API provider type, API module + """ + + from . import (coverages, environmental_data_retrieval, itemtypes, maps, + processes, tiles, stac) + + return { + 'coverage': coverages, + 'edr': environmental_data_retrieval, + 'itemtypes': itemtypes, + 'map': maps, + 'process': processes, + 'tile': tiles, + 'stac': stac + } + + +def apply_gzip(headers: dict, content: Union[str, bytes]) -> Union[str, bytes]: + """ + Compress content if requested in header. + """ + charset = CHARSET[0] + if F_GZIP in headers.get('Content-Encoding', []): + try: + if isinstance(content, bytes): + # bytes means Content-Type needs to be set upstream + content = compress(content) + else: + headers['Content-Type'] = \ + f"{headers['Content-Type']}; charset={charset}" + content = compress(content.encode(charset)) + except TypeError as err: + headers.pop('Content-Encoding') + LOGGER.error(f'Error in compression: {err}') + return content + + +class APIRequest: + """ + Transforms an incoming server-specific Request into an object + with some generic helper methods and properties. + + This allows writing straightforward API functions supporting all + web platforms such as with this example: + + .. code-block:: python + + def example_method(api: API, request: APIRequest, custom_arg): + headers = request.get_response_headers() + + # generate response_body here + + return headers, HTTPStatus.OK, response_body + + + Basic request validation is done automatically by web platform specific + adapters such as ``execute_from_flask`` in flask_app.py . If you want to + support custom formats (e.g. ``f=xml``), it needs to be registered with + `skip_valid_check=True` and you can use the following code for custom + validation. If `xml` was requested, we set the `Content-Type` ourselves. + For the standard formats, the `APIRequest` object sets the `Content-Type`. + + .. code-block:: python + + def example_method(api: API, request: APIRequest, custom_arg): + if not request.is_valid(['xml']): + return api.get_format_exception(request) + + content_type = 'application/xml' if request.format == 'xml' else None + headers = request.get_response_headers(content_type) + + # generate response_body here + + return headers, HTTPStatus.OK, response_body + + Note that you don't *have* to call :meth:`is_valid`, but that you can also + perform a custom check on the requested output format by looking at the + :attr:`format` property. + Other query parameters are available through the :attr:`params` property as + a `dict`. The request body is available through the :attr:`data` property. + + :param request: The web platform specific Request instance. + :param supported_locales: List or set of supported Locale instances. + """ + def __init__(self, request, supported_locales): + # Set default request data + self._data = b'' + + # Copy request query parameters + self._args = self._get_params(request) + + # Get path info + if hasattr(request, 'scope'): + self._path_info = request.scope['path'].strip('/') + elif hasattr(request.headers, 'environ'): + self._path_info = request.headers.environ['PATH_INFO'].strip('/') + elif hasattr(request, 'path_info'): + self._path_info = request.path_info + + # Extract locale from params or headers + self._raw_locale, self._locale = self._get_locale(request.headers, + supported_locales) + + # Determine format + self._format = self._get_format(request.headers) + + # Get received headers + self._headers = self.get_request_headers(request.headers) + + @classmethod + def from_flask(cls, request, supported_locales) -> 'APIRequest': + """Factory class similar to with_data, but only for flask requests""" + api_req = cls(request, supported_locales) + api_req._data = request.data + return api_req + + @classmethod + async def from_starlette(cls, request, supported_locales) -> 'APIRequest': + """Factory class similar to with_data, but only for starlette requests + """ + api_req = cls(request, supported_locales) + api_req._data = await request.body() + return api_req + + @classmethod + def from_django(cls, request, supported_locales) -> 'APIRequest': + """Factory class similar to with_data, but only for django requests""" + api_req = cls(request, supported_locales) + api_req._data = request.body + return api_req + + @staticmethod + def _get_params(request): + """ + Extracts the query parameters from the `Request` object. + + :param request: A Flask or Starlette Request instance + :returns: `ImmutableMultiDict` or empty `dict` + """ + + if hasattr(request, 'args'): + # Return ImmutableMultiDict from Flask request + return request.args + elif hasattr(request, 'query_params'): + # Return ImmutableMultiDict from Starlette request + return request.query_params + elif hasattr(request, 'GET'): + # Return QueryDict from Django GET request + return request.GET + elif hasattr(request, 'POST'): + # Return QueryDict from Django GET request + return request.POST + LOGGER.debug('No query parameters found') + return {} + + def _get_locale(self, headers, supported_locales): + """ + Detects locale from "lang=" param or `Accept-Language` + header. Returns a tuple of (raw, locale) if found in params or headers. + Returns a tuple of (raw default, default locale) if not found. + + :param headers: A dict with Request headers + :param supported_locales: List or set of supported Locale instances + :returns: A tuple of (str, Locale) + """ + + raw = None + try: + default_locale = l10n.str2locale(supported_locales[0]) + except (TypeError, IndexError, l10n.LocaleError) as err: + # This should normally not happen, since the API class already + # loads the supported languages from the config, which raises + # a LocaleError if any of these languages are invalid. + LOGGER.error(err) + raise ValueError(f"{self.__class__.__name__} must be initialized" + f"with a list of valid supported locales") + + for func, mapping in ((l10n.locale_from_params, self._args), + (l10n.locale_from_headers, headers)): + loc_str = func(mapping) + if loc_str: + if not raw: + # This is the first-found locale string: set as raw + raw = loc_str + # Check if locale string is a good match for the UI + loc = l10n.best_match(loc_str, supported_locales) + is_override = func is l10n.locale_from_params + if loc != default_locale or is_override: + return raw, loc + + return raw, default_locale + + def _get_format(self, headers) -> Union[str, None]: + """ + Get `Request` format type from query parameters or headers. + + :param headers: Dict of Request headers + :returns: format value or None if not found/specified + """ + + # Optional f=html or f=json query param + # Overrides Accept header and might differ from FORMAT_TYPES + format_ = (self._args.get('f') or '').strip() + if format_: + return format_ + + # Format not specified: get from Accept headers (MIME types) + # e.g. format_ = 'text/html' + h = headers.get('accept', headers.get('Accept', '')).strip() # noqa + (fmts, mimes) = zip(*FORMAT_TYPES.items()) + # basic support for complex types (i.e. with "q=0.x") + for type_ in (t.split(';')[0].strip() for t in h.split(',') if t): + if type_ in mimes: + idx_ = mimes.index(type_) + format_ = fmts[idx_] + break + + return format_ or None + + @property + def data(self) -> bytes: + """Returns the additional data send with the Request (bytes)""" + return self._data + + @property + def params(self) -> dict: + """Returns the Request query parameters dict""" + return self._args + + @property + def path_info(self) -> str: + """Returns the web server request path info part""" + return self._path_info + + @property + def locale(self) -> l10n.Locale: + """ + Returns the user-defined locale from the request object. + If no locale has been defined or if it is invalid, + the default server locale is returned. + + .. note:: The locale here determines the language in which pygeoapi + should return its responses. This may not be the language + that the user requested. It may also not be the language + that is supported by a collection provider, for example. + For this reason, you should pass the `raw_locale` property + to the :func:`l10n.get_plugin_locale` function, so that + the best match for the provider can be determined. + + :returns: babel.core.Locale + """ + + return self._locale + + @property + def raw_locale(self) -> Union[str, None]: + """ + Returns the raw locale string from the `Request` object. + If no "lang" query parameter or `Accept-Language` header was found, + `None` is returned. + Pass this value to the :func:`l10n.get_plugin_locale` function to let + the provider determine a best match for the locale, which may be + different from the locale used by pygeoapi's UI. + + :returns: a locale string or None + """ + + return self._raw_locale + + @property + def format(self) -> Union[str, None]: + """ + Returns the content type format from the + request query parameters or headers. + + :returns: Format name or None + """ + + return self._format + + @property + def headers(self) -> dict: + """ + Returns the dictionary of the headers from + the request. + + :returns: Request headers dictionary + """ + + return self._headers + + def get_linkrel(self, format_: str) -> str: + """ + Returns the hyperlink relationship (rel) attribute value for + the given API format string. + + The string is compared against the request format and if it matches, + the value 'self' is returned. Otherwise, 'alternate' is returned. + However, if `format_` is 'json' and *no* request format was found, + the relationship 'self' is returned as well (JSON is the default). + + :param format_: The format to compare the request format against. + :returns: A string 'self' or 'alternate'. + """ + + fmt = format_.lower() + if fmt == self._format or (fmt == F_JSON and not self._format): + return 'self' + return 'alternate' + + def is_valid(self, additional_formats=None) -> bool: + """ + Returns True if: + - the format is not set (None) + - the requested format is supported + - the requested format exists in a list if additional formats + + .. note:: Format names are matched in a case-insensitive manner. + + :param additional_formats: Optional additional supported formats list + + :returns: bool + """ + + if not self._format: + return True + if self._format in FORMAT_TYPES.keys(): + return True + if self._format in (f.lower() for f in (additional_formats or ())): + return True + return False + + def get_response_headers(self, force_lang: l10n.Locale = None, + force_type: str = None, + force_encoding: str = None, + **custom_headers) -> dict: + """ + Prepares and returns a dictionary with Response object headers. + + This method always adds a 'Content-Language' header, where the value + is determined by the 'lang' query parameter or 'Accept-Language' + header from the request. + If no language was requested, the default pygeoapi language is used, + unless a `force_lang` override was specified (see notes below). + + A 'Content-Type' header is also always added to the response. + If the user does not specify `force_type`, the header is based on + the `format` APIRequest property. If that is invalid, the default MIME + type `application/json` is used. + + ..note:: If a `force_lang` override is applied, that language + is always set as the 'Content-Language', regardless of + a 'lang' query parameter or 'Accept-Language' header. + If an API response always needs to be in the same + language, 'force_lang' should be set to that language. + + :param force_lang: An optional Content-Language header override. + :param force_type: An optional Content-Type header override. + :param force_encoding: An optional Content-Encoding header override. + :returns: A header dict + """ + + headers = HEADERS.copy() + headers.update(**custom_headers) + l10n.set_response_language(headers, force_lang or self._locale) + if force_type: + # Set custom MIME type if specified + headers['Content-Type'] = force_type + elif self.is_valid() and self._format: + # Set MIME type for valid formats + headers['Content-Type'] = FORMAT_TYPES[self._format] + + if F_GZIP in FORMAT_TYPES: + if force_encoding: + headers['Content-Encoding'] = force_encoding + elif F_GZIP in self._headers.get('Accept-Encoding', ''): + headers['Content-Encoding'] = F_GZIP + + return headers + + def get_request_headers(self, headers) -> dict: + """ + Obtains and returns a dictionary with Request object headers. + + This method adds the headers of the original request and + makes them available to the API object. + + :returns: A header dict + """ + + headers_ = {item[0]: item[1] for item in headers.items()} + return headers_ + + +class API: + """API object""" + + def __init__(self, config, openapi): + """ + constructor + + :param config: configuration dict + :param openapi: openapi dict + + :returns: `pygeoapi.API` instance + """ + + self.config = config + self.openapi = openapi + self.api_headers = get_api_rules(self.config).response_headers + self.base_url = get_base_url(self.config) + self.prefetcher = UrlPrefetcher() + + CHARSET[0] = config['server'].get('encoding', 'utf-8') + if config['server'].get('gzip'): + FORMAT_TYPES[F_GZIP] = 'application/gzip' + FORMAT_TYPES.move_to_end(F_JSON) + + # Process language settings (first locale is default!) + self.locales = l10n.get_locales(config) + self.default_locale = self.locales[0] + + if 'templates' not in self.config['server']: + self.config['server']['templates'] = {'path': TEMPLATES} + + if 'pretty_print' not in self.config['server']: + self.config['server']['pretty_print'] = False + + self.pretty_print = self.config['server']['pretty_print'] + + setup_logger(self.config['logging']) + + # Create config clone for HTML templating with modified base URL + self.tpl_config = deepcopy(self.config) + self.tpl_config['server']['url'] = self.base_url + + self.manager = get_manager(self.config) + LOGGER.info('Process manager plugin loaded') + + def get_exception(self, status, headers, format_, code, + description) -> Tuple[dict, int, str]: + """ + Exception handler + + :param status: HTTP status code + :param headers: dict of HTTP response headers + :param format_: format string + :param code: OGC API exception code + :param description: OGC API exception code + + :returns: tuple of headers, status, and message + """ + + exception_info = sys.exc_info() + LOGGER.error( + description, + exc_info=exception_info if exception_info[0] is not None else None + ) + exception = { + 'code': code, + 'type': code, + 'description': description + } + + if format_ == F_HTML: + headers['Content-Type'] = FORMAT_TYPES[F_HTML] + content = render_j2_template( + self.tpl_config, self.config['server']['templates'], + 'exception.html', exception, SYSTEM_LOCALE) + else: + content = to_json(exception, self.pretty_print) + + return headers, status, content + + def get_format_exception(self, request) -> Tuple[dict, int, str]: + """ + Returns a format exception. + + :param request: An APIRequest instance. + + :returns: tuple of (headers, status, message) + """ + + # Content-Language is in the system locale (ignore language settings) + headers = request.get_response_headers(SYSTEM_LOCALE, + **self.api_headers) + msg = 'Invalid format requested' + LOGGER.error(f'{msg}: {request.format}') + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, + request.format, 'InvalidParameterValue', msg) + + def get_collections_url(self) -> str: + return f"{self.base_url}/collections" + + def get_dataset_templates(self, dataset) -> dict: + templates = self.config['resources'][dataset].get('templates') + + return templates or self.tpl_config['server']['templates'] + + @staticmethod + def _create_crs_transform_spec( + config: dict, + query_crs_uri: Optional[str] = None, + ) -> Union[None, CrsTransformSpec]: + """Create a `CrsTransformSpec` instance based on provider config and + *crs* query parameter. + + :param config: Provider config dictionary. + :type config: dict + :param query_crs_uri: Uniform resource identifier of the coordinate + reference system (CRS) specified in query parameter (if specified). + :type query_crs_uri: str, optional + + :raises ValueError: Error raised if the CRS specified in the query + parameter is not in the list of supported CRSs of the provider. + :raises `CRSError`: Error raised if no CRS could be identified from the + query *crs* parameter (URI). + + :returns: `CrsTransformSpec` instance if the CRS specified in query + parameter differs from the storage CRS, else `None`. + :rtype: Union[None, CrsTransformSpec] + """ + # Get storage/default CRS for Collection. + storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) + + if not query_crs_uri: + if storage_crs_uri in DEFAULT_CRS_LIST: + # Could be that storageCRS is + # http://www.opengis.net/def/crs/OGC/1.3/CRS84h + query_crs_uri = storage_crs_uri + else: + query_crs_uri = DEFAULT_CRS + LOGGER.debug(f'no crs parameter, using default: {query_crs_uri}') + + supported_crs_list = get_supported_crs_list(config, DEFAULT_CRS_LIST) + # Check that the crs specified by the query parameter is supported. + if query_crs_uri not in supported_crs_list: + raise ValueError( + f'CRS {query_crs_uri!r} not supported for this ' + 'collection. List of supported CRSs: ' + f'{", ".join(supported_crs_list)}.' + ) + crs_out = get_crs_from_uri(query_crs_uri) + + storage_crs = get_crs_from_uri(storage_crs_uri) + # Check if the crs specified in query parameter differs from the + # storage crs. + if str(storage_crs) != str(crs_out): + LOGGER.debug( + f'CRS transformation: {storage_crs} -> {crs_out}' + ) + return CrsTransformSpec( + source_crs_uri=storage_crs_uri, + source_crs_wkt=storage_crs.to_wkt(), + target_crs_uri=query_crs_uri, + target_crs_wkt=crs_out.to_wkt(), + ) + else: + LOGGER.debug('No CRS transformation') + return None + + @staticmethod + def _set_content_crs_header( + headers: dict, + config: dict, + query_crs_uri: Optional[str] = None, + ): + """Set the *Content-Crs* header in responses from providers of Feature + type. + + :param headers: Response headers dictionary. + :type headers: dict + :param config: Provider config dictionary. + :type config: dict + :param query_crs_uri: Uniform resource identifier of the coordinate + reference system specified in query parameter (if specified). + :type query_crs_uri: str, optional + """ + if query_crs_uri: + content_crs_uri = query_crs_uri + else: + # If empty use default CRS + storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) + if storage_crs_uri in DEFAULT_CRS_LIST: + # Could be that storageCRS is one of the defaults like + # http://www.opengis.net/def/crs/OGC/1.3/CRS84h + content_crs_uri = storage_crs_uri + else: + content_crs_uri = DEFAULT_CRS + + headers['Content-Crs'] = f'<{content_crs_uri}>' + + +@jsonldify +def landing_page(api: API, + request: APIRequest) -> Tuple[dict, int, str]: + """ + Provide API landing page + + :param request: A request object + + :returns: tuple of headers, status code, content + """ + + fcm = { + 'links': [], + 'title': l10n.translate( + api.config['metadata']['identification']['title'], + request.locale), + 'description': + l10n.translate( + api.config['metadata']['identification']['description'], + request.locale) + } + + LOGGER.debug('Creating links') + # TODO: put title text in config or translatable files? + fcm['links'] = [{ + 'rel': 'about', + 'type': 'text/html', + 'title': l10n.translate( + api.config['metadata']['identification']['title'], + request.locale), + 'href': api.config['metadata']['identification']['url'] + }, { + 'rel': request.get_linkrel(F_JSON), + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate('This document as JSON', request.locale), + 'href': f"{api.base_url}?f={F_JSON}" + }, { + 'rel': request.get_linkrel(F_JSONLD), + 'type': FORMAT_TYPES[F_JSONLD], + 'title': l10n.translate('This document as RDF (JSON-LD)', request.locale), # noqa + 'href': f"{api.base_url}?f={F_JSONLD}" + }, { + 'rel': request.get_linkrel(F_HTML), + 'type': FORMAT_TYPES[F_HTML], + 'title': l10n.translate('This document as HTML', request.locale), + 'href': f"{api.base_url}?f={F_HTML}", + 'hreflang': api.default_locale + }, { + 'rel': 'service-desc', + 'type': 'application/vnd.oai.openapi+json;version=3.0', + 'title': l10n.translate('The OpenAPI definition as JSON', request.locale), # noqa + 'href': f"{api.base_url}/openapi" + }, { + 'rel': 'service-doc', + 'type': FORMAT_TYPES[F_HTML], + 'title': l10n.translate('The OpenAPI definition as HTML', request.locale), # noqa + 'href': f"{api.base_url}/openapi?f={F_HTML}", + 'hreflang': api.default_locale + }, { + 'rel': 'conformance', + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate('Conformance', request.locale), + 'href': f"{api.base_url}/conformance" + }, { + 'rel': 'data', + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate('Collections', request.locale), + 'href': api.get_collections_url() + }, { + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/processes', + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate('Processes', request.locale), + 'href': f"{api.base_url}/processes" + }, { + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate('Jobs', request.locale), + 'href': f"{api.base_url}/jobs" + }, { + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-schemes', + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate('The list of supported tiling schemes as JSON', request.locale), # noqa + 'href': f"{api.base_url}/TileMatrixSets?f=json" + }, { + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-schemes', + 'type': FORMAT_TYPES[F_HTML], + 'title': l10n.translate('The list of supported tiling schemes as HTML', request.locale), # noqa + 'href': f"{api.base_url}/TileMatrixSets?f=html" + }] + + headers = request.get_response_headers(**api.api_headers) + if request.format == F_HTML: # render + + for resource_type in ['collection', 'process', 'stac-collection']: + fcm[resource_type] = False + + found = filter_dict_by_key_value(api.config['resources'], + 'type', resource_type) + if found: + fcm[resource_type] = True + if resource_type == 'collection': # check for tiles + for key, value in found.items(): + if filter_providers_by_type(value['providers'], + 'tile'): + fcm['tile'] = True + + content = render_j2_template( + api.tpl_config, api.config['server']['templates'], + 'landing_page.html', fcm, request.locale) + + return headers, HTTPStatus.OK, content + + if request.format == F_JSONLD: + return headers, HTTPStatus.OK, to_json( + api.fcmld, api.pretty_print) + + return headers, HTTPStatus.OK, to_json(fcm, api.pretty_print) + + +def openapi_(api: API, request: APIRequest) -> Tuple[dict, int, str]: + """ + Provide OpenAPI document + + :param request: A request object + :param openapi: dict of OpenAPI definition + + :returns: tuple of headers, status code, content + """ + headers = request.get_response_headers(**api.api_headers) + + if request.format == F_HTML: + template = 'openapi/swagger.html' + if request._args.get('ui') == 'redoc': + template = 'openapi/redoc.html' + + path = f'{api.base_url}/openapi' + data = { + 'openapi-document-path': path + } + content = render_j2_template( + api.tpl_config, api.config['server']['templates'], template, data, + request.locale) + + return headers, HTTPStatus.OK, content + + headers['Content-Type'] = 'application/vnd.oai.openapi+json;version=3.0' # noqa + + if isinstance(api.openapi, dict): + return headers, HTTPStatus.OK, to_json(api.openapi, + api.pretty_print) + else: + return headers, HTTPStatus.OK, api.openapi + + +def conformance(api, request: APIRequest) -> Tuple[dict, int, str]: + """ + Provide conformance definition + + :param request: A request object + + :returns: tuple of headers, status code, content + """ + + apis_dict = all_apis() + + conformance_list = CONFORMANCE_CLASSES + + for key, value in api.config['resources'].items(): + if value['type'] == 'process': + conformance_list.extend( + apis_dict['process'].CONFORMANCE_CLASSES) + else: + for provider in value['providers']: + if provider['type'] in apis_dict: + conformance_list.extend( + apis_dict[provider['type']].CONFORMANCE_CLASSES) + if provider['type'] == 'feature': + conformance_list.extend( + apis_dict['itemtypes'].CONFORMANCE_CLASSES_FEATURES) # noqa + if provider['type'] == 'record': + conformance_list.extend( + apis_dict['itemtypes'].CONFORMANCE_CLASSES_RECORDS) + + conformance = { + 'conformsTo': sorted(list(set(conformance_list))) + } + + headers = request.get_response_headers(**api.api_headers) + if request.format == F_HTML: # render + content = render_j2_template( + api.tpl_config, api.config['server']['templates'], + 'conformance.html', conformance, request.locale) + + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(conformance, api.pretty_print) + + +@jsonldify +def describe_collections(api: API, request: APIRequest, + dataset=None) -> Tuple[dict, int, str]: + """ + Provide collection metadata + + :param request: A request object + :param dataset: name of collection + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(**api.api_headers) + + fcm = { + 'collections': [], + 'links': [] + } + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if all([dataset is not None, dataset not in collections.keys()]): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + if dataset is not None: + collections_dict = { + k: v for k, v in collections.items() if k == dataset + } + else: + collections_dict = collections + + LOGGER.debug('Creating collections') + for k, v in collections_dict.items(): + if v.get('visibility', 'default') == 'hidden': + LOGGER.debug(f'Skipping hidden layer: {k}') + continue + collection_data = get_provider_default(v['providers']) + collection_data_type = collection_data['type'] + + collection_data_format = None + + if 'format' in collection_data: + collection_data_format = collection_data['format'] + + is_vector_tile = (collection_data_type == 'tile' and + collection_data_format['name'] not + in [F_PNG, F_JPEG]) + + collection = { + 'id': k, + 'title': l10n.translate(v['title'], request.locale), + 'description': l10n.translate(v['description'], request.locale), # noqa + 'keywords': l10n.translate(v['keywords'], request.locale), + 'links': [] + } + + bbox = v['extents']['spatial']['bbox'] + # The output should be an array of bbox, so if the user only + # provided a single bbox, wrap it in a array. + if not isinstance(bbox[0], list): + bbox = [bbox] + collection['extent'] = { + 'spatial': { + 'bbox': bbox + } + } + if 'crs' in v['extents']['spatial']: + collection['extent']['spatial']['crs'] = \ + v['extents']['spatial']['crs'] + + t_ext = v.get('extents', {}).get('temporal', {}) + if t_ext: + begins = dategetter('begin', t_ext) + ends = dategetter('end', t_ext) + collection['extent']['temporal'] = { + 'interval': [[begins, ends]] + } + if 'trs' in t_ext: + collection['extent']['temporal']['trs'] = t_ext['trs'] + + LOGGER.debug('Processing configured collection links') + for link in l10n.translate(v.get('links', []), request.locale): + lnk = { + 'type': link['type'], + 'rel': link['rel'], + 'title': l10n.translate(link['title'], request.locale), + 'href': l10n.translate(link['href'], request.locale), + } + if 'hreflang' in link: + lnk['hreflang'] = l10n.translate( + link['hreflang'], request.locale) + content_length = link.get('length', 0) + + if lnk['rel'] == 'enclosure' and content_length == 0: + # Issue HEAD request for enclosure links without length + lnk_headers = api.prefetcher.get_headers(lnk['href']) + content_length = int(lnk_headers.get('content-length', 0)) + content_type = lnk_headers.get('content-type', lnk['type']) + if content_length == 0: + # Skip this (broken) link + LOGGER.debug(f"Enclosure {lnk['href']} is invalid") + continue + if content_type != lnk['type']: + # Update content type if different from specified + lnk['type'] = content_type + LOGGER.debug( + f"Fixed media type for enclosure {lnk['href']}") + + if content_length > 0: + lnk['length'] = content_length + + collection['links'].append(lnk) + + # TODO: provide translations + LOGGER.debug('Adding JSON and HTML link relations') + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'root', + 'title': l10n.translate('The landing page of this server as JSON', request.locale), # noqa + 'href': f"{api.base_url}?f={F_JSON}" + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'root', + 'title': l10n.translate('The landing page of this server as HTML', request.locale), # noqa + 'href': f"{api.base_url}?f={F_HTML}" + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': l10n.translate('This document as JSON', request.locale), # noqa + 'href': f'{api.get_collections_url()}/{k}?f={F_JSON}' + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': l10n.translate('This document as RDF (JSON-LD)', request.locale), # noqa + 'href': f'{api.get_collections_url()}/{k}?f={F_JSONLD}' + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': l10n.translate('This document as HTML', request.locale), # noqa + 'href': f'{api.get_collections_url()}/{k}?f={F_HTML}' + }) + + if collection_data_type == 'record': + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/ogc-catalog', + 'title': l10n.translate('Record catalogue as JSON', request.locale), # noqa + 'href': f'{api.get_collections_url()}/{k}?f={F_JSON}' + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/ogc-catalog', + 'title': l10n.translate('Record catalogue as HTML', request.locale), # noqa + 'href': f'{api.get_collections_url()}/{k}?f={F_HTML}' + }) + + if collection_data_type in ['feature', 'coverage', 'record']: + collection['links'].append({ + 'type': 'application/schema+json', + 'rel': f'{OGC_RELTYPES_BASE}/schema', + 'title': l10n.translate('Schema of collection in JSON', request.locale), # noqa + 'href': f'{api.get_collections_url()}/{k}/schema?f={F_JSON}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': f'{OGC_RELTYPES_BASE}/schema', + 'title': l10n.translate('Schema of collection in HTML', request.locale), # noqa + 'href': f'{api.get_collections_url()}/{k}/schema?f={F_HTML}' # noqa + }) + + if is_vector_tile or collection_data_type in ['feature', 'record']: + # TODO: translate + collection['itemType'] = collection_data_type + LOGGER.debug('Adding feature/record based links') + collection['links'].append({ + 'type': 'application/schema+json', + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/queryables', + 'title': l10n.translate('Queryables for this collection as JSON', request.locale), # noqa + 'href': f'{api.get_collections_url()}/{k}/queryables?f={F_JSON}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/queryables', + 'title': l10n.translate('Queryables for this collection as HTML', request.locale), # noqa + 'href': f'{api.get_collections_url()}/{k}/queryables?f={F_HTML}' # noqa + }) + collection['links'].append({ + 'type': 'application/geo+json', + 'rel': 'items', + 'title': l10n.translate('Items as GeoJSON', request.locale), # noqa + 'href': f'{api.get_collections_url()}/{k}/items?f={F_JSON}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': 'items', + 'title': l10n.translate('Items as RDF (GeoJSON-LD)', request.locale), # noqa + 'href': f'{api.get_collections_url()}/{k}/items?f={F_JSONLD}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'items', + 'title': l10n.translate('Items as HTML', request.locale), # noqa + 'href': f'{api.get_collections_url()}/{k}/items?f={F_HTML}' # noqa + }) + + # OAPIF Part 2 - list supported CRSs and StorageCRS + if collection_data_type in ['edr', 'feature']: + collection['crs'] = get_supported_crs_list(collection_data, DEFAULT_CRS_LIST) # noqa + collection['storageCRS'] = collection_data.get('storage_crs', DEFAULT_STORAGE_CRS) # noqa + if 'storage_crs_coordinate_epoch' in collection_data: + collection['storageCrsCoordinateEpoch'] = collection_data.get('storage_crs_coordinate_epoch') # noqa + + elif collection_data_type == 'coverage': + # TODO: translate + LOGGER.debug('Adding coverage based links') + collection['links'].append({ + 'type': 'application/prs.coverage+json', + 'rel': f'{OGC_RELTYPES_BASE}/coverage', + 'title': l10n.translate('Coverage data', request.locale), + 'href': f'{api.get_collections_url()}/{k}/coverage?f={F_JSON}' # noqa + }) + if collection_data_format is not None: + title_ = l10n.translate('Coverage data as', request.locale) # noqa + title_ = f"{title_} {collection_data_format['name']}" + collection['links'].append({ + 'type': collection_data_format['mimetype'], + 'rel': f'{OGC_RELTYPES_BASE}/coverage', + 'title': title_, + 'href': f"{api.get_collections_url()}/{k}/coverage?f={collection_data_format['name']}" # noqa + }) + if dataset is not None: + LOGGER.debug('Creating extended coverage metadata') + try: + provider_def = get_provider_by_type( + api.config['resources'][k]['providers'], + 'coverage') + p = load_plugin('provider', provider_def) + except ProviderConnectionError: + msg = 'connection error (check logs)' + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, + headers, request.format, + 'NoApplicableCode', msg) + except ProviderTypeError: + pass + else: + collection['extent']['spatial']['grid'] = [{ + 'cellsCount': p._coverage_properties['width'], + 'resolution': p._coverage_properties['resx'] + }, { + 'cellsCount': p._coverage_properties['height'], + 'resolution': p._coverage_properties['resy'] + }] + if 'time_range' in p._coverage_properties: + collection['extent']['temporal'] = { + 'interval': [p._coverage_properties['time_range']] + } + if 'restime' in p._coverage_properties: + collection['extent']['temporal']['grid'] = { + 'resolution': p._coverage_properties['restime'] # noqa + } + if 'uad' in p._coverage_properties: + collection['extent'].update(p._coverage_properties['uad']) # noqa + + try: + tile = get_provider_by_type(v['providers'], 'tile') + p = load_plugin('provider', tile) + except ProviderConnectionError: + msg = 'connection error (check logs)' + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, + headers, request.format, + 'NoApplicableCode', msg) + except ProviderTypeError: + tile = None + + if tile: + # TODO: translate + + LOGGER.debug('Adding tile links') + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': f'http://www.opengis.net/def/rel/ogc/1.0/tilesets-{p.tile_type}', # noqa + 'title': l10n.translate('Tiles as JSON', request.locale), + 'href': f'{api.get_collections_url()}/{k}/tiles?f={F_JSON}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': f'http://www.opengis.net/def/rel/ogc/1.0/tilesets-{p.tile_type}', # noqa + 'title': l10n.translate('Tiles as HTML', request.locale), + 'href': f'{api.get_collections_url()}/{k}/tiles?f={F_HTML}' # noqa + }) + + try: + map_ = get_provider_by_type(v['providers'], 'map') + except ProviderTypeError: + map_ = None + + if map_: + LOGGER.debug('Adding map links') + + map_mimetype = map_['format']['mimetype'] + map_format = map_['format']['name'] + + title_ = l10n.translate('Map as', request.locale) + title_ = f"{title_} {map_format}" + + collection['links'].append({ + 'type': map_mimetype, + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/map', + 'title': title_, + 'href': f"{api.get_collections_url()}/{k}/map?f={map_format}" # noqa + }) + + try: + edr = get_provider_by_type(v['providers'], 'edr') + p = load_plugin('provider', edr) + except ProviderConnectionError: + msg = 'connection error (check logs)' + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'NoApplicableCode', msg) + except ProviderTypeError: + edr = None + + if edr: + # TODO: translate + LOGGER.debug('Adding EDR links') + collection['data_queries'] = {} + parameters = p.get_fields() + if parameters: + collection['parameter_names'] = {} + for key, value in parameters.items(): + collection['parameter_names'][key] = { + 'id': key, + 'type': 'Parameter', + 'name': value['title'], + 'observedProperty': { + 'label': { + 'id': key, + 'en': value['title'] + }, + }, + 'unit': { + 'label': { + 'en': value['title'] + }, + 'symbol': { + 'value': value['x-ogc-unit'], + 'type': 'http://www.opengis.net/def/uom/UCUM/' # noqa + } + } + } + + for qt in p.get_query_types(): + data_query = { + 'link': { + 'href': f'{api.get_collections_url()}/{k}/{qt}', + 'rel': 'data', + 'variables': { + 'query_type': qt + } + } + } + collection['data_queries'][qt] = data_query + + title1 = l10n.translate('query for this collection as JSON', request.locale) # noqa + title1 = f'{qt} {title1}' + title2 = l10n.translate('query for this collection as HTML', request.locale) # noqa + title2 = f'{qt} {title2}' + + collection['links'].append({ + 'type': 'application/json', + 'rel': 'data', + 'title': title1, + 'href': f'{api.get_collections_url()}/{k}/{qt}?f={F_JSON}' + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'data', + 'title': title2, + 'href': f'{api.get_collections_url()}/{k}/{qt}?f={F_HTML}' + }) + + if dataset is not None and k == dataset: + fcm = collection + break + + fcm['collections'].append(collection) + + if dataset is None: + # TODO: translate + fcm['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': l10n.translate('This document as JSON', request.locale), # noqa + 'href': f'{api.get_collections_url()}?f={F_JSON}' + }) + fcm['links'].append({ + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': l10n.translate('This document as RDF (JSON-LD)', request.locale), # noqa + 'href': f'{api.get_collections_url()}?f={F_JSONLD}' + }) + fcm['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': l10n.translate('This document as HTML', request.locale), # noqa + 'href': f'{api.get_collections_url()}?f={F_HTML}' + }) + + if request.format == F_HTML: # render + fcm['collections_path'] = api.get_collections_url() + if dataset is not None: + tpl_config = api.get_dataset_templates(dataset) + content = render_j2_template(api.tpl_config, tpl_config, + 'collections/collection.html', + fcm, request.locale) + else: + content = render_j2_template( + api.tpl_config, api.config['server']['templates'], + 'collections/index.html', fcm, request.locale) + + return headers, HTTPStatus.OK, content + + if request.format == F_JSONLD: + jsonld = api.fcmld.copy() + if dataset is not None: + jsonld['dataset'] = jsonldify_collection(api, fcm, + request.locale) + else: + jsonld['dataset'] = [ + jsonldify_collection(api, c, request.locale) + for c in fcm.get('collections', []) + ] + return headers, HTTPStatus.OK, to_json(jsonld, api.pretty_print) + + return headers, HTTPStatus.OK, to_json(fcm, api.pretty_print) + + +def get_collection_schema(api: API, request: Union[APIRequest, Any], + dataset) -> Tuple[dict, int, str]: + """ + Returns a collection schema + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(**api.api_headers) + + if any([dataset is None, + dataset not in api.config['resources'].keys()]): + + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection schema') + try: + LOGGER.debug('Loading feature provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'feature')) + except ProviderTypeError: + try: + LOGGER.debug('Loading coverage provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'coverage')) # noqa + except ProviderTypeError: + LOGGER.debug('Loading record provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'record')) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + schema = { + 'type': 'object', + 'title': l10n.translate( + api.config['resources'][dataset]['title'], request.locale), + 'properties': {}, + '$schema': 'http://json-schema.org/draft/2019-09/schema', + '$id': f'{api.get_collections_url()}/{dataset}/schema' + } + + if p.type != 'coverage': + schema['properties']['geometry'] = { + 'format': 'geometry-any', + 'x-ogc-role': 'primary-geometry', + #GA Customisation - Add $ref link to ensure geojson definition is accessible. + #To be reviewed when upgrading beyond version 0.20.0 + '$ref': 'https://geojson.org/schema/Geometry.json' + } + + for k, v in p.fields.items(): + schema['properties'][k] = v + if v.get('format') is None: + schema['properties'][k].pop('format', None) + + if k == p.id_field: + schema['properties'][k]['x-ogc-role'] = 'id' + if k == p.time_field: + schema['properties'][k]['x-ogc-role'] = 'primary-instant' + + if request.format == F_HTML: # render + tpl_config = api.get_dataset_templates(dataset) + schema['title'] = l10n.translate( + api.config['resources'][dataset]['title'], request.locale) + + schema['collections_path'] = api.get_collections_url() + schema['dataset_path'] = f'{api.get_collections_url()}/{dataset}' + + content = render_j2_template(api.tpl_config, tpl_config, + 'collections/schema.html', + schema, request.locale) + + return headers, HTTPStatus.OK, content + + headers['Content-Type'] = 'application/schema+json' + + return headers, HTTPStatus.OK, to_json(schema, api.pretty_print) + + +def validate_bbox(value=None) -> list: + """ + Helper function to validate bbox parameter + + :param value: `list` of minx, miny, maxx, maxy + + :returns: bbox as `list` of `float` values + """ + + if value is None: + LOGGER.debug('bbox is empty') + return [] + + bbox = value.split(',') + + if len(bbox) not in [4, 6]: + msg = 'bbox should be either 4 values (minx,miny,maxx,maxy) ' \ + 'or 6 values (minx,miny,minz,maxx,maxy,maxz)' + LOGGER.debug(msg) + raise ValueError(msg) + + try: + bbox = [float(c) for c in bbox] + except ValueError as err: + msg = 'bbox values must be numbers' + err.args = (msg,) + LOGGER.debug(msg) + raise + + if (len(bbox) == 4 and bbox[1] > bbox[3]) \ + or (len(bbox) == 6 and bbox[1] > bbox[4]): + msg = 'miny should be less than maxy' + LOGGER.debug(msg) + raise ValueError(msg) + + if (len(bbox) == 4 and bbox[0] > bbox[2]) \ + or (len(bbox) == 6 and bbox[0] > bbox[3]): + msg = 'minx is greater than maxx (possibly antimeridian bbox)' + LOGGER.debug(msg) + + if len(bbox) == 6 and bbox[2] > bbox[5]: + msg = 'minz should be less than maxz' + LOGGER.debug(msg) + raise ValueError(msg) + + return bbox + + +def validate_datetime(resource_def, datetime_=None) -> str: + """ + Helper function to validate temporal parameter + + :param resource_def: `dict` of configuration resource definition + :param datetime_: `str` of datetime parameter + + :returns: `str` of datetime input, if valid + """ + + # TODO: pass datetime to query as a `datetime` object + # we would need to ensure partial dates work accordingly + # as well as setting '..' values to `None` so that underlying + # providers can just assume a `datetime.datetime` object + # + # NOTE: needs testing when passing partials from API to backend + + datetime_invalid = False + + if datetime_ is not None and 'temporal' in resource_def: + + dateparse_begin = partial(dateparse, default=datetime.min) + dateparse_end = partial(dateparse, default=datetime.max) + unix_epoch = datetime(1970, 1, 1, 0, 0, 0) + dateparse_ = partial(dateparse, default=unix_epoch) + + te = resource_def['temporal'] + + try: + if te['begin'] is not None and te['begin'].tzinfo is None: + te['begin'] = te['begin'].replace(tzinfo=pytz.UTC) + if te['end'] is not None and te['end'].tzinfo is None: + te['end'] = te['end'].replace(tzinfo=pytz.UTC) + except AttributeError: + msg = 'Configured times should be RFC3339' + LOGGER.error(msg) + raise ValueError(msg) + + if '/' in datetime_: # envelope + LOGGER.debug('detected time range') + LOGGER.debug('Validating time windows') + + # normalize "" to ".." (actually changes datetime_) + datetime_ = re.sub(r'^/', '../', datetime_) + datetime_ = re.sub(r'/$', '/..', datetime_) + + datetime_begin, datetime_end = datetime_.split('/') + if datetime_begin != '..': + datetime_begin = dateparse_begin(datetime_begin) + if datetime_begin.tzinfo is None: + datetime_begin = datetime_begin.replace( + tzinfo=pytz.UTC) + + if datetime_end != '..': + datetime_end = dateparse_end(datetime_end) + if datetime_end.tzinfo is None: + datetime_end = datetime_end.replace(tzinfo=pytz.UTC) + + datetime_invalid = any([ + (te['end'] is not None and datetime_begin != '..' and + datetime_begin > te['end']), + (te['begin'] is not None and datetime_end != '..' and + datetime_end < te['begin']) + ]) + + else: # time instant + LOGGER.debug('detected time instant') + datetime__ = dateparse_(datetime_) + if datetime__ != '..': + if datetime__.tzinfo is None: + datetime__ = datetime__.replace(tzinfo=pytz.UTC) + datetime_invalid = any([ + (te['begin'] is not None and datetime__ != '..' and + datetime__ < te['begin']), + (te['end'] is not None and datetime__ != '..' and + datetime__ > te['end']) + ]) + + if datetime_invalid: + msg = 'datetime parameter out of range' + LOGGER.debug(msg) + raise ValueError(msg) + + return datetime_ + + +def validate_subset(value: str) -> dict: + """ + Helper function to validate subset parameter + + :param value: `subset` parameter + + :returns: dict of axis/values + """ + + subsets = {} + + for s in value.split(','): + LOGGER.debug(f'Processing subset {s}') + m = re.search(r'(.*)\((.*)\)', s) + subset_name, values = m.group(1, 2) + + if '"' in values: + LOGGER.debug('Values are strings') + if values.count('"') % 2 != 0: + msg = 'Invalid format: subset should be like axis("min"[:"max"])' # noqa + LOGGER.error(msg) + raise ValueError(msg) + try: + LOGGER.debug('Value is an interval') + m = re.search(r'"(\S+)":"(\S+)"', values) + values = list(m.group(1, 2)) + except AttributeError: + LOGGER.debug('Value is point') + m = re.search(r'"(.*)"', values) + values = [m.group(1)] + else: + LOGGER.debug('Values are numbers') + try: + LOGGER.debug('Value is an interval') + m = re.search(r'(\S+):(\S+)', values) + values = list(m.group(1, 2)) + except AttributeError: + LOGGER.debug('Value is point') + values = [values] + + subsets[subset_name] = list(map(get_typed_value, values)) + + return subsets + + +def evaluate_limit(requested: Union[None, int], server_limits: dict, + collection_limits: dict) -> int: + """ + Helper function to evaluate limit parameter + + :param requested: the limit requested by the client + :param server_limits: `dict` of server limits + :param collection_limits: `dict` of collection limits + + :returns: `int` of evaluated limit + """ + + effective_limits = ChainMap(collection_limits, server_limits) + + default = effective_limits.get('default_items', 10) + max_ = effective_limits.get('max_items', 10) + on_exceed = effective_limits.get('on_exceed', 'throttle') + + LOGGER.debug(f'Requested limit: {requested}') + LOGGER.debug(f'Default limit: {default}') + LOGGER.debug(f'Maximum limit: {max_}') + LOGGER.debug(f'On exceed: {on_exceed}') + + if requested is None: + LOGGER.debug('no limit requested; returning default') + return default + + requested2 = get_typed_value(requested) + if not isinstance(requested2, int): + raise ValueError('limit value should be an integer') + + if requested2 <= 0: + raise ValueError('limit value should be strictly positive') + elif requested2 > max_ and on_exceed == 'error': + raise RuntimeError('Limit exceeded; throwing errror') + else: + LOGGER.debug('limit requested') + return min(requested2, max_) diff --git a/pygeoapi/api/coverages.py b/pygeoapi/api/coverages.py new file mode 100644 index 000000000..c6687042c --- /dev/null +++ b/pygeoapi/api/coverages.py @@ -0,0 +1,249 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import logging +from http import HTTPStatus +from typing import Tuple + +from pygeoapi import l10n +from pygeoapi.plugin import load_plugin +from pygeoapi.provider.base import ProviderGenericError, ProviderTypeError +from pygeoapi.util import ( + filter_dict_by_key_value, get_provider_by_type, to_json +) + +from . import ( + APIRequest, API, F_JSON, SYSTEM_LOCALE, validate_bbox, validate_datetime, + validate_subset +) + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/oas30', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/html', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/geodata-coverage', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-subset', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-rangesubset', # noqa + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-bbox', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-datetime' +] + + +def get_collection_coverage( + api: API, request: APIRequest, dataset) -> Tuple[dict, int, str]: + """ + Returns a subset of a collection coverage + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + query_args = {} + format_ = request.format or F_JSON + + # Force response content type and language (en-US only) headers + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + LOGGER.debug('Loading provider') + try: + collection_def = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'coverage') + + p = load_plugin('provider', collection_def) + except KeyError: + msg = 'collection does not exist' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, format_, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + LOGGER.debug('Processing bbox parameter') + + bbox = request.params.get('bbox') + + if bbox is None: + bbox = [] + else: + try: + bbox = validate_bbox(bbox) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, format_, + 'InvalidParameterValue', msg) + + query_args['bbox'] = bbox + + LOGGER.debug('Processing bbox-crs parameter') + + bbox_crs = request.params.get('bbox-crs') + if bbox_crs is not None: + query_args['bbox_crs'] = bbox_crs + + LOGGER.debug('Processing datetime parameter') + + datetime_ = request.params.get('datetime') + + try: + datetime_ = validate_datetime( + api.config['resources'][dataset]['extents'], datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + query_args['datetime_'] = datetime_ + query_args['format_'] = format_ + + properties = request.params.get('properties') + if properties: + LOGGER.debug('Processing properties parameter') + query_args['properties'] = [rs for + rs in properties.split(',') if rs] + LOGGER.debug(f"Fields: {query_args['properties']}") + + for a in query_args['properties']: + if a not in p.fields: + msg = 'Invalid field specified' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + if 'subset' in request.params: + LOGGER.debug('Processing subset parameter') + try: + subsets = validate_subset(request.params['subset'] or '') + except (AttributeError, ValueError) as err: + msg = f'Invalid subset: {err}' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + if not set(subsets.keys()).issubset(p.axes): + msg = 'Invalid axis name' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + query_args['subsets'] = subsets + LOGGER.debug(f"Subsets: {query_args['subsets']}") + + LOGGER.debug('Querying coverage') + try: + data = p.query(**query_args) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + mt = collection_def['format']['name'] + if format_ == mt: # native format + if p.filename is not None: + cd = f'attachment; filename="{p.filename}"' + headers['Content-Disposition'] = cd + + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + elif format_ == F_JSON: + headers['Content-Type'] = 'application/prs.coverage+json' + return headers, HTTPStatus.OK, to_json(data, api.pretty_print) + else: + return api.get_format_exception(request) + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in get_visible_collections(cfg).items(): + try: + load_plugin('provider', get_provider_by_type( + collections[k]['providers'], 'coverage')) + except ProviderTypeError: + LOGGER.debug('collection is not coverage based') + continue + + coverage_path = f'/collections/{k}/coverage' + title = l10n.translate(v['title'], locale) + description = l10n.translate(v['description'], locale) + + paths[coverage_path] = { + 'get': { + 'summary': f'Get {title} coverage', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Coverage', + 'parameters': [ + {'$ref': '#/components/parameters/lang'}, + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/bbox'}, + {'$ref': '#/components/parameters/bbox-crs'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + + return [{'name': 'coverages'}], {'paths': paths} diff --git a/pygeoapi/api/environmental_data_retrieval.py b/pygeoapi/api/environmental_data_retrieval.py new file mode 100644 index 000000000..e78170368 --- /dev/null +++ b/pygeoapi/api/environmental_data_retrieval.py @@ -0,0 +1,601 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from http import HTTPStatus +import logging +from typing import Tuple +import urllib + +from shapely.errors import ShapelyError +from shapely.wkt import loads as shapely_loads + +from pygeoapi import l10n +from pygeoapi.api import evaluate_limit +from pygeoapi.plugin import load_plugin, PLUGINS +from pygeoapi.provider.base import ( + ProviderGenericError, ProviderItemNotFoundError) +from pygeoapi.util import ( + filter_providers_by_type, get_provider_by_type, render_j2_template, + to_json, filter_dict_by_key_value +) + +from . import (APIRequest, API, F_COVERAGEJSON, F_HTML, F_JSON, F_JSONLD, + validate_datetime, validate_bbox) + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-edr-1/1.0/conf/core' +] + + +def get_collection_edr_instances(api: API, request: APIRequest, dataset, + instance_id=None) -> Tuple[dict, int, str]: + """ + Queries collection EDR instances + + :param request: APIRequest instance with query params + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + data = { + 'instances': [], + 'links': [] + } + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + headers = request.get_response_headers(api.default_locale, + **api.api_headers) + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + uri = f'{api.get_collections_url()}/{dataset}' + + LOGGER.debug('Loading provider') + try: + p = load_plugin('provider', get_provider_by_type( + collections[dataset]['providers'], 'edr')) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + if instance_id is not None: + try: + instances = [p.get_instance(instance_id)] + except ProviderItemNotFoundError: + msg = 'Instance not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + else: + instances = p.instances() + + for instance in instances: + instance_dict = { + 'id': instance, + 'links': [{ + 'href': f'{uri}/instances/{instance}?f={F_JSON}', + 'rel': request.get_linkrel(F_JSON), + 'type': 'application/json' + }, { + 'href': f'{uri}/instances/{instance}?f={F_HTML}', + 'rel': request.get_linkrel(F_HTML), + 'type': 'text/html' + }, { + 'href': f'{uri}?f={F_HTML}', + 'rel': 'collection', + 'title': collections[dataset]['title'], + 'type': 'text/html' + }, { + 'href': f'{uri}?f={F_JSON}', + 'rel': 'collection', + 'title': collections[dataset]['title'], + 'type': 'application/json' + }], + 'data_queries': {} + } + + for qt in p.get_query_types(): + if qt == 'instances': + continue + data_query = { + 'link': { + 'href': f'{uri}/instances/{instance}/{qt}', + 'rel': 'data', + 'title': f'{qt} query' + } + } + instance_dict['data_queries'][qt] = data_query + + data['instances'].append(instance_dict) + + if instance_id is not None: + data = data['instances'][0] + data.pop('instances', None) + links_uri = f'{uri}/instances/{instance_id}' + else: + links_uri = f'{uri}/instances' + + if instance_id is None: + data['links'].extend([{ + 'href': f'{links_uri}?f={F_JSON}', + 'rel': request.get_linkrel(F_JSON), + 'type': 'application/json' + }, { + 'href': f'{links_uri}?f={F_HTML}', + 'rel': request.get_linkrel(F_HTML), + 'type': 'text/html' + }]) + + if request.format == F_HTML: # render + tpl_config = api.get_dataset_templates(dataset) + + serialized_query_params = '' + for k, v in request.params.items(): + if k != 'f': + serialized_query_params += '&' + serialized_query_params += urllib.parse.quote(k, safe='') + serialized_query_params += '=' + serialized_query_params += urllib.parse.quote(str(v), safe=',') + + if instance_id is None: + uri = f'{uri}/instances' + else: + uri = f'{uri}/instances/{instance_id}' + + data['query_type'] = 'instances' + data['query_path'] = uri + data['title'] = collections[dataset]['title'] + data['description'] = collections[dataset]['description'] + data['keywords'] = collections[dataset]['keywords'] + data['collections_path'] = api.get_collections_url() + + if instance_id is None: + data['dataset_path'] = data['collections_path'] + '/' + uri.split('/')[-2] # noqa + template = 'collections/edr/instances.html' + else: + data['dataset_path'] = data['collections_path'] + '/' + uri.split('/')[-3] # noqa + template = 'collections/edr/instance.html' + + data['links'] = [{ + 'rel': 'collection', + 'title': collections[dataset]['title'], + 'href': f"{data['dataset_path']}?f={F_JSON}", + 'type': 'text/html' + }, { + 'rel': 'collection', + 'title': collections[dataset]['title'], + 'href': f"{data['dataset_path']}?f={F_HTML}", + 'type': 'application/json' + }, { + 'type': 'application/json', + 'rel': 'alternate', + 'title': l10n.translate('This document as JSON', request.locale), + 'href': f'{uri}?f={F_JSON}{serialized_query_params}' + }, { + 'type': 'application/ld+json', + 'rel': 'alternate', + 'title': l10n.translate('This document as JSON-LD', request.locale), # noqa + 'href': f'{uri}?f={F_JSONLD}{serialized_query_params}' + }] + + content = render_j2_template(api.tpl_config, tpl_config, template, + data, api.default_locale) + else: + content = to_json(data, api.pretty_print) + + return headers, HTTPStatus.OK, content + + +def get_collection_edr_query(api: API, request: APIRequest, + dataset, instance, query_type, + location_id=None) -> Tuple[dict, int, str]: + """ + Queries collection EDR + + :param request: APIRequest instance with query params + :param dataset: dataset name + :param instance: instance name + :param query_type: EDR query type + :param location_id: location id of a /location/ query + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + headers = request.get_response_headers(api.default_locale, + **api.api_headers) + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Loading provider') + try: + p = load_plugin('provider', get_provider_by_type( + collections[dataset]['providers'], 'edr')) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + if instance is not None and not p.get_instance(instance): + msg = 'Invalid instance identifier' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, + request.format, 'InvalidParameterValue', msg) + + if query_type not in p.get_query_types(): + msg = 'Unsupported query type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + datetime_ = validate_datetime(collections[dataset]['extents'], + datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing parameter-name parameter') + parameternames = request.params.get('parameter-name') or [] + if isinstance(parameternames, str): + parameternames = parameternames.split(',') + + bbox = None + if query_type in ['cube', 'locations']: + LOGGER.debug('Processing cube bbox') + try: + bbox = validate_bbox(request.params.get('bbox')) + if not bbox and query_type == 'cube': + raise ValueError('bbox parameter required by cube queries') + except ValueError as err: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', str(err)) + + LOGGER.debug('Processing coords parameter') + wkt = request.params.get('coords') + + if wkt: + try: + wkt = shapely_loads(wkt) + except ShapelyError: + msg = 'invalid coords parameter' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + elif query_type not in ['cube', 'locations']: + msg = 'missing coords parameter' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + within = within_units = None + if query_type == 'radius': + LOGGER.debug('Processing within / within-units parameters') + within = request.params.get('within') + within_units = request.params.get('within-units') + + LOGGER.debug('Processing z parameter') + z = request.params.get('z') + + if parameternames and not any((fld in parameternames) + for fld in p.get_fields().keys()): + msg = 'Invalid parameter-name' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing limit parameter') + if api.config['server'].get('limit') is not None: + msg = ('server.limit is no longer supported! ' + 'Please use limits at the server or collection ' + 'level (RFC5)') + LOGGER.warning(msg) + try: + limit = evaluate_limit(request.params.get('limit'), + api.config['server'].get('limits', {}), + collections[dataset].get('limits', {})) + except ValueError as err: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', str(err)) + + query_args = dict( + query_type=query_type, + instance=instance, + format_=request.format, + datetime_=datetime_, + select_properties=parameternames, + wkt=wkt, + z=z, + bbox=bbox, + within=within, + within_units=within_units, + limit=limit, + location_id=location_id + ) + + try: + data = p.query(**query_args) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + if request.format == F_HTML: # render + tpl_config = api.get_dataset_templates(dataset) + + uri = f'{api.get_collections_url()}/{dataset}/{query_type}' + serialized_query_params = '' + for k, v in request.params.items(): + if k != 'f': + serialized_query_params += '&' + serialized_query_params += urllib.parse.quote(k, safe='') + serialized_query_params += '=' + serialized_query_params += urllib.parse.quote(str(v), safe=',') + + data['query_type'] = query_type.capitalize() + data['query_path'] = uri + data['dataset_path'] = '/'.join(uri.split('/')[:-1]) + data['collections_path'] = api.get_collections_url() + + data['links'] = [{ + 'rel': 'collection', + 'title': collections[dataset]['title'], + 'href': data['dataset_path'] + }, { + 'type': 'application/prs.coverage+json', + 'rel': request.get_linkrel(F_COVERAGEJSON), + 'title': l10n.translate('This document as CoverageJSON', request.locale), # noqa + 'href': f'{uri}?f={F_COVERAGEJSON}{serialized_query_params}' + }, { + 'type': 'application/ld+json', + 'rel': 'alternate', + 'title': l10n.translate('This document as JSON-LD', request.locale), # noqa + 'href': f'{uri}?f={F_JSONLD}{serialized_query_params}' + }] + + content = render_j2_template(api.tpl_config, tpl_config, + 'collections/edr/query.html', data, + api.default_locale) + else: + content = to_json(data, api.pretty_print) + + return headers, HTTPStatus.OK, content + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + LOGGER.debug('setting up edr endpoints') + + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in get_visible_collections(cfg).items(): + edr_extension = filter_providers_by_type( + collections[k]['providers'], 'edr') + + if edr_extension: + collection_name_path = f'/collections/{k}' + + ep = load_plugin('provider', edr_extension) + + edr_query_endpoints = [] + + for qt in [qt for qt in ep.get_query_types() if qt != 'locations']: + edr_query_endpoints.append({ + 'path': f'{collection_name_path}/{qt}', + 'qt': qt, + 'op_id': f'query{qt.capitalize()}{k.capitalize()}' + }) + if 'instances' in ep.get_query_types() and qt != 'instances': + edr_query_endpoints.append({ + 'path': f'{collection_name_path}/instances/{{instanceId}}/{qt}', # noqa + 'qt': qt, + 'op_id': f'query{qt.capitalize()}Instance{k.capitalize()}' # noqa + }) + + for eqe in edr_query_endpoints: + if eqe['qt'] == 'cube': + spatial_parameter = { + 'description': 'Only features that have a geometry that intersects the bounding box are selected.The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or depth).', # noqa + 'explode': False, + 'in': 'query', + 'name': 'bbox', + 'required': True, + 'schema': { + 'items': { + 'type': 'number' + }, + 'maxItems': 6, + 'minItems': 4, + 'type': 'array' + }, + 'style': 'form' + } + else: + spatial_parameter = { + '$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{eqe['qt']}Coords.yaml" # noqa + } + paths[eqe['path']] = { + 'get': { + 'summary': f"query {v['description']} by {eqe['qt']}", + 'description': v['description'], + 'tags': [k], + 'operationId': eqe['op_id'], + 'parameters': [ + spatial_parameter, + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"}, # noqa + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': { + 'description': 'Response', + 'content': { + 'application/prs.coverage+json': { + 'schema': { + '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa + } + } + } + } + } + } + } + if 'instanceId' in eqe['path']: + paths[eqe['path']]['get']['parameters'].insert(0, { + '$ref': f"{OPENAPI_YAML['oaedr']}/parameters/instanceId.yaml"} # noqa + ) + + if 'instances' in ep.get_query_types(): + paths[f'{collection_name_path}/instances'] = { + 'get': { + 'summary': f"Get pre-defined instances of {v['description']}", # noqa + 'description': v['description'], + 'tags': [k], + 'operationId': f'getInstances{k.capitalize()}', + 'parameters': [ + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + paths[f'{collection_name_path}/instances/{{instanceId}}'] = { + 'get': { + 'summary': f"Get {v['description']} instance", + 'description': v['description'], + 'tags': [k], + 'operationId': f'getInstance{k.capitalize()}', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/instanceId.yaml"}, # noqa + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa + } + } + } + + if 'locations' in ep.get_query_types(): + paths[f'{collection_name_path}/locations'] = { + 'get': { + 'summary': f"Get pre-defined locations of {v['description']}", # noqa + 'description': v['description'], + 'tags': [k], + 'operationId': f'getLocations{k.capitalize()}', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/bbox.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + paths[f'{collection_name_path}/locations/{{locId}}'] = { + 'get': { + 'summary': f"query {v['description']} by location", + 'description': v['description'], + 'tags': [k], + 'operationId': f'getLocation{k.capitalize()}', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/locationId.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': { + 'description': 'Response', + 'content': { + 'application/prs.coverage+json': { + 'schema': { + '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa + } + } + } + } + } + } + } + + return [{'name': 'edr'}], {'paths': paths} diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py new file mode 100644 index 000000000..fa97bc392 --- /dev/null +++ b/pygeoapi/api/itemtypes.py @@ -0,0 +1,1356 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from copy import deepcopy +from datetime import datetime +from http import HTTPStatus +import logging +from typing import Any, Tuple, Union, Optional +import urllib.parse + +from pygeofilter.parsers.ecql import parse as parse_ecql_text +from pygeofilter.parsers.cql2_json import parse as parse_cql2_json +from pyproj.exceptions import CRSError + +from pygeoapi import l10n +from pygeoapi.api import evaluate_limit +from pygeoapi.formatter.base import FormatterSerializationError +from pygeoapi.linked_data import geojson2jsonld +from pygeoapi.plugin import load_plugin, PLUGINS +from pygeoapi.provider.base import ( + ProviderGenericError, ProviderTypeError, SchemaType) + +from pygeoapi.util import (CrsTransformSpec, filter_providers_by_type, + filter_dict_by_key_value, get_crs_from_uri, + get_provider_by_type, get_supported_crs_list, + modify_pygeofilter, render_j2_template, str2bool, + to_json, transform_bbox) + +from . import ( + APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD, + validate_bbox, validate_datetime +) + +LOGGER = logging.getLogger(__name__) + +OGC_RELTYPES_BASE = 'http://www.opengis.net/def/rel/ogc/1.0' + +DEFAULT_CRS_LIST = [ + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', +] + +DEFAULT_CRS = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' +DEFAULT_STORAGE_CRS = DEFAULT_CRS + +CONFORMANCE_CLASSES_FEATURES = [ + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/html', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson', + 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs', + 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables', + 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables-query-parameters', # noqa + 'http://www.opengis.net/spec/ogcapi-features-4/1.0/conf/create-replace-delete', # noqa + 'http://www.opengis.net/spec/ogcapi-features-5/1.0/conf/schemas', + 'http://www.opengis.net/spec/ogcapi-features-5/1.0/conf/core-roles-features' # noqa +] + +CONFORMANCE_CLASSES_RECORDS = [ + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/sorting', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/opensearch', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/json', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/html' +] + + +def get_collection_queryables(api: API, request: Union[APIRequest, Any], + dataset=None) -> Tuple[dict, int, str]: + """ + Provide collection queryables + + :param request: A request object + :param dataset: name of collection + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(**api.api_headers) + + if any([dataset is None, + dataset not in api.config['resources'].keys()]): + + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection queryables') + + p = None + for pt in ['feature', 'coverage', 'record']: + try: + LOGGER.debug(f'Loading {pt} provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], pt)) + break + except ProviderTypeError: + LOGGER.debug(f'Providing type {pt} not found') + + if p is None: + msg = 'queryables not available for this collection' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableError', msg) + + queryables = { + 'type': 'object', + 'title': l10n.translate( + api.config['resources'][dataset]['title'], request.locale), + 'properties': {}, + '$schema': 'http://json-schema.org/draft/2019-09/schema', + '$id': f'{api.get_collections_url()}/{dataset}/queryables' + } + + if p.fields: + queryables['properties']['geometry'] = { + 'format': 'geometry-any', + 'x-ogc-role': 'primary-geometry', + #GA Customisation - Add $ref link to ensure geojson definition is accessible. + #To be reviewed when upgrading beyond version 0.20.0 + '$ref': 'https://geojson.org/schema/Geometry.json' + } + + for k, v in p.fields.items(): + show_field = False + if p.properties: + if k in p.properties: + show_field = True + else: + show_field = True + + if show_field: + queryables['properties'][k] = { + 'title': k, + 'type': v['type'] + } + if v.get('format') is not None: + queryables['properties'][k]['format'] = v['format'] + if 'values' in v: + queryables['properties'][k]['enum'] = v['values'] + + if k == p.id_field: + queryables['properties'][k]['x-ogc-role'] = 'id' + if k == p.time_field: + queryables['properties'][k]['x-ogc-role'] = 'primary-instant' # noqa + + if request.format == F_HTML: # render + tpl_config = api.get_dataset_templates(dataset) + + queryables['title'] = l10n.translate( + api.config['resources'][dataset]['title'], request.locale) + + queryables['collections_path'] = api.get_collections_url() + queryables['dataset_path'] = f'{api.get_collections_url()}/{dataset}' + + content = render_j2_template(api.tpl_config, tpl_config, + 'collections/queryables.html', + queryables, request.locale) + + return headers, HTTPStatus.OK, content + + headers['Content-Type'] = 'application/schema+json' + + return headers, HTTPStatus.OK, to_json(queryables, api.pretty_print) + + +def get_collection_items( + api: API, request: Union[APIRequest, Any], + dataset) -> Tuple[dict, int, str]: + """ + Queries collection + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + + properties = [] + reserved_fieldnames = ['bbox', 'bbox-crs', 'crs', 'f', 'lang', 'limit', + 'offset', 'resulttype', 'datetime', 'sortby', + 'properties', 'skipGeometry', 'q', + 'filter', 'filter-lang', 'filter-crs'] + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Processing offset parameter') + try: + offset = int(request.params.get('offset')) + if offset < 0: + msg = 'offset value should be positive or zero' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ValueError: + msg = 'offset value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + offset = 0 + + LOGGER.debug('Processing limit parameter') + if api.config['server'].get('limit') is not None: + msg = ('server.limit is no longer supported! ' + 'Please use limits at the server or collection ' + 'level (RFC5)') + LOGGER.warning(msg) + try: + limit = evaluate_limit(request.params.get('limit'), + api.config['server'].get('limits', {}), + collections[dataset].get('limits', {})) + except ValueError as err: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', str(err)) + + resulttype = request.params.get('resulttype') or 'results' + + LOGGER.debug('Processing bbox parameter') + + bbox = request.params.get('bbox') + + if bbox is None: + bbox = [] + else: + try: + bbox = validate_bbox(bbox) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + datetime_ = validate_datetime(collections[dataset]['extents'], + datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('processing q parameter') + q = request.params.get('q') or None + + LOGGER.debug('Loading provider') + + provider_def = None + try: + provider_type = 'feature' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + try: + provider_type = 'record' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + msg = 'Invalid provider type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + crs_transform_spec = None + if provider_type == 'feature': + # crs query parameter is only available for OGC API - Features + # right now, not for OGC API - Records. + LOGGER.debug('Processing crs parameter') + query_crs_uri = request.params.get('crs') + try: + crs_transform_spec = create_crs_transform_spec( + provider_def, query_crs_uri, + ) + except (ValueError, CRSError) as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + set_content_crs_header(headers, provider_def, query_crs_uri) + + LOGGER.debug('Processing bbox-crs parameter') + bbox_crs = request.params.get('bbox-crs') + if bbox_crs is not None: + # Validate bbox-crs parameter + if len(bbox) == 0: + msg = 'bbox-crs specified without bbox parameter' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + + if len(bbox_crs) == 0: + msg = 'bbox-crs specified but is empty' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + + supported_crs_list = get_supported_crs_list(provider_def, DEFAULT_CRS_LIST) # noqa + if bbox_crs not in supported_crs_list: + msg = f'bbox-crs {bbox_crs} not supported for this collection' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + elif len(bbox) > 0: + # bbox but no bbox-crs param: assume bbox is in default CRS + bbox_crs = DEFAULT_CRS + + # Transform bbox to storageCRS + # when bbox-crs different from storageCRS. + if len(bbox) > 0: + try: + # Get a pyproj CRS instance for the Collection's Storage CRS + storage_crs = provider_def.get('storage_crs', DEFAULT_STORAGE_CRS) # noqa + + # Do the (optional) Transform to the Storage CRS + bbox = transform_bbox(bbox, bbox_crs, storage_crs) + except CRSError as e: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', str(e)) + + LOGGER.debug('processing property parameters') + for k, v in request.params.items(): + if k not in reserved_fieldnames: + if k in list(p.fields.keys()): + LOGGER.debug(f'Adding property filter {k}={v}') + else: + LOGGER.debug(f'Adding additional property filter {k}={v}') + + properties.append((k, v)) + + LOGGER.debug('processing sort parameter') + val = request.params.get('sortby') + + if val is not None: + sortby = [] + sorts = val.split(',') + for s in sorts: + prop = s + order = '+' + if s[0] in ['+', '-']: + order = s[0] + prop = s[1:] + + if prop not in p.fields.keys(): + msg = 'bad sort property' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + sortby.append({'property': prop, 'order': order}) + else: + sortby = [] + + LOGGER.debug('processing properties parameter') + val = request.params.get('properties') + + if val is not None: + select_properties = val.split(',') + properties_to_check = set(p.properties) | set(p.fields.keys()) + + if (len(list(set(select_properties) - + set(properties_to_check))) > 0): + msg = 'unknown properties specified' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + select_properties = [] + + LOGGER.debug('processing skipGeometry parameter') + val = request.params.get('skipGeometry') + if val is not None: + skip_geometry = str2bool(val) + else: + skip_geometry = False + + LOGGER.debug('Processing filter-crs parameter') + filter_crs_uri = request.params.get('filter-crs', DEFAULT_CRS) + + LOGGER.debug('processing filter parameter') + cql_text = request.params.get('filter') + + if cql_text is not None: + try: + filter_ = parse_ecql_text(cql_text) + filter_ = modify_pygeofilter( + filter_, + filter_crs_uri=filter_crs_uri, + storage_crs_uri=provider_def.get('storage_crs'), + geometry_column_name=provider_def.get('geom_field'), + ) + except Exception: + msg = 'Bad CQL text' + LOGGER.error(f'{msg}: {cql_text}') + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + elif request.data: + try: + request_data = request.data.decode() + filter_ = parse_cql2_json(request_data) + filter_ = modify_pygeofilter( + filter_, + filter_crs_uri=filter_crs_uri, + storage_crs_uri=provider_def.get('storage_crs'), + geometry_column_name=provider_def.get('geom_field'), + ) + except Exception: + msg = 'Bad CQL JSON' + LOGGER.error(f'{msg}: {request_data}') + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + filter_ = None + + LOGGER.debug('Processing filter-lang parameter') + filter_lang = request.params.get('filter-lang') + # Currently only cql-text is handled, but it is optional + if filter_lang not in [None, 'cql-json', 'cql-text']: + msg = 'Invalid filter language' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + # Get provider locale (if any) + prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) + + LOGGER.debug('Querying provider') + LOGGER.debug(f'offset: {offset}') + LOGGER.debug(f'limit: {limit}') + LOGGER.debug(f'resulttype: {resulttype}') + LOGGER.debug(f'sortby: {sortby}') + LOGGER.debug(f'bbox: {bbox}') + if provider_type == 'feature': + LOGGER.debug(f'crs: {query_crs_uri}') + LOGGER.debug(f'datetime: {datetime_}') + LOGGER.debug(f'properties: {properties}') + LOGGER.debug(f'select properties: {select_properties}') + LOGGER.debug(f'skipGeometry: {skip_geometry}') + LOGGER.debug(f'language: {prv_locale}') + LOGGER.debug(f'q: {q}') + LOGGER.debug(f'cql_text: {cql_text}') + LOGGER.debug(f'filter_: {filter_}') + LOGGER.debug(f'filter-lang: {filter_lang}') + LOGGER.debug(f'filter-crs: {filter_crs_uri}') + + try: + content = p.query(offset=offset, limit=limit, + resulttype=resulttype, bbox=bbox, + datetime_=datetime_, properties=properties, + sortby=sortby, skip_geometry=skip_geometry, + select_properties=select_properties, + crs_transform_spec=crs_transform_spec, + q=q, language=prv_locale, filterq=filter_) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + serialized_query_params = '' + for k, v in request.params.items(): + if k not in ('f', 'offset'): + serialized_query_params += '&' + serialized_query_params += urllib.parse.quote(k, safe='') + serialized_query_params += '=' + serialized_query_params += urllib.parse.quote(str(v), safe=',') + + if 'links' not in content: + content['links'] = [] + + # TODO: translate titles + uri = f'{api.get_collections_url()}/{dataset}/items' + content['links'].extend([{ + 'type': 'application/geo+json', + 'rel': request.get_linkrel(F_JSON), + 'title': l10n.translate('This document as GeoJSON', request.locale), + 'href': f'{uri}?f={F_JSON}{serialized_query_params}' + }, { + 'rel': request.get_linkrel(F_JSONLD), + 'type': FORMAT_TYPES[F_JSONLD], + 'title': l10n.translate('This document as RDF (JSON-LD)', request.locale), # noqa + 'href': f'{uri}?f={F_JSONLD}{serialized_query_params}' + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': l10n.translate('This document as HTML', request.locale), + 'href': f'{uri}?f={F_HTML}{serialized_query_params}' + }]) + + if offset > 0: + prev = max(0, offset - limit) + content['links'].append( + { + 'type': 'application/geo+json', + 'rel': 'prev', + 'title': l10n.translate('Items (prev)', request.locale), + 'href': f'{uri}?offset={prev}{serialized_query_params}' + }) + + next_link = False + + if content.get('numberMatched', -1) > (limit + offset): + next_link = True + elif len(content['features']) == limit: + next_link = True + + if next_link: + next_ = offset + limit + next_href = f'{uri}?offset={next_}{serialized_query_params}' + content['links'].append( + { + 'type': 'application/geo+json', + 'rel': 'next', + 'title': l10n.translate('Items (next)', request.locale), + 'href': next_href + }) + + content['links'].append( + { + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate( + collections[dataset]['title'], request.locale), + 'rel': 'collection', + 'href': '/'.join(uri.split('/')[:-1]) + }) + + content['timeStamp'] = datetime.utcnow().strftime( + '%Y-%m-%dT%H:%M:%S.%fZ') + + # Set response language to requested provider locale + # (if it supports language) and/or otherwise the requested pygeoapi + # locale (or fallback default locale) + l10n.set_response_language(headers, prv_locale, request.locale) + + if request.format == F_HTML: # render + tpl_config = api.get_dataset_templates(dataset) + # For constructing proper URIs to items + + content['items_path'] = uri + content['dataset_path'] = '/'.join(uri.split('/')[:-1]) + content['collections_path'] = api.get_collections_url() + + content['offset'] = offset + + content['id_field'] = p.id_field + if p.uri_field is not None: + content['uri_field'] = p.uri_field + if p.title_field is not None: + content['title_field'] = l10n.translate(p.title_field, + request.locale) + # If title exists, use it as id in html templates + content['id_field'] = content['title_field'] + content = render_j2_template(api.tpl_config, tpl_config, + 'collections/items/index.html', + content, request.locale) + return headers, HTTPStatus.OK, content + elif request.format == 'csv': # render + formatter = load_plugin('formatter', + {'name': 'CSV', 'geom': True}) + + try: + content = formatter.write( + data=content, + options={ + 'provider_def': get_provider_by_type( + collections[dataset]['providers'], + 'feature') + } + ) + except FormatterSerializationError: + msg = 'Error serializing output' + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + headers['Content-Type'] = formatter.mimetype + + if p.filename is None: + filename = f'{dataset}.csv' + else: + filename = f'{p.filename}' + + cd = f'attachment; filename="{filename}"' + headers['Content-Disposition'] = cd + + return headers, HTTPStatus.OK, content + + elif request.format == F_JSONLD: + content = geojson2jsonld( + api, content, dataset, id_field=(p.uri_field or 'id') + ) + + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + +def manage_collection_item( + api: API, request: APIRequest, + action, dataset, identifier=None) -> Tuple[dict, int, str]: + """ + Adds an item to a collection + + :param request: A request object + :param action: an action among 'create', 'update', 'delete', 'options' + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Loading provider') + try: + provider_def = get_provider_by_type( + collections[dataset]['providers'], 'feature') + p = load_plugin('provider', provider_def) + except ProviderTypeError: + try: + provider_def = get_provider_by_type( + collections[dataset]['providers'], 'record') + p = load_plugin('provider', provider_def) + except ProviderTypeError: + msg = 'Invalid provider type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if action == 'options': + headers['Allow'] = 'HEAD, GET' + if p.editable: + if identifier is None: + headers['Allow'] += ', POST' + else: + headers['Allow'] += ', PUT, DELETE' + return headers, HTTPStatus.OK, '' + + if not p.editable: + msg = 'Collection is not editable' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if action in ['create', 'update'] and not request.data: + msg = 'No data found' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if action == 'create': + LOGGER.debug('Creating item') + try: + identifier = p.create(request.data) + except TypeError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + headers['Location'] = f'{api.get_collections_url()}/{dataset}/items/{identifier}' # noqa + + return headers, HTTPStatus.CREATED, '' + + if action == 'update': + LOGGER.debug('Updating item') + try: + _ = p.update(identifier, request.data) + except TypeError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + return headers, HTTPStatus.NO_CONTENT, '' + + if action == 'delete': + LOGGER.debug('Deleting item') + try: + _ = p.delete(identifier) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + return headers, HTTPStatus.OK, '' + + +def get_collection_item(api: API, request: APIRequest, + dataset, identifier) -> Tuple[dict, int, str]: + """ + Get a single collection item + + :param request: A request object + :param dataset: dataset name + :param identifier: item identifier + + :returns: tuple of headers, status code, content + """ + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + LOGGER.debug('Processing query parameters') + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Loading provider') + + try: + provider_type = 'feature' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + try: + provider_type = 'record' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + msg = 'Invalid provider type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + crs_transform_spec = None + if provider_type == 'feature': + # crs query parameter is only available for OGC API - Features + # right now, not for OGC API - Records. + LOGGER.debug('Processing crs parameter') + query_crs_uri = request.params.get('crs') + try: + crs_transform_spec = create_crs_transform_spec( + provider_def, query_crs_uri, + ) + except (ValueError, CRSError) as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + set_content_crs_header(headers, provider_def, query_crs_uri) + + # Get provider language (if any) + prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) + + try: + LOGGER.debug(f'Fetching id {identifier}') + content = p.get( + identifier, + language=prv_locale, + crs_transform_spec=crs_transform_spec, + ) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + if content is None: + msg = 'identifier not found' + return api.get_exception(HTTPStatus.BAD_REQUEST, headers, + request.format, 'NotFound', msg) + + uri = content['properties'].get(p.uri_field) if p.uri_field else \ + f'{api.get_collections_url()}/{dataset}/items/{identifier}' + + if 'links' not in content: + content['links'] = [] + + content['links'].extend([{ + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'root', + 'title': l10n.translate('The landing page of this server as JSON', request.locale), # noqa + 'href': f"{api.base_url}?f={F_JSON}" + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'root', + 'title': l10n.translate('The landing page of this server as HTML', request.locale), # noqa + 'href': f"{api.base_url}?f={F_HTML}" + }, { + 'rel': request.get_linkrel(F_JSON), + 'type': 'application/geo+json', + 'title': l10n.translate('This document as JSON', request.locale), + 'href': f'{uri}?f={F_JSON}' + }, { + 'rel': request.get_linkrel(F_JSONLD), + 'type': FORMAT_TYPES[F_JSONLD], + 'title': l10n.translate('This document as RDF (JSON-LD)', request.locale), # noqa + 'href': f'{uri}?f={F_JSONLD}' + }, { + 'rel': request.get_linkrel(F_HTML), + 'type': FORMAT_TYPES[F_HTML], + 'title': l10n.translate('This document as HTML', request.locale), + 'href': f'{uri}?f={F_HTML}' + }, { + 'rel': 'collection', + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate(collections[dataset]['title'], + request.locale), + 'href': f'{api.get_collections_url()}/{dataset}' + }]) + + link_request_format = ( + request.format if request.format is not None else F_JSON + ) + if 'prev' in content: + content['links'].append({ + 'rel': 'prev', + 'type': FORMAT_TYPES[link_request_format], + 'href': f"{api.get_collections_url()}/{dataset}/items/{content['prev']}?f={link_request_format}" # noqa + }) + if 'next' in content: + content['links'].append({ + 'rel': 'next', + 'type': FORMAT_TYPES[link_request_format], + 'href': f"{api.get_collections_url()}/{dataset}/items/{content['next']}?f={link_request_format}" # noqa + }) + + # Set response language to requested provider locale + # (if it supports language) and/or otherwise the requested pygeoapi + # locale (or fallback default locale) + l10n.set_response_language(headers, prv_locale, request.locale) + + # GA customisation - Add PID LinkAdd commentMore actions + if collections[dataset]['other']: + content['pid'] = f'{collections[dataset]["other"]["href"]}{identifier}' + # End GA customisation + + if request.format == F_HTML: # render + tpl_config = api.get_dataset_templates(dataset) + content['title'] = l10n.translate(collections[dataset]['title'], + request.locale) + content['id_field'] = p.id_field + if p.uri_field is not None: + content['uri_field'] = p.uri_field + if p.title_field is not None: + content['title_field'] = l10n.translate(p.title_field, + request.locale) + content['collections_path'] = api.get_collections_url() + + content = render_j2_template(api.tpl_config, tpl_config, + 'collections/items/item.html', + content, request.locale) + return headers, HTTPStatus.OK, content + + elif request.format == F_JSONLD: + content = geojson2jsonld( + api, content, dataset, uri, (p.uri_field or 'id') + ) + + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + +def create_crs_transform_spec( + config: dict, query_crs_uri: Optional[str] = None) -> Union[None, CrsTransformSpec]: # noqa + """ + Create a `CrsTransformSpec` instance based on provider config and + *crs* query parameter. + + :param config: Provider config dictionary. + :type config: dict + :param query_crs_uri: Uniform resource identifier of the coordinate + reference system (CRS) specified in query parameter (if specified). + :type query_crs_uri: str, optional + + :raises ValueError: Error raised if the CRS specified in the query + parameter is not in the list of supported CRSs of the provider. + :raises `CRSError`: Error raised if no CRS could be identified from the + query *crs* parameter (URI). + + :returns: `CrsTransformSpec` instance if the CRS specified in query + parameter differs from the storage CRS, else `None`. + :rtype: Union[None, CrsTransformSpec] + """ + + # Get storage/default CRS for Collection. + storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) + + if not query_crs_uri: + if storage_crs_uri in DEFAULT_CRS_LIST: + # Could be that storageCRS is + # http://www.opengis.net/def/crs/OGC/1.3/CRS84h + query_crs_uri = storage_crs_uri + else: + query_crs_uri = DEFAULT_CRS + LOGGER.debug(f'no crs parameter, using default: {query_crs_uri}') + + supported_crs_list = get_supported_crs_list(config, DEFAULT_CRS_LIST) + # Check that the crs specified by the query parameter is supported. + if query_crs_uri not in supported_crs_list: + raise ValueError( + f'CRS {query_crs_uri!r} not supported for this ' + 'collection. List of supported CRSs: ' + f'{", ".join(supported_crs_list)}.' + ) + crs_out = get_crs_from_uri(query_crs_uri) + + storage_crs = get_crs_from_uri(storage_crs_uri) + # Check if the crs specified in query parameter differs from the + # storage crs. + if str(storage_crs) != str(crs_out): + LOGGER.debug( + f'CRS transformation: {storage_crs} -> {crs_out}' + ) + return CrsTransformSpec( + source_crs_uri=storage_crs_uri, + source_crs_wkt=storage_crs.to_wkt(), + target_crs_uri=query_crs_uri, + target_crs_wkt=crs_out.to_wkt(), + ) + else: + LOGGER.debug('No CRS transformation') + return None + + +def set_content_crs_header( + headers: dict, config: dict, query_crs_uri: Optional[str] = None): + """ + Set the *Content-Crs* header in responses from providers of Feature type. + + :param headers: Response headers dictionary. + :type headers: dict + :param config: Provider config dictionary. + :type config: dict + :param query_crs_uri: Uniform resource identifier of the coordinate + reference system specified in query parameter (if specified). + :type query_crs_uri: str, optional + + :returns: None + """ + + if query_crs_uri: + content_crs_uri = query_crs_uri + else: + # If empty use default CRS + storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) + if storage_crs_uri in DEFAULT_CRS_LIST: + # Could be that storageCRS is one of the defaults like + # http://www.opengis.net/def/crs/OGC/1.3/CRS84h + content_crs_uri = storage_crs_uri + else: + content_crs_uri = DEFAULT_CRS + + headers['Content-Crs'] = f'<{content_crs_uri}>' + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + properties = { + 'name': 'properties', + 'in': 'query', + 'description': 'The properties that should be included for each feature. The parameter value is a comma-separated list of property names.', # noqa + 'required': False, + 'style': 'form', + 'explode': False, + 'schema': { + 'type': 'array', + 'items': { + 'type': 'string' + } + } + } + + LOGGER.debug('setting up collection endpoints') + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in get_visible_collections(cfg).items(): + try: + ptype = None + + if filter_providers_by_type( + collections[k]['providers'], 'feature'): + ptype = 'feature' + + if filter_providers_by_type( + collections[k]['providers'], 'record'): + ptype = 'record' + + p = load_plugin('provider', get_provider_by_type( + collections[k]['providers'], ptype)) + + collection_name_path = f'/collections/{k}' + items_path = f'/collections/{k}/items' + title = l10n.translate(v['title'], locale) + description = l10n.translate(v['description'], locale) + + coll_properties = deepcopy(properties) + + coll_properties['schema']['items']['enum'] = list(p.fields.keys()) + + paths[items_path] = { + 'get': { + 'summary': f'Get {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Features', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'}, + {'$ref': '#/components/parameters/bbox'}, + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/limit"}, # noqa + {'$ref': '#/components/parameters/crs'}, # noqa + {'$ref': '#/components/parameters/bbox-crs'}, + coll_properties, + {'$ref': '#/components/parameters/vendorSpecificParameters'}, # noqa + {'$ref': '#/components/parameters/skipGeometry'}, + {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/sortby.yaml"}, # noqa + {'$ref': '#/components/parameters/offset'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + }, + 'options': { + 'summary': f'Options for {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'options{k.capitalize()}Features', + 'responses': { + '200': {'description': 'options response'} + } + } + } + + if p.editable: + LOGGER.debug('Provider is editable; adding post') + + paths[items_path]['post'] = { + 'summary': f'Add {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'add{k.capitalize()}Features', + 'requestBody': { + 'description': 'Adds item to collection', + 'content': { + 'application/geo+json': { + 'schema': {} + } + }, + 'required': True + }, + 'responses': { + '201': {'description': 'Successful creation'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + + try: + schema_ref = p.get_schema(SchemaType.create) + paths[items_path]['post']['requestBody']['content'][schema_ref[0]] = { # noqa + 'schema': schema_ref[1] + } + except Exception as err: + LOGGER.debug(err) + + if ptype == 'record': + paths[items_path]['get']['parameters'].append( + {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/q.yaml"}) + if p.fields: + schema_path = f'{collection_name_path}/schema' + + paths[schema_path] = { + 'get': { + 'summary': f'Get {title} schema', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Schema', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/Queryables'}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa + } + } + } + + queryables_path = f'{collection_name_path}/queryables' + + paths[queryables_path] = { + 'get': { + 'summary': f'Get {title} queryables', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Queryables', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/Queryables'}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa + } + } + } + + if p.time_field is not None: + paths[items_path]['get']['parameters'].append( + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa + + for field, type_ in p.fields.items(): + + if p.properties and field not in p.properties: + LOGGER.debug('Provider specified not to advertise property') # noqa + continue + + if field == 'q' and ptype == 'record': + LOGGER.debug('q parameter already declared, skipping') + continue + + if type_ == 'date': + schema = { + 'type': 'string', + 'format': 'date' + } + elif type_ == 'float': + schema = { + 'type': 'number', + 'format': 'float' + } + elif type_ == 'long': + schema = { + 'type': 'integer', + 'format': 'int64' + } + else: + schema = type_ + + if schema.get('format') is None: + schema.pop('format', None) + + path_ = f'{collection_name_path}/items' + paths[path_]['get']['parameters'].append({ + 'name': field, + 'in': 'query', + 'required': False, + 'schema': schema, + 'style': 'form', + 'explode': False + }) + + paths[f'{collection_name_path}/items/{{featureId}}'] = { + 'get': { + 'summary': f'Get {title} item by id', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Feature', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa + {'$ref': '#/components/parameters/crs'}, # noqa + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Feature"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + }, + 'options': { + 'summary': f'Options for {title} item by id', + 'description': description, + 'tags': [k], + 'operationId': f'options{k.capitalize()}Feature', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa + ], + 'responses': { + '200': {'description': 'options response'} + } + } + } + + try: + schema_ref = p.get_schema() + paths[f'{collection_name_path}/items/{{featureId}}']['get']['responses']['200'] = { # noqa + 'content': { + schema_ref[0]: { + 'schema': schema_ref[1] + } + } + } + except Exception as err: + LOGGER.debug(err) + + if p.editable: + LOGGER.debug('Provider is editable; adding put/delete') + put_path = f'{collection_name_path}/items/{{featureId}}' # noqa + paths[put_path]['put'] = { # noqa + 'summary': f'Update {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'update{k.capitalize()}Features', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa + ], + 'requestBody': { + 'description': 'Updates item in collection', + 'content': { + 'application/geo+json': { + 'schema': {} + } + }, + 'required': True + }, + 'responses': { + '204': {'$ref': '#/components/responses/204'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + + try: + schema_ref = p.get_schema(SchemaType.replace) + paths[put_path]['put']['requestBody']['content'][schema_ref[0]] = { # noqa + 'schema': schema_ref[1] + } + except Exception as err: + LOGGER.debug(err) + + paths[f'{collection_name_path}/items/{{featureId}}']['delete'] = { # noqa + 'summary': f'Delete {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'delete{k.capitalize()}Features', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa + ], + 'responses': { + '200': {'description': 'Successful delete'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + + except ProviderTypeError: + LOGGER.debug('collection is not feature/item based') + + return [{'name': 'records'}, {'name': 'features'}], {'paths': paths} diff --git a/pygeoapi/api/maps.py b/pygeoapi/api/maps.py new file mode 100644 index 000000000..036892fe2 --- /dev/null +++ b/pygeoapi/api/maps.py @@ -0,0 +1,336 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from copy import deepcopy +from http import HTTPStatus +import logging +from typing import Tuple + +from pygeoapi.openapi import get_oas_30_parameters +from pygeoapi.plugin import load_plugin +from pygeoapi.provider.base import ProviderGenericError +from pygeoapi.util import ( + get_provider_by_type, to_json, filter_providers_by_type, + filter_dict_by_key_value +) + +from . import APIRequest, API, validate_datetime + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-maps-1/1.0/conf/core' +] + + +def get_collection_map(api: API, request: APIRequest, + dataset, style=None) -> Tuple[dict, int, str]: + """ + Returns a subset of a collection map + + :param request: A request object + :param dataset: dataset name + :param style: style name + + :returns: tuple of headers, status code, content + """ + + query_args = { + 'crs': 'CRS84' + } + + format_ = request.format or 'png' + headers = request.get_response_headers(**api.api_headers) + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Loading provider') + try: + collection_def = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'map') + + p = load_plugin('provider', collection_def) + except KeyError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'collection does not exist' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.NOT_FOUND, to_json( + exception, api.pretty_print) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + query_args['format_'] = request.params.get('f', 'png') + query_args['style'] = style + query_args['crs'] = request.params.get('bbox-crs', 4326) + query_args['transparent'] = request.params.get('transparent', True) + + try: + query_args['width'] = int(request.params.get('width', 500)) + query_args['height'] = int(request.params.get('height', 300)) + except ValueError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'invalid width/height' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + exception, api.pretty_print) + + LOGGER.debug('Processing bbox parameter') + try: + bbox = request.params.get('bbox').split(',') + if len(bbox) != 4: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'bbox values should be minx,miny,maxx,maxy' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + exception, api.pretty_print) + except AttributeError: + bbox = api.config['resources'][dataset]['extents']['spatial']['bbox'] # noqa + try: + query_args['bbox'] = [float(c) for c in bbox] + except ValueError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'bbox values must be numbers' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + exception, api.pretty_print) + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + query_args['datetime_'] = validate_datetime( + api.config['resources'][dataset]['extents'], datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Generating map') + try: + data = p.query(**query_args) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + mt = collection_def['format']['name'] + + if format_ == mt: + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + elif format_ in [None, 'html']: + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + else: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'invalid format parameter' + } + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + data, api.pretty_print) + + +def get_collection_map_legend(api: API, request: APIRequest, + dataset, style=None) -> Tuple[dict, int, str]: + """ + Returns a subset of a collection map legend + + :param request: A request object + :param dataset: dataset name + :param style: style name + + :returns: tuple of headers, status code, content + """ + + format_ = 'png' + headers = request.get_response_headers(**api.api_headers) + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Loading provider') + try: + collection_def = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'map') + + p = load_plugin('provider', collection_def) + except KeyError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'collection does not exist' + } + LOGGER.error(exception) + return headers, HTTPStatus.NOT_FOUND, to_json( + exception, api.pretty_print) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + LOGGER.debug('Generating legend') + try: + data = p.get_legend(style, request.params.get('f', 'png')) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + mt = collection_def['format']['name'] + + if format_ == mt: + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + else: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'invalid format parameter' + } + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + data, api.pretty_print) + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + LOGGER.debug('setting up maps endpoints') + + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + parameters = get_oas_30_parameters(cfg, locale) + for k, v in get_visible_collections(cfg).items(): + map_extension = filter_providers_by_type( + collections[k]['providers'], 'map') + + if map_extension: + mp = load_plugin('provider', map_extension) + + map_f = deepcopy(parameters['f']) + map_f['schema']['enum'] = [map_extension['format']['name']] + map_f['schema']['default'] = map_extension['format']['name'] + + pth = f'/collections/{k}/map' + paths[pth] = { + 'get': { + 'summary': 'Get map', + 'description': f"{v['description']} map", + 'tags': [k], + 'operationId': 'getMap', + 'parameters': [ + {'$ref': '#/components/parameters/bbox'}, + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + { + 'name': 'width', + 'in': 'query', + 'description': 'Response image width', + 'required': False, + 'schema': { + 'type': 'integer', + }, + 'style': 'form', + 'explode': False + }, + { + 'name': 'height', + 'in': 'query', + 'description': 'Response image height', + 'required': False, + 'schema': { + 'type': 'integer', + }, + 'style': 'form', + 'explode': False + }, + { + 'name': 'transparent', + 'in': 'query', + 'description': 'Background transparency of map (default=true).', # noqa + 'required': False, + 'schema': { + 'type': 'boolean', + 'default': True, + }, + 'style': 'form', + 'explode': False + }, + {'$ref': '#/components/parameters/bbox-crs-epsg'}, + map_f + ], + 'responses': { + '200': { + 'description': 'Response', + 'content': { + 'application/json': {} + } + }, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa + } + } + } + if mp.time_field is not None: + paths[pth]['get']['parameters'].append( + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa + + return [{'name': 'maps'}], {'paths': paths} diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py new file mode 100644 index 000000000..dcf99d541 --- /dev/null +++ b/pygeoapi/api/processes.py @@ -0,0 +1,844 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# Francesco Martinelli +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# Copyright (c) 2024 Francesco Martinelli +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from copy import deepcopy +from datetime import datetime, timezone +from http import HTTPStatus +import json +import logging +from typing import Tuple +import urllib.parse + +from pygeoapi import l10n +from pygeoapi.api import evaluate_limit +from pygeoapi.util import ( + json_serial, render_j2_template, JobStatus, RequestedProcessExecutionMode, + to_json, DATETIME_FORMAT) +from pygeoapi.process.base import ( + JobNotFoundError, JobResultNotFoundError, ProcessorExecuteError +) +from pygeoapi.process.manager.base import get_manager, Subscriber + +from . import ( + APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD, +) + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/ogc-process-description', # noqa + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/json', + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/oas30', + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/callback' +] + + +def describe_processes(api: API, request: APIRequest, + process=None) -> Tuple[dict, int, str]: + """ + Provide processes metadata + + :param request: A request object + :param process: process identifier, defaults to None to obtain + information about all processes + + :returns: tuple of headers, status code, content + """ + + processes = [] + + headers = request.get_response_headers(**api.api_headers) + + if process is not None: + if process not in api.manager.processes.keys(): + msg = 'Identifier not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'NoSuchProcess', msg) + + if len(api.manager.processes) > 0: + if process is not None: + relevant_processes = [process] + else: + LOGGER.debug('Processing limit parameter') + if api.config['server'].get('limit') is not None: + msg = ('server.limit is no longer supported! ' + 'Please use limits at the server or collection ' + 'level (RFC5)') + LOGGER.warning(msg) + try: + limit = evaluate_limit(request.params.get('limit'), + api.config['server'].get('limits', {}), + {}) + relevant_processes = list(api.manager.processes)[:limit] + except ValueError as err: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', str(err)) + + for key in relevant_processes: + p = api.manager.get_processor(key) + p2 = l10n.translate_struct(deepcopy(p.metadata), + request.locale) + p2['id'] = key + + if process is None: + p2.pop('inputs') + p2.pop('outputs') + p2.pop('example', None) + + p2['jobControlOptions'] = ['sync-execute'] + if api.manager.is_async: + p2['jobControlOptions'].append('async-execute') + + p2['outputTransmission'] = ['value'] + p2['links'] = p2.get('links', []) + + jobs_url = f"{api.base_url}/jobs" + process_url = f"{api.base_url}/processes/{key}" + + # TODO translation support + link = { + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'href': f'{process_url}?f={F_JSON}', + 'title': l10n.translate('Process description as JSON', request.locale), # noqa + 'hreflang': api.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'href': f'{process_url}?f={F_HTML}', + 'title': l10n.translate('Process description as HTML', request.locale), # noqa + 'hreflang': api.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', + 'href': f'{jobs_url}?f={F_HTML}', + 'title': l10n.translate('Jobs list as HTML', request.locale), # noqa + 'hreflang': api.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', + 'href': f'{jobs_url}?f={F_JSON}', + 'title': l10n.translate('Jobs list as JSON', request.locale), # noqa + 'hreflang': api.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/execute', + 'href': f'{process_url}/execution?f={F_JSON}', + 'title': l10n.translate('Execution for this process as JSON', request.locale), # noqa + 'hreflang': api.default_locale + } + p2['links'].append(link) + + processes.append(p2) + + if process is not None: + response = processes[0] + else: + process_url = f"{api.base_url}/processes" + response = { + 'processes': processes, + 'links': [{ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': l10n.translate('This document as JSON', request.locale), # noqa + 'href': f'{process_url}?f={F_JSON}' + }, { + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': l10n.translate('This document as RDF (JSON-LD)', request.locale), # noqa + 'href': f'{process_url}?f={F_JSONLD}' + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': l10n.translate('This document as HTML', request.locale), # noqa + 'href': f'{process_url}?f={F_HTML}' + }] + } + + if request.format == F_HTML: # render + if process is not None: + tpl_config = api.get_dataset_templates(process) + response = render_j2_template(api.tpl_config, tpl_config, + 'processes/process.html', + response, request.locale) + else: + response = render_j2_template( + api.tpl_config, api.config['server']['templates'], + 'processes/index.html', response, request.locale) + + return headers, HTTPStatus.OK, response + + return headers, HTTPStatus.OK, to_json(response, api.pretty_print) + + +# TODO: get_jobs doesn't have tests +def get_jobs(api: API, request: APIRequest, + job_id=None) -> Tuple[dict, int, str]: + """ + Get process jobs + + :param request: A request object + :param job_id: id of job + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + LOGGER.debug('Processing limit parameter') + try: + limit = evaluate_limit(request.params.get('limit'), + api.config['server'].get('limits', {}), + {}) + except ValueError as err: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', str(err)) + + LOGGER.debug('Processing offset parameter') + try: + offset = int(request.params.get('offset')) + if offset < 0: + msg = 'offset value should be positive or zero' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + offset = 0 + except ValueError: + msg = 'offset value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if job_id is None: + jobs_data = api.manager.get_jobs(limit=limit, offset=offset) + # TODO: For pagination to work, the provider has to do the sorting. + # Here we do sort again in case the provider doesn't support + # pagination yet and always returns all jobs. + jobs = sorted(jobs_data['jobs'], + key=lambda k: k['started'], + reverse=True) + numberMatched = jobs_data['numberMatched'] + + else: + try: + jobs = [api.manager.get_job(job_id)] + except JobNotFoundError: + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, + 'InvalidParameterValue', job_id) + numberMatched = 1 + + serialized_jobs = { + 'jobs': [], + 'links': [{ + 'href': f"{api.base_url}/jobs?f={F_HTML}", + 'rel': request.get_linkrel(F_HTML), + 'type': FORMAT_TYPES[F_HTML], + 'title': l10n.translate('Jobs list as HTML', request.locale) + }, { + 'href': f"{api.base_url}/jobs?f={F_JSON}", + 'rel': request.get_linkrel(F_JSON), + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate('Jobs list as JSON', request.locale) + }] + } + for job_ in jobs: + job2 = { + 'type': 'process', + 'processID': job_['process_id'], + 'jobID': job_['identifier'], + 'status': job_['status'], + 'message': job_['message'], + 'progress': job_['progress'], + 'parameters': job_.get('parameters'), + 'created': job_['created'], + 'started': job_['started'], + 'finished': job_['finished'], + 'updated': job_['updated'] + } + + # TODO: translate + if JobStatus[job_['status']] in ( + JobStatus.successful, JobStatus.running, JobStatus.accepted): + + job_result_url = f"{api.base_url}/jobs/{job_['identifier']}/results" # noqa + + job2['links'] = [{ + 'href': f'{job_result_url}?f={F_HTML}', + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', + 'type': FORMAT_TYPES[F_HTML], + 'title': l10n.translate(f'Results of job as HTML', request.locale), # noqa + }, { + 'href': f'{job_result_url}?f={F_JSON}', + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate(f'Results of job as JSON', request.locale), # noqa + }] + + if job_['mimetype'] not in (FORMAT_TYPES[F_JSON], + FORMAT_TYPES[F_HTML]): + + job2['links'].append({ + 'href': job_result_url, + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', # noqa + 'type': job_['mimetype'], + 'title': f"Results of job {job_id} as {job_['mimetype']}" # noqa + }) + + serialized_jobs['jobs'].append(job2) + + serialized_query_params = '' + for k, v in request.params.items(): + if k not in ('f', 'offset'): + serialized_query_params += '&' + serialized_query_params += urllib.parse.quote(k, safe='') + serialized_query_params += '=' + serialized_query_params += urllib.parse.quote(str(v), safe=',') + + uri = f'{api.base_url}/jobs' + + if offset > 0: + prev = max(0, offset - limit) + serialized_jobs['links'].append( + { + 'href': f'{uri}?offset={prev}{serialized_query_params}', + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'prev', + 'title': l10n.translate('Items (prev)', request.locale), + }) + + next_link = False + + if numberMatched > (limit + offset): + next_link = True + elif len(jobs) == limit: + next_link = True + + if next_link: + next_ = offset + limit + next_href = f'{uri}?offset={next_}{serialized_query_params}' + serialized_jobs['links'].append( + { + 'href': next_href, + 'rel': 'next', + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate('Items (next)', request.locale), + }) + + if job_id is None: + j2_template = 'jobs/index.html' + else: + serialized_jobs = serialized_jobs['jobs'][0] + j2_template = 'jobs/job.html' + + if request.format == F_HTML: + data = { + 'jobs': serialized_jobs, + 'offset': offset, + 'now': datetime.now(timezone.utc).strftime(DATETIME_FORMAT) + } + response = render_j2_template( + api.tpl_config, api.config['server']['templates'], j2_template, + data, request.locale) + + return headers, HTTPStatus.OK, response + + return headers, HTTPStatus.OK, to_json(serialized_jobs, + api.pretty_print) + + +def execute_process(api: API, request: APIRequest, + process_id) -> Tuple[dict, int, str]: + """ + Execute process + + :param request: A request object + :param process_id: id of process + + :returns: tuple of headers, status code, content + """ + + # Responses are always in US English only + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + if process_id not in api.manager.processes: + msg = 'identifier not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'NoSuchProcess', msg) + + data = request.data + if not data: + # TODO not all processes require input, e.g. time-dependent or + # random value generators + msg = 'missing request data' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'MissingParameterValue', msg) + + try: + # Parse bytes data, if applicable + data = data.decode() + LOGGER.debug(data) + except (UnicodeDecodeError, AttributeError): + pass + + try: + data = json.loads(data) + except (json.decoder.JSONDecodeError, TypeError): + # Input does not appear to be valid JSON + msg = 'invalid request data' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + data_dict = data.get('inputs', {}) + LOGGER.debug(data_dict) + + requested_outputs = data.get('outputs') + LOGGER.debug(f'outputs: {requested_outputs}') + + requested_response = data.get('response', 'raw') + + subscriber = None + subscriber_dict = data.get('subscriber') + if subscriber_dict: + try: + success_uri = subscriber_dict['successUri'] + except KeyError: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'MissingParameterValue', 'Missing successUri') + else: + subscriber = Subscriber( + # NOTE: successUri is mandatory according to the standard + success_uri=success_uri, + in_progress_uri=subscriber_dict.get('inProgressUri'), + failed_uri=subscriber_dict.get('failedUri'), + ) + + try: + execution_mode = RequestedProcessExecutionMode( + request.headers.get('Prefer', request.headers.get('prefer')) + ) + except ValueError: + execution_mode = None + try: + LOGGER.debug('Executing process') + result = api.manager.execute_process( + process_id, data_dict, execution_mode=execution_mode, + requested_outputs=requested_outputs, + subscriber=subscriber, + requested_response=requested_response) + job_id, mime_type, outputs, status, additional_headers = result + headers.update(additional_headers or {}) + + if api.manager.is_async: + headers['Location'] = f'{api.base_url}/jobs/{job_id}' + + except ProcessorExecuteError as err: + return api.get_exception( + err.http_status_code, headers, + request.format, err.ogc_exception_code, err.message) + + response = {} + if status == JobStatus.failed: + response = outputs + + if requested_response == 'raw': + headers['Content-Type'] = mime_type + response = outputs + elif status not in (JobStatus.failed, JobStatus.accepted): + response = outputs + + if status == JobStatus.accepted: + http_status = HTTPStatus.CREATED + elif status == JobStatus.failed: + http_status = HTTPStatus.BAD_REQUEST + else: + http_status = HTTPStatus.OK + + if mime_type == 'application/json' or requested_response == 'document': + response2 = to_json(response, api.pretty_print) + else: + response2 = response + + return headers, http_status, response2 + + +def get_job_result(api: API, request: APIRequest, + job_id) -> Tuple[dict, int, str]: + """ + Get result of job (instance of a process) + + :param request: A request object + :param job_id: ID of job + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + try: + job = api.manager.get_job(job_id) + except JobNotFoundError: + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'NoSuchJob', job_id + ) + + status = JobStatus[job['status']] + + if status == JobStatus.running: + msg = 'job still running' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'ResultNotReady', msg) + + elif status == JobStatus.accepted: + # NOTE: this case is not mentioned in the specification + msg = 'job accepted but not yet running' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'ResultNotReady', msg) + + elif status == JobStatus.failed: + msg = 'job failed' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + try: + mimetype, job_output = api.manager.get_job_result(job_id) + except JobResultNotFoundError: + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'JobResultNotFound', job_id + ) + + if mimetype not in (None, FORMAT_TYPES[F_JSON]): + headers['Content-Type'] = mimetype + content = job_output + else: + if request.format == F_JSON: + content = json.dumps(job_output, sort_keys=True, indent=4, + default=json_serial) + else: + # HTML + headers['Content-Type'] = "text/html" + data = { + 'job': {'id': job_id}, + 'result': job_output + } + content = render_j2_template( + api.config, api.config['server']['templates'], + 'jobs/results/index.html', data, request.locale) + + return headers, HTTPStatus.OK, content + + +def delete_job(api: API, request: APIRequest, job_id) -> Tuple[dict, int, str]: + """ + Delete a process job + + :param job_id: job identifier + + :returns: tuple of headers, status code, content + """ + + response_headers = request.get_response_headers( + SYSTEM_LOCALE, **api.api_headers) + try: + success = api.manager.delete_job(job_id) + except JobNotFoundError: + return api.get_exception( + HTTPStatus.NOT_FOUND, response_headers, request.format, + 'NoSuchJob', job_id + ) + else: + if success: + http_status = HTTPStatus.OK + jobs_url = f"{api.base_url}/jobs" + + response = { + 'jobID': job_id, + 'status': JobStatus.dismissed.value, + 'message': 'Job dismissed', + 'progress': 100, + 'links': [{ + 'href': jobs_url, + 'rel': 'up', + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate('The job list for the current process', request.locale) # noqa + }] + } + else: + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, response_headers, + request.format, 'InternalError', job_id + ) + LOGGER.info(response) + # TODO: this response does not have any headers + return {}, http_status, to_json(response, api.pretty_print) + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML + + LOGGER.debug('setting up processes endpoints') + + oas = {'tags': []} + + paths = {} + + process_manager = get_manager(cfg) + + if len(process_manager.processes) > 0: + paths['/processes'] = { + 'get': { + 'summary': 'Processes', + 'description': 'Processes', + 'tags': ['server'], + 'operationId': 'getProcesses', + 'parameters': [ + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ProcessList.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + LOGGER.debug('setting up processes') + + for k, v in process_manager.processes.items(): + if k.startswith('_'): + LOGGER.debug(f'Skipping hidden layer: {k}') + continue + name = l10n.translate(k, locale) + p = process_manager.get_processor(k) + md_desc = l10n.translate(p.metadata['description'], locale) + process_name_path = f'/processes/{name}' + tag = { + 'name': name, + 'description': md_desc, + 'externalDocs': {} + } + for link in p.metadata.get('links', []): + if link['type'] == 'information': + translated_link = l10n.translate(link, locale) + tag['externalDocs']['description'] = translated_link[ + 'type'] + tag['externalDocs']['url'] = translated_link['url'] + break + if len(tag['externalDocs']) == 0: + del tag['externalDocs'] + + oas['tags'].append(tag) + + paths[process_name_path] = { + 'get': { + 'summary': 'Get process metadata', + 'description': md_desc, + 'tags': [name], + 'operationId': f'describe{name.capitalize()}Process', + 'parameters': [ + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + paths[f'{process_name_path}/execution'] = { + 'post': { + 'summary': f"Process {l10n.translate(p.metadata['title'], locale)} execution", # noqa + 'description': md_desc, + 'tags': [name], + 'operationId': f'execute{name.capitalize()}Job', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '201': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ExecuteAsync.yaml"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ServerError.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + }, + 'requestBody': { + 'description': 'Mandatory execute request JSON', + 'required': True, + 'content': { + 'application/json': { + 'schema': { + '$ref': f"{OPENAPI_YAML['oapip']}/schemas/execute.yaml" # noqa + } + } + } + } + } + } + + try: + first_key = list(p.metadata['outputs'])[0] + p_output = p.metadata['outputs'][first_key] + + if p_output.get('schema') is not None: + LOGGER.debug('Adding output schema') + content_media_type = p_output['schema'].pop('contentMediaType', 'application/json') # noqa + paths[f'{process_name_path}/execution']['post']['responses']['200'] = { # noqa + 'description': 'Process output schema', + 'content': { + content_media_type: { + 'schema': p_output['schema'] + } + } + } + except (IndexError, KeyError): + LOGGER.debug('No output defined') + + if 'example' in p.metadata: + paths[f'{process_name_path}/execution']['post']['requestBody']['content']['application/json']['example'] = p.metadata['example'] # noqa + + name_in_path = { + 'name': 'jobId', + 'in': 'path', + 'description': 'job identifier', + 'required': True, + 'schema': { + 'type': 'string' + } + } + + paths['/jobs'] = { + 'get': { + 'summary': 'Retrieve jobs list', + 'description': 'Retrieve a list of jobs', + 'tags': ['jobs'], + 'operationId': 'getJobs', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + paths['/jobs/{jobId}'] = { + 'get': { + 'summary': 'Retrieve job details', + 'description': 'Retrieve job details', + 'tags': ['jobs'], + 'parameters': [ + name_in_path, + {'$ref': '#/components/parameters/f'} + ], + 'operationId': 'getJob', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + }, + 'delete': { + 'summary': 'Cancel / delete job', + 'description': 'Cancel / delete job', + 'tags': ['jobs'], + 'parameters': [ + name_in_path + ], + 'operationId': 'deleteJob', + 'responses': { + '204': {'$ref': '#/components/responses/204'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + }, + } + + paths['/jobs/{jobId}/results'] = { + 'get': { + 'summary': 'Retrieve job results', + 'description': 'Retrieve job results', + 'tags': ['jobs'], + 'parameters': [ + name_in_path, + {'$ref': '#/components/parameters/f'} + ], + 'operationId': 'getJobResults', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + return [{'name': 'processes'}, {'name': 'jobs'}], {'paths': paths} diff --git a/pygeoapi/api/stac.py b/pygeoapi/api/stac.py new file mode 100644 index 000000000..716862fc8 --- /dev/null +++ b/pygeoapi/api/stac.py @@ -0,0 +1,248 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from http import HTTPStatus +import logging +from typing import Tuple + +from pygeoapi import l10n +from pygeoapi.plugin import load_plugin + +from pygeoapi.provider.base import ( + ProviderConnectionError, ProviderNotFoundError +) +from pygeoapi.util import ( + get_provider_by_type, to_json, filter_dict_by_key_value, + render_j2_template +) + +from . import APIRequest, API, FORMAT_TYPES, F_JSON, F_HTML + + +LOGGER = logging.getLogger(__name__) + + +CONFORMANCE_CLASSES = [] + + +# TODO: no tests for this? +def get_stac_root(api: API, request: APIRequest) -> Tuple[dict, int, str]: + """ + Provide STAC root page + + :param request: APIRequest instance with query params + + :returns: tuple of headers, status code, content + """ + headers = request.get_response_headers(**api.api_headers) + + id_ = 'pygeoapi-stac' + stac_version = '1.0.0-rc.2' + stac_url = f'{api.base_url}/stac' + + content = { + 'id': id_, + 'type': 'Catalog', + 'stac_version': stac_version, + 'title': l10n.translate( + api.config['metadata']['identification']['title'], + request.locale), + 'description': l10n.translate( + api.config['metadata']['identification']['description'], + request.locale), + 'links': [] + } + + stac_collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'stac-collection') + + for key, value in stac_collections.items(): + content['links'].append({ + 'rel': 'child', + 'href': f'{stac_url}/{key}?f={F_JSON}', + 'type': FORMAT_TYPES[F_JSON] + }) + content['links'].append({ + 'rel': 'child', + 'href': f'{stac_url}/{key}', + 'type': FORMAT_TYPES[F_HTML] + }) + + if request.format == F_HTML: # render + content = render_j2_template( + api.tpl_config, api.config['server']['templates'], + 'stac/collection.html', content, request.locale) + + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + +# TODO: no tests for this? +def get_stac_path(api: API, request: APIRequest, + path) -> Tuple[dict, int, str]: + """ + Provide STAC resource path + + :param request: APIRequest instance with query params + + :returns: tuple of headers, status code, content + """ + headers = request.get_response_headers(**api.api_headers) + + dataset = None + LOGGER.debug(f'Path: {path}') + dir_tokens = path.split('/') + if dir_tokens: + dataset = dir_tokens[0] + + stac_collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'stac-collection') + + if dataset not in stac_collections: + msg = 'Collection not found' + return api.get_exception(HTTPStatus.NOT_FOUND, headers, + request.format, 'NotFound', msg) + + LOGGER.debug('Loading provider') + try: + p = load_plugin('provider', get_provider_by_type( + stac_collections[dataset]['providers'], 'stac')) + except ProviderConnectionError: + msg = 'connection error (check logs)' + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'NoApplicableCode', msg) + + id_ = f'{dataset}-stac' + stac_version = '1.0.0-rc.2' + + content = { + 'id': id_, + 'type': 'Catalog', + 'stac_version': stac_version, + 'description': l10n.translate( + stac_collections[dataset]['description'], request.locale), + 'links': [] + } + try: + stac_data = p.get_data_path( + f'{api.base_url}/stac', + path, + path.replace(dataset, '', 1) + ) + except ProviderNotFoundError: + msg = 'resource not found' + return api.get_exception(HTTPStatus.NOT_FOUND, headers, + request.format, 'NotFound', msg) + except Exception: + msg = 'data query error' + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'NoApplicableCode', msg) + + if isinstance(stac_data, dict): + content.update(stac_data) + content['links'].extend( + stac_collections[dataset].get('links', [])) + + if request.format == F_HTML: # render + content['path'] = path + if 'assets' in content: # item view + if content['type'] == 'Collection': + content = render_j2_template( + api.tpl_config, api.config['server']['templates'], + 'stac/collection_base.html', + content, request.locale) + elif content['type'] == 'Feature': + content = render_j2_template( + api.tpl_config, api.config['server']['templattes'], + 'stac/item.html', content, request.locale) + else: + msg = f'Unknown STAC type {content.type}' + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, + headers, + request.format, + 'NoApplicableCode', + msg) + else: + content = render_j2_template( + api.tpl_config, api.config['server']['templates'], + 'stac/catalog.html', content, request.locale) + + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + else: # send back file + headers.pop('Content-Type', None) + return headers, HTTPStatus.OK, stac_data + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + LOGGER.debug('setting up STAC') + stac_collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'stac-collection') + paths = {} + if stac_collections: + paths['/stac'] = { + 'get': { + 'summary': 'SpatioTemporal Asset Catalog', + 'description': 'SpatioTemporal Asset Catalog', + 'tags': ['stac'], + 'operationId': 'getStacCatalog', + 'parameters': [], + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + 'default': {'$ref': '#/components/responses/default'} + } + } + } + return [{'name': 'stac'}], {'paths': paths} diff --git a/pygeoapi/api/tiles.py b/pygeoapi/api/tiles.py new file mode 100644 index 000000000..3a02db3d4 --- /dev/null +++ b/pygeoapi/api/tiles.py @@ -0,0 +1,546 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import logging +from http import HTTPStatus +from typing import Tuple + +from pygeoapi import l10n +from pygeoapi.plugin import load_plugin +from pygeoapi.models.provider.base import (TilesMetadataFormat, + TileMatrixSetEnum) +from pygeoapi.provider.base import ( + ProviderGenericError, ProviderTypeError +) +from pygeoapi.provider.tile import ProviderTileNotFoundError + +from pygeoapi.util import ( + get_provider_by_type, to_json, filter_dict_by_key_value, + filter_providers_by_type, render_j2_template +) + +from . import ( + APIRequest, API, FORMAT_TYPES, F_JSON, F_HTML, SYSTEM_LOCALE, F_JSONLD +) + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/mvt', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tileset', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tilesets-list', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/oas30', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/geodata-tilesets' +] + + +def get_collection_tiles(api: API, request: APIRequest, + dataset=None) -> Tuple[dict, int, str]: + """ + Provide collection tiles + + :param request: A request object + :param dataset: name of collection + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + if any([dataset is None, + dataset not in api.config['resources'].keys()]): + + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection tiles') + LOGGER.debug('Loading provider') + try: + t = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'tile') + p = load_plugin('provider', t) + except (KeyError, ProviderTypeError): + msg = 'Invalid collection tiles' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + tiles = { + 'links': [], + 'tilesets': [] + } + + tiles['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': l10n.translate('This document as JSON', request.locale), + 'href': f'{api.get_collections_url()}/{dataset}/tiles?f={F_JSON}' + }) + tiles['links'].append({ + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': l10n.translate('This document as RDF (JSON-LD)', request.locale), # noqa + 'href': f'{api.get_collections_url()}/{dataset}/tiles?f={F_JSONLD}' + }) + tiles['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': l10n.translate('This document as HTML', request.locale), + 'href': f'{api.get_collections_url()}/{dataset}/tiles?f={F_HTML}' + }) + + tile_services = p.get_tiles_service( + baseurl=api.base_url, + servicepath=f'{api.get_collections_url()}/{dataset}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f={p.format_type}' # noqa + ) + + for service in tile_services['links']: + tiles['links'].append(service) + + tiling_schemes = p.get_tiling_schemes() + + datatype = 'vector' + + if t['format']['mimetype'].startswith('image'): + datatype = 'map' + + for matrix in tiling_schemes: + tile_matrix = { + 'title': dataset, + 'tileMatrixSetURI': matrix.tileMatrixSetURI, + 'crs': matrix.crs, + 'dataType': datatype, + 'links': [] + } + tile_matrix['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme', + 'title': l10n.translate('TileMatrixSet definition in JSON', request.locale), # noqa + 'href': f'{api.base_url}/TileMatrixSets/{matrix.tileMatrixSet}?f={F_JSON}' # noqa + }) + tile_matrix['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_JSON}', + 'href': f'{api.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_JSON}' # noqa + }) + tile_matrix['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_HTML}', + 'href': f'{api.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_HTML}' # noqa + }) + + tiles['tilesets'].append(tile_matrix) + + if request.format == F_HTML: # render + tpl_config = api.get_dataset_templates(dataset) + tiles['id'] = dataset + tiles['title'] = l10n.translate( + api.config['resources'][dataset]['title'], SYSTEM_LOCALE) + tiles['tilesets'] = [ + scheme.tileMatrixSet for scheme in p.get_tiling_schemes()] + tiles['bounds'] = \ + api.config['resources'][dataset]['extents']['spatial']['bbox'] + tiles['minzoom'] = p.options['zoom']['min'] + tiles['maxzoom'] = p.options['zoom']['max'] + tiles['collections_path'] = api.get_collections_url() + tiles['tile_type'] = p.tile_type + + content = render_j2_template(api.tpl_config, tpl_config, + 'collections/tiles/index.html', tiles, + request.locale) + + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(tiles, api.pretty_print) + + +def get_collection_tiles_data( + api: API, request: APIRequest, + dataset=None, matrix_id=None, + z_idx=None, y_idx=None, x_idx=None) -> Tuple[dict, int, str]: + """ + Get collection items tiles + + :param request: A request object + :param dataset: dataset name + :param matrix_id: matrix identifier + :param z_idx: z index + :param y_idx: y index + :param x_idx: x index + + :returns: tuple of headers, status code, content + """ + + format_ = request.format + if not format_: + return api.get_format_exception(request) + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + LOGGER.debug('Processing tiles') + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Loading tile provider') + try: + t = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'tile') + p = load_plugin('provider', t) + + format_ = p.format_type + headers['Content-Type'] = format_ + + LOGGER.debug(f'Fetching tileset id {matrix_id} and tile {z_idx}/{y_idx}/{x_idx}') # noqa + content = p.get_tiles(layer=p.get_layer(), tileset=matrix_id, + z=z_idx, y=y_idx, x=x_idx, format_=format_) + if content is None: + msg = 'identifier not found' + return api.get_exception( + HTTPStatus.NO_CONTENT, headers, format_, 'NocContent', msg) + else: + return headers, HTTPStatus.OK, content + + # @TODO: figure out if the spec requires to return json errors + except KeyError: + msg = 'Invalid collection tiles' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + except ProviderTileNotFoundError: + msg = 'Tile not found' + LOGGER.info(msg) + return headers, HTTPStatus.NOT_FOUND, msg + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + +def get_collection_tiles_metadata( + api: API, request: APIRequest, + dataset=None, matrix_id=None) -> Tuple[dict, int, str]: + """ + Get collection items tiles + + :param request: A request object + :param dataset: dataset name + :param matrix_id: matrix identifier + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid([TilesMetadataFormat.TILEJSON]): + return api.get_format_exception(request) + headers = request.get_response_headers(**api.api_headers) + + if any([dataset is None, + dataset not in api.config['resources'].keys()]): + + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection tiles') + LOGGER.debug('Loading provider') + try: + t = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'tile') + p = load_plugin('provider', t) + except KeyError: + msg = 'Invalid collection tiles' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + # Get provider language (if any) + prv_locale = l10n.get_plugin_locale(t, request.raw_locale) + + # Set response language to requested provider locale + # (if it supports language) and/or otherwise the requested pygeoapi + # locale (or fallback default locale) + l10n.set_response_language(headers, prv_locale, request.locale) + + tiles_metadata = p.get_metadata( + dataset=dataset, server_url=api.base_url, + layer=p.get_layer(), tileset=matrix_id, + metadata_format=request._format, title=l10n.translate( + api.config['resources'][dataset]['title'], + request.locale), + description=l10n.translate( + api.config['resources'][dataset]['description'], + request.locale), + language=prv_locale) + + if request.format == F_HTML: # render + tpl_config = api.get_dataset_templates(dataset) + content = render_j2_template(api.tpl_config, tpl_config, + 'collections/tiles/metadata.html', + tiles_metadata, request.locale) + + else: + content = to_json(tiles_metadata, api.pretty_print) + + return headers, HTTPStatus.OK, content + + +def tilematrixsets(api: API, + request: APIRequest) -> Tuple[dict, int, str]: + """ + Provide tileMatrixSets definition + + :param request: A request object + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(**api.api_headers) + + # Retrieve available TileMatrixSets + enums = [e.value for e in TileMatrixSetEnum] + + tms = {"tileMatrixSets": []} + + for e in enums: + tms['tileMatrixSets'].append({ + "title": e.title, + "id": e.tileMatrixSet, + "uri": e.tileMatrixSetURI, + "links": [ + { + "rel": "self", + "type": "text/html", + "title": f"The HTML representation of the {e.tileMatrixSet} tile matrix set", # noqa + "href": f"{api.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=html" # noqa + }, + { + "rel": "self", + "type": "application/json", + "title": f"The JSON representation of the {e.tileMatrixSet} tile matrix set", # noqa + "href": f"{api.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=json" # noqa + } + ] + }) + + tms['links'] = [{ + "rel": "alternate", + "type": "text/html", + "title": l10n.translate('This document as HTML', request.locale), + "href": f"{api.base_url}/tileMatrixSets?f=html" + }, { + "rel": "self", + "type": "application/json", + "title": l10n.translate('This document as JSON', request.locale), + "href": f"{api.base_url}/tileMatrixSets?f=json" + }] + + if request.format == F_HTML: # render + content = render_j2_template(api.tpl_config, api.tpl_config, + 'tilematrixsets/index.html', + tms, request.locale) + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(tms, api.pretty_print) + + +def tilematrixset(api: API, + request: APIRequest, + tileMatrixSetId) -> Tuple[dict, + int, str]: + """ + Provide tile matrix definition + + :param request: A request object + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(**api.api_headers) + + # Retrieve relevant TileMatrixSet + enums = [e.value for e in TileMatrixSetEnum] + enum = None + + try: + for e in enums: + if tileMatrixSetId == e.tileMatrixSet: + enum = e + if not enum: + raise ValueError('could not find this tilematrixset') + except ValueError as err: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', str(err)) + + tms = { + "title": enum.tileMatrixSet, + "crs": enum.crs, + "id": enum.tileMatrixSet, + "uri": enum.tileMatrixSetURI, + "orderedAxes": enum.orderedAxes, + "wellKnownScaleSet": enum.wellKnownScaleSet, + "tileMatrices": enum.tileMatrices + } + + if request.format == F_HTML: # render + content = render_j2_template(api.tpl_config, api.tpl_config, + 'tilematrixsets/tilematrixset.html', + tms, request.locale) + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(tms, api.pretty_print) + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + paths = {} + + LOGGER.debug('setting up tiles endpoints') + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in get_visible_collections(cfg).items(): + tile_extension = filter_providers_by_type( + collections[k]['providers'], 'tile') + + if tile_extension: + tp = load_plugin('provider', tile_extension) + + tiles_path = f'/collections/{k}/tiles' + title = l10n.translate(v['title'], locale) + description = l10n.translate(v['description'], locale) + + datatype = 'vector' + + if tile_extension['format']['mimetype'].startswith('image'): + datatype = 'map' + + paths[tiles_path] = { + 'get': { + 'summary': f'Fetch a {title} tiles description', + 'description': description, + 'tags': [k], + 'operationId': f'describe{k.capitalize()}.collection.{datatype}.getTileSetsList', # noqa + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/Tiles'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + + tiles_data_path = f'{tiles_path}/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}' # noqa + + paths[tiles_data_path] = { + 'get': { + 'summary': f'Get a {title} tile', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}.collection.{datatype}.getTile', # noqa + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrixSetId"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrix"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileRow"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileCol"}, # noqa + { + 'name': 'f', + 'in': 'query', + 'description': 'The optional f parameter indicates the output format which the server shall provide as part of the response document.', # noqa + 'required': False, + 'schema': { + 'type': 'string', + 'enum': [tp.format_type], + 'default': tp.format_type + }, + 'style': 'form', + 'explode': False + } + ], + 'responses': { + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + mimetype = tile_extension['format']['mimetype'] + paths[tiles_data_path]['get']['responses']['200'] = { + 'description': 'successful operation', + 'content': { + mimetype: { + 'schema': { + 'type': 'string', + 'format': 'binary' + } + } + } + } + + return [{'name': 'tiles'}], {'paths': paths} diff --git a/pygeoapi/config.py b/pygeoapi/config.py index 07d861525..d1bf5c390 100644 --- a/pygeoapi/config.py +++ b/pygeoapi/config.py @@ -4,7 +4,7 @@ # Francesco Bartoli # # Copyright (c) 2022 Tom Kralidis -# Copyright (c) 2023 Francesco Bartoli +# Copyright (c) 2024 Francesco Bartoli # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation diff --git a/pygeoapi/django_/urls.py b/pygeoapi/django_/urls.py index faf8bf469..5736b2dea 100644 --- a/pygeoapi/django_/urls.py +++ b/pygeoapi/django_/urls.py @@ -8,7 +8,7 @@ # Copyright (c) 2022 Francesco Bartoli # Copyright (c) 2022 Luca Delucchi # Copyright (c) 2022 Krishna Lodha -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -96,6 +96,11 @@ def apply_slash_rule(url: str): views.collections, name='collection-detail', ), + path( + apply_slash_rule('collections//schema/'), + views.collection_schema, + name='collection-schema', + ), path( apply_slash_rule('collections//queryables/'), views.collection_queryables, @@ -116,16 +121,6 @@ def apply_slash_rule(url: str): views.collection_coverage, name='collection-coverage', ), - path( - apply_slash_rule('collections//coverage/domainset/'), # noqa - views.collection_coverage_domainset, - name='collection-coverage-domainset', - ), - path( - apply_slash_rule('collections//coverage/rangetype/'), # noqa - views.collection_coverage_rangetype, - name='collection-coverage-rangetype', - ), path( 'collections//map', views.collection_map, @@ -152,8 +147,8 @@ def apply_slash_rule(url: str): name='collection-tiles-metadata', ), path( - 'collections//tiles/\ - ///', + 'collections//tiles/' + + '///', views.collection_item_tiles, name='collection-item-tiles', ), @@ -187,6 +182,26 @@ def apply_slash_rule(url: str): views.get_collection_edr_query, name='collection-edr-corridor', ), + path( + 'collections//locations/', + views.get_collection_edr_query, + name='collection-edr-corridor', + ), + path( + 'collections//locations', + views.get_collection_edr_query, + name='collection-edr-corridor', + ), + path( + 'collections//instances/', + views.get_collection_edr_query, + name='collection-edr-instance', + ), + path( + 'collections//instances', + views.get_collection_edr_query, + name='collection-edr-instances', + ), path( 'collections//instances//position', views.get_collection_edr_query, @@ -217,8 +232,20 @@ def apply_slash_rule(url: str): views.get_collection_edr_query, name='collection-edr-instance-corridor', ), + path( + 'collections//instances/locations/', # noqa + views.get_collection_edr_query, + name='collection-edr-corridor', + ), + path( + 'collections//instances/locations', + views.get_collection_edr_query, + name='collection-edr-corridor', + ), path(apply_slash_rule('processes/'), views.processes, name='processes'), path('processes/', views.processes, name='process-detail'), + path('processes//execution', views.process_execution, + name='process-execution'), path(apply_slash_rule('jobs/'), views.jobs, name='jobs'), path('jobs/', views.jobs, name='job'), path( diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index 7f6896297..682ef51ce 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -8,7 +8,7 @@ # Copyright (c) 2022 Francesco Bartoli # Copyright (c) 2022 Luca Delucchi # Copyright (c) 2022 Krishna Lodha -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2025 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -35,11 +35,23 @@ """Integration module for Django""" -from typing import Tuple, Dict, Mapping, Optional +from typing import Optional, Union + from django.conf import settings from django.http import HttpRequest, HttpResponse -from pygeoapi.api import API +from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api as core_api +import pygeoapi.api.coverages as coverages_api +import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.itemtypes as itemtypes_api +import pygeoapi.api.maps as maps_api +import pygeoapi.api.processes as processes_api +import pygeoapi.api.stac as stac_api +import pygeoapi.api.tiles as tiles_api + +if settings.PYGEOAPI_CONFIG['server'].get('admin'): + import pygeoapi.admin as admin_api def landing_page(request: HttpRequest) -> HttpResponse: @@ -51,10 +63,7 @@ def landing_page(request: HttpRequest) -> HttpResponse: :returns: Django HTTP Response """ - response_ = _feed_response(request, 'landing_page') - response = _to_django_response(*response_) - - return response + return execute_from_django(core_api.landing_page, request) def openapi(request: HttpRequest) -> HttpResponse: @@ -66,10 +75,7 @@ def openapi(request: HttpRequest) -> HttpResponse: :returns: Django HTTP Response """ - response_ = _feed_response(request, 'openapi_') - response = _to_django_response(*response_) - - return response + return execute_from_django(core_api.openapi_, request) def conformance(request: HttpRequest) -> HttpResponse: @@ -80,11 +86,7 @@ def conformance(request: HttpRequest) -> HttpResponse: :returns: Django HTTP Response """ - - response_ = _feed_response(request, 'conformance') - response = _to_django_response(*response_) - - return response + return execute_from_django(core_api.conformance, request) def tilematrixsets(request: HttpRequest, @@ -98,15 +100,13 @@ def tilematrixsets(request: HttpRequest, :returns: Django HTTP Response """ - response = None - if tilematrixset_id is None: - response_ = _feed_response(request, 'tilematrixsets') + response_ = execute_from_django(tiles_api.tilematrixsets, request) else: - response_ = _feed_response(request, 'tilematrixset', tilematrixset_id) - response = _to_django_response(*response_) + response_ = execute_from_django(tiles_api.tilematrixsets, request, + tilematrixset_id) - return response + return response_ def collections(request: HttpRequest, @@ -120,10 +120,23 @@ def collections(request: HttpRequest, :returns: Django HTTP Response """ - response_ = _feed_response(request, 'describe_collections', collection_id) - response = _to_django_response(*response_) + return execute_from_django(core_api.describe_collections, request, + collection_id) - return response + +def collection_schema(request: HttpRequest, + collection_id: Optional[str] = None) -> HttpResponse: + """ + OGC API collections schema endpoint + + :request Django HTTP Request + :param collection_id: collection identifier + + :returns: Django HTTP Response + """ + + return execute_from_django(core_api.get_collection_schema, request, + collection_id) def collection_queryables(request: HttpRequest, @@ -137,12 +150,9 @@ def collection_queryables(request: HttpRequest, :returns: Django HTTP Response """ - response_ = _feed_response( - request, 'get_collection_queryables', collection_id + return execute_from_django( + itemtypes_api.get_collection_queryables, request, collection_id ) - response = _to_django_response(*response_) - - return response def collection_items(request: HttpRequest, collection_id: str) -> HttpResponse: @@ -156,26 +166,28 @@ def collection_items(request: HttpRequest, collection_id: str) -> HttpResponse: """ if request.method == 'GET': - response_ = _feed_response( + response_ = execute_from_django( + itemtypes_api.get_collection_items, request, - 'get_collection_items', collection_id, + skip_valid_check=True, ) elif request.method == 'POST': if request.content_type is not None: if request.content_type == 'application/geo+json': - response_ = _feed_response(request, 'manage_collection_item', - request, 'create', collection_id) + response_ = execute_from_django( + itemtypes_api.manage_collection_item, request, + 'create', collection_id, skip_valid_check=True) else: - response_ = _feed_response(request, 'post_collection_items', - request, collection_id) + response_ = execute_from_django( + itemtypes_api.get_collection_items, + request, collection_id, skip_valid_check=True,) elif request.method == 'OPTIONS': - response_ = _feed_response(request, 'manage_collection_item', - request, 'options', collection_id) + response_ = execute_from_django(itemtypes_api.manage_collection_item, + request, 'options', collection_id, + skip_valid_check=True) - response = _to_django_response(*response_) - - return response + return response_ def collection_map(request: HttpRequest, collection_id: str): @@ -187,11 +199,9 @@ def collection_map(request: HttpRequest, collection_id: str): :returns: HTTP response """ - response_ = _feed_response(request, 'get_collection_map', collection_id) - - response = _to_django_response(*response_) - - return response + return execute_from_django( + maps_api.get_collection_map, request, collection_id + ) def collection_style_map(request: HttpRequest, collection_id: str, @@ -205,13 +215,9 @@ def collection_style_map(request: HttpRequest, collection_id: str, :returns: HTTP response """ - response_ = _feed_response(request, 'get_collection_map', + return execute_from_django(maps_api.get_collection_map, request, collection_id, style_id) - response = _to_django_response(*response_) - - return response - def collection_item(request: HttpRequest, collection_id: str, item_id: str) -> HttpResponse: @@ -226,27 +232,22 @@ def collection_item(request: HttpRequest, """ if request.method == 'GET': - response_ = _feed_response( - request, 'get_collection_item', collection_id, item_id - ) + response_ = execute_from_django(itemtypes_api.get_collection_item, + request, collection_id, item_id) elif request.method == 'PUT': - response_ = _feed_response( - request, 'manage_collection_item', request, 'update', - collection_id, item_id - ) + response_ = execute_from_django(itemtypes_api.manage_collection_item, + request, 'update', collection_id, + item_id, skip_valid_check=True) elif request.method == 'DELETE': - response_ = _feed_response( - request, 'manage_collection_item', request, 'delete', - collection_id, item_id - ) + response_ = execute_from_django(itemtypes_api.manage_collection_item, + request, 'delete', collection_id, + item_id, skip_valid_check=True) elif request.method == 'OPTIONS': - response_ = _feed_response( - request, 'manage_collection_item', request, 'options', - collection_id, item_id) + response_ = execute_from_django(itemtypes_api.manage_collection_item, + request, 'options', collection_id, + item_id, skip_valid_check=True) - response = _to_django_response(*response_) - - return response + return response_ def collection_coverage(request: HttpRequest, @@ -260,50 +261,10 @@ def collection_coverage(request: HttpRequest, :returns: Django HTTP response """ - response_ = _feed_response( - request, 'get_collection_coverage', collection_id + return execute_from_django( + coverages_api.get_collection_coverage, request, collection_id, + skip_valid_check=True ) - response = _to_django_response(*response_) - - return response - - -def collection_coverage_domainset(request: HttpRequest, - collection_id: str) -> HttpResponse: - """ - OGC API - Coverages coverage domainset endpoint - - :request Django HTTP Request - :param collection_id: collection identifier - - :returns: Django HTTP response - """ - - response_ = _feed_response( - request, 'get_collection_coverage_domainset', collection_id - ) - response = _to_django_response(*response_) - - return response - - -def collection_coverage_rangetype(request: HttpRequest, - collection_id: str) -> HttpResponse: - """ - OGC API - Coverages coverage rangetype endpoint - - :request Django HTTP Request - :param collection_id: collection identifier - - :returns: Django HTTP response - """ - - response_ = _feed_response( - request, 'get_collection_coverage_rangetype', collection_id - ) - response = _to_django_response(*response_) - - return response def collection_tiles(request: HttpRequest, collection_id: str) -> HttpResponse: @@ -316,10 +277,8 @@ def collection_tiles(request: HttpRequest, collection_id: str) -> HttpResponse: :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_collection_tiles', collection_id) - response = _to_django_response(*response_) - - return response + return execute_from_django(tiles_api.get_collection_tiles, request, + collection_id) def collection_tiles_metadata(request: HttpRequest, collection_id: str, @@ -334,20 +293,16 @@ def collection_tiles_metadata(request: HttpRequest, collection_id: str, :returns: Django HTTP response """ - response_ = _feed_response( - request, - 'get_collection_tiles_metadata', - collection_id, - tileMatrixSetId, + return execute_from_django( + tiles_api.get_collection_tiles_metadata, + request, collection_id, tileMatrixSetId, + skip_valid_check=True ) - response = _to_django_response(*response_) - - return response def collection_item_tiles(request: HttpRequest, collection_id: str, tileMatrixSetId: str, tileMatrix: str, - tileRow: str, tileCol: str,) -> HttpResponse: + tileRow: str, tileCol: str) -> HttpResponse: """ OGC API - Tiles collection tiles data endpoint @@ -361,18 +316,16 @@ def collection_item_tiles(request: HttpRequest, collection_id: str, :returns: Django HTTP response """ - response_ = _feed_response( + return execute_from_django( + tiles_api.get_collection_tiles_data, request, - 'get_collection_tiles_metadata', collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol, + skip_valid_check=True ) - response = _to_django_response(*response_) - - return response def processes(request: HttpRequest, @@ -386,10 +339,22 @@ def processes(request: HttpRequest, :returns: Django HTTP response """ - response_ = _feed_response(request, 'describe_processes', process_id) - response = _to_django_response(*response_) + return execute_from_django(processes_api.describe_processes, request, + process_id) - return response + +def process_execution(request: HttpRequest, process_id: str) -> HttpResponse: + """ + OGC API - Processes execution endpoint + + :request Django HTTP Request + :param process_id: process identifier + + :returns: Django HTTP response + """ + + return execute_from_django(processes_api.execute_process, request, + process_id) def jobs(request: HttpRequest, job_id: Optional[str] = None) -> HttpResponse: @@ -403,10 +368,17 @@ def jobs(request: HttpRequest, job_id: Optional[str] = None) -> HttpResponse: :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_jobs', job_id) - response = _to_django_response(*response_) + if job_id is None: + response_ = execute_from_django(processes_api.get_jobs, request) + else: + if request.method == 'DELETE': # dismiss job + response_ = execute_from_django(processes_api.delete_job, request, + job_id) + else: # Return status of a specific job + response_ = execute_from_django(processes_api.get_jobs, request, + job_id) - return response + return response_ def job_results(request: HttpRequest, @@ -420,10 +392,7 @@ def job_results(request: HttpRequest, :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_job_result', job_id) - response = _to_django_response(*response_) - - return response + return execute_from_django(processes_api.get_job_result, request, job_id) def job_results_resource(request: HttpRequest, process_id: str, job_id: str, @@ -438,40 +407,49 @@ def job_results_resource(request: HttpRequest, process_id: str, job_id: str, :returns: Django HTTP response """ - response_ = _feed_response( - request, - 'get_job_result_resource', - job_id, - resource - ) - response = _to_django_response(*response_) - - return response + # TODO: this api method does not exist + return execute_from_django(processes_api.get_job_result_resource, + request, job_id, resource) -def get_collection_edr_query(request: HttpRequest, collection_id: str, - instance_id: str) -> HttpResponse: +def get_collection_edr_query( + request: HttpRequest, collection_id: str, + instance_id: Optional[str] = None, + location_id: Optional[str] = None +) -> HttpResponse: """ OGC API - EDR endpoint - :request Django HTTP Request - :param job_id: job identifier - :param resource: job resource + :param request: Django HTTP Request + :param collection_id: collection identifier + :param instance_id: optional instance identifier. default is None + :param location_id: optional location identifier. default is None :returns: Django HTTP response """ - query_type = request.path.split('/')[-1] - response_ = _feed_response( + if (request.path.endswith('instances') or + (instance_id is not None and + request.path.endswith(instance_id))): + return execute_from_django( + edr_api.get_collection_edr_instances, request, collection_id, + instance_id + ) + + if location_id: + query_type = 'locations' + else: + query_type = request.path.split('/')[-1] + + return execute_from_django( + edr_api.get_collection_edr_query, request, - 'get_collection_edr_query', collection_id, instance_id, - query_type + query_type, + location_id, + skip_valid_check=True ) - response = _to_django_response(*response_) - - return response def stac_catalog_root(request: HttpRequest) -> HttpResponse: @@ -483,10 +461,7 @@ def stac_catalog_root(request: HttpRequest) -> HttpResponse: :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_stac_root') - response = _to_django_response(*response_) - - return response + return execute_from_django(stac_api.get_stac_root, request) def stac_catalog_path(request: HttpRequest, path: str) -> HttpResponse: @@ -499,10 +474,7 @@ def stac_catalog_path(request: HttpRequest, path: str) -> HttpResponse: :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_stac_path', path) - response = _to_django_response(*response_) - - return response + return execute_from_django(stac_api.get_stac_path, request, path) def admin_config(request: HttpRequest) -> HttpResponse: @@ -513,13 +485,13 @@ def admin_config(request: HttpRequest) -> HttpResponse: """ if request.method == 'GET': - return _feed_response(request, 'get_admin_config') + return execute_from_django(admin_api.get_config_, request) elif request.method == 'PUT': - return _feed_response(request, 'put_admin_config') + return execute_from_django(admin_api.put_config, request) elif request.method == 'PATCH': - return _feed_response(request, 'patch_admin_config') + return execute_from_django(admin_api.patch_config, request) def admin_config_resources(request: HttpRequest) -> HttpResponse: @@ -530,10 +502,10 @@ def admin_config_resources(request: HttpRequest) -> HttpResponse: """ if request.method == 'GET': - return _feed_response(request, 'get_admin_config_resources') + return execute_from_django(admin_api.get_resources, request) elif request.method == 'POST': - return _feed_response(request, 'put_admin_config_resources') + return execute_from_django(admin_api.put_resource, request) def admin_config_resource(request: HttpRequest, @@ -545,42 +517,43 @@ def admin_config_resource(request: HttpRequest, """ if request.method == 'GET': - return _feed_response(request, 'put_admin_config_resource', - resource_id) + return execute_from_django(admin_api.get_resource, request, + resource_id) elif request.method == 'DELETE': - return _feed_response(request, 'delete_admin_config_resource', - resource_id) + return execute_from_django(admin_api.delete_resource, request, + resource_id) elif request.method == 'PUT': - return _feed_response(request, 'put_admin_config_resource', - resource_id) + return execute_from_django(admin_api.put_resource, request, + resource_id) elif request.method == 'PATCH': - return _feed_response(request, 'patch_admin_config_resource', - resource_id) + return execute_from_django(admin_api.patch_resource, request, + resource_id) -def _feed_response(request: HttpRequest, api_definition: str, - *args, **kwargs) -> Tuple[Dict, int, str]: - """Use pygeoapi api to process the input request""" +def execute_from_django(api_function, request: HttpRequest, *args, + skip_valid_check=False) -> HttpResponse: - if 'admin' in api_definition and settings.PYGEOAPI_CONFIG['server'].get('admin'): # noqa + api_: API | "Admin" + if settings.PYGEOAPI_CONFIG['server'].get('admin'): # noqa from pygeoapi.admin import Admin api_ = Admin(settings.PYGEOAPI_CONFIG, settings.OPENAPI_DOCUMENT) else: api_ = API(settings.PYGEOAPI_CONFIG, settings.OPENAPI_DOCUMENT) - api = getattr(api_, api_definition) - - return api(request, *args, **kwargs) - + api_request = APIRequest.from_django(request, api_.locales) + content: Union[str, bytes] + if not skip_valid_check and not api_request.is_valid(): + headers, status, content = api_.get_format_exception(api_request) + else: -def _to_django_response(headers: Mapping, status_code: int, - content: str) -> HttpResponse: - """Convert API payload to a django response""" + headers, status, content = api_function(api_, api_request, *args) + content = apply_gzip(headers, content) - response = HttpResponse(content, status=status_code) + # Convert API payload to a django response + response = HttpResponse(content, status=status) for key, value in headers.items(): response[key] = value diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index ac5946268..ab661a356 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -3,7 +3,7 @@ # Authors: Tom Kralidis # Norman Barker # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -31,12 +31,21 @@ """Flask module providing the route paths to the api""" import os +from typing import Union import click - -from flask import Flask, Blueprint, make_response, request, send_from_directory - -from pygeoapi.api import API +from flask import (Flask, Blueprint, make_response, request, + send_from_directory, Response, Request) + +from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api as core_api +import pygeoapi.api.coverages as coverages_api +import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.itemtypes as itemtypes_api +import pygeoapi.api.maps as maps_api +import pygeoapi.api.processes as processes_api +import pygeoapi.api.stac as stac_api +import pygeoapi.api.tiles as tiles_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_mimetype, get_api_rules @@ -48,6 +57,7 @@ API_RULES = get_api_rules(CONFIG) if CONFIG['server'].get('admin'): + import pygeoapi.admin as admin_api from pygeoapi.admin import Admin STATIC_FOLDER = 'static' @@ -63,13 +73,18 @@ static_folder=STATIC_FOLDER, url_prefix=API_RULES.get_url_prefix('flask') ) -ADMIN_BLUEPRINT = Blueprint('admin', __name__, static_folder=STATIC_FOLDER) +ADMIN_BLUEPRINT = Blueprint( + 'admin', + __name__, + static_folder=STATIC_FOLDER, + url_prefix=API_RULES.get_url_prefix('flask') +) # CORS: optionally enable from config. if CONFIG['server'].get('cors', False): try: from flask_cors import CORS - CORS(APP) + CORS(APP, CORS_EXPOSE_HEADERS=['*']) except ModuleNotFoundError: print('Python package flask-cors required for CORS support') @@ -101,23 +116,43 @@ def schemas(path): dirname_ = os.path.dirname(full_filepath) basename_ = os.path.basename(full_filepath) - # TODO: better sanitization? - path_ = dirname_.replace('..', '').replace('//', '') + path_ = dirname_.replace('..', '').replace('//', '').replace('./', '') + + if '..' in path_: + return 'Invalid path', 400 + return send_from_directory(path_, basename_, mimetype=get_mimetype(basename_)) -def get_response(result: tuple): +def execute_from_flask(api_function, request: Request, *args, + skip_valid_check=False, + alternative_api=None + ) -> Response: """ - Creates a Flask Response object and updates matching headers. + Executes API function from Flask - :param result: The result of the API call. - This should be a tuple of (headers, status, content). + :param api_function: API function + :param request: request object + :param *args: variable length additional arguments + :param skip_validity_check: bool + :param alternative_api: specify custom api instance such as Admin - :returns: A Response instance. + :returns: A Response instance """ - headers, status, content = result + actual_api = api_ if alternative_api is None else alternative_api + + api_request = APIRequest.from_flask(request, actual_api.locales) + + content: Union[str, bytes] + + if not skip_valid_check and not api_request.is_valid(): + headers, status, content = actual_api.get_format_exception(api_request) + else: + headers, status, content = api_function(actual_api, api_request, *args) + content = apply_gzip(headers, content) + response = make_response(content, status) if headers: @@ -132,7 +167,7 @@ def landing_page(): :returns: HTTP response """ - return get_response(api_.landing_page(request)) + return execute_from_flask(core_api.landing_page, request) @BLUEPRINT.route('/openapi') @@ -142,7 +177,8 @@ def openapi(): :returns: HTTP response """ - return get_response(api_.openapi_(request)) + + return execute_from_flask(core_api.openapi_, request) @BLUEPRINT.route('/conformance') @@ -152,7 +188,8 @@ def conformance(): :returns: HTTP response """ - return get_response(api_.conformance(request)) + + return execute_from_flask(core_api.conformance, request) @BLUEPRINT.route('/TileMatrixSets/') @@ -161,9 +198,12 @@ def get_tilematrix_set(tileMatrixSetId=None): OGC API TileMatrixSet endpoint :param tileMatrixSetId: identifier of tile matrix set + :returns: HTTP response """ - return get_response(api_.tilematrixset(request, tileMatrixSetId)) + + return execute_from_flask(tiles_api.tilematrixset, request, + tileMatrixSetId) @BLUEPRINT.route('/TileMatrixSets') @@ -173,7 +213,8 @@ def get_tilematrix_sets(): :returns: HTTP response """ - return get_response(api_.tilematrixsets(request)) + + return execute_from_flask(tiles_api.tilematrixsets, request) @BLUEPRINT.route('/collections') @@ -186,19 +227,37 @@ def collections(collection_id=None): :returns: HTTP response """ - return get_response(api_.describe_collections(request, collection_id)) + + return execute_from_flask(core_api.describe_collections, request, + collection_id) + + +@BLUEPRINT.route('/collections//schema') +def collection_schema(collection_id): + """ + OGC API - collections schema endpoint + + :param collection_id: collection identifier + + :returns: HTTP response + """ + + return execute_from_flask(core_api.get_collection_schema, request, + collection_id) @BLUEPRINT.route('/collections//queryables') def collection_queryables(collection_id=None): """ - OGC API collections querybles endpoint + OGC API collections queryables endpoint :param collection_id: collection identifier :returns: HTTP response """ - return get_response(api_.get_collection_queryables(request, collection_id)) + + return execute_from_flask(itemtypes_api.get_collection_queryables, request, + collection_id) @BLUEPRINT.route('/collections//items', @@ -218,37 +277,41 @@ def collection_items(collection_id, item_id=None): """ if item_id is None: - if request.method == 'GET': # list items - return get_response( - api_.get_collection_items(request, collection_id)) - elif request.method == 'POST': # filter or manage items + if request.method == 'POST': # filter or manage items if request.content_type is not None: if request.content_type == 'application/geo+json': - return get_response( - api_.manage_collection_item(request, 'create', - collection_id)) + return execute_from_flask( + itemtypes_api.manage_collection_item, + request, 'create', collection_id, + skip_valid_check=True) else: - return get_response( - api_.post_collection_items(request, collection_id)) + return execute_from_flask( + itemtypes_api.get_collection_items, request, + collection_id, skip_valid_check=True) elif request.method == 'OPTIONS': - return get_response( - api_.manage_collection_item(request, 'options', collection_id)) + return execute_from_flask( + itemtypes_api.manage_collection_item, request, 'options', + collection_id, skip_valid_check=True) + else: # GET: list items + return execute_from_flask(itemtypes_api.get_collection_items, + request, collection_id, + skip_valid_check=True) elif request.method == 'DELETE': - return get_response( - api_.manage_collection_item(request, 'delete', - collection_id, item_id)) + return execute_from_flask(itemtypes_api.manage_collection_item, + request, 'delete', collection_id, item_id, + skip_valid_check=True) elif request.method == 'PUT': - return get_response( - api_.manage_collection_item(request, 'update', - collection_id, item_id)) + return execute_from_flask(itemtypes_api.manage_collection_item, + request, 'update', collection_id, item_id, + skip_valid_check=True) elif request.method == 'OPTIONS': - return get_response( - api_.manage_collection_item(request, 'options', - collection_id, item_id)) + return execute_from_flask(itemtypes_api.manage_collection_item, + request, 'options', collection_id, item_id, + skip_valid_check=True) else: - return get_response( - api_.get_collection_item(request, collection_id, item_id)) + return execute_from_flask(itemtypes_api.get_collection_item, request, + collection_id, item_id) @BLUEPRINT.route('/collections//coverage') @@ -260,33 +323,9 @@ def collection_coverage(collection_id): :returns: HTTP response """ - return get_response(api_.get_collection_coverage(request, collection_id)) - - -@BLUEPRINT.route('/collections//coverage/domainset') -def collection_coverage_domainset(collection_id): - """ - OGC API - Coverages coverage domainset endpoint - - :param collection_id: collection identifier - - :returns: HTTP response - """ - return get_response(api_.get_collection_coverage_domainset( - request, collection_id)) - -@BLUEPRINT.route('/collections//coverage/rangetype') -def collection_coverage_rangetype(collection_id): - """ - OGC API - Coverages coverage rangetype endpoint - - :param collection_id: collection identifier - - :returns: HTTP response - """ - return get_response(api_.get_collection_coverage_rangetype( - request, collection_id)) + return execute_from_flask(coverages_api.get_collection_coverage, request, + collection_id, skip_valid_check=True) @BLUEPRINT.route('/collections//tiles') @@ -298,8 +337,9 @@ def get_collection_tiles(collection_id=None): :returns: HTTP response """ - return get_response(api_.get_collection_tiles( - request, collection_id)) + + return execute_from_flask(tiles_api.get_collection_tiles, request, + collection_id) @BLUEPRINT.route('/collections//tiles/') @@ -313,8 +353,10 @@ def get_collection_tiles_metadata(collection_id=None, tileMatrixSetId=None): :returns: HTTP response """ - return get_response(api_.get_collection_tiles_metadata( - request, collection_id, tileMatrixSetId)) + + return execute_from_flask(tiles_api.get_collection_tiles_metadata, + request, collection_id, tileMatrixSetId, + skip_valid_check=True) @BLUEPRINT.route('/collections//tiles/\ @@ -332,8 +374,12 @@ def get_collection_tiles_data(collection_id=None, tileMatrixSetId=None, :returns: HTTP response """ - return get_response(api_.get_collection_tiles_data( - request, collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol)) + + return execute_from_flask( + tiles_api.get_collection_tiles_data, + request, collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol, + skip_valid_check=True, + ) @BLUEPRINT.route('/collections//map') @@ -348,15 +394,9 @@ def collection_map(collection_id, style_id=None): :returns: HTTP response """ - headers, status_code, content = api_.get_collection_map( - request, collection_id, style_id) - - response = make_response(content, status_code) - - if headers: - response.headers = headers - - return response + return execute_from_flask( + maps_api.get_collection_map, request, collection_id, style_id + ) @BLUEPRINT.route('/processes') @@ -369,7 +409,9 @@ def get_processes(process_id=None): :returns: HTTP response """ - return get_response(api_.describe_processes(request, process_id)) + + return execute_from_flask(processes_api.describe_processes, request, + process_id) @BLUEPRINT.route('/jobs') @@ -385,12 +427,13 @@ def get_jobs(job_id=None): """ if job_id is None: - return get_response(api_.get_jobs(request)) + return execute_from_flask(processes_api.get_jobs, request) else: if request.method == 'DELETE': # dismiss job - return get_response(api_.delete_job(request, job_id)) + return execute_from_flask(processes_api.delete_job, request, + job_id) else: # Return status of a specific job - return get_response(api_.get_jobs(request, job_id)) + return execute_from_flask(processes_api.get_jobs, request, job_id) @BLUEPRINT.route('/processes//execution', methods=['POST']) @@ -403,7 +446,8 @@ def execute_process_jobs(process_id): :returns: HTTP response """ - return get_response(api_.execute_process(request, process_id)) + return execute_from_flask(processes_api.execute_process, request, + process_id) @BLUEPRINT.route('/jobs//results', @@ -416,22 +460,8 @@ def get_job_result(job_id=None): :returns: HTTP response """ - return get_response(api_.get_job_result(request, job_id)) - -@BLUEPRINT.route('/jobs//results/', - methods=['GET']) -def get_job_result_resource(job_id, resource): - """ - OGC API - Processes job result resource endpoint - - :param job_id: job identifier - :param resource: job resource - - :returns: HTTP response - """ - return get_response(api_.get_job_result_resource( - request, job_id, resource)) + return execute_from_flask(processes_api.get_job_result, request, job_id) @BLUEPRINT.route('/collections//position') @@ -440,24 +470,47 @@ def get_job_result_resource(job_id, resource): @BLUEPRINT.route('/collections//radius') @BLUEPRINT.route('/collections//trajectory') @BLUEPRINT.route('/collections//corridor') +@BLUEPRINT.route('/collections//locations/') +@BLUEPRINT.route('/collections//locations') @BLUEPRINT.route('/collections//instances//position') # noqa @BLUEPRINT.route('/collections//instances//area') # noqa @BLUEPRINT.route('/collections//instances//cube') # noqa @BLUEPRINT.route('/collections//instances//radius') # noqa @BLUEPRINT.route('/collections//instances//trajectory') # noqa @BLUEPRINT.route('/collections//instances//corridor') # noqa -def get_collection_edr_query(collection_id, instance_id=None): +@BLUEPRINT.route('/collections//instances//locations/') # noqa +@BLUEPRINT.route('/collections//instances//locations') # noqa +@BLUEPRINT.route('/collections//instances/') +@BLUEPRINT.route('/collections//instances') +def get_collection_edr_query(collection_id, instance_id=None, + location_id=None): """ OGC EDR API endpoints :param collection_id: collection identifier :param instance_id: instance identifier + :param location_id: location id of a /locations/ query :returns: HTTP response """ - query_type = request.path.split('/')[-1] - return get_response(api_.get_collection_edr_query(request, collection_id, - instance_id, query_type)) + + if (request.path.endswith('instances') or + (instance_id is not None and + request.path.endswith(instance_id))): + return execute_from_flask( + edr_api.get_collection_edr_instances, request, collection_id, + instance_id + ) + + if location_id: + query_type = 'locations' + else: + query_type = request.path.split('/')[-1] + + return execute_from_flask( + edr_api.get_collection_edr_query, request, collection_id, instance_id, + query_type, location_id, skip_valid_check=True + ) @BLUEPRINT.route('/stac') @@ -467,7 +520,8 @@ def stac_catalog_root(): :returns: HTTP response """ - return get_response(api_.get_stac_root(request)) + + return execute_from_flask(stac_api.get_stac_root, request) @BLUEPRINT.route('/stac/') @@ -479,7 +533,8 @@ def stac_catalog_path(path): :returns: HTTP response """ - return get_response(api_.get_stac_path(request, path)) + + return execute_from_flask(stac_api.get_stac_path, request, path) @ADMIN_BLUEPRINT.route('/admin/config', methods=['GET', 'PUT', 'PATCH']) @@ -491,13 +546,16 @@ def admin_config(): """ if request.method == 'GET': - return get_response(admin_.get_config(request)) + return execute_from_flask(admin_api.get_config_, request, + alternative_api=admin_) elif request.method == 'PUT': - return get_response(admin_.put_config(request)) + return execute_from_flask(admin_api.put_config, request, + alternative_api=admin_) elif request.method == 'PATCH': - return get_response(admin_.patch_config(request)) + return execute_from_flask(admin_api.patch_config, request, + alternative_api=admin_) @ADMIN_BLUEPRINT.route('/admin/config/resources', methods=['GET', 'POST']) @@ -509,10 +567,12 @@ def admin_config_resources(): """ if request.method == 'GET': - return get_response(admin_.get_resources(request)) + return execute_from_flask(admin_api.get_resources, request, + alternative_api=admin_) elif request.method == 'POST': - return get_response(admin_.post_resource(request)) + return execute_from_flask(admin_api.post_resource, request, + alternative_api=admin_) @ADMIN_BLUEPRINT.route( @@ -526,16 +586,24 @@ def admin_config_resource(resource_id): """ if request.method == 'GET': - return get_response(admin_.get_resource(request, resource_id)) + return execute_from_flask(admin_api.get_resource, request, + resource_id, + alternative_api=admin_) elif request.method == 'DELETE': - return get_response(admin_.delete_resource(request, resource_id)) + return execute_from_flask(admin_api.delete_resource, request, + resource_id, + alternative_api=admin_) elif request.method == 'PUT': - return get_response(admin_.put_resource(request, resource_id)) + return execute_from_flask(admin_api.put_resource, request, + resource_id, + alternative_api=admin_) elif request.method == 'PATCH': - return get_response(admin_.patch_resource(request, resource_id)) + return execute_from_flask(admin_api.patch_resource, request, + resource_id, + alternative_api=admin_) APP.register_blueprint(BLUEPRINT) diff --git a/pygeoapi/formatter/csv_.py b/pygeoapi/formatter/csv_.py index 664bc8807..51a6ded17 100644 --- a/pygeoapi/formatter/csv_.py +++ b/pygeoapi/formatter/csv_.py @@ -27,11 +27,10 @@ # # ================================================================= +import csv import io import logging -import unicodecsv as csv - from pygeoapi.formatter.base import BaseFormatter, FormatterSerializationError LOGGER = logging.getLogger(__name__) @@ -83,10 +82,11 @@ def write(self, options: dict = {}, data: dict = None) -> str: # TODO: implement wkt geometry serialization LOGGER.debug('not a point geometry, skipping') + print("JJJ", fields) LOGGER.debug(f'CSV fields: {fields}') try: - output = io.BytesIO() + output = io.StringIO() writer = csv.DictWriter(output, fields) writer.writeheader() @@ -101,7 +101,7 @@ def write(self, options: dict = {}, data: dict = None) -> str: LOGGER.error(err) raise FormatterSerializationError('Error writing CSV output') - return output.getvalue() + return output.getvalue().encode('utf-8') def __repr__(self): return f' {self.name}' diff --git a/pygeoapi/l10n.py b/pygeoapi/l10n.py index f363abd4c..908888e45 100644 --- a/pygeoapi/l10n.py +++ b/pygeoapi/l10n.py @@ -66,7 +66,7 @@ def str2locale(value, silent: bool = False) -> Union[Locale, None]: :returns: babel.core.Locale or None - :raises: LocaleError + :raises LocaleError: """ if isinstance(value, Locale): @@ -103,7 +103,7 @@ def locale2str(value: Locale) -> str: :returns: A string containing a web locale (e.g. 'fr-CH') or language tag (e.g. 'de'). - :raises: LocaleError + :raises LocaleError: """ if not isinstance(value, Locale): @@ -145,7 +145,7 @@ def best_match(accept_languages, available_locales) -> Locale: :returns: babel.core.Locale - :raises: LocaleError + :raises LocaleError: """ def get_match(locale_, available_locales_): @@ -254,7 +254,7 @@ def translate(value, language: Union[Locale, str]): :returns: A translated string or the original value. - :raises: LocaleError + :raises LocaleError: """ nested_dicts = isinstance(value, dict) and any(isinstance(v, dict) @@ -386,7 +386,7 @@ def set_response_language(headers: dict, *locale_: Locale): Multiple locales can be set for this header. Note that duplicates will be removed. - :raises: LocaleError if no valid Babel Locale was found. + :raises LocaleError: if no valid Babel Locale was found. """ if not hasattr(headers, '__setitem__'): @@ -422,7 +422,7 @@ def add_locale(url, locale_) -> str: :returns: A new URL with a 'lang=' query parameter. - :raises: requests.exceptions.MissingSchema + :raises requests.exceptions.MissingSchema: """ loc = str2locale(locale_, True) diff --git a/pygeoapi/linked_data.py b/pygeoapi/linked_data.py index b79a72aaf..83bad1e2a 100644 --- a/pygeoapi/linked_data.py +++ b/pygeoapi/linked_data.py @@ -31,11 +31,10 @@ Returns content as linked data representations """ -import json import logging from typing import Callable -from pygeoapi.util import is_url, render_j2_template +from pygeoapi.util import is_url, render_j2_template, url_join from pygeoapi import l10n from shapely.geometry import shape from shapely.ops import unary_union @@ -189,30 +188,22 @@ def geojson2jsonld(cls, data: dict, dataset: str, :returns: string of rendered JSON (GeoJSON-LD) """ - LOGGER.debug('Fetching context and template from resource configuration') - jsonld = cls.config['resources'][dataset].get('linked-data', {}) - ds_url = f"{cls.get_collections_url()}/{dataset}" - - context = jsonld.get('context', []).copy() - template = jsonld.get('item_template', None) + LOGGER.debug('Fetching context from resource configuration') + context = cls.config['resources'][dataset].get('context', []).copy() + templates = cls.get_dataset_templates(dataset) defaultVocabulary = { 'schema': 'https://schema.org/', + 'gsp': 'http://www.opengis.net/ont/geosparql#', 'type': '@type' } if identifier: - # Single jsonld - defaultVocabulary.update({ - 'gsp': 'http://www.opengis.net/ont/geosparql#' - }) - # Expand properties block data.update(data.pop('properties')) # Include multiple geometry encodings if (data.get('geometry') is not None): - data['type'] = 'schema:Place' jsonldify_geometry(data) data['@id'] = identifier @@ -224,6 +215,7 @@ def geojson2jsonld(cls, data: dict, dataset: str, 'FeatureCollection': 'schema:itemList' }) + ds_url = url_join(cls.get_collections_url(), dataset) data['@id'] = ds_url for i, feature in enumerate(data['features']): @@ -233,9 +225,15 @@ def geojson2jsonld(cls, data: dict, dataset: str, if not is_url(str(identifier_)): identifier_ = f"{ds_url}/items/{feature['id']}" # noqa + # Include multiple geometry encodings + if feature.get('geometry') is not None: + jsonldify_geometry(feature) + data['features'][i] = { '@id': identifier_, - 'type': 'schema:Place' + 'type': 'schema:Place', + **feature.pop('properties'), + **feature } if data.get('timeStamp', False): @@ -248,14 +246,21 @@ def geojson2jsonld(cls, data: dict, dataset: str, **data } - if None in (template, identifier): - return ldjsonData + if identifier: + # Render jsonld template for single item + LOGGER.debug('Rendering JSON-LD item template') + content = render_j2_template( + cls.tpl_config, templates, + 'collections/items/item.jsonld', ldjsonData) + else: - # Render jsonld template for single item with template configured - LOGGER.debug(f'Rendering JSON-LD template: {template}') - content = render_j2_template(cls.config, template, ldjsonData) - ldjsonData = json.loads(content) - return ldjsonData + # Render jsonld template for /items + LOGGER.debug('Rendering JSON-LD items template') + content = render_j2_template( + cls.tpl_config, templates, + 'collections/items/index.jsonld', ldjsonData) + + return content def jsonldify_geometry(feature: dict) -> None: @@ -268,6 +273,8 @@ def jsonldify_geometry(feature: dict) -> None: :returns: None """ + feature['type'] = 'schema:Place' + geo = feature.get('geometry') geom = shape(geo) @@ -284,7 +291,11 @@ def jsonldify_geometry(feature: dict) -> None: } # Schema geometry - feature['schema:geo'] = geom2schemageo(geom) + try: + feature['schema:geo'] = geom2schemageo(geom) + except AttributeError: + msg = f'Unable to parse schema geometry for {feature["id"]}' + LOGGER.warning(msg) def geom2schemageo(geom: shape) -> dict: diff --git a/pygeoapi/models/config.py b/pygeoapi/models/config.py index d527ba14f..2aa6717b7 100644 --- a/pygeoapi/models/config.py +++ b/pygeoapi/models/config.py @@ -3,8 +3,10 @@ # ================================================================= # # Authors: Sander Schaminee +# Francesco Bartoli # # Copyright (c) 2023 Sander Schaminee +# Copyright (c) 2024 Francesco Bartoli # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -34,7 +36,7 @@ class APIRules(BaseModel): """ Pydantic model for API design rules that must be adhered to. """ - api_version: str = Field(pattern=r'^\d+\.\d+\..+$', + api_version: str = Field(regex=r'^\d+\.\d+\..+$', description="Semantic API version number.") url_prefix: str = Field( "", @@ -60,11 +62,11 @@ def create(**rules_config) -> 'APIRules': """ Returns a new APIRules instance for the current API version and configured rules. """ obj = { - k: v for k, v in rules_config.items() if k in APIRules.model_fields + k: v for k, v in rules_config.items() if k in APIRules.__fields__ } # Validation will fail if required `api_version` is missing # or if `api_version` is not a semantic version number - return APIRules.model_validate(obj) + return APIRules.parse_obj(obj) @property def response_headers(self) -> dict: diff --git a/pygeoapi/models/cql.py b/pygeoapi/models/cql.py deleted file mode 100644 index c5838fc9e..000000000 --- a/pygeoapi/models/cql.py +++ /dev/null @@ -1,507 +0,0 @@ -# ****************************** -*- -# flake8: noqa -# generated by datamodel-codegen: -# filename: cql-schema.json -# timestamp: 2021-03-13T21:05:20+00:00 -# ================================================================= -# -# Authors: Francesco Bartoli -# -# Copyright (c) 2021 Francesco Bartoli -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# ================================================================= - -from datetime import date, datetime -from typing import Any, List, Optional, Union -from typing_extensions import Literal - -from pydantic import BaseModel, Field, RootModel - - -class CQLModel(RootModel): - root: 'Union[\n ComparisonPredicate,\n SpatialPredicate,\n TemporalPredicate,\n AndExpression\n ]' - - -class AndExpression(BaseModel): - and_: 'List[ComparisonPredicate]' = Field(..., alias='and') - - -class NotExpression(BaseModel): - not_: 'List[Any]' = Field(..., alias='not') - - -class OrExpression(BaseModel): - or_: 'List[Any]' = Field(..., alias='or') - - -class PropertyRef(BaseModel): - property: 'Optional[str]' = None - - -class ScalarLiteral(RootModel): - root: 'Union[str, float, bool]' - - -class Bbox(RootModel): - root: 'List[float]' - - -class LinestringCoordinate(RootModel): - root: 'List[Any]' - - -class Linestring(BaseModel): - type: Literal['LineString'] - coordinates: 'List[LinestringCoordinate]' = Field(...) - bbox: 'Optional[List[float]]' = Field(None) - - -class MultilineStringCoordinate(RootModel): - root: 'List[Any]' - - -class Multilinestring(BaseModel): - type: Literal['MultiLineString'] - coordinates: 'List[List[MultilineStringCoordinate]]' - bbox: 'Optional[List[float]]' = Field(None) - - -class Multipoint(BaseModel): - type: Literal['MultiPoint'] - coordinates: 'List[List[float]]' - bbox: 'Optional[List[float]]' = Field(None) - - -class MultipolygonCoordinateItem(RootModel): - root: 'List[Any]' - - -class Multipolygon(BaseModel): - type: Literal['MultiPolygon'] - coordinates: 'List[List[List[MultipolygonCoordinateItem]]]' - bbox: 'Optional[List[float]]' = Field(None) - - -class Point(BaseModel): - type: Literal['Point'] - coordinates: 'List[float]' = Field(...) - bbox: 'Optional[List[float]]' = Field(None) - - -class PolygonCoordinatesItem(RootModel): - root: 'List[Any]' - - -class Polygon(BaseModel): - type: Literal['Polygon'] - coordinates: 'List[List[PolygonCoordinatesItem]]' - bbox: 'Optional[List[float]]' = Field(None) - - -class TimeString(RootModel): - root: 'Union[date, datetime]' - - -class EnvelopeLiteral(BaseModel): - bbox: 'Bbox' - - -class GeometryLiteral(RootModel): - root: 'Union[\n Point, Linestring, Polygon, Multipoint, Multilinestring, Multipolygon\n ]' - - -class TypedTimeString(BaseModel): - datetime: 'TimeString' - - -class PeriodString(RootModel): - root: 'List[Union[TimeString, str]]' = Field(...) - - -class SpatialLiteral(RootModel): - root: 'Union[GeometryLiteral, EnvelopeLiteral]' - - -class TemporalLiteral(RootModel): - root: 'Union[TimeString, PeriodString]' - - -class TypedPeriodString(BaseModel): - datetime: 'PeriodString' - - -class TypedTemporalLiteral(RootModel): - root: 'Union[TypedTimeString, TypedPeriodString]' - - -class ArrayPredicate(RootModel): - root: 'Union[\n AequalsExpression,\n AcontainsExpression,\n AcontainedByExpression,\n AoverlapsExpression,\n ]' - - -class ComparisonPredicate(RootModel): - root: 'Union[\n BinaryComparisonPredicate,\n IsLikePredicate,\n IsBetweenPredicate,\n IsInListPredicate,\n IsNullPredicate,\n ]' - - -class SpatialPredicate(RootModel): - root: 'Union[\n IntersectsExpression,\n EqualsExpression,\n DisjointExpression,\n TouchesExpression,\n WithinExpression,\n OverlapsExpression,\n CrossesExpression,\n ContainsExpression,\n ]' - - -class TemporalPredicate(RootModel): - root: 'Union[\n BeforeExpression,\n AfterExpression,\n MeetsExpression,\n MetbyExpression,\n ToverlapsExpression,\n OverlappedbyExpression,\n BeginsExpression,\n BegunbyExpression,\n DuringExpression,\n TcontainsExpression,\n EndsExpression,\n EndedbyExpression,\n TequalsExpression,\n AnyinteractsExpression,\n ]' - - -class AcontainedByExpression(BaseModel): - acontainedBy: 'ArrayExpression' - - -class AcontainsExpression(BaseModel): - acontains: 'ArrayExpression' - - -class AequalsExpression(BaseModel): - aequals: 'ArrayExpression' - - -class AfterExpression(BaseModel): - after: 'TemporalOperands' - - -class AnyinteractsExpression(BaseModel): - anyinteracts: 'TemporalOperands' - - -class AoverlapsExpression(BaseModel): - aoverlaps: 'ArrayExpression' - - -class BeforeExpression(BaseModel): - before: 'TemporalOperands' - - -class BeginsExpression(BaseModel): - begins: 'TemporalOperands' - - -class BegunbyExpression(BaseModel): - begunby: 'TemporalOperands' - - -class BinaryComparisonPredicate(RootModel): - root: 'Union[\n EqExpression, LtExpression, GtExpression, LteExpression, GteExpression\n ]' - - -class ContainsExpression(BaseModel): - contains: 'SpatialOperands' - - -class CrossesExpression(BaseModel): - crosses: 'SpatialOperands' - - -class DisjointExpression(BaseModel): - disjoint: 'SpatialOperands' - - -class DuringExpression(BaseModel): - during: 'TemporalOperands' - - -class EndedbyExpression(BaseModel): - endedby: 'TemporalOperands' - - -class EndsExpression(BaseModel): - ends: 'TemporalOperands' - - -class EqualsExpression(BaseModel): - equals: 'SpatialOperands' - - -class IntersectsExpression(BaseModel): - intersects: 'SpatialOperands' - - -class Between(BaseModel): - value: 'ValueExpression' - lower: 'Optional[ScalarExpression]' = Field(None) - upper: 'Optional[ScalarExpression]' = Field(None) - - -class IsBetweenPredicate(BaseModel): - between: 'Between' - - -class In(BaseModel): - value: 'ValueExpression' - list: 'List[ValueExpression]' - nocase: 'Optional[bool]' = True - - -class IsInListPredicate(BaseModel): - in_: 'In' = Field(..., alias='in') - - -class IsLikePredicate(BaseModel): - like: 'ScalarOperands' - wildcard: 'Optional[str]' = '%' - singleChar: 'Optional[str]' = '.' - escapeChar: 'Optional[str]' = '\\' - nocase: 'Optional[bool]' = True - - -class IsNullPredicate(BaseModel): - isNull: 'ScalarExpression' - - -class MeetsExpression(BaseModel): - meets: 'TemporalOperands' - - -class MetbyExpression(BaseModel): - metby: 'TemporalOperands' - - -class OverlappedbyExpression(BaseModel): - overlappedby: 'TemporalOperands' - - -class OverlapsExpression(BaseModel): - overlaps: 'SpatialOperands' - - -class TcontainsExpression(BaseModel): - tcontains: 'TemporalOperands' - - -class TequalsExpression(BaseModel): - tequals: 'TemporalOperands' - - -class TouchesExpression(BaseModel): - touches: 'SpatialOperands' - - -class ToverlapsExpression(BaseModel): - toverlaps: 'TemporalOperands' - - -class WithinExpression(BaseModel): - within: 'SpatialOperands' - - -class ArrayExpression(RootModel): - root: 'List[Union[PropertyRef, FunctionRef, ArrayLiteral]]' = Field( - ... # , max_items=2, min_items=2 - ) - - -class EqExpression(BaseModel): - eq: 'ScalarOperands' - - -class GtExpression(BaseModel): - gt: 'ScalarOperands' - - -class GteExpression(BaseModel): - gte: 'ScalarOperands' - - -class LtExpression(BaseModel): - lt: 'ScalarOperands' - - -class LteExpression(BaseModel): - lte: 'ScalarOperands' - - -class ScalarExpression(RootModel): - root: 'Union[ScalarLiteral, PropertyRef,\n FunctionRef, ArithmeticExpression]' - - -class ScalarOperands(RootModel): - root: 'List[ScalarExpression]' = Field(...) - - -class SpatialOperands(RootModel): - root: 'List[GeomExpression]' = Field(...) - - -class TemporalOperands(RootModel): - root: 'List[TemporalExpression]' = Field(...) - # , max_items=2, min_items=2) - - -class ValueExpression(RootModel): - root: 'Union[ScalarExpression, SpatialLiteral, TypedTemporalLiteral]' - - -class ArithmeticExpression(RootModel): - root: 'Union[AddExpression, SubExpression, MulExpression, DivExpression]' - - -class ArrayLiteral(RootModel): - root: 'List[\n Union[\n ScalarLiteral,\n SpatialLiteral,\n TypedTemporalLiteral,\n PropertyRef,\n FunctionRef,\n ArithmeticExpression,\n ArrayLiteral,\n ]\n ]' - - -class FunctionRef(BaseModel): - function: 'Function' - - -class GeomExpression(RootModel): - root: 'Union[SpatialLiteral, PropertyRef, FunctionRef]' - - -class TemporalExpression(RootModel): - root: 'Union[TemporalLiteral, PropertyRef, FunctionRef]' - - -# here -class AddExpression(BaseModel): - add_: 'ArithmeticOperands' = Field(..., alias='+') - - -# here -class DivExpression(BaseModel): - div_: 'Optional[ArithmeticOperands]' = Field(None, alias='/') - - -class Function(BaseModel): - name: 'str' - arguments: 'Optional[\n List[\n Union[\n ScalarLiteral,\n SpatialLiteral,\n TypedTemporalLiteral,\n PropertyRef,\n FunctionRef,\n ArithmeticExpression,\n ArrayLiteral,\n ]\n ]\n ]' = None - - -# here -class MulExpression(BaseModel): - mul_: 'ArithmeticOperands' = Field(..., alias='*') - - -# here -class SubExpression(BaseModel): - sub_: 'ArithmeticOperands' = Field(..., alias='-') - - -class ArithmeticOperands(RootModel): - root: 'List[\n Union[ArithmeticExpression, PropertyRef, FunctionRef, float]\n ]' = Field(...) - - -CQLModel.model_rebuild() -AndExpression.model_rebuild() -ArrayPredicate.model_rebuild() -ComparisonPredicate.model_rebuild() -SpatialPredicate.model_rebuild() -TemporalPredicate.model_rebuild() -AcontainedByExpression.model_rebuild() -AcontainsExpression.model_rebuild() -AequalsExpression.model_rebuild() -AfterExpression.model_rebuild() -AnyinteractsExpression.model_rebuild() -AoverlapsExpression.model_rebuild() -BeforeExpression.model_rebuild() -BeginsExpression.model_rebuild() -BegunbyExpression.model_rebuild() -BinaryComparisonPredicate.model_rebuild() -ContainsExpression.model_rebuild() -CrossesExpression.model_rebuild() -DisjointExpression.model_rebuild() -DuringExpression.model_rebuild() -EndedbyExpression.model_rebuild() -EndsExpression.model_rebuild() -EqualsExpression.model_rebuild() -IntersectsExpression.model_rebuild() -Between.model_rebuild() -In.model_rebuild() -IsBetweenPredicate.model_rebuild() -IsLikePredicate.model_rebuild() -IsNullPredicate.model_rebuild() -ValueExpression.model_rebuild() -MeetsExpression.model_rebuild() -MetbyExpression.model_rebuild() -OverlappedbyExpression.model_rebuild() -OverlapsExpression.model_rebuild() -TcontainsExpression.model_rebuild() -TequalsExpression.model_rebuild() -TouchesExpression.model_rebuild() -ToverlapsExpression.model_rebuild() -WithinExpression.model_rebuild() -ArrayExpression.model_rebuild() -EqExpression.model_rebuild() -GtExpression.model_rebuild() -GteExpression.model_rebuild() -LtExpression.model_rebuild() -LteExpression.model_rebuild() -ScalarExpression.model_rebuild() -ScalarOperands.model_rebuild() -SpatialOperands.model_rebuild() -TemporalOperands.model_rebuild() -ArithmeticExpression.model_rebuild() -ArrayLiteral.model_rebuild() -ScalarLiteral.model_rebuild() -PropertyRef.model_rebuild() -FunctionRef.model_rebuild() -AddExpression.model_rebuild() -DivExpression.model_rebuild() -MulExpression.model_rebuild() -SubExpression.model_rebuild() - - -def get_next_node(obj): - logical_op = None - if obj.__repr_name__() == 'AndExpression': - next_node = obj.and_ - logical_op = 'and' - elif obj.__repr_name__() == 'OrExpression': - next_node = obj.or_ - logical_op = 'or' - elif obj.__repr_name__() == 'NotExpression': - next_node = obj.not_ - logical_op = 'not' - elif obj.__repr_name__() == 'ComparisonPredicate': - next_node = obj.root - elif obj.__repr_name__() == 'SpatialPredicate': - next_node = obj.root - elif obj.__repr_name__() == 'TemporalPredicate': - next_node = obj.root - elif obj.__repr_name__() == 'IsBetweenPredicate': - next_node = obj.between - elif obj.__repr_name__() == 'Between': - next_node = obj.value - elif obj.__repr_name__() == 'ValueExpression': - next_node = obj.root or obj.lower or obj.upper - elif obj.__repr_name__() == 'ScalarExpression': - next_node = obj.root - elif obj.__repr_name__() == 'ScalarLiteral': - next_node = obj.root - elif obj.__repr_name__() == 'PropertyRef': - next_node = obj.property - elif obj.__repr_name__() == 'BinaryComparisonPredicate': - next_node = obj.root - elif obj.__repr_name__() == 'EqExpression': - next_node = obj.eq - logical_op = 'eq' - else: - raise ValueError("Object not valid") - - return (logical_op, next_node) diff --git a/pygeoapi/models/openapi.py b/pygeoapi/models/openapi.py index ffe587f04..d07923612 100644 --- a/pygeoapi/models/openapi.py +++ b/pygeoapi/models/openapi.py @@ -4,7 +4,7 @@ # # Authors: Francesco Bartoli # -# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2024 Francesco Bartoli # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -31,7 +31,7 @@ from enum import Enum -from pydantic import RootModel +from pydantic import BaseModel class SupportedFormats(Enum): @@ -39,5 +39,5 @@ class SupportedFormats(Enum): YAML = "yaml" -class OAPIFormat(RootModel): - root: SupportedFormats = SupportedFormats.YAML +class OAPIFormat(BaseModel): + __root__: SupportedFormats = SupportedFormats.YAML diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index d697df206..bd4fe1775 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -4,7 +4,7 @@ # Authors: Francesco Bartoli # Authors: Ricardo Garcia Silva # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # Copyright (c) 2022 Francesco Bartoli # Copyright (c) 2023 Ricardo Garcia Silva # @@ -44,12 +44,9 @@ import yaml from pygeoapi import l10n +from pygeoapi.api import all_apis from pygeoapi.models.openapi import OAPIFormat -from pygeoapi.plugin import load_plugin -from pygeoapi.process.manager.base import get_manager -from pygeoapi.provider.base import ProviderTypeError, SchemaType -from pygeoapi.util import (filter_dict_by_key_value, get_provider_by_type, - filter_providers_by_type, to_json, yaml_load, +from pygeoapi.util import (filter_dict_by_key_value, to_json, yaml_load, get_api_rules, get_base_url) LOGGER = logging.getLogger(__name__) @@ -68,7 +65,14 @@ THISDIR = os.path.dirname(os.path.realpath(__file__)) -def get_ogc_schemas_location(server_config): +def get_ogc_schemas_location(server_config: dict) -> str: + """ + Determine OGC schemas location + + :param server_config: `dict` of server configuration + + :returns: `str` of OGC schemas location + """ osl = server_config.get('ogc_schemas_location') @@ -85,7 +89,7 @@ def get_ogc_schemas_location(server_config): # TODO: remove this function once OGC API - Processing is final -def gen_media_type_object(media_type, api_type, path): +def gen_media_type_object(media_type: str, api_type: str, path: str) -> dict: """ Generates an OpenAPI Media Type Object @@ -110,7 +114,8 @@ def gen_media_type_object(media_type, api_type, path): # TODO: remove this function once OGC API - Processing is final -def gen_response_object(description, media_type, api_type, path): +def gen_response_object(description: str, media_type: str, + api_type: str, path: str) -> dict: """ Generates an OpenAPI Response Object @@ -129,20 +134,110 @@ def gen_response_object(description, media_type, api_type, path): return response -def get_oas_30(cfg): +def gen_contact(cfg: dict) -> dict: + """ + Generates an OpenAPI contact object with OGC extensions + based on OGC API - Records contact + + :param cfg: `dict` of configuration + + :returns: `dict` of OpenAPI contact object + """ + + has_addresses = False + has_phones = False + + contact = { + 'name': cfg['metadata']['provider']['name'] + } + + for key in ['url', 'email']: + if key in cfg['metadata']['provider']: + contact[key] = cfg['metadata']['provider'][key] + + contact['x-ogc-serviceContact'] = { + 'name': cfg['metadata']['contact']['name'], + 'addresses': [] + } + + if 'position' in cfg['metadata']['contact']: + contact['x-ogc-serviceContact']['position'] = cfg['metadata']['contact']['position'] # noqa + + if any(address in ['address', 'city', 'stateorprovince', 'postalcode', 'country'] for address in cfg['metadata']['contact']): # noqa + has_addresses = True + + if has_addresses: + address = {} + if 'address' in cfg['metadata']['contact']: + address['deliveryPoint'] = [cfg['metadata']['contact']['address']] + + if 'city' in cfg['metadata']['contact']: + address['city'] = cfg['metadata']['contact']['city'] + + if 'stateorprovince' in cfg['metadata']['contact']: + address['administrativeArea'] = cfg['metadata']['contact']['stateorprovince'] # noqa + + if 'postalCode' in cfg['metadata']['contact']: + address['administrativeArea'] = cfg['metadata']['contact']['postalCode'] # noqa + + if 'country' in cfg['metadata']['contact']: + address['administrativeArea'] = cfg['metadata']['contact']['country'] # noqa + + contact['x-ogc-serviceContact']['addresses'].append(address) + + if any(phone in ['phone', 'fax'] for phone in cfg['metadata']['contact']): + has_phones = True + contact['x-ogc-serviceContact']['phones'] = [] + + if has_phones: + if 'phone' in cfg['metadata']['contact']: + contact['x-ogc-serviceContact']['phones'].append({ + 'type': 'main', 'value': cfg['metadata']['contact']['phone'] + }) + + if 'fax' in cfg['metadata']['contact']: + contact['x-ogc-serviceContact']['phones'].append({ + 'type': 'fax', 'value': cfg['metadata']['contact']['fax'] + }) + + if 'email' in cfg['metadata']['contact']: + contact['x-ogc-serviceContact']['emails'] = [{ + 'value': cfg['metadata']['contact']['email'] + }] + + if 'url' in cfg['metadata']['contact']: + contact['x-ogc-serviceContact']['links'] = [{ + 'type': 'text/html', + 'href': cfg['metadata']['contact']['url'] + }] + + if 'instructions' in cfg['metadata']['contact']: + contact['x-ogc-serviceContact']['contactInstructions'] = cfg['metadata']['contact']['instructions'] # noqa + + if 'hours' in cfg['metadata']['contact']: + contact['x-ogc-serviceContact']['hoursOfService'] = cfg['metadata']['contact']['hours'] # noqa + + if 'role' in cfg['metadata']['contact']: + contact['x-ogc-serviceContact']['hoursOfService'] = cfg['metadata']['contact']['role'] # noqa + + return contact + + +def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: """ Generates an OpenAPI 3.0 Document :param cfg: configuration object + :param fail_on_invalid_collection: `bool` of whether to fail on an invalid + collection - :returns: OpenAPI definition YAML dict + :returns: dict of OpenAPI definition """ paths = {} # TODO: make openapi multilingual (default language only for now) - server_locales = l10n.get_locales(cfg) - locale_ = server_locales[0] + locale_ = l10n.get_locales(cfg)[0] api_rules = get_api_rules(cfg) @@ -161,11 +256,7 @@ def get_oas_30(cfg): 'x-keywords': l10n.translate(cfg['metadata']['identification']['keywords'], locale_), # noqa 'termsOfService': cfg['metadata']['identification']['terms_of_service'], - 'contact': { - 'name': cfg['metadata']['provider']['name'], - 'url': cfg['metadata']['provider']['url'], - 'email': cfg['metadata']['contact']['email'] - }, + 'contact': gen_contact(cfg), 'license': { 'name': cfg['metadata']['license']['name'], 'url': cfg['metadata']['license']['url'] @@ -272,11 +363,6 @@ def get_oas_30(cfg): 'url': cfg['metadata']['identification']['url']} } ) - oas['tags'].append({ - 'name': 'stac', - 'description': 'SpatioTemporal Asset Catalog' - } - ) oas['components'] = { 'responses': { @@ -299,149 +385,7 @@ def get_oas_30(cfg): } } }, - 'parameters': { - 'f': { - 'name': 'f', - 'in': 'query', - 'description': 'The optional f parameter indicates the output format which the server shall provide as part of the response document. The default format is GeoJSON.', # noqa - 'required': False, - 'schema': { - 'type': 'string', - 'enum': ['json', 'html', 'jsonld'], - 'default': 'json' - }, - 'style': 'form', - 'explode': False - }, - 'lang': { - 'name': 'lang', - 'in': 'query', - 'description': 'The optional lang parameter instructs the server return a response in a certain language, if supported. If the language is not among the available values, the Accept-Language header language will be used if it is supported. If the header is missing, the default server language is used. Note that providers may only support a single language (or often no language at all), that can be different from the server language. Language strings can be written in a complex (e.g. "fr-CA,fr;q=0.9,en-US;q=0.8,en;q=0.7"), simple (e.g. "de") or locale-like (e.g. "de-CH" or "fr_BE") fashion.', # noqa - 'required': False, - 'schema': { - 'type': 'string', - 'enum': [l10n.locale2str(sl) for sl in server_locales], - 'default': l10n.locale2str(locale_) - } - }, - 'properties': { - 'name': 'properties', - 'in': 'query', - 'description': 'The properties that should be included for each feature. The parameter value is a comma-separated list of property names.', # noqa - 'required': False, - 'style': 'form', - 'explode': False, - 'schema': { - 'type': 'array', - 'items': { - 'type': 'string' - } - } - }, - 'skipGeometry': { - 'name': 'skipGeometry', - 'in': 'query', - 'description': 'This option can be used to skip response geometries for each feature.', # noqa - 'required': False, - 'style': 'form', - 'explode': False, - 'schema': { - 'type': 'boolean', - 'default': False - } - }, - 'crs': { - 'name': 'crs', - 'in': 'query', - 'description': 'Indicates the coordinate reference system for the results.', # noqa - 'style': 'form', - 'required': False, - 'explode': False, - 'schema': { - 'format': 'uri', - 'type': 'string' - } - }, - 'bbox': { - 'name': 'bbox', - 'in': 'query', - 'description': 'Only features that have a geometry that intersects the bounding box are selected.' # noqa - 'The bounding box is provided as four or six numbers, depending on whether the ' # noqa - 'coordinate reference system includes a vertical axis (height or depth).', # noqa - 'required': False, - 'style': 'form', - 'explode': False, - 'schema': { - 'type': 'array', - 'minItems': 4, - 'maxItems': 6, - 'items': { - 'type': 'number' - } - } - }, - 'bbox-crs': { - 'name': 'bbox-crs', - 'in': 'query', - 'description': 'Indicates the coordinate reference system for the given bbox coordinates.', # noqa - 'style': 'form', - 'required': False, - 'explode': False, - 'schema': { - 'format': 'uri', - 'type': 'string' - } - }, - # FIXME: This is not compatible with the bbox-crs definition in - # OGCAPI Features Part 2! - # We need to change the mapscript provider and - # get_collection_map() method in the API! - # So this is for de map-provider only. - 'bbox-crs-epsg': { - 'name': 'bbox-crs', - 'in': 'query', - 'description': 'Indicates the EPSG for the given bbox coordinates.', # noqa - 'required': False, - 'style': 'form', - 'explode': False, - 'schema': { - 'type': 'integer', - 'default': 4326 - } - }, - 'offset': { - 'name': 'offset', - 'in': 'query', - 'description': 'The optional offset parameter indicates the index within the result set from which the server shall begin presenting results in the response document. The first element has an index of 0 (default).', # noqa - 'required': False, - 'schema': { - 'type': 'integer', - 'minimum': 0, - 'default': 0 - }, - 'style': 'form', - 'explode': False - }, - 'vendorSpecificParameters': { - 'name': 'vendorSpecificParameters', - 'in': 'query', - 'description': 'Additional "free-form" parameters that are not explicitly defined', # noqa - 'schema': { - 'type': 'object', - 'additionalProperties': True - }, - 'style': 'form' - }, - 'resourceId': { - 'name': 'resourceId', - 'in': 'path', - 'description': 'Configuration resource identifier', - 'required': True, - 'schema': { - 'type': 'string' - } - } - }, + 'parameters': get_oas_30_parameters(cfg=cfg, locale_=locale_), 'schemas': { # TODO: change this schema once OGC will definitively publish it 'queryable': { @@ -498,16 +442,10 @@ def get_oas_30(cfg): items_f = deepcopy(oas['components']['parameters']['f']) items_f['schema']['enum'].append('csv') - items_l = deepcopy(oas['components']['parameters']['lang']) LOGGER.debug('setting up datasets') - collections = filter_dict_by_key_value(cfg['resources'], - 'type', 'collection') - for k, v in collections.items(): - if v.get('visibility', 'default') == 'hidden': - LOGGER.debug(f'Skipping hidden layer: {k}') - continue + for k, v in get_visible_collections(cfg).items(): name = l10n.translate(k, locale_) title = l10n.translate(v['title'], locale_) desc = l10n.translate(v['description'], locale_) @@ -546,782 +484,227 @@ def get_oas_30(cfg): } } - LOGGER.debug('setting up collection endpoints') - try: - ptype = None - - if filter_providers_by_type( - collections[k]['providers'], 'feature'): - ptype = 'feature' - - if filter_providers_by_type( - collections[k]['providers'], 'record'): - ptype = 'record' - - p = load_plugin('provider', get_provider_by_type( - collections[k]['providers'], ptype)) - - items_path = f'{collection_name_path}/items' - - coll_properties = deepcopy(oas['components']['parameters']['properties']) # noqa - - coll_properties['schema']['items']['enum'] = list(p.fields.keys()) - - paths[items_path] = { - 'get': { - 'summary': f'Get {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Features', - 'parameters': [ - items_f, - items_l, - {'$ref': '#/components/parameters/bbox'}, - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/limit"}, # noqa - {'$ref': '#/components/parameters/crs'}, # noqa - {'$ref': '#/components/parameters/bbox-crs'}, # noqa - coll_properties, - {'$ref': '#/components/parameters/vendorSpecificParameters'}, # noqa - {'$ref': '#/components/parameters/skipGeometry'}, - {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/sortby.yaml"}, # noqa - {'$ref': '#/components/parameters/offset'}, - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - }, - 'options': { - 'summary': f'Options for {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'options{name.capitalize()}Features', - 'responses': { - '200': {'description': 'options response'} - } - } - } - - if p.editable: - LOGGER.debug('Provider is editable; adding post') - - paths[items_path]['post'] = { - 'summary': f'Add {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'add{name.capitalize()}Features', - 'requestBody': { - 'description': 'Adds item to collection', - 'content': { - 'application/geo+json': { - 'schema': {} + oas['components']['responses'].update({ + 'Tiles': { + 'description': 'Retrieves the tiles description for this collection', # noqa + 'content': { + 'application/json': { + 'schema': { + '$ref': '#/components/schemas/tiles' } - }, - 'required': True - }, - 'responses': { - '201': {'description': 'Successful creation'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - - try: - schema_ref = p.get_schema(SchemaType.create) - paths[items_path]['post']['requestBody']['content'][schema_ref[0]] = { # noqa - 'schema': schema_ref[1] - } - except Exception as err: - LOGGER.debug(err) - - if ptype == 'record': - paths[items_path]['get']['parameters'].append( - {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/q.yaml"}) - if p.fields: - queryables_path = f'{collection_name_path}/queryables' - - paths[queryables_path] = { - 'get': { - 'summary': f'Get {title} queryables', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Queryables', - 'parameters': [ - items_f, - items_l - ], - 'responses': { - '200': {'$ref': '#/components/responses/Queryables'}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa } } } - - if p.time_field is not None: - paths[items_path]['get']['parameters'].append( - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa - - for field, type_ in p.fields.items(): - - if p.properties and field not in p.properties: - LOGGER.debug('Provider specified not to advertise property') # noqa - continue - - if field == 'q' and ptype == 'record': - LOGGER.debug('q parameter already declared, skipping') - continue - - if type_ == 'date': - schema = { - 'type': 'string', - 'format': 'date' - } - elif type_ == 'float': - schema = { - 'type': 'number', - 'format': 'float' - } - elif type_ == 'long': - schema = { - 'type': 'integer', - 'format': 'int64' - } - else: - schema = type_ - - path_ = f'{collection_name_path}/items' - paths[path_]['get']['parameters'].append({ - 'name': field, - 'in': 'query', - 'required': False, - 'schema': schema, - 'style': 'form', - 'explode': False - }) - - paths[f'{collection_name_path}/items/{{featureId}}'] = { - 'get': { - 'summary': f'Get {title} item by id', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Feature', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa - {'$ref': '#/components/parameters/crs'}, # noqa - {'$ref': '#/components/parameters/f'}, - {'$ref': '#/components/parameters/lang'} - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Feature"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - }, - 'options': { - 'summary': f'Options for {title} item by id', - 'description': desc, - 'tags': [name], - 'operationId': f'options{name.capitalize()}Feature', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa - ], - 'responses': { - '200': {'description': 'options response'} - } - } } + ) - try: - schema_ref = p.get_schema() - paths[f'{collection_name_path}/items/{{featureId}}']['get']['responses']['200'] = { # noqa - 'content': { - schema_ref[0]: { - 'schema': schema_ref[1] + oas['components']['schemas'].update({ + 'tilematrixsetlink': { + 'type': 'object', + 'required': ['tileMatrixSet'], + 'properties': { + 'tileMatrixSet': { + 'type': 'string' + }, + 'tileMatrixSetURI': { + 'type': 'string' } } - } - except Exception as err: - LOGGER.debug(err) - - if p.editable: - LOGGER.debug('Provider is editable; adding put/delete') - put_path = f'{collection_name_path}/items/{{featureId}}' # noqa - paths[put_path]['put'] = { # noqa - 'summary': f'Update {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'update{name.capitalize()}Features', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa + }, + 'tiles': { + 'type': 'object', + 'required': [ + 'tileMatrixSetLinks', + 'links' ], - 'requestBody': { - 'description': 'Updates item in collection', - 'content': { - 'application/geo+json': { - 'schema': {} + 'properties': { + 'tileMatrixSetLinks': { + 'type': 'array', + 'items': { + '$ref': '#/components/schemas/tilematrixsetlink' # noqa } }, - 'required': True - }, - 'responses': { - '204': {'$ref': '#/components/responses/204'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + 'links': { + 'type': 'array', + 'items': {'$ref': f"{OPENAPI_YAML['oapit']}#/components/schemas/link"} # noqa + } } } + } + ) - try: - schema_ref = p.get_schema(SchemaType.replace) - paths[put_path]['put']['requestBody']['content'][schema_ref[0]] = { # noqa - 'schema': schema_ref[1] - } - except Exception as err: - LOGGER.debug(err) - - paths[f'{collection_name_path}/items/{{featureId}}']['delete'] = { # noqa - 'summary': f'Delete {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'delete{name.capitalize()}Features', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa - ], - 'responses': { - '200': {'description': 'Successful delete'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } + oas['paths'] = paths - except ProviderTypeError: - LOGGER.debug('collection is not feature based') + for api_name, api_module in all_apis().items(): + LOGGER.debug(f'Adding OpenAPI definitions for {api_name}') - LOGGER.debug('setting up coverage endpoints') try: - load_plugin('provider', get_provider_by_type( - collections[k]['providers'], 'coverage')) - - coverage_path = f'{collection_name_path}/coverage' - - paths[coverage_path] = { - 'get': { - 'summary': f'Get {title} coverage', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Coverage', - 'parameters': [ - items_f, - items_l, - {'$ref': '#/components/parameters/bbox'}, - {'$ref': '#/components/parameters/bbox-crs'}, # noqa - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - - coverage_domainset_path = f'{collection_name_path}/coverage/domainset' # noqa - - paths[coverage_domainset_path] = { - 'get': { - 'summary': f'Get {title} coverage domain set', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}CoverageDomainSet', - 'parameters': [ - items_f, - items_l - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oacov']}/schemas/cis_1.1/domainSet.yaml"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } + sub_tags, sub_paths = api_module.get_oas_30(cfg, locale_) + oas['paths'].update(sub_paths['paths']) + oas['tags'].extend(sub_tags) + except Exception as err: + if fail_on_invalid_collection: + raise + else: + LOGGER.warning(f'Resource not added to OpenAPI: {err}') - coverage_rangetype_path = f'{collection_name_path}/coverage/rangetype' # noqa - - paths[coverage_rangetype_path] = { - 'get': { - 'summary': f'Get {title} coverage range type', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}CoverageRangeType', - 'parameters': [ - items_f, - items_l - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oacov']}/schemas/cis_1.1/rangeType.yaml"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - except ProviderTypeError: - LOGGER.debug('collection is not coverage based') - - LOGGER.debug('setting up tiles endpoints') - tile_extension = filter_providers_by_type( - collections[k]['providers'], 'tile') - - if tile_extension: - tp = load_plugin('provider', tile_extension) - oas['components']['responses'].update({ - 'Tiles': { - 'description': 'Retrieves the tiles description for this collection', # noqa - 'content': { - 'application/json': { - 'schema': { - '$ref': '#/components/schemas/tiles' - } - } - } - } - } - ) - - oas['components']['schemas'].update({ - 'tilematrixsetlink': { - 'type': 'object', - 'required': ['tileMatrixSet'], - 'properties': { - 'tileMatrixSet': { - 'type': 'string' - }, - 'tileMatrixSetURI': { - 'type': 'string' - } - } - }, - 'tiles': { - 'type': 'object', - 'required': [ - 'tileMatrixSetLinks', - 'links' - ], - 'properties': { - 'tileMatrixSetLinks': { - 'type': 'array', - 'items': { - '$ref': '#/components/schemas/tilematrixsetlink' # noqa - } - }, - 'links': { - 'type': 'array', - 'items': {'$ref': f"{OPENAPI_YAML['oapit']}#/components/schemas/link"} # noqa - } - } - } - } - ) - - tiles_path = f'{collection_name_path}/tiles' - - paths[tiles_path] = { - 'get': { - 'summary': f'Fetch a {title} tiles description', - 'description': desc, - 'tags': [name], - 'operationId': f'describe{name.capitalize()}Tiles', - 'parameters': [ - items_f, - # items_l TODO: is this useful? - ], - 'responses': { - '200': {'$ref': '#/components/responses/Tiles'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } + if cfg['server'].get('admin', False): + schema_dict = get_config_schema() + oas['definitions'] = schema_dict['definitions'] + LOGGER.debug('Adding admin endpoints') + oas['paths'].update(get_admin(cfg)) - tiles_data_path = f'{collection_name_path}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}' # noqa - - paths[tiles_data_path] = { - 'get': { - 'summary': f'Get a {title} tile', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Tiles', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrixSetId"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrix"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileRow"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileCol"}, # noqa - { - 'name': 'f', - 'in': 'query', - 'description': 'The optional f parameter indicates the output format which the server shall provide as part of the response document.', # noqa - 'required': False, - 'schema': { - 'type': 'string', - 'enum': [tp.format_type], - 'default': tp.format_type - }, - 'style': 'form', - 'explode': False - } - ], - 'responses': { - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - mimetype = tile_extension['format']['mimetype'] - paths[tiles_data_path]['get']['responses']['200'] = { - 'description': 'successful operation', - 'content': { - mimetype: { - 'schema': { - 'type': 'string', - 'format': 'binary' - } - } - } - } + return oas - LOGGER.debug('setting up edr endpoints') - edr_extension = filter_providers_by_type( - collections[k]['providers'], 'edr') - - if edr_extension: - ep = load_plugin('provider', edr_extension) - - edr_query_endpoints = [] - - for qt in ep.get_query_types(): - edr_query_endpoints.append({ - 'path': f'{collection_name_path}/{qt}', - 'qt': qt, - 'op_id': f'query{qt.capitalize()}{k.capitalize()}' - }) - if ep.instances: - edr_query_endpoints.append({ - 'path': f'{collection_name_path}/instances/{{instanceId}}/{qt}', # noqa - 'qt': qt, - 'op_id': f'query{qt.capitalize()}Instance{k.capitalize()}' # noqa - }) - - for eqe in edr_query_endpoints: - if eqe['qt'] == 'cube': - spatial_parameter = 'bbox' - else: - spatial_parameter = f"{eqe['qt']}Coords" - paths[eqe['path']] = { - 'get': { - 'summary': f"query {v['description']} by {eqe['qt']}", # noqa - 'description': v['description'], - 'tags': [k], - 'operationId': eqe['op_id'], - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{spatial_parameter}.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"}, # noqa - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': { - 'description': 'Response', - 'content': { - 'application/prs.coverage+json': { - 'schema': { - '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa - } - } - } - } - } - } - } - LOGGER.debug('setting up maps endpoints') - map_extension = filter_providers_by_type( - collections[k]['providers'], 'map') - - if map_extension: - mp = load_plugin('provider', map_extension) - - map_f = deepcopy(oas['components']['parameters']['f']) - map_f['schema']['enum'] = [map_extension['format']['name']] - map_f['schema']['default'] = map_extension['format']['name'] - - pth = f'/collections/{k}/map' - paths[pth] = { - 'get': { - 'summary': 'Get map', - 'description': f"{v['description']} map", - 'tags': [k], - 'operationId': 'getMap', - 'parameters': [ - {'$ref': '#/components/parameters/bbox'}, - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa - { - 'name': 'width', - 'in': 'query', - 'description': 'Response image width', - 'required': False, - 'schema': { - 'type': 'integer', - }, - 'style': 'form', - 'explode': False - }, - { - 'name': 'height', - 'in': 'query', - 'description': 'Response image height', - 'required': False, - 'schema': { - 'type': 'integer', - }, - 'style': 'form', - 'explode': False - }, - { - 'name': 'transparent', - 'in': 'query', - 'description': 'Background transparency of map (default=true).', # noqa - 'required': False, - 'schema': { - 'type': 'boolean', - 'default': True, - }, - 'style': 'form', - 'explode': False - }, - {'$ref': '#/components/parameters/bbox-crs-epsg'}, - map_f - ], - 'responses': { - '200': { - 'description': 'Response', - 'content': { - 'application/json': {} - } - }, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa - } - } - } - if mp.time_field is not None: - paths[pth]['get']['parameters'].append( - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa - - LOGGER.debug('setting up STAC') - stac_collections = filter_dict_by_key_value(cfg['resources'], - 'type', 'stac-collection') - if stac_collections: - paths['/stac'] = { - 'get': { - 'summary': 'SpatioTemporal Asset Catalog', - 'description': 'SpatioTemporal Asset Catalog', - 'tags': ['stac'], - 'operationId': 'getStacCatalog', - 'parameters': [], - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - 'default': {'$ref': '#/components/responses/default'} - } - } - } - - process_manager = get_manager(cfg) +def get_oas_30_parameters(cfg: dict, locale_: str): + server_locales = l10n.get_locales(cfg) - if len(process_manager.processes) > 0: - paths['/processes'] = { - 'get': { - 'summary': 'Processes', - 'description': 'Processes', - 'tags': ['server'], - 'operationId': 'getProcesses', - 'parameters': [ - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ProcessList.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} - } + oas_30_parameters = { + 'f': { + 'name': 'f', + 'in': 'query', + 'description': 'The optional f parameter indicates the output format which the server shall provide as part of the response document. The default format is GeoJSON.', # noqa + 'required': False, + 'schema': { + 'type': 'string', + 'enum': ['json', 'html', 'jsonld'], + 'default': 'json' + }, + 'style': 'form', + 'explode': False + }, + 'lang': { + 'name': 'lang', + 'in': 'query', + 'description': 'The optional lang parameter instructs the server return a response in a certain language, if supported. If the language is not among the available values, the Accept-Language header language will be used if it is supported. If the header is missing, the default server language is used. Note that providers may only support a single language (or often no language at all), that can be different from the server language. Language strings can be written in a complex (e.g. "fr-CA,fr;q=0.9,en-US;q=0.8,en;q=0.7"), simple (e.g. "de") or locale-like (e.g. "de-CH" or "fr_BE") fashion.', # noqa + 'required': False, + 'schema': { + 'type': 'string', + 'enum': [l10n.locale2str(sl) for sl in server_locales], + 'default': l10n.locale2str(locale_) } - } - LOGGER.debug('setting up processes') - - for k, v in process_manager.processes.items(): - if k.startswith('_'): - LOGGER.debug(f'Skipping hidden layer: {k}') - continue - name = l10n.translate(k, locale_) - p = process_manager.get_processor(k) - md_desc = l10n.translate(p.metadata['description'], locale_) - process_name_path = f'/processes/{name}' - tag = { - 'name': name, - 'description': md_desc, # noqa - 'externalDocs': {} + }, + 'skipGeometry': { + 'name': 'skipGeometry', + 'in': 'query', + 'description': 'This option can be used to skip response geometries for each feature.', # noqa + 'required': False, + 'style': 'form', + 'explode': False, + 'schema': { + 'type': 'boolean', + 'default': False } - for link in p.metadata.get('links', []): - if link['type'] == 'information': - translated_link = l10n.translate(link, locale_) - tag['externalDocs']['description'] = translated_link[ - 'type'] - tag['externalDocs']['url'] = translated_link['url'] - break - if len(tag['externalDocs']) == 0: - del tag['externalDocs'] - - oas['tags'].append(tag) - - paths[process_name_path] = { - 'get': { - 'summary': 'Get process metadata', - 'description': md_desc, - 'tags': [name], - 'operationId': f'describe{name.capitalize()}Process', - 'parameters': [ - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - 'default': {'$ref': '#/components/responses/default'} - } - } + }, + 'crs': { + 'name': 'crs', + 'in': 'query', + 'description': 'Indicates the coordinate reference system for the results.', # noqa + 'style': 'form', + 'required': False, + 'explode': False, + 'schema': { + 'format': 'uri', + 'type': 'string' } - - paths[f'{process_name_path}/execution'] = { - 'post': { - 'summary': f"Process {l10n.translate(p.metadata['title'], locale_)} execution", # noqa - 'description': md_desc, - 'tags': [name], - 'operationId': f'execute{name.capitalize()}Job', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '201': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ExecuteAsync.yaml"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ServerError.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} - }, - 'requestBody': { - 'description': 'Mandatory execute request JSON', - 'required': True, - 'content': { - 'application/json': { - 'schema': { - '$ref': f"{OPENAPI_YAML['oapip']}/schemas/execute.yaml" # noqa - } - } - } - } + }, + 'bbox': { + 'name': 'bbox', + 'in': 'query', + 'description': 'Only features that have a geometry that intersects the bounding box are selected.' # noqa + 'The bounding box is provided as four or six numbers, depending on whether the ' # noqa + 'coordinate reference system includes a vertical axis (height or depth).', # noqa + 'required': False, + 'style': 'form', + 'explode': False, + 'schema': { + 'type': 'array', + 'minItems': 4, + 'maxItems': 6, + 'items': { + 'type': 'number' } } - if 'example' in p.metadata: - paths[f'{process_name_path}/execution']['post']['requestBody']['content']['application/json']['example'] = p.metadata['example'] # noqa - - name_in_path = { - 'name': 'jobId', - 'in': 'path', - 'description': 'job identifier', - 'required': True, - 'schema': { - 'type': 'string' - } + }, + 'bbox-crs': { + 'name': 'bbox-crs', + 'in': 'query', + 'description': 'Indicates the coordinate reference system for the given bbox coordinates.', # noqa + 'style': 'form', + 'required': False, + 'explode': False, + 'schema': { + 'format': 'uri', + 'type': 'string' } - - paths['/jobs'] = { - 'get': { - 'summary': 'Retrieve jobs list', - 'description': 'Retrieve a list of jobs', - 'tags': ['jobs'], - 'operationId': 'getJobs', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} - } + }, + # FIXME: This is not compatible with the bbox-crs definition in + # OGCAPI Features Part 2! + # We need to change the mapscript provider and + # get_collection_map() method in the API! + # So this is for de map-provider only. + 'bbox-crs-epsg': { + 'name': 'bbox-crs', + 'in': 'query', + 'description': 'Indicates the EPSG for the given bbox coordinates.', # noqa + 'required': False, + 'style': 'form', + 'explode': False, + 'schema': { + 'type': 'integer', + 'default': 4326 } - } - - paths['/jobs/{jobId}'] = { - 'get': { - 'summary': 'Retrieve job details', - 'description': 'Retrieve job details', - 'tags': ['jobs'], - 'parameters': [ - name_in_path, - {'$ref': '#/components/parameters/f'} - ], - 'operationId': 'getJob', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } + }, + 'offset': { + 'name': 'offset', + 'in': 'query', + 'description': 'The optional offset parameter indicates the index within the result set from which the server shall begin presenting results in the response document. The first element has an index of 0 (default).', # noqa + 'required': False, + 'schema': { + 'type': 'integer', + 'minimum': 0, + 'default': 0 }, - 'delete': { - 'summary': 'Cancel / delete job', - 'description': 'Cancel / delete job', - 'tags': ['jobs'], - 'parameters': [ - name_in_path - ], - 'operationId': 'deleteJob', - 'responses': { - '204': {'$ref': '#/components/responses/204'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } + 'style': 'form', + 'explode': False + }, + 'vendorSpecificParameters': { + 'name': 'vendorSpecificParameters', + 'in': 'query', + 'description': 'Additional "free-form" parameters that are not explicitly defined', # noqa + 'schema': { + 'type': 'object', + 'additionalProperties': True }, + 'style': 'form' + }, + 'resourceId': { + 'name': 'resourceId', + 'in': 'path', + 'description': 'Configuration resource identifier', + 'required': True, + 'schema': { + 'type': 'string' + } } + } + if len(list(cfg['resources'].keys())) > 0: + oas_30_parameters['resourceId']['schema']['default'] = list( + cfg['resources'].keys() + )[0] + return oas_30_parameters - paths['/jobs/{jobId}/results'] = { - 'get': { - 'summary': 'Retrieve job results', - 'description': 'Retrive job resiults', - 'tags': ['jobs'], - 'parameters': [ - name_in_path, - {'$ref': '#/components/parameters/f'} - ], - 'operationId': 'getJobResults', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } - } - } - - tag = { - 'name': 'jobs', - 'description': 'Process jobs', - } - oas['tags'].insert(1, tag) - - oas['paths'] = paths - if cfg['server'].get('admin', False): - schema_dict = get_config_schema() - oas['definitions'] = schema_dict['definitions'] - LOGGER.debug('Adding admin endpoints') - oas['paths'].update(get_admin()) +def get_visible_collections(cfg: dict) -> dict: + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') - return oas + return { + k: v + for k, v in collections.items() + if v.get('visibility', 'default') != 'hidden' + } def get_config_schema(): @@ -1332,12 +715,25 @@ def get_config_schema(): return yaml_load(fh2) -def get_admin(): +def get_admin(cfg: dict) -> dict: schema_dict = get_config_schema() paths = {} + res_eg_key = next(iter(cfg['resources'])) + res_eg = { + res_eg_key: cfg['resources'][res_eg_key] + } + if 'extents' in res_eg[res_eg_key]: + res_eg_eg_key = 'extents' + elif 'type' in res_eg[res_eg_key]: + res_eg_eg_key = 'type' + + res_eg[res_eg_key]['patch_example'] = { + res_eg_eg_key: res_eg[res_eg_key][res_eg_eg_key] + } + paths['/admin/config'] = { 'get': { 'summary': 'Get admin configuration', @@ -1350,6 +746,7 @@ def get_admin(): ], 'responses': { '200': { + 'description': 'Successful response', 'content': { 'application/json': { 'schema': schema_dict @@ -1367,6 +764,7 @@ def get_admin(): 'description': 'Updates admin configuration', 'content': { 'application/json': { + 'example': cfg, 'schema': schema_dict } }, @@ -1387,6 +785,7 @@ def get_admin(): 'description': 'Updates admin configuration', 'content': { 'application/json': { + 'example': {'metadata': cfg['metadata']}, 'schema': schema_dict } }, @@ -1411,6 +810,7 @@ def get_admin(): ], 'responses': { '200': { + 'description': 'Successful response', 'content': { 'application/json': { 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa @@ -1428,6 +828,7 @@ def get_admin(): 'description': 'Adds resource to configuration', 'content': { 'application/json': { + 'example': {'new-collection': cfg['resources'][res_eg_key]}, # noqa 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa } }, @@ -1453,6 +854,7 @@ def get_admin(): ], 'responses': { '200': { + 'description': 'Successful response', 'content': { 'application/json': { 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa @@ -1473,6 +875,7 @@ def get_admin(): 'description': 'Updates admin configuration resource', 'content': { 'application/json': { + 'example': res_eg[res_eg_key], 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa } }, @@ -1496,6 +899,7 @@ def get_admin(): 'description': 'Updates admin configuration resource', 'content': { 'application/json': { + 'example': res_eg[res_eg_key]['patch_example'], 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa } }, @@ -1526,23 +930,27 @@ def get_admin(): return paths -def get_oas(cfg, version='3.0'): +def get_oas(cfg: dict, fail_on_invalid_collection: bool = True, + version='3.0') -> dict: """ Stub to generate OpenAPI Document - :param cfg: configuration object + :param cfg: `dict` configuration + :param fail_on_invalid_collection: `bool` of whether to fail on an + invalid collection :param version: version of OpenAPI (default 3.0) - :returns: OpenAPI definition YAML dict + :returns: `dict` of OpenAPI definition """ if version == '3.0': - return get_oas_30(cfg) + return get_oas_30( + cfg, fail_on_invalid_collection=fail_on_invalid_collection) else: raise RuntimeError('OpenAPI version not supported') -def validate_openapi_document(instance_dict): +def validate_openapi_document(instance_dict: dict) -> bool: """ Validate an OpenAPI document against the OpenAPI schema @@ -1562,27 +970,36 @@ def validate_openapi_document(instance_dict): def generate_openapi_document(cfg_file: Union[Path, io.TextIOWrapper], - output_format: OAPIFormat): + output_format: OAPIFormat, + fail_on_invalid_collection: bool = True) -> str: """ Generate an OpenAPI document from the configuration file - :param cfg_file: configuration Path instance + :param cfg_file: configuration Path instance (`str` of filepath + or parsed `dict`) :param output_format: output format for OpenAPI document + :param fail_on_invalid_collection: `bool` of whether to fail on an + invalid collection - :returns: content of the OpenAPI document in the output - format requested + :returns: `str` of the OpenAPI document in the output format requested """ + + LOGGER.debug(f'Loading configuration {cfg_file}') + if isinstance(cfg_file, Path): with cfg_file.open(mode="r") as cf: s = yaml_load(cf) else: s = yaml_load(cfg_file) + pretty_print = s['server'].get('pretty_print', False) + oas = get_oas(s, fail_on_invalid_collection=fail_on_invalid_collection) + if output_format == 'yaml': - content = yaml.safe_dump(get_oas(s), default_flow_style=False) + content = yaml.safe_dump(oas, default_flow_style=False) else: - content = to_json(get_oas(s), pretty=pretty_print) + content = to_json(oas, pretty=pretty_print) return content @@ -1595,6 +1012,17 @@ def load_openapi_document() -> dict: pygeoapi_openapi = os.environ.get('PYGEOAPI_OPENAPI') + if pygeoapi_openapi is None: + msg = 'PYGEOAPI_OPENAPI environment not set' + LOGGER.error(msg) + raise RuntimeError(msg) + + if not os.path.exists(pygeoapi_openapi): + msg = (f'OpenAPI document {pygeoapi_openapi} does not exist. ' + 'Please generate before starting pygeoapi') + LOGGER.error(msg) + raise RuntimeError(msg) + with open(pygeoapi_openapi, encoding='utf8') as ff: if pygeoapi_openapi.endswith(('.yaml', '.yml')): openapi_ = yaml_load(ff) @@ -1613,17 +1041,21 @@ def openapi(): @click.command() @click.pass_context @click.argument('config_file', type=click.File(encoding='utf-8')) +@click.option('--fail-on-invalid-collection/--no-fail-on-invalid-collection', + '-fic', default=True, help='Fail on invalid collection') @click.option('--format', '-f', 'format_', type=click.Choice(['json', 'yaml']), default='yaml', help='output format (json|yaml)') @click.option('--output-file', '-of', type=click.File('w', encoding='utf-8'), help='Name of output file') -def generate(ctx, config_file, output_file, format_='yaml'): +def generate(ctx, config_file, output_file, format_='yaml', + fail_on_invalid_collection=True): """Generate OpenAPI Document""" if config_file is None: raise click.ClickException('--config/-c required') - content = generate_openapi_document(config_file, format_) + content = generate_openapi_document( + config_file, format_, fail_on_invalid_collection) if output_file is None: click.echo(content) diff --git a/pygeoapi/plugin.py b/pygeoapi/plugin.py index 312017a12..08324cf64 100644 --- a/pygeoapi/plugin.py +++ b/pygeoapi/plugin.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -51,16 +51,22 @@ 'MapScript': 'pygeoapi.provider.mapscript_.MapScriptProvider', 'MongoDB': 'pygeoapi.provider.mongo.MongoProvider', 'MVT-tippecanoe': 'pygeoapi.provider.mvt_tippecanoe.MVTTippecanoeProvider', # noqa: E501 - 'MVT-elastic': 'pygeoapi.provider.mvt_elastic.MVTElasticProvider', # noqa: E501 + 'MVT-elastic': 'pygeoapi.provider.mvt_elastic.MVTElasticProvider', + 'MVT-proxy': 'pygeoapi.provider.mvt_proxy.MVTProxyProvider', 'OracleDB': 'pygeoapi.provider.oracle.OracleProvider', 'OGR': 'pygeoapi.provider.ogr.OGRProvider', + 'OpenSearch': 'pygeoapi.provider.opensearch_.OpenSearchProvider', + 'Parquet': 'pygeoapi.provider.parquet.ParquetProvider', 'PostgreSQL': 'pygeoapi.provider.postgresql.PostgreSQLProvider', 'rasterio': 'pygeoapi.provider.rasterio_.RasterioProvider', 'SensorThings': 'pygeoapi.provider.sensorthings.SensorThingsProvider', + 'SensorThingsEDR': 'pygeoapi.provider.sensorthings_edr.SensorThingsEDRProvider', # noqa: E501 'SQLiteGPKG': 'pygeoapi.provider.sqlite.SQLiteGPKGProvider', 'Socrata': 'pygeoapi.provider.socrata.SODAServiceProvider', + 'TinyDB': 'pygeoapi.provider.tinydb_.TinyDBProvider', 'TinyDBCatalogue': 'pygeoapi.provider.tinydb_.TinyDBCatalogueProvider', 'WMSFacade': 'pygeoapi.provider.wms_facade.WMSFacadeProvider', + 'WMTSFacade': 'pygeoapi.provider.wmts_facade.WMTSFacadeProvider', 'xarray': 'pygeoapi.provider.xarray_.XarrayProvider', 'xarray-edr': 'pygeoapi.provider.xarray_edr.XarrayEDRProvider' }, @@ -68,12 +74,15 @@ 'CSV': 'pygeoapi.formatter.csv_.CSVFormatter' }, 'process': { - 'HelloWorld': 'pygeoapi.process.hello_world.HelloWorldProcessor' + 'HelloWorld': 'pygeoapi.process.hello_world.HelloWorldProcessor', + 'ShapelyFunctions': 'pygeoapi.process.shapely_functions.ShapelyFunctionsProcessor', # noqa: E501 + 'Echo': 'pygeoapi.process.echo.EchoProcessor' }, 'process_manager': { 'Dummy': 'pygeoapi.process.manager.dummy.DummyManager', 'MongoDB': 'pygeoapi.process.manager.mongodb_.MongoDBManager', - 'TinyDB': 'pygeoapi.process.manager.tinydb_.TinyDBManager' + 'TinyDB': 'pygeoapi.process.manager.tinydb_.TinyDBManager', + 'PostgreSQL': 'pygeoapi.process.manager.postgresql.PostgreSQLManager' } } diff --git a/pygeoapi/process/base.py b/pygeoapi/process/base.py index 2403d1712..87c05a3cd 100644 --- a/pygeoapi/process/base.py +++ b/pygeoapi/process/base.py @@ -1,8 +1,10 @@ # ================================================================= # # Authors: Tom Kralidis +# Francesco Martinelli # # Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2024 Francesco Martinelli # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -28,7 +30,7 @@ # ================================================================= import logging -from typing import Any, Tuple +from typing import Any, Tuple, Optional from pygeoapi.error import GenericError @@ -47,17 +49,40 @@ def __init__(self, processor_def: dict, process_metadata: dict): :returns: pygeoapi.processor.base.BaseProvider """ + self.name = processor_def['name'] self.metadata = process_metadata + self.supports_outputs = False + + def set_job_id(self, job_id: str) -> None: + """ + Set the job_id within the processor + To be implemented by derived classes where required. + + :param job_id: the job_id assigned to the request by the Manager. + The function should be called by the Manager upon + assigning the job_id. The job_id is intended to be used + by derived classes, e.g. to write temporary files where + filenames contains the string job_id. - def execute(self, data: dict) -> Tuple[str, Any]: + :returns: `None` + """ + + pass + + def execute(self, data: dict, outputs: Optional[dict] = None + ) -> Tuple[str, Any]: """ execute the process :param data: Dict with the input data that the process needs in order to execute - + :param outputs: `dict` or `list` to optionally specify the subset of + required outputs - defaults to all outputs. + The value of any key may be an object and include the + property `transmissionMode` - defaults to `value`. :returns: tuple of MIME type and process response + (string or bytes, or dict) """ raise NotImplementedError() diff --git a/pygeoapi/process/echo.py b/pygeoapi/process/echo.py index 8c3f04a38..eff009087 100644 --- a/pygeoapi/process/echo.py +++ b/pygeoapi/process/echo.py @@ -1,8 +1,10 @@ # ================================================================= # # Authors: Alexander Pilz +# Tom Kralidis # # Copyright (c) 2023 Alexander Pilz +# Copyright (c) 2023 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -26,6 +28,7 @@ # OTHER DEALINGS IN THE SOFTWARE. # # ================================================================= + import logging import time @@ -35,40 +38,40 @@ #: Process metadata and description PROCESS_METADATA = { - "id": "echo", - "title": "Echo Process", - "description": "Testable Echo process.", - "version": "1.0.0", - "jobControlOptions": [ - "async-execute", - "sync-execute" + 'id': 'echo', + 'title': 'Echo Process', + 'description': 'Testable Echo process.', + 'version': '1.0.0', + 'jobControlOptions': [ + 'async-execute', + 'sync-execute' ], - "outputTransmission": [ - "value", - "reference" + 'outputTransmission': [ + 'value', + 'reference' ], - "inputs": { - "echoInput": { - "title": "Echo value", - "description": "Value to be echoed back.", - "minOccurs": 1, - "maxOccurs": 1, - "schema": { - "type": "string", - "enum": [ - "Echo", - "Test", - "42" + 'inputs': { + 'echoInput': { + 'title': 'Echo value', + 'description': 'Value to be echoed back.', + 'minOccurs': 1, + 'maxOccurs': 1, + 'schema': { + 'type': 'string', + 'enum': [ + 'Echo', + 'Test', + '42' ] }}, - "pause": { - "title": "Pause value", - "description": "Value to control the processing time.", - "minOccurs": 1, - "maxOccurs": 1, - "schema": { - "type": "float", - "enum": [ + 'pause': { + 'title': 'Pause value', + 'description': 'Value to control the processing time.', + 'minOccurs': 1, + 'maxOccurs': 1, + 'schema': { + 'type': 'number', + 'enum': [ 5.5, 10.25, 42.0 @@ -76,14 +79,14 @@ } } }, - "outputs": { - "echoOutput": { - "schema": { - "type": "string" + 'outputs': { + 'echoOutput': { + 'schema': { + 'type': 'string' } } }, - "links": [{ + 'links': [{ 'type': 'text/html', 'rel': 'about', 'title': 'information', @@ -99,7 +102,7 @@ } -class echoProcessor(BaseProcessor): +class EchoProcessor(BaseProcessor): """Echo Processor example""" def __init__(self, processor_def): """ @@ -107,33 +110,35 @@ def __init__(self, processor_def): :param processor_def: provider definition - :returns: pygeoapi.process.echo.echoProcessor + :returns: pygeoapi.process.echo.EchoProcessor """ super().__init__(processor_def, PROCESS_METADATA) - def execute(self, data): + def execute(self, data, outputs=None): mimetype = 'application/json' - echo = data.get('echoInput', None) - pause = data.get('pause', None) + echo = data.get('echoInput') + pause = data.get('pause') if echo is None: raise ProcessorExecuteError( 'Cannot run process without echo value') + if not isinstance(echo, str): raise ProcessorExecuteError( - 'Cannot run process with echo not of type String') + 'Cannot run process with echo not of type string') outputs = { 'id': 'echoOutput', 'value': echo } + if pause is not None and isinstance(pause, float): time.sleep(pause) return mimetype, outputs def __repr__(self): - return ' {}'.format(self.name) + return f' {self.name}' diff --git a/pygeoapi/process/hello_world.py b/pygeoapi/process/hello_world.py index 9d1962916..4577929aa 100644 --- a/pygeoapi/process/hello_world.py +++ b/pygeoapi/process/hello_world.py @@ -1,8 +1,10 @@ # ================================================================= # # Authors: Tom Kralidis +# Francesco Martinelli # # Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2024 Francesco Martinelli # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -69,7 +71,6 @@ }, 'minOccurs': 1, 'maxOccurs': 1, - 'metadata': None, # TODO how to use? 'keywords': ['full name', 'personal'] }, 'message': { @@ -80,7 +81,6 @@ }, 'minOccurs': 0, 'maxOccurs': 1, - 'metadata': None, 'keywords': ['message'] } }, @@ -117,9 +117,9 @@ def __init__(self, processor_def): """ super().__init__(processor_def, PROCESS_METADATA) + self.supports_outputs = True - def execute(self, data): - + def execute(self, data, outputs=None): mimetype = 'application/json' name = data.get('name') @@ -129,12 +129,14 @@ def execute(self, data): message = data.get('message', '') value = f'Hello {name}! {message}'.strip() - outputs = { - 'id': 'echo', - 'value': value - } + produced_outputs = {} + if not bool(outputs) or 'echo' in outputs: + produced_outputs = { + 'id': 'echo', + 'value': value + } - return mimetype, outputs + return mimetype, produced_outputs def __repr__(self): return f' {self.name}' diff --git a/pygeoapi/process/manager/__init__.py b/pygeoapi/process/manager/__init__.py index 2019a54b7..565d98e85 100644 --- a/pygeoapi/process/manager/__init__.py +++ b/pygeoapi/process/manager/__init__.py @@ -29,35 +29,4 @@ # # ================================================================= -import logging -from typing import Dict - -from pygeoapi.plugin import load_plugin -from pygeoapi.process.manager.base import BaseManager - -LOGGER = logging.getLogger(__name__) - - -def get_manager(config: Dict) -> BaseManager: - """Instantiate process manager from the supplied configuration. - - :param config: pygeoapi configuration - - :returns: The pygeoapi process manager object - """ - manager_conf = config.get('server', {}).get( - 'manager', - { - 'name': 'Dummy', - 'connection': None, - 'output_dir': None - } - ) - processes_conf = {} - for id_, resource_conf in config.get('resources', {}).items(): - if resource_conf.get('type') == 'process': - processes_conf[id_] = resource_conf - manager_conf['processes'] = processes_conf - if manager_conf.get('name') == 'Dummy': - LOGGER.info('Starting dummy manager') - return load_plugin('process_manager', manager_conf) +"""OGC process manager package""" diff --git a/pygeoapi/process/manager/base.py b/pygeoapi/process/manager/base.py index d0d782470..d3d3285b0 100644 --- a/pygeoapi/process/manager/base.py +++ b/pygeoapi/process/manager/base.py @@ -2,9 +2,11 @@ # # Authors: Tom Kralidis # Ricardo Garcia Silva +# Francesco Martinelli # -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # (c) 2023 Ricardo Garcia Silva +# (c) 2024 Francesco Martinelli # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -30,7 +32,6 @@ # ================================================================= import collections -from datetime import datetime import json import logging from multiprocessing import dummy @@ -38,6 +39,8 @@ from typing import Any, Dict, Tuple, Optional, OrderedDict import uuid +import requests + from pygeoapi.plugin import load_plugin from pygeoapi.process.base import ( BaseProcessor, @@ -46,10 +49,12 @@ UnknownProcessError, ) from pygeoapi.util import ( - DATETIME_FORMAT, + get_current_datetime, JobStatus, ProcessExecutionMode, RequestedProcessExecutionMode, + RequestedResponse, + Subscriber ) LOGGER = logging.getLogger(__name__) @@ -70,6 +75,7 @@ def __init__(self, manager_def: dict): self.name = manager_def['name'] self.is_async = False + self.supports_subscribing = False self.connection = manager_def.get('connection') self.output_dir = manager_def.get('output_dir') @@ -85,12 +91,12 @@ def __init__(self, manager_def: dict): for id_, process_conf in manager_def.get('processes', {}).items(): self.processes[id_] = dict(process_conf) - def get_processor(self, process_id: str) -> Optional[BaseProcessor]: + def get_processor(self, process_id: str) -> BaseProcessor: """Instantiate a processor. :param process_id: Identifier of the process - :raise UnknownProcessError: if the processor cannot be created + :raises UnknownProcessError: if the processor cannot be created :returns: instance of the processor """ @@ -101,14 +107,21 @@ def get_processor(self, process_id: str) -> Optional[BaseProcessor]: else: return load_plugin('process', process_conf['processor']) - def get_jobs(self, status: JobStatus = None) -> list: + def get_jobs(self, + status: JobStatus = None, + limit: Optional[int] = None, + offset: Optional[int] = None + ) -> dict: """ Get process jobs, optionally filtered by status :param status: job status (accepted, running, successful, failed, results) (default is all) + :param limit: number of jobs to return + :param offset: pagination offset - :returns: `list` of jobs (identifier, status, process identifier) + :returns: dict of list of jobs (identifier, status, process identifier) + and numberMatched """ raise NotImplementedError() @@ -142,8 +155,8 @@ def get_job(self, job_id: str) -> dict: :param job_id: job identifier - :raises: JobNotFoundError: if the job_id does not correspond to a - known job + :raises JobNotFoundError: if the job_id does not correspond to a + known job :returns: `dict` of job result """ @@ -155,10 +168,10 @@ def get_job_result(self, job_id: str) -> Tuple[str, Any]: :param job_id: job identifier - :raises: JobNotFoundError: if the job_id does not correspond to a - known job - :raises: JobResultNotFoundError: if the job-related result cannot - be returned + :raises JobNotFoundError: if the job_id does not correspond to a + known job + :raises JobResultNotFoundError: if the job-related result cannot + be returned :returns: `tuple` of mimetype and raw output """ @@ -170,15 +183,19 @@ def delete_job(self, job_id: str) -> bool: :param job_id: job identifier - :raises: JobNotFoundError: if the job_id does not correspond to a - known job + :raises JobNotFoundError: if the job_id does not correspond to a + known job :returns: `bool` of status result """ raise JobNotFoundError() def _execute_handler_async(self, p: BaseProcessor, job_id: str, - data_dict: dict) -> Tuple[str, None, JobStatus]: + data_dict: dict, + requested_outputs: Optional[dict] = None, + subscriber: Optional[Subscriber] = None, + requested_response: Optional[RequestedResponse] = RequestedResponse.raw.value # noqa + ) -> Tuple[str, None, JobStatus]: """ This private execution handler executes a process in a background thread using `multiprocessing.dummy` @@ -188,19 +205,35 @@ def _execute_handler_async(self, p: BaseProcessor, job_id: str, :param p: `pygeoapi.process` object :param job_id: job identifier :param data_dict: `dict` of data parameters + :param requested_outputs: `dict` optionally specifying the subset of + required outputs - defaults to all outputs. + The value of any key may be an object and + include the property `transmissionMode` + (defaults to `value`) + Note: 'optional' is for backward + compatibility. + :param subscriber: optional `Subscriber` specifying callback URLs + :param requested_response: `RequestedResponse` optionally specifying + raw or document (default is `raw`) :returns: tuple of None (i.e. initial response payload) and JobStatus.accepted (i.e. initial job status) """ - _process = dummy.Process( - target=self._execute_handler_sync, - args=(p, job_id, data_dict) - ) + + args = (p, job_id, data_dict, requested_outputs, subscriber, + requested_response) + + _process = dummy.Process(target=self._execute_handler_sync, args=args) _process.start() + return 'application/json', None, JobStatus.accepted def _execute_handler_sync(self, p: BaseProcessor, job_id: str, - data_dict: dict) -> Tuple[str, Any, JobStatus]: + data_dict: dict, + requested_outputs: Optional[dict] = None, + subscriber: Optional[Subscriber] = None, + requested_response: Optional[RequestedResponse] = RequestedResponse.raw.value # noqa + ) -> Tuple[str, Any, JobStatus]: """ Synchronous execution handler @@ -211,27 +244,28 @@ def _execute_handler_sync(self, p: BaseProcessor, job_id: str, :param p: `pygeoapi.process` object :param job_id: job identifier :param data_dict: `dict` of data parameters + :param requested_outputs: `dict` optionally specifying the subset of + required outputs - defaults to all outputs. + The value of any key may be an object and + include the property `transmissionMode` + (defaults to `value`) + Note: 'optional' is for backward + compatibility. + :param subscriber: optional `Subscriber` specifying callback URLs + :param requested_response: `RequestedResponse` optionally specifying + raw or document (default is `raw`) :returns: tuple of MIME type, response payload and status """ - process_id = p.metadata['id'] - current_status = JobStatus.accepted + extra_execute_parameters = {} - job_metadata = { - 'identifier': job_id, - 'process_id': process_id, - 'job_start_datetime': datetime.utcnow().strftime( - DATETIME_FORMAT), - 'job_end_datetime': None, - 'status': current_status.value, - 'location': None, - 'mimetype': None, - 'message': 'Job accepted and ready for execution', - 'progress': 5 - } + # only pass requested_outputs if supported, + # otherwise this breaks existing processes + if p.supports_outputs: + extra_execute_parameters['outputs'] = requested_outputs - self.add_job(job_metadata) + self._send_in_progress_notification(subscriber) try: if self.output_dir is not None: @@ -241,9 +275,15 @@ def _execute_handler_sync(self, p: BaseProcessor, job_id: str, job_filename = None current_status = JobStatus.running - jfmt, outputs = p.execute(data_dict) + jfmt, outputs = p.execute(data_dict, **extra_execute_parameters) + + if requested_response == RequestedResponse.document.value: + outputs = { + 'outputs': [outputs] + } self.update_job(job_id, { + 'updated': get_current_datetime(), 'status': current_status.value, 'message': 'Writing job output', 'progress': 95 @@ -251,7 +291,7 @@ def _execute_handler_sync(self, p: BaseProcessor, job_id: str, if self.output_dir is not None: LOGGER.debug(f'writing output to {job_filename}') - if isinstance(outputs, dict): + if isinstance(outputs, (dict, list)): mode = 'w' data = json.dumps(outputs, sort_keys=True, indent=4) encoding = 'utf-8' @@ -265,8 +305,8 @@ def _execute_handler_sync(self, p: BaseProcessor, job_id: str, current_status = JobStatus.successful job_update_metadata = { - 'job_end_datetime': datetime.utcnow().strftime( - DATETIME_FORMAT), + 'finished': get_current_datetime(), + 'updated': get_current_datetime(), 'status': current_status.value, 'location': str(job_filename), 'mimetype': jfmt, @@ -275,11 +315,12 @@ def _execute_handler_sync(self, p: BaseProcessor, job_id: str, } self.update_job(job_id, job_update_metadata) + self._send_success_notification(subscriber, outputs=outputs) except Exception as err: # TODO assess correct exception type and description to help users # NOTE, the /results endpoint should return the error HTTP status - # for jobs that failed, ths specification says that failing jobs + # for jobs that failed, the specification says that failing jobs # must still be able to be retrieved with their error message # intact, and the correct HTTP error status at the /results # endpoint, even if the /result endpoint correctly returns the @@ -289,22 +330,24 @@ def _execute_handler_sync(self, p: BaseProcessor, job_id: str, current_status = JobStatus.failed code = 'InvalidParameterValue' outputs = { + 'type': code, 'code': code, - 'description': 'Error updating job' + 'description': f'Error executing process: {err}' } - LOGGER.error(err) + LOGGER.exception(err) job_metadata = { - 'job_end_datetime': datetime.utcnow().strftime( - DATETIME_FORMAT), + 'finished': get_current_datetime(), + 'updated': get_current_datetime(), 'status': current_status.value, 'location': None, - 'mimetype': None, + 'mimetype': 'application/octet-stream', 'message': f'{code}: {outputs["description"]}' } jfmt = 'application/json' self.update_job(job_id, job_metadata) + self._send_failed_notification(subscriber) return jfmt, outputs, current_status @@ -312,7 +355,10 @@ def execute_process( self, process_id: str, data_dict: dict, - execution_mode: Optional[RequestedProcessExecutionMode] = None + execution_mode: Optional[RequestedProcessExecutionMode] = None, + requested_outputs: Optional[dict] = None, + subscriber: Optional[Subscriber] = None, + requested_response: Optional[RequestedResponse] = RequestedResponse.raw.value # noqa ) -> Tuple[str, Any, JobStatus, Optional[Dict[str, str]]]: """ Default process execution handler @@ -320,10 +366,21 @@ def execute_process( :param process_id: process identifier :param data_dict: `dict` of data parameters :param execution_mode: `str` optionally specifying sync or async - processing. - - :raises: UnknownProcessError if the input process_id does not - correspond to a known process + processing. + :param requested_outputs: `dict` optionally specifying the subset of + required outputs - defaults to all outputs. + The value of any key may be an object and + include the property `transmissionMode` + (default is `value`) + Note: 'optional' is for backward + compatibility. + :param subscriber: `Subscriber` optionally specifying callback urls + :param requested_response: `RequestedResponse` optionally specifying + raw or document (default is `raw`) + + + :raises UnknownProcessError: if the input process_id does not + correspond to a known process :returns: tuple of job_id, MIME type, response payload, status and optionally additional HTTP headers to include in the final response @@ -331,6 +388,11 @@ def execute_process( job_id = str(uuid.uuid1()) processor = self.get_processor(process_id) + processor.set_job_id(job_id) + extra_execute_handler_parameters = { + 'requested_response': requested_response + } + if execution_mode == RequestedProcessExecutionMode.respond_async: job_control_options = processor.metadata.get( 'jobControlOptions', []) @@ -363,11 +425,64 @@ def execute_process( LOGGER.debug('Synchronous execution') handler = self._execute_handler_sync response_headers = None + + # Add Job before returning any response. + current_status = JobStatus.accepted + job_metadata = { + 'type': 'process', + 'identifier': job_id, + 'process_id': process_id, + 'created': get_current_datetime(), + 'started': get_current_datetime(), + 'updated': get_current_datetime(), + 'finished': None, + 'status': current_status.value, + 'location': None, + 'mimetype': 'application/octet-stream', + 'message': 'Job accepted and ready for execution', + 'progress': 5 + } + self.add_job(job_metadata) + + # only pass subscriber if supported, otherwise this breaks + # existing managers + if self.supports_subscribing: + extra_execute_handler_parameters['subscriber'] = subscriber + # TODO: handler's response could also be allowed to include more HTTP # headers - mime_type, outputs, status = handler(processor, job_id, data_dict) + mime_type, outputs, status = handler( + processor, + job_id, + data_dict, + requested_outputs, + **extra_execute_handler_parameters) + return job_id, mime_type, outputs, status, response_headers + def _send_in_progress_notification(self, subscriber: Optional[Subscriber]): + if subscriber and subscriber.in_progress_uri: + response = requests.post(subscriber.in_progress_uri, json={}) + LOGGER.debug( + f'In progress notification response: {response.status_code}' + ) + + def _send_success_notification( + self, subscriber: Optional[Subscriber], outputs: Any + ): + if subscriber: + response = requests.post(subscriber.success_uri, json=outputs) + LOGGER.debug( + f'Success notification response: {response.status_code}' + ) + + def _send_failed_notification(self, subscriber: Optional[Subscriber]): + if subscriber and subscriber.failed_uri: + response = requests.post(subscriber.failed_uri, json={}) + LOGGER.debug( + f'Failed notification response: {response.status_code}' + ) + def __repr__(self): return f' {self.name}' diff --git a/pygeoapi/process/manager/dummy.py b/pygeoapi/process/manager/dummy.py index b27507b7d..6528f53a7 100644 --- a/pygeoapi/process/manager/dummy.py +++ b/pygeoapi/process/manager/dummy.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -33,8 +33,10 @@ from pygeoapi.process.manager.base import BaseManager from pygeoapi.util import ( - RequestedProcessExecutionMode, JobStatus, + RequestedProcessExecutionMode, + RequestedResponse, + Subscriber ) LOGGER = logging.getLogger(__name__) @@ -54,23 +56,30 @@ def __init__(self, manager_def: dict): super().__init__(manager_def) - def get_jobs(self, status: JobStatus = None) -> list: + def get_jobs(self, status: JobStatus = None, limit=None, offset=None + ) -> dict: """ Get process jobs, optionally filtered by status :param status: job status (accepted, running, successful, failed, results) (default is all) + :param limit: number of jobs to return + :param offset: pagination offset - :returns: `list` of jobs (identifier, status, process identifier) + :returns: dict of list of jobs (identifier, status, process identifier) + and numberMatched """ - return [] + return {'jobs': [], 'numberMatched': 0} def execute_process( self, process_id: str, data_dict: dict, - execution_mode: Optional[RequestedProcessExecutionMode] = None + execution_mode: Optional[RequestedProcessExecutionMode] = None, + requested_outputs: Optional[dict] = None, + subscriber: Optional[Subscriber] = None, + requested_response: Optional[RequestedResponse] = RequestedResponse.raw.value # noqa ) -> Tuple[str, str, Any, JobStatus, Optional[Dict[str, str]]]: """ Default process execution handler @@ -78,9 +87,19 @@ def execute_process( :param process_id: process identifier :param data_dict: `dict` of data parameters :param execution_mode: requested execution mode - + :param requested_outputs: `dict` optionally specify the subset of + required outputs - defaults to all outputs. + The value of any key may be an object and include the property + `transmissionMode` - defaults to `value`. + Note: 'optional' is for backward compatibility. + :param subscriber: `Subscriber` optionally specifying callback urls + :param requested_response: `RequestedResponse` optionally specifying + raw or document (default is `raw`) + + :raises UnknownProcessError: if the input process_id does not + correspond to a known process :returns: tuple of job_id, MIME type, response payload, status and - optionally additional HTTP headers to include in the + optionally additional HTTP headers to include in the final response """ @@ -94,17 +113,27 @@ def execute_process( LOGGER.debug('Dummy manager does not support asynchronous') LOGGER.debug('Forcing synchronous execution') + self._send_in_progress_notification(subscriber) processor = self.get_processor(process_id) try: - jfmt, outputs = processor.execute(data_dict) + jfmt, outputs = processor.execute( + data_dict, outputs=requested_outputs) current_status = JobStatus.successful - except Exception: + self._send_success_notification(subscriber, outputs) + except Exception as err: outputs = { 'code': 'InvalidParameterValue', - 'description': 'Error updating job' + 'description': f'Error executing process: {err}' } current_status = JobStatus.failed - LOGGER.exception('Process failed') + LOGGER.exception(err) + self._send_failed_notification(subscriber) + + if requested_response == RequestedResponse.document.value: + outputs = { + 'outputs': [outputs] + } + job_id = str(uuid.uuid1()) return job_id, jfmt, outputs, current_status, response_headers diff --git a/pygeoapi/process/manager/mongodb_.py b/pygeoapi/process/manager/mongodb_.py index 7e26aa770..44bce6dbe 100644 --- a/pygeoapi/process/manager/mongodb_.py +++ b/pygeoapi/process/manager/mongodb_.py @@ -32,6 +32,7 @@ from pymongo import MongoClient +from pygeoapi.api import FORMAT_TYPES, F_JSON, F_JSONLD from pygeoapi.process.base import ( JobNotFoundError, JobResultNotFoundError, @@ -45,6 +46,7 @@ class MongoDBManager(BaseManager): def __init__(self, manager_def): super().__init__(manager_def) self.is_async = True + self.supports_subscribing = True def _connect(self): try: @@ -69,7 +71,7 @@ def destroy(self): exc_info=(traceback)) return False - def get_jobs(self, status=None): + def get_jobs(self, status=None, limit=None, offset=None): try: self._connect() database = self.db.job_manager_pygeoapi @@ -79,7 +81,10 @@ def get_jobs(self, status=None): else: jobs = list(collection.find({})) LOGGER.info("JOBMANAGER - MongoDB jobs queried") - return jobs + return { + 'jobs': jobs, + 'numberMatched': len(jobs) + } except Exception: LOGGER.error("JOBMANAGER - get_jobs error", exc_info=(traceback)) @@ -147,8 +152,16 @@ def get_job_result(self, job_id): if entry["status"] != "successful": LOGGER.info("JOBMANAGER - job not finished or failed") return (None,) - with open(entry["location"], "r") as file: - data = json.load(file) + if not entry["location"]: + LOGGER.warning(f"job {job_id!r} - unknown result location") + raise JobResultNotFoundError() + if entry["mimetype"] in (None, FORMAT_TYPES[F_JSON], + FORMAT_TYPES[F_JSONLD]): + with open(entry["location"], "r") as file: + data = json.load(file) + else: + with open(entry["location"], "rb") as file: + data = file.read() LOGGER.info("JOBMANAGER - MongoDB job result queried") return entry["mimetype"], data except Exception as err: diff --git a/pygeoapi/process/manager/postgresql.py b/pygeoapi/process/manager/postgresql.py new file mode 100644 index 000000000..16d25ab8f --- /dev/null +++ b/pygeoapi/process/manager/postgresql.py @@ -0,0 +1,309 @@ +# ================================================================= +# +# Authors: Francesco Martinelli +# +# Copyright (c) 2024 Francesco Martinelli +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +# Requires postgresql database structure. +# Create the database: +# e.g. +# CREATE DATABASE test +# WITH TEMPLATE = template0 +# ENCODING = 'UTF8' +# LOCALE = 'en_US.UTF-8'; +# ALTER DATABASE test OWNER TO postgres; +# +# Import dump: +# psql -U postgres -h 127.0.0.1 -p 5432 test < +# tests/data/postgres_manager_full_structure.backup.sql + +import json +import logging +from pathlib import Path +from typing import Any, Tuple + +from sqlalchemy import insert, update, delete +from sqlalchemy.engine import make_url +from sqlalchemy.orm import Session + +from pygeoapi.api import FORMAT_TYPES, F_JSON, F_JSONLD +from pygeoapi.process.base import ( + JobNotFoundError, + JobResultNotFoundError, + ProcessorGenericError +) +from pygeoapi.process.manager.base import BaseManager +from pygeoapi.provider.postgresql import get_engine, get_table_model +from pygeoapi.util import JobStatus + + +LOGGER = logging.getLogger(__name__) + + +class PostgreSQLManager(BaseManager): + """PostgreSQL Manager""" + + def __init__(self, manager_def: dict): + """ + Initialize object + + :param manager_def: manager definition + + :returns: `pygeoapi.process.manager.postgresqs.PostgreSQLManager` + """ + + super().__init__(manager_def) + self.is_async = True + self.id_field = 'identifier' + self.supports_subscribing = True + self.connection = manager_def['connection'] + + try: + self.db_search_path = tuple(self.connection.get('search_path', + ['public'])) + except Exception: + self.db_search_path = ('public',) + + try: + LOGGER.debug('Connecting to database') + if isinstance(self.connection, str): + _url = make_url(self.connection) + self._engine = get_engine( + _url.host, + _url.port, + _url.database, + _url.username, + _url.password) + else: + self._engine = get_engine(**self.connection) + except Exception as err: + msg = 'Test connecting to DB failed' + LOGGER.error(f'{msg}: {err}') + raise ProcessorGenericError(msg) + + try: + LOGGER.debug('Getting table model') + self.table_model = get_table_model( + 'jobs', + self.id_field, + self.db_search_path, + self._engine + ) + except Exception as err: + msg = 'Table model fetch failed' + LOGGER.error(f'{msg}: {err}') + raise ProcessorGenericError(msg) + + def get_jobs(self, status: JobStatus = None, limit=None, offset=None + ) -> dict: + """ + Get jobs + + :param status: job status (accepted, running, successful, + failed, results) (default is all) + :param limit: number of jobs to return + :param offset: pagination offset + + :returns: dict of list of jobs (identifier, status, process identifier) + and numberMatched + """ + + LOGGER.debug('Querying for jobs') + with Session(self._engine) as session: + results = session.query(self.table_model) + if status is not None: + column = getattr(self.table_model, 'status') + results = results.filter(column == status.value) + + jobs = [r.__dict__ for r in results.all()] + return { + 'jobs': jobs, + 'numberMatched': len(jobs) + } + + def add_job(self, job_metadata: dict) -> str: + """ + Add a job + + :param job_metadata: `dict` of job metadata + + :returns: identifier of added job + """ + + LOGGER.debug('Adding job') + with Session(self._engine) as session: + try: + session.execute(insert(self.table_model) + .values(**job_metadata)) + session.commit() + except Exception as err: + session.rollback() + msg = 'Insert failed' + LOGGER.error(f'{msg}: {err}') + raise ProcessorGenericError(msg) + + return job_metadata['identifier'] + + def update_job(self, job_id: str, update_dict: dict) -> bool: + """ + Updates a job + + :param job_id: job identifier + :param update_dict: `dict` of property updates + + :returns: `bool` of status result + """ + + rowcount = 0 + + LOGGER.debug('Updating job') + with Session(self._engine) as session: + try: + column = getattr(self.table_model, self.id_field) + stmt = ( + update(self.table_model) + .where(column == job_id) + .values(**update_dict) + ) + result = session.execute(stmt) + session.commit() + rowcount = result.rowcount + except Exception as err: + session.rollback() + msg = 'Update failed' + LOGGER.error(f'{msg}: {err}') + raise ProcessorGenericError(msg) + + return rowcount == 1 + + def get_job(self, job_id: str) -> dict: + """ + Get a single job + + :param job_id: job identifier + + :raises JobNotFoundError: if the job_id does not correspond to a + known job + :returns: `dict` # `pygeoapi.process.manager.Job` + """ + + LOGGER.debug('Querying for job') + with Session(self._engine) as session: + results = session.query(self.table_model) + column = getattr(self.table_model, self.id_field) + results = session.query(self.table_model).filter(column == job_id) + + first = results.first() + + if first is not None: + return first.__dict__ + else: + raise JobNotFoundError() + + def delete_job(self, job_id: str) -> bool: + """ + Deletes a job + + :param job_id: job identifier + + :raises JobNotFoundError: if the job_id does not correspond to a + known job + :return `bool` of status result + """ + + rowcount = 0 + + # get result file if present for deletion + job_result = self.get_job(job_id) + location = job_result.get('location') + + LOGGER.debug('Deleting job') + with Session(self._engine) as session: + try: + column = getattr(self.table_model, self.id_field) + stmt = ( + delete(self.table_model) + .where(column == job_id) + ) + result = session.execute(stmt) + session.commit() + rowcount = result.rowcount + except Exception as err: + session.rollback() + msg = 'Delete failed' + LOGGER.error(f'{msg}: {err}') + raise ProcessorGenericError(msg) + + # delete result file if present + if None not in [location, self.output_dir]: + try: + Path(location).unlink() + except FileNotFoundError: + pass + + return rowcount == 1 + + def get_job_result(self, job_id: str) -> Tuple[str, Any]: + """ + Get a job's status, and actual output of executing the process + + :param job_id: job identifier + + :raises JobNotFoundError: if the job_id does not correspond to a + known job + :raises JobResultNotFoundError: if the job-related result cannot + be returned + :returns: `tuple` of mimetype and raw output + """ + + job_result = self.get_job(job_id) + location = job_result.get('location') + mimetype = job_result.get('mimetype') + job_status = JobStatus[job_result['status']] + + if job_status != JobStatus.successful: + # Job is incomplete + return (None,) + if not location: + LOGGER.warning(f'job {job_id!r} - unknown result location') + raise JobResultNotFoundError() + else: + try: + location = Path(location) + if mimetype in (None, FORMAT_TYPES[F_JSON], + FORMAT_TYPES[F_JSONLD]): + with location.open('r', encoding='utf-8') as fh: + result = json.load(fh) + else: + with location.open('rb') as fh: + result = fh.read() + except (TypeError, FileNotFoundError, json.JSONDecodeError): + raise JobResultNotFoundError() + else: + return mimetype, result + + def __repr__(self): + return f' {self.name}' diff --git a/pygeoapi/process/manager/tinydb_.py b/pygeoapi/process/manager/tinydb_.py index c55b7916f..b04d29a49 100644 --- a/pygeoapi/process/manager/tinydb_.py +++ b/pygeoapi/process/manager/tinydb_.py @@ -37,6 +37,7 @@ import tinydb from filelock import FileLock +from pygeoapi.api import FORMAT_TYPES, F_JSON, F_JSONLD from pygeoapi.process.base import ( JobNotFoundError, JobResultNotFoundError, @@ -61,6 +62,7 @@ def __init__(self, manager_def: dict): super().__init__(manager_def) self.is_async = True + self.supports_subscribing = True @contextmanager def _db(self): @@ -81,20 +83,35 @@ def destroy(self) -> bool: return True - def get_jobs(self, status: JobStatus = None) -> list: + def get_jobs(self, status: JobStatus = None, limit=None, offset=None + ) -> dict: """ Get jobs :param status: job status (accepted, running, successful, failed, results) (default is all) + :param limit: number of jobs to return + :param offset: pagination offset - :returns: 'list` of jobs (identifier, status, process identifier) + :returns: dict of list of jobs (identifier, status, process identifier) + and numberMatched """ with self._db() as db: jobs_list = db.all() - return jobs_list + number_matched = len(jobs_list) + + if offset: + jobs_list = jobs_list[offset:] + + if limit: + jobs_list = jobs_list[:limit] + + return { + 'jobs': jobs_list, + 'numberMatched': number_matched + } def add_job(self, job_metadata: dict) -> str: """ @@ -131,8 +148,8 @@ def delete_job(self, job_id: str) -> bool: :param job_id: job identifier - :raises: JobNotFoundError: if the job_id does not correspond to a - known job + :raises JobNotFoundError: if the job_id does not correspond to a + known job :return `bool` of status result """ # delete result file if present @@ -152,8 +169,8 @@ def get_job(self, job_id: str) -> dict: :param job_id: job identifier - :raises: JobNotFoundError: if the job_id does not correspond to a - known job + :raises JobNotFoundError: if the job_id does not correspond to a + known job :returns: `dict` # `pygeoapi.process.manager.Job` """ @@ -174,10 +191,10 @@ def get_job_result(self, job_id: str) -> Tuple[str, Any]: :param job_id: job identifier - :raises: JobNotFoundError: if the job_id does not correspond to a - known job - :raises: JobResultNotFoundError: if the job-related result cannot - be returned + :raises JobNotFoundError: if the job_id does not correspond to a + known job + :raises JobResultNotFoundError: if the job-related result cannot + be returned :returns: `tuple` of mimetype and raw output """ @@ -195,8 +212,13 @@ def get_job_result(self, job_id: str) -> Tuple[str, Any]: else: try: location = Path(location) - with location.open('r', encoding='utf-8') as filehandler: - result = json.load(filehandler) + if mimetype in (None, FORMAT_TYPES[F_JSON], + FORMAT_TYPES[F_JSONLD]): + with location.open('r', encoding='utf-8') as filehandler: + result = json.load(filehandler) + else: + with location.open('rb') as filehandler: + result = filehandler.read() except (TypeError, FileNotFoundError, json.JSONDecodeError): raise JobResultNotFoundError() else: diff --git a/pygeoapi/process/shapely_functions.py b/pygeoapi/process/shapely_functions.py new file mode 100644 index 000000000..e9047cc9a --- /dev/null +++ b/pygeoapi/process/shapely_functions.py @@ -0,0 +1,412 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Emmanuel Jolaiya +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2024 Emmanuel Jolaiya +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import enum +import logging + +from typing import Any, Dict, List, Tuple, Union + +from shapely.geometry import mapping, shape +from shapely.geometry.base import BaseGeometry +from shapely.prepared import prep +from shapely.wkt import loads + +from pygeoapi.process.base import BaseProcessor, ProcessorExecuteError + + +LOGGER = logging.getLogger(__name__) + + +class SupportedShapelyOperations(enum.Enum): + """ + Enum for the supported shapely operations + """ + + # Measurement operations. + # Ref: https://shapely.readthedocs.io/en/stable/measurement.html + + MEASUREMENT_AREA = "measurement:area" + MEASUREMENT_DISTANCE = "measurement:distance" + MEASUREMENT_BOUNDS = "measurement:bounds" + + # Predicate operations. + # Ref: https://shapely.readthedocs.io/en/stable/predicates.html + + PREDICATES_COVERS = "predicates:covers" + PREDICATES_WITHIN = "predicates:within" + + # Set operations. + # Ref: https://shapely.readthedocs.io/en/stable/set_operations.html + + SET_DIFFERENCE = "set:difference" + SET_UNION = "set:union" + + # Constructive operations. + # Ref: https://shapely.readthedocs.io/en/stable/constructive.html + + CONSTRUCTIVE_BUFFER = "constructive:buffer" + CONSTRUCTIVE_CENTROID = "constructive:centroid" + + +class SupportedFormats(enum.Enum): + WKT = "wkt" + """Well Known Text""" + GeoJSON_GEOMETRY = "geojson" + """GeoJSON Geometry""" + + +class SupportedGeometryTypes(enum.Enum): + POINT = "Point" + POLYGON = "Polygon" + MULTI_POLYGON = "MultiPolygon" + + +#: Process metadata and description +PROCESS_METADATA = { + "version": "0.2.0", + "id": "shapely-functions", + "title": { + "en": "Shapely Functions", + "fr": "Fonctions galbées", + "es": "Funciones bien formadas" + }, + "description": { + "en": "An example process that takes one or more input geometry " + "(WKT or GeoJSON geometry), applies the specified shapely" + " operation on the input, and returns the result " + "in the specified output_format.", + "fr": "Un exemple de processus qui prend une ou plusieurs géométries" + " d'entrée (géométrie WKT ou GeoJSON) applique l'opération " + "galbée spécifiée sur l'entrée et renvoie le " + "résultat dans le format de sortie spécifié.", + "es": "Un proceso de ejemplo que toma una o más geometrías" + "de entrada (geometría WKT o GeoJSON) aplica la operación" + "de forma especificada en la entrada y devuelve el resultado" + "en el formato de salida especificado." + }, + "jobControlOptions": ["sync-execute", "async-execute"], + "keywords": [ + "shapely functions", + "measurement", + "predicates", + "set operations", + "constructive ops" + ], + "links": [ + { + "type": "text/html", + "rel": "about", + "title": "information", + "href": "https://shapely.readthedocs.io", + "hreflang": "en-US" + }, + ], + "inputs": { + "operation": { + "title": "Shapely operation", + "description": "The shapely operation to perform. Namespace of the" + "function category is used to avoid mixup in function names.", + "schema": { + "type": "string" + }, + "minOccurs": 1, + "maxOccurs": 1, + "enum": [ops.value for ops in SupportedShapelyOperations] + }, + "geoms": { + "title": "Input geometry", + "description": "The representation of the object" + " as GeoJSON geometry or a WKT.", + "schema": { + "type": "array", + "description": "An array of the geometries. " + "The order of the geometries matter. The first " + "element is considered geom A, second element as geom B, " + "in that order.", + "minItems": 1, + "maxItems": 2, + "items": { + "oneOf": [ + # Can be a WKT string + { + "type": "string", + }, + # or a GeoJSON geometry + { + "oneOf": [ + {"format": "geojson-geometry"}, + { + "$ref": "http://schemas.opengis.net/" + "ogcapi/features/part1/1.0/openapi/" + "schemas/geometryGeoJSON.yaml" + }, + ] + }, + ] + }, + }, + "minOccurs": 1, + "maxOccurs": 1, + }, + "output_format": { + "title": "The output format of the geometry", + "description": "The output format of the process result. " + "If the shapely operation does not return a geometry, then " + " this is ignored.", + "schema": { + "type": "string" + }, + "minOccurs": 0, + "maxOccurs": 1, + "enum": [v.value for v in SupportedFormats] + }, + }, + "outputs": { + "result": { + "title": "Shapely operation result", + "description": "The result of the shapely operation " + "performed in the process.", + "schema": { + "type": "object", + "contentMediaType": "application/json" + }, + }, + }, + "example": { + "inputs": { + "operation": "predicates:within", + "geoms": [ + { + "coordinates": [ + [ + [80.58983993887352, 24.07996699713047], + [80.58983993887352, 20.291958215654446], + [85.04987314197012, 20.291958215654446], + [85.04987314197012, 24.07996699713047], + [80.58983993887352, 24.07996699713047], + ] + ], + "type": "Polygon", + }, + { + "coordinates": [ + [ + [80.58983993887352, 24.07996699713047], + [80.58983993887352, 20.291958215654446], + [85.04987314197012, 20.291958215654446], + [85.04987314197012, 24.07996699713047], + [80.58983993887352, 24.07996699713047], + ] + ], + "type": "Polygon" + }, + ], + } + }, +} + + +class ShapelyFunctionsProcessor(BaseProcessor): + """Shapely Functions Processor example""" + + def __init__(self, processor_def): + """ + Initialize object + + :param processor_def: provider definition + + :returns: pygeoapi.process.shapely_functions.ShapelyFunctionsProcessor + """ + self.shapely_operations = [ops.value for ops in SupportedShapelyOperations] # noqa: E501 + self.supported_formats = [fmt.value for fmt in SupportedFormats] + super().__init__(processor_def, PROCESS_METADATA) + + def execute(self, data, outputs=None) -> Tuple[str, Dict[str, Any]]: + mimetype = "application/json" + operation = data.get("operation") + output_format = data.get("output_format") + geometries = data.get("geoms") + + # validations + if geometries is None: + raise ProcessorExecuteError("Cannot process without valid geometries.") # noqa: E501 + + if not isinstance(geometries, list): + raise ProcessorExecuteError( + """Cannot process without valid geometries. + `geoms` must be an array of WKT + or GeoJSON geometry objects.""" + ) + + # output_format could be optional, + # because some operations return only value and not geometry + nongeom_operations = { + SupportedShapelyOperations.MEASUREMENT_AREA.value, + SupportedShapelyOperations.MEASUREMENT_DISTANCE.value, + SupportedShapelyOperations.MEASUREMENT_BOUNDS.value, + SupportedShapelyOperations.PREDICATES_COVERS.value, + SupportedShapelyOperations.PREDICATES_WITHIN.value + } + if output_format is None and operation not in nongeom_operations: + raise ProcessorExecuteError("Cannot process without an `output_format`.") # noqa: E501 + + if ( + output_format is not None + and output_format.lower() not in self.supported_formats + ): + raise ProcessorExecuteError( + f"""Invalid `output_format` provided. + Supported options are {self.supported_formats}, + but got {output_format}.""" + ) + + self.output_format = output_format + + if operation is None: + raise ProcessorExecuteError("Cannot process without a valid operation.") # noqa: E501 + + if operation not in self.shapely_operations: + raise ProcessorExecuteError( + f"""Invalid shapely operation provided. + Supported operations are : {self.shapely_operations}""" + ) + + requires_single_geom = { + SupportedShapelyOperations.MEASUREMENT_AREA.value, + SupportedShapelyOperations.MEASUREMENT_BOUNDS.value, + SupportedShapelyOperations.CONSTRUCTIVE_BUFFER.value, + SupportedShapelyOperations.CONSTRUCTIVE_CENTROID.value + } + + if operation in requires_single_geom and len(geometries) > 1: + raise ProcessorExecuteError( + f"""Too many geometries. The {operation} + operation requires only one geometry, + but {len(geometries)} was provided.""" + ) + + if operation not in requires_single_geom and len(geometries) < 2: + raise ProcessorExecuteError( + f"""Too few geometries. The {operation} operation + requires at least two geometry, + but {len(geometries)} was provided.""" + ) + + parsed_geoms = [] + + for geom in geometries: + if isinstance(geom, str): + parsed_geoms.append(loads(geom)) + else: + parsed_geoms.append(shape(geom)) + + result = self.perform_operation(parsed_geoms, operation) + + return mimetype, result + + def parse_result(self, geom: BaseGeometry) -> Union[str, Dict[str, Any]]: + """ + Convert a shapely geometry into the specified format by the client. + + Args: + geom (BaseGeometry): The shapely geometry to convert. + + Returns: + Union[str,Dict[str,Any]]: The resulting geometry + in the specified format. + """ + return ( + geom.wkt + if self.output_format == SupportedFormats.WKT.value + else mapping(geom) + ) + + def perform_operation(self, parsed_geoms: List[BaseGeometry], operation: str): # noqa: E501 + """ + Perform the exact shapely operation specified by the client. + + Args: + parsed_geoms (List[BaseGeometry]): An array of shapely geometries. + operation (str): The exact shapely operation to perform. + + Raises: + ProcessorExecuteError: Server error if given an invalid parameter. + + Returns: + Dict[str,Any]: The response object with the operation result, + to be returned to the client. + """ + result = {"operation": operation, "result": None} + + # prep the first geom for performance improvement + prep(parsed_geoms[0]) + if operation == SupportedShapelyOperations.MEASUREMENT_AREA.value: + if parsed_geoms[0].type not in [ + SupportedGeometryTypes.POLYGON.value, + SupportedGeometryTypes.MULTI_POLYGON.value + ]: + raise ProcessorExecuteError( + f"""Invalid geometry type.{operation} + operation only works on + `Polygon and MultiPolygon` geometry.""" + ) + result.update({"result": parsed_geoms[0].area}) + elif operation == SupportedShapelyOperations.MEASUREMENT_BOUNDS.value: + result.update({"result": parsed_geoms[0].bounds}) + elif operation == SupportedShapelyOperations.MEASUREMENT_DISTANCE.value: # noqa: E501 + result.update({"result": parsed_geoms[0].distance(parsed_geoms[1])}) # noqa: E501 + elif operation == SupportedShapelyOperations.PREDICATES_COVERS.value: + result.update({"result": parsed_geoms[0].covers(parsed_geoms[1])}) + elif operation == SupportedShapelyOperations.PREDICATES_WITHIN.value: + result.update({"result": parsed_geoms[0].within(parsed_geoms[1])}) + elif operation == SupportedShapelyOperations.SET_DIFFERENCE.value: + result.update( + { + "result": self.parse_result( + parsed_geoms[0].difference(parsed_geoms[1]) + ) + } + ) + elif operation == SupportedShapelyOperations.SET_UNION.value: + result.update( + { + "result": self.parse_result(parsed_geoms[0].union(parsed_geoms[1])) # noqa: E501 + } + ) + elif operation == SupportedShapelyOperations.CONSTRUCTIVE_BUFFER.value: + # todo - how do we receive kwargs from the user? + result.update({"result": self.parse_result(parsed_geoms[0].buffer(10))}) # noqa: E501 + else: + result.update({"result": self.parse_result(parsed_geoms[0].centroid)}) # noqa: E501 + return result + + def __repr__(self): + return f" {self.name}" diff --git a/pygeoapi/provider/base.py b/pygeoapi/provider/base.py index 4f8728dda..5f9456870 100644 --- a/pygeoapi/provider/base.py +++ b/pygeoapi/provider/base.py @@ -73,7 +73,7 @@ def __init__(self, provider_def): self.title_field = provider_def.get('title_field') self.properties = provider_def.get('properties', []) self.file_types = provider_def.get('file_types', []) - self.fields = {} + self._fields = {} self.filename = None # for coverage providers @@ -85,13 +85,31 @@ def get_fields(self): """ Get provider field information (names, types) - Example response: {'field1': 'string', 'field2': 'number'}} + Example response: + {'field1': {'type': 'string'}, 'field2': {'type': 'number'}} - :returns: dict of field names and their associated JSON Schema typess + :returns: dict of field names and their associated JSON Schema types """ raise NotImplementedError() + @property + def fields(self) -> dict: + """ + Store provider field information (names, types) + + Example response: + {'field1': {'type': 'string'}, 'field2': {'type': 'number'}} + + :returns: dict of dicts (field names and their + associated JSON Schema definitions) + """ + + if hasattr(self, '_fields'): + return self._fields + else: + return self.get_fields() + def get_schema(self, schema_type: SchemaType = SchemaType.item): """ Get provider schema model @@ -181,24 +199,6 @@ def delete(self, identifier): raise NotImplementedError() - def get_coverage_domainset(self): - """ - Provide coverage domainset - - :returns: CIS JSON object of domainset metadata - """ - - raise NotImplementedError() - - def get_coverage_rangetype(self): - """ - Provide coverage rangetype - - :returns: CIS JSON object of rangetype metadata - """ - - raise NotImplementedError() - def _load_and_prepare_item(self, item, identifier=None, accept_missing_identifier=False, raise_if_exists=True): diff --git a/pygeoapi/provider/base_edr.py b/pygeoapi/provider/base_edr.py index 3e7a259cb..7ac8792a0 100644 --- a/pygeoapi/provider/base_edr.py +++ b/pygeoapi/provider/base_edr.py @@ -29,10 +29,14 @@ import logging -from pygeoapi.provider.base import BaseProvider +from pygeoapi.provider.base import BaseProvider, ProviderInvalidDataError LOGGER = logging.getLogger(__name__) +EDR_QUERY_TYPES = ['position', 'radius', 'area', 'cube', + 'trajectory', 'corridor', 'items', + 'locations', 'instances'] + class BaseEDRProvider(BaseProvider): """Base EDR Provider""" @@ -48,13 +52,18 @@ def __init__(self, provider_def): :returns: pygeoapi.provider.base_edr.BaseEDRProvider """ - super().__init__(provider_def) + BaseProvider.__init__(self, provider_def) - self.instances = [] +# self.instances = [] @classmethod def register(cls): def inner(fn): + if fn.__name__ not in EDR_QUERY_TYPES: + msg = 'Invalid EDR Query type' + LOGGER.error(msg) + raise ProviderInvalidDataError(msg) + cls.query_types.append(fn.__name__) return fn return inner diff --git a/pygeoapi/provider/base_mvt.py b/pygeoapi/provider/base_mvt.py index 23ef01c5c..0618909bb 100644 --- a/pygeoapi/provider/base_mvt.py +++ b/pygeoapi/provider/base_mvt.py @@ -122,7 +122,7 @@ def get_tiles(self, layer=None, tileset=None, def get_html_metadata(self, dataset, server_url, layer, tileset, title, description, keywords, **kwargs): """ - Gets tile metadata informations in html format + Gets tile metadata information in html format :param dataset: dataset name :param server_url: server base url @@ -248,3 +248,45 @@ def get_tms_links(self): ] } return links + + def get_tilematrixset(self, tileMatrixSetId): + """ + Get tilematrixset + + :param tileMatrixSetId: tilematrixsetid str + + :returns: tilematrixset enum object + """ + + enums = [e.value for e in TileMatrixSetEnum] + enum = None + + try: + for e in enums: + if tileMatrixSetId == e.tileMatrixSet: + enum = e + if not enum: + raise ValueError('could not find this tilematrixset') + return enum + + except ValueError as err: + LOGGER.error(err) + + def is_in_limits(self, tilematrixset, z, x, y): + """ + Is within the limits of the tilematrixset + + :param z: tilematrix + :param x: x + :param y: y + + :returns: wether this tile is within the tile matrix + set limits (Boolean) + """ + + try: + if int(x) < tilematrixset.tileMatrices[int(z)]['matrixWidth'] and int(y) < tilematrixset.tileMatrices[int(z)]['matrixHeight']: # noqa + return True + return False + except ValueError as err: + LOGGER.error(err) diff --git a/pygeoapi/provider/csv_.py b/pygeoapi/provider/csv_.py index 88d56c56a..b51c1ebcb 100644 --- a/pygeoapi/provider/csv_.py +++ b/pygeoapi/provider/csv_.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2025 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -32,8 +32,9 @@ import itertools import logging -from pygeoapi.provider.base import (BaseProvider, ProviderQueryError, - ProviderItemNotFoundError) +from pygeoapi.provider.base import (BaseProvider, ProviderInvalidQueryError, + ProviderItemNotFoundError, + ProviderQueryError) from pygeoapi.util import get_typed_value, crs_transform LOGGER = logging.getLogger(__name__) @@ -54,7 +55,7 @@ def __init__(self, provider_def): super().__init__(provider_def) self.geometry_x = provider_def['geometry']['x_field'] self.geometry_y = provider_def['geometry']['y_field'] - self.fields = self.get_fields() + self.get_fields() def get_fields(self): """ @@ -62,32 +63,31 @@ def get_fields(self): :returns: dict of fields """ - - LOGGER.debug('Treating all columns as string types') - with open(self.data) as ff: - LOGGER.debug('Serializing DictReader') - data_ = csv.DictReader(ff) - fields = {} - - row = next(data_) - - for key, value in row.items(): - LOGGER.debug(f'key: {key}, value: {value}') - value2 = get_typed_value(value) - if key in [self.geometry_x, self.geometry_y]: - continue - if key == self.id_field: - type_ = 'string' - elif isinstance(value2, float): - type_ = 'number' - elif isinstance(value2, int): - type_ = 'integer' - else: - type_ = 'string' - - fields[key] = {'type': type_} - - return fields + if not self._fields: + LOGGER.debug('Treating all columns as string types') + with open(self.data) as ff: + LOGGER.debug('Serializing DictReader') + data_ = csv.DictReader(ff) + + row = next(data_) + + for key, value in row.items(): + LOGGER.debug(f'key: {key}, value: {value}') + value2 = get_typed_value(value) + if key in [self.geometry_x, self.geometry_y]: + continue + if key == self.id_field: + type_ = 'string' + elif isinstance(value2, float): + type_ = 'number' + elif isinstance(value2, int): + type_ = 'integer' + else: + type_ = 'string' + + self._fields[key] = {'type': type_} + + return self._fields def _load(self, offset=0, limit=10, resulttype='results', identifier=None, bbox=[], datetime_=None, properties=[], @@ -121,6 +121,12 @@ def _load(self, offset=0, limit=10, resulttype='results', LOGGER.debug('Serializing DictReader') data_ = csv.DictReader(ff) if properties: + for prop in properties: + if prop[0] not in data_.fieldnames: + msg = 'Invalid fieldname' + LOGGER.error(msg) + raise ProviderInvalidQueryError(msg) + data_ = filter( lambda p: all( [p[prop[0]] == prop[1] for prop in properties]), data_) @@ -130,16 +136,17 @@ def _load(self, offset=0, limit=10, resulttype='results', feature_collection['numberMatched'] = len(list(data_)) return feature_collection LOGGER.debug('Slicing CSV rows') - for row in itertools.islice(data_, offset, offset+limit): + for row in itertools.islice(data_, 0, None): try: coordinates = [ float(row.pop(self.geometry_x)), float(row.pop(self.geometry_y)), ] except ValueError: - msg = f'Skipping row with invalid geometry: {row.get(self.id_field)}' # noqa - LOGGER.error(msg) - continue + msg = f'Row with invalid geometry: {row.get(self.id_field)}, setting coordinates to None' # noqa + LOGGER.warning(msg) + coordinates = None + feature = {'type': 'Feature'} feature['id'] = row.pop(self.id_field) if not skip_geometry: @@ -178,6 +185,9 @@ def _load(self, offset=0, limit=10, resulttype='results', elif identifier is not None and found: return result + features_returned = feature_collection['features'][offset:offset+limit] + feature_collection['features'] = features_returned + feature_collection['numberReturned'] = len( feature_collection['features']) diff --git a/pygeoapi/provider/csw_facade.py b/pygeoapi/provider/csw_facade.py index cfb5bb826..69cd0ddee 100644 --- a/pygeoapi/provider/csw_facade.py +++ b/pygeoapi/provider/csw_facade.py @@ -69,7 +69,8 @@ def __init__(self, provider_def): 'language': ('dc:language', 'language') } - self.fields = self.get_fields() + self._fields = {} + self.get_fields() def get_fields(self): """ @@ -78,17 +79,17 @@ def get_fields(self): :returns: dict of fields """ - fields = {} - date_fields = ['date', 'created', 'updated'] + if not self._fields: + date_fields = ['date', 'created', 'updated'] - for key in self.record_mappings.keys(): - LOGGER.debug(f'key: {key}') - fields[key] = {'type': 'string'} + for key in self.record_mappings.keys(): + LOGGER.debug(f'key: {key}') + self._fields[key] = {'type': 'string'} - if key in date_fields: - fields[key]['format'] = 'date-time' + if key in date_fields: + self._fields[key]['format'] = 'date-time' - return fields + return self._fields @crs_transform def query(self, offset=0, limit=10, resulttype='results', diff --git a/pygeoapi/provider/elasticsearch_.py b/pygeoapi/provider/elasticsearch_.py index 2b146e1b3..4c0ae20c1 100644 --- a/pygeoapi/provider/elasticsearch_.py +++ b/pygeoapi/provider/elasticsearch_.py @@ -1,9 +1,10 @@ # ================================================================= # # Authors: Tom Kralidis +# Francesco Bartoli # -# Copyright (c) 2023 Tom Kralidis -# Copyright (c) 2021 Francesco Bartoli +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2024 Francesco Bartoli # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -35,13 +36,14 @@ import uuid from elasticsearch import Elasticsearch, exceptions, helpers -from elasticsearch_dsl import Search, Q +from pygeofilter.backends.elasticsearch import to_filter + +from elasticsearch_dsl import Search from pygeoapi.provider.base import (BaseProvider, ProviderConnectionError, ProviderQueryError, ProviderItemNotFoundError) -from pygeoapi.models.cql import CQLModel, get_next_node -from pygeoapi.util import get_envelope, crs_transform +from pygeoapi.util import crs_transform LOGGER = logging.getLogger(__name__) @@ -86,7 +88,7 @@ def __init__(self, provider_def): LOGGER.debug('Grabbing field information') try: - self.fields = self.get_fields() + self.get_fields() except exceptions.NotFoundError as err: LOGGER.error(err) raise ProviderQueryError(err) @@ -97,38 +99,56 @@ def get_fields(self): :returns: dict of fields """ + if not self._fields: + ii = self.es.indices.get(index=self.index_name, + allow_no_indices=False) + + LOGGER.debug(f'Response: {ii}') + try: + if '*' not in self.index_name: + mappings = ii[self.index_name]['mappings'] + p = mappings['properties']['properties'] + else: + LOGGER.debug('Wildcard index; setting from first match') + index_name_ = list(ii.keys())[0] + p = ii[index_name_]['mappings']['properties']['properties'] + except KeyError: + LOGGER.warning('Trying for alias') + alias_name = next(iter(ii)) + p = ii[alias_name]['mappings']['properties']['properties'] + except IndexError: + LOGGER.warning('could not get fields; returning empty set') + return {} - fields_ = {} - ii = self.es.indices.get(index=self.index_name, allow_no_indices=False) + self._fields = self.get_nested_fields(p, self._fields) + return self._fields - LOGGER.debug(f'Response: {ii}') - try: - if '*' not in self.index_name: - p = ii[self.index_name]['mappings']['properties']['properties'] + def get_nested_fields(self, properties, fields, prev_field=None): + """ + Get Elasticsearch fields (names, types) for all nested properties + + :param properties: `dict` of Elasticsearch mappings properties + :param fields: `dict` of fields in the current iteration + :param prev_field: name of the parent field + + :returns: dict of fields + """ + for k, v in properties['properties'].items(): + + cur_field = k if prev_field is None else f'{prev_field}.{k}' + + if isinstance(v, dict) and 'properties' in v: + fields = self.get_nested_fields(v, fields, cur_field) else: - LOGGER.debug('Wildcard index; setting from first match') - index_name_ = list(ii.keys())[0] - p = ii[index_name_]['mappings']['properties']['properties'] - except KeyError: - LOGGER.warning('Trying for alias') - alias_name = next(iter(ii)) - p = ii[alias_name]['mappings']['properties']['properties'] - except IndexError: - LOGGER.warning('could not get fields; returning empty set') - return {} - - for k, v in p['properties'].items(): - if 'type' in v: if v['type'] == 'text': - fields_[k] = {'type': 'string'} + fields[cur_field] = {'type': 'string'} elif v['type'] == 'date': - fields_[k] = {'type': 'string', 'format': 'date'} + fields[cur_field] = {'type': 'string', 'format': 'date'} elif v['type'] in ('float', 'long'): - fields_[k] = {'type': 'number', 'format': v['type']} + fields[cur_field] = {'type': 'number', 'format': v['type']} else: - fields_[k] = {'type': v['type']} - - return fields_ + fields[cur_field] = {'type': v['type']} + return fields @crs_transform def query(self, offset=0, limit=10, resulttype='results', @@ -294,7 +314,7 @@ def query(self, offset=0, limit=10, resulttype='results', try: LOGGER.debug('querying Elasticsearch') if filterq: - LOGGER.debug(f'adding cql object: {filterq.model_dump_json()}') + LOGGER.debug(f'adding cql object: {filterq}') query = update_query(input_query=query, cql=filterq) LOGGER.debug(json.dumps(query, indent=4)) @@ -486,8 +506,8 @@ def esdoc2geojson(self, doc): try: feature_thinned['properties'][p] = feature_['properties'][p] # noqa except KeyError as err: - LOGGER.error(err) - raise ProviderQueryError() + msg = f'Property missing {err}; continuing' + LOGGER.warning(msg) if feature_thinned: return feature_thinned @@ -566,183 +586,8 @@ def __repr__(self): return f' {self.data}' -class ESQueryBuilder: - def __init__(self): - self._operation = None - self.must_value = {} - self.should_value = {} - self.mustnot_value = {} - self.filter_value = {} - - def must(self, must_value): - self.must_value = must_value - return self - - def should(self, should_value): - self.should_value = should_value - return self - - def must_not(self, mustnot_value): - self.mustnot_value = mustnot_value - return self - - def filter(self, filter_value): - self.filter_value = filter_value - return self - - @property - def operation(self): - return self._operation - - @operation.setter - def operation(self, value): - self._operation = value - - def build(self): - if self.must_value: - must_clause = self.must_value or {} - if self.should_value: - should_clause = self.should_value or {} - if self.mustnot_value: - mustnot_clause = self.mustnot_value or {} - if self.filter_value: - filter_clause = self.filter_value or {} - else: - filter_clause = {} - - # to figure out how to deal with logical operations - # return match_clause & range_clause - clauses = must_clause or should_clause or mustnot_clause - filters = filter_clause - if self.operation == 'and': - res = Q( - 'bool', - must=[clause for clause in clauses], - filter=[filter for filter in filters]) - elif self.operation == 'or': - res = Q( - 'bool', - should=[clause for clause in clauses], - filter=[filter for filter in filters]) - elif self.operation == 'not': - res = Q( - 'bool', - must_not=[clause for clause in clauses], - filter=[filter for filter in filters]) - else: - if filters: - res = Q( - 'bool', - must=[clauses], - filter=[filters]) - else: - res = Q( - 'bool', - must=[clauses]) - - return res - - -def _build_query(q, cql): - - # this would be handled by the AST with the traverse of CQL model - op, node = get_next_node(cql.root) - q.operation = op - if isinstance(node, list): - query_list = [] - for elem in node: - op, next_node = get_next_node(elem) - if not getattr(next_node, 'between', 0) == 0: - property = next_node.between.value.root.root.property - lower = next_node.between.lower.root.root - upper = next_node.between.upper.root.root - query_list.append(Q( - { - 'range': - { - f'{property}': { - 'gte': lower, 'lte': upper - } - } - } - )) - if not getattr(next_node, 'root', 0) == 0: - scalars = tuple(next_node.root.eq.root) - property = scalars[0].root.property - value = scalars[1].root.root - query_list.append(Q( - {'match': {f'{property}': f'{value}'}} - )) - q.must(query_list) - elif not getattr(node, 'between', 0) == 0: - property = node.between.value.root.root.property - lower = None - if not getattr(node.between.lower, - 'root', 0) == 0: - lower = node.between.lower.root.root - upper = None - if not getattr(node.between.upper, - 'root', 0) == 0: - upper = node.between.upper.root.root - query = Q( - { - 'range': - { - f'{property}': { - 'gte': lower, 'lte': upper - } - } - } - ) - q.must(query) - elif not getattr(node, 'root', 0) == 0: - next_op, next_node = get_next_node(node) - if not getattr(next_node, 'eq', 0) == 0: - scalars = tuple(next_node.eq.root) - property = scalars[0].root.property - value = scalars[1].root.root - query = Q( - {'match': {f'{property}': f'{value}'}} - ) - q.must(query) - elif not getattr(node, 'intersects', 0) == 0: - property = node.intersects.root[0].root.property - if property == 'geometry': - geom_type = node.intersects.root[ - 1].root.root.root.type - if geom_type == 'Polygon': - coordinates = node.intersects.root[ - 1].root.root.root.coordinates - coords_list = [ - poly_coords.root for poly_coords in coordinates[0] - ] - filter_ = Q( - { - 'geo_shape': { - 'geometry': { - 'shape': { - 'type': 'envelope', - 'coordinates': get_envelope( - coords_list) - }, - 'relation': 'intersects' - } - } - } - ) - query_all = Q( - {'match_all': {}} - ) - q.must(query_all) - q.filter(filter_) - return q.build() - - -def update_query(input_query: Dict, cql: CQLModel): +def update_query(input_query: Dict, cql): s = Search.from_dict(input_query) - query = ESQueryBuilder() - output_query = _build_query(query, cql) - s = s.query(output_query) - + s = s.query(to_filter(cql)) LOGGER.debug(f'Enhanced query: {json.dumps(s.to_dict())}') return s.to_dict() diff --git a/pygeoapi/provider/erddap.py b/pygeoapi/provider/erddap.py index a81876e1f..2fc71c064 100644 --- a/pygeoapi/provider/erddap.py +++ b/pygeoapi/provider/erddap.py @@ -51,6 +51,7 @@ from pygeoapi.provider.base import ( BaseProvider, ProviderNotFoundError, ProviderQueryError) +from pygeoapi.util import crs_transform LOGGER = logging.getLogger(__name__) @@ -61,25 +62,27 @@ def __init__(self, provider_def): LOGGER.debug('Setting provider query filters') self.filters = self.options.get('filters') - self.fields = self.get_fields() + self.get_fields() def get_fields(self): - LOGGER.debug('Fetching one feature for field definitions') - properties = self.query(limit=1)['features'][0]['properties'] + if not self._fields: + LOGGER.debug('Fetching one feature for field definitions') + properties = self.query(limit=1)['features'][0]['properties'] - for key, value in properties.items(): - LOGGER.debug(f'Field: {key}={value}') + for key, value in properties.items(): + LOGGER.debug(f'Field: {key}={value}') - data_type = type(value).__name__ + data_type = type(value).__name__ - if data_type == 'str': - data_type = 'string' - if data_type == 'float': - data_type = 'number' - properties[key] = {'type': data_type} + if data_type == 'str': + data_type = 'string' + if data_type == 'float': + data_type = 'number' + self._fields[key] = {'type': data_type} - return properties + return self._fields + @crs_transform def query(self, offset=0, limit=10, resulttype='results', bbox=[], datetime_=None, properties=[], sortby=[], select_properties=[], skip_geometry=False, q=None, @@ -164,6 +167,7 @@ def query(self, offset=0, limit=10, resulttype='results', 'numberReturned': returned } + @crs_transform def get(self, identifier, **kwargs): query_params = [] diff --git a/pygeoapi/provider/esri.py b/pygeoapi/provider/esri.py index 0d22a8805..47d74e2b9 100644 --- a/pygeoapi/provider/esri.py +++ b/pygeoapi/provider/esri.py @@ -62,8 +62,9 @@ def __init__(self, provider_def): self.crs = provider_def.get('crs', '4326') self.username = provider_def.get('username') self.password = provider_def.get('password') + self.token_url = provider_def.get('token_service', ARCGIS_URL) + self.token_referer = provider_def.get('referer', GENERATE_TOKEN_URL) self.token = None - self.session = Session() self.login() @@ -76,7 +77,7 @@ def get_fields(self): :returns: `dict` of fields """ - if not self.fields: + if not self._fields: # Load fields params = {'f': 'pjson'} resp = self.get_response(self.data, params=params) @@ -102,9 +103,9 @@ def get_fields(self): raise ProviderTypeError(msg) for _ in resp['fields']: - self.fields.update({_['name']: {'type': _['type']}}) + self._fields.update({_['name']: {'type': _['type']}}) - return self.fields + return self._fields @crs_transform def query(self, offset=0, limit=10, resulttype='results', @@ -194,16 +195,15 @@ def login(self): msg = 'Missing ESRI login information, not setting token' LOGGER.debug(msg) return - params = { 'f': 'pjson', 'username': self.username, 'password': self.password, - 'referer': ARCGIS_URL + 'referer': self.token_referer } LOGGER.debug('Logging in') - with self.session.post(GENERATE_TOKEN_URL, data=params) as r: + with self.session.post(self.token_url, data=params) as r: self.token = r.json().get('token') # https://enterprise.arcgis.com/en/server/latest/administer/windows/about-arcgis-tokens.htm self.session.headers.update({ diff --git a/pygeoapi/provider/geojson.py b/pygeoapi/provider/geojson.py index 180cf0746..257166a7c 100644 --- a/pygeoapi/provider/geojson.py +++ b/pygeoapi/provider/geojson.py @@ -68,7 +68,7 @@ def __init__(self, provider_def): """initializer""" super().__init__(provider_def) - self.fields = self.get_fields() + self.get_fields() def get_fields(self): """ @@ -77,23 +77,24 @@ def get_fields(self): :returns: dict of fields """ - fields = {} - LOGGER.debug('Treating all columns as string types') - if os.path.exists(self.data): - with open(self.data) as src: - data = json.loads(src.read()) - for key, value in data['features'][0]['properties'].items(): - if isinstance(value, float): - type_ = 'number' - elif isinstance(value, int): - type_ = 'integer' - else: - type_ = 'string' - - fields[key] = {'type': type_} - else: - LOGGER.warning(f'File {self.data} does not exist.') - return fields + if not self._fields: + LOGGER.debug('Treating all columns as string types') + if os.path.exists(self.data): + with open(self.data) as src: + data = json.loads(src.read()) + for key, value in data['features'][0]['properties'].items(): + if isinstance(value, float): + type_ = 'number' + elif isinstance(value, int): + type_ = 'integer' + else: + type_ = 'string' + + self._fields[key] = {'type': type_} + else: + LOGGER.warning(f'File {self.data} does not exist.') + + return self._fields def _load(self, skip_geometry=None, properties=[], select_properties=[]): """Load and validate the source GeoJSON file diff --git a/pygeoapi/provider/hateoas.py b/pygeoapi/provider/hateoas.py index 307119d46..dfb98123f 100644 --- a/pygeoapi/provider/hateoas.py +++ b/pygeoapi/provider/hateoas.py @@ -120,6 +120,9 @@ def get_data_path(self, baseurl, urlpath, entrypath): try: jsondata = _get_json_data(f'{data_path}/collection.json') resource_type = 'Collection' + for key in ['license', 'extent', 'id']: + if key in jsondata: + content[key] = jsondata[key] except Exception: try: filename = os.path.basename(data_path) @@ -148,7 +151,7 @@ def get_data_path(self, baseurl, urlpath, entrypath): child_links.append({ 'rel': 'child', 'href': newpath, - 'type': 'text/html', + 'type': 'application/json', 'created': "-", 'entry:type': 'Catalog' }) @@ -156,7 +159,7 @@ def get_data_path(self, baseurl, urlpath, entrypath): child_links.append({ 'rel': 'child', 'href': newpath, - 'type': 'text/html', + 'type': 'application/json', 'created': "-", 'entry:type': 'Collection' }) diff --git a/pygeoapi/provider/mapscript_.py b/pygeoapi/provider/mapscript_.py index a459b1e3b..b1d237225 100644 --- a/pygeoapi/provider/mapscript_.py +++ b/pygeoapi/provider/mapscript_.py @@ -35,6 +35,7 @@ from pygeoapi.provider.base import (BaseProvider, ProviderConnectionError, ProviderQueryError) +from pygeoapi.util import str2bool LOGGER = logging.getLogger(__name__) @@ -74,13 +75,17 @@ def __init__(self, provider_def): file_extension = self.data.split('.')[-1] - if file_extension in ['shp', 'tif']: - LOGGER.debug('Setting built-in MapServer driver') - self._layer.data = self.data + if str2bool(self.options.get('tileindex', False)): + LOGGER.debug('Setting tileindex') + self._layer.tileindex = self.data else: - LOGGER.debug('Setting OGR driver') - self._layer.setConnectionType(mapscript.MS_OGR, 'OGR') - self._layer.connection = self.data + if file_extension in ['shp', 'tif']: + LOGGER.debug('Setting built-in MapServer driver') + self._layer.data = self.data + else: + LOGGER.debug('Setting OGR driver') + self._layer.setConnectionType(mapscript.MS_OGR, 'OGR') + self._layer.connection = self.data self._layer.type = getattr(mapscript, self.options['type']) @@ -111,6 +116,9 @@ def __init__(self, provider_def): cls.updateFromString(cls_def) self._layer.insertClass(cls) + if self.options['type'] == 'MS_LAYER_RASTER': + self._layer.addProcessing('SCALE=AUTO') + except MapServerError as err: LOGGER.warning(err) raise ProviderConnectionError('Cannot connect to map service') diff --git a/pygeoapi/provider/mongo.py b/pygeoapi/provider/mongo.py index f97a724f1..ca258018c 100644 --- a/pygeoapi/provider/mongo.py +++ b/pygeoapi/provider/mongo.py @@ -31,7 +31,6 @@ from datetime import datetime import logging -from bson import Code from pymongo import MongoClient from pymongo import GEOSPHERE from pymongo import ASCENDING, DESCENDING @@ -67,7 +66,7 @@ def __init__(self, provider_def): self.featuredb = dbclient.get_default_database() self.collection = provider_def['collection'] self.featuredb[self.collection].create_index([("geometry", GEOSPHERE)]) - self.fields = self.get_fields() + self.get_fields() def get_fields(self): """ @@ -76,21 +75,24 @@ def get_fields(self): :returns: dict of fields """ - map = Code( - "function() { for (var key in this.properties) " - "{ emit(key, null); } }") - reduce = Code("function(key, stuff) { return null; }") - result = self.featuredb[self.collection].map_reduce( - map, reduce, "myresults") - - # prepare a dictionary with fields - # set the field type to 'string'. - # by operating without a schema, mongo can query any data type. - fields = {} - for i in result.distinct('_id'): - fields[i] = {'type': 'string'} - - return (fields) + if not self._fields: + pipeline = [ + {"$project": {"properties": 1}}, + {"$unwind": "$properties"}, + {"$group": {"_id": "$properties", "count": {"$sum": 1}}}, + {"$project": {"_id": 1}} + ] + + result = list(self.featuredb[self.collection].aggregate(pipeline)) + + # prepare a dictionary with fields + # set the field type to 'string'. + # by operating without a schema, mongo can query any data type. + for i in result: + for key in result[0]['_id'].keys(): + self._fields[key] = {'type': 'string'} + + return self._fields def _get_feature_list(self, filterObj, sortList=[], skip=0, maxitems=1, skip_geometry=False): diff --git a/pygeoapi/provider/mvt_elastic.py b/pygeoapi/provider/mvt_elastic.py index df739c18e..027009197 100644 --- a/pygeoapi/provider/mvt_elastic.py +++ b/pygeoapi/provider/mvt_elastic.py @@ -35,8 +35,9 @@ from pygeoapi.provider.base import (ProviderConnectionError, ProviderGenericError, ProviderInvalidQueryError) +from pygeoapi.provider.tile import ProviderTileNotFoundError from pygeoapi.models.provider.base import ( - TileSetMetadata, LinkType) + TileSetMetadata, TileMatrixSetEnum, LinkType) from pygeoapi.util import is_url, url_join LOGGER = logging.getLogger(__name__) @@ -119,6 +120,13 @@ def get_layer(self): LOGGER.debug('Removing leading "/"') return layer[1:] + def get_tiling_schemes(self): + + "Only WebMercatorQuad tiling scheme is supported in elastic" + return [ + TileMatrixSetEnum.WEBMERCATORQUAD.value + ] + def get_tiles_service(self, baseurl=None, servicepath=None, dirpath=None, tile_type=None): """ @@ -166,8 +174,15 @@ def get_tiles(self, layer=None, tileset=None, try: with requests.Session() as session: + data = {'fields': ['*']} session.get(base_url) - resp = session.get(f'{base_url}/{layer}/{z}/{y}/{x}{url_query}') # noqa + resp = session.get(f'{base_url}/{layer}/{z}/{y}/{x}{url_query}', json=data) # noqa + + if resp.status_code == 404: + if (self.is_in_limits(TileMatrixSetEnum.WEBMERCATORQUAD.value, z, x, y)): # noqa + return None + raise ProviderTileNotFoundError + resp.raise_for_status() return resp.content except requests.exceptions.RequestException as e: @@ -224,7 +239,7 @@ def get_default_metadata(self, dataset, server_url, layer, tileset, tiling_scheme = LinkType(href=tiling_scheme_url, rel="http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", # noqa - type=tiling_scheme_url_type, + type_=tiling_scheme_url_type, title=tiling_scheme_url_title) if tiling_scheme is None: @@ -240,7 +255,7 @@ def get_default_metadata(self, dataset, server_url, layer, tileset, service_url_link_type = "application/vnd.mapbox-vector-tile" service_url_link_title = f'{tileset} vector tiles for {layer}' service_url_link = LinkType(href=service_url, rel="item", - type=service_url_link_type, + type_=service_url_link_type, title=service_url_link_title) links.append(tiling_scheme) @@ -249,11 +264,3 @@ def get_default_metadata(self, dataset, server_url, layer, tileset, content.links = links return content.dict(exclude_none=True) - - def get_vendor_metadata(self, dataset, server_url, layer, tileset, - title, description, keywords, **kwargs): - """ - Gets tile metadata in tilejson format - """ - LOGGER.debug("Get tilejson metadata") - return "" diff --git a/pygeoapi/provider/mvt_proxy.py b/pygeoapi/provider/mvt_proxy.py new file mode 100644 index 000000000..3d20d3ced --- /dev/null +++ b/pygeoapi/provider/mvt_proxy.py @@ -0,0 +1,262 @@ +# ================================================================= +# +# Authors: Antonio Cerciello +# +# Copyright (c) 2024 Antonio Cerciello +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import logging +import requests +from urllib.parse import urlparse + +from pygeoapi.provider.base_mvt import BaseMVTProvider +from pygeoapi.provider.base import (ProviderConnectionError, + ProviderGenericError, + ProviderInvalidQueryError) +from pygeoapi.provider.tile import ProviderTileNotFoundError +from pygeoapi.models.provider.base import ( + TileSetMetadata, LinkType) +from pygeoapi.util import is_url, url_join + +LOGGER = logging.getLogger(__name__) + + +class MVTProxyProvider(BaseMVTProvider): + """ + MVT Proxy Provider + Provider for serving tiles rendered with an external + tiles provider + """ + + def __init__(self, BaseMVTProvider): + """ + Initialize object + + :param provider_def: provider definition + + :returns: pygeoapi.provider.mvt_proxy.pygeoapi/provider/mvt_proxy.py # noqa + """ + + super().__init__(BaseMVTProvider) + + if not is_url(self.data): + msg = 'Wrong input format for MVT' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + url = urlparse(self.data) + baseurl = f'{url.scheme}://{url.netloc}' + param_type = '?f=mvt' + layer = f'/{self.get_layer()}' + + LOGGER.debug('Extracting layer name from URL') + LOGGER.debug(f'Layer: {layer}') + + tilepath = f'{layer}/tiles' + servicepath = f'{tilepath}/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}{param_type}' # noqa + + self._service_url = url_join(baseurl, servicepath) + + self._service_metadata_url = url_join( + self.service_url.split('{tileMatrix}/{tileRow}/{tileCol}')[0], + 'metadata') + + def __repr__(self): + return f' {self.data}' + + @property + def service_url(self): + return self._service_url + + @property + def service_metadata_url(self): + return self._service_metadata_url + + def get_layer(self): + """ + Extracts layer name from url + + :returns: layer name + """ + + if not is_url(self.data): + msg = 'Wrong input format for MVT' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + url = urlparse(self.data) + + if ('/{z}/{x}/{y}' not in url.path): + msg = 'Wrong input format for MVT' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + layer = url.path.split('/{z}/{x}/{y}')[0] + + LOGGER.debug(layer) + LOGGER.debug('Removing leading "/"') + return layer[1:] + + def get_tiles_service(self, baseurl=None, servicepath=None, + dirpath=None, tile_type=None): + """ + Gets mvt service description + + :param baseurl: base URL of endpoint + :param servicepath: base path of URL + :param dirpath: directory basepath (equivalent of URL) + :param tile_type: tile format type + + :returns: `dict` of item tile service + """ + + super().get_tiles_service(baseurl, servicepath, + dirpath, tile_type) + + self._service_url = servicepath + return self.get_tms_links() + + def get_tiles(self, layer=None, tileset=None, + z=None, y=None, x=None, format_=None): + """ + Gets tile + + :param layer: mvt tile layer + :param tileset: mvt tileset + :param z: z index + :param y: y index + :param x: x index + :param format_: tile format + + :returns: an encoded mvt tile + """ + if format_ == 'mvt': + format_ = self.format_type + + if is_url(self.data): + url = urlparse(self.data) + base_url = f'{url.scheme}://{url.netloc}' + + if url.query: + url_query = f'?{url.query}' + else: + url_query = '' + + resp = None + + try: + with requests.Session() as session: + session.get(base_url) + if '.' in url.path: + resp = session.get(f'{base_url}/{layer}/{z}/{y}/{x}.{format_}{url_query}') # noqa + else: + resp = session.get(f'{base_url}/{layer}/{z}/{y}/{x}{url_query}') # noqa + + if resp.status_code == 404: + if (self.is_in_limits(self.get_tilematrixset(tileset), z, x, y)): # noqa + return None + raise ProviderTileNotFoundError + + resp.raise_for_status() + return resp.content + except requests.exceptions.RequestException as e: + LOGGER.debug(e) + if resp and resp.status_code < 500: + raise ProviderInvalidQueryError # Client is sending an invalid request # noqa + raise ProviderGenericError # Server error + else: + msg = 'Wrong input format for MVT' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + def get_html_metadata(self, dataset, server_url, layer, tileset, + title, description, keywords, **kwargs): + + service_url = url_join( + server_url, + f'collections/{dataset}/tiles/{tileset}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f=mvt') # noqa + metadata_url = url_join( + server_url, + f'collections/{dataset}/tiles/{tileset}/metadata') + + metadata = dict() + metadata['id'] = dataset + metadata['title'] = title + metadata['tileset'] = tileset + metadata['collections_path'] = service_url + metadata['json_url'] = f'{metadata_url}?f=json' + + return metadata + + def get_default_metadata(self, dataset, server_url, layer, tileset, + title, description, keywords, **kwargs): + + service_url = url_join( + server_url, + f'collections/{dataset}/tiles/{tileset}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f=mvt') # noqa + + content = {} + tiling_schemes = self.get_tiling_schemes() + # Default values + tileMatrixSetURI = tiling_schemes[0].tileMatrixSetURI + crs = tiling_schemes[0].crs + # Checking the selected matrix in configured tiling_schemes + for schema in tiling_schemes: + if (schema.tileMatrixSet == tileset): + crs = schema.crs + tileMatrixSetURI = schema.tileMatrixSetURI + + tiling_scheme_url = url_join( + server_url, f'/TileMatrixSets/{schema.tileMatrixSet}') + tiling_scheme_url_type = "application/json" + tiling_scheme_url_title = f'{schema.tileMatrixSet} tile matrix set definition' # noqa + + tiling_scheme = LinkType(href=tiling_scheme_url, + rel="http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", # noqa + type_=tiling_scheme_url_type, + title=tiling_scheme_url_title) + + if tiling_scheme is None: + msg = f'Could not identify a valid tiling schema' # noqa + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + content = TileSetMetadata(title=title, description=description, + keywords=keywords, crs=crs, + tileMatrixSetURI=tileMatrixSetURI) + + links = [] + service_url_link_type = "application/vnd.mapbox-vector-tile" + service_url_link_title = f'{tileset} vector tiles for {layer}' + service_url_link = LinkType(href=service_url, rel="item", + type_=service_url_link_type, + title=service_url_link_title) + + links.append(tiling_scheme) + links.append(service_url_link) + + content.links = links + + return content.dict(exclude_none=True) diff --git a/pygeoapi/provider/mvt_tippecanoe.py b/pygeoapi/provider/mvt_tippecanoe.py index 207524153..2cf855903 100644 --- a/pygeoapi/provider/mvt_tippecanoe.py +++ b/pygeoapi/provider/mvt_tippecanoe.py @@ -40,7 +40,7 @@ ProviderGenericError) from pygeoapi.provider.base_mvt import BaseMVTProvider from pygeoapi.models.provider.base import ( - TileSetMetadata, LinkType) + TileSetMetadata, TileMatrixSetEnum, LinkType) from pygeoapi.models.provider.mvt import MVTTilesJson from pygeoapi.util import is_url, url_join @@ -72,14 +72,13 @@ def __init__(self, BaseMVTProvider): if is_url(self.data): url = urlparse(self.data) baseurl = f'{url.scheme}://{url.netloc}' - extension = Path(url.path).suffix # e.g. ".pbf" layer = f'/{self.get_layer()}' LOGGER.debug('Extracting layer name from URL') LOGGER.debug(f'Layer: {layer}') tilepath = f'{layer}/tiles' - servicepath = f'{tilepath}/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}{extension}' # noqa + servicepath = f'{tilepath}/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f=mvt' # noqa self._service_url = url_join(baseurl, servicepath) @@ -137,6 +136,13 @@ def get_layer(self): else: return Path(self.data).name + def get_tiling_schemes(self): + + "Only WebMercatorQuad tiling scheme is supported in elastic" + return [ + TileMatrixSetEnum.WEBMERCATORQUAD.value + ] + def get_tiles_service(self, baseurl=None, servicepath=None, dirpath=None, tile_type=None): """ @@ -180,6 +186,12 @@ def get_tiles_from_url(self, layer=None, tileset=None, with requests.Session() as session: session.get(base_url) resp = session.get(f'{base_url}/{layer}/{z}/{y}/{x}{extension}') # noqa + + if resp.status_code == 404: + if (self.is_in_limits(TileMatrixSetEnum.WEBMERCATORQUAD.value, z, x, y)): # noqa + return None + raise ProviderTileNotFoundError + resp.raise_for_status() return resp.content except requests.exceptions.RequestException as e: @@ -208,6 +220,8 @@ def get_tiles_from_disk(self, layer=None, tileset=None, with open(service_url_path, mode='rb') as tile: return tile.read() except FileNotFoundError as err: + if (self.is_in_limits(TileMatrixSetEnum.WEBMERCATORQUAD.value, z, x, y)): # noqa + return None raise ProviderTileNotFoundError(err) def get_tiles(self, layer=None, tileset=None, @@ -289,7 +303,7 @@ def get_html_metadata(self, dataset, server_url, layer, tileset, content.tiles = service_url content.vector_layers = json.loads( metadata_json_content["json"])["vector_layers"] - metadata['metadata'] = content.model_dump() + metadata['metadata'] = content.dict() # Some providers may not implement tilejson metadata metadata['tilejson_url'] = f'{metadata_url}?f=tilejson' except ProviderConnectionError: @@ -326,7 +340,7 @@ def get_default_metadata(self, dataset, server_url, layer, tileset, tiling_scheme = LinkType(href=tiling_scheme_url, rel="http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", # noqa - type=tiling_scheme_url_type, + type_=tiling_scheme_url_type, title=tiling_scheme_url_title) if tiling_scheme is None: @@ -343,7 +357,7 @@ def get_default_metadata(self, dataset, server_url, layer, tileset, service_url_link_type = "application/vnd.mapbox-vector-tile" service_url_link_title = f'{tileset} vector tiles for {layer}' service_url_link = LinkType(href=service_url, rel="item", - type=service_url_link_type, + type_=service_url_link_type, title=service_url_link_title) links.append(tiling_scheme) @@ -351,7 +365,7 @@ def get_default_metadata(self, dataset, server_url, layer, tileset, content.links = links - return content.model_dump(exclude_none=True) + return content.dict(exclude_none=True) def get_vendor_metadata(self, dataset, server_url, layer, tileset, title, description, keywords, **kwargs): @@ -370,7 +384,7 @@ def get_vendor_metadata(self, dataset, server_url, layer, tileset, content.tiles = service_url content.vector_layers = json.loads( metadata_json_content["json"])["vector_layers"] - return content.model_dump() + return content.dict() except ProviderConnectionError: msg = f'No tiles metadata json available: {self.service_metadata_url}' # noqa LOGGER.error(msg) diff --git a/pygeoapi/provider/ogr.py b/pygeoapi/provider/ogr.py index 1584e6dcc..3132c2cb8 100644 --- a/pygeoapi/provider/ogr.py +++ b/pygeoapi/provider/ogr.py @@ -188,7 +188,7 @@ def __init__(self, provider_def): self.conn = None LOGGER.debug('Grabbing field information') - self.fields = self.get_fields() + self.get_fields() def _list_open_options(self): return [ @@ -260,43 +260,43 @@ def get_fields(self): :returns: dict of fields """ - fields = {} - try: - layer_defn = self._get_layer().GetLayerDefn() - for fld in range(layer_defn.GetFieldCount()): - field_defn = layer_defn.GetFieldDefn(fld) - fieldName = field_defn.GetName() - fieldTypeCode = field_defn.GetType() - fieldType = field_defn.GetFieldTypeName(fieldTypeCode) + if not self._fields: + try: + layer_defn = self._get_layer().GetLayerDefn() + for fld in range(layer_defn.GetFieldCount()): + field_defn = layer_defn.GetFieldDefn(fld) + fieldName = field_defn.GetName() + fieldTypeCode = field_defn.GetType() + fieldType = field_defn.GetFieldTypeName(fieldTypeCode) - fieldName2 = fieldType.lower() + fieldName2 = fieldType.lower() - if fieldName2 == 'integer64': - fieldName2 = 'integer' - elif fieldName2 == 'real': - fieldName2 = 'number' + if fieldName2 == 'integer64': + fieldName2 = 'integer' + elif fieldName2 == 'real': + fieldName2 = 'number' - fields[fieldName] = {'type': fieldName2} + self._fields[fieldName] = {'type': fieldName2} - if fieldName2 == 'datetime': - fields[fieldName] = { - 'type': 'string', - 'format': 'date-time' - } + if fieldName2 == 'datetime': + self._fields[fieldName] = { + 'type': 'string', + 'format': 'date-time' + } - # fieldWidth = layer_defn.GetFieldDefn(fld).GetWidth() - # GetPrecision = layer_defn.GetFieldDefn(fld).GetPrecision() + # fieldWidth = layer_defn.GetFieldDefn(fld).GetWidth() + # GetPrecision = layer_defn.GetFieldDefn(fld).GetPrecision() # noqa - except RuntimeError as err: - LOGGER.error(err) - raise ProviderConnectionError(err) - except Exception as err: - LOGGER.error(err) + except RuntimeError as err: + LOGGER.error(err) + raise ProviderConnectionError(err) + except Exception as err: + LOGGER.error(err) - finally: - self._close() + finally: + self._close() - return fields + return self._fields def query(self, offset=0, limit=10, resulttype='results', bbox=[], datetime_=None, properties=[], sortby=[], diff --git a/pygeoapi/provider/opensearch_.py b/pygeoapi/provider/opensearch_.py new file mode 100644 index 000000000..3e84de0e8 --- /dev/null +++ b/pygeoapi/provider/opensearch_.py @@ -0,0 +1,569 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Francesco Bartoli +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2024 Francesco Bartoli +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +from typing import Dict +from collections import OrderedDict +import json +import logging +import uuid + +from opensearchpy import OpenSearch, helpers +from opensearch_dsl import Search + +from pygeofilter.backends.opensearch import to_filter + +from pygeoapi.provider.base import (BaseProvider, ProviderConnectionError, + ProviderQueryError, + ProviderItemNotFoundError) +from pygeoapi.util import crs_transform + + +LOGGER = logging.getLogger(__name__) + + +class OpenSearchProvider(BaseProvider): + """OpenSearch Provider""" + + def __init__(self, provider_def): + """ + Initialize object + + :param provider_def: provider definition + + :returns: pygeoapi.provider.opensearch_.OpenSearchProvider + """ + + super().__init__(provider_def) + + self.select_properties = [] + + self.os_host, self.index_name = self.data.rsplit('/', 1) + + LOGGER.debug('Setting OpenSearch properties') + + LOGGER.debug(f'host: {self.os_host}') + LOGGER.debug(f'index: {self.index_name}') + + LOGGER.debug('Connecting to OpenSearch') + self.os_ = OpenSearch(self.os_host, verify_certs=0) + if not self.os_.ping(): + msg = f'Cannot connect to OpenSearch: {self.os_host}' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + LOGGER.debug('Determining OpenSearch version') + v = self.os_.info()['version']['number'][:3] + LOGGER.debug(f'OpenSearch version: {v}') + + LOGGER.debug('Grabbing field information') + try: + self.get_fields() + except Exception as err: + LOGGER.error(err) + raise ProviderQueryError(err) + + def get_fields(self): + """ + Get provider field information (names, types) + + :returns: dict of fields + """ + if not self._fields: + ii = self.os_.indices.get(index=self.index_name, + allow_no_indices=False) + + LOGGER.debug(f'Response: {ii}') + try: + if '*' not in self.index_name: + mappings = ii[self.index_name]['mappings'] + p = mappings['properties']['properties'] + else: + LOGGER.debug('Wildcard index; setting from first match') + index_name_ = list(ii.keys())[0] + p = ii[index_name_]['mappings']['properties']['properties'] + except KeyError: + LOGGER.warning('Trying for alias') + alias_name = next(iter(ii)) + p = ii[alias_name]['mappings']['properties']['properties'] + except IndexError: + LOGGER.warning('could not get fields; returning empty set') + return {} + + for k, v in p['properties'].items(): + if 'type' in v: + if v['type'] == 'text': + self._fields[k] = {'type': 'string'} + elif v['type'] == 'date': + self._fields[k] = {'type': 'string', 'format': 'date'} + elif v['type'] in ('float', 'long'): + self._fields[k] = {'type': 'number', + 'format': v['type']} + else: + self._fields[k] = {'type': v['type']} + + return self._fields + + @crs_transform + def query(self, offset=0, limit=10, resulttype='results', + bbox=[], datetime_=None, properties=[], sortby=[], + select_properties=[], skip_geometry=False, q=None, + filterq=None, **kwargs): + """ + query OpenSearch index + + :param offset: starting record to return (default 0) + :param limit: number of records to return (default 10) + :param resulttype: return results or hit limit (default results) + :param bbox: bounding box [minx,miny,maxx,maxy] + :param datetime_: temporal (datestamp or extent) + :param properties: list of tuples (name, value) + :param sortby: list of dicts (property, order) + :param select_properties: list of property names + :param skip_geometry: bool of whether to skip geometry (default False) + :param q: full-text search term(s) + :param filterq: filter object + + :returns: dict of 0..n GeoJSON features + """ + + self.select_properties = select_properties + + query = {'track_total_hits': True, 'query': {'bool': {'filter': []}}} + filter_ = [] + + feature_collection = { + 'type': 'FeatureCollection', + 'features': [] + } + + if resulttype == 'hits': + LOGGER.debug('hits only specified') + limit = 0 + + if bbox: + LOGGER.debug('processing bbox parameter') + minx, miny, maxx, maxy = bbox + bbox_filter = { + 'geo_shape': { + 'geometry': { + 'shape': { + 'type': 'envelope', + 'coordinates': [[minx, maxy], [maxx, miny]] + }, + 'relation': 'intersects' + } + } + } + + query['query']['bool']['filter'].append(bbox_filter) + + if datetime_ is not None: + LOGGER.debug('processing datetime parameter') + if self.time_field is None: + LOGGER.error('time_field not enabled for collection') + raise ProviderQueryError() + + time_field = self.mask_prop(self.time_field) + + if '/' in datetime_: # envelope + LOGGER.debug('detected time range') + time_begin, time_end = datetime_.split('/') + + range_ = { + 'range': { + time_field: { + 'gte': time_begin, + 'lte': time_end + } + } + } + if time_begin == '..': + range_['range'][time_field].pop('gte') + elif time_end == '..': + range_['range'][time_field].pop('lte') + + filter_.append(range_) + + else: # time instant + LOGGER.debug('detected time instant') + filter_.append({'match': {time_field: datetime_}}) + + LOGGER.debug(filter_) + query['query']['bool']['filter'].append(*filter_) + + if properties: + LOGGER.debug('processing properties') + for prop in properties: + prop_name = self.mask_prop(prop[0]) + pf = { + 'match': { + prop_name: { + 'query': prop[1] + } + } + } + query['query']['bool']['filter'].append(pf) + + if '|' not in prop[1]: + pf['match'][prop_name]['minimum_should_match'] = '100%' + + if sortby: + LOGGER.debug('processing sortby') + query['sort'] = [] + for sort in sortby: + LOGGER.debug(f'processing sort object: {sort}') + + sp = sort['property'] + + if (self.fields[sp]['type'] == 'string' + and self.fields[sp].get('format') != 'date'): + LOGGER.debug('setting OpenSearch .raw on property') + sort_property = f'{self.mask_prop(sp)}.raw' + else: + sort_property = self.mask_prop(sp) + + sort_order = 'asc' + if sort['order'] == '-': + sort_order = 'desc' + + sort_ = { + sort_property: { + 'order': sort_order + } + } + query['sort'].append(sort_) + + if q is not None: + LOGGER.debug('Adding free-text search') + query['query']['bool']['must'] = {'query_string': {'query': q}} + + query['_source'] = { + 'excludes': [ + 'properties._metadata-payload', + 'properties._metadata-schema', + 'properties._metadata-format' + ] + } + + if self.properties or self.select_properties: + LOGGER.debug('filtering properties') + + all_properties = self.get_properties() + + query['_source'] = { + 'includes': list(map(self.mask_prop, all_properties)) + } + + query['_source']['includes'].append('id') + query['_source']['includes'].append('type') + query['_source']['includes'].append('geometry') + + if skip_geometry: + LOGGER.debug('excluding geometry') + try: + query['_source']['excludes'] = ['geometry'] + except KeyError: + query['_source'] = {'excludes': ['geometry']} + try: + LOGGER.debug('querying OpenSearch') + if filterq: + LOGGER.debug(f'adding cql object: {filterq}') + query = update_query(input_query=query, cql=filterq) + LOGGER.debug(json.dumps(query, indent=4)) + + LOGGER.debug('Testing for OpenSearch scrolling') + if offset + limit > 10000: + gen = helpers.scan(client=self.os_, query=query, + preserve_order=True, + index=self.index_name) + results = {'hits': {'total': limit, 'hits': []}} + for i in range(offset + limit): + try: + if i >= offset: + results['hits']['hits'].append(next(gen)) + else: + next(gen) + except StopIteration: + break + + matched = len(results['hits']['hits']) + offset + returned = len(results['hits']['hits']) + else: + es_results = self.os_.search(index=self.index_name, + from_=offset, size=limit, + body=query) + results = es_results + matched = es_results['hits']['total']['value'] + returned = len(es_results['hits']['hits']) + + except ValueError: + pass + + feature_collection['numberMatched'] = matched + + if resulttype == 'hits': + return feature_collection + + feature_collection['numberReturned'] = returned + + LOGGER.debug('serializing features') + for feature in results['hits']['hits']: + feature_ = self.osdoc2geojson(feature) + feature_collection['features'].append(feature_) + + return feature_collection + + @crs_transform + def get(self, identifier, **kwargs): + """ + Get OpenSearch document by id + + :param identifier: feature id + + :returns: dict of single GeoJSON feature + """ + + try: + LOGGER.debug(f'Fetching identifier {identifier}') + result = self.os_.get(index=self.index_name, id=identifier) + LOGGER.debug('Serializing feature') + feature_ = self.osdoc2geojson(result) + except Exception as err: + LOGGER.debug(f'Not found via OpenSearch id query: {err}') + LOGGER.debug('Trying via a real query') + + query = { + 'query': { + 'bool': { + 'filter': [{ + 'match_phrase': { + '_id': identifier + } + }] + } + } + } + + LOGGER.debug(f'Query: {query}') + try: + result = self.os_search(index=self.index_name, **query) + if len(result['hits']['hits']) == 0: + LOGGER.error(err) + raise ProviderItemNotFoundError(err) + LOGGER.debug('Serializing feature') + feature_ = self.osdoc2geojson(result['hits']['hits'][0]) + except Exception as err2: + LOGGER.error(err2) + raise ProviderItemNotFoundError(err2) + except Exception as err: + LOGGER.error(err) + return None + + return feature_ + + def create(self, item): + """ + Create a new item + + :param item: `dict` of new item + + :returns: identifier of created item + """ + + identifier, json_data = self._load_and_prepare_item( + item, accept_missing_identifier=True) + if identifier is None: + # If there is no incoming identifier, allocate a random one + identifier = str(uuid.uuid4()) + json_data["id"] = identifier + + LOGGER.debug(f'Inserting data with identifier {identifier}') + _ = self.os_.index(index=self.index_name, id=identifier, + body=json_data) + LOGGER.debug('Item added') + + return identifier + + def update(self, identifier, item): + """ + Updates an existing item + + :param identifier: feature id + :param item: `dict` of partial or full item + + :returns: `bool` of update result + """ + + LOGGER.debug(f'Updating item {identifier}') + identifier, json_data = self._load_and_prepare_item( + item, identifier, raise_if_exists=False) + + _ = self.os_index(index=self.index_name, id=identifier, body=json_data) + + return True + + def delete(self, identifier): + """ + Deletes an existing item + + :param identifier: item id + + :returns: `bool` of deletion result + """ + + LOGGER.debug(f'Deleting item {identifier}') + _ = self.os_delete(index=self.index_name, id=identifier) + + return True + + def osdoc2geojson(self, doc): + """ + generate GeoJSON `dict` from OpenSearch document + :param doc: `dict` of OpenSearch document + + :returns: GeoJSON `dict` + """ + + feature_ = {} + feature_thinned = {} + + LOGGER.debug('Fetching id and geometry from GeoJSON document') + feature_ = doc['_source'] + + try: + id_ = doc['_source']['properties'][self.id_field] + except KeyError as err: + LOGGER.debug(f'Missing field: {err}') + id_ = doc['_source'].get('id', doc['_id']) + + feature_['id'] = id_ + feature_['geometry'] = doc['_source'].get('geometry') + + if self.properties or self.select_properties: + LOGGER.debug('Filtering properties') + all_properties = self.get_properties() + + feature_thinned = { + 'id': id_, + 'type': feature_['type'], + 'geometry': feature_.get('geometry'), + 'properties': OrderedDict() + } + for p in all_properties: + try: + feature_thinned['properties'][p] = feature_['properties'][p] # noqa + except KeyError as err: + LOGGER.error(err) + raise ProviderQueryError() + + if feature_thinned: + return feature_thinned + else: + return feature_ + + def mask_prop(self, property_name): + """ + generate property name based on OpenSearch backend setup + + :param property_name: property name + + :returns: masked property name + """ + + return f'properties.{property_name}' + + def get_properties(self): + all_properties = [] + + LOGGER.debug(f'configured properties: {self.properties}') + LOGGER.debug(f'selected properties: {self.select_properties}') + + if not self.properties and not self.select_properties: + all_properties = self.get_fields() + if self.properties and self.select_properties: + all_properties = self.properties and self.select_properties + else: + all_properties = self.properties or self.select_properties + + LOGGER.debug(f'resulting properties: {all_properties}') + return all_properties + + def __repr__(self): + return f' {self.data}' + + +class OpenSearchCatalogueProvider(OpenSearchProvider): + """OpenSearch Provider""" + + def __init__(self, provider_def): + super().__init__(provider_def) + + def _excludes(self): + return [ + 'properties._metadata-anytext' + ] + + def get_fields(self): + fields = super().get_fields() + for i in self._excludes(): + if i in fields: + del fields[i] + + fields['q'] = {'type': 'string'} + + return fields + + def query(self, offset=0, limit=10, resulttype='results', + bbox=[], datetime_=None, properties=[], sortby=[], + select_properties=[], skip_geometry=False, q=None, + filterq=None, **kwargs): + + records = super().query( + offset=offset, limit=limit, + resulttype=resulttype, bbox=bbox, + datetime_=datetime_, properties=properties, + sortby=sortby, + select_properties=select_properties, + skip_geometry=skip_geometry, + q=q) + + return records + + def __repr__(self): + return f' {self.data}' + + +def update_query(input_query: Dict, cql): + s = Search.from_dict(input_query) + s = s.query(to_filter(cql)) + + LOGGER.debug(f'Enhanced query: {json.dumps(s.to_dict())}') + return s.to_dict() diff --git a/pygeoapi/provider/oracle.py b/pygeoapi/provider/oracle.py index b899e154a..c8c973774 100644 --- a/pygeoapi/provider/oracle.py +++ b/pygeoapi/provider/oracle.py @@ -1,9 +1,10 @@ # ================================================================= # # Authors: Andreas Kosubek +# Authors: Moritz Langer # # Copyright (c) 2023 Andreas Kosubek -# +# Copyright (c) 2024 Moritz Langer # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without @@ -30,9 +31,12 @@ import importlib import json import logging +import os +import threading +from typing import Optional + import oracledb import pyproj -from typing import Optional from pygeoapi.api import DEFAULT_STORAGE_CRS @@ -54,6 +58,41 @@ class DatabaseConnection: """Database connection class to be used as 'with' statement. The class returns a connection object. """ + pool = None # Class-level connection pool + lock = threading.Lock() + + @classmethod + def create_pool(cls, conn_dict, oracle_pool_min, oracle_pool_max): + """Initialize the connection pool for the class + Lock is implemented before function call at __init__""" + dsn = cls._make_dsn(conn_dict) + + connect_kwargs = { + 'dsn': dsn, + 'min': oracle_pool_min, + 'max': oracle_pool_max, + 'increment': 1 + } + + # Create the pool + if conn_dict.get("external_auth") == "wallet": + # If Auth is via Wallet you need to save a wallet under + # the directory returned by this bash command if apache is used + # cat /etc/passwd |grep apache + # except another directory is specified in the sqlnet.ora file + LOGGER.debug("Connection pool from wallet.") + connect_kwargs["externalauth"] = True + connect_kwargs["homogeneous"] = False + + else: + LOGGER.debug("Connection pool from user and password.") + connect_kwargs["user"] = conn_dict["user"] + connect_kwargs["password"] = conn_dict["password"] + + p = oracledb.create_pool(**connect_kwargs) + LOGGER.debug("Connection pool created successfully") + + return p def __init__(self, conn_dic, table, properties=[], context="query"): """ @@ -88,106 +127,137 @@ def __init__(self, conn_dic, table, properties=[], context="query"): ) self.properties = [item.lower() for item in properties] self.fields = {} # Dict of columns. Key is col name, value is type - self.conn = None - - def __enter__(self): - try: - if self.conn_dict.get("init_oracle_client", False): - oracledb.init_oracle_client() - - # Connect with tnsnames.ora entry and Login with Oracle Wallet - if self.conn_dict.get("external_auth") == "wallet": - LOGGER.debug( - "Oracle connect with tnsnames.ora entry \ - and login with Oracle Wallet" - ) - - if "tns_name" not in self.conn_dict: - raise ProviderConnectionError( - "tns_name must be set for external authentication!" + oracle_pool_min = int(os.environ.get('ORACLE_POOL_MIN', 0)) + oracle_pool_max = int(os.environ.get('ORACLE_POOL_MAX', 0)) + # Initialize the connection pool if it hasn't been initialized + if oracle_pool_min and oracle_pool_max: + LOGGER.debug("Found environment variables for session pooling:") + LOGGER.debug(f"ORACLE_POOL_MIN: {oracle_pool_min}") + LOGGER.debug(f"ORACLE_POOL_MAX: {oracle_pool_max}") + with DatabaseConnection.lock: + if DatabaseConnection.pool is None: + LOGGER.debug(f"self.conn_dict contains {self.conn_dict}") + DatabaseConnection.pool = DatabaseConnection.create_pool( + self.conn_dict, oracle_pool_min, oracle_pool_max + ) + LOGGER.debug( + "Initialized connection pool with " + f"{DatabaseConnection.pool.max} connections" ) - dsn = self.conn_dict["tns_name"] + @staticmethod + def _make_dsn(conn_dict): + if conn_dict.get("init_oracle_client", False): + oracledb.init_oracle_client() - # Connect with SERVICE_NAME - if "service_name" in self.conn_dict: - LOGGER.debug( - f"Oracle connect with service_name: \ - {self.conn_dict['service_name']}" + # Connect with tnsnames.ora entry and Login with Oracle Wallet + if conn_dict.get("external_auth") == "wallet": + LOGGER.debug( + "Oracle connect with tnsnames.ora entry \ + and login with Oracle Wallet" + ) + if "tns_name" not in conn_dict: + raise ProviderConnectionError( + "tns_name must be set for external authentication!" ) - if "host" not in self.conn_dict: - raise ProviderConnectionError( - "Host must be set for connection with service_name!" - ) + dsn = conn_dict["tns_name"] - dsn = oracledb.makedsn( - self.conn_dict["host"], - self.conn_dict.get("port", 1521), - service_name=self.conn_dict["service_name"], - ) + # Connect with SERVICE_NAME + if "service_name" in conn_dict: + LOGGER.debug( + f"Oracle connect with service_name: \ + {conn_dict['service_name']}" + ) - # Connect with SID - elif "sid" in self.conn_dict: - LOGGER.debug( - f"Oracle connect with sid: {self.conn_dict['sid']}" + if "host" not in conn_dict: + raise ProviderConnectionError( + "Host must be set for connection with service_name!" ) - if "host" not in self.conn_dict: - raise ProviderConnectionError( - "Host must be set for connection with sid!" - ) - - dsn = oracledb.makedsn( - self.conn_dict["host"], - self.conn_dict.get("port", 1521), - sid=self.conn_dict["sid"], - ) + dsn = oracledb.makedsn( + conn_dict["host"], + conn_dict.get("port", 1521), + service_name=conn_dict["service_name"], + ) - # Connect with tnsnames.ora entry - elif "tns_name" in self.conn_dict: - LOGGER.debug( - f"Oracle connect with tns_name: \ - {self.conn_dict['tns_name']}" - ) - dsn = self.conn_dict["tns_name"] + # Connect with SID + elif "sid" in conn_dict: + LOGGER.debug( + f"Oracle connect with sid: {conn_dict['sid']}" + ) - else: + if "host" not in conn_dict: raise ProviderConnectionError( - "One of service_name, sid or tns_name must be specified!" + "Host must be set for connection with sid!" ) - LOGGER.debug(f"Oracle DSN string: {dsn}") + dsn = oracledb.makedsn( + conn_dict["host"], + conn_dict.get("port", 1521), + sid=conn_dict["sid"], + ) - # Connect with tnsnames.ora entry and Login with Oracle Wallet - if self.conn_dict.get("external_auth") == "wallet": - self.conn = oracledb.connect(externalauth=True, dsn=dsn) + # Connect with tnsnames.ora entry + elif "tns_name" in conn_dict: + LOGGER.debug( + f"Oracle connect with tns_name: \ + {conn_dict['tns_name']}" + ) + dsn = conn_dict["tns_name"] - # Connect with tnsnames.ora entry, - # TNS_ADMIN is set via configuration - if "tns_admin" in self.conn_dict: - self.conn = oracledb.connect( - user=self.conn_dict["user"], - password=self.conn_dict["password"], - dsn=dsn, - config_dir=self.conn_dict["tns_admin"], - ) + else: + raise ProviderConnectionError( + "One of service_name, sid or tns_name must be specified!" + ) + + LOGGER.debug(f"Oracle DSN string: {dsn}") - # Connect with user / password via dsn string - # When dsn is a TNS name, the environment variable TNS_ADMIN must - # be set (Path to tnsnames.ora file) + return dsn + + def __enter__(self): + """Acquires a connection from the pool.""" + try: + if DatabaseConnection.pool: + self.conn = DatabaseConnection.pool.acquire() + LOGGER.debug("Connection acquired from pool .") + LOGGER.debug(f"Connection from pool is {self.conn}.") else: - self.conn = oracledb.connect( - user=self.conn_dict["user"], - password=self.conn_dict["password"], - dsn=dsn, - ) + dsn = self._make_dsn(self.conn_dict) + LOGGER.debug(f"Created dsn for single connection with params: {dsn}") # noqa + # Connect with tnsnames.ora entry and Login with Oracle Wallet # noqa + if self.conn_dict.get("external_auth") == "wallet": + self.conn = oracledb.connect(externalauth=True, dsn=dsn) + + # Connect with tnsnames.ora entry, + # TNS_ADMIN is set via configuration + if "tns_admin" in self.conn_dict: + self.conn = oracledb.connect( + user=self.conn_dict["user"], + password=self.conn_dict["password"], + dsn=dsn, + config_dir=self.conn_dict["tns_admin"], + ) + + # Connect with user / password via dsn string + # When dsn is a TNS name, the environment variable TNS_ADMIN must # noqa + # be set (Path to tnsnames.ora file) + else: + self.conn = oracledb.connect( + user=self.conn_dict["user"], + password=self.conn_dict["password"], + dsn=dsn, + ) except oracledb.DatabaseError as e: - LOGGER.error( - f"Couldn't connect to Oracle using:{str(self.conn_dict)}" - ) - LOGGER.error(e) + if DatabaseConnection.pool: + LOGGER.error("Couldn't acquire a connection from the pool.") + LOGGER.error(e) + else: + LOGGER.error( + f"Couldn't connect to Oracle using:{str(self.conn_dict)}" + ) + LOGGER.error(e) raise ProviderConnectionError(e) # Check if table name has schema/owner inside @@ -204,36 +274,47 @@ def __enter__(self): LOGGER.debug("Table: " + table) if self.context == "query": - column_list = self._get_table_columns(schema, table) - - # When self.properties is set, then the result would be filtered - if self.properties: - column_list = [ - col - for col in column_list - if col[0].lower() - in [item.lower() for item in self.properties] - ] - - # Concatenate column names with ', ' - self.columns = ", ".join([item[0].lower() for item in column_list]) + columns = dict(self._get_table_columns(schema, table)) # Populate dictionary for columns with column type - for k, v in dict(column_list).items(): + # NOTE: we want all columns available here because they are + # used for filtering in the where clause, not only + # the ones that are returned to the client. + for k, v in columns.items(): self.fields[k.lower()] = {"type": v} + filtered_columns = set(self.fields) + if self.properties: + filtered_columns &= {k.lower() for k in self.properties} + + # fields which are part of the output + self.filtered_fields = { + k: v for k, v in self.fields.items() if k in filtered_columns + } + return self def __exit__(self, exc_type, exc_val, exc_tb): - # some logic to commit/rollback - self.conn.close() + """ + Releases the connection back to the pool. + """ + try: + if DatabaseConnection.pool: + DatabaseConnection.pool.release(self.conn) + LOGGER.debug("Connection released back to pool.") + else: + self.conn.close() + LOGGER.debug("Single Connection closed") + except oracledb.DatabaseError as e: + LOGGER.error("Error closing the connection.") + LOGGER.error(e) def _get_table_columns(self, schema, table): """ Returns an array with all column names and data types from Oracle table ALL_TAB_COLUMNS. Lookup for public and private synonyms. - Throws ProviderGenericError when table not exist or accesable. + Throws ProviderGenericError when table not exist or accessible. """ sql = """ @@ -327,6 +408,7 @@ def __init__(self, provider_def): self.geom = provider_def["geom_field"] self.properties = [item.lower() for item in self.properties] self.mandatory_properties = provider_def.get("mandatory_properties") + self.extra_properties = provider_def.get("extra_properties", []) # SQL manipulator properties self.sql_manipulator = provider_def.get("sql_manipulator") @@ -367,12 +449,12 @@ def get_fields(self): """ LOGGER.debug("Get available fields/properties") - if not self.fields: + if not self._fields: with DatabaseConnection( self.conn_dic, self.table, properties=self.properties ) as db: - self.fields = db.fields - return self.fields + self._fields = db.fields + return self._fields def _get_where_clauses( self, @@ -496,6 +578,12 @@ def _get_orderby(self, sortby): return f"ORDER BY {','.join(ret)}" + def _get_extra_columns_expression(self): + """Returns part of SELECT clause for extra properties""" + return "".join( + f", {e_prop}" for e_prop in self.extra_properties + ) + def _output_type_handler( self, cursor, name, default_type, size, precision, scale ): @@ -525,6 +613,38 @@ def _get_srid_from_crs(self, crs): return srid + def _process_query_with_sql_manipulator_sup( + self, db, sql_query, bind_variables, extra_params, **query_args + ): + """ + Apply the SQL manipulation plugin to process the SQL query. + + :param db: Database connection instance + :param sql_query: The SQL query to process + :param bind_variables: Query bind variables + :param extra_params: Additional parameters for manipulation + :param query_args: Other dynamic arguments required for processing + :return: Processed SQL query and bind variables + """ + if self.sql_manipulator: + LOGGER.debug(f"sql_manipulator: {self.sql_manipulator}") + manipulation_class = _class_factory(self.sql_manipulator) + + # Pass all arguments to the process_query method + sql_query, bind_variables = manipulation_class.process_query( + db=db, + sql_query=sql_query, + bind_variables=bind_variables, + sql_manipulator_options=self.sql_manipulator_options, + **query_args, + extra_params=extra_params, + ) + + for placeholder in ["#HINTS#", "#JOIN#", "#WHERE#"]: + sql_query = sql_query.replace(placeholder, "") + + return sql_query, bind_variables + def query( self, offset=0, @@ -559,6 +679,19 @@ def query( :returns: GeoJSON FeaturesCollection """ + LOGGER.debug(f"properties contains: {properties}") + + # NOTE: properties contains field keys plus extra params + # need to split them up here + filtered_properties = [] + extra_params = {} + for (key, value) in properties: + if key in self.fields.keys(): + filtered_properties.append((key, value)) + else: + extra_params[key] = value + + properties = filtered_properties # Check mandatory filter properties property_dict = dict(properties) @@ -574,8 +707,6 @@ def query( f"Missing mandatory filter property: {mand_col}" ) - # GA customisation - unindented until Line 755 (pagination) - # if resulttype == "hits": with DatabaseConnection( self.conn_dic, self.table, @@ -596,9 +727,36 @@ def query( # because of getFields ... sql_query = f"SELECT COUNT(1) AS hits \ FROM {self.table} \ - {where_dict['clause']}" + {where_dict['clause']} #WHERE#" + + # Assign where_dict["properties"] to bind_variables + bind_variables = {**where_dict["properties"]} + + # Default values for the process_query function (sql_manipulator) + query_args = { + "offset": offset, + "limit": limit, + "resulttype": resulttype, + "bbox": bbox, + "datetime_": datetime_, + "properties": properties, + "sortby": sortby, + "skip_geometry": skip_geometry, + "select_properties": select_properties, + "crs_transform_spec": crs_transform_spec, + "q": q, + "language": language, + "filterq": filterq, + } + + # Apply the SQL manipulation plugin + extra_params["geom"] = self.geom + sql_query, bind_variables = self._process_query_with_sql_manipulator_sup( # noqa: E501 + db, sql_query, bind_variables, extra_params, **query_args + ) + try: - cursor.execute(sql_query, where_dict["properties"]) + cursor.execute(sql_query, bind_variables) except oracledb.Error as err: LOGGER.error( f"Error executing sql_query: {sql_query}: {err}" @@ -608,9 +766,6 @@ def query( hits = cursor.fetchone()[0] LOGGER.debug(f"hits: {str(hits)}") - # GA customisation - desable the return below (pagination) - # return self._response_feature_hits(hits) - with DatabaseConnection( self.conn_dic, self.table, properties=self.properties ) as db: @@ -621,10 +776,10 @@ def query( # Create column list. # Uses columns field that was generated in the Connection class # or the configured columns from the Yaml file. - props = ( - db.columns - if select_properties == [] - else ", ".join([p for p in select_properties]) + props = ", ".join( + select_properties + if select_properties + else db.filtered_fields ) where_dict = self._get_where_clauses( @@ -659,7 +814,7 @@ def query( LOGGER.debug(f"target_srid: {target_srid}") # Build geometry column call - # When a different output CRS is definded, the geometry + # When a different output CRS is defined, the geometry # geometry column would be transformed. if skip_geometry: geom = "" @@ -682,14 +837,16 @@ def query( # SQL manipulation class paging_bind = {} if limit > 0: - sql_query = f"SELECT #HINTS# {props} {geom} \ + sql_query = f"SELECT #HINTS# {props} \ + {self._get_extra_columns_expression()} {geom} \ FROM {self.table} t1 #JOIN# \ {where_dict['clause']} #WHERE# \ {orderby} \ OFFSET :offset ROWS FETCH NEXT :limit ROWS ONLY" paging_bind = {"offset": offset, "limit": limit} else: - sql_query = f"SELECT #HINTS# {props} {geom} \ + sql_query = f"SELECT #HINTS# {props} \ + {self._get_extra_columns_expression()} {geom} \ FROM {self.table} t1 #JOIN# \ {where_dict['clause']} #WHERE# \ {orderby}" @@ -697,35 +854,10 @@ def query( # Create dictionary for sql bind variables bind_variables = {**where_dict["properties"], **paging_bind} - # SQL manipulation plugin - if self.sql_manipulator: - LOGGER.debug("sql_manipulator: " + self.sql_manipulator) - manipulation_class = _class_factory(self.sql_manipulator) - sql_query, bind_variables = manipulation_class.process_query( - db, - sql_query, - bind_variables, - self.sql_manipulator_options, - offset, - limit, - resulttype, - bbox, - datetime_, - properties, - sortby, - skip_geometry, - select_properties, - crs_transform_spec, - q, - language, - filterq, - ) - - # Clean up placeholders that aren't used by the - # manipulation class. - sql_query = sql_query.replace("#HINTS#", "") - sql_query = sql_query.replace("#JOIN#", "") - sql_query = sql_query.replace("#WHERE#", "") + # Apply the SQL manipulation plugin + sql_query, bind_variables = self._process_query_with_sql_manipulator_sup( # noqa: E501 + db, sql_query, bind_variables, extra_params, **query_args + ) LOGGER.debug(f"SQL Query: {sql_query}") LOGGER.debug(f"Bind variables: {bind_variables}") @@ -850,7 +982,7 @@ def get(self, identifier, crs_transform_spec=None, **kwargs): LOGGER.debug(f"target_srid: {target_srid}") # Build geometry column call - # When a different output CRS is definded, the geometry + # When a different output CRS is defined, the geometry # geometry column would be transformed. if source_srid != target_srid: crs_dict = {"target_srid": target_srid} @@ -862,7 +994,10 @@ def get(self, identifier, crs_transform_spec=None, **kwargs): else: geom_sql = f", t1.{self.geom}.get_geojson() AS geometry " - sql_query = f"SELECT {db.columns} {geom_sql} \ + columns = ", ".join(db.filtered_fields) + sql_query = f"SELECT {columns} \ + {self._get_extra_columns_expression()} \ + {geom_sql} \ FROM {self.table} t1 \ WHERE {self.id_field} = :in_id" @@ -975,15 +1110,13 @@ def create(self, request_data): columns = [ col for col in columns - if col.lower() in [field.lower() for field in self.fields] + if col.lower() in db.filtered_fields ] - # Flter function to get only properties who are + # Filter function to get only properties who are # in the column list def filter_binds(pair): - return pair[0].lower() in [ - field.lower() for field in self.fields - ] + return pair[0].lower() in db.filtered_fields # Filter bind variables bind_variables = dict( @@ -1074,15 +1207,13 @@ def update(self, identifier, request_data): columns = [ col for col in columns - if col.lower() in [field.lower() for field in self.fields] + if col.lower() in db.filtered_fields ] - # Flter function to get only properties who are + # Filter function to get only properties who are # in the column list def filter_binds(pair): - return pair[0].lower() in [ - field.lower() for field in self.fields - ] + return pair[0].lower() in db.filtered_fields # Filter bind variables bind_variables = dict( diff --git a/pygeoapi/provider/parquet.py b/pygeoapi/provider/parquet.py new file mode 100644 index 000000000..1d6314c27 --- /dev/null +++ b/pygeoapi/provider/parquet.py @@ -0,0 +1,458 @@ +# ================================================================= +# +# Authors: Leo Ghignone +# +# Copyright (c) 2024 Leo Ghignone +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +from itertools import chain +import json +import logging + +from dateutil.parser import isoparse +import geopandas as gpd +import pyarrow +import pyarrow.compute as pc +import pyarrow.dataset +import s3fs + +from pygeoapi.provider.base import ( + BaseProvider, + ProviderConnectionError, + ProviderGenericError, + ProviderItemNotFoundError, + ProviderQueryError, +) +from pygeoapi.util import crs_transform + +LOGGER = logging.getLogger(__name__) + + +def arrow_to_pandas_type(arrow_type): + pd_type = arrow_type.to_pandas_dtype() + try: + # Needed for specific types such as dtype(' pc.scalar(minx)) + & (pc.field(self.miny) > pc.scalar(miny)) + & (pc.field(self.maxx) < pc.scalar(maxx)) + & (pc.field(self.maxy) < pc.scalar(maxy)) + ) + + if datetime_ is not None: + if self.time_field is None: + msg = ( + 'Dataset does not have a time field, ' + 'querying by datetime is not supported.' + ) + raise ProviderQueryError(msg) + timefield = pc.field(self.time_field) + if '/' in datetime_: + begin, end = datetime_.split('/') + if begin != '..': + begin = isoparse(begin) + filter = filter & (timefield >= begin) + if end != '..': + end = isoparse(end) + filter = filter & (timefield <= end) + else: + target_time = isoparse(datetime_) + filter = filter & (timefield == target_time) + + if properties: + LOGGER.debug('processing properties') + for name, value in properties: + field = self.ds.schema.field(name) + pd_type = arrow_to_pandas_type(field.type) + expr = pc.field(name) == pc.scalar(pd_type(value)) + + filter = filter & expr + + if len(select_properties) == 0: + select_properties = self.ds.schema.names + else: # Load id and geometry together with any specified columns + if self.has_geometry and 'geometry' not in select_properties: + select_properties.append('geometry') + if self.id_field not in select_properties: + select_properties.insert(0, self.id_field) + + if skip_geometry: + select_properties.remove('geometry') + + # Make response based on resulttype specified + if resulttype == 'hits': + LOGGER.debug('hits only specified') + result = self._response_feature_hits(filter) + elif resulttype == 'results': + LOGGER.debug('results specified') + result = self._response_feature_collection( + filter, offset, limit, columns=select_properties + ) + else: + LOGGER.error(f'Invalid resulttype: {resulttype}') + + except RuntimeError as err: + LOGGER.error(err) + raise ProviderQueryError(err) + except ProviderConnectionError as err: + LOGGER.error(err) + raise ProviderConnectionError(err) + except Exception as err: + LOGGER.error(err) + raise ProviderGenericError(err) + + return result + + @crs_transform + def get(self, identifier, **kwargs): + """ + Get Feature by id + + :param identifier: feature id + + :returns: a single feature + """ + result = None + try: + LOGGER.debug(f'Fetching identifier {identifier}') + id_type = arrow_to_pandas_type( + self.ds.schema.field(self.id_field).type) + batches = self._read_parquet( + filter=( + pc.field(self.id_field) == pc.scalar(id_type(identifier)) + ) + ) + + for batch in batches: + if batch.num_rows > 0: + assert ( + batch.num_rows == 1 + ), f'Multiple items found with ID {identifier}' + row = batch.to_pandas() + break + else: + raise ProviderItemNotFoundError(f'ID {identifier} not found') + + if self.has_geometry: + geom = gpd.GeoSeries.from_wkb(row['geometry'], crs=self.crs) + else: + geom = [None] + gdf = gpd.GeoDataFrame(row, geometry=geom) + LOGGER.debug('results computed') + + # Grab the collection from geopandas geo_interface + result = gdf.__geo_interface__['features'][0] + + except RuntimeError as err: + LOGGER.error(err) + raise ProviderQueryError(err) + except ProviderConnectionError as err: + LOGGER.error(err) + raise ProviderConnectionError(err) + except ProviderItemNotFoundError as err: + LOGGER.error(err) + raise ProviderItemNotFoundError(err) + except Exception as err: + LOGGER.error(err) + raise ProviderGenericError(err) + + return result + + def __repr__(self): + return f' {self.data}' + + def _response_feature_collection(self, filter, offset, limit, + columns=None): + """ + Assembles output from query as + GeoJSON FeatureCollection structure. + + :returns: GeoJSON FeatureCollection + """ + + LOGGER.debug(f'offset:{offset}, limit:{limit}') + + try: + batches, scanner = self._read_parquet( + filter=filter, columns=columns, return_scanner=True + ) + + # Discard batches until offset is reached + counted = 0 + for batch in batches: + if counted + batch.num_rows > offset: + # Slice current batch to start from the requested row + batch = batch.slice(offset=offset - counted) + # Build a new generator yielding the current batch + # and all following ones + + batches = chain([batch], batches) + break + else: + counted += batch.num_rows + + # batches is a generator, it will now be either fully spent + # or set to the new generator starting from offset + + # Get the next `limit+1` rows + # The extra row is used to check if a "next" link is needed + # (when numberMatched > offset + limit) + batches_list = [] + read = 0 + + for batch in batches: + read += batch.num_rows + if read > limit: + batches_list.append(batch.slice(0, limit + 1)) + break + else: + batches_list.append(batch) + + # Passing schema from scanner in case no rows are returned + table = pyarrow.Table.from_batches( + batches_list, schema=scanner.projected_schema + ) + + rp = table.to_pandas() + + number_matched = offset + len(rp) + + # Remove the extra row + if len(rp) > limit: + rp = rp.iloc[:-1] + + if 'geometry' not in rp.columns: + # We need a null geometry column to create a GeoDataFrame + rp['geometry'] = None + geom = gpd.GeoSeries.from_wkb(rp['geometry']) + else: + geom = gpd.GeoSeries.from_wkb(rp['geometry'], crs=self.crs) + + gdf = gpd.GeoDataFrame(rp, geometry=geom) + LOGGER.debug('results computed') + result = gdf.__geo_interface__ + + # Add numberMatched to generate "next" link + result['numberMatched'] = number_matched + + return result + + except RuntimeError as error: + LOGGER.error(error) + raise error + + def _response_feature_hits(self, filter): + """ + Assembles GeoJSON hits from row count + + :returns: GeoJSON FeaturesCollection + """ + + try: + scanner = pyarrow.dataset.Scanner.from_dataset(self.ds, + filter=filter) + return { + 'type': 'FeatureCollection', + 'numberMatched': scanner.count_rows(), + 'features': [], + } + except Exception as error: + LOGGER.error(error) + raise error diff --git a/pygeoapi/provider/postgresql.py b/pygeoapi/provider/postgresql.py index 47d9b7e59..6a52881ad 100644 --- a/pygeoapi/provider/postgresql.py +++ b/pygeoapi/provider/postgresql.py @@ -6,11 +6,13 @@ # John A Stevenson # Colin Blackburn # Francesco Bartoli +# Bernhard Mallinger # # Copyright (c) 2018 Jorge Samuel Mendes de Jesus -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2025 Tom Kralidis # Copyright (c) 2022 John A Stevenson and Colin Blackburn # Copyright (c) 2023 Francesco Bartoli +# Copyright (c) 2024 Bernhard Mallinger # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -48,29 +50,33 @@ # gunzip < tests/data/hotosm_bdi_waterways.sql.gz | # psql -U postgres -h 127.0.0.1 -p 5432 test +from copy import deepcopy +from datetime import datetime +from decimal import Decimal +import functools import logging -from copy import deepcopy from geoalchemy2 import Geometry # noqa - this isn't used explicitly but is needed to process Geometry columns from geoalchemy2.functions import ST_MakeEnvelope -from geoalchemy2.shape import to_shape +from geoalchemy2.shape import to_shape, from_shape from pygeofilter.backends.sqlalchemy.evaluate import to_filter import pyproj import shapely -from sqlalchemy import create_engine, MetaData, PrimaryKeyConstraint, asc, desc +from sqlalchemy import create_engine, MetaData, PrimaryKeyConstraint, asc, \ + desc, delete from sqlalchemy.engine import URL -from sqlalchemy.exc import InvalidRequestError, OperationalError +from sqlalchemy.exc import ConstraintColumnNotFoundError, \ + InvalidRequestError, OperationalError from sqlalchemy.ext.automap import automap_base from sqlalchemy.orm import Session, load_only from sqlalchemy.sql.expression import and_ from pygeoapi.provider.base import BaseProvider, \ - ProviderConnectionError, ProviderQueryError, ProviderItemNotFoundError + ProviderConnectionError, ProviderInvalidDataError, ProviderQueryError, \ + ProviderItemNotFoundError from pygeoapi.util import get_transform_from_crs -_ENGINE_STORE = {} -_TABLE_MODEL_STORE = {} LOGGER = logging.getLogger(__name__) @@ -102,14 +108,36 @@ def __init__(self, provider_def): LOGGER.debug(f'ID field: {self.id_field}') LOGGER.debug(f'Geometry field: {self.geom}') + # conforming to the docs: + # https://docs.pygeoapi.io/en/latest/data-publishing/ogcapi-features.html#connection-examples # noqa + self.storage_crs = provider_def.get( + 'storage_crs', + 'https://www.opengis.net/def/crs/OGC/0/CRS84' + ) + LOGGER.debug(f'Configured Storage CRS: {self.storage_crs}') + # Read table information from database options = None if provider_def.get('options'): options = provider_def['options'] self._store_db_parameters(provider_def['data'], options) - self._engine, self.table_model = self._get_engine_and_table_model() + self._engine = get_engine( + self.db_host, + self.db_port, + self.db_name, + self.db_user, + self._db_password, + **(self.db_options or {}) + ) + self.table_model = get_table_model( + self.table, + self.id_field, + self.db_search_path, + self._engine + ) + LOGGER.debug(f'DB connection: {repr(self._engine.url)}') - self.fields = self.get_fields() + self.get_fields() def query(self, offset=0, limit=10, resulttype='results', bbox=[], datetime_=None, properties=[], sortby=[], @@ -140,6 +168,7 @@ def query(self, offset=0, limit=10, resulttype='results', property_filters = self._get_property_filters(properties) cql_filters = self._get_cql_filters(filterq) bbox_filter = self._get_bbox_filter(bbox) + time_filter = self._get_datetime_filter(datetime_) order_by_clauses = self._get_order_by_clauses(sortby, self.table_model) selected_properties = self._select_properties_clause(select_properties, skip_geometry) @@ -151,15 +180,10 @@ def query(self, offset=0, limit=10, resulttype='results', .filter(property_filters) .filter(cql_filters) .filter(bbox_filter) - .order_by(*order_by_clauses) - .options(selected_properties) - .offset(offset)) + .filter(time_filter) + .options(selected_properties)) matched = results.count() - if limit < matched: - returned = limit - else: - returned = matched LOGGER.debug(f'Found {matched} result(s)') @@ -168,14 +192,16 @@ def query(self, offset=0, limit=10, resulttype='results', 'type': 'FeatureCollection', 'features': [], 'numberMatched': matched, - 'numberReturned': returned + 'numberReturned': 0 } if resulttype == "hits" or not results: - response['numberReturned'] = 0 return response + crs_transform_out = self._get_crs_transform(crs_transform_spec) - for item in results.limit(limit): + + for item in results.order_by(*order_by_clauses).offset(offset).limit(limit): # noqa + response['numberReturned'] += 1 response['features'].append( self._sqlalchemy_to_feature(item, crs_transform_out) ) @@ -188,39 +214,64 @@ def get_fields(self): :returns: dict of fields """ + LOGGER.debug('Get available fields/properties') # sql-schema only allows these types, so we need to map from sqlalchemy # string, number, integer, object, array, boolean, null, # https://json-schema.org/understanding-json-schema/reference/type.html column_type_map = { - str: 'string', + bool: 'boolean', + datetime: 'string', + Decimal: 'number', + dict: 'object', float: 'number', int: 'integer', - bool: 'boolean', + str: 'string' + } + default_type = 'string' + + # https://json-schema.org/understanding-json-schema/reference/string#built-in-formats # noqa + column_format_map = { + 'date': 'date', + 'interval': 'duration', + 'time': 'time', + 'timestamp': 'date-time' } - default_value = 'string' def _column_type_to_json_schema_type(column_type): try: python_type = column_type.python_type except NotImplementedError: LOGGER.warning(f'Unsupported column type {column_type}') - return default_value + return default_type else: try: return column_type_map[python_type] except KeyError: LOGGER.warning(f'Unsupported column type {column_type}') - return default_value + return default_type - return { - str(column.name): { - 'type': _column_type_to_json_schema_type(column.type) - } - for column in self.table_model.__table__.columns - if column.name != self.geom # Exclude geometry column - } + def _column_format_to_json_schema_format(column_type): + try: + ct = str(column_type).lower() + return column_format_map[ct] + except KeyError: + LOGGER.debug('No string format detected') + return None + + if not self._fields: + for column in self.table_model.__table__.columns: + LOGGER.debug(f'Testing {column.name}') + if column.name == self.geom: + continue + + self._fields[str(column.name)] = { + 'type': _column_type_to_json_schema_type(column.type), + 'format': _column_format_to_json_schema_format(column.type) + } + + return self._fields def get(self, identifier, crs_transform_spec=None, **kwargs): """ @@ -269,120 +320,76 @@ def get(self, identifier, crs_transform_spec=None, **kwargs): return feature - def _store_db_parameters(self, parameters, options): - self.db_user = parameters.get('user') - self.db_host = parameters.get('host') - self.db_port = parameters.get('port', 5432) - self.db_name = parameters.get('dbname') - self.db_search_path = parameters.get('search_path', ['public']) - self._db_password = parameters.get('password') - self.db_options = options - - def _get_engine_and_table_model(self): + def create(self, item): """ - Create a SQL Alchemy engine for the database and reflect the table - model. Use existing versions from stores if available to allow reuse - of Engine connection pool and save expensive table reflection. + Create a new item + + :param item: `dict` of new item + + :returns: identifier of created item """ - # One long-lived engine is used per database URL: - # https://docs.sqlalchemy.org/en/14/core/connections.html#basic-usage - engine_store_key = (self.db_user, self.db_host, self.db_port, - self.db_name) - try: - engine = _ENGINE_STORE[engine_store_key] - except KeyError: - conn_str = URL.create( - 'postgresql+psycopg2', - username=self.db_user, - password=self._db_password, - host=self.db_host, - port=self.db_port, - database=self.db_name - ) - conn_args = { - 'client_encoding': 'utf8', - 'application_name': 'pygeoapi' - } - if self.db_options: - conn_args.update(self.db_options) - engine = create_engine( - conn_str, - connect_args=conn_args, - pool_pre_ping=True) - _ENGINE_STORE[engine_store_key] = engine - - # Reuse table model if one exists - table_model_store_key = (self.db_host, self.db_port, self.db_name, - self.table) - try: - table_model = _TABLE_MODEL_STORE[table_model_store_key] - except KeyError: - table_model = self._reflect_table_model(engine) - _TABLE_MODEL_STORE[table_model_store_key] = table_model - return engine, table_model + identifier, json_data = self._load_and_prepare_item( + item, accept_missing_identifier=True) + + new_instance = self._feature_to_sqlalchemy(json_data, identifier) + with Session(self._engine) as session: + session.add(new_instance) + session.commit() + result_id = getattr(new_instance, self.id_field) + + # NOTE: need to use id from instance in case it's generated + return result_id - def _reflect_table_model(self, engine): + def update(self, identifier, item): """ - Reflect database metadata to create a SQL Alchemy model corresponding - to target table. This requires a database query and is expensive to - perform. + Updates an existing item + + :param identifier: feature id + :param item: `dict` of partial or full item + + :returns: `bool` of update result """ - metadata = MetaData() - # Look for table in the first schema in the search path - try: - schema = self.db_search_path[0] - metadata.reflect( - bind=engine, schema=schema, only=[self.table], views=True) - except OperationalError: - msg = (f"Could not connect to {repr(engine.url)} " - "(password hidden).") - raise ProviderConnectionError(msg) - except InvalidRequestError: - msg = (f"Table '{self.table}' not found in schema '{schema}' " - f"on {repr(engine.url)}.") - raise ProviderQueryError(msg) - - # Create SQLAlchemy model from reflected table - # It is necessary to add the primary key constraint because SQLAlchemy - # requires it to reflect the table, but a view in a PostgreSQL database - # does not have a primary key defined. - sqlalchemy_table_def = metadata.tables[f'{schema}.{self.table}'] - try: - sqlalchemy_table_def.append_constraint( - PrimaryKeyConstraint(self.id_field) - ) - except KeyError: - msg = (f"No such id_field column ({self.id_field}) on " - f"{schema}.{self.table}.") - raise ProviderQueryError(msg) - - Base = automap_base(metadata=metadata) - Base.prepare( - name_for_scalar_relationship=self._name_for_scalar_relationship, - ) - TableModel = getattr(Base.classes, self.table) + identifier, json_data = self._load_and_prepare_item( + item, raise_if_exists=False) + + new_instance = self._feature_to_sqlalchemy(json_data, identifier) + with Session(self._engine) as session: + session.merge(new_instance) + session.commit() - return TableModel + return True - @staticmethod - def _name_for_scalar_relationship( - base, local_cls, referred_cls, constraint, - ): - """Function used when automapping classes and relationships from - database schema and fixes potential naming conflicts. + def delete(self, identifier): """ - name = referred_cls.__name__.lower() - local_table = local_cls.__table__ - if name in local_table.columns: - newname = name + '_' - LOGGER.debug( - f'Already detected column name {name!r} in table ' - f'{local_table!r}. Using {newname!r} for relationship name.' + Deletes an existing item + + :param identifier: item id + + :returns: `bool` of deletion result + """ + with Session(self._engine) as session: + id_column = getattr(self.table_model, self.id_field) + result = session.execute( + delete(self.table_model) + .where(id_column == identifier) ) - return newname - return name + session.commit() + + return result.rowcount > 0 + + def _store_db_parameters(self, parameters, options): + self.db_user = parameters.get('user') + self.db_host = parameters.get('host') + self.db_port = parameters.get('port', 5432) + self.db_name = parameters.get('dbname') + # db_search_path gets converted to a tuple here in order to ensure it + # is hashable - which allows us to use functools.cache() when + # reflecting the table definition from the DB + self.db_search_path = tuple(parameters.get('search_path', ['public'])) + self._db_password = parameters.get('password') + self.db_options = options def _sqlalchemy_to_feature(self, item, crs_transform_out=None): feature = { @@ -408,6 +415,26 @@ def _sqlalchemy_to_feature(self, item, crs_transform_out=None): return feature + def _feature_to_sqlalchemy(self, json_data, identifier=None): + attributes = {**json_data['properties']} + # 'identifier' key maybe be present in geojson properties, but might + # not be a valid db field + attributes.pop('identifier', None) + attributes[self.geom] = from_shape( + shapely.geometry.shape(json_data['geometry']), + # NOTE: for some reason, postgis in the github action requires + # explicit crs information. i think it's valid to assume 4326: + # https://portal.ogc.org/files/108198#feature-crs + srid=pyproj.CRS.from_user_input(self.storage_crs).to_epsg() + ) + attributes[self.id_field] = identifier + + try: + return self.table_model(**attributes) + except Exception as e: + LOGGER.exception('Failed to create db model') + raise ProviderInvalidDataError(str(e)) + def _get_order_by_clauses(self, sort_by, table_model): # Build sort_by clauses if provided clauses = [] @@ -459,6 +486,29 @@ def _get_bbox_filter(self, bbox): return bbox_filter + def _get_datetime_filter(self, datetime_): + if datetime_ in (None, '../..'): + return True + else: + if self.time_field is None: + LOGGER.error('time_field not enabled for collection') + raise ProviderQueryError() + + time_column = getattr(self.table_model, self.time_field) + + if '/' in datetime_: # envelope + LOGGER.debug('detected time range') + time_begin, time_end = datetime_.split('/') + if time_begin == '..': + datetime_filter = time_column <= time_end + elif time_end == '..': + datetime_filter = time_column >= time_begin + else: + datetime_filter = time_column.between(time_begin, time_end) + else: + datetime_filter = time_column == datetime_ + return datetime_filter + def _select_properties_clause(self, select_properties, skip_geometry): # List the column names that we want if select_properties: @@ -495,3 +545,91 @@ def _get_crs_transform(self, crs_transform_spec=None): else: crs_transform = None return crs_transform + + +@functools.cache +def get_engine( + host: str, + port: str, + database: str, + user: str, + password: str, + **connection_options +): + """Create SQL Alchemy engine.""" + conn_str = URL.create( + 'postgresql+psycopg2', + username=user, + password=password, + host=host, + port=int(port), + database=database + ) + conn_args = { + 'client_encoding': 'utf8', + 'application_name': 'pygeoapi', + **connection_options, + } + engine = create_engine( + conn_str, + connect_args=conn_args, + pool_pre_ping=True) + return engine + + +@functools.cache +def get_table_model( + table_name: str, + id_field: str, + db_search_path: tuple[str], + engine, +): + """Reflect table.""" + metadata = MetaData() + + # Look for table in the first schema in the search path + schema = db_search_path[0] + try: + metadata.reflect( + bind=engine, schema=schema, only=[table_name], views=True) + except OperationalError: + raise ProviderConnectionError( + f"Could not connect to {repr(engine.url)} (password hidden).") + except InvalidRequestError: + raise ProviderQueryError( + f"Table '{table_name}' not found in schema '{schema}' " + f"on {repr(engine.url)}." + ) + + # Create SQLAlchemy model from reflected table + # It is necessary to add the primary key constraint because SQLAlchemy + # requires it to reflect the table, but a view in a PostgreSQL database + # does not have a primary key defined. + sqlalchemy_table_def = metadata.tables[f'{schema}.{table_name}'] + try: + sqlalchemy_table_def.append_constraint(PrimaryKeyConstraint(id_field)) + except (ConstraintColumnNotFoundError, KeyError): + raise ProviderQueryError( + f"No such id_field column ({id_field}) on {schema}.{table_name}.") + + _Base = automap_base(metadata=metadata) + _Base.prepare( + name_for_scalar_relationship=_name_for_scalar_relationship, + ) + return getattr(_Base.classes, table_name) + + +def _name_for_scalar_relationship(base, local_cls, referred_cls, constraint): + """Function used when automapping classes and relationships from + database schema and fixes potential naming conflicts. + """ + name = referred_cls.__name__.lower() + local_table = local_cls.__table__ + if name in local_table.columns: + newname = name + '_' + LOGGER.debug( + f'Already detected column name {name!r} in table ' + f'{local_table!r}. Using {newname!r} for relationship name.' + ) + return newname + return name diff --git a/pygeoapi/provider/rasterio_.py b/pygeoapi/provider/rasterio_.py index 7414fcc3f..3b0fbc2c7 100644 --- a/pygeoapi/provider/rasterio_.py +++ b/pygeoapi/provider/rasterio_.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -59,107 +59,39 @@ def __init__(self, provider_def): self.axes = self._coverage_properties['axes'] self.crs = self._coverage_properties['bbox_crs'] self.num_bands = self._coverage_properties['num_bands'] - self.fields = [str(num) for num in range(1, self.num_bands+1)] + self.get_fields() self.native_format = provider_def['format']['name'] except Exception as err: LOGGER.warning(err) raise ProviderConnectionError(err) - def get_coverage_domainset(self, *args, **kwargs): - """ - Provide coverage domainset - :returns: CIS JSON object of domainset metadata - """ - - domainset = { - 'type': 'DomainSet', - 'generalGrid': { - 'type': 'GeneralGridCoverage', - 'srsName': self._coverage_properties['bbox_crs'], - 'axisLabels': [ - self._coverage_properties['x_axis_label'], - self._coverage_properties['y_axis_label'] - ], - 'axis': [{ - 'type': 'RegularAxis', - 'axisLabel': self._coverage_properties['x_axis_label'], - 'lowerBound': self._coverage_properties['bbox'][0], - 'upperBound': self._coverage_properties['bbox'][2], - 'uomLabel': self._coverage_properties['bbox_units'], - 'resolution': self._coverage_properties['resx'] - }, { - 'type': 'RegularAxis', - 'axisLabel': self._coverage_properties['y_axis_label'], - 'lowerBound': self._coverage_properties['bbox'][1], - 'upperBound': self._coverage_properties['bbox'][3], - 'uomLabel': self._coverage_properties['bbox_units'], - 'resolution': self._coverage_properties['resy'] - }], - 'gridLimits': { - 'type': 'GridLimits', - 'srsName': 'http://www.opengis.net/def/crs/OGC/0/Index2D', - 'axisLabels': ['i', 'j'], - 'axis': [{ - 'type': 'IndexAxis', - 'axisLabel': 'i', - 'lowerBound': 0, - 'upperBound': self._coverage_properties['width'] - }, { - 'type': 'IndexAxis', - 'axisLabel': 'j', - 'lowerBound': 0, - 'upperBound': self._coverage_properties['height'] - }] - } - }, - '_meta': { - 'tags': self._coverage_properties['tags'] - } - } - - return domainset + def get_fields(self): + if not self._fields: + for i, dtype in zip(self._data.indexes, self._data.dtypes): + LOGGER.debug(f'Adding field for band {i}') + i2 = str(i) - def get_coverage_rangetype(self, *args, **kwargs): - """ - Provide coverage rangetype - :returns: CIS JSON object of rangetype metadata - """ - - rangetype = { - 'type': 'DataRecord', - 'field': [] - } - - for i, dtype, nodataval in zip(self._data.indexes, self._data.dtypes, - self._data.nodatavals): - LOGGER.debug(f'Determing rangetype for band {i}') - - name, units = None, None - if self._data.units[i-1] is None: parameter = _get_parameter_metadata( self._data.profile['driver'], self._data.tags(i)) + name = parameter['description'] - units = parameter['unit_label'] - - rangetype['field'].append({ - 'id': i, - 'type': 'Quantity', - 'name': name, - 'encodingInfo': { - 'dataType': f'http://www.opengis.net/def/dataType/OGC/0/{dtype}' # noqa - }, - 'nodata': nodataval, - 'uom': { - 'id': f'http://www.opengis.net/def/uom/UCUM/{units}', - 'type': 'UnitReference', - 'code': units - }, - '_meta': { - 'tags': self._data.tags(i) + units = parameter.get('unit_label') + + dtype2 = dtype + if dtype.startswith('float'): + dtype2 = 'number' + elif dtype.startswith('int'): + dtype2 = 'integer' + + self._fields[i2] = { + 'title': name, + 'type': dtype2, + '_meta': self._data.tags(i) } - }) + if units is not None: + self._fields[i2]['x-ogc-unit'] = units - return rangetype + return self._fields def query(self, properties=[], subsets={}, bbox=None, bbox_crs=4326, datetime_=None, format_='json', **kwargs): @@ -310,16 +242,15 @@ def query(self, properties=[], subsets={}, bbox=None, bbox_crs=4326, out_meta['units'] = _data.units LOGGER.debug('Serializing data in memory') - with MemoryFile() as memfile: - with memfile.open(**out_meta) as dest: - dest.write(out_image) - - if format_ == 'json': - LOGGER.debug('Creating output in CoverageJSON') - out_meta['bands'] = args['indexes'] - return self.gen_covjson(out_meta, out_image) - - else: # return data in native format + if format_ == 'json': + LOGGER.debug('Creating output in CoverageJSON') + out_meta['bands'] = args['indexes'] + return self.gen_covjson(out_meta, out_image) + + else: # return data in native format + with MemoryFile() as memfile: + with memfile.open(**out_meta) as dest: + dest.write(out_image) LOGGER.debug('Returning data in native format') return memfile.read() diff --git a/pygeoapi/provider/sensorthings.py b/pygeoapi/provider/sensorthings.py index 6b497b44b..4d99705bb 100644 --- a/pygeoapi/provider/sensorthings.py +++ b/pygeoapi/provider/sensorthings.py @@ -3,7 +3,7 @@ # Authors: Benjamin Webb # Authors: Tom Kralidis # -# Copyright (c) 2023 Benjamin Webb +# Copyright (c) 2025 Benjamin Webb # Copyright (c) 2022 Tom Kralidis # # Permission is hereby granted, free of charge, to any person @@ -30,14 +30,18 @@ # ================================================================= from json.decoder import JSONDecodeError -import os import logging from requests import Session +from requests.exceptions import ConnectionError +from urllib.parse import urlparse +from pygeoapi.config import get_config from pygeoapi.provider.base import ( - BaseProvider, ProviderQueryError, ProviderConnectionError) + BaseProvider, ProviderQueryError, ProviderConnectionError, + ProviderInvalidDataError) from pygeoapi.util import ( - yaml_load, url_join, get_provider_default, crs_transform, get_base_url) + url_join, get_provider_default, crs_transform, get_base_url, + get_typed_value) LOGGER = logging.getLogger(__name__) @@ -51,10 +55,10 @@ _EXPAND = { 'Things': 'Locations,Datastreams', 'Observations': 'Datastream,FeatureOfInterest', + 'ObservedProperties': 'Datastreams/Thing/Locations', 'Datastreams': """ Sensor ,ObservedProperty - ,Thing ,Thing/Locations ,Observations( $select=@iot.id; @@ -71,6 +75,7 @@ class SensorThingsProvider(BaseProvider): """SensorThings API (STA) Provider""" + expand = EXPAND def __init__(self, provider_def): """ @@ -82,63 +87,11 @@ def __init__(self, provider_def): :returns: pygeoapi.provider.sensorthings.SensorThingsProvider """ LOGGER.debug('Setting SensorThings API (STA) provider') - + self.linked_entity = {} super().__init__(provider_def) - self.data.rstrip('/') - try: - self.entity = provider_def['entity'] - self._url = url_join(self.data, self.entity) - except KeyError: - LOGGER.debug('Attempting to parse Entity from provider data') - if not self._get_entity(self.data): - raise RuntimeError('Entity type required') - self.entity = self._get_entity(self.data) - self._url = self.data - self.data = self._url.rstrip(f'/{self.entity}') - LOGGER.debug(f'STA endpoint: {self.data}, Entity: {self.entity}') - - # Default id - if self.id_field: - LOGGER.debug(f'Using id field: {self.id_field}') - else: - LOGGER.debug('Using default @iot.id for id field') - self.id_field = '@iot.id' - - # Create intra-links - self.links = {} - self.intralink = provider_def.get('intralink', False) - if self.intralink and provider_def.get('rel_link'): - # For pytest - self.rel_link = provider_def['rel_link'] - - elif self.intralink: - # Read from pygeoapi config - with open(os.getenv('PYGEOAPI_CONFIG'), encoding='utf8') as fh: - CONFIG = yaml_load(fh) - self.rel_link = get_base_url(CONFIG) - - for (name, rs) in CONFIG['resources'].items(): - pvs = rs.get('providers') - p = get_provider_default(pvs) - e = p.get('entity') or self._get_entity(p['data']) - if any([ - not pvs, # No providers in resource - not p.get('intralink'), # No configuration for intralinks - not e, # No STA entity found - self.data not in p.get('data') # No common STA endpoint - ]): - continue - - if p.get('uri_field'): - LOGGER.debug(f'Linking {e} with field: {p["uri_field"]}') - else: - LOGGER.debug(f'Linking {e} with collection: {name}') - self.links[e] = { - 'cnm': name, # OAPI collection name, - 'cid': p.get('id_field', '@iot.id'), # OAPI id_field - 'uri': p.get('uri_field') # STA uri_field - } + self._generate_mappings(provider_def) + LOGGER.debug(f'STA endpoint: {self.data}, Entity: {self.entity}') # Start session self.http = Session() @@ -150,22 +103,26 @@ def get_fields(self): :returns: dict of fields """ - if not self.fields: - r = self._get_response(self._url, {'$top': 1}) + if not self._fields: try: + r = self._get_response(self._url, {'$top': 1}) results = r['value'][0] except IndexError: LOGGER.warning('could not get fields; returning empty set') return {} + except (ConnectionError, ProviderConnectionError): + msg = f'Unable to contact SensorThings endpoint at {self._url}' + LOGGER.error(msg) + raise ProviderConnectionError(msg) for (n, v) in results.items(): if isinstance(v, (int, float)) or \ (isinstance(v, (dict, list)) and n in ENTITY): - self.fields[n] = {'type': 'number'} + self._fields[n] = {'type': 'number'} elif isinstance(v, str): - self.fields[n] = {'type': 'string'} + self._fields[n] = {'type': 'string'} - return self.fields + return self._fields @crs_transform def query(self, offset=0, limit=10, resulttype='results', @@ -205,6 +162,65 @@ def get(self, identifier, **kwargs): response = self._get_response(f'{self._url}({identifier})') return self._make_feature(response) + def create(self, item): + """ + Create a new item + + :param item: `dict` of new item + + :returns: identifier of created item + """ + response = self.http.post(self._url, json=item) + + if response.status_code == 201: + location = response.headers.get("Location") + iotid = location[location.find("(")+1:location.find(")")] + + LOGGER.debug(f'Feature created with @iot.id: {iotid}') + return get_typed_value(iotid) + else: + msg = f"Failed to create item: {response.text}" + raise ProviderInvalidDataError(msg) + + def update(self, identifier, item): + """ + Updates an existing item + + :param identifier: feature id + :param item: `dict` of partial or full item + + :returns: `bool` of update result + """ + id = f"'{identifier}'" \ + if isinstance(identifier, str) else str(identifier) + LOGGER.debug(f'Updating @iot.id: {id}') + response = self.http.put(f"{self._url}({id})", json=item) + + if response.status_code == 200: + return True + else: + msg = f'Failed to update item: {response.text}' + raise ProviderConnectionError(msg) + + def delete(self, identifier): + """ + Deletes an existing item + + :param identifier: item id + + :returns: `bool` of deletion result + """ + id = f"'{identifier}'" \ + if isinstance(identifier, str) else str(identifier) + LOGGER.debug(f'Deleting @iot.id: {id}') + response = self.http.delete(f"{self._url}({id})") + + if response.status_code == 200: + return True + else: + msg = f"Failed to delete item: {response.text}" + raise ProviderConnectionError(msg) + def _load(self, offset=0, limit=10, resulttype='results', bbox=[], datetime_=None, properties=[], sortby=[], select_properties=[], skip_geometry=False, q=None): @@ -258,8 +274,13 @@ def _load(self, offset=0, limit=10, resulttype='results', v = response.get('value') while len(v) < limit: try: + # Ensure we only use provided network location + next_ = urlparse(response['@iot.nextLink'])._replace( + scheme=self.parsed_url.scheme, + netloc=self.parsed_url.netloc + ).geturl() + LOGGER.debug('Fetching next set of values') - next_ = response['@iot.nextLink'] response = self._get_response(next_) v.extend(response['value']) except (ProviderConnectionError, KeyError): @@ -272,17 +293,19 @@ def _load(self, offset=0, limit=10, resulttype='results', return fc - def _make_feature(self, entity, select_properties=[], skip_geometry=False): + def _make_feature(self, feature, select_properties=[], skip_geometry=False, + entity=None): """ Private function: Create feature from entity - :param entity: `dict` of STA entity + :param feature: `dict` of STA entity :param select_properties: list of property names :param skip_geometry: bool of whether to skip geometry (default False) + :param entity: SensorThings entity name :returns: dict of GeoJSON Feature """ - _ = entity.pop(self.id_field) + _ = feature.pop(self.id_field) id = f"'{_}'" if isinstance(_, str) else str(_) f = { 'type': 'Feature', 'id': id, 'properties': {}, 'geometry': None @@ -290,28 +313,35 @@ def _make_feature(self, entity, select_properties=[], skip_geometry=False): # Make geometry if not skip_geometry: - f['geometry'] = self._geometry(entity) + f['geometry'] = self._geometry(feature, entity) # Fill properties block try: f['properties'] = self._expand_properties( - entity, select_properties) + feature, select_properties, entity) except KeyError as err: LOGGER.error(err) raise ProviderQueryError(err) return f - def _get_response(self, url, params={}): + def _get_response(self, url, params={}, entity=None, expand=None): """ Private function: Get STA response :param url: request url :param params: query parameters + :param entity: SensorThings entity name + :param expand: SensorThings expand query + :returns: STA response """ - params.update({'$expand': EXPAND[self.entity]}) + if expand: + params.update({'$expand': expand}) + else: + entity_ = entity or self.entity + params.update({'$expand': self.expand[entity_]}) r = self.http.get(url, params=params) @@ -327,13 +357,15 @@ def _get_response(self, url, params={}): return response - def _make_filter(self, properties, bbox=[], datetime_=None): + def _make_filter(self, properties, bbox=[], datetime_=None, + entity=None): """ Private function: Make STA filter from query properties :param properties: list of tuples (name, value) :param bbox: bounding box [minx,miny,maxx,maxy] :param datetime_: temporal (datestamp or extent) + :param entity: SensorThings entity name :returns: STA $filter string of properties """ @@ -345,16 +377,8 @@ def _make_filter(self, properties, bbox=[], datetime_=None): ret.append(f'{name} eq {value}') if bbox: - minx, miny, maxx, maxy = bbox - bbox_ = f'POLYGON (({minx} {miny}, {maxx} {miny}, \ - {maxx} {maxy}, {minx} {maxy}, {minx} {miny}))' - if self.entity == 'Things': - loc = 'Locations/location' - elif self.entity == 'Datastreams': - loc = 'Thing/Locations/location' - elif self.entity == 'Observations': - loc = 'FeatureOfInterest/feature' - ret.append(f"st_within({loc}, geography'{bbox_}')") + entity_ = entity or self.entity + ret.append(self._make_bbox(bbox, entity_)) if datetime_ is not None: if self.time_field is None: @@ -373,6 +397,20 @@ def _make_filter(self, properties, bbox=[], datetime_=None): return ' and '.join(ret) + @staticmethod + def _make_bbox(bbox, entity): + minx, miny, maxx, maxy = bbox + bbox_ = f'POLYGON(({minx} {miny},{maxx} {miny},{maxx} {maxy},{minx} {maxy},{minx} {miny}))' # noqa + if entity == 'Things': + loc = 'Locations/location' + elif entity == 'Datastreams': + loc = 'Thing/Locations/location' + elif entity == 'Observations': + loc = 'FeatureOfInterest/feature' + elif entity == 'ObservedProperties': + loc = 'Datastreams/observedArea' + return f"st_within({loc},geography'{bbox_}')" + def _make_orderby(self, sortby): """ Private function: Make STA filter from query properties @@ -393,79 +431,90 @@ def _make_orderby(self, sortby): return ','.join(ret) - def _geometry(self, entity): + def _geometry(self, feature, entity=None): """ Private function: Retrieve STA geometry - :param entity: SensorThings entity + :param feature: SensorThings entity + :param entity: SensorThings entity name :returns: GeoJSON Geometry for feature """ + entity_ = entity or self.entity try: - if self.entity == 'Things': - return entity['Locations'][0]['location'] + if entity_ == 'Things': + return feature['Locations'][0]['location'] - elif self.entity == 'Observations': - return entity['FeatureOfInterest'].pop('feature') + elif entity_ == 'Observations': + return feature['FeatureOfInterest'].pop('feature') - elif self.entity == 'Datastreams': + elif entity_ == 'Datastreams': try: - return entity['Observations'][0]['FeatureOfInterest'].pop('feature') # noqa + return feature['Observations'][0]['FeatureOfInterest'].pop('feature') # noqa except (KeyError, IndexError): - return entity['Thing'].pop('Locations')[0]['location'] + return feature['Thing'].pop('Locations')[0]['location'] + + elif entity_ == 'ObservedProperties': + return feature['Datastreams'][0]['Thing']['Locations'][0]['location'] # noqa except (KeyError, IndexError): LOGGER.warning('No geometry found') return None - def _expand_properties(self, entity, keys=(), uri=''): + def _expand_properties(self, feature, keys=(), uri='', + entity=None): """ Private function: Parse STA entity into feature - :param entity: SensorThings entity + :param feature: `dict` of SensorThings entity :param keys: keys used in properties block :param uri: uri of STA entity + :param entity: SensorThings entity name :returns: dict of SensorThings feature properties """ - LOGGER.debug('Adding extra properties') - # Properties filter & display keys = (() if not self.properties and not keys else set(self.properties) | set(keys)) - if self.entity == 'Things': - self._expand_location(entity) - elif 'Thing' in entity.keys(): - self._expand_location(entity['Thing']) + entity = entity or self.entity + if entity == 'Things': + self._expand_location(feature) + elif 'Thing' in feature.keys(): + self._expand_location(feature['Thing']) # Retain URI if present - if entity.get('properties') and self.uri_field: - uri = entity['properties'] + try: + if self.uri_field is not None: + uri = feature['properties'][self.uri_field] + except KeyError: + msg = f'Unable to find uri field: {self.uri_field}' + LOGGER.error(msg) + raise ProviderInvalidDataError(msg) # Create intra links - LOGGER.debug('Creating intralinks') - for k, v in entity.items(): - if k in self.links: - entity[k] = [self._get_uri(_v, **self.links[k]) for _v in v] + for k, v in feature.items(): + if k in self.linked_entity: + feature[k] = [self._get_uri(_v, **self.linked_entity[k]) + for _v in v] LOGGER.debug(f'Created link for {k}') - elif f'{k}s' in self.links: - entity[k] = self._get_uri(v, **self.links[f'{k}s']) + elif f'{k}s' in self.linked_entity: + feature[k] = \ + self._get_uri(v, **self.linked_entity[f'{k}s']) LOGGER.debug(f'Created link for {k}') # Make properties block - LOGGER.debug('Making properties block') - if entity.get('properties'): - entity.update(entity.pop('properties')) + if feature.get('properties'): + feature.update(feature.pop('properties')) if keys: - ret = {k: entity.pop(k) for k in keys} - entity = ret + ret = {k: feature.pop(k) for k in keys} + feature = ret - if self.uri_field is not None and uri != '': - entity[self.uri_field] = uri + if self.uri_field is not None: + feature[self.uri_field] = uri - return entity + return feature @staticmethod def _expand_location(entity): @@ -517,5 +566,74 @@ def _get_entity(uri): else: return '' + def _generate_mappings(self, provider_def: dict): + """ + Generate mappings for the STA entity and set up intra-links. + + This function sets up the necessary mappings and configurations for + the STA entity based on the provided provider definition. + + :param provider_def: `dict` of provider definition containing + configuration details for the STA entity. + """ + self.data.rstrip('/') + try: + self.entity = provider_def['entity'] + self._url = url_join(self.data, self.entity) + except KeyError: + LOGGER.debug('Attempting to parse Entity from provider data') + if not self._get_entity(self.data): + raise RuntimeError('Entity type required') + self.entity = self._get_entity(self.data) + self._url = self.data + self.data = self._url.rstrip(f'/{self.entity}') + + self.parsed_url = urlparse(self.data) + + # Default id + if self.id_field: + LOGGER.debug(f'Using id field: {self.id_field}') + else: + LOGGER.debug('Using default @iot.id for id field') + self.id_field = '@iot.id' + + # Custom expand + if provider_def.get('expand'): + EXPAND[self.entity] = provider_def['expand'] + + # Create intra-links + self.intralink = provider_def.get('intralink', False) + if self.intralink and provider_def.get('rel_link'): + # For pytest + self.rel_link = provider_def['rel_link'] + + elif self.intralink: + # Read from pygeoapi config + CONFIG = get_config() + self.rel_link = get_base_url(CONFIG) + + for name, rs in CONFIG['resources'].items(): + pvs = rs.get('providers') + p = get_provider_default(pvs) + e = p.get('entity') or self._get_entity(p['data']) + if any([ + not pvs, # No providers in resource + not p.get('intralink'), # No configuration for intralinks + not e, # No STA entity found + self.data not in p.get('data') # No common STA endpoint + ]): + continue + + if p.get('uri_field'): + LOGGER.debug(f'Linking {e} with field: {p["uri_field"]}') + else: + LOGGER.debug(f'Linking {e} with collection: {name}') + + self.linked_entity[e] = { + 'cnm': name, # OAPI collection name, + 'cid': p.get('id_field', '@iot.id'), # OAPI id_field + 'uri': p.get('uri_field') # STA uri_field + } + def __repr__(self): return f' {self.data}, {self.entity}' diff --git a/pygeoapi/provider/sensorthings_edr.py b/pygeoapi/provider/sensorthings_edr.py new file mode 100644 index 000000000..de943765e --- /dev/null +++ b/pygeoapi/provider/sensorthings_edr.py @@ -0,0 +1,442 @@ +# ================================================================= +# +# Authors: Ben Webb +# +# Copyright (c) 2025 Ben Webb +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import logging + +from pygeoapi.provider.base import ProviderNoDataError +from pygeoapi.provider.base_edr import BaseEDRProvider +from pygeoapi.provider.sensorthings import SensorThingsProvider + +LOGGER = logging.getLogger(__name__) + +GEOGRAPHIC_CRS = { + 'coordinates': ['x', 'y'], + 'system': { + 'type': 'GeographicCRS', + 'id': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' + } +} + +TEMPORAL_RS = { + 'coordinates': ['t'], + 'system': {'type': 'TemporalRS', 'calendar': 'Gregorian'} +} + + +class SensorThingsEDRProvider(BaseEDRProvider, SensorThingsProvider): + def __init__(self, provider_def): + """ + Initialize the SensorThingsEDRProvider instance. + + :param provider_def: provider definition + """ + + provider_def['entity'] = 'ObservedProperties' + BaseEDRProvider.__init__(self, provider_def) + SensorThingsProvider.__init__(self, provider_def) + self.expand['ObservedProperties'] = ( + 'Datastreams/Thing/Locations,Datastreams/Observations' + ) + self.time_field = 'Datastreams/Observations/resultTime' + self._fields = {} + self.get_fields() + + def get_fields(self): + """ + Retrieve and store ObservedProperties field definitions. + + :returns: A dictionary mapping field IDs to their properties. + """ + + if not self._fields: + r = self._get_response( + self._url, entity='ObservedProperties', expand='Datastreams' + ) + try: + _ = r['value'][0] + except IndexError: + LOGGER.warning('could not get fields; returning empty set') + return {} + + for feature in r['value']: + id = str(feature['@iot.id']) + key = feature['name'] + try: + uom = feature['Datastreams'][0]['unitOfMeasurement'] + except IndexError: + continue + + self._fields[id] = { + 'type': 'number', + 'title': key, + 'x-ogc-unit': uom['symbol'] + } + + return self._fields + + @BaseEDRProvider.register() + def items(self, **kwargs): + """ + Retrieve a collection of items. + + :param kwargs: Additional parameters for the request. + :returns: A GeoJSON representation of the items. + """ + + # This method is empty due to the way pygeoapi handles items requests + # We implement this method inside of the feature provider + pass + + @BaseEDRProvider.register() + def locations( + self, + select_properties: list = [], + bbox: list = [], + datetime_: str = None, + location_id: str = None, + **kwargs + ): + """ + Extract and return location data from ObservedProperties. + + :param select_properties: List of properties to include. + :param bbox: Bounding box geometry for spatial queries. + :param datetime_: Temporal filter for observations. + :param location_id: Identifier of the location to filter by. + + :returns: A GeoJSON FeatureCollection of locations. + """ + + fc = {'type': 'FeatureCollection', 'features': []} + + params = {} + expand = [ + 'Datastreams($select=description,name,unitOfMeasurement)', + 'Datastreams/Thing($select=@iot.id)', + 'Datastreams/Thing/Locations($select=location)' + ] + + if select_properties: + properties = [ + ['@iot.id', f"'{p}'" if isinstance(p, str) else p] + for p in select_properties + ] + ret = [f'{name} eq {value}' for (name, value) in properties] + params['$filter'] = ' or '.join(ret) + + filter_ = f'$filter={self._make_dtf(datetime_)};' if datetime_ else '' + if location_id: + expand[0] = ( + f'Datastreams($filter=Thing/@iot.id eq {location_id};' + '$select=description,name,unitOfMeasurement)' + ) + expand.append( + f'Datastreams/Observations({filter_}$orderby=phenomenonTime;' + '$select=result,phenomenonTime,resultTime)' + ) + else: + expand.append( + f'Datastreams/Observations({filter_}$select=result;$top=1)') + + if bbox: + geom_filter = self._make_bbox(bbox, 'Datastreams') + expand[0] = f'Datastreams($filter={geom_filter})' + + expand = ','.join(expand) + response = self._get_response( + url=self._url, params=params, + entity='ObservedProperties', expand=expand + ) + + if location_id: + return self._make_coverage_collection(response) + + for property in response['value']: + for datastream in property['Datastreams']: + if len(datastream['Observations']) == 0: + continue + + fc['features'].append( + self._make_feature(datastream['Thing'], entity='Things') + ) + + return fc + + @BaseEDRProvider.register() + def cube( + self, + select_properties: list = [], + bbox: list = [], + datetime_: str = None, + **kwargs + ): + """ + Extract and return coverage data from ObservedProperties. + + :param select_properties: List of properties to include. + :param bbox: Bounding box geometry for spatial queries. + :param datetime_: Temporal filter for observations. + + :returns: A CovJSON CoverageCollection. + """ + + params = {} + + geom_filter = self._make_bbox(bbox, 'Datastreams') + expand = [ + ( + f'Datastreams($filter={geom_filter};' + '$select=description,name,unitOfMeasurement)' + ), + 'Datastreams/Thing($select=@iot.id)', + 'Datastreams/Thing/Locations($select=location)' + ] + + if select_properties: + properties = [ + ['@iot.id', f"'{p}'" if isinstance(p, str) else p] + for p in select_properties + ] + ret = [f'{name} eq {value}' for (name, value) in properties] + params['$filter'] = ' or '.join(ret) + + filter_ = f'$filter={self._make_dtf(datetime_)};' if datetime_ else '' + expand.append( + f'Datastreams/Observations({filter_}$orderby=phenomenonTime;' + '$select=result,phenomenonTime,resultTime)' + ) + + expand = ','.join(expand) + response = self._get_response( + url=self._url, params=params, + entity='ObservedProperties', expand=expand + ) + + return self._make_coverage_collection(response) + + @BaseEDRProvider.register() + def area( + self, + wkt: str, + select_properties: list = [], + datetime_: str = None, + **kwargs + ): + """ + Extract and return coverage data from a specified area. + + :param wkt: Well-Known Text (WKT) representation of the + geometry for the area. + :param select_properties: List of properties to include + in the response. + :param datetime_: Temporal filter for observations. + + :returns: A CovJSON CoverageCollection. + """ + + params = {} + + expand = [ + ( + 'Datastreams($filter=st_within(' + f"Thing/Locations/location,geography'{wkt}');" + '$select=description,name,unitOfMeasurement)' + ), + 'Datastreams/Thing($select=@iot.id)', + 'Datastreams/Thing/Locations($select=location)' + ] + + if select_properties: + properties = [ + ['@iot.id', f"'{p}'" if isinstance(p, str) else p] + for p in select_properties + ] + ret = [f'{name} eq {value}' for (name, value) in properties] + params['$filter'] = ' or '.join(ret) + + filter_ = f'$filter={self._make_dtf(datetime_)};' if datetime_ else '' + expand.append( + f'Datastreams/Observations({filter_}$orderby=phenomenonTime;' + '$select=result,phenomenonTime,resultTime)' + ) + + expand = ','.join(expand) + response = self._get_response( + url=self._url, params=params, + entity='ObservedProperties', expand=expand + ) + + return self._make_coverage_collection(response) + + def _generate_coverage(self, datastream: dict, id: str) -> dict: + """ + Generate a coverage object for a datastream. + + :param datastream: The datastream data to generate coverage for. + :param id: The ID to use for the coverage. + + :returns: A dict containing the coverage object. + """ + + times, values = self._expand_observations(datastream) + thing = datastream['Thing'] + coords = thing['Locations'][0]['location']['coordinates'] + length = len(values) + + return { + 'type': 'Coverage', + 'id': str(thing['@iot.id']), + 'domain': { + 'type': 'Domain', + 'domainType': 'PointSeries', + 'axes': { + 'x': {'values': [coords[0]]}, + 'y': {'values': [coords[1]]}, + 't': {'values': times} + }, + 'referencing': [GEOGRAPHIC_CRS, TEMPORAL_RS] + }, + 'ranges': { + id: { + 'type': 'NdArray', + 'dataType': 'float', + 'axisNames': ['t'], + 'shape': [length], + 'values': values + } + } + }, length + + @staticmethod + def _generate_paramters(datastream: dir, label: str) -> dict: + """ + Generate parameters for a given datastream. + + :param datastream: The datastream data to generate parameters for. + :param label: The label for the parameter. + + :returns: A dictionary containing the parameter definition. + """ + + return { + 'type': 'Parameter', + 'description': {'en': datastream['description']}, + 'observedProperty': {'id': label, 'label': {'en': label}}, + 'unit': { + 'label': {'en': datastream['unitOfMeasurement']['name']}, + 'symbol': datastream['unitOfMeasurement']['symbol'] + } + } + + @staticmethod + def _make_dtf(datetime_: str) -> str: + """ + Create a Date-Time filter for querying. + + :param datetime_: Temporal filter. + + :returns: A string datetime filter for use in queries. + """ + + dtf_r = [] + if '/' in datetime_: + time_start, time_end = datetime_.split('/') + if time_start != '..': + dtf_r.append(f'phenomenonTime ge {time_start}') + + if time_end != '..': + dtf_r.append(f'phenomenonTime le {time_end}') + + else: + dtf_r.append(f'phenomenonTime eq {datetime_}') + + return ' and '.join(dtf_r) + + def _make_coverage_collection(self, response): + """ + Build a CoverageCollection from the SensorThings API response. + + :param response: source response from SensorThings. + + :returns: The updated CoverageCollection object. + """ + + cc = { + 'type': 'CoverageCollection', + 'domainType': 'PointSeries', + 'parameters': {}, + 'coverages': [] + } + + for feature in response['value']: + if len(feature['Datastreams']) == 0: + continue + + id = feature['name'].replace(' ', '+') + + for datastream in feature['Datastreams']: + coverage, length = self._generate_coverage(datastream, id) + if length > 0: + cc['coverages'].append(coverage) + cc['parameters'][id] = self._generate_paramters( + datastream, feature['name'] + ) + + if cc['parameters'] == {} or cc['coverages'] == []: + msg = 'No data found' + LOGGER.warning(msg) + raise ProviderNoDataError(msg) + + return cc + + @staticmethod + def _expand_observations(datastream: dict): + """ + Expand and extract observation times and values from a datastream. + + :param datastream: The datastream containing observations. + + :returns: A tuple containing lists of times and values. + """ + + times = [] + values = [] + # TODO: Expand observations when 'Observations@iot.nextLink' + # or '@iot.nextLink' is present + for obs in datastream['Observations']: + resultTime = obs['resultTime'] or obs['phenomenonTime'] + if obs['result'] is not None and resultTime: + try: + result = float(obs['result']) + except ValueError: + result = obs['result'] + times.append(resultTime) + values.append(result) + + return times, values diff --git a/pygeoapi/provider/socrata.py b/pygeoapi/provider/socrata.py index 0f402a735..9d2292549 100644 --- a/pygeoapi/provider/socrata.py +++ b/pygeoapi/provider/socrata.py @@ -75,7 +75,7 @@ def get_fields(self): :returns: dict of fields """ - if not self.fields: + if not self._fields: try: [dataset] = self.client.datasets(ids=[self.resource_id]) @@ -87,9 +87,9 @@ def get_fields(self): fields = self.properties or resource[FIELD_NAME] for field in fields: idx = resource[FIELD_NAME].index(field) - self.fields[field] = {'type': resource[DATA_TYPE][idx]} + self._fields[field] = {'type': resource[DATA_TYPE][idx]} - return self.fields + return self._fields @crs_transform def query(self, offset=0, limit=10, resulttype='results', diff --git a/pygeoapi/provider/sqlite.py b/pygeoapi/provider/sqlite.py index d7ae81d16..9ba796bb9 100644 --- a/pygeoapi/provider/sqlite.py +++ b/pygeoapi/provider/sqlite.py @@ -88,7 +88,7 @@ def get_fields(self): :returns: dict of fields """ - if not self.fields: + if not self._fields: results = self.cursor.execute( f'PRAGMA table_info({self.table})').fetchall() for item in results: @@ -100,16 +100,16 @@ def get_fields(self): json_type = 'string' if json_type is not None: - self.fields[item['name']] = {'type': json_type} + self._fields[item['name']] = {'type': json_type} - return self.fields + return self._fields def __get_where_clauses(self, properties=[], bbox=[]): """ Generarates WHERE conditions to be implemented in query. Private method mainly associated with query method. - Method returns part of the SQL query, plus tupple to be used + Method returns part of the SQL query, plus tuple to be used in the sqlite query method :param properties: list of tuples (name, value) diff --git a/pygeoapi/provider/tinydb_.py b/pygeoapi/provider/tinydb_.py index 3bac4c059..502b6a385 100644 --- a/pygeoapi/provider/tinydb_.py +++ b/pygeoapi/provider/tinydb_.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2025 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -32,17 +32,20 @@ import os import uuid +from dateutil.parser import parse as parse_date from shapely.geometry import shape from tinydb import TinyDB, Query, where from pygeoapi.provider.base import (BaseProvider, ProviderConnectionError, + ProviderInvalidQueryError, ProviderItemNotFoundError) +from pygeoapi.util import crs_transform, get_typed_value LOGGER = logging.getLogger(__name__) -class TinyDBCatalogueProvider(BaseProvider): - """TinyDB Catalogue Provider""" +class TinyDBProvider(BaseProvider): + """TinyDB Provider""" def __init__(self, provider_def): """ @@ -50,15 +53,13 @@ def __init__(self, provider_def): :param provider_def: provider definition - :returns: pygeoapi.provider.tinydb_.TinyDBCatalogueProvider + :returns: pygeoapi.provider.tinydb_.TinyDBProvider """ - self.excludes = [ - '_metadata-anytext', - ] - super().__init__(provider_def) + self._excludes = [] + LOGGER.debug(f'Connecting to TinyDB db at {self.data}') if not os.path.exists(self.data): @@ -74,7 +75,7 @@ def __init__(self, provider_def): else: self.db = TinyDB(self.data) - self.fields = self.get_fields() + self.get_fields() def get_fields(self): """ @@ -83,22 +84,39 @@ def get_fields(self): :returns: dict of fields """ - fields = {} - - try: - r = self.db.all()[0] - except IndexError as err: - LOGGER.debug(err) - return fields - - for p in r['properties'].keys(): - if p not in self.excludes: - fields[p] = {'type': 'string'} - - fields['q'] = {'type': 'string'} - - return fields - + if not self._fields: + try: + r = self.db.all()[0] + except IndexError as err: + LOGGER.debug(err) + return {} + + for key, value in r['properties'].items(): + if key not in self._excludes: + typed_value = get_typed_value(str(value)) + if isinstance(typed_value, float): + typed_value_type = 'number' + elif isinstance(typed_value, int): + typed_value_type = 'integer' + else: + typed_value_type = 'string' + + self._fields[key] = {'type': typed_value_type} + + try: + LOGGER.debug('Attempting to detect date types') + _ = parse_date(value) + if len(value) > 11: + self._fields[key]['format'] = 'date-time' + else: + self._fields[key]['format'] = 'date' + except Exception: + LOGGER.debug('No date types detected') + pass + + return self._fields + + @crs_transform def query(self, offset=0, limit=10, resulttype='results', bbox=[], datetime_=None, properties=[], sortby=[], select_properties=[], skip_geometry=False, q=None, **kwargs): @@ -164,11 +182,13 @@ def query(self, offset=0, limit=10, resulttype='results', if properties: LOGGER.debug('processing properties') for prop in properties: - QUERY.append(f"(Q.properties['{prop[0]}']=='{prop[1]}')") + if isinstance(prop[1], str): + value = f"'{prop[1]}'" + else: + value = prop[1] + QUERY.append(f"(Q.properties['{prop[0]}']=={value})") - if q is not None: - for t in q.split(): - QUERY.append(f"(Q.properties['_metadata-anytext'].search('{t}', flags=re.IGNORECASE))") # noqa + QUERY = self._add_search_query(QUERY, q) QUERY_STRING = '&'.join(QUERY) LOGGER.debug(f'QUERY_STRING: {QUERY_STRING}') @@ -178,7 +198,12 @@ def query(self, offset=0, limit=10, resulttype='results', LOGGER.debug('querying database') if len(QUERY) > 0: LOGGER.debug(f'running eval on {SEARCH_STRING}') - results = eval(SEARCH_STRING) + try: + results = eval(SEARCH_STRING) + except SyntaxError as err: + msg = 'Invalid query' + LOGGER.error(f'{msg}: {err}') + raise ProviderInvalidQueryError(msg) else: results = self.db.all() @@ -188,7 +213,7 @@ def query(self, offset=0, limit=10, resulttype='results', return feature_collection for r in results: - for e in self.excludes: + for e in self._excludes: try: del r['properties'][e] except KeyError: @@ -219,6 +244,7 @@ def query(self, offset=0, limit=10, resulttype='results', return feature_collection + @crs_transform def get(self, identifier, **kwargs): """ Get TinyDB document by id @@ -235,7 +261,7 @@ def get(self, identifier, **kwargs): if record is None: raise ProviderItemNotFoundError('record does not exist') - for e in self.excludes: + for e in self._excludes: try: del record['properties'][e] except KeyError: @@ -259,14 +285,7 @@ def create(self, item): identifier = str(uuid.uuid4()) json_data["id"] = identifier - try: - json_data['properties']['_metadata-anytext'] = ''.join([ - json_data['properties']['title'], - json_data['properties']['description'] - ]) - except KeyError: - LOGGER.debug('Missing title and description') - json_data['properties']['_metadata_anytext'] = '' + json_data = self._add_extra_fields(json_data) LOGGER.debug(f'Inserting data with identifier {identifier}') result = self.db.insert(json_data) @@ -306,17 +325,71 @@ def delete(self, identifier): return True - def _bbox(input_bbox, record_bbox): + def _add_extra_fields(self, json_data: dict) -> dict: """ - Test whether one bbox intersects another + Helper function to add extra fields to an item payload - :param input_bbox: `list` of minx,miny,maxx,maxy - :param record_bbox: `list` of minx,miny,maxx,maxy + :param json_data: `dict` of JSON data - :returns: `bool` of result + :returns: `dict` of updated JSON data """ - return True + return json_data + + def _add_search_query(self, query: list, search_term: str = None) -> str: + """ + Helper function to add extra query predicates + + :param query: `list` of query predicates + :param search_term: `str` of search term + + :returns: `list` of updated query predicates + """ + + return query + + def __repr__(self): + return f' {self.data}' + + +class TinyDBCatalogueProvider(TinyDBProvider): + """TinyDB Catalogue Provider""" + + def __init__(self, provider_def): + super().__init__(provider_def) + + LOGGER.debug('Refreshing fields') + self._excludes = ['_metadata-anytext'] + self._fields = {} + self.get_fields() + + def get_fields(self): + fields = super().get_fields() + + fields['q'] = {'type': 'string'} + + return fields + + def _add_extra_fields(self, json_data: dict) -> dict: + LOGGER.debug('Adding catalogue anytext property') + try: + json_data['properties']['_metadata-anytext'] = ''.join([ + json_data['properties']['title'], + json_data['properties']['description'] + ]) + except KeyError: + LOGGER.debug('Missing title and description') + json_data['properties']['_metadata_anytext'] = '' + + return json_data + + def _add_search_query(self, query: list, search_term: str = None) -> str: + if search_term is not None: + LOGGER.debug('catalogue q= query') + for t in search_term.split(): + query.append(f"(Q.properties['_metadata-anytext'].search('{t}', flags=re.IGNORECASE))") # noqa + + return query def __repr__(self): return f' {self.data}' diff --git a/pygeoapi/provider/wms_facade.py b/pygeoapi/provider/wms_facade.py index 8771b6395..fa3ffd064 100644 --- a/pygeoapi/provider/wms_facade.py +++ b/pygeoapi/provider/wms_facade.py @@ -84,7 +84,9 @@ def query(self, style=None, bbox=[-180, -90, 180, 90], width=500, self._transparent = 'TRUE' - if crs in [4326, 'CRS;84']: + version = self.options.get('version', '1.3.0') + + if crs in [4326, 'CRS;84'] and version == '1.3.0': LOGGER.debug('Swapping 4326 axis order to WMS 1.3 mode (yx)') bbox2 = ','.join(str(c) for c in [bbox[1], bbox[0], bbox[3], bbox[2]]) @@ -106,12 +108,14 @@ def query(self, style=None, bbox=[-180, -90, 180, 90], width=500, if not transparent: self._transparent = 'FALSE' + crs_param = 'crs' if version == '1.3.0' else 'srs' + params = { - 'version': '1.3.0', + 'version': version, 'service': 'WMS', 'request': 'GetMap', 'bbox': bbox2, - 'crs': CRS_CODES[crs], + crs_param: CRS_CODES[crs], 'layers': self.options['layer'], 'styles': self.options.get('style', 'default'), 'width': width, @@ -128,7 +132,7 @@ def query(self, style=None, bbox=[-180, -90, 180, 90], width=500, else: request_url = '?'.join([self.data, urlencode(params)]) - LOGGER.debug(f'WMS 1.3.0 request url: {request_url}') + LOGGER.debug(f'WMS {version} request url: {request_url}') response = requests.get(request_url) diff --git a/pygeoapi/provider/wmts_facade.py b/pygeoapi/provider/wmts_facade.py new file mode 100644 index 000000000..51d1421e6 --- /dev/null +++ b/pygeoapi/provider/wmts_facade.py @@ -0,0 +1,281 @@ +# ================================================================= +# +# Authors: Francesco Bartoli +# Authors: Tom Kralidis +# Authors: Simon Seyock +# +# Copyright (c) 2020 Francesco Bartoli +# Copyright (c) 2023 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import logging + +import requests +from urllib.parse import urlparse, urlencode + +from pygeoapi.provider.tile import (ProviderTileNotFoundError, + BaseTileProvider) +from pygeoapi.provider.base import ProviderConnectionError +from pygeoapi.models.provider.base import ( + TileMatrixSetEnum, TilesMetadataFormat, TileSetMetadata, LinkType) +from pygeoapi.util import is_url, url_join + +LOGGER = logging.getLogger(__name__) + + +class WMTSFacadeProvider(BaseTileProvider): + """WMTS Provider""" + + def __init__(self, provider_def): + """ + Initialize object + + :param provider_def: provider definition + + :returns: pygeoapi.provider.WMTSFacadeProvider + """ + + if provider_def['format']['name'] not in ['png', 'jpeg']: + raise RuntimeError("WMTS format has to be 'png' or 'jpeg'") + + super().__init__(provider_def) + + self.tile_type = 'raster' + + def __repr__(self): + return f' {self.data}' + + @property + def service_url(self): + return self._service_url + + @property + def service_metadata_url(self): + return self._service_metadata_url + + def get_tiling_schemes(self): + + tile_matrix_set_links_list = [ + TileMatrixSetEnum.WORLDCRS84QUAD.value, + TileMatrixSetEnum.WEBMERCATORQUAD.value + ] + tile_matrix_set_links = [ + item for item in tile_matrix_set_links_list + if item.tileMatrixSet == self.options['scheme'] + ] + + return tile_matrix_set_links + + def get_tiles_service(self, baseurl=None, servicepath=None, + dirpath=None, tile_type=None): + """ + Gets service description + + :param baseurl: base URL of endpoint + :param servicepath: base path of URL + :param dirpath: directory basepath (equivalent of URL) + :param tile_type: tile format type + + :returns: `dict` of item tile service + """ + + url = urlparse(self.data) + baseurl = baseurl or f'{url.scheme}://{url.netloc}' + tile_type = self.format_type + basepath = url.path.split('/{z}/{x}/{y}')[0] + servicepath = servicepath or f'{basepath}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f={tile_type}' # noqa + + if servicepath.startswith(baseurl): + self._service_url = servicepath + else: + self._service_url = url_join(baseurl, servicepath) + + tile_matrix_set = self.service_url.split( + '/{tileMatrix}/{tileRow}/{tileCol}')[0] + self._service_metadata_url = url_join(tile_matrix_set, 'metadata') + + links = { + 'links': [ + { + 'type': self.mimetype, + 'rel': 'item', + 'title': 'This collection as image tiles', + 'href': self.service_url + }, + { + 'type': 'application/json', + 'rel': 'describedby', + 'title': 'Collection metadata in TileJSON format', + 'href': f'{self.service_metadata_url}?f=json' + } + ] + } + return links + + def get_tiles(self, layer=None, tileset=None, + z=None, y=None, x=None, format_=None): + """ + Gets tile + + :param layer: The layer name + :param tileset: The tileset name + :param z: z index + :param y: y index + :param x: x index + :param format_: tile format + + :returns: an image tile + """ + if is_url(self.data): + params = { + 'service': 'WMTS', + 'request': 'GetTile', + 'version': '1.0.0', + 'format': self.mimetype, + 'layer': self.options['wmts_layer'], + 'tileMatrixSet': self.options['wmts_tile_matrix_set'], + 'tileMatrix': z, + 'tileRow': y, + 'tileCol': x, + 'style': '' + } + + if '?' in self.data: + request_url = '&'.join([self.data, urlencode(params)]) + else: + request_url = '?'.join([self.data, urlencode(params)]) + + LOGGER.debug(f'WMTS 1.0.0 request url: {request_url}') + + with requests.Session() as session: + resp = session.get(request_url) + + if resp.status_code == 400: + raise ProviderTileNotFoundError + + return resp.content + else: + msg = f'Wrong data path configuration: {self.data}' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + def get_metadata(self, dataset, server_url, layer=None, + tileset=None, metadata_format=None, title=None, + description=None, keywords=None, **kwargs): + """ + Gets tiles metadata + + :param dataset: dataset name + :param server_url: server base url + :param layer: mvt tile layer name + :param tileset: mvt tileset name + :param metadata_format: format for metadata, + enum TilesMetadataFormat + :param title: title name + :param description: description name + :param keywords: keywords list + + :returns: `dict` of JSON metadata + """ + + if metadata_format.upper() == TilesMetadataFormat.JSON: + return self.get_default_metadata(dataset, server_url, layer, + tileset, title, description, + keywords, **kwargs) + elif metadata_format.upper() == TilesMetadataFormat.HTML: + return self.get_html_metadata(dataset, server_url, layer, + tileset, title, description, + keywords, **kwargs) + else: + raise NotImplementedError(f"_{metadata_format.upper()}_ is not supported") # noqa + + def get_html_metadata(self, dataset, server_url, layer, tileset, + title, description, keywords, **kwargs): + + service_url = url_join( + server_url, + f'collections/{dataset}/tiles/{tileset}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f={self.format_type}') # noqa + metadata_url = url_join( + server_url, + f'collections/{dataset}/tiles/{tileset}/metadata') + + metadata = dict() + metadata['id'] = dataset + metadata['title'] = title + metadata['tileset'] = tileset + metadata['collections_path'] = service_url + metadata['json_url'] = f'{metadata_url}?f=json' + + return metadata + + def get_default_metadata(self, dataset, server_url, layer, tileset, + title, description, keywords, **kwargs): + + service_url = url_join( + server_url, + f'collections/{dataset}/tiles/{tileset}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f={self.format_type}') # noqa + + content = {} + tiling_schemes = self.get_tiling_schemes() + # Default values + tileMatrixSetURI = tiling_schemes[0].tileMatrixSetURI + crs = tiling_schemes[0].crs + # Checking the selected matrix in configured tiling_schemes + for schema in tiling_schemes: + if (schema.tileMatrixSet == tileset): + crs = schema.crs + tileMatrixSetURI = schema.tileMatrixSetURI + + tiling_scheme_url = url_join( + server_url, f'/TileMatrixSets/{schema.tileMatrixSet}') + tiling_scheme_url_type = "application/json" + tiling_scheme_url_title = f'{schema.tileMatrixSet} tile matrix set definition' # noqa + + tiling_scheme = LinkType(href=tiling_scheme_url, + rel="http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", # noqa + type_=tiling_scheme_url_type, + title=tiling_scheme_url_title) + + if tiling_scheme is None: + msg = f'Could not identify a valid tiling schema' # noqa + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + content = TileSetMetadata(title=title, description=description, + keywords=keywords, crs=crs, + tileMatrixSetURI=tileMatrixSetURI) + + links = [] + service_url_link_title = f'{tileset} raster tiles for {dataset}' + service_url_link = LinkType(href=service_url, rel="item", + type_=self.mimetype, + title=service_url_link_title) + + links.append(tiling_scheme) + links.append(service_url_link) + + content.links = links + + return content.dict(exclude_none=True) diff --git a/pygeoapi/provider/xarray_.py b/pygeoapi/provider/xarray_.py index 44398c760..9ed2726b1 100644 --- a/pygeoapi/provider/xarray_.py +++ b/pygeoapi/provider/xarray_.py @@ -37,12 +37,16 @@ import xarray import fsspec import numpy as np +import pyproj +from pyproj.exceptions import CRSError + +from pygeoapi.api import DEFAULT_STORAGE_CRS from pygeoapi.provider.base import (BaseProvider, ProviderConnectionError, ProviderNoDataError, ProviderQueryError) -from pygeoapi.util import read_data +from pygeoapi.util import get_crs_from_uri, read_data LOGGER = logging.getLogger(__name__) @@ -81,125 +85,43 @@ def __init__(self, provider_def): else: data_to_open = self.data - self._data = open_func(data_to_open) + try: + self._data = open_func(data_to_open) + except ValueError as err: + # Manage non-cf-compliant time dimensions + if 'time' in str(err): + self._data = open_func(self.data, decode_times=False) + else: + raise err + + self.storage_crs = self._parse_storage_crs(provider_def) self._coverage_properties = self._get_coverage_properties() - self.axes = [self._coverage_properties['x_axis_label'], - self._coverage_properties['y_axis_label'], - self._coverage_properties['time_axis_label']] + self.axes = self._coverage_properties['axes'] - self.fields = self._coverage_properties['fields'] + self.get_fields() except Exception as err: LOGGER.warning(err) raise ProviderConnectionError(err) - def get_coverage_domainset(self, *args, **kwargs): - """ - Provide coverage domainset - - :returns: CIS JSON object of domainset metadata - """ - - c_props = self._coverage_properties - domainset = { - 'type': 'DomainSet', - 'generalGrid': { - 'type': 'GeneralGridCoverage', - 'srsName': c_props['bbox_crs'], - 'axisLabels': [ - c_props['x_axis_label'], - c_props['y_axis_label'], - c_props['time_axis_label'] - ], - 'axis': [{ - 'type': 'RegularAxis', - 'axisLabel': c_props['x_axis_label'], - 'lowerBound': c_props['bbox'][0], - 'upperBound': c_props['bbox'][2], - 'uomLabel': c_props['bbox_units'], - 'resolution': c_props['resx'] - }, { - 'type': 'RegularAxis', - 'axisLabel': c_props['y_axis_label'], - 'lowerBound': c_props['bbox'][1], - 'upperBound': c_props['bbox'][3], - 'uomLabel': c_props['bbox_units'], - 'resolution': c_props['resy'] - }, - { - 'type': 'RegularAxis', - 'axisLabel': c_props['time_axis_label'], - 'lowerBound': c_props['time_range'][0], - 'upperBound': c_props['time_range'][1], - 'uomLabel': c_props['restime'], - 'resolution': c_props['restime'] - } - ], - 'gridLimits': { - 'type': 'GridLimits', - 'srsName': 'http://www.opengis.net/def/crs/OGC/0/Index2D', - 'axisLabels': ['i', 'j'], - 'axis': [{ - 'type': 'IndexAxis', - 'axisLabel': 'i', - 'lowerBound': 0, - 'upperBound': c_props['width'] - }, { - 'type': 'IndexAxis', - 'axisLabel': 'j', - 'lowerBound': 0, - 'upperBound': c_props['height'] - }] - } - }, - '_meta': { - 'tags': self._data.attrs - } - } - - return domainset - - def get_coverage_rangetype(self, *args, **kwargs): - """ - Provide coverage rangetype - - :returns: CIS JSON object of rangetype metadata - """ - - rangetype = { - 'type': 'DataRecord', - 'field': [] - } - - for name, var in self._data.variables.items(): - LOGGER.debug(f'Determining rangetype for {name}') - - desc, units = None, None - if len(var.shape) >= 3: - parameter = self._get_parameter_metadata( - name, var.attrs) - desc = parameter['description'] - units = parameter['unit_label'] - - rangetype['field'].append({ - 'id': name, - 'type': 'Quantity', - 'name': var.attrs.get('long_name') or desc, - 'encodingInfo': { - 'dataType': f'http://www.opengis.net/def/dataType/OGC/0/{var.dtype}' # noqa - }, - 'nodata': 'null', - 'uom': { - 'id': f'http://www.opengis.net/def/uom/UCUM/{units}', - 'type': 'UnitReference', - 'code': units - }, - '_meta': { - 'tags': var.attrs + def get_fields(self): + if not self._fields: + for key, value in self._data.variables.items(): + if key not in self._data.coords: + LOGGER.debug('Adding variable') + dtype = value.dtype + if dtype.name.startswith('float'): + dtype = 'number' + elif dtype.name.startswith('int'): + dtype = 'integer' + + self._fields[key] = { + 'type': dtype, + 'title': value.attrs.get('long_name'), + 'x-ogc-unit': value.attrs.get('units') } - }) - return rangetype + return self._fields def query(self, properties=[], subsets={}, bbox=[], bbox_crs=4326, datetime_=None, format_='json', **kwargs): @@ -224,13 +146,13 @@ def query(self, properties=[], subsets={}, bbox=[], bbox_crs=4326, return read_data(self.data) if len(properties) < 1: - properties = self.fields + properties = self.fields.keys() data = self._data[[*properties]] - if any([self._coverage_properties['x_axis_label'] in subsets, - self._coverage_properties['y_axis_label'] in subsets, - self._coverage_properties['time_axis_label'] in subsets, + if any([self._coverage_properties.get('x_axis_label') in subsets, + self._coverage_properties.get('y_axis_label') in subsets, + self._coverage_properties.get('time_axis_label') in subsets, datetime_ is not None]): LOGGER.debug('Creating spatio-temporal subset') @@ -249,18 +171,36 @@ def query(self, properties=[], subsets={}, bbox=[], bbox_crs=4326, self._coverage_properties['y_axis_label'] in subsets, len(bbox) > 0]): msg = 'bbox and subsetting by coordinates are exclusive' - LOGGER.warning(msg) + LOGGER.error(msg) raise ProviderQueryError(msg) else: - query_params[self._coverage_properties['x_axis_label']] = \ - slice(bbox[0], bbox[2]) - query_params[self._coverage_properties['y_axis_label']] = \ - slice(bbox[1], bbox[3]) + x_axis_label = self._coverage_properties['x_axis_label'] + x_coords = data.coords[x_axis_label] + if x_coords.values[0] > x_coords.values[-1]: + LOGGER.debug( + 'Reversing slicing of x axis from high to low' + ) + query_params[x_axis_label] = slice(bbox[2], bbox[0]) + else: + query_params[x_axis_label] = slice(bbox[0], bbox[2]) + y_axis_label = self._coverage_properties['y_axis_label'] + y_coords = data.coords[y_axis_label] + if y_coords.values[0] > y_coords.values[-1]: + LOGGER.debug( + 'Reversing slicing of y axis from high to low' + ) + query_params[y_axis_label] = slice(bbox[3], bbox[1]) + else: + query_params[y_axis_label] = slice(bbox[1], bbox[3]) LOGGER.debug('bbox_crs is not currently handled') if datetime_ is not None: - if self._coverage_properties['time_axis_label'] in subsets: + if self._coverage_properties['time_axis_label'] is None: + msg = 'Dataset does not contain a time axis' + LOGGER.error(msg) + raise ProviderQueryError(msg) + elif self._coverage_properties['time_axis_label'] in subsets: msg = 'datetime and temporal subsetting are exclusive' LOGGER.error(msg) raise ProviderQueryError(msg) @@ -282,13 +222,15 @@ def query(self, properties=[], subsets={}, bbox=[], bbox_crs=4326, LOGGER.warning(err) raise ProviderQueryError(err) - if (any([data.coords[self.x_field].size == 0, - data.coords[self.y_field].size == 0, - data.coords[self.time_field].size == 0])): + if any(size == 0 for size in data.sizes.values()): msg = 'No data found' LOGGER.warning(msg) raise ProviderNoDataError(msg) + if format_ == 'json': + # json does not support float32 + data = _convert_float32_to_float64(data) + out_meta = { 'bbox': [ data.coords[self.x_field].values[0], @@ -296,18 +238,20 @@ def query(self, properties=[], subsets={}, bbox=[], bbox_crs=4326, data.coords[self.x_field].values[-1], data.coords[self.y_field].values[-1] ], - "time": [ - _to_datetime_string(data.coords[self.time_field].values[0]), - _to_datetime_string(data.coords[self.time_field].values[-1]) - ], "driver": "xarray", - "height": data.dims[self.y_field], - "width": data.dims[self.x_field], - "time_steps": data.dims[self.time_field], + "height": data.sizes[self.y_field], + "width": data.sizes[self.x_field], "variables": {var_name: var.attrs for var_name, var in data.variables.items()} } + if self.time_field is not None: + out_meta['time'] = [ + _to_datetime_string(data.coords[self.time_field].values[0]), + _to_datetime_string(data.coords[self.time_field].values[-1]), + ] + out_meta["time_steps"] = data.sizes[self.time_field] + LOGGER.debug('Serializing data in memory') if format_ == 'json': LOGGER.debug('Creating output in CoverageJSON') @@ -316,26 +260,32 @@ def query(self, properties=[], subsets={}, bbox=[], bbox_crs=4326, LOGGER.debug('Returning data in native zarr format') return _get_zarr_data(data) else: # return data in native format - with tempfile.TemporaryFile() as fp: + with tempfile.NamedTemporaryFile() as fp: LOGGER.debug('Returning data in native NetCDF format') - fp.write(data.to_netcdf()) + data.to_netcdf( + fp.name + ) # we need to pass a string to be able to use the "netcdf4" engine # noqa fp.seek(0) return fp.read() - def gen_covjson(self, metadata, data, range_type): + def gen_covjson(self, metadata, data, fields): """ Generate coverage as CoverageJSON representation :param metadata: coverage metadata :param data: rasterio DatasetReader object - :param range_type: range type list + :param fields: fields :returns: dict of CoverageJSON representation """ LOGGER.debug('Creating CoverageJSON domain') minx, miny, maxx, maxy = metadata['bbox'] - mint, maxt = metadata['time'] + + selected_fields = { + key: value for key, value in self.fields.items() + if key in fields + } try: tmp_min = data.coords[self.y_field].values[0] @@ -366,11 +316,6 @@ def gen_covjson(self, metadata, data, range_type): 'start': maxy, 'stop': miny, 'num': metadata['height'] - }, - self.time_field: { - 'start': mint, - 'stop': maxt, - 'num': metadata['time_steps'] } }, 'referencing': [{ @@ -385,42 +330,51 @@ def gen_covjson(self, metadata, data, range_type): 'ranges': {} } - for variable in range_type: - pm = self._get_parameter_metadata( - variable, self._data[variable].attrs) + if self.time_field is not None: + mint, maxt = metadata['time'] + cj['domain']['axes'][self.time_field] = { + 'start': mint, + 'stop': maxt, + 'num': metadata['time_steps'], + } + for key, value in selected_fields.items(): parameter = { 'type': 'Parameter', - 'description': pm['description'], + 'description': value['title'], 'unit': { - 'symbol': pm['unit_label'] + 'symbol': value['x-ogc-unit'] }, 'observedProperty': { - 'id': pm['observed_property_id'], + 'id': key, 'label': { - 'en': pm['observed_property_name'] + 'en': value['title'] } } } - cj['parameters'][pm['id']] = parameter + cj['parameters'][key] = parameter data = data.fillna(None) - data = _convert_float32_to_float64(data) try: - for key in cj['parameters'].keys(): + for key, value in selected_fields.items(): cj['ranges'][key] = { 'type': 'NdArray', - 'dataType': str(self._data[variable].dtype), + 'dataType': value['type'], 'axisNames': [ - 'y', 'x', self._coverage_properties['time_axis_label'] + 'y', 'x' ], 'shape': [metadata['height'], - metadata['width'], - metadata['time_steps']] + metadata['width']] } cj['ranges'][key]['values'] = data[key].values.flatten().tolist() # noqa + + if self.time_field is not None: + cj['ranges'][key]['axisNames'].append( + self._coverage_properties['time_axis_label'] + ) + cj['ranges'][key]['shape'].append(metadata['time_steps']) except IndexError as err: LOGGER.warning(err) raise ProviderQueryError('Invalid query parameter') @@ -430,19 +384,21 @@ def gen_covjson(self, metadata, data, range_type): def _get_coverage_properties(self): """ Helper function to normalize coverage properties + :param provider_def: provider definition :returns: `dict` of coverage properties """ time_var, y_var, x_var = [None, None, None] + for coord in self._data.coords: if coord.lower() == 'time': time_var = coord continue - if self._data.coords[coord].attrs['units'] == 'degrees_north': + if self._data.coords[coord].attrs.get('units') == 'degrees_north': y_var = coord continue - if self._data.coords[coord].attrs['units'] == 'degrees_east': + if self._data.coords[coord].attrs.get('units') == 'degrees_east': x_var = coord continue @@ -454,7 +410,7 @@ def _get_coverage_properties(self): self.time_field = time_var # It would be preferable to use CF attributes to get width - # resolution etc but for now a generic approach is used to asess + # resolution etc but for now a generic approach is used to assess # all of the attributes based on lat lon vars properties = { @@ -464,47 +420,60 @@ def _get_coverage_properties(self): self._data.coords[self.x_field].values[-1], self._data.coords[self.y_field].values[-1], ], - 'time_range': [ - _to_datetime_string( - self._data.coords[self.time_field].values[0] - ), - _to_datetime_string( - self._data.coords[self.time_field].values[-1] - ) - ], 'bbox_crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', 'crs_type': 'GeographicCRS', 'x_axis_label': self.x_field, 'y_axis_label': self.y_field, - 'time_axis_label': self.time_field, - 'width': self._data.dims[self.x_field], - 'height': self._data.dims[self.y_field], - 'time': self._data.dims[self.time_field], - 'time_duration': self.get_time_coverage_duration(), + 'width': self._data.sizes[self.x_field], + 'height': self._data.sizes[self.y_field], 'bbox_units': 'degrees', - 'resx': np.abs(self._data.coords[self.x_field].values[1] - - self._data.coords[self.x_field].values[0]), - 'resy': np.abs(self._data.coords[self.y_field].values[1] - - self._data.coords[self.y_field].values[0]), - 'restime': self.get_time_resolution() + 'resx': np.abs( + self._data.coords[self.x_field].values[1] + - self._data.coords[self.x_field].values[0] + ), + 'resy': np.abs( + self._data.coords[self.y_field].values[1] + - self._data.coords[self.y_field].values[0] + ), } - if 'crs' in self._data.variables.keys(): - properties['bbox_crs'] = f'http://www.opengis.net/def/crs/OGC/1.3/{self._data.crs.epsg_code}' # noqa - - properties['inverse_flattening'] = self._data.crs.\ - inverse_flattening + if self.time_field is not None: + properties['time_axis_label'] = self.time_field + properties['time_range'] = [ + _to_datetime_string( + self._data.coords[self.time_field].values[0] + ), + _to_datetime_string( + self._data.coords[self.time_field].values[-1] + ), + ] + properties['time'] = self._data.sizes[self.time_field] + properties['time_duration'] = self.get_time_coverage_duration() + properties['restime'] = self.get_time_resolution() + + # Update properties based on the xarray's CRS + epsg_code = self.storage_crs.to_epsg() + LOGGER.debug(f'{epsg_code}') + if epsg_code == 4326 or self.storage_crs == 'OGC:CRS84': + pass + LOGGER.debug('Confirmed default of WGS 84') + else: + properties['bbox_crs'] = \ + f'https://www.opengis.net/def/crs/EPSG/0/{epsg_code}' + properties['inverse_flattening'] = \ + self.storage_crs.ellipsoid.inverse_flattening + if self.storage_crs.is_projected: + properties['crs_type'] = 'ProjectedCRS' - properties['crs_type'] = 'ProjectedCRS' + LOGGER.debug(f'properties: {properties}') properties['axes'] = [ properties['x_axis_label'], - properties['y_axis_label'], - properties['time_axis_label'] + properties['y_axis_label'] ] - properties['fields'] = [name for name in self._data.variables - if len(self._data.variables[name].shape) >= 3] + if self.time_field is not None: + properties['axes'].append(properties['time_axis_label']) return properties @@ -532,7 +501,8 @@ def get_time_resolution(self): :returns: time resolution string """ - if self._data[self.time_field].size > 1: + if self.time_field is not None \ + and self._data[self.time_field].size > 1: time_diff = (self._data[self.time_field][1] - self._data[self.time_field][0]) @@ -549,6 +519,9 @@ def get_time_coverage_duration(self): :returns: time coverage duration string """ + if self.time_field is None: + return None + dur = self._data[self.time_field][-1] - self._data[self.time_field][0] ms_difference = dur.values.astype('timedelta64[ms]').astype(np.double) @@ -564,6 +537,71 @@ def get_time_coverage_duration(self): return ', '.join(times) + def _parse_grid_mapping(self): + """ + Identifies grid_mapping. + + :returns: name of xarray data variable that contains CRS information. + """ + LOGGER.debug('Parsing grid mapping...') + spatiotemporal_dims = (self.time_field, self.y_field, self.x_field) + LOGGER.debug(spatiotemporal_dims) + grid_mapping_name = None + for var_name, var in self._data.variables.items(): + if all(dim in var.dims for dim in spatiotemporal_dims): + try: + grid_mapping_name = self._data[var_name].attrs['grid_mapping'] # noqa + LOGGER.debug(f'Grid mapping: {grid_mapping_name}') + except KeyError as err: + LOGGER.debug(err) + LOGGER.debug('No grid mapping information found.') + return grid_mapping_name + + def _parse_storage_crs( + self, + provider_def: dict + ) -> pyproj.CRS: + """ + Parse the storage CRS from an xarray dataset. + + :param provider_def: provider definition + + :returns: `pyproj.CRS` instance parsed from dataset + """ + storage_crs = None + + try: + storage_crs = provider_def['storage_crs'] + crs_function = pyproj.CRS.from_user_input + except KeyError as err: + LOGGER.debug(err) + LOGGER.debug('No storage_crs found. Attempting to parse the CRS.') + + if storage_crs is None: + grid_mapping = self._parse_grid_mapping() + if grid_mapping is not None: + storage_crs = self._data[grid_mapping].attrs + crs_function = pyproj.CRS.from_cf + elif 'crs' in self._data.variables.keys(): + storage_crs = self._data['crs'].attrs + crs_function = pyproj.CRS.from_dict + else: + storage_crs = DEFAULT_STORAGE_CRS + crs_function = get_crs_from_uri + LOGGER.debug('Failed to parse dataset CRS. Assuming WGS84.') + + LOGGER.debug(f'Parsing CRS {storage_crs} with {crs_function}') + try: + crs = crs_function(storage_crs) + except CRSError as err: + LOGGER.debug(f'Unable to parse projection with pyproj: {err}') + LOGGER.debug('Assuming default WGS84.') + crs = get_crs_from_uri(DEFAULT_STORAGE_CRS) + + LOGGER.debug(crs) + + return crs + def _to_datetime_string(datetime_obj): """ @@ -646,7 +684,7 @@ def _convert_float32_to_float64(data): for var_name in data.variables: if data[var_name].dtype == 'float32': og_attrs = data[var_name].attrs - data[var_name] = data[var_name].astype('float64') + data[var_name] = data[var_name].astype('float64', copy=False) data[var_name].attrs = og_attrs return data diff --git a/pygeoapi/provider/xarray_edr.py b/pygeoapi/provider/xarray_edr.py index f3c3e1af8..f5bf543f2 100644 --- a/pygeoapi/provider/xarray_edr.py +++ b/pygeoapi/provider/xarray_edr.py @@ -57,15 +57,6 @@ def __init__(self, provider_def): BaseEDRProvider.__init__(self, provider_def) XarrayProvider.__init__(self, provider_def) - def get_fields(self): - """ - Get provider field information (names, types) - - :returns: dict of dicts of parameters - """ - - return self.get_coverage_rangetype() - @BaseEDRProvider.register() def position(self, **kwargs): """ @@ -90,14 +81,14 @@ def position(self, **kwargs): wkt = kwargs.get('wkt') if wkt is not None: LOGGER.debug('Processing WKT') - LOGGER.debug(f'Geometry type: {wkt.type}') - if wkt.type == 'Point': + LOGGER.debug(f'Geometry type: {wkt.geom_type}') + if wkt.geom_type == 'Point': query_params[self._coverage_properties['x_axis_label']] = wkt.x query_params[self._coverage_properties['y_axis_label']] = wkt.y - elif wkt.type == 'LineString': + elif wkt.geom_type == 'LineString': query_params[self._coverage_properties['x_axis_label']] = wkt.xy[0] # noqa query_params[self._coverage_properties['y_axis_label']] = wkt.xy[1] # noqa - elif wkt.type == 'Polygon': + elif wkt.geom_type == 'Polygon': query_params[self._coverage_properties['x_axis_label']] = slice(wkt.bounds[0], wkt.bounds[2]) # noqa query_params[self._coverage_properties['y_axis_label']] = slice(wkt.bounds[1], wkt.bounds[3]) # noqa pass @@ -118,7 +109,7 @@ def position(self, **kwargs): try: if select_properties: - self.fields = select_properties + self._fields = {k: v for k, v in self._fields.items() if k in select_properties} # noqa data = self._data[[*select_properties]] else: data = self._data @@ -147,11 +138,11 @@ def position(self, **kwargs): raise ProviderNoDataError() try: - height = data.dims[self.y_field] + height = data.sizes[self.y_field] except KeyError: height = 1 try: - width = data.dims[self.x_field] + width = data.sizes[self.x_field] except KeyError: width = 1 time, time_steps = self._parse_time_metadata(data, kwargs) @@ -215,7 +206,7 @@ def cube(self, **kwargs): LOGGER.debug(f'query parameters: {query_params}') try: if select_properties: - self.fields = select_properties + self._fields = {k: v for k, v in self._fields.items() if k in select_properties} # noqa data = self._data[[*select_properties]] else: data = self._data @@ -224,8 +215,8 @@ def cube(self, **kwargs): except KeyError: raise ProviderNoDataError() - height = data.dims[self.y_field] - width = data.dims[self.x_field] + height = data.sizes[self.y_field] + width = data.sizes[self.x_field] time, time_steps = self._parse_time_metadata(data, kwargs) out_meta = { diff --git a/pygeoapi/schemas/config/pygeoapi-config-0.x.yml b/pygeoapi/schemas/config/pygeoapi-config-0.x.yml index 5d567c93a..ba566b716 100644 --- a/pygeoapi/schemas/config/pygeoapi-config-0.x.yml +++ b/pygeoapi/schemas/config/pygeoapi-config-0.x.yml @@ -60,8 +60,38 @@ properties: default: false limit: type: integer - description: server limit on number of items to return default: 10 + description: "limit of items to return. DEPRECATED: use limits instead" + limits: &x-limits + type: object + description: server level data limiting + properties: + max_items: + type: integer + description: maximum limit of items to return for item type providers + minimum: 1 + default: 10 + default_items: + type: integer + description: default limit of items to return for item type providers + minimum: 1 + default: 10 + max_distance_x: + type: number + description: maximum distance in x for all data providers + max_distance_y: + type: number + description: maximum distance in y for all data providers + max_distance_units: + type: string + description: maximum distance units as per UCUM https://ucum.org/ucum#section-Tables-of-Terminal-Symbols + on_exceed: + type: string + description: how to handle limit exceeding + default: throttle + enum: + - error + - throttle templates: type: object description: optional configuration to specify a different set of templates for HTML pages. Recommend using absolute paths. Omit this to use the default provided templates @@ -336,9 +366,6 @@ properties: type: object description: linked data configuration properties: - item_template: - type: string - description: path to JSON-LD Jinja2 template context: type: array description: additional JSON-LD context @@ -411,8 +438,15 @@ properties: type: [string, 'null'] format: date-time nullable: true + trs: + type: string + description: temporal reference system of features + default: 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' required: - spatial + limits: + <<: *x-limits + description: collection level data limiting providers: type: array description: required connection information diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index 29db43e16..91496b452 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -5,7 +5,7 @@ # Abdulazeez Abdulazeez Adeshina # # Copyright (c) 2020 Francesco Bartoli -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2025 Tom Kralidis # Copyright (c) 2022 Abdulazeez Abdulazeez Adeshina # # Permission is hereby granted, free of charge, to any person @@ -38,7 +38,6 @@ from pathlib import Path import click - from starlette.routing import Route, Mount from starlette.staticfiles import StaticFiles from starlette.applications import Starlette @@ -50,7 +49,15 @@ ) import uvicorn -from pygeoapi.api import API +from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api as core_api +import pygeoapi.api.coverages as coverages_api +import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.itemtypes as itemtypes_api +import pygeoapi.api.maps as maps_api +import pygeoapi.api.processes as processes_api +import pygeoapi.api.stac as stac_api +import pygeoapi.api.tiles as tiles_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_api_rules @@ -63,6 +70,7 @@ OPENAPI = load_openapi_document() if CONFIG['server'].get('admin'): + import pygeoapi.admin as admin_api from pygeoapi.admin import Admin p = Path(__file__) @@ -96,27 +104,7 @@ def call_api_threadsafe( return api_call(*args) -async def get_response( - api_call, - *args, -) -> Union[Response, JSONResponse, HTMLResponse]: - """ - Creates a Starlette Response object and updates matching headers. - - Runs the core api handler in a separate thread in order to avoid - blocking the main event loop. - - :param result: The result of the API call. - This should be a tuple of (headers, status, content). - - :returns: A Response instance. - """ - - loop = asyncio.get_running_loop() - result = await loop.run_in_executor( - None, call_api_threadsafe, loop, api_call, *args) - - headers, status, content = result +def _to_response(headers, status, content): if headers['Content-Type'] == 'text/html': response = HTMLResponse(content=content, status_code=status) else: @@ -130,6 +118,30 @@ async def get_response( return response +async def execute_from_starlette(api_function, request: Request, *args, + skip_valid_check=False, + alternative_api=None + ) -> Response: + actual_api = api_ if alternative_api is None else alternative_api + api_request = await APIRequest.from_starlette(request, actual_api.locales) + content: Union[str, bytes] + if not skip_valid_check and not api_request.is_valid(): + headers, status, content = actual_api.get_format_exception(api_request) + else: + + loop = asyncio.get_running_loop() + headers, status, content = await loop.run_in_executor( + None, call_api_threadsafe, loop, api_function, + actual_api, api_request, *args) + # NOTE: that gzip currently doesn't work in starlette + # https://github.com/geopython/pygeoapi/issues/1591 + content = apply_gzip(headers, content) + + response = _to_response(headers, status, content) + + return response + + async def landing_page(request: Request): """ OGC API landing page endpoint @@ -138,7 +150,7 @@ async def landing_page(request: Request): :returns: Starlette HTTP Response """ - return await get_response(api_.landing_page, request) + return await execute_from_starlette(core_api.landing_page, request) async def openapi(request: Request): @@ -149,7 +161,7 @@ async def openapi(request: Request): :returns: Starlette HTTP Response """ - return await get_response(api_.openapi_, request) + return await execute_from_starlette(core_api.openapi_, request) async def conformance(request: Request): @@ -160,7 +172,7 @@ async def conformance(request: Request): :returns: Starlette HTTP Response """ - return await get_response(api_.conformance, request) + return await execute_from_starlette(core_api.conformance, request) async def get_tilematrix_set(request: Request, tileMatrixSetId=None): @@ -173,8 +185,9 @@ async def get_tilematrix_set(request: Request, tileMatrixSetId=None): if 'tileMatrixSetId' in request.path_params: tileMatrixSetId = request.path_params['tileMatrixSetId'] - return await get_response( - api_.tilematrixset, request, tileMatrixSetId) + return await execute_from_starlette( + tiles_api.tilematrixset, request, tileMatrixSetId, + ) async def get_tilematrix_sets(request: Request): @@ -183,7 +196,23 @@ async def get_tilematrix_sets(request: Request): :returns: HTTP response """ - return await get_response(api_.tilematrixsets, request) + return await execute_from_starlette(tiles_api.tilematrixsets, request) + + +async def collection_schema(request: Request, collection_id=None): + """ + OGC API collections schema endpoint + + :param request: Starlette Request instance + :param collection_id: collection identifier + + :returns: Starlette HTTP Response + """ + if 'collection_id' in request.path_params: + collection_id = request.path_params['collection_id'] + + return await execute_from_starlette(core_api.get_collection_schema, + request, collection_id) async def collection_queryables(request: Request, collection_id=None): @@ -197,8 +226,10 @@ async def collection_queryables(request: Request, collection_id=None): """ if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_queryables, request, collection_id) + + return await execute_from_starlette( + itemtypes_api.get_collection_queryables, request, collection_id, + ) async def get_collection_tiles(request: Request, collection_id=None): @@ -212,8 +243,9 @@ async def get_collection_tiles(request: Request, collection_id=None): """ if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_tiles, request, collection_id) + + return await execute_from_starlette( + tiles_api.get_collection_tiles, request, collection_id) async def get_collection_tiles_metadata(request: Request, collection_id=None, @@ -230,9 +262,10 @@ async def get_collection_tiles_metadata(request: Request, collection_id=None, collection_id = request.path_params['collection_id'] if 'tileMatrixSetId' in request.path_params: tileMatrixSetId = request.path_params['tileMatrixSetId'] - return await get_response( - api_.get_collection_tiles_metadata, request, - collection_id, tileMatrixSetId + + return await execute_from_starlette( + tiles_api.get_collection_tiles_metadata, request, + collection_id, tileMatrixSetId, skip_valid_check=True, ) @@ -261,9 +294,10 @@ async def get_collection_items_tiles(request: Request, collection_id=None, tileRow = request.path_params['tileRow'] if 'tileCol' in request.path_params: tileCol = request.path_params['tileCol'] - return await get_response( - api_.get_collection_tiles_data, request, collection_id, - tileMatrixSetId, tile_matrix, tileRow, tileCol + return await execute_from_starlette( + tiles_api.get_collection_tiles_data, request, collection_id, + tileMatrixSetId, tile_matrix, tileRow, tileCol, + skip_valid_check=True, ) @@ -283,46 +317,48 @@ async def collection_items(request: Request, collection_id=None, item_id=None): if 'item_id' in request.path_params: item_id = request.path_params['item_id'] if item_id is None: - if request.method == 'GET': # list items - return await get_response( - api_.get_collection_items, request, collection_id) - elif request.method == 'POST': # filter or manage items + if request.method == 'POST': # filter or manage items content_type = request.headers.get('content-type') if content_type is not None: if content_type == 'application/geo+json': - return await get_response( - api_.manage_collection_item, request, - 'create', collection_id) + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, + 'create', collection_id, skip_valid_check=True) else: - return await get_response( - api_.post_collection_items, + return await execute_from_starlette( + itemtypes_api.get_collection_items, request, - collection_id + collection_id, + skip_valid_check=True, ) elif request.method == 'OPTIONS': - return await get_response( - api_.manage_collection_item, request, - 'options', collection_id + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, + 'options', collection_id, skip_valid_check=True, ) + else: # GET: list items + return await execute_from_starlette( + itemtypes_api.get_collection_items, request, collection_id, + skip_valid_check=True) elif request.method == 'DELETE': - return await get_response( - api_.manage_collection_item, request, 'delete', - collection_id, item_id + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, 'delete', + collection_id, item_id, skip_valid_check=True, ) elif request.method == 'PUT': - return await get_response( - api_.manage_collection_item, request, 'update', - collection_id, item_id + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, 'update', + collection_id, item_id, skip_valid_check=True, ) elif request.method == 'OPTIONS': - return await get_response( - api_.manage_collection_item, request, 'options', - collection_id, item_id + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, 'options', + collection_id, item_id, skip_valid_check=True, ) else: - return await get_response( - api_.get_collection_item, request, collection_id, item_id) + return await execute_from_starlette( + itemtypes_api.get_collection_item, request, collection_id, item_id) async def collection_coverage(request: Request, collection_id=None): @@ -337,44 +373,12 @@ async def collection_coverage(request: Request, collection_id=None): if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_coverage, request, collection_id) + return await execute_from_starlette( + coverages_api.get_collection_coverage, request, collection_id, + skip_valid_check=True) -async def collection_coverage_domainset(request: Request, collection_id=None): - """ - OGC API - Coverages coverage domainset endpoint - - :param request: Starlette Request instance - :param collection_id: collection identifier - - :returns: Starlette HTTP Response - """ - if 'collection_id' in request.path_params: - collection_id = request.path_params['collection_id'] - - return await get_response( - api_.get_collection_coverage_domainset, request, collection_id) - - -async def collection_coverage_rangetype(request: Request, collection_id=None): - """ - OGC API - Coverages coverage rangetype endpoint - - :param request: Starlette Request instance - :param collection_id: collection identifier - - :returns: Starlette HTTP Response - """ - - if 'collection_id' in request.path_params: - collection_id = request.path_params['collection_id'] - - return await get_response( - api_.get_collection_coverage_rangetype, request, collection_id) - - -async def collection_map(request: Request, collection_id, style_id=None): +async def collection_map(request: Request, collection_id=None, style_id=None): """ OGC API - Maps map render endpoint @@ -389,8 +393,9 @@ async def collection_map(request: Request, collection_id, style_id=None): if 'style_id' in request.path_params: style_id = request.path_params['style_id'] - return await get_response( - api_.get_collection_map, request, collection_id, style_id) + return await execute_from_starlette( + maps_api.get_collection_map, request, collection_id, style_id + ) async def get_processes(request: Request, process_id=None): @@ -405,7 +410,8 @@ async def get_processes(request: Request, process_id=None): if 'process_id' in request.path_params: process_id = request.path_params['process_id'] - return await get_response(api_.describe_processes, request, process_id) + return await execute_from_starlette(processes_api.describe_processes, + request, process_id) async def get_jobs(request: Request, job_id=None): @@ -422,12 +428,14 @@ async def get_jobs(request: Request, job_id=None): job_id = request.path_params['job_id'] if job_id is None: # list of submit job - return await get_response(api_.get_jobs, request) + return await execute_from_starlette(processes_api.get_jobs, request) else: # get or delete job if request.method == 'DELETE': - return await get_response(api_.delete_job, job_id) + return await execute_from_starlette(processes_api.delete_job, + request, job_id) else: # Return status of a specific job - return await get_response(api_.get_jobs, request, job_id) + return await execute_from_starlette(processes_api.get_jobs, + request, job_id) async def execute_process_jobs(request: Request, process_id=None): @@ -443,7 +451,8 @@ async def execute_process_jobs(request: Request, process_id=None): if 'process_id' in request.path_params: process_id = request.path_params['process_id'] - return await get_response(api_.execute_process, request, process_id) + return await execute_from_starlette(processes_api.execute_process, + request, process_id) async def get_job_result(request: Request, job_id=None): @@ -459,36 +468,17 @@ async def get_job_result(request: Request, job_id=None): if 'job_id' in request.path_params: job_id = request.path_params['job_id'] - return await get_response(api_.get_job_result, request, job_id) + return await execute_from_starlette(processes_api.get_job_result, + request, job_id) -async def get_job_result_resource(request: Request, - job_id=None, resource=None): - """ - OGC API - Processes job result resource endpoint - - :param request: Starlette Request instance - :param job_id: job identifier - :param resource: job resource - - :returns: HTTP response - """ - - if 'job_id' in request.path_params: - job_id = request.path_params['job_id'] - if 'resource' in request.path_params: - resource = request.path_params['resource'] - - return await get_response( - api_.get_job_result_resource, request, job_id, resource) - - -async def get_collection_edr_query(request: Request, collection_id=None, instance_id=None): # noqa +async def get_collection_edr_query(request: Request, collection_id=None, instance_id=None, location_id=None): # noqa """ OGC EDR API endpoints :param collection_id: collection identifier :param instance_id: instance identifier + :param location_id: location id of a /locations/ query :returns: HTTP response """ @@ -499,10 +489,24 @@ async def get_collection_edr_query(request: Request, collection_id=None, instanc if 'instance_id' in request.path_params: instance_id = request.path_params['instance_id'] - query_type = request["path"].split('/')[-1] # noqa - return await get_response( - api_.get_collection_edr_query, request, collection_id, - instance_id, query_type + if (request.url.path.endswith('instances') or + (instance_id is not None and + request.url.path.endswith(instance_id))): + return await execute_from_starlette( + edr_api.get_collection_edr_instances, request, collection_id, + instance_id + ) + + if 'location_id' in request.path_params: + location_id = request.path_params['location_id'] + query_type = 'locations' + else: + query_type = request['path'].split('/')[-1] + + return await execute_from_starlette( + edr_api.get_collection_edr_query, request, collection_id, + instance_id, query_type, location_id, + skip_valid_check=True, ) @@ -517,8 +521,9 @@ async def collections(request: Request, collection_id=None): """ if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.describe_collections, request, collection_id) + + return await execute_from_starlette(core_api.describe_collections, request, + collection_id) async def stac_catalog_root(request: Request): @@ -529,7 +534,7 @@ async def stac_catalog_root(request: Request): :returns: Starlette HTTP response """ - return await get_response(api_.get_stac_root, request) + return await execute_from_starlette(stac_api.get_stac_root, request) async def stac_catalog_path(request: Request): @@ -541,7 +546,7 @@ async def stac_catalog_path(request: Request): :returns: Starlette HTTP response """ path = request.path_params["path"] - return await get_response(api_.get_stac_path, request, path) + return await execute_from_starlette(stac_api.get_stac_path, request, path) async def admin_config(request: Request): @@ -552,11 +557,14 @@ async def admin_config(request: Request): """ if request.method == 'GET': - return await get_response(ADMIN.get_config, request) + return await execute_from_starlette(admin_api.get_config_, request, + alternative_api=ADMIN) elif request.method == 'PUT': - return await get_response(ADMIN.put_config, request) + return await execute_from_starlette(admin_api.put_config, request, + alternative_api=ADMIN) elif request.method == 'PATCH': - return await get_response(ADMIN.patch_config, request) + return await execute_from_starlette(admin_api.patch_config, request, + alternative_api=ADMIN) async def admin_config_resources(request: Request): @@ -567,9 +575,11 @@ async def admin_config_resources(request: Request): """ if request.method == 'GET': - return await get_response(ADMIN.get_resources, request) + return await execute_from_starlette(admin_api.get_resources, request, + alternative_api=ADMIN) elif request.method == 'POST': - return await get_response(ADMIN.put_resource, request) + return await execute_from_starlette(admin_api.put_resource, request, + alternative_api=ADMIN) async def admin_config_resource(request: Request, resource_id: str): @@ -585,17 +595,21 @@ async def admin_config_resource(request: Request, resource_id: str): resource_id = request.path_params['resource_id'] if request.method == 'GET': - return await get_response( - ADMIN.get_resource, request, resource_id) + return await execute_from_starlette(admin_api.get_resource, request, + resource_id, + alternative_api=ADMIN) elif request.method == 'PUT': - return await get_response( - ADMIN.put_resource, request, resource_id) + return await execute_from_starlette(admin_api.put_resource, request, + resource_id, + alternative_api=ADMIN) elif request.method == 'PATCH': - return await get_response( - ADMIN.patch_resource, request, resource_id) + return await execute_from_starlette(admin_api.patch_resource, request, + resource_id, + alternative_api=ADMIN) elif request.method == 'DELETE': - return await get_response( - ADMIN.delete_resource, request, resource_id) + return await execute_from_starlette(admin_api.delete_resource, request, + resource_id, + alternative_api=ADMIN) class ApiRulesMiddleware: @@ -637,6 +651,7 @@ async def __call__(self, scope: Scope, Route('/conformance', conformance), Route('/TileMatrixSets/{tileMatrixSetId}', get_tilematrix_set), Route('/TileMatrixSets', get_tilematrix_sets), + Route('/collections/{collection_id:path}/schema', collection_schema), Route('/collections/{collection_id:path}/queryables', collection_queryables), # noqa Route('/collections/{collection_id:path}/tiles', get_collection_tiles), Route('/collections/{collection_id:path}/tiles/{tileMatrixSetId}', get_collection_tiles_metadata), # noqa @@ -645,8 +660,6 @@ async def __call__(self, scope: Scope, Route('/collections/{collection_id:path}/items', collection_items, methods=['GET', 'POST', 'OPTIONS']), # noqa Route('/collections/{collection_id:path}/items/{item_id:path}', collection_items, methods=['GET', 'PUT', 'DELETE', 'OPTIONS']), # noqa Route('/collections/{collection_id:path}/coverage', collection_coverage), # noqa - Route('/collections/{collection_id:path}/coverage/domainset', collection_coverage_domainset), # noqa - Route('/collections/{collection_id:path}/coverage/rangetype', collection_coverage_rangetype), # noqa Route('/collections/{collection_id:path}/map', collection_map), Route('/collections/{collection_id:path}/styles/{style_id:path}/map', collection_map), # noqa Route('/processes', get_processes), @@ -655,23 +668,28 @@ async def __call__(self, scope: Scope, Route('/jobs/{job_id}', get_jobs, methods=['GET', 'DELETE']), Route('/processes/{process_id}/execution', execute_process_jobs, methods=['POST']), # noqa Route('/jobs/{job_id}/results', get_job_result), - Route('/jobs/{job_id}/results/{resource}', get_job_result_resource), Route('/collections/{collection_id:path}/position', get_collection_edr_query), # noqa Route('/collections/{collection_id:path}/area', get_collection_edr_query), Route('/collections/{collection_id:path}/cube', get_collection_edr_query), Route('/collections/{collection_id:path}/radius', get_collection_edr_query), # noqa Route('/collections/{collection_id:path}/trajectory', get_collection_edr_query), # noqa Route('/collections/{collection_id:path}/corridor', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/locations', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/locations/{location_id}', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/instances', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/instances/{instance_id}', get_collection_edr_query), # noqa Route('/collections/{collection_id:path}/instances/{instance_id}/position', get_collection_edr_query), # noqa Route('/collections/{collection_id:path}/instances/{instance_id}/area', get_collection_edr_query), # noqa Route('/collections/{collection_id:path}/instances/{instance_id}/cube', get_collection_edr_query), # noqa Route('/collections/{collection_id:path}/instances/{instance_id}/radius', get_collection_edr_query), # noqa Route('/collections/{collection_id:path}/instances/{instance_id}/trajectory', get_collection_edr_query), # noqa Route('/collections/{collection_id:path}/instances/{instance_id}/corridor', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/instances/{instance_id}/locations', get_collection_edr_query), # noqa + Route('/collections/{collection_id:path}/instances/{instance_id}/locations/{location_id}', get_collection_edr_query), # noqa Route('/collections', collections), Route('/collections/{collection_id:path}', collections), Route('/stac', stac_catalog_root), - Route('/stac/{path:path}', stac_catalog_path), + Route('/stac/{path:path}', stac_catalog_path) ] admin_routes = [ @@ -710,7 +728,12 @@ async def __call__(self, scope: Scope, # CORS: optionally enable from config. if CONFIG['server'].get('cors', False): from starlette.middleware.cors import CORSMiddleware - APP.add_middleware(CORSMiddleware, allow_origins=['*']) + APP.add_middleware( + CORSMiddleware, + allow_origins=['*'], + allow_methods=['*'], + expose_headers=['*'] + ) try: OGC_SCHEMAS_LOCATION = Path(CONFIG['server']['ogc_schemas_location']) diff --git a/pygeoapi/static/css/default.css b/pygeoapi/static/css/default.css index 0dc6c3461..8a4546fdf 100644 --- a/pygeoapi/static/css/default.css +++ b/pygeoapi/static/css/default.css @@ -30,6 +30,15 @@ main { height: 400px; } +#coverages-map { + width: 100%; + height: 80vh; +} + +.c3-tooltip-container { + z-index: 300; +} + /* cancel mini-css header>button uppercase */ header button, header [type="button"], header .button, header [role="button"] { text-transform: none; diff --git a/pygeoapi/templates/_base.html b/pygeoapi/templates/_base.html index c4e9ef62e..e61566aca 100644 --- a/pygeoapi/templates/_base.html +++ b/pygeoapi/templates/_base.html @@ -1,5 +1,5 @@ - + {% block title %}{% endblock %}{% if not self.title() %}{{ config['metadata']['identification']['title'] }}{% endif %} @@ -37,7 +37,7 @@
-
+
{% trans %}Home{% endtrans %} {% endblock %} - + {% set links_found = namespace(json=0, jsonld=0) %} {% for link in data['links'] %} - {% if link['rel'] == 'alternate' and link['type'] and link['type'] in ['application/json', 'application/geo+json'] %} + {% if link['rel'] == 'alternate' and link['type'] and link['type'] in ['application/json', 'application/geo+json', 'application/prs.coverage+json'] %} {% set links_found.json = 1 %} {% trans %}json{% endtrans %} {% elif link['rel'] == 'alternate' and link['type'] and link['type'] == 'application/ld+json' %} diff --git a/pygeoapi/templates/collections/collection.html b/pygeoapi/templates/collections/collection.html index fa2fade16..668d76e34 100644 --- a/pygeoapi/templates/collections/collection.html +++ b/pygeoapi/templates/collections/collection.html @@ -64,6 +64,14 @@

{% trans %}Queryables{% endtrans %}

{% trans %}Display Queryables of{% endtrans %} "{{ data['title'] }}"
+

{% trans %}Schema{% endtrans %}

+ {% for provider in config['resources'][data['id']]['providers'] %} {% if 'tile' in provider['type'] %}

{% trans %}Tiles{% endtrans %}

@@ -77,6 +85,25 @@

{% trans %}Tiles{% endtrans %}

{% endif %} {% endfor %} {% endif %} + + {% if 'parameter_names' in data %} +

Parameters

+ + + + + + + {% for parameter in data['parameter_names'].values() %} + + + + + + {% endfor %} +
idnameunits
{{ parameter['id'] }}{{ parameter['name'] }}{{ parameter['unit']['symbol']['value'] }}
+ {% endif %} +

{% trans %}Links{% endtrans %}

    {% for link in data['links'] %} @@ -86,7 +113,7 @@

    {% trans %}Links{% endtrans %}

    {% endfor %}
- - {% endif %} + + {% endif %}#} {% endblock %} @@ -120,6 +147,14 @@

{% trans %}Storage CRS{% endtrans %}

} )); + {# if this collection has a map representation, add it to the map #} + {% for link in data['links'] %} + {% if link['rel'] == 'http://www.opengis.net/def/rel/ogc/1.0/map' and link['href'] %} + L.tileLayer.wms("{{ link['href'] }}", {"opacity": .7, "transparent": true, "crs": L.CRS.EPSG4326}).addTo(map); + {% endif %} + {% endfor %} + + var bbox_layer = L.polygon([ ['{{ data['extent']['spatial']['bbox'][0][1] }}', '{{ data['extent']['spatial']['bbox'][0][0] }}'], ['{{ data['extent']['spatial']['bbox'][0][3] }}', '{{ data['extent']['spatial']['bbox'][0][0] }}'], diff --git a/pygeoapi/templates/collections/coverage/domainset.html b/pygeoapi/templates/collections/coverage/domainset.html deleted file mode 100644 index 8ca91a482..000000000 --- a/pygeoapi/templates/collections/coverage/domainset.html +++ /dev/null @@ -1,37 +0,0 @@ -{% extends "_base.html" %} -{% block title %}{{ super() }} {{ data['title'] }} {% endblock %} -{% block crumbs %}{{ super() }} -/ {% trans %}Collections{% endtrans %} -/ {{ data['title'] }} -{% endblock %} -{% block body %} -
-

{{ data['title'] }}

-

{{ data['description'] }}

-

{% trans %}Coverage domain set{% endtrans %}

-

{% trans %}Axis labels{% endtrans %}

-
    - {% for al in data['generalGrid']['axisLabels'] %} -
  • {{ al }}
    • - {% endfor %} -
-

{% trans %}Extent{% endtrans %}

-
    -
  • minx: {{ data['generalGrid']['axis'][0]['lowerBound'] }}
    • -
  • miny: {{ data['generalGrid']['axis'][1]['lowerBound'] }}
    • -
  • maxx: {{ data['generalGrid']['axis'][0]['upperBound'] }}
    • -
  • maxy: {{ data['generalGrid']['axis'][1]['upperBound'] }}
    • -
  • {% trans %}Coordinate reference system{% endtrans %}: {{ data['generalGrid']['srsName'] }}
    • -
-

{% trans %}Size{% endtrans %}

-
    -
  • {% trans %}width{% endtrans %}: {{ data['generalGrid']['gridLimits']['axis'][0]['upperBound'] }}
    • -
  • {% trans %}height{% endtrans %}: {{ data['generalGrid']['gridLimits']['axis'][1]['upperBound'] }}
    • -
-

{% trans %}Resolution{% endtrans %}

-
    -
  • {% trans %}x{% endtrans %}: {{ data['generalGrid']['axis'][0]['resolution'] }}
    • -
  • {% trans %}y{% endtrans %}: {{ data['generalGrid']['axis'][1]['resolution'] }}
    • -
-
-{% endblock %} diff --git a/pygeoapi/templates/collections/coverage/rangetype.html b/pygeoapi/templates/collections/coverage/rangetype.html deleted file mode 100644 index d047594a1..000000000 --- a/pygeoapi/templates/collections/coverage/rangetype.html +++ /dev/null @@ -1,19 +0,0 @@ -{% extends "_base.html" %} -{% block title %}{{ super() }} {{ data['title'] }} {% endblock %} -{% block crumbs %}{{ super() }} -/ {% trans %}Collections{% endtrans %} -/ {{ data['title'] }} -{% endblock %} -{% block body %} -
-

{{ data['title'] }}

-

{{ data['description'] }}

-

{% trans %}Coverage range type{% endtrans %}

-

{% trans %}Fields{% endtrans %}

-
    - {% for field in data['field'] %} -
  • {{ field['id'] }}: {{ field['name'] }} ({{ field['definition'] }})
    • - {% endfor %} -
-
-{% endblock %} diff --git a/pygeoapi/templates/collections/edr/instance.html b/pygeoapi/templates/collections/edr/instance.html new file mode 100644 index 000000000..5f4054d69 --- /dev/null +++ b/pygeoapi/templates/collections/edr/instance.html @@ -0,0 +1,42 @@ +{% extends "_base.html" %} +{% block title %}{{ super() }} {{ data['title'] }} {% endblock %} +{% block crumbs %}{{ super() }} +/ {% trans %}Collections{% endtrans %} +{% for link in data['links'] %} + {% if link.rel == 'collection' %} / + {{ link['title'] | truncate( 25 ) }} + {% set col_title = link['title'] %} + {% endif %} +{% endfor %} +/ {% trans %}Instances{% endtrans %} +/ {{ data['id'] }} +{% endblock %} +{% block body %} +
+

{{ data['title'] }}

+

{{ data['description'] }}

+

+ {% for kw in data['keywords'] %} + {{ kw }} + {% endfor %} +

+

{% trans %}Instance{% endtrans %} {{ data['id'] }}

+
+

{% trans %}Links{% endtrans %}

+ +{% endblock %} diff --git a/pygeoapi/templates/collections/edr/instances.html b/pygeoapi/templates/collections/edr/instances.html new file mode 100644 index 000000000..a3e5393dc --- /dev/null +++ b/pygeoapi/templates/collections/edr/instances.html @@ -0,0 +1,29 @@ +{% extends "_base.html" %} +{% block title %}{{ super() }} {{ data['title'] }} {% endblock %} +{% block crumbs %}{{ super() }} +/ {% trans %}Collections{% endtrans %} +{% for link in data['links'] %} + {% if link.rel == 'collection' %} / + {{ link['title'] | truncate( 25 ) }} + {% set col_title = link['title'] %} + {% endif %} +{% endfor %} +/ {% trans %}Instances{% endtrans %} +{% endblock %} +{% block body %} +
+

{{ data['title'] }}

+

{{ data['description'] }}

+

+ {% for kw in data['keywords'] %} + {{ kw }} + {% endfor %} +

+

{% trans %}Instances{% endtrans %}

+ +
+{% endblock %} diff --git a/pygeoapi/templates/collections/edr/query.html b/pygeoapi/templates/collections/edr/query.html index e73519bc8..73c4e5b48 100644 --- a/pygeoapi/templates/collections/edr/query.html +++ b/pygeoapi/templates/collections/edr/query.html @@ -8,54 +8,281 @@ {% set col_title = link['title'] %} {% endif %} {% endfor %} -/ {% trans %}Items{% endtrans %} +/ {% trans query_type=data.query_type %}{{ query_type }}{% endtrans %} {% endblock %} {% block extrahead %} + {% if data.type == "Coverage" or data.type == "CoverageCollection" %} + + + + {% elif data.type == "Feature" or data.type == "FeatureCollection" %} + + + + {% endif %} {% endblock %} {% block body %}
-
+ {% if data.features or data.coverages or data.ranges or data.references %} +
+ {% else %} +
+

{% trans %}No items{% endtrans %}

+
+ {% endif %}
{% endblock %} {% block extrafoot %} {% if data %} {% endif %} {% endblock %} diff --git a/pygeoapi/templates/collections/index.html b/pygeoapi/templates/collections/index.html index 5336df3f4..ae3913917 100644 --- a/pygeoapi/templates/collections/index.html +++ b/pygeoapi/templates/collections/index.html @@ -4,31 +4,60 @@ / {% trans %}Collections{% endtrans %} {% endblock %} {% block body %} -
-

Collections in this service

+
+

{% trans %}Data collections in this service{% endtrans %}

- {% for col in data['collections'] %} + {% if col.get('itemType') != 'record' %} - + {% endif %} {% endfor %}
{% trans %}Name{% endtrans %}{% trans %}Type{% endtrans %} {% trans %}Description{% endtrans %}
{{ col['title'] | striptags | truncate }} {{ col["itemType"] }} {{ col['description'] | striptags | truncate }}
+ {# GA Customisation - Disable Record Collections +
+

{% trans %}Record collections in this service{% endtrans %}

+ + + + + + + + + {% for col in data['collections'] %} + {% if col.get('itemType') == 'record' %} + + + + + {% endif %} + {% endfor %} + +
{% trans %}Name{% endtrans %}{% trans %}Description{% endtrans %}
+ + {{ col['title'] | striptags | truncate }} + + {{ col['description'] | striptags | truncate }} +
+
+ #} {% endblock %} diff --git a/pygeoapi/templates/collections/items/index.html b/pygeoapi/templates/collections/items/index.html index 5de3a63fd..789f8958c 100644 --- a/pygeoapi/templates/collections/items/index.html +++ b/pygeoapi/templates/collections/items/index.html @@ -22,8 +22,8 @@

{% for l in data['links'] if l.rel == 'collection' %} {{ l['title'] }} {% endfor %}

- + {# GA customisation disable trans +

{% trans %}Items in this collection{% endtrans %}.

#}
{% if data['features'] %} @@ -36,7 +36,9 @@

{% for l in data['links'] if l.rel == 'collection' %} {{ l['title'] }} {% en
- {% trans %}Warning: Higher limits not recommended!{% endtrans %} + {% if data['numberMatched'] %} +

{% trans %}Items in this collection{% endtrans %}: {{ data['numberMatched'] }}

+ {% endif %}
@@ -48,6 +50,7 @@

{% for l in data['links'] if l.rel == 'collection' %} {{ l['title'] }} {% en +

{% trans %}Warning: Higher limits not recommended!{% endtrans %}