From 86a9d62dbc31e9abeb6201c324f7365fc8ce466a Mon Sep 17 00:00:00 2001 From: Lyn Long Date: Tue, 3 Feb 2026 13:49:42 +1100 Subject: [PATCH 1/4] update wms_layers --- .../server/core/service/wfs/WfsServer.java | 80 ++------ .../server/core/service/wms/WmsServer.java | 189 +++++++++++++----- .../server/core/util/GeoserverUtils.java | 86 ++++++++ 3 files changed, 242 insertions(+), 113 deletions(-) create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/util/GeoserverUtils.java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java index 8eae715b..a17b5fa9 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java @@ -21,17 +21,17 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; -import org.springframework.web.util.UriComponentsBuilder; -import org.springframework.web.util.UriUtils; import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; + import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.DOWNLOADABLE_FIELDS; import static au.org.aodn.ogcapi.server.core.service.wfs.WfsDefaultParam.WFS_LINK_MARKER; +import static au.org.aodn.ogcapi.server.core.util.GeoserverUtils.extractLayernameOrTypenameFromUrl; +import static au.org.aodn.ogcapi.server.core.util.GeoserverUtils.roughlyMatch; @Slf4j public class WfsServer { @@ -133,6 +133,7 @@ protected Optional> getAllFeatureServerUrls(String collectionId) { return Optional.empty(); } } + /** * Find the url that is able to get WFS call, this can be found in ai:Group * @@ -154,6 +155,7 @@ public Optional getFeatureServerUrlByTitle(String collectionId, String l return Optional.empty(); } } + /** * Find the url that is able to get WFS call, this can be found in ai:Group * @@ -170,7 +172,7 @@ public Optional getFeatureServerUrlByTitleOrQueryParam(String collection .filter(link -> link.getAiGroup() != null) .filter(link -> link.getAiGroup().contains(WFS_LINK_MARKER)) .filter(link -> { - Optional name = extractTypenameFromUrl(link.getHref()); + Optional name = extractLayernameOrTypenameFromUrl(link.getHref()); return link.getTitle().equalsIgnoreCase(layerName) || (name.isPresent() && roughlyMatch(name.get(), layerName)); }) @@ -180,59 +182,7 @@ public Optional getFeatureServerUrlByTitleOrQueryParam(String collection return Optional.empty(); } } - /** - * Fuzzy match utility to compare layer names, ignoring namespace prefixes - * For example: "underway:nuyina_underway_202122020" matches "nuyina_underway_202122020" - * For example: "abc/cde" matches "abc" - * - * @param text1 - First text to compare - * @param text2 - Second text to compare - * @return true if texts match (after removing namespace prefix) and subfix - */ - protected boolean roughlyMatch(String text1, String text2) { - if (text1 == null || text2 == null) { - return false; - } - - // Remove namespace prefix (text before ":") - String normalized1 = text1.contains(":") ? text1.substring(text1.indexOf(":") + 1) : text1; - String normalized2 = text2.contains(":") ? text2.substring(text2.indexOf(":") + 1) : text2; - - // Remove "/" and anything follows - normalized1 = normalized1.split("/")[0]; - normalized2 = normalized2.split("/")[0]; - - return normalized1.equals(normalized2); - } - /** - * Extract typename from WFS URL query parameters - * - * @param url - The WFS URL - * @return typename if found, empty otherwise - */ - protected Optional extractTypenameFromUrl(String url) { - try { - UriComponentsBuilder builder = UriComponentsBuilder.fromUriString(url); - var queryParams = builder.build().getQueryParams(); - // Try different parameter name variations - List typeNames = queryParams.get("typeName"); - if (typeNames == null || typeNames.isEmpty()) { - typeNames = queryParams.get("TYPENAME"); - } - if (typeNames == null || typeNames.isEmpty()) { - typeNames = queryParams.get("typename"); - } - if (typeNames != null && !typeNames.isEmpty()) { - // URL decode the typename (e.g., "underway%3Aunderway_60" -> "underway:underway_60") - String typename = UriUtils.decode(typeNames.get(0), StandardCharsets.UTF_8); - return Optional.of(typename); - } - } catch (Exception e) { - log.debug("Failed to extract typename from URL: {}", url, e); - } - return Optional.empty(); - } /** * Filter WMS layers based on matching with WFS links * Matching logic: @@ -281,7 +231,7 @@ public List filterLayersByWfsLinks(String collectionId, List typename = extractTypenameFromUrl(wfsLink.getHref()); + Optional typename = extractLayernameOrTypenameFromUrl(wfsLink.getHref()); if (typename.isPresent()) { if (roughlyMatch(typename.get(), layer.getName()) || roughlyMatch(typename.get(), layer.getTitle())) { @@ -299,14 +249,14 @@ public List filterLayersByWfsLinks(String collectionId, List aodn_map = filteredLayers.stream().filter(l -> - l.getName().endsWith("_aodn_map") || l.getTitle().endsWith("_aodn_map") - ).toList(); - if(!aodn_map.isEmpty()) { - filteredLayers = aodn_map; - } +// // Very specific logic for AODN, we favor any layer name ends with _aodn_map, so we display +// // map layer similar to old portal, if we cannot find any then display what we have +// List aodn_map = filteredLayers.stream().filter(l -> +// l.getName().endsWith("_aodn_map") || l.getTitle().endsWith("_aodn_map") +// ).toList(); +// if (!aodn_map.isEmpty()) { +// filteredLayers = aodn_map; +// } log.info("Filtered {} layers out of {} based on WFS link matching", filteredLayers.size(), layers.size()); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java index f6408a0a..0c91a5f4 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java @@ -35,6 +35,8 @@ import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.CACHE_WMS_MAP_TILE; import static au.org.aodn.ogcapi.server.core.service.wms.WmsDefaultParam.WMS_LINK_MARKER; +import static au.org.aodn.ogcapi.server.core.util.GeoserverUtils.extractLayernameOrTypenameFromUrl; +import static au.org.aodn.ogcapi.server.core.util.GeoserverUtils.roughlyMatch; @Slf4j public class WmsServer { @@ -72,11 +74,13 @@ public WmsServer(HttpEntity entity) { pretendUserEntity = entity; } + /** * This function is used to append the CQL filter to the geonetwork query, it will guess the correct dataTime field by * some logic, so that if user select filter by range, it works. In case of issue please debug the logic as we are * dealing with different non-standard name - * @param uuid - The uuid of metadata + * + * @param uuid - The uuid of metadata * @param request - The request object to the map * @return - The CQL combined the wfs cql and the dateTime query. */ @@ -85,13 +89,12 @@ protected String createCQLFilter(String uuid, FeatureRequest request) { // If the metadata record have wfs url query, we will use it and analysis it and extract the CQL part if exist Optional wfsUrl = wfsServer.getFeatureServerUrlByTitleOrQueryParam(uuid, request.getLayerName()); - if(wfsUrl.isPresent()) { + if (wfsUrl.isPresent()) { UriComponents wfsUrlComponents = UriComponentsBuilder.fromUriString(wfsUrl.get()).build(); // Extract the CQL if existing in the WFS, we need to apply it to the WMS as well - if(wfsUrlComponents.getQueryParams().get("cql_filter") != null) { + if (wfsUrlComponents.getQueryParams().get("cql_filter") != null) { cql = wfsUrlComponents.getQueryParams().get("cql_filter").get(0); - } - else if(wfsUrlComponents.getQueryParams().get("CQL_FILTER") != null) { + } else if (wfsUrlComponents.getQueryParams().get("CQL_FILTER") != null) { cql = wfsUrlComponents.getQueryParams().get("CQL_FILTER").get(0); } } @@ -136,31 +139,31 @@ else if(wfsUrlComponents.getQueryParams().get("CQL_FILTER") != null) { .filter(v -> Stream.of("juld", "time").anyMatch(k -> v.getName().equalsIgnoreCase(k))) .toList(); - if(individual.size() == 1) { + if (individual.size() == 1) { log.debug("Map datetime field to name to [{}]", individual.get(0).getName()); String timeCql = String.format("CQL_FILTER=%s DURING %s", individual.get(0).getName(), request.getDatetime()); return "".equalsIgnoreCase(cql) ? timeCql : timeCql + " AND " + cql; } } - } - else if(target.size() == 1) { + } else if (target.size() == 1) { log.debug("Map datetime field to name to the only dateTime field [{}]", target.get(0).getName()); String timeCql = String.format("CQL_FILTER=%s DURING %s", target.get(0).getName(), request.getDatetime()); return "".equalsIgnoreCase(cql) ? timeCql : timeCql + " AND " + cql; } } log.error("No date time field found for uuid {}, result will not be bounded by date time even specified", uuid); - } - catch (DownloadableFieldsNotFoundException dfnf) { + } catch (DownloadableFieldsNotFoundException dfnf) { // Without field, we cannot create a valid CQL filte targeting a dateTime, so just return existing CQL if exist } } return "".equalsIgnoreCase(cql) ? "" : String.format("CQL_FILTER=%s", cql); } + /** * Create the full WMS url to fetch the tiles image - * @param url - The url from the metadata, it may point to the wms server only without specifying the remain details, this function will do a smart lookup - * @param uuid - The UUID of the metadata which use to find the WFS links + * + * @param url - The url from the metadata, it may point to the wms server only without specifying the remain details, this function will do a smart lookup + * @param uuid - The UUID of the metadata which use to find the WFS links * @param request - The request like bbox and other param say datetime, layerName (where layerName is not reliable and need lookup internally) * @return - The final URl to do the query */ @@ -177,7 +180,7 @@ protected List createMapQueryUrl(String url, String uuid, FeatureRequest param.putAll(wmsDefaultParam.getWms()); } else if (pathSegments.get(pathSegments.size() - 1).equalsIgnoreCase("ncwms")) { param.putAll(wmsDefaultParam.getNcwms()); - if(request.getDatetime() != null) { + if (request.getDatetime() != null) { param.put("TIME", request.getDatetime()); } } @@ -229,8 +232,7 @@ protected List createMapQueryUrl(String url, String uuid, FeatureRequest String target = builder.build().toUriString(); log.debug("Url to ncWms geoserver {}", target); urls.add(target); - } - else { + } else { // Cannot set cql in param as it contains value like "/" which is not allow in UriComponent checks // but server must use "/" in param and cannot encode it to %2F, so to avoid exception in the // build() call, we append the cql after the construction. @@ -296,7 +298,7 @@ protected List createMapDescribeUrl(String url, String uuid, FeatureRequ log.debug("Url to wms geoserver for describe layer {}", target); urls.add(target); - if (pathSegments.get(pathSegments.size() - 1).equalsIgnoreCase("ncwms") && request.getLayerName().contains("/")) { + if (pathSegments.get(pathSegments.size() - 1).equalsIgnoreCase("ncwms") && request.getLayerName() != null && request.getLayerName().contains("/")) { // Special handle for ncwms, the layer name may be incorrect with /xxx suffix // for example srs_ghrsst_l4_gamssa_url/analysed_sst, we need to remove the /xxxx // Generate more url to test which one works @@ -312,21 +314,24 @@ protected List createMapDescribeUrl(String url, String uuid, FeatureRequ } return null; } + /** * Some URL provided will miss the workspace in url event the layername is xxx:yyy where xxx is workspace * it is a typo in the metadata but manual fix will be very time consuming, so we can safely assume rewrite * the URL will work as it is a geoserver standard. - * @param url - URl that may or may not missing the work space + * + * @param url - URl that may or may not missing the work space * @param request - Request that contains layer name * @return - A rewrite URL or original URL depends on logic */ + // TODO: get wms link by uuid and extract layername for workspace protected static String rewriteUrlWithWorkSpace(String url, FeatureRequest request) { - if(request.getLayerName().contains(":")) { + if (request.getLayerName() != null && request.getLayerName().contains(":")) { String workspace = request.getLayerName().split(":")[0]; UriComponents components = UriComponentsBuilder.fromUriString(url).build(); String workspacePatternInURL = String.format("/%s/", workspace); - if(components.getPath() != null && !components.getPath().contains(workspacePatternInURL)) { + if (components.getPath() != null && !components.getPath().contains(workspacePatternInURL)) { // Need rewrite, get a writable list List segments = new ArrayList<>(components.getPathSegments()); segments.add(segments.size() - 1, workspace); @@ -340,13 +345,16 @@ protected static String rewriteUrlWithWorkSpace(String url, FeatureRequest reque .toUriString(); } } + return url; } + /** * Create the URL to WMS to get the map feature, in this case it will be the content of the popup when you click * the map. - * @param url - URL to WMS - * @param uuid - UUID of record + * + * @param url - URL to WMS + * @param uuid - UUID of record * @param request - Feature requested * @return - List of URL point to the wms queuing map features */ @@ -531,6 +539,7 @@ public DescribeLayerResponse describeLayer(String collectionId, FeatureRequest r } return null; } + /** * Get the wms image/png tile * @@ -552,8 +561,7 @@ public byte[] getMapTile(String collectionId, FeatureRequest request) throws URI if (response.getStatusCode().is2xxSuccessful()) { if (response.getHeaders().getContentType() != null && response.getHeaders().getContentType().getType().equals("image")) { return response.getBody(); - } - else { + } else { // Something wrong from the server likely syntax error throw new URISyntaxException(response.getBody() != null ? new String(response.getBody(), StandardCharsets.UTF_8) : "", url); } @@ -562,10 +570,12 @@ public byte[] getMapTile(String collectionId, FeatureRequest request) throws URI } return null; } + /** * Query the field using WMS's DescriberLayer function to find out the associated WFS layer and fields + * * @param collectionId - The uuid of the metadata that hold this WMS link - * @param request - Request item for this WMS layer, usually layer name, size, etc. + * @param request - Request item for this WMS layer, usually layer name, size, etc. * @return - The fields contained in this WMS layer, we are particular interest in the date time field for subsetting */ public List getDownloadableFields(String collectionId, FeatureRequest request) { @@ -581,6 +591,7 @@ public List getDownloadableFields(String collectionId, F return wfsServer.getDownloadableFields(collectionId, request, null); } } + /** * Fetch raw layers from WMS GetCapabilities - cached by URL, that is query all layer supported by this WMS server. * This allows multiple collections sharing the same WMS server to use cached results @@ -638,13 +649,95 @@ public List fetchCapabilitiesLayersByUrl(String wmsServerUrl) { return Collections.emptyList(); } + + /** + * Filter WMS layers based on matching with WFS links + * Matching logic: + * 1. Primary: link.title matches layer.name OR layer.title (fuzzy match) + * 2. Fallback: extract typename from link URI, then typename matches layer.name OR layer.title (fuzzy match) + * + * @param collectionId - The uuid + * @param layers - List of layers to filter + * @return Filtered list of WMS layers that have matching WFS links + */ + public List filterLayersByWmsLinks(String collectionId, List layers) { + ElasticSearchBase.SearchResult result = search.searchCollections(collectionId); + + if (result.getCollections().isEmpty()) { + log.info("Return empty layers if as no collection found for collectionId: {}", collectionId); + return Collections.emptyList(); + } + + StacCollectionModel model = result.getCollections().get(0); + + // Filter WMS links where ai:group == "Data Access > wms" + List wmsLinks = model.getLinks() + .stream() + .filter(link -> link.getAiGroup() != null) + .filter(link -> link.getAiGroup().contains(WMS_LINK_MARKER)) + .toList(); + + // Filter WMS layers based on matching with WFS links + List filteredLayers = new ArrayList<>(); + + log.debug("=== Starting to match {} layers ===", layers.size()); + for (LayerInfo layer : layers) { + boolean matched = false; + + for (LinkModel wmsLink : wmsLinks) { + // Primary match: link.title matches layer.name OR layer.title + if (wmsLink.getTitle() != null) { + if (roughlyMatch(wmsLink.getTitle(), layer.getName()) || + roughlyMatch(wmsLink.getTitle(), layer.getTitle())) { + log.debug(" ✓ Primary match found - WMS title '{}' matches layer '{}'", + wmsLink.getTitle(), layer.getName()); + matched = true; + break; // This will skip the next if block + } + } + + // Sometimes the wms link.title is wrong but the link.url contains all information + // Fallback match: extract layername from link URI + if (wmsLink.getHref() != null) { + Optional layername = extractLayernameOrTypenameFromUrl(wmsLink.getHref()); + if (layername.isPresent()) { + if (roughlyMatch(layername.get(), layer.getName()) || + roughlyMatch(layername.get(), layer.getTitle())) { + log.debug(" ✓ Fallback match found - layername '{}' matches layer '{}'", + layername.get(), layer.getName()); + matched = true; + break; + } + } + } + } + + if (matched) { + filteredLayers.add(layer); + } + } + +// // Very specific logic for AODN, we favor any layer name ends with _aodn_map, so we display +// // map layer similar to old portal, if we cannot find any then display what we have +// List aodn_map = filteredLayers.stream().filter(l -> +// l.getName().endsWith("_aodn_map") || l.getTitle().endsWith("_aodn_map") +// ).toList(); +// if (!aodn_map.isEmpty()) { +// filteredLayers = aodn_map; +// } + + log.info("Filtered {} wms layers out of {} based on WMS link matching", + filteredLayers.size(), layers.size()); + return filteredLayers; + } + /** * Get filtered layers from WMS GetCapabilities for a specific collection, we use this function because we do not * trust the WMS layer value because it can be wrong, we use the WFS link to infer the layer and therefore the layer * name return will be operational with WFS function. - *

+ *

* First fetches all layers (cached by URL), then filters by WFS links (cached by UUID) - *

+ *

* Sometimes the URL provided by WMS link is not optimal, for example * ... * will result in timeout due to too big query, if layername inside request have format xxx:yyyy then @@ -664,29 +757,29 @@ public List getCapabilitiesLayers(String collectionId, FeatureRequest List allLayers = self.fetchCapabilitiesLayersByUrl(url); if (!allLayers.isEmpty()) { - // Filter layers based on WFS link matching - List filteredLayers = wfsServer.filterLayersByWfsLinks(collectionId, allLayers); - - // If filteredLayers empty, that means no layer have wfs operation, but that does not mean - // the layer cannot serve for display only. - if(filteredLayers.isEmpty() && request.getLayerName() != null) { - DescribeLayerResponse dr = describeLayer(collectionId, request); - if(dr != null) { - // That means at least layer is valid just not works with wfs, we should keep the - // original layername instead showing the lookup name as it can be different from - // what is mentioned in the metadata which people get confused. - filteredLayers = List.of( - LayerInfo.builder() - .name(request.getLayerName()) - .title(request.getLayerName()) - .queryable("0") - .build() - ); - } - } + // Filter layers based on WMS link matching + List filteredLayers = filterLayersByWmsLinks(collectionId, allLayers); + +// // If filteredLayers empty, that means no layer have wfs operation, but that does not mean +// // the layer cannot serve for display only. +// if (filteredLayers.isEmpty() && request.getLayerName() != null) { +// DescribeLayerResponse dr = describeLayer(collectionId, request); +// if (dr != null) { +// // That means at least layer is valid just not works with wfs, we should keep the +// // original layername instead showing the lookup name as it can be different from +// // what is mentioned in the metadata which people get confused. +// filteredLayers = List.of( +// LayerInfo.builder() +// .name(request.getLayerName()) +// .title(request.getLayerName()) +// .queryable("0") +// .build() +// ); +// } +// } // Special case for NCWMS layer where we need to call GetMetadata to find the related points for gridded data - if(mapServerUrl.get().contains("/ncwms")) { + if (mapServerUrl.get().contains("/ncwms")) { filteredLayers.forEach(layer -> { // For each ncwms layer, we attach the metadata UriComponentsBuilder builder = UriComponentsBuilder.fromUriString(mapServerUrl.get()); @@ -700,7 +793,7 @@ public List getCapabilitiesLayers(String collectionId, FeatureRequest builder.queryParam("layerName", layer.getName()); ResponseEntity response = restTemplate.exchange(builder.toUriString(), HttpMethod.GET, pretendUserEntity, String.class); - if(response.getStatusCode().is2xxSuccessful()) { + if (response.getStatusCode().is2xxSuccessful()) { try { layer.setNcWmsLayerInfo(objectMapper.readValue(response.getBody(), NcWmsLayerInfo.class)); } catch (JsonProcessingException e) { diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/GeoserverUtils.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/GeoserverUtils.java new file mode 100644 index 00000000..ef2b0aa5 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/GeoserverUtils.java @@ -0,0 +1,86 @@ +package au.org.aodn.ogcapi.server.core.util; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.util.UriComponentsBuilder; +import org.springframework.web.util.UriUtils; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Optional; + +@Slf4j +public class GeoserverUtils { + /** + * Fuzzy match utility to compare layer names, ignoring namespace prefixes + * For example: "underway:nuyina_underway_202122020" matches "nuyina_underway_202122020" + * For example: "abc/cde" matches "abc" + * + * @param text1 - First text to compare + * @param text2 - Second text to compare + * @return true if texts match (after removing namespace prefix) and subfix + */ + public static boolean roughlyMatch(String text1, String text2) { + if (text1 == null || text2 == null) { + return false; + } + + // Remove namespace prefix (text before ":") + String normalized1 = text1.contains(":") ? text1.substring(text1.indexOf(":") + 1) : text1; + String normalized2 = text2.contains(":") ? text2.substring(text2.indexOf(":") + 1) : text2; + + // Remove "/" and anything follows + normalized1 = normalized1.split("/")[0]; + normalized2 = normalized2.split("/")[0]; + + return normalized1.equals(normalized2); + } + + /** + * Extract typename from WFS or WMS URL query parameters. + * For WFS URLs, looks for typeName/TYPENAME/typename parameter. + * For WMS URLs, looks for layers/LAYERS parameter. + * + * @param url - The WFS or WMS URL + * @return typename/layer name if found, empty otherwise + */ + public static Optional extractLayernameOrTypenameFromUrl(String url) { + try { + UriComponentsBuilder builder = UriComponentsBuilder.fromUriString(url); + var queryParams = builder.build().getQueryParams(); + + // Try WFS parameter name variations (typeName) + List typeNames = queryParams.get("typeName"); + if (typeNames == null || typeNames.isEmpty()) { + typeNames = queryParams.get("TYPENAME"); + } + if (typeNames == null || typeNames.isEmpty()) { + typeNames = queryParams.get("typename"); + } + if (typeNames != null && !typeNames.isEmpty()) { + // URL decode the typename (e.g., "underway%3Aunderway_60" -> "underway:underway_60") + String typename = UriUtils.decode(typeNames.get(0), StandardCharsets.UTF_8); + return Optional.of(typename); + } + + // Try WMS parameter name variations (layers) + // TODO: the layers might be joined layernames + List layerNames = queryParams.get("layers"); + if (layerNames == null || layerNames.isEmpty()) { + layerNames = queryParams.get("LAYERS"); + } + if (layerNames == null || layerNames.isEmpty()) { + layerNames = queryParams.get("Layers"); + } + if (layerNames != null && !layerNames.isEmpty()) { + // URL decode the layer name + String layerName = UriUtils.decode(layerNames.get(0), StandardCharsets.UTF_8); + return Optional.of(layerName); + } + } catch (Exception e) { + log.debug("Failed to extract typename/layer from URL: {}", url, e); + } + return Optional.empty(); + } + + +} From 886a995c1f8ba0445dd367d04eb43ed48a03b97f Mon Sep 17 00:00:00 2001 From: Lyn Long Date: Tue, 3 Feb 2026 16:12:28 +1100 Subject: [PATCH 2/4] implement wfs_layers --- .../core/configuration/CacheConfig.java | 12 +- .../core/model/enumeration/FeatureId.java | 3 +- .../core/model/ogc/wfs/FeatureTypeInfo.java | 70 ++++++++ .../ogc/wfs/WfsGetCapabilitiesResponse.java | 29 ++++ .../ogcapi/server/core/service/CacheWarm.java | 82 +++++++-- .../server/core/service/wfs/WfsServer.java | 158 ++++++++++++++---- .../server/core/service/wms/WmsServer.java | 30 +--- .../aodn/ogcapi/server/features/RestApi.java | 3 + .../ogcapi/server/features/RestServices.java | 16 ++ .../server/service/wfs/WfsServerTest.java | 60 ++----- 10 files changed, 336 insertions(+), 127 deletions(-) create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wfs/FeatureTypeInfo.java create mode 100644 server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wfs/WfsGetCapabilitiesResponse.java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java index 4722b352..5878ff31 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java @@ -2,6 +2,7 @@ import au.org.aodn.ogcapi.server.core.service.CacheNoLandGeometry; import au.org.aodn.ogcapi.server.core.service.CacheWarm; +import au.org.aodn.ogcapi.server.core.service.wfs.WfsServer; import au.org.aodn.ogcapi.server.core.service.wms.WmsServer; import au.org.aodn.ogcapi.server.core.util.GeometryUtils; import org.ehcache.config.builders.*; @@ -29,6 +30,7 @@ public class CacheConfig { public static final String CACHE_WMS_MAP_TILE = "cache-wms-map_tile"; public static final String GET_CAPABILITIES_WMS_LAYERS = "get-capabilities-wms-layers"; + public static final String GET_CAPABILITIES_WFS_LAYERS = "get-capabilities-wfs-layers"; public static final String DOWNLOADABLE_FIELDS = "downloadable-fields"; public static final String ALL_NO_LAND_GEOMETRY = "all-noland-geometry"; public static final String ALL_PARAM_VOCABS = "parameter-vocabs"; @@ -103,6 +105,12 @@ public JCacheCacheManager cacheManager() throws IOException { ResourcePoolsBuilder.heap(20) ).withExpiry(ExpiryPolicyBuilder.timeToLiveExpiration(Duration.ofHours(24))) ) + .withCache(GET_CAPABILITIES_WFS_LAYERS, + CacheConfigurationBuilder.newCacheConfigurationBuilder( + Object.class, Object.class, + ResourcePoolsBuilder.heap(20) + ).withExpiry(ExpiryPolicyBuilder.timeToLiveExpiration(Duration.ofHours(24))) + ) .build(); @@ -119,8 +127,8 @@ public JCacheCacheManager cacheManager() throws IOException { } @Bean - public CacheWarm createCacheWarm(WmsServer wmsServer, CacheNoLandGeometry cacheNoLandGeometry, GeometryUtils geometryUtils) { + public CacheWarm createCacheWarm(WmsServer wmsServer, WfsServer wfsServer, CacheNoLandGeometry cacheNoLandGeometry, GeometryUtils geometryUtils) { GeometryUtils.setSelf(geometryUtils); - return new CacheWarm(wmsServer, cacheNoLandGeometry, geometryUtils); + return new CacheWarm(wmsServer, wfsServer, cacheNoLandGeometry, geometryUtils); } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureId.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureId.java index b6c0e22e..75ed6fa2 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureId.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/FeatureId.java @@ -8,7 +8,8 @@ public enum FeatureId { wms_downloadable_fields("wms_downloadable_fields"), // Query field based on value from wms describe layer query wms_map_tile("wms_map_tile"), wms_map_feature("wms_map_feature"), - wms_layers("wms_layers"); // Get all available layers from WMS GetCapabilities + wms_layers("wms_layers"), // Get all available layers from WMS GetCapabilities + wfs_layers("wfs_layers"); // Get all available feature types from WFS GetCapabilities public final String featureId; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wfs/FeatureTypeInfo.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wfs/FeatureTypeInfo.java new file mode 100644 index 00000000..98586980 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wfs/FeatureTypeInfo.java @@ -0,0 +1,70 @@ +package au.org.aodn.ogcapi.server.core.model.ogc.wfs; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@JacksonXmlRootElement(localName = "FeatureType") +public class FeatureTypeInfo { + + @JacksonXmlProperty(localName = "Name") + protected String name; + + @JacksonXmlProperty(localName = "Title") + protected String title; + + @JsonProperty("abstract") + @JacksonXmlProperty(localName = "Abstract") + private String abstract_; + + @JacksonXmlProperty(localName = "Keywords", namespace = "http://www.opengis.net/ows/1.1") + private Keywords keywords; + + @JacksonXmlProperty(localName = "DefaultCRS") + private String defaultCRS; + + @JacksonXmlProperty(localName = "WGS84BoundingBox", namespace = "http://www.opengis.net/ows/1.1") + private WGS84BoundingBox wgs84BoundingBox; + + @JacksonXmlProperty(localName = "MetadataURL") + private MetadataURL metadataURL; + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static class Keywords { + @JacksonXmlElementWrapper(useWrapping = false) + @JacksonXmlProperty(localName = "Keyword", namespace = "http://www.opengis.net/ows/1.1") + private List keyword; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static class WGS84BoundingBox { + @JacksonXmlProperty(localName = "LowerCorner", namespace = "http://www.opengis.net/ows/1.1") + private String lowerCorner; + + @JacksonXmlProperty(localName = "UpperCorner", namespace = "http://www.opengis.net/ows/1.1") + private String upperCorner; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static class MetadataURL { + @JacksonXmlProperty(isAttribute = true, localName = "href", namespace = "http://www.w3.org/1999/xlink") + private String href; + } +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wfs/WfsGetCapabilitiesResponse.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wfs/WfsGetCapabilitiesResponse.java new file mode 100644 index 00000000..45037509 --- /dev/null +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/wfs/WfsGetCapabilitiesResponse.java @@ -0,0 +1,29 @@ +package au.org.aodn.ogcapi.server.core.model.ogc.wfs; + +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@JacksonXmlRootElement(localName = "WFS_Capabilities", namespace = "http://www.opengis.net/wfs/2.0") +public class WfsGetCapabilitiesResponse { + + @JacksonXmlProperty(localName = "FeatureTypeList") + private FeatureTypeList featureTypeList; + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static class FeatureTypeList { + @JacksonXmlElementWrapper(useWrapping = false) + @JacksonXmlProperty(localName = "FeatureType") + private List featureTypes; + } +} diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheWarm.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheWarm.java index d90abd58..76101d29 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheWarm.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheWarm.java @@ -1,6 +1,7 @@ package au.org.aodn.ogcapi.server.core.service; import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; +import au.org.aodn.ogcapi.server.core.service.wfs.WfsServer; import au.org.aodn.ogcapi.server.core.service.wms.WmsServer; import au.org.aodn.ogcapi.server.core.util.GeometryUtils; import lombok.extern.slf4j.Slf4j; @@ -10,9 +11,13 @@ import org.springframework.retry.annotation.Backoff; import org.springframework.retry.annotation.Retryable; import org.springframework.scheduling.annotation.Scheduled; + import java.util.List; import java.util.Map; +import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.GET_CAPABILITIES_WFS_LAYERS; +import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.GET_CAPABILITIES_WMS_LAYERS; + /** * Some WMS server response very slow for GetCapabilities operation, so we warm some * of those operation for the target server. @@ -21,7 +26,7 @@ public class CacheWarm { // Hardcode server list as not expect to change much overtime, add more if needed, the operation is // heavy so we warm the cache when start - protected List getCapabilitiesUrls = List.of( + protected List getWmsCapabilitiesUrls = List.of( "https://data.aad.gov.au/geoserver/underway/wms", "https://www.cmar.csiro.au/geoserver/caab/wms", "https://www.cmar.csiro.au/geoserver/ereefs/wms", @@ -34,7 +39,18 @@ public class CacheWarm { "https://geoserver.apps.aims.gov.au/aims/wms", "https://geoserver.apps.aims.gov.au/reefcloud/wms" ); + + // WFS server URLs for cache warming - using /ows endpoint which supports both WMS and WFS + protected List getWfsCapabilitiesUrls = List.of( + "https://geoserver-123.aodn.org.au/geoserver/ows", + "https://geoserver.imas.utas.edu.au/geoserver/ows", + "https://www.cmar.csiro.au/geoserver/ows", + "https://geoserver.apps.aims.gov.au/aims/ows", + "https://data.aad.gov.au/geoserver/underway/ows" + ); + protected WmsServer wmsServer; + protected WfsServer wfsServer; protected GeometryUtils geometryUtils; protected CacheNoLandGeometry cacheNoLandGeometry; @@ -43,21 +59,36 @@ public class CacheWarm { protected CacheWarm self; public CacheWarm(WmsServer wmsServer, + WfsServer wfsServer, CacheNoLandGeometry cacheNoLandGeometry, GeometryUtils geometryUtils) { this.cacheNoLandGeometry = cacheNoLandGeometry; this.geometryUtils = geometryUtils; this.wmsServer = wmsServer; + this.wfsServer = wfsServer; } + /** - * Scheduled task to refresh specific keys 5 sec after starts and then every 23 hours + * Scheduled task to refresh WMS cache 5 sec after starts and then every 23 hours */ @Scheduled(initialDelay = 5000, fixedRate = 23 * 60 * 60 * 1000) - public void evictSpecificCacheEntries() { - // Evict specific keys one by one - getCapabilitiesUrls.forEach(url -> { - self.evictGetCapabilities(url); - self.warmGetCapabilities(url); + public void evictAndWarmWmsCache() { + // Evict and warm WMS cache entries one by one + getWmsCapabilitiesUrls.forEach(url -> { + self.evictWmsGetCapabilities(url); + self.warmWmsGetCapabilities(url); + }); + } + + /** + * Scheduled task to refresh WFS cache 10 sec after starts and then every 23 hours + */ + @Scheduled(initialDelay = 10000, fixedRate = 23 * 60 * 60 * 1000) + public void evictAndWarmWfsCache() { + // Evict and warm WFS cache entries one by one + getWfsCapabilitiesUrls.forEach(url -> { + self.evictWfsGetCapabilities(url); + self.warmWfsGetCapabilities(url); }); } @@ -71,24 +102,41 @@ public void keepWarmNoLandGeometryAndGeometryUtil() { } - @CacheEvict(value = au.org.aodn.ogcapi.server.core.configuration.CacheConfig.GET_CAPABILITIES_WMS_LAYERS, key = "#key") - protected void evictGetCapabilities(String key){ + // ==================== WMS Cache Methods ==================== + + @CacheEvict(value = GET_CAPABILITIES_WMS_LAYERS, key = "#key") + public void evictWmsGetCapabilities(String key) { // @CacheEvict handles the eviction - log.info("Evicting cache {} for key {}", - au.org.aodn.ogcapi.server.core.configuration.CacheConfig.GET_CAPABILITIES_WMS_LAYERS, - key - ); + log.info("Evicting WMS cache {} for key {}", GET_CAPABILITIES_WMS_LAYERS, key); } @Retryable(maxAttempts = 5, backoff = @Backoff(delay = 1000)) - protected void warmGetCapabilities(String url) { + protected void warmWmsGetCapabilities(String url) { try { // Call and warm cache wmsServer.fetchCapabilitiesLayersByUrl(url); - log.info("Cache GetCapabilities warm success for {}", url); + log.info("WMS Cache GetCapabilities warm success for {}", url); + } catch (RuntimeException rte) { + log.warn("WMS Cache GetCapabilities warm failed for {}, call will be slow", url); } - catch(RuntimeException rte) { - log.warn("Cache GetCapabilities warm failed for {}, call will be slow", url); + } + + // ==================== WFS Cache Methods ==================== + + @CacheEvict(value = GET_CAPABILITIES_WFS_LAYERS, key = "#key") + public void evictWfsGetCapabilities(String key) { + // @CacheEvict handles the eviction + log.info("Evicting WFS cache {} for key {}", GET_CAPABILITIES_WFS_LAYERS, key); + } + + @Retryable(maxAttempts = 5, backoff = @Backoff(delay = 1000)) + protected void warmWfsGetCapabilities(String url) { + try { + // Call and warm cache + wfsServer.fetchCapabilitiesFeatureTypesByUrl(url); + log.info("WFS Cache GetCapabilities warm success for {}", url); + } catch (RuntimeException rte) { + log.warn("WFS Cache GetCapabilities warm failed for {}, call will be slow", url); } } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java index a17b5fa9..d08435d6 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java @@ -5,8 +5,9 @@ import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; import au.org.aodn.ogcapi.server.core.model.ogc.FeatureRequest; import au.org.aodn.ogcapi.server.core.model.ogc.wfs.WfsDescribeFeatureTypeResponse; +import au.org.aodn.ogcapi.server.core.model.ogc.wfs.WfsGetCapabilitiesResponse; +import au.org.aodn.ogcapi.server.core.model.ogc.wfs.FeatureTypeInfo; import au.org.aodn.ogcapi.server.core.model.ogc.wfs.DownloadableFieldModel; -import au.org.aodn.ogcapi.server.core.model.ogc.wms.LayerInfo; import au.org.aodn.ogcapi.server.core.service.ElasticSearchBase; import au.org.aodn.ogcapi.server.core.service.Search; import au.org.aodn.ogcapi.server.core.util.RestTemplateUtils; @@ -15,12 +16,16 @@ import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.Cacheable; +import org.springframework.context.annotation.Lazy; import org.springframework.http.HttpEntity; import org.springframework.http.HttpMethod; import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; import java.net.URISyntaxException; import java.util.ArrayList; @@ -29,6 +34,7 @@ import java.util.Optional; import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.DOWNLOADABLE_FIELDS; +import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.GET_CAPABILITIES_WFS_LAYERS; import static au.org.aodn.ogcapi.server.core.service.wfs.WfsDefaultParam.WFS_LINK_MARKER; import static au.org.aodn.ogcapi.server.core.util.GeoserverUtils.extractLayernameOrTypenameFromUrl; import static au.org.aodn.ogcapi.server.core.util.GeoserverUtils.roughlyMatch; @@ -43,6 +49,10 @@ public class WfsServer { protected Search search; protected HttpEntity pretendUserEntity; + @Lazy + @Autowired + protected WfsServer self; + public WfsServer(Search search, DownloadableFieldsService downloadableFieldsService, RestTemplate restTemplate, @@ -184,48 +194,103 @@ public Optional getFeatureServerUrlByTitleOrQueryParam(String collection } /** - * Filter WMS layers based on matching with WFS links + * Fetch raw feature types from WFS GetCapabilities - cached by URL. + * This allows multiple collections sharing the same WFS server to use cached results. + * + * @param wfsServerUrl - The WFS server base URL + * @return - List of all FeatureTypeInfo objects from GetCapabilities (unfiltered) + */ + @Cacheable(value = GET_CAPABILITIES_WFS_LAYERS) + public List fetchCapabilitiesFeatureTypesByUrl(String wfsServerUrl) { + try { + // Parse the base URL to construct GetCapabilities request + UriComponents components = UriComponentsBuilder.fromUriString(wfsServerUrl).build(); + + // Build GetCapabilities URL + UriComponentsBuilder builder = UriComponentsBuilder + .newInstance() + .scheme("https") // hardcode to be https to avoid redirect + .port(components.getPort()) + .host(components.getHost()) + .path(components.getPath() != null ? components.getPath() : "/geoserver/ows") + .queryParam("service", "wfs") + .queryParam("request", "GetCapabilities"); + + String url = builder.build().toUriString(); + log.debug("WFS GetCapabilities URL: {}", url); + + // Make the HTTPS call + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, pretendUserEntity, String.class); + + if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) { + // Parse XML response + WfsGetCapabilitiesResponse capabilitiesResponse = xmlMapper.readValue( + response.getBody(), + WfsGetCapabilitiesResponse.class + ); + + // Extract all feature types + if (capabilitiesResponse != null + && capabilitiesResponse.getFeatureTypeList() != null + && capabilitiesResponse.getFeatureTypeList().getFeatureTypes() != null) { + + List featureTypes = capabilitiesResponse.getFeatureTypeList().getFeatureTypes(); + + log.info("Fetched and cached WFS get-capabilities feature types: {} ", featureTypes.size()); + return featureTypes; + } + } + } catch (RestClientException | JsonProcessingException e) { + log.error("Error fetching WFS GetCapabilities for URL: {}", wfsServerUrl, e); + throw new RuntimeException(e); + } + + return Collections.emptyList(); + } + + /** + * Filter feature types based on matching with WFS links from ElasticSearch * Matching logic: - * 1. Primary: link.title matches layer.name OR layer.title (fuzzy match) - * 2. Fallback: extract typename from link URI, then typename matches layer.name OR layer.title (fuzzy match) + * 1. Primary: link.title matches featureType.name OR featureType.title (fuzzy match) + * 2. Fallback: extract typename from link URI, then typename matches featureType.name OR featureType.title (fuzzy match) * * @param collectionId - The uuid - * @param layers - List of layers to filter - * @return Filtered list of WMS layers that have matching WFS links + * @param featureTypes - List of feature types to filter + * @return Filtered list of feature types that have matching WFS links */ - public List filterLayersByWfsLinks(String collectionId, List layers) { + public List filterFeatureTypesByWfsLinks(String collectionId, List featureTypes) { ElasticSearchBase.SearchResult result = search.searchCollections(collectionId); if (result.getCollections().isEmpty()) { - log.info("Return empty layers if as no collection found for collectionId: {}", collectionId); + log.info("Return empty feature types as no collection found for collectionId: {}", collectionId); return Collections.emptyList(); } StacCollectionModel model = result.getCollections().get(0); - // Filter WFS links where ai:group == "Data Access > wfs" + // Filter WFS links where ai:group contains "Data Access > wfs" List wfsLinks = model.getLinks() .stream() .filter(link -> link.getAiGroup() != null) .filter(link -> link.getAiGroup().contains(WFS_LINK_MARKER)) .toList(); - // Filter WMS layers based on matching with WFS links - List filteredLayers = new ArrayList<>(); + // Filter feature types based on matching with WFS links + List filteredFeatureTypes = new ArrayList<>(); - log.debug("=== Starting to match {} layers ===", layers.size()); - for (LayerInfo layer : layers) { + log.debug("=== Starting to match {} feature types ===", featureTypes.size()); + for (FeatureTypeInfo featureType : featureTypes) { boolean matched = false; for (LinkModel wfsLink : wfsLinks) { - // Primary match: link.title matches layer.name OR layer.title + // Primary match: link.title matches featureType.name OR featureType.title if (wfsLink.getTitle() != null) { - if (roughlyMatch(wfsLink.getTitle(), layer.getName()) || - roughlyMatch(wfsLink.getTitle(), layer.getTitle())) { - log.debug(" ✓ Primary match found - WFS title '{}' matches layer '{}'", - wfsLink.getTitle(), layer.getName()); + if (roughlyMatch(wfsLink.getTitle(), featureType.getName()) || + roughlyMatch(wfsLink.getTitle(), featureType.getTitle())) { + log.debug(" ✓ Primary match found - WFS title '{}' matches feature type '{}'", + wfsLink.getTitle(), featureType.getName()); matched = true; - break; // This will skip the next if block + break; } } @@ -233,10 +298,10 @@ public List filterLayersByWfsLinks(String collectionId, List typename = extractLayernameOrTypenameFromUrl(wfsLink.getHref()); if (typename.isPresent()) { - if (roughlyMatch(typename.get(), layer.getName()) || - roughlyMatch(typename.get(), layer.getTitle())) { - log.debug(" ✓ Fallback match found - typename '{}' matches layer '{}'", - typename.get(), layer.getName()); + if (roughlyMatch(typename.get(), featureType.getName()) || + roughlyMatch(typename.get(), featureType.getTitle())) { + log.debug(" ✓ Fallback match found - typename '{}' matches feature type '{}'", + typename.get(), featureType.getName()); matched = true; break; } @@ -245,21 +310,42 @@ public List filterLayersByWfsLinks(String collectionId, List getCapabilitiesFeatureTypes(String collectionId, FeatureRequest request) { + Optional> wfsServerUrls = getAllFeatureServerUrls(collectionId); + + if (wfsServerUrls.isPresent() && !wfsServerUrls.get().isEmpty()) { + // Use the first WFS server URL + String wfsServerUrl = wfsServerUrls.get().get(0); + + // Fetch all feature types from GetCapabilities (this call is cached by URL) + List allFeatureTypes = self.fetchCapabilitiesFeatureTypesByUrl(wfsServerUrl); + + if (!allFeatureTypes.isEmpty()) { + // Filter feature types based on WFS link matching + List filteredFeatureTypes = filterFeatureTypesByWfsLinks(collectionId, allFeatureTypes); + + log.debug("Returning feature types {}", filteredFeatureTypes); + return filteredFeatureTypes; } } -// // Very specific logic for AODN, we favor any layer name ends with _aodn_map, so we display -// // map layer similar to old portal, if we cannot find any then display what we have -// List aodn_map = filteredLayers.stream().filter(l -> -// l.getName().endsWith("_aodn_map") || l.getTitle().endsWith("_aodn_map") -// ).toList(); -// if (!aodn_map.isEmpty()) { -// filteredLayers = aodn_map; -// } - - log.info("Filtered {} layers out of {} based on WFS link matching", - filteredLayers.size(), layers.size()); - return filteredLayers; + return Collections.emptyList(); } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java index 0c91a5f4..e0bdfc7a 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java @@ -34,6 +34,7 @@ import java.util.stream.Stream; import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.CACHE_WMS_MAP_TILE; +import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.GET_CAPABILITIES_WMS_LAYERS; import static au.org.aodn.ogcapi.server.core.service.wms.WmsDefaultParam.WMS_LINK_MARKER; import static au.org.aodn.ogcapi.server.core.util.GeoserverUtils.extractLayernameOrTypenameFromUrl; import static au.org.aodn.ogcapi.server.core.util.GeoserverUtils.roughlyMatch; @@ -599,7 +600,7 @@ public List getDownloadableFields(String collectionId, F * @param wmsServerUrl - The WMS server base URL * @return - List of all LayerInfo objects from GetCapabilities (unfiltered) */ - @Cacheable(value = au.org.aodn.ogcapi.server.core.configuration.CacheConfig.GET_CAPABILITIES_WMS_LAYERS) + @Cacheable(value = GET_CAPABILITIES_WMS_LAYERS) public List fetchCapabilitiesLayersByUrl(String wmsServerUrl) { try { // Parse the base URL to construct GetCapabilities request @@ -717,15 +718,6 @@ public List filterLayersByWmsLinks(String collectionId, List aodn_map = filteredLayers.stream().filter(l -> -// l.getName().endsWith("_aodn_map") || l.getTitle().endsWith("_aodn_map") -// ).toList(); -// if (!aodn_map.isEmpty()) { -// filteredLayers = aodn_map; -// } - log.info("Filtered {} wms layers out of {} based on WMS link matching", filteredLayers.size(), layers.size()); return filteredLayers; @@ -760,24 +752,6 @@ public List getCapabilitiesLayers(String collectionId, FeatureRequest // Filter layers based on WMS link matching List filteredLayers = filterLayersByWmsLinks(collectionId, allLayers); -// // If filteredLayers empty, that means no layer have wfs operation, but that does not mean -// // the layer cannot serve for display only. -// if (filteredLayers.isEmpty() && request.getLayerName() != null) { -// DescribeLayerResponse dr = describeLayer(collectionId, request); -// if (dr != null) { -// // That means at least layer is valid just not works with wfs, we should keep the -// // original layername instead showing the lookup name as it can be different from -// // what is mentioned in the metadata which people get confused. -// filteredLayers = List.of( -// LayerInfo.builder() -// .name(request.getLayerName()) -// .title(request.getLayerName()) -// .queryable("0") -// .build() -// ); -// } -// } - // Special case for NCWMS layer where we need to call GetMetadata to find the related points for gridded data if (mapServerUrl.get().contains("/ncwms")) { filteredLayers.forEach(layer -> { diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java index e088ec01..d2c106c8 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestApi.java @@ -140,6 +140,9 @@ public ResponseEntity getFeature( case wms_layers -> { return featuresService.getWmsLayers(collectionId, request); } + case wfs_layers -> { + return featuresService.getWfsLayers(collectionId, request); + } default -> { return ResponseEntity.status(HttpStatus.NOT_IMPLEMENTED).build(); } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java index 2917ea7d..422b8b1d 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java @@ -3,6 +3,7 @@ import au.org.aodn.ogcapi.features.model.Collection; import au.org.aodn.ogcapi.server.core.exception.DownloadableFieldsNotFoundException; import au.org.aodn.ogcapi.server.core.model.ogc.FeatureRequest; +import au.org.aodn.ogcapi.server.core.model.ogc.wfs.FeatureTypeInfo; import au.org.aodn.ogcapi.server.core.model.ogc.wms.FeatureInfoResponse; import au.org.aodn.ogcapi.server.core.model.ogc.wms.LayerInfo; import au.org.aodn.ogcapi.server.core.service.DasService; @@ -150,6 +151,21 @@ public ResponseEntity getWmsLayers(String collectionId, FeatureRequest reques ResponseEntity.notFound().build() : ResponseEntity.ok(result); } + + /** + * This is used to get all available feature types from WFS GetCapabilities + * + * @param collectionId - The uuid of dataset + * @param request - Request to get feature types + * @return - List of available feature types with name, title, abstract, etc. + */ + public ResponseEntity getWfsLayers(String collectionId, FeatureRequest request) { + List result = wfsServer.getCapabilitiesFeatureTypes(collectionId, request); + + return result.isEmpty() ? + ResponseEntity.notFound().build() : + ResponseEntity.ok(result); + } /** * @param collectionID - uuid * @param from - diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/WfsServerTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/WfsServerTest.java index 360e42c0..9e1ed002 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/WfsServerTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/WfsServerTest.java @@ -2,7 +2,7 @@ import au.org.aodn.ogcapi.server.core.model.LinkModel; import au.org.aodn.ogcapi.server.core.model.StacCollectionModel; -import au.org.aodn.ogcapi.server.core.model.ogc.wms.LayerInfo; +import au.org.aodn.ogcapi.server.core.model.ogc.wfs.FeatureTypeInfo; import au.org.aodn.ogcapi.server.core.service.ElasticSearchBase; import au.org.aodn.ogcapi.server.core.service.Search; import au.org.aodn.ogcapi.server.core.service.wfs.DownloadableFieldsService; @@ -53,23 +53,24 @@ public void setUp() { void cleanUp() throws Exception { closeableMock.close(); } + /** * Test null case where the dataset have the collection id not found */ @Test - void noCollection_returnsEmptyLayers() { + void noCollection_returnsEmptyFeatureTypes() { ElasticSearchBase.SearchResult result = new ElasticSearchBase.SearchResult<>(); result.setCollections(Collections.emptyList()); when(mockSearch.searchCollections(anyString())).thenReturn(result); WfsServer server = new WfsServer(mockSearch, downloadableFieldsService, restTemplate, new RestTemplateUtils(restTemplate), entity); - List layers = Collections.singletonList(LayerInfo.builder().build()); - assertEquals(Collections.emptyList(), server.filterLayersByWfsLinks("id", layers)); + List featureTypes = Collections.singletonList(FeatureTypeInfo.builder().build()); + assertEquals(Collections.emptyList(), server.filterFeatureTypesByWfsLinks("id", featureTypes)); } @Test - void noWfsLinks_returnsEmptyLayers() { + void noWfsLinks_returnsEmptyFeatureTypes() { StacCollectionModel model = mock(StacCollectionModel.class); when(model.getLinks()).thenReturn(Collections.emptyList()); @@ -80,51 +81,24 @@ void noWfsLinks_returnsEmptyLayers() { WfsServer server = new WfsServer(mockSearch, downloadableFieldsService, restTemplate, new RestTemplateUtils(restTemplate), entity); - List layers = Collections.singletonList(LayerInfo.builder().build()); - assertEquals(Collections.emptyList(), server.filterLayersByWfsLinks("id", layers)); + List featureTypes = Collections.singletonList(FeatureTypeInfo.builder().build()); + assertEquals(Collections.emptyList(), server.filterFeatureTypesByWfsLinks("id", featureTypes)); } - /** - * The function should fine one because title name matches - */ - @Test - void primaryTitleMatch_filtersMatchingLayers() { - LinkModel wfsLink = LinkModel.builder() - .title("test_layer") - .aiGroup(WFS_LINK_MARKER) - .href("http://example.com?wfs").build(); - StacCollectionModel model = StacCollectionModel.builder().links(List.of(wfsLink)).build(); - var layers = List.of( - LayerInfo.builder().title("test_layer").name("").build(), - LayerInfo.builder().title("other").build() - ); - - ElasticSearchBase.SearchResult result = new ElasticSearchBase.SearchResult<>(); - result.setCollections(List.of(model)); - when(mockSearch.searchCollections(anyString())).thenReturn(result); - - WfsServer server = new WfsServer(mockSearch, downloadableFieldsService, restTemplate, new RestTemplateUtils(restTemplate), entity); - - List info = server.filterLayersByWfsLinks("id", layers); - assertEquals(1, info.size(), "Layer count match"); - assertEquals(layers.get(0), info.get(0), "Layer test_layer found"); - } /** - * The function will scan the layer that match if there exist layers where name ends with _aodn_map, then - * only return those, otherwise return layers found without _aodn_map sufix. This make the portal works like - * old portal where they setup layer for portal with sufix _aodn_map + * The function should find one because title name matches */ @Test - void primaryTitleMatch_filtersPreferAodnMapLayers() { + void primaryTitleMatch_filtersMatchingFeatureTypes() { LinkModel wfsLink = LinkModel.builder() - .title("test_layer") + .title("test_feature_type") .aiGroup(WFS_LINK_MARKER) .href("http://example.com?wfs").build(); StacCollectionModel model = StacCollectionModel.builder().links(List.of(wfsLink)).build(); - var layers = List.of( - LayerInfo.builder().title("test_layer").name("").build(), - LayerInfo.builder().title("layer:test_layer_aodn_map").name("").build() + var featureTypes = List.of( + FeatureTypeInfo.builder().title("test_feature_type").name("").build(), + FeatureTypeInfo.builder().title("other").build() ); ElasticSearchBase.SearchResult result = new ElasticSearchBase.SearchResult<>(); @@ -133,8 +107,8 @@ void primaryTitleMatch_filtersPreferAodnMapLayers() { WfsServer server = new WfsServer(mockSearch, downloadableFieldsService, restTemplate, new RestTemplateUtils(restTemplate), entity); - List info = server.filterLayersByWfsLinks("id", layers); - assertEquals(1, info.size(), "Layer count match"); - assertEquals(layers.get(0), info.get(0), "Layer layer:test_layer_aodn_map found"); + List info = server.filterFeatureTypesByWfsLinks("id", featureTypes); + assertEquals(1, info.size(), "FeatureType count match"); + assertEquals(featureTypes.get(0), info.get(0), "FeatureType test_feature_type found"); } } From 7f498cc69532a7cc437de58bd896a1e3d68824c7 Mon Sep 17 00:00:00 2001 From: Lyn Long Date: Tue, 3 Feb 2026 16:22:25 +1100 Subject: [PATCH 3/4] update function description --- .../ogcapi/server/core/service/wms/WmsServer.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java index e0bdfc7a..f4857cfc 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wms/WmsServer.java @@ -655,11 +655,11 @@ public List fetchCapabilitiesLayersByUrl(String wmsServerUrl) { * Filter WMS layers based on matching with WFS links * Matching logic: * 1. Primary: link.title matches layer.name OR layer.title (fuzzy match) - * 2. Fallback: extract typename from link URI, then typename matches layer.name OR layer.title (fuzzy match) + * 2. Fallback: extract layername from link URI, then layername matches layer.name OR layer.title (fuzzy match) * * @param collectionId - The uuid * @param layers - List of layers to filter - * @return Filtered list of WMS layers that have matching WFS links + * @return Filtered list of WMS layers that have matching metadata WMS links */ public List filterLayersByWmsLinks(String collectionId, List layers) { ElasticSearchBase.SearchResult result = search.searchCollections(collectionId); @@ -724,12 +724,8 @@ public List filterLayersByWmsLinks(String collectionId, List - * First fetches all layers (cached by URL), then filters by WFS links (cached by UUID) - *

+ * Get filtered layers from WMS GetCapabilities for a specific collection + * First fetches all layers (cached by URL), then filters by WMS links * Sometimes the URL provided by WMS link is not optimal, for example * ... * will result in timeout due to too big query, if layername inside request have format xxx:yyyy then From fa6fff0051c2778159afc23020b57e794eb9edcb Mon Sep 17 00:00:00 2001 From: Lyn Long Date: Tue, 3 Feb 2026 16:26:58 +1100 Subject: [PATCH 4/4] update typo --- .../aodn/ogcapi/server/core/configuration/CacheConfig.java | 4 ++-- .../au/org/aodn/ogcapi/server/core/service/CacheWarm.java | 6 +++--- .../org/aodn/ogcapi/server/core/service/wfs/WfsServer.java | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java index 5878ff31..c2342289 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java @@ -30,7 +30,7 @@ public class CacheConfig { public static final String CACHE_WMS_MAP_TILE = "cache-wms-map_tile"; public static final String GET_CAPABILITIES_WMS_LAYERS = "get-capabilities-wms-layers"; - public static final String GET_CAPABILITIES_WFS_LAYERS = "get-capabilities-wfs-layers"; + public static final String GET_CAPABILITIES_WFS_FEATURE_TYPES = "get-capabilities-wfs-feature-types"; public static final String DOWNLOADABLE_FIELDS = "downloadable-fields"; public static final String ALL_NO_LAND_GEOMETRY = "all-noland-geometry"; public static final String ALL_PARAM_VOCABS = "parameter-vocabs"; @@ -105,7 +105,7 @@ public JCacheCacheManager cacheManager() throws IOException { ResourcePoolsBuilder.heap(20) ).withExpiry(ExpiryPolicyBuilder.timeToLiveExpiration(Duration.ofHours(24))) ) - .withCache(GET_CAPABILITIES_WFS_LAYERS, + .withCache(GET_CAPABILITIES_WFS_FEATURE_TYPES, CacheConfigurationBuilder.newCacheConfigurationBuilder( Object.class, Object.class, ResourcePoolsBuilder.heap(20) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheWarm.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheWarm.java index 76101d29..3948d892 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheWarm.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/CacheWarm.java @@ -15,7 +15,7 @@ import java.util.List; import java.util.Map; -import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.GET_CAPABILITIES_WFS_LAYERS; +import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.GET_CAPABILITIES_WFS_FEATURE_TYPES; import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.GET_CAPABILITIES_WMS_LAYERS; /** @@ -123,10 +123,10 @@ protected void warmWmsGetCapabilities(String url) { // ==================== WFS Cache Methods ==================== - @CacheEvict(value = GET_CAPABILITIES_WFS_LAYERS, key = "#key") + @CacheEvict(value = GET_CAPABILITIES_WFS_FEATURE_TYPES, key = "#key") public void evictWfsGetCapabilities(String key) { // @CacheEvict handles the eviction - log.info("Evicting WFS cache {} for key {}", GET_CAPABILITIES_WFS_LAYERS, key); + log.info("Evicting WFS cache {} for key {}", GET_CAPABILITIES_WFS_FEATURE_TYPES, key); } @Retryable(maxAttempts = 5, backoff = @Backoff(delay = 1000)) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java index d08435d6..c1ac75ae 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java @@ -34,7 +34,7 @@ import java.util.Optional; import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.DOWNLOADABLE_FIELDS; -import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.GET_CAPABILITIES_WFS_LAYERS; +import static au.org.aodn.ogcapi.server.core.configuration.CacheConfig.GET_CAPABILITIES_WFS_FEATURE_TYPES; import static au.org.aodn.ogcapi.server.core.service.wfs.WfsDefaultParam.WFS_LINK_MARKER; import static au.org.aodn.ogcapi.server.core.util.GeoserverUtils.extractLayernameOrTypenameFromUrl; import static au.org.aodn.ogcapi.server.core.util.GeoserverUtils.roughlyMatch; @@ -200,7 +200,7 @@ public Optional getFeatureServerUrlByTitleOrQueryParam(String collection * @param wfsServerUrl - The WFS server base URL * @return - List of all FeatureTypeInfo objects from GetCapabilities (unfiltered) */ - @Cacheable(value = GET_CAPABILITIES_WFS_LAYERS) + @Cacheable(value = GET_CAPABILITIES_WFS_FEATURE_TYPES) public List fetchCapabilitiesFeatureTypesByUrl(String wfsServerUrl) { try { // Parse the base URL to construct GetCapabilities request