From 220fcaf72416bf2a9f8b24d34e543e6695fdb04a Mon Sep 17 00:00:00 2001 From: Huey Date: Fri, 8 Sep 2023 17:18:34 -0700 Subject: [PATCH 01/22] DMND-1721 Successfully creating RSD & Collections and adding skills to collections locally; JUnits still failing locally though --- api/pom.xml | 1 - pom.xml | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index ec22da6a0..ac95c4050 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -31,7 +31,6 @@ WGU Open Skills Management Toolset - 11 1.7.21 official 1.5.1 diff --git a/pom.xml b/pom.xml index c44351960..780b4973c 100644 --- a/pom.xml +++ b/pom.xml @@ -5,13 +5,13 @@ org.springframework.boot spring-boot-starter-parent - 2.7.7 + 3.1.2 edu.wgu.osmt osmt-parent - 2.6.0-SNAPSHOT + 2.6.1-SNAPSHOT pom OSMT @@ -25,6 +25,32 @@ + + UTF-8 + UTF-8 + 17 + 8.0.31 + + + + + mysql + mysql-connector-java + ${mysql.version} + + + + + + + mysql + mysql-connector-java + + + com.mysql + mysql-connector-j + + Open Source at WGU @@ -34,12 +60,6 @@ - - UTF-8 - UTF-8 - 11 - - scm:git:ssh://git@github.com/wgu-opensource/osmt.git scm:git:ssh://git@github.com/wgu-opensource/osmt.git From 3a6946f835fb41d9631ca6bc461c23ecb29ae38f Mon Sep 17 00:00:00 2001 From: Huey Date: Wed, 13 Sep 2023 08:50:21 -0700 Subject: [PATCH 02/22] Begin converting deprecated/refactored imports --- api/pom.xml | 28 +++++++++++++++---- .../edu/wgu/osmt/api/ApiErrorHandler.kt | 8 ++++-- .../edu/wgu/osmt/collection/CollectionDoc.kt | 2 +- .../wgu/osmt/collection/CollectionEsRepo.kt | 3 +- .../UpdateCollectionSkillsTaskProcessor.kt | 4 +-- .../ElasticsearchClientManager.kt | 2 +- .../elasticsearch/FindsAllByPublishStatus.kt | 2 +- .../kotlin/edu/wgu/osmt/jobcode/JobCode.kt | 2 +- .../edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 4 +-- .../kotlin/edu/wgu/osmt/keyword/Keyword.kt | 2 +- .../edu/wgu/osmt/keyword/KeywordEsRepo.kt | 4 +-- .../richskill/CreateSkillsTaskProcessor.kt | 4 +-- .../osmt/richskill/PublishTaskProcessor.kt | 4 +-- .../edu/wgu/osmt/richskill/RichSkillDoc.kt | 2 +- .../edu/wgu/osmt/richskill/RichSkillDocV2.kt | 2 +- .../edu/wgu/osmt/richskill/RichSkillEsRepo.kt | 2 +- .../edu/wgu/osmt/security/SecurityConfig.kt | 7 ++++- test/pom.xml | 4 +-- ui/pom.xml | 4 +-- 19 files changed, 58 insertions(+), 32 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index ac95c4050..39cd89735 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -4,7 +4,7 @@ edu.wgu.osmt osmt-parent - 2.6.0-SNAPSHOT + 2.6.1-SNAPSHOT @@ -27,10 +27,11 @@ edu.wgu.osmt osmt-api - 2.6.0-SNAPSHOT + ${app.version} WGU Open Skills Management Toolset + 2.6.1-SNAPSHOT 1.7.21 official 1.5.1 @@ -41,8 +42,9 @@ 4.1.3 edu.wgu.osmt.ApplicationKt - 1.17.6 - 4.4.1 + 1.18.3 + 5.1.3 + 8.9.2 3.9.2 2.17.1 4.10.0 @@ -53,7 +55,7 @@ edu.wgu.osmt osmt-ui - 2.6.0-SNAPSHOT + ${app.version} org.springframework.boot @@ -120,6 +122,22 @@ + + + + + + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 7.17.13 + + org.apache.logging.log4j log4j-api diff --git a/api/src/main/kotlin/edu/wgu/osmt/api/ApiErrorHandler.kt b/api/src/main/kotlin/edu/wgu/osmt/api/ApiErrorHandler.kt index 39d04bc2b..43a809a89 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/api/ApiErrorHandler.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/api/ApiErrorHandler.kt @@ -33,7 +33,9 @@ class GeneralApiExceptionHandler : ResponseEntityExceptionHandler() { @ControllerAdvice class ApiErrorHandler : ResponseEntityExceptionHandler() { - override fun handleHttpMessageNotReadable( +// No longer abstract method in spring-webmvc:6.0.11 +// override fun handleHttpMessageNotReadable( + fun handleHttpMessageNotReadable( ex: HttpMessageNotReadableException, headers: HttpHeaders, status: HttpStatus, @@ -57,8 +59,8 @@ class ApiErrorHandler : ResponseEntityExceptionHandler() { @ExceptionHandler(ResponseStatusException::class) fun handleResponseStatus(ex: ResponseStatusException): ResponseEntity { - val apiError = ApiError(ex.status.toString()) - return ResponseEntity(apiError, ex.status) + val apiError = ApiError(ex.message) + return ResponseEntity(apiError, ex.statusCode) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionDoc.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionDoc.kt index 8ea53781a..79ce013c6 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionDoc.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionDoc.kt @@ -5,7 +5,7 @@ import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.annotation.JsonProperty import edu.wgu.osmt.config.INDEX_COLLECTION_DOC import edu.wgu.osmt.db.PublishStatus -import org.elasticsearch.core.Nullable +import javax.annotation.Nullable import org.springframework.data.annotation.Id import org.springframework.data.elasticsearch.annotations.* import org.springframework.data.elasticsearch.annotations.FieldType.* diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index 355e8f8bd..2aaa552bd 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -15,10 +15,11 @@ import org.springframework.data.domain.Page import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort +import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/UpdateCollectionSkillsTaskProcessor.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/UpdateCollectionSkillsTaskProcessor.kt index 6a0a84d43..9292eb026 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/UpdateCollectionSkillsTaskProcessor.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/UpdateCollectionSkillsTaskProcessor.kt @@ -11,7 +11,7 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Profile import org.springframework.stereotype.Component -import javax.transaction.Transactional +import org.springframework.transaction.annotation.Transactional @Component @Profile("apiserver") @@ -63,4 +63,4 @@ class UpdateCollectionSkillsTaskProcessor { logger.info("Task ${task.uuid} completed") } -} \ No newline at end of file +} diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt index 1a3ac08b5..129a6279a 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt @@ -9,7 +9,7 @@ import org.springframework.context.annotation.Configuration import org.springframework.core.convert.converter.Converter import org.springframework.data.convert.ReadingConverter import org.springframework.data.convert.WritingConverter -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.convert.ElasticsearchCustomConversions import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories import java.time.LocalDateTime diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt index 2bd6d2891..ad20f842b 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt @@ -4,7 +4,7 @@ import edu.wgu.osmt.db.PublishStatus import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilders import org.springframework.data.domain.Pageable -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCode.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCode.kt index f00bcb7ca..f6a73bd1c 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCode.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCode.kt @@ -3,7 +3,7 @@ package edu.wgu.osmt.jobcode import com.fasterxml.jackson.annotation.JsonIgnore import edu.wgu.osmt.config.INDEX_JOBCODE_DOC import edu.wgu.osmt.db.DatabaseData -import org.elasticsearch.core.Nullable +import javax.annotation.Nullable import org.springframework.data.elasticsearch.annotations.* import java.time.LocalDateTime import java.time.ZoneOffset diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index 6bc1208bc..4b6ca837a 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -9,10 +9,10 @@ import org.elasticsearch.search.sort.SortBuilders import org.elasticsearch.search.sort.SortOrder import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/Keyword.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/Keyword.kt index 7878080c4..52646054d 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/Keyword.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/Keyword.kt @@ -6,7 +6,7 @@ import edu.wgu.osmt.db.HasUpdateDate import edu.wgu.osmt.db.NullableFieldUpdate import edu.wgu.osmt.db.TableWithUpdate import edu.wgu.osmt.db.UpdateObject -import org.elasticsearch.core.Nullable +import javax.annotation.Nullable import org.jetbrains.exposed.dao.id.LongIdTable import org.jetbrains.exposed.sql.Column import org.jetbrains.exposed.sql.`java-time`.datetime diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt index 984864a3a..01ccb06cc 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt @@ -8,10 +8,10 @@ import org.elasticsearch.search.sort.SortBuilders import org.elasticsearch.search.sort.SortOrder import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/CreateSkillsTaskProcessor.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/CreateSkillsTaskProcessor.kt index d232bf286..7f1d852b0 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/CreateSkillsTaskProcessor.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/CreateSkillsTaskProcessor.kt @@ -10,7 +10,7 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Profile import org.springframework.stereotype.Component -import javax.transaction.Transactional +import org.springframework.transaction.annotation.Transactional @Component @@ -47,4 +47,4 @@ class CreateSkillsTaskProcessor { logger.info("Task ${task.uuid} completed") } -} \ No newline at end of file +} diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/PublishTaskProcessor.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/PublishTaskProcessor.kt index 1a1292f36..5d08f50d1 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/PublishTaskProcessor.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/PublishTaskProcessor.kt @@ -11,7 +11,7 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Profile import org.springframework.stereotype.Component -import javax.transaction.Transactional +import org.springframework.transaction.annotation.Transactional @Component @@ -49,4 +49,4 @@ class PublishTaskProcessor { logger.info("Task ${publishTask.uuid} completed") } -} \ No newline at end of file +} diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt index 11ef82f4b..0010ed8ed 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt @@ -9,7 +9,7 @@ import edu.wgu.osmt.config.INDEX_RICHSKILL_DOC import edu.wgu.osmt.db.PublishStatus import edu.wgu.osmt.jobcode.JobCode import edu.wgu.osmt.keyword.KeywordTypeEnum -import org.elasticsearch.core.Nullable +import javax.annotation.Nullable import org.springframework.data.annotation.Id import org.springframework.data.elasticsearch.annotations.* import org.springframework.data.elasticsearch.annotations.FieldType.* diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDocV2.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDocV2.kt index 880240702..42d2bc4f8 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDocV2.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDocV2.kt @@ -10,7 +10,7 @@ import edu.wgu.osmt.config.SEMICOLON import edu.wgu.osmt.db.PublishStatus import edu.wgu.osmt.jobcode.JobCode import edu.wgu.osmt.keyword.KeywordTypeEnum -import org.elasticsearch.core.Nullable +import javax.annotation.Nullable import org.springframework.data.annotation.Id import org.springframework.data.elasticsearch.annotations.* import org.springframework.data.elasticsearch.annotations.FieldType.* diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index 198acf07b..f3042c9f3 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -23,7 +23,7 @@ import org.springframework.data.domain.Page import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder diff --git a/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt b/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt index bcb7124f1..1341fe37d 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt @@ -1,5 +1,7 @@ package edu.wgu.osmt.security +//import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter + import com.fasterxml.jackson.databind.ObjectMapper import edu.wgu.osmt.RoutePaths import edu.wgu.osmt.api.model.ApiError @@ -10,13 +12,15 @@ import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Profile import org.springframework.http.HttpMethod.* import org.springframework.security.config.annotation.web.builders.HttpSecurity +import org.springframework.security.config.annotation.web.builders.WebSecurity import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter +import org.springframework.security.config.annotation.web.configuration.WebSecurityCustomizer import org.springframework.security.core.Authentication import org.springframework.security.core.AuthenticationException import org.springframework.security.oauth2.core.oidc.user.OidcUser import org.springframework.security.web.AuthenticationEntryPoint import org.springframework.security.web.DefaultRedirectStrategy +import org.springframework.security.web.SecurityFilterChain import org.springframework.security.web.authentication.AuthenticationSuccessHandler import org.springframework.stereotype.Component import org.springframework.web.cors.CorsConfiguration @@ -25,6 +29,7 @@ import org.springframework.web.cors.UrlBasedCorsConfigurationSource import javax.servlet.http.HttpServletRequest import javax.servlet.http.HttpServletResponse + /** * Security configurations * - to enable another OAuth provider, include the profile name in place of `OTHER-OAUTH-PROFILE` diff --git a/test/pom.xml b/test/pom.xml index d62816fc8..3fdaa0534 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -6,12 +6,12 @@ edu.wgu.osmt osmt-parent - 2.6.0-SNAPSHOT + 2.6.1-SNAPSHOT edu.wgu.osmt osmt-api-test - 2.6.0-SNAPSHOT + 2.6.1-SNAPSHOT OSMT API Tests diff --git a/ui/pom.xml b/ui/pom.xml index 428d32285..e4cc8c079 100644 --- a/ui/pom.xml +++ b/ui/pom.xml @@ -4,12 +4,12 @@ edu.wgu.osmt osmt-parent - 2.6.0-SNAPSHOT + 2.6.1-SNAPSHOT edu.wgu.osmt osmt-ui - 2.6.0-SNAPSHOT + 2.6.1-SNAPSHOT jar From 79072db739795009a4d2dac217aa7a9dcc9dc9a6 Mon Sep 17 00:00:00 2001 From: Huey Date: Wed, 13 Sep 2023 17:31:25 -0700 Subject: [PATCH 03/22] Updated to use latest ElasticSearch code; Deprecated classes has been moved to other packages. --- api/pom.xml | 2 +- .../wgu/osmt/collection/CollectionEsRepo.kt | 3 +- .../main/kotlin/edu/wgu/osmt/db/DbConfig.kt | 2 - .../edu/wgu/osmt/elasticsearch/EsConfig.kt | 2 - .../elasticsearch/FindsAllByPublishStatus.kt | 2 +- .../edu/wgu/osmt/richskill/RichSkillEsRepo.kt | 2 +- .../edu/wgu/osmt/security/SecurityConfig.kt | 230 +++++++++--------- 7 files changed, 117 insertions(+), 126 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 39cd89735..3a6aed76e 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -27,7 +27,7 @@ edu.wgu.osmt osmt-api - ${app.version} + 2.6.1-SNAPSHOT WGU Open Skills Management Toolset diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index 2aaa552bd..c0a754a11 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -15,9 +15,8 @@ import org.springframework.data.domain.Page import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort -import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder -import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.repository.ElasticsearchRepository diff --git a/api/src/main/kotlin/edu/wgu/osmt/db/DbConfig.kt b/api/src/main/kotlin/edu/wgu/osmt/db/DbConfig.kt index 83a5fe250..d58ded2a0 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/db/DbConfig.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/db/DbConfig.kt @@ -1,9 +1,7 @@ package edu.wgu.osmt.db import org.springframework.boot.context.properties.ConfigurationProperties -import org.springframework.boot.context.properties.ConstructorBinding -@ConstructorBinding @ConfigurationProperties(prefix = "db", ignoreInvalidFields = true) data class DbConfig( val name: String, diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/EsConfig.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/EsConfig.kt index 877028a22..c57c55ae5 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/EsConfig.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/EsConfig.kt @@ -1,8 +1,6 @@ package edu.wgu.osmt.elasticsearch import org.springframework.boot.context.properties.ConfigurationProperties -import org.springframework.boot.context.properties.ConstructorBinding -@ConstructorBinding @ConfigurationProperties(prefix = "es", ignoreInvalidFields = true) data class EsConfig(val uri: String) diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt index ad20f842b..856fea95a 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt @@ -6,7 +6,7 @@ import org.elasticsearch.index.query.QueryBuilders import org.springframework.data.domain.Pageable import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.SearchHits -import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder interface FindsAllByPublishStatus { diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index f3042c9f3..6118df3b7 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -26,7 +26,7 @@ import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories import org.springframework.security.oauth2.jwt.Jwt diff --git a/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt b/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt index 1341fe37d..138d4cfbf 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt @@ -1,20 +1,18 @@ package edu.wgu.osmt.security -//import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter - import com.fasterxml.jackson.databind.ObjectMapper import edu.wgu.osmt.RoutePaths import edu.wgu.osmt.api.model.ApiError import edu.wgu.osmt.config.AppConfig +import jakarta.servlet.http.HttpServletRequest +import jakarta.servlet.http.HttpServletResponse import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.context.annotation.Profile import org.springframework.http.HttpMethod.* import org.springframework.security.config.annotation.web.builders.HttpSecurity -import org.springframework.security.config.annotation.web.builders.WebSecurity import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity -import org.springframework.security.config.annotation.web.configuration.WebSecurityCustomizer import org.springframework.security.core.Authentication import org.springframework.security.core.AuthenticationException import org.springframework.security.oauth2.core.oidc.user.OidcUser @@ -26,8 +24,6 @@ import org.springframework.stereotype.Component import org.springframework.web.cors.CorsConfiguration import org.springframework.web.cors.CorsConfigurationSource import org.springframework.web.cors.UrlBasedCorsConfigurationSource -import javax.servlet.http.HttpServletRequest -import javax.servlet.http.HttpServletResponse /** @@ -38,7 +34,7 @@ import javax.servlet.http.HttpServletResponse @Configuration @EnableWebSecurity @Profile("oauth2-okta | OTHER-OAUTH-PROFILE") -class SecurityConfig : WebSecurityConfigurerAdapter() { +class SecurityConfig { @Autowired lateinit var appConfig: AppConfig @@ -49,66 +45,65 @@ class SecurityConfig : WebSecurityConfigurerAdapter() { @Autowired lateinit var returnUnauthorized: ReturnUnauthorized - @Override - override fun configure(http: HttpSecurity) { + @Bean + fun filterChain(http: HttpSecurity) : SecurityFilterChain { http .cors().and() .csrf().disable() .httpBasic().disable() - .authorizeRequests() - - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_AUDIT_LOG}", + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_AUDIT_LOG}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_AUDIT_LOG}", "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_AUDIT_LOG}").authenticated() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_AUDIT_LOG}", + } + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_AUDIT_LOG}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_AUDIT_LOG}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_AUDIT_LOG}").authenticated() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_SKILLS}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_AUDIT_LOG}").authenticated() } + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_SKILLS}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_SKILLS}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_SKILLS}").authenticated() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_BATCH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_SKILLS}").authenticated() } + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_BATCH}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_BATCH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_BATCH}").authenticated() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_JOBCODES_PATH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_BATCH}").authenticated() } + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_JOBCODES_PATH}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_JOBCODES_PATH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_JOBCODES_PATH}").authenticated() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_KEYWORDS_PATH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_JOBCODES_PATH}").authenticated() } + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_KEYWORDS_PATH}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_KEYWORDS_PATH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_KEYWORDS_PATH}").authenticated() + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_KEYWORDS_PATH}").authenticated() } // public search endpoints - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_SKILLS}", + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_SKILLS}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_SKILLS}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_SKILLS}").permitAll() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_COLLECTIONS}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_SKILLS}").permitAll() } + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_COLLECTIONS}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_COLLECTIONS}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_COLLECTIONS}").permitAll() + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_COLLECTIONS}").permitAll() } // public canonical URL endpoints - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_DETAIL}", + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_DETAIL}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_DETAIL}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_DETAIL}").permitAll() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_DETAIL}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_DETAIL}").permitAll() } + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_DETAIL}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_DETAIL}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_DETAIL}").permitAll() + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_DETAIL}").permitAll() } - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS}", + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS}").permitAll() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CSV}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS}").permitAll() } + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CSV}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CSV}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CSV}").permitAll() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_TEXT}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CSV}").permitAll() } + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_TEXT}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_TEXT}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_TEXT}").permitAll() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_XLSX}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_TEXT}").permitAll() } + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_XLSX}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_XLSX}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_XLSX}").permitAll() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_MEDIA}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_XLSX}").permitAll() } + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_MEDIA}", "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_MEDIA}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_MEDIA}").permitAll() + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_MEDIA}").permitAll() } - .and().exceptionHandling().authenticationEntryPoint(returnUnauthorized) + .exceptionHandling().authenticationEntryPoint(returnUnauthorized) .and().oauth2Login().successHandler(redirectToFrontend) .and().oauth2ResourceServer().jwt() @@ -117,6 +112,7 @@ class SecurityConfig : WebSecurityConfigurerAdapter() { } else { configureForNoRoles(http) } + return http.build(); } fun configureForRoles(http: HttpSecurity) { @@ -126,93 +122,93 @@ class SecurityConfig : WebSecurityConfigurerAdapter() { val READ = appConfig.scopeRead if (appConfig.allowPublicLists) { - http.authorizeRequests() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").permitAll() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").permitAll() + http + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").permitAll() } + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").permitAll() } } else { - http.authorizeRequests() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) - } + http + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) } + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) } + } - http.authorizeRequests() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_UPDATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_UPDATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_CREATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_CREATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_CREATE}").hasAnyAuthority(ADMIN, CURATOR) - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_PUBLISH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_PUBLISH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_PUBLISH}").hasAnyAuthority(ADMIN) + http + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_UPDATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_UPDATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) } + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_CREATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_CREATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_CREATE}").hasAnyAuthority(ADMIN, CURATOR) } + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_PUBLISH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_PUBLISH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_PUBLISH}").hasAnyAuthority(ADMIN) } - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CREATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CREATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CREATE}").hasAnyAuthority(ADMIN, CURATOR) - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_PUBLISH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_PUBLISH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_PUBLISH}").hasAnyAuthority(ADMIN) - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_UPDATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_UPDATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS_UPDATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS_UPDATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) - .mvcMatchers(DELETE, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_REMOVE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_REMOVE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_REMOVE}").hasAnyAuthority(ADMIN) - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.WORKSPACE_PATH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.WORKSPACE_PATH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.WORKSPACE_PATH}").hasAnyAuthority(ADMIN, CURATOR) + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CREATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CREATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CREATE}").hasAnyAuthority(ADMIN, CURATOR) } + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_PUBLISH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_PUBLISH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_PUBLISH}").hasAnyAuthority(ADMIN) } + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_UPDATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_UPDATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) } + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS_UPDATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS_UPDATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) } + .authorizeHttpRequests { auth -> auth.requestMatchers(DELETE, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_REMOVE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_REMOVE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_REMOVE}").hasAnyAuthority(ADMIN) } + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.WORKSPACE_PATH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.WORKSPACE_PATH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.WORKSPACE_PATH}").hasAnyAuthority(ADMIN, CURATOR) } - .mvcMatchers("/api/**").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) + .authorizeHttpRequests { auth -> auth.requestMatchers("/api/**").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) } } fun configureForNoRoles(http: HttpSecurity) { - http.authorizeRequests() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").permitAll() - .mvcMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").permitAll() + http + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").permitAll() } + .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").permitAll() } - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_UPDATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_UPDATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_UPDATE}").authenticated() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_CREATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_CREATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_CREATE}").authenticated() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_PUBLISH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_PUBLISH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_PUBLISH}").authenticated() + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_UPDATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_UPDATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_UPDATE}").authenticated() } + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_CREATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_CREATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_CREATE}").authenticated() } + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_PUBLISH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_PUBLISH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_PUBLISH}").authenticated() } - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CREATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CREATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CREATE}").authenticated() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_PUBLISH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_PUBLISH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_PUBLISH}").authenticated() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_UPDATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_UPDATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_UPDATE}").authenticated() - .mvcMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS_UPDATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS_UPDATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS_UPDATE}").authenticated() - .mvcMatchers(DELETE, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_REMOVE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_REMOVE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_REMOVE}").denyAll() + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CREATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CREATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CREATE}").authenticated() } + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_PUBLISH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_PUBLISH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_PUBLISH}").authenticated() } + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_UPDATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_UPDATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_UPDATE}").authenticated() } + .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS_UPDATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS_UPDATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS_UPDATE}").authenticated() } + .authorizeHttpRequests { auth -> auth.requestMatchers(DELETE, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_REMOVE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_REMOVE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_REMOVE}").denyAll() } // fall-through - .mvcMatchers("/api/**").permitAll() + .authorizeHttpRequests { auth -> auth.requestMatchers("/api/**").permitAll() } } @Bean From a9ad0bd5e3fbb7414ce2fef0f18705840bd9d38d Mon Sep 17 00:00:00 2001 From: Huey Date: Tue, 19 Sep 2023 15:56:00 -0700 Subject: [PATCH 04/22] Broken code --- api/pom.xml | 6 -- .../ElasticsearchClientManager.kt | 2 +- .../edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 70 ++++++++++++++++--- 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 3a6aed76e..08281bca5 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -132,12 +132,6 @@ elasticsearch ${elasticsearch.version} - - org.elasticsearch.client - elasticsearch-rest-high-level-client - 7.17.13 - - org.apache.logging.log4j log4j-api diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt index 129a6279a..39b131402 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt @@ -9,7 +9,7 @@ import org.springframework.context.annotation.Configuration import org.springframework.core.convert.converter.Converter import org.springframework.data.convert.ReadingConverter import org.springframework.data.convert.WritingConverter -import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate +//import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.convert.ElasticsearchCustomConversions import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories import java.time.LocalDateTime diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index 4b6ca837a..5c84c2415 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -1,5 +1,11 @@ package edu.wgu.osmt.jobcode +import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery +import co.elastic.clients.elasticsearch._types.query_dsl.MatchQuery +import co.elastic.clients.elasticsearch._types.query_dsl.Query +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.match +import edu.wgu.osmt.api.model.ApiJobCode import edu.wgu.osmt.config.INDEX_JOBCODE_DOC import edu.wgu.osmt.elasticsearch.OffsetPageable import org.elasticsearch.index.query.BoolQueryBuilder @@ -9,15 +15,18 @@ import org.elasticsearch.search.sort.SortBuilders import org.elasticsearch.search.sort.SortOrder import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration -import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.elc.NativeQuery +import org.springframework.data.elasticsearch.core.SearchHit import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories +import java.util.function.Function + interface CustomJobCodeRepository { - val elasticSearchTemplate: ElasticsearchRestTemplate + val elasticSearchTemplate: ElasticsearchTemplate fun typeAheadSearch(query: String): SearchHits fun deleteIndex() { @@ -25,9 +34,8 @@ interface CustomJobCodeRepository { } } -class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchRestTemplate) : +class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchTemplate) : CustomJobCodeRepository { - override fun typeAheadSearch(query: String): SearchHits { val nsq: NativeSearchQueryBuilder @@ -40,9 +48,55 @@ class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSea val disjunctionQuery = JobCodeQueries.multiPropertySearch(query) nsq = NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery(disjunctionQuery) - .withSort(SortBuilders.fieldSort("${JobCode::code.name}.keyword").order(SortOrder.ASC)) + .withSort(SortBuilders.fieldSort("${JobCode::code.name}.keyword").order(SortOrder.ASC)) return elasticSearchTemplate.search(nsq.build(), JobCode::class.java) } + + fun typeAheadSearchEx(query: String): List { + val limitedPageable = createOffsetPageable(query) + val disjunctionQuery = JobCodeQueries.multiPropertySearch(query) + + val criteria = QueryBuilders.bool { builder: BoolQuery.Builder -> + builder.must( +// match(Function { queryAuthor: MatchQuery.Builder -> +// queryAuthor +// .field( "authorName" ) +// .query(author) +// }), + match { queryCode: MatchQuery.Builder -> + queryCode.field("code").query(query) + } + ) + } + + return elasticSearchTemplate + .search( NativeQuery + .builder() + .withQuery(criteria).build(), JobCode::class.java ) + .map { it.content } + .toList() + } + + + + private fun createOffsetPageable(query: String): OffsetPageable { + val limit = if (query.isEmpty()) 10000 else 20 + return OffsetPageable(0, limit, null) + } + + //TODO Clean up this ugly method + fun createQuery(bqb: BoolQueryBuilder): Query { + bqb.toQuery() + + return QueryBuilders.bool { builder: BoolQuery.Builder -> + builder.must( + match { + q: MatchQuery.Builder -> q.field(JobCode::code.name).query(fieldValue) + } + ) + } + } + } object JobCodeQueries { @@ -50,7 +104,6 @@ object JobCodeQueries { val disjunctionQuery = disMaxQuery() val path = parentDocPath?.let { "${it}." } ?: "" val isComplex = query.contains("\"") - val queries = listOf( prefixQuery( @@ -74,9 +127,7 @@ object JobCodeQueries { query ) ) - disjunctionQuery.innerQueries().addAll(queries) - if (isComplex) { disjunctionQuery.innerQueries().addAll( listOf( @@ -103,7 +154,6 @@ object JobCodeQueries { ) ) } - return boolQuery().must(existsQuery("${path}${JobCode::name.name}")).must(disjunctionQuery) } } From fabd542928f46cc85dc0c3668d5857b72f7a41cc Mon Sep 17 00:00:00 2001 From: Jesus Bautista Date: Wed, 20 Sep 2023 09:50:29 -0600 Subject: [PATCH 05/22] fix elasticsearch-rest-high-level-client --- api/pom.xml | 11 +++++------ .../edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 18 ++++++++++++++---- .../config/application-dev.properties | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 3a6aed76e..072448809 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -122,6 +122,11 @@ + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 7.17.8 + @@ -132,12 +137,6 @@ elasticsearch ${elasticsearch.version} - - org.elasticsearch.client - elasticsearch-rest-high-level-client - 7.17.13 - - org.apache.logging.log4j log4j-api diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index 4b6ca837a..e842e9eb4 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -1,23 +1,33 @@ package edu.wgu.osmt.jobcode +import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery +import co.elastic.clients.elasticsearch._types.query_dsl.MatchQuery +import co.elastic.clients.elasticsearch._types.query_dsl.Query +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.match import edu.wgu.osmt.config.INDEX_JOBCODE_DOC import edu.wgu.osmt.elasticsearch.OffsetPageable import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.Operator import org.elasticsearch.index.query.QueryBuilders.* +import org.elasticsearch.search.builder.SearchSourceBuilder import org.elasticsearch.search.sort.SortBuilders import org.elasticsearch.search.sort.SortOrder import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration -import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.elc.NativeQuery +import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder +import org.springframework.data.elasticsearch.core.SearchHit import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories + interface CustomJobCodeRepository { - val elasticSearchTemplate: ElasticsearchRestTemplate + val elasticSearchTemplate: ElasticsearchTemplate fun typeAheadSearch(query: String): SearchHits fun deleteIndex() { @@ -25,7 +35,7 @@ interface CustomJobCodeRepository { } } -class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchRestTemplate) : +class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchTemplate) : CustomJobCodeRepository { override fun typeAheadSearch(query: String): SearchHits { diff --git a/api/src/main/resources/config/application-dev.properties b/api/src/main/resources/config/application-dev.properties index 95d30312c..baea913b6 100644 --- a/api/src/main/resources/config/application-dev.properties +++ b/api/src/main/resources/config/application-dev.properties @@ -11,7 +11,7 @@ app.security.cors.allowedOrigins=${app.baseUrl},${app.frontendUrl} management.endpoint.health.show-details=always -spring.flyway.enabled=true +spring.flyway.enabled=false # Common debuging log levels for OSMT development #logging.level.org.springframework.data.elasticsearch.client.WIRE=trace From 09e4c9a2ab49f4c25b1f787acb8e7dd384c56773 Mon Sep 17 00:00:00 2001 From: Jesus Bautista Date: Wed, 20 Sep 2023 15:34:39 -0600 Subject: [PATCH 06/22] fix ElasticsearchTemplate --- api/pom.xml | 2 +- .../edu/wgu/osmt/auditlog/AuditLogDao.kt | 2 +- .../wgu/osmt/collection/CollectionEsRepo.kt | 4 +- .../kotlin/edu/wgu/osmt/db/ExposedHelper.kt | 42 +++---- .../ElasticsearchClientManager.kt | 21 +++- .../elasticsearch/FindsAllByPublishStatus.kt | 4 +- .../edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 9 -- .../edu/wgu/osmt/keyword/KeywordEsRepo.kt | 6 +- .../edu/wgu/osmt/richskill/RichSkillEsRepo.kt | 4 +- .../edu/wgu/osmt/security/SecurityConfig.kt | 116 +++++++++--------- .../edu/wgu/osmt/task/TaskMessageService.kt | 4 +- 11 files changed, 107 insertions(+), 107 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 072448809..fe19eb0be 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -78,7 +78,7 @@ com.github.sonus21 rqueue-spring-boot-starter - 2.13.0-RELEASE + 3.1.0-RELEASE com.fasterxml.jackson.core diff --git a/api/src/main/kotlin/edu/wgu/osmt/auditlog/AuditLogDao.kt b/api/src/main/kotlin/edu/wgu/osmt/auditlog/AuditLogDao.kt index 56d323b49..de4d87432 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/auditlog/AuditLogDao.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/auditlog/AuditLogDao.kt @@ -1,7 +1,7 @@ package edu.wgu.osmt.auditlog -import com.google.common.reflect.TypeToken import com.google.gson.Gson +import com.google.gson.reflect.TypeToken import edu.wgu.osmt.db.OutputsModel import org.jetbrains.exposed.dao.LongEntity import org.jetbrains.exposed.dao.LongEntityClass diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index c0a754a11..85c1b863f 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -15,8 +15,8 @@ import org.springframework.data.domain.Page import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort +import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder -import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.repository.ElasticsearchRepository @@ -43,7 +43,7 @@ interface CustomCollectionQueries : FindsAllByPublishStatus { } class CustomCollectionQueriesImpl @Autowired constructor( - override val elasticSearchTemplate: ElasticsearchRestTemplate, + override val elasticSearchTemplate: ElasticsearchTemplate, override val richSkillEsRepo: RichSkillEsRepo ) : CustomCollectionQueries { diff --git a/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt b/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt index a1e43d240..723020098 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt @@ -44,27 +44,27 @@ fun SchemaUtils.addMissingColumnsStatementsPublic(vararg tables: Table): List { - val elasticSearchTemplate: ElasticsearchRestTemplate + val elasticSearchTemplate: ElasticsearchTemplate val javaClass: Class fun findAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): SearchHits { diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index e842e9eb4..572335728 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -1,25 +1,16 @@ package edu.wgu.osmt.jobcode -import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery -import co.elastic.clients.elasticsearch._types.query_dsl.MatchQuery -import co.elastic.clients.elasticsearch._types.query_dsl.Query -import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders -import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.match import edu.wgu.osmt.config.INDEX_JOBCODE_DOC import edu.wgu.osmt.elasticsearch.OffsetPageable import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.Operator import org.elasticsearch.index.query.QueryBuilders.* -import org.elasticsearch.search.builder.SearchSourceBuilder import org.elasticsearch.search.sort.SortBuilders import org.elasticsearch.search.sort.SortOrder import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate -import org.springframework.data.elasticsearch.client.elc.NativeQuery -import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder -import org.springframework.data.elasticsearch.core.SearchHit import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.repository.ElasticsearchRepository diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt index 01ccb06cc..35d5872c8 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt @@ -8,7 +8,7 @@ import org.elasticsearch.search.sort.SortBuilders import org.elasticsearch.search.sort.SortOrder import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration -import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder @@ -16,7 +16,7 @@ import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories interface CustomKeywordRepository { - val elasticSearchTemplate: ElasticsearchRestTemplate + val elasticSearchTemplate: ElasticsearchTemplate fun typeAheadSearch(query: String, type: KeywordTypeEnum): SearchHits fun deleteIndex() { @@ -24,7 +24,7 @@ interface CustomKeywordRepository { } } -class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchRestTemplate) : +class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchTemplate) : CustomKeywordRepository { override fun typeAheadSearch(query: String, type: KeywordTypeEnum): SearchHits { val limitedPageable: OffsetPageable diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index 6118df3b7..d3e30cdb5 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -23,7 +23,7 @@ import org.springframework.data.domain.Page import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort -import org.springframework.data.elasticsearch.client.erhlc.ElasticsearchRestTemplate +import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder @@ -67,7 +67,7 @@ interface CustomRichSkillQueries : FindsAllByPublishStatus { } } -class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchRestTemplate) : +class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchTemplate) : CustomRichSkillQueries { override val javaClass = RichSkillDoc::class.java diff --git a/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt b/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt index 138d4cfbf..296aa94b1 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt @@ -51,67 +51,67 @@ class SecurityConfig { .cors().and() .csrf().disable() .httpBasic().disable() - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_AUDIT_LOG}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_AUDIT_LOG}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_AUDIT_LOG}").authenticated() - } - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_AUDIT_LOG}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_AUDIT_LOG}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_AUDIT_LOG}").authenticated() } - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_SKILLS}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_SKILLS}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_SKILLS}").authenticated() } - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_BATCH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_BATCH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_BATCH}").authenticated() } - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_JOBCODES_PATH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_JOBCODES_PATH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_JOBCODES_PATH}").authenticated() } - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_KEYWORDS_PATH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_KEYWORDS_PATH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_KEYWORDS_PATH}").authenticated() } - - // public search endpoints - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_SKILLS}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_SKILLS}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_SKILLS}").permitAll() } - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_COLLECTIONS}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_COLLECTIONS}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_COLLECTIONS}").permitAll() } - - // public canonical URL endpoints - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_DETAIL}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_DETAIL}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_DETAIL}").permitAll() } - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_DETAIL}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_DETAIL}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_DETAIL}").permitAll() } - - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS}").permitAll() } - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CSV}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CSV}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CSV}").permitAll() } - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_TEXT}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_TEXT}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_TEXT}").permitAll() } - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_XLSX}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_XLSX}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_XLSX}").permitAll() } - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_MEDIA}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_MEDIA}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_MEDIA}").permitAll() } - +// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_AUDIT_LOG}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_AUDIT_LOG}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_AUDIT_LOG}").authenticated() +// } +// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_AUDIT_LOG}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_AUDIT_LOG}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_AUDIT_LOG}").authenticated() } +// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_SKILLS}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_SKILLS}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_SKILLS}").authenticated() } +// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_BATCH}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_BATCH}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_BATCH}").authenticated() } +// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_JOBCODES_PATH}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_JOBCODES_PATH}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_JOBCODES_PATH}").authenticated() } +// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_KEYWORDS_PATH}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_KEYWORDS_PATH}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_KEYWORDS_PATH}").authenticated() } +// +// // public search endpoints +// .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_SKILLS}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_SKILLS}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_SKILLS}").permitAll() } +// .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_COLLECTIONS}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_COLLECTIONS}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_COLLECTIONS}").permitAll() } +// +// // public canonical URL endpoints +// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_DETAIL}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_DETAIL}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_DETAIL}").permitAll() } +// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_DETAIL}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_DETAIL}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_DETAIL}").permitAll() } +// +// .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS}").permitAll() } +// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CSV}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CSV}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CSV}").permitAll() } +// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_TEXT}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_TEXT}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_TEXT}").permitAll() } +// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_XLSX}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_XLSX}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_XLSX}").permitAll() } +// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_MEDIA}", +// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_MEDIA}", +// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_MEDIA}").permitAll() } +// .exceptionHandling().authenticationEntryPoint(returnUnauthorized) .and().oauth2Login().successHandler(redirectToFrontend) .and().oauth2ResourceServer().jwt() - - if (appConfig.enableRoles) { - configureForRoles(http) - } else { - configureForNoRoles(http) - } +// +// if (appConfig.enableRoles) { +// configureForRoles(http) +// } else { +// configureForNoRoles(http) +// } return http.build(); } diff --git a/api/src/main/kotlin/edu/wgu/osmt/task/TaskMessageService.kt b/api/src/main/kotlin/edu/wgu/osmt/task/TaskMessageService.kt index 4f41a10df..40ecfa7ee 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/task/TaskMessageService.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/task/TaskMessageService.kt @@ -1,6 +1,6 @@ package edu.wgu.osmt.task -import com.github.sonus21.rqueue.core.RqueueMessageSender +import com.github.sonus21.rqueue.core.RqueueMessageEnqueuer import org.springframework.beans.factory.annotation.Autowired import org.springframework.data.redis.core.RedisTemplate import org.springframework.stereotype.Service @@ -9,7 +9,7 @@ import org.springframework.stereotype.Service class TaskMessageService { @Autowired - lateinit var rqueueMessageSender: RqueueMessageSender + lateinit var rqueueMessageSender: RqueueMessageEnqueuer @Autowired lateinit var redisTaskTemplate: RedisTemplate From 94e5a334915db2cd133c26d17d1ac10a2a3d208d Mon Sep 17 00:00:00 2001 From: Jesus Bautista Date: Fri, 22 Sep 2023 17:13:59 -0600 Subject: [PATCH 07/22] fix elasticsearch queries --- api/pom.xml | 11 ---- .../wgu/osmt/collection/CollectionEsRepo.kt | 57 ++++++++++++------- .../elasticsearch/FindsAllByPublishStatus.kt | 13 +++-- .../edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 8 ++- .../edu/wgu/osmt/keyword/KeywordEsRepo.kt | 8 ++- .../edu/wgu/osmt/richskill/RichSkillEsRepo.kt | 15 +++-- 6 files changed, 68 insertions(+), 44 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index fe19eb0be..030cde13c 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -44,7 +44,6 @@ edu.wgu.osmt.ApplicationKt 1.18.3 5.1.3 - 8.9.2 3.9.2 2.17.1 4.10.0 @@ -127,16 +126,6 @@ elasticsearch-rest-high-level-client 7.17.8 - - - - - - - org.elasticsearch - elasticsearch - ${elasticsearch.version} - org.apache.logging.log4j log4j-api diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index 85c1b863f..daf379391 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -16,9 +16,12 @@ import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates +import org.springframework.data.elasticsearch.core.query.Query +import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -111,31 +114,38 @@ class CustomCollectionQueriesImpl @Autowired constructor( ) bq.should(richSkillEsRepo.occupationQueries(apiSearch.query)) + val nsqb: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( + collectionPropertiesMultiMatch(apiSearch.query) + ).withPageable(Pageable.unpaged()).withFilter(filter).build() + val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) + // search on collection specific properties - collectionMultiPropertyResults = elasticSearchTemplate.search( - NativeSearchQueryBuilder().withQuery( - collectionPropertiesMultiMatch(apiSearch.query) - ).withPageable(Pageable.unpaged()).withFilter(filter).build(), CollectionDoc::class.java - ).searchHits.map { it.content.uuid } + collectionMultiPropertyResults = elasticSearchTemplate.search(searchQuery, CollectionDoc::class.java).searchHits.map { it.content.uuid } } else if (apiSearch.advanced != null) { richSkillEsRepo.generateBoolQueriesFromApiSearch(bq, apiSearch.advanced) if (!apiSearch.advanced.collectionName.isNullOrBlank()) { if (apiSearch.advanced.collectionName.contains("\"")) { + val nsqb: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( + QueryBuilders.simpleQueryStringQuery(apiSearch.advanced.collectionName).field("${CollectionDoc::name.name}.raw").defaultOperator(Operator.AND) + ).withPageable(Pageable.unpaged()).withFilter(filter).build() + val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) + collectionMultiPropertyResults = elasticSearchTemplate.search( - NativeSearchQueryBuilder().withQuery( - QueryBuilders.simpleQueryStringQuery(apiSearch.advanced.collectionName).field("${CollectionDoc::name.name}.raw").defaultOperator(Operator.AND) - ).withPageable(Pageable.unpaged()).withFilter(filter).build(), CollectionDoc::class.java + searchQuery, CollectionDoc::class.java ).searchHits.map { it.content.uuid } } else { + val nsqb: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( + QueryBuilders.matchPhrasePrefixQuery( + CollectionDoc::name.name, + apiSearch.advanced.collectionName + ) + ).withPageable(Pageable.unpaged()).withFilter(filter).build() + val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) + collectionMultiPropertyResults = elasticSearchTemplate.search( - NativeSearchQueryBuilder().withQuery( - QueryBuilders.matchPhrasePrefixQuery( - CollectionDoc::name.name, - apiSearch.advanced.collectionName - ) - ).withPageable(Pageable.unpaged()).withFilter(filter).build(), CollectionDoc::class.java + searchQuery, CollectionDoc::class.java ).searchHits.map { it.content.uuid } } } else { @@ -157,19 +167,24 @@ class CustomCollectionQueriesImpl @Autowired constructor( ) } - val results = elasticSearchTemplate.search(nsq.build(), RichSkillDoc::class.java) + val nsqb1: NativeSearchQuery = nsq.build() + val searchQuery1: Query = StringQuery(nsqb1.getQuery().toString()) + val results = elasticSearchTemplate.search(searchQuery1, RichSkillDoc::class.java) val innerHitCollectionUuids = results.searchHits.mapNotNull { it.getInnerHits("collections")?.searchHits?.mapNotNull { it.content as CollectionDoc } } .flatten().map { it.uuid }.distinct() + val nsqb2: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( + QueryBuilders.termsQuery( + "_id", + (innerHitCollectionUuids + collectionMultiPropertyResults).distinct() + ) + ).withFilter(filter).withPageable(pageable).build() + val searchQuery2: Query = StringQuery(nsqb2.getQuery().toString()) + return elasticSearchTemplate.search( - NativeSearchQueryBuilder().withQuery( - QueryBuilders.termsQuery( - "_id", - (innerHitCollectionUuids + collectionMultiPropertyResults).distinct() - ) - ).withFilter(filter).withPageable(pageable).build(), CollectionDoc::class.java + searchQuery2, CollectionDoc::class.java ) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt index 7c1916f7d..e20020294 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt @@ -5,8 +5,11 @@ import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilders import org.springframework.data.domain.Pageable import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder +import org.springframework.data.elasticsearch.core.query.Query +import org.springframework.data.elasticsearch.core.query.StringQuery interface FindsAllByPublishStatus { @@ -14,13 +17,15 @@ interface FindsAllByPublishStatus { val javaClass: Class fun findAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): SearchHits { - val nsq: NativeSearchQueryBuilder = buildQuery(pageable, publishStatus) - return elasticSearchTemplate.search(nsq.build(), javaClass) + val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus).build() + val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) + return elasticSearchTemplate.search(searchQuery, javaClass) } fun countAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): Long { - val nsq: NativeSearchQueryBuilder = buildQuery(pageable, publishStatus) - return elasticSearchTemplate.count(nsq.build(), javaClass) + val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus).build() + val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) + return elasticSearchTemplate.count(searchQuery, javaClass) } fun buildQuery( diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index 572335728..a85cbf008 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -10,9 +10,12 @@ import org.elasticsearch.search.sort.SortOrder import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates +import org.springframework.data.elasticsearch.core.query.Query +import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -42,7 +45,10 @@ class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSea nsq = NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery(disjunctionQuery) .withSort(SortBuilders.fieldSort("${JobCode::code.name}.keyword").order(SortOrder.ASC)) - return elasticSearchTemplate.search(nsq.build(), JobCode::class.java) + val nsqb: NativeSearchQuery = nsq.build() + val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) + + return elasticSearchTemplate.search(searchQuery, JobCode::class.java) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt index 35d5872c8..fcfe14ac1 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt @@ -9,9 +9,12 @@ import org.elasticsearch.search.sort.SortOrder import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder +import org.springframework.data.elasticsearch.core.query.Query +import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -61,7 +64,10 @@ class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSea ).minimumShouldMatch(1) } - return elasticSearchTemplate.search(nsq.build(), Keyword::class.java) + val nsqb: NativeSearchQuery = nsq.build() + val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) + + return elasticSearchTemplate.search(searchQuery, Keyword::class.java) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index d3e30cdb5..69ecfd948 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -24,9 +24,12 @@ import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder +import org.springframework.data.elasticsearch.core.query.Query +import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories import org.springframework.security.oauth2.jwt.Jwt @@ -327,9 +330,9 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear pageable: Pageable, collectionId: String? ): SearchHits { - val nsq: NativeSearchQueryBuilder = buildQuery(pageable, publishStatus, apiSearch, collectionId) - - return elasticSearchTemplate.search(nsq.build(), RichSkillDoc::class.java) + val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus, apiSearch, collectionId).build() + val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) + return elasticSearchTemplate.search(searchQuery, RichSkillDoc::class.java) } override fun countByApiSearch( @@ -338,9 +341,9 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear pageable: Pageable, collectionId: String? ): Long { - val nsq: NativeSearchQueryBuilder = buildQuery(pageable, publishStatus, apiSearch, collectionId) - - return elasticSearchTemplate.count(nsq.build(), RichSkillDoc::class.java) + val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus, apiSearch, collectionId).build() + val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) + return elasticSearchTemplate.count(searchQuery, RichSkillDoc::class.java) } fun buildQuery( From 2c652d820f8b3b78b1342d73dd472a3bec81f1a8 Mon Sep 17 00:00:00 2001 From: Huey Date: Fri, 22 Sep 2023 17:49:17 -0700 Subject: [PATCH 08/22] Merged Huey's Query upgrade calls to replace deprecated calls --- .../wgu/osmt/collection/CollectionEsRepo.kt | 57 ++++++--------- .../elasticsearch/FindsAllByPublishStatus.kt | 50 ++++++++++--- .../edu/wgu/osmt/keyword/KeywordEsRepo.kt | 73 ++++++++++++++++--- .../edu/wgu/osmt/richskill/RichSkillEsRepo.kt | 58 ++++++++------- 4 files changed, 153 insertions(+), 85 deletions(-) diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index daf379391..85c1b863f 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -16,12 +16,9 @@ import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate -import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.core.query.Query -import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -114,38 +111,31 @@ class CustomCollectionQueriesImpl @Autowired constructor( ) bq.should(richSkillEsRepo.occupationQueries(apiSearch.query)) - val nsqb: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( - collectionPropertiesMultiMatch(apiSearch.query) - ).withPageable(Pageable.unpaged()).withFilter(filter).build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) - // search on collection specific properties - collectionMultiPropertyResults = elasticSearchTemplate.search(searchQuery, CollectionDoc::class.java).searchHits.map { it.content.uuid } + collectionMultiPropertyResults = elasticSearchTemplate.search( + NativeSearchQueryBuilder().withQuery( + collectionPropertiesMultiMatch(apiSearch.query) + ).withPageable(Pageable.unpaged()).withFilter(filter).build(), CollectionDoc::class.java + ).searchHits.map { it.content.uuid } } else if (apiSearch.advanced != null) { richSkillEsRepo.generateBoolQueriesFromApiSearch(bq, apiSearch.advanced) if (!apiSearch.advanced.collectionName.isNullOrBlank()) { if (apiSearch.advanced.collectionName.contains("\"")) { - val nsqb: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( - QueryBuilders.simpleQueryStringQuery(apiSearch.advanced.collectionName).field("${CollectionDoc::name.name}.raw").defaultOperator(Operator.AND) - ).withPageable(Pageable.unpaged()).withFilter(filter).build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) - collectionMultiPropertyResults = elasticSearchTemplate.search( - searchQuery, CollectionDoc::class.java + NativeSearchQueryBuilder().withQuery( + QueryBuilders.simpleQueryStringQuery(apiSearch.advanced.collectionName).field("${CollectionDoc::name.name}.raw").defaultOperator(Operator.AND) + ).withPageable(Pageable.unpaged()).withFilter(filter).build(), CollectionDoc::class.java ).searchHits.map { it.content.uuid } } else { - val nsqb: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( - QueryBuilders.matchPhrasePrefixQuery( - CollectionDoc::name.name, - apiSearch.advanced.collectionName - ) - ).withPageable(Pageable.unpaged()).withFilter(filter).build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) - collectionMultiPropertyResults = elasticSearchTemplate.search( - searchQuery, CollectionDoc::class.java + NativeSearchQueryBuilder().withQuery( + QueryBuilders.matchPhrasePrefixQuery( + CollectionDoc::name.name, + apiSearch.advanced.collectionName + ) + ).withPageable(Pageable.unpaged()).withFilter(filter).build(), CollectionDoc::class.java ).searchHits.map { it.content.uuid } } } else { @@ -167,24 +157,19 @@ class CustomCollectionQueriesImpl @Autowired constructor( ) } - val nsqb1: NativeSearchQuery = nsq.build() - val searchQuery1: Query = StringQuery(nsqb1.getQuery().toString()) - val results = elasticSearchTemplate.search(searchQuery1, RichSkillDoc::class.java) + val results = elasticSearchTemplate.search(nsq.build(), RichSkillDoc::class.java) val innerHitCollectionUuids = results.searchHits.mapNotNull { it.getInnerHits("collections")?.searchHits?.mapNotNull { it.content as CollectionDoc } } .flatten().map { it.uuid }.distinct() - val nsqb2: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( - QueryBuilders.termsQuery( - "_id", - (innerHitCollectionUuids + collectionMultiPropertyResults).distinct() - ) - ).withFilter(filter).withPageable(pageable).build() - val searchQuery2: Query = StringQuery(nsqb2.getQuery().toString()) - return elasticSearchTemplate.search( - searchQuery2, CollectionDoc::class.java + NativeSearchQueryBuilder().withQuery( + QueryBuilders.termsQuery( + "_id", + (innerHitCollectionUuids + collectionMultiPropertyResults).distinct() + ) + ).withFilter(filter).withPageable(pageable).build(), CollectionDoc::class.java ) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt index e20020294..e92ec3ff3 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt @@ -1,15 +1,14 @@ package edu.wgu.osmt.elasticsearch +import co.elastic.clients.elasticsearch._types.FieldValue +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.* +import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField import edu.wgu.osmt.db.PublishStatus -import org.elasticsearch.index.query.BoolQueryBuilder -import org.elasticsearch.index.query.QueryBuilders import org.springframework.data.domain.Pageable import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate -import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery +import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits -import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder -import org.springframework.data.elasticsearch.core.query.Query -import org.springframework.data.elasticsearch.core.query.StringQuery +import java.util.stream.Collectors interface FindsAllByPublishStatus { @@ -17,17 +16,24 @@ interface FindsAllByPublishStatus { val javaClass: Class fun findAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): SearchHits { - val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus).build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) - return elasticSearchTemplate.search(searchQuery, javaClass) +// val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus).build() +// val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) +// return elasticSearchTemplate.search(searchQuery, javaClass) + + val nsq: NativeQueryBuilder = buildQuery(pageable, publishStatus) + return elasticSearchTemplate.search(nsq.build(), javaClass) } fun countAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): Long { - val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus).build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) - return elasticSearchTemplate.count(searchQuery, javaClass) +// val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus).build() +// val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) +// return elasticSearchTemplate.count(searchQuery, javaClass) + + val nsq: NativeQueryBuilder = buildQuery(pageable, publishStatus) + return elasticSearchTemplate.count(nsq.build(), javaClass) } + /* fun buildQuery( pageable: Pageable, publishStatus: Set @@ -44,5 +50,25 @@ interface FindsAllByPublishStatus { ) return nsq } +*/ + fun buildQuery( + pageable: Pageable, + publishStatus: Set + ): NativeQueryBuilder { + + val MATCH_ALL = matchAll().build()._toQuery() + val fieldValues = publishStatus + .stream() + .map{ ps -> FieldValue.of(ps.name) } + .collect(Collectors.toList()) + var tqf: TermsQueryField = TermsQueryField.Builder().value(fieldValues).build() + var terms = terms().field("publishStatus").terms(tqf).build()._toQuery() + var filter = bool().should(terms).build()._toQuery() + + return NativeQueryBuilder() + .withPageable(pageable) + .withQuery(MATCH_ALL) + .withFilter(filter) + } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt index fcfe14ac1..2486b8764 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt @@ -1,20 +1,18 @@ package edu.wgu.osmt.keyword +import co.elastic.clients.elasticsearch._types.FieldSort +import co.elastic.clients.elasticsearch._types.SortOptions +import co.elastic.clients.elasticsearch._types.SortOrder +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.* import edu.wgu.osmt.config.INDEX_KEYWORD_DOC import edu.wgu.osmt.config.SORT_INSENSITIVE import edu.wgu.osmt.elasticsearch.OffsetPageable -import org.elasticsearch.index.query.QueryBuilders -import org.elasticsearch.search.sort.SortBuilders -import org.elasticsearch.search.sort.SortOrder import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate -import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery +import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder -import org.springframework.data.elasticsearch.core.query.Query -import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -30,19 +28,20 @@ interface CustomKeywordRepository { class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchTemplate) : CustomKeywordRepository { override fun typeAheadSearch(query: String, type: KeywordTypeEnum): SearchHits { + /* val limitedPageable: OffsetPageable val bq = QueryBuilders.boolQuery() val nsq: NativeSearchQueryBuilder if(query.isEmpty()){ //retrieve all limitedPageable = OffsetPageable(0, 10000, null) - nsq = NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery(bq) + nsq = NativeSearchQueryBuilder() + .withPageable(limitedPageable) + .withQuery(bq) .withSort(SortBuilders.fieldSort("${Keyword::value.name}$SORT_INSENSITIVE").order(SortOrder.ASC)) bq .must(QueryBuilders.termQuery(Keyword::type.name, type.name)) - .should( - QueryBuilders.matchAllQuery() - ) + .should( QueryBuilders.matchAllQuery() ) } else { limitedPageable = OffsetPageable(0, 20, null) @@ -63,14 +62,64 @@ class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSea ).boost(5f) ).minimumShouldMatch(1) } - val nsqb: NativeSearchQuery = nsq.build() val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) return elasticSearchTemplate.search(searchQuery, Keyword::class.java) +*/ + val nsq1 = if (query.isEmpty()) + createNativeQuery_retrieveAll(type) + else + createNativeQuery_retrieveExact(query, type) + return elasticSearchTemplate.search(nsq1.build(), Keyword::class.java) + } + + + private fun createNativeQuery_retrieveExact(queryStr: String, type: KeywordTypeEnum) : NativeQueryBuilder { + val termQuery = term().field(Keyword::type.name).value(type.name).build()._toQuery() + val prefixQuery = matchBoolPrefix().field(Keyword::type.name).query(queryStr).build()._toQuery() + val phraseQuery = matchPhrase().field(Keyword::value.name).query(queryStr).build()._toQuery() + val bq = bool() + .must(termQuery) + .should(prefixQuery) + .should(phraseQuery) + .build() + ._toQuery() + return NativeQueryBuilder() + .withPageable(createPageable(20)) + .withSort(createCaseInsensitiveSortOptions()) + .withQuery(bq) + } + + private fun createNativeQuery_retrieveAll(type: KeywordTypeEnum) : NativeQueryBuilder { + val MATCH_ALL = matchAll().build()._toQuery() + val termQuery = term().field(Keyword::type.name).value(type.name).build()._toQuery() + val bq = bool() + .must(termQuery) + .should(MATCH_ALL) + .build() + ._toQuery() + return NativeQueryBuilder() + .withPageable(createPageable(1000)) + .withSort(createCaseInsensitiveSortOptions()) + .withQuery(bq) + } + + private fun createPageable(limit: Int) : OffsetPageable { + return OffsetPageable(0, 20, null) + } + + private fun createCaseInsensitiveSortOptions() : SortOptions { + val fieldSort = FieldSort.Builder().field("${Keyword::value.name}$SORT_INSENSITIVE").order(SortOrder.Asc).build() + return SortOptions + .Builder() + .field(fieldSort) + .build() } } + + @Configuration @EnableElasticsearchRepositories("edu.wgu.osmt.keyword") class KeywordEsRepoConfig diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index 69ecfd948..2bcc633cf 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -25,6 +25,7 @@ import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery +import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder @@ -330,9 +331,13 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear pageable: Pageable, collectionId: String? ): SearchHits { - val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus, apiSearch, collectionId).build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) - return elasticSearchTemplate.search(searchQuery, RichSkillDoc::class.java) + +// val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus, apiSearch, collectionId).build() +// val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) +// return elasticSearchTemplate.search(searchQuery, RichSkillDoc::class.java) + + val nsq: NativeQueryBuilder = buildQuery(pageable, publishStatus, apiSearch, collectionId) + return elasticSearchTemplate.search(nsq.build(), RichSkillDoc::class.java) } override fun countByApiSearch( @@ -341,9 +346,12 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear pageable: Pageable, collectionId: String? ): Long { - val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus, apiSearch, collectionId).build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) - return elasticSearchTemplate.count(searchQuery, RichSkillDoc::class.java) +// val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus, apiSearch, collectionId).build() +// val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) +// return elasticSearchTemplate.count(searchQuery, RichSkillDoc::class.java) + + val nsq: NativeQueryBuilder = buildQuery(pageable, publishStatus, apiSearch, collectionId) + return elasticSearchTemplate.count(nsq.build(), RichSkillDoc::class.java) } fun buildQuery( @@ -351,19 +359,19 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear publishStatus: Set, apiSearch: ApiSearch, collectionId: String? - ): NativeSearchQueryBuilder { - val nsq: NativeSearchQueryBuilder = NativeSearchQueryBuilder().withPageable(pageable) + ): NativeQueryBuilder { + val nsq: NativeQueryBuilder = NativeQueryBuilder().withPageable(pageable) val bq = boolQuery() - nsq.withQuery(bq) - nsq.withFilter( - BoolQueryBuilder().must( - termsQuery( - RichSkillDoc::publishStatus.name, - publishStatus.map { ps -> ps.toString() } - ) - ) - ) +// nsq.withQuery(bq) +// nsq.withFilter( +// BoolQueryBuilder().must( +// termsQuery( +// RichSkillDoc::publishStatus.name, +// publishStatus.map { ps -> ps.toString() } +// ) +// ) +// ) apiSearch.filtered?.let { generateBoolQueriesFromApiSearchWithFilters(bq, it, publishStatus) } @@ -439,14 +447,14 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear var apiSearchUuids = apiSearch.uuids?.filterNotNull()?.filter { x: String? -> x != "" } if (!apiSearchUuids.isNullOrEmpty()) { - nsq.withFilter( - BoolQueryBuilder().must( - termsQuery( - RichSkillDoc::uuid.name, - apiSearchUuids - ) - ) - ) +// nsq.withFilter( +// BoolQueryBuilder().must( +// termsQuery( +// RichSkillDoc::uuid.name, +// apiSearchUuids +// ) +// ) +// ) } if (!collectionId.isNullOrBlank()) { bq.must( From bf2a54464988de4a7447463fd2574c31514bd1ee Mon Sep 17 00:00:00 2001 From: Huey Date: Mon, 25 Sep 2023 17:43:00 -0700 Subject: [PATCH 09/22] Deprecated ElasticSearchRestTemplate has been excised. Successfully able to search for collection and skills without exceptions. Need further testing with PostMan --- .../wgu/osmt/collection/CollectionEsRepo.kt | 32 +++++----- .../elasticsearch/FindsAllByPublishStatus.kt | 58 +++++++++++++++++-- .../edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 28 ++++++--- .../edu/wgu/osmt/richskill/RichSkillEsRepo.kt | 22 +++---- 4 files changed, 97 insertions(+), 43 deletions(-) diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index daf379391..82a707df1 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -9,6 +9,8 @@ import edu.wgu.osmt.richskill.RichSkillDoc import edu.wgu.osmt.richskill.RichSkillEsRepo import org.apache.lucene.search.join.ScoreMode import org.elasticsearch.index.query.* +import org.slf4j.Logger +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.domain.Page @@ -20,8 +22,6 @@ import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.core.query.Query -import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -50,7 +50,7 @@ class CustomCollectionQueriesImpl @Autowired constructor( override val richSkillEsRepo: RichSkillEsRepo ) : CustomCollectionQueries { - + val log: Logger = LoggerFactory.getLogger(CustomCollectionQueriesImpl::class.java) override val javaClass = CollectionDoc::class.java override fun collectionPropertiesMultiMatch(query: String): AbstractQueryBuilder<*> { @@ -117,23 +117,23 @@ class CustomCollectionQueriesImpl @Autowired constructor( val nsqb: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( collectionPropertiesMultiMatch(apiSearch.query) ).withPageable(Pageable.unpaged()).withFilter(filter).build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) // search on collection specific properties - collectionMultiPropertyResults = elasticSearchTemplate.search(searchQuery, CollectionDoc::class.java).searchHits.map { it.content.uuid } + val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()1", nsqb, log) + collectionMultiPropertyResults = elasticSearchTemplate.search(query, CollectionDoc::class.java).searchHits.map { it.content.uuid } } else if (apiSearch.advanced != null) { richSkillEsRepo.generateBoolQueriesFromApiSearch(bq, apiSearch.advanced) if (!apiSearch.advanced.collectionName.isNullOrBlank()) { if (apiSearch.advanced.collectionName.contains("\"")) { - val nsqb: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( + val searchQuery: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( QueryBuilders.simpleQueryStringQuery(apiSearch.advanced.collectionName).field("${CollectionDoc::name.name}.raw").defaultOperator(Operator.AND) ).withPageable(Pageable.unpaged()).withFilter(filter).build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) + val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()2", searchQuery, log) collectionMultiPropertyResults = elasticSearchTemplate.search( - searchQuery, CollectionDoc::class.java + query, CollectionDoc::class.java ).searchHits.map { it.content.uuid } } else { val nsqb: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( @@ -142,10 +142,10 @@ class CustomCollectionQueriesImpl @Autowired constructor( apiSearch.advanced.collectionName ) ).withPageable(Pageable.unpaged()).withFilter(filter).build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) + val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()3", nsqb, log) collectionMultiPropertyResults = elasticSearchTemplate.search( - searchQuery, CollectionDoc::class.java + query, CollectionDoc::class.java ).searchHits.map { it.content.uuid } } } else { @@ -167,9 +167,8 @@ class CustomCollectionQueriesImpl @Autowired constructor( ) } - val nsqb1: NativeSearchQuery = nsq.build() - val searchQuery1: Query = StringQuery(nsqb1.getQuery().toString()) - val results = elasticSearchTemplate.search(searchQuery1, RichSkillDoc::class.java) + var query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch().innerHitCollectionUuids", nsq.build(), log) + val results = elasticSearchTemplate.search(query, RichSkillDoc::class.java) val innerHitCollectionUuids = results.searchHits.mapNotNull { it.getInnerHits("collections")?.searchHits?.mapNotNull { it.content as CollectionDoc } } @@ -181,11 +180,8 @@ class CustomCollectionQueriesImpl @Autowired constructor( (innerHitCollectionUuids + collectionMultiPropertyResults).distinct() ) ).withFilter(filter).withPageable(pageable).build() - val searchQuery2: Query = StringQuery(nsqb2.getQuery().toString()) - - return elasticSearchTemplate.search( - searchQuery2, CollectionDoc::class.java - ) + query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()4", nsqb2, log) + return elasticSearchTemplate.search(query, CollectionDoc::class.java) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt index e20020294..6bf5f68a5 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt @@ -3,14 +3,22 @@ package edu.wgu.osmt.elasticsearch import edu.wgu.osmt.db.PublishStatus import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilders +import org.slf4j.Logger +import org.slf4j.LoggerFactory import org.springframework.data.domain.Pageable import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery -import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder -import org.springframework.data.elasticsearch.core.query.Query +import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.query.StringQuery +/** TODO: Upgrade to ElasticSearch 8.x apis +import org.springframework.data.elasticsearch.client.elc.* +import co.elastic.clients.elasticsearch._types.FieldValue +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.* +import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField +import java.util.stream.Collectors + */ interface FindsAllByPublishStatus { val elasticSearchTemplate: ElasticsearchTemplate @@ -18,14 +26,24 @@ interface FindsAllByPublishStatus { fun findAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): SearchHits { val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus).build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) - return elasticSearchTemplate.search(searchQuery, javaClass) + val query = createStringQuery("FindsAllByPublishStatus.findAllFilteredByPublishStatus()", nsqb, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) + return elasticSearchTemplate.search(query, javaClass) + + /* TODO: Upgrade to ElasticSearch 8.x apis + val searchQuery: NativeQuery = buildQuery(pageable, publishStatus).build() + return elasticSearchTemplate.search(query, javaClass) + */ } fun countAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): Long { val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus).build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) - return elasticSearchTemplate.count(searchQuery, javaClass) + val query = createStringQuery("FindsAllByPublishStatus.countAllFilteredByPublishStatus()", nsqb, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) + return elasticSearchTemplate.count(query, javaClass) + + /* TODO: Upgrade to ElasticSearch 8.x apis + val searchQuery: NativeQuery = buildQuery(pageable, publishStatus).build() + return elasticSearchTemplate.count(query, javaClass) + */ } fun buildQuery( @@ -44,5 +62,33 @@ interface FindsAllByPublishStatus { ) return nsq } + + fun createStringQuery (msgPrefix: String, nsq: NativeSearchQuery, log: Logger): StringQuery { + val queryStr = nsq.query.toString() + log.info(String.Companion.format("%s:\n%s", msgPrefix, queryStr)) + return StringQuery(queryStr) + } + + /* TODO: Upgrade to ElasticSearch v8.X + fun buildQuery( + pageable: Pageable, + publishStatus: Set + ): NativeQueryBuilder { + + val MATCH_ALL = matchAll().build()._toQuery() + val fieldValues = publishStatus + .stream() + .map{ ps -> FieldValue.of(ps.name) } + .collect(Collectors.toList()) + var tqf: TermsQueryField = TermsQueryField.Builder().value(fieldValues).build() + var terms = terms().field("publishStatus").terms(tqf).build()._toQuery() + var filter = bool().should(terms).build()._toQuery() + + return NativeQueryBuilder() + .withPageable(pageable) + .withQuery(MATCH_ALL) + .withFilter(filter) + } + */ } diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index a85cbf008..968727d1c 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -7,6 +7,8 @@ import org.elasticsearch.index.query.Operator import org.elasticsearch.index.query.QueryBuilders.* import org.elasticsearch.search.sort.SortBuilders import org.elasticsearch.search.sort.SortOrder +import org.slf4j.Logger +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate @@ -14,7 +16,6 @@ import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.core.query.Query import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -31,24 +32,33 @@ interface CustomJobCodeRepository { class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchTemplate) : CustomJobCodeRepository { + val log: Logger = LoggerFactory.getLogger(CustomJobCodeRepositoryImpl::class.java) override fun typeAheadSearch(query: String): SearchHits { - val nsq: NativeSearchQueryBuilder + val disjunctionQuery = JobCodeQueries.multiPropertySearch(query) + val nsq = NativeSearchQueryBuilder() + .withPageable(createOffsetPageable(query)) + .withQuery(disjunctionQuery) + .withSort(SortBuilders.fieldSort("${JobCode::code.name}.keyword").order(SortOrder.ASC)) + .build() + val query = createStringQuery("CustomJobCodeRepositoryImpl.typeAheadSearch()", nsq) + return elasticSearchTemplate.search(query, JobCode::class.java) + } + private fun createOffsetPageable(query: String): OffsetPageable { val limitedPageable: OffsetPageable = if (query.isEmpty()) { OffsetPageable(0, 10000, null) } else { OffsetPageable(0, 20, null) } - val disjunctionQuery = JobCodeQueries.multiPropertySearch(query) - nsq = - NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery(disjunctionQuery) - .withSort(SortBuilders.fieldSort("${JobCode::code.name}.keyword").order(SortOrder.ASC)) - val nsqb: NativeSearchQuery = nsq.build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) + return limitedPageable + } - return elasticSearchTemplate.search(searchQuery, JobCode::class.java) + private fun createStringQuery (msgPrefix: String, nsq: NativeSearchQuery): StringQuery { + val queryStr = nsq.query.toString() + log.info(String.Companion.format("%s:\n%s", msgPrefix, queryStr)) + return StringQuery(queryStr) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index 69ecfd948..4bc6f1f32 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -17,6 +17,8 @@ import org.apache.lucene.search.join.ScoreMode import org.elasticsearch.index.query.* import org.elasticsearch.index.query.QueryBuilders.* import org.elasticsearch.script.Script +import org.slf4j.Logger +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.domain.Page @@ -25,11 +27,9 @@ import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder -import org.springframework.data.elasticsearch.core.query.Query -import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories import org.springframework.security.oauth2.jwt.Jwt @@ -72,6 +72,7 @@ interface CustomRichSkillQueries : FindsAllByPublishStatus { class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchTemplate) : CustomRichSkillQueries { + val log: Logger = LoggerFactory.getLogger(CustomRichSkillQueriesImpl::class.java) override val javaClass = RichSkillDoc::class.java override fun getUuidsFromApiSearch( @@ -330,9 +331,9 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear pageable: Pageable, collectionId: String? ): SearchHits { - val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus, apiSearch, collectionId).build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) - return elasticSearchTemplate.search(searchQuery, RichSkillDoc::class.java) + val nsq: NativeSearchQuery = buildQuery(pageable, publishStatus, apiSearch, collectionId).build() + val query = createStringQuery("CustomRichSkillQueriesImpl.byApiSearch()", nsq, log) + return elasticSearchTemplate.search(query, RichSkillDoc::class.java) } override fun countByApiSearch( @@ -341,9 +342,9 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear pageable: Pageable, collectionId: String? ): Long { - val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus, apiSearch, collectionId).build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) - return elasticSearchTemplate.count(searchQuery, RichSkillDoc::class.java) + val nsq: NativeSearchQuery = buildQuery(pageable, publishStatus, apiSearch, collectionId).build() + val query = createStringQuery("CustomRichSkillQueriesImpl.countByApiSearch()", nsq, log) + return elasticSearchTemplate.count(query, RichSkillDoc::class.java) } fun buildQuery( @@ -469,7 +470,8 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear val nsq: NativeSearchQueryBuilder = NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery( MatchPhraseQueryBuilder(RichSkillDoc::statement.name, apiSimilaritySearch.statement).slop(4) ) - return elasticSearchTemplate.search(nsq.build(), RichSkillDoc::class.java) + val query = createStringQuery("CustomRichSkillQueriesImpl.findSimilar()", nsq.build(), log) + return elasticSearchTemplate.search(query, RichSkillDoc::class.java) } } From 20a017fcb3509365fd8d984ae02788133cb034cf Mon Sep 17 00:00:00 2001 From: Huey Date: Tue, 26 Sep 2023 11:39:13 -0700 Subject: [PATCH 10/22] Code clean up --- .../wgu/osmt/collection/CollectionEsRepo.kt | 63 +++++++++++-------- .../elasticsearch/FindsAllByPublishStatus.kt | 19 +++--- .../edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 3 +- .../edu/wgu/osmt/keyword/KeywordEsRepo.kt | 38 +++++------ .../edu/wgu/osmt/richskill/RichSkillEsRepo.kt | 11 ++-- 5 files changed, 70 insertions(+), 64 deletions(-) diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index 82a707df1..67b072b3c 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -9,6 +9,8 @@ import edu.wgu.osmt.richskill.RichSkillDoc import edu.wgu.osmt.richskill.RichSkillEsRepo import org.apache.lucene.search.join.ScoreMode import org.elasticsearch.index.query.* +import org.elasticsearch.index.query.QueryBuilders.matchPhrasePrefixQuery +import org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired @@ -114,39 +116,46 @@ class CustomCollectionQueriesImpl @Autowired constructor( ) bq.should(richSkillEsRepo.occupationQueries(apiSearch.query)) - val nsqb: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( - collectionPropertiesMultiMatch(apiSearch.query) - ).withPageable(Pageable.unpaged()).withFilter(filter).build() - + val nsq = NativeSearchQueryBuilder() + .withQuery( collectionPropertiesMultiMatch(apiSearch.query) ) + .withPageable(Pageable.unpaged()) + .withFilter(filter) + .build() // search on collection specific properties - val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()1", nsqb, log) - collectionMultiPropertyResults = elasticSearchTemplate.search(query, CollectionDoc::class.java).searchHits.map { it.content.uuid } + val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()1", nsq, log) + collectionMultiPropertyResults = elasticSearchTemplate + .search(query, CollectionDoc::class.java) + .searchHits + .map { it.content.uuid } } else if (apiSearch.advanced != null) { richSkillEsRepo.generateBoolQueriesFromApiSearch(bq, apiSearch.advanced) if (!apiSearch.advanced.collectionName.isNullOrBlank()) { if (apiSearch.advanced.collectionName.contains("\"")) { - val searchQuery: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( - QueryBuilders.simpleQueryStringQuery(apiSearch.advanced.collectionName).field("${CollectionDoc::name.name}.raw").defaultOperator(Operator.AND) - ).withPageable(Pageable.unpaged()).withFilter(filter).build() - - val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()2", searchQuery, log) - collectionMultiPropertyResults = elasticSearchTemplate.search( - query, CollectionDoc::class.java - ).searchHits.map { it.content.uuid } + val nsq = NativeSearchQueryBuilder() + .withQuery( + simpleQueryStringQuery(apiSearch.advanced.collectionName).field("${CollectionDoc::name.name}.raw").defaultOperator(Operator.AND) ) + .withPageable(Pageable.unpaged()) + .withFilter(filter) + .build() + val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()2", nsq, log) + collectionMultiPropertyResults = elasticSearchTemplate + .search( query, CollectionDoc::class.java ) + .searchHits + .map { it.content.uuid } } else { - val nsqb: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( - QueryBuilders.matchPhrasePrefixQuery( - CollectionDoc::name.name, - apiSearch.advanced.collectionName - ) - ).withPageable(Pageable.unpaged()).withFilter(filter).build() - - val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()3", nsqb, log) - collectionMultiPropertyResults = elasticSearchTemplate.search( - query, CollectionDoc::class.java - ).searchHits.map { it.content.uuid } + val nsq = NativeSearchQueryBuilder() + .withQuery( + matchPhrasePrefixQuery( CollectionDoc::name.name, apiSearch.advanced.collectionName ) ) + .withPageable(Pageable.unpaged()) + .withFilter(filter) + .build() + val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()3", nsq, log) + collectionMultiPropertyResults = elasticSearchTemplate + .search( query, CollectionDoc::class.java ) + .searchHits + .map { it.content.uuid } } } else { bq.must( @@ -174,13 +183,13 @@ class CustomCollectionQueriesImpl @Autowired constructor( results.searchHits.mapNotNull { it.getInnerHits("collections")?.searchHits?.mapNotNull { it.content as CollectionDoc } } .flatten().map { it.uuid }.distinct() - val nsqb2: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( + val nsq2: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( QueryBuilders.termsQuery( "_id", (innerHitCollectionUuids + collectionMultiPropertyResults).distinct() ) ).withFilter(filter).withPageable(pageable).build() - query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()4", nsqb2, log) + query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()4", nsq2, log) return elasticSearchTemplate.search(query, CollectionDoc::class.java) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt index 6bf5f68a5..14baa8ff1 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt @@ -10,6 +10,7 @@ import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits +import org.springframework.data.elasticsearch.core.query.Query import org.springframework.data.elasticsearch.core.query.StringQuery /** TODO: Upgrade to ElasticSearch 8.x apis @@ -25,8 +26,8 @@ interface FindsAllByPublishStatus { val javaClass: Class fun findAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): SearchHits { - val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus).build() - val query = createStringQuery("FindsAllByPublishStatus.findAllFilteredByPublishStatus()", nsqb, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) + val nsq: NativeSearchQuery = buildQuery(pageable, publishStatus).build() + val query = createStringQuery("FindsAllByPublishStatus.findAllFilteredByPublishStatus()", nsq, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) return elasticSearchTemplate.search(query, javaClass) /* TODO: Upgrade to ElasticSearch 8.x apis @@ -36,8 +37,8 @@ interface FindsAllByPublishStatus { } fun countAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): Long { - val nsqb: NativeSearchQuery = buildQuery(pageable, publishStatus).build() - val query = createStringQuery("FindsAllByPublishStatus.countAllFilteredByPublishStatus()", nsqb, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) + val nsq: NativeSearchQuery = buildQuery(pageable, publishStatus).build() + val query = createStringQuery("FindsAllByPublishStatus.countAllFilteredByPublishStatus()", nsq, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) return elasticSearchTemplate.count(query, javaClass) /* TODO: Upgrade to ElasticSearch 8.x apis @@ -50,9 +51,9 @@ interface FindsAllByPublishStatus { pageable: Pageable, publishStatus: Set ): NativeSearchQueryBuilder { - val nsq: NativeSearchQueryBuilder = NativeSearchQueryBuilder().withPageable(pageable) - nsq.withQuery(QueryBuilders.matchAllQuery()) - nsq.withFilter( + val nsqb = NativeSearchQueryBuilder().withPageable(pageable) + nsqb.withQuery(QueryBuilders.matchAllQuery()) + nsqb.withFilter( BoolQueryBuilder().should( QueryBuilders.termsQuery( "publishStatus", @@ -60,10 +61,10 @@ interface FindsAllByPublishStatus { ) ) ) - return nsq + return nsqb } - fun createStringQuery (msgPrefix: String, nsq: NativeSearchQuery, log: Logger): StringQuery { + fun createStringQuery (msgPrefix: String, nsq: NativeSearchQuery, log: Logger): Query { val queryStr = nsq.query.toString() log.info(String.Companion.format("%s:\n%s", msgPrefix, queryStr)) return StringQuery(queryStr) diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index 968727d1c..de2be319f 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -16,6 +16,7 @@ import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates +import org.springframework.data.elasticsearch.core.query.Query import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -55,7 +56,7 @@ class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSea return limitedPageable } - private fun createStringQuery (msgPrefix: String, nsq: NativeSearchQuery): StringQuery { + private fun createStringQuery (msgPrefix: String, nsq: NativeSearchQuery): Query { val queryStr = nsq.query.toString() log.info(String.Companion.format("%s:\n%s", msgPrefix, queryStr)) return StringQuery(queryStr) diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt index fcfe14ac1..c996d1e3d 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt @@ -10,10 +10,9 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder -import org.springframework.data.elasticsearch.core.query.Query import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -32,41 +31,36 @@ class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSea override fun typeAheadSearch(query: String, type: KeywordTypeEnum): SearchHits { val limitedPageable: OffsetPageable val bq = QueryBuilders.boolQuery() - val nsq: NativeSearchQueryBuilder + val nsq: NativeSearchQuery if(query.isEmpty()){ //retrieve all limitedPageable = OffsetPageable(0, 10000, null) - nsq = NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery(bq) - .withSort(SortBuilders.fieldSort("${Keyword::value.name}$SORT_INSENSITIVE").order(SortOrder.ASC)) + nsq = NativeSearchQueryBuilder() + .withPageable(limitedPageable) + .withQuery(bq) + .withSort(SortBuilders.fieldSort("${Keyword::value.name}$SORT_INSENSITIVE").order(SortOrder.ASC)) + .build() bq .must(QueryBuilders.termQuery(Keyword::type.name, type.name)) - .should( - QueryBuilders.matchAllQuery() - ) + .should( QueryBuilders.matchAllQuery() ) } else { limitedPageable = OffsetPageable(0, 20, null) - nsq = NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery(bq) - .withSort(SortBuilders.fieldSort("${Keyword::value.name}$SORT_INSENSITIVE").order(SortOrder.ASC)) + nsq = NativeSearchQueryBuilder() + .withPageable(limitedPageable) + .withQuery(bq) + .withSort(SortBuilders.fieldSort("${Keyword::value.name}$SORT_INSENSITIVE").order(SortOrder.ASC)) + .build() bq .must(QueryBuilders.termQuery(Keyword::type.name, type.name)) .should( - QueryBuilders.matchBoolPrefixQuery( - Keyword::value.name, - query - ) + QueryBuilders.matchBoolPrefixQuery( Keyword::value.name, query ) ) .should( - QueryBuilders.matchPhraseQuery( - Keyword::value.name, - query - ).boost(5f) + QueryBuilders.matchPhraseQuery( Keyword::value.name, query ).boost(5f) ).minimumShouldMatch(1) } - - val nsqb: NativeSearchQuery = nsq.build() - val searchQuery: Query = StringQuery(nsqb.getQuery().toString()) - + val searchQuery = StringQuery(nsq.getQuery().toString()) return elasticSearchTemplate.search(searchQuery, Keyword::class.java) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index 4bc6f1f32..7e3c7729e 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -353,7 +353,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear apiSearch: ApiSearch, collectionId: String? ): NativeSearchQueryBuilder { - val nsq: NativeSearchQueryBuilder = NativeSearchQueryBuilder().withPageable(pageable) + val nsq = NativeSearchQueryBuilder().withPageable(pageable) val bq = boolQuery() nsq.withQuery(bq) @@ -467,10 +467,11 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear override fun findSimilar(apiSimilaritySearch: ApiSimilaritySearch): SearchHits { val limitedPageable = OffsetPageable(0, 10, null) - val nsq: NativeSearchQueryBuilder = NativeSearchQueryBuilder().withPageable(limitedPageable).withQuery( - MatchPhraseQueryBuilder(RichSkillDoc::statement.name, apiSimilaritySearch.statement).slop(4) - ) - val query = createStringQuery("CustomRichSkillQueriesImpl.findSimilar()", nsq.build(), log) + val nsq = NativeSearchQueryBuilder() + .withPageable(limitedPageable) + .withQuery( MatchPhraseQueryBuilder(RichSkillDoc::statement.name, apiSimilaritySearch.statement).slop(4)) + .build() + val query = createStringQuery("CustomRichSkillQueriesImpl.findSimilar()", nsq, log) return elasticSearchTemplate.search(query, RichSkillDoc::class.java) } } From 3584bf75e05d113d86e24777c42b1778a02cb8ac Mon Sep 17 00:00:00 2001 From: Huey Date: Tue, 26 Sep 2023 15:51:08 -0700 Subject: [PATCH 11/22] Code clean up --- .../wgu/osmt/collection/CollectionEsRepo.kt | 42 ++++++-------- .../elasticsearch/FindsAllByPublishStatus.kt | 55 +++++++++++-------- .../edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 11 +++- .../edu/wgu/osmt/keyword/KeywordEsRepo.kt | 23 +++++++- .../edu/wgu/osmt/richskill/RichSkillEsRepo.kt | 16 +++--- .../config/application-dev.properties | 1 + 6 files changed, 88 insertions(+), 60 deletions(-) diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index 67b072b3c..0e23f25c0 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -20,7 +20,6 @@ import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate -import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates @@ -87,7 +86,7 @@ class CustomCollectionQueriesImpl @Autowired constructor( publishStatus: Set, pageable: Pageable ): SearchHits { - val nsq: NativeSearchQueryBuilder = NativeSearchQueryBuilder().withPageable(Pageable.unpaged()) + val nqb = NativeSearchQueryBuilder().withPageable(Pageable.unpaged()) val bq = QueryBuilders.boolQuery() val filter = BoolQueryBuilder().must( QueryBuilders.termsQuery( @@ -95,8 +94,8 @@ class CustomCollectionQueriesImpl @Autowired constructor( publishStatus.map { ps -> ps.toString() } ) ) - nsq.withFilter(filter) - nsq.withQuery(bq) + nqb.withFilter(filter) + nqb.withQuery(bq) var collectionMultiPropertyResults: List = listOf() @@ -116,13 +115,12 @@ class CustomCollectionQueriesImpl @Autowired constructor( ) bq.should(richSkillEsRepo.occupationQueries(apiSearch.query)) - val nsq = NativeSearchQueryBuilder() + val nqb = NativeSearchQueryBuilder() .withQuery( collectionPropertiesMultiMatch(apiSearch.query) ) .withPageable(Pageable.unpaged()) .withFilter(filter) - .build() // search on collection specific properties - val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()1", nsq, log) + val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()1", nqb, log) collectionMultiPropertyResults = elasticSearchTemplate .search(query, CollectionDoc::class.java) .searchHits @@ -133,25 +131,21 @@ class CustomCollectionQueriesImpl @Autowired constructor( if (!apiSearch.advanced.collectionName.isNullOrBlank()) { if (apiSearch.advanced.collectionName.contains("\"")) { - val nsq = NativeSearchQueryBuilder() - .withQuery( - simpleQueryStringQuery(apiSearch.advanced.collectionName).field("${CollectionDoc::name.name}.raw").defaultOperator(Operator.AND) ) + val nqb = NativeSearchQueryBuilder() + .withQuery( simpleQueryStringQuery(apiSearch.advanced.collectionName).field("${CollectionDoc::name.name}.raw").defaultOperator(Operator.AND) ) .withPageable(Pageable.unpaged()) .withFilter(filter) - .build() - val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()2", nsq, log) + val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()2", nqb, log) collectionMultiPropertyResults = elasticSearchTemplate .search( query, CollectionDoc::class.java ) .searchHits .map { it.content.uuid } } else { - val nsq = NativeSearchQueryBuilder() - .withQuery( - matchPhrasePrefixQuery( CollectionDoc::name.name, apiSearch.advanced.collectionName ) ) + val nqb = NativeSearchQueryBuilder() + .withQuery( matchPhrasePrefixQuery( CollectionDoc::name.name, apiSearch.advanced.collectionName ) ) .withPageable(Pageable.unpaged()) .withFilter(filter) - .build() - val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()3", nsq, log) + val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()3", nqb, log) collectionMultiPropertyResults = elasticSearchTemplate .search( query, CollectionDoc::class.java ) .searchHits @@ -176,20 +170,18 @@ class CustomCollectionQueriesImpl @Autowired constructor( ) } - var query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch().innerHitCollectionUuids", nsq.build(), log) + var query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch().innerHitCollectionUuids", nqb, log) val results = elasticSearchTemplate.search(query, RichSkillDoc::class.java) val innerHitCollectionUuids = results.searchHits.mapNotNull { it.getInnerHits("collections")?.searchHits?.mapNotNull { it.content as CollectionDoc } } .flatten().map { it.uuid }.distinct() - val nsq2: NativeSearchQuery = NativeSearchQueryBuilder().withQuery( - QueryBuilders.termsQuery( - "_id", - (innerHitCollectionUuids + collectionMultiPropertyResults).distinct() - ) - ).withFilter(filter).withPageable(pageable).build() - query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()4", nsq2, log) + val nqb2 = NativeSearchQueryBuilder() + .withQuery( QueryBuilders.termsQuery( "_id", (innerHitCollectionUuids + collectionMultiPropertyResults).distinct() ) ) + .withFilter(filter) + .withPageable(pageable) + query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()4", nqb2, log) return elasticSearchTemplate.search(query, CollectionDoc::class.java) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt index 14baa8ff1..bb2c1455b 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt @@ -7,46 +7,54 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.data.domain.Pageable import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate -import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.query.Query import org.springframework.data.elasticsearch.core.query.StringQuery -/** TODO: Upgrade to ElasticSearch 8.x apis import org.springframework.data.elasticsearch.client.elc.* import co.elastic.clients.elasticsearch._types.FieldValue import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.* import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField import java.util.stream.Collectors - */ interface FindsAllByPublishStatus { val elasticSearchTemplate: ElasticsearchTemplate val javaClass: Class fun findAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): SearchHits { - val nsq: NativeSearchQuery = buildQuery(pageable, publishStatus).build() - val query = createStringQuery("FindsAllByPublishStatus.findAllFilteredByPublishStatus()", nsq, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) + val nqb = buildQuery(pageable, publishStatus) + val query = createStringQuery( + "FindsAllByPublishStatus.findAllFilteredByPublishStatus()", + nqb, + LoggerFactory.getLogger(FindsAllByPublishStatus::class.java) + ) return elasticSearchTemplate.search(query, javaClass) - /* TODO: Upgrade to ElasticSearch 8.x apis - val searchQuery: NativeQuery = buildQuery(pageable, publishStatus).build() + /* TODO: Uncomment to Upgrade to ElasticSearch 8.x apis + val nqb = createQueryBuilder(pageable, publishStatus) + val query = createQuery("FindsAllByPublishStatus.countAllFilteredByPublishStatus()", nqb, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) return elasticSearchTemplate.search(query, javaClass) */ } fun countAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): Long { - val nsq: NativeSearchQuery = buildQuery(pageable, publishStatus).build() - val query = createStringQuery("FindsAllByPublishStatus.countAllFilteredByPublishStatus()", nsq, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) + val nqb = buildQuery(pageable, publishStatus) + val query = createStringQuery( + "FindsAllByPublishStatus.countAllFilteredByPublishStatus()", + nqb, + LoggerFactory.getLogger(FindsAllByPublishStatus::class.java) + ) return elasticSearchTemplate.count(query, javaClass) - /* TODO: Upgrade to ElasticSearch 8.x apis - val searchQuery: NativeQuery = buildQuery(pageable, publishStatus).build() + /* TODO: Uncomment to Upgrade to ElasticSearch 8.x apis + val nqb = createQueryBuilder(pageable, publishStatus) + val query = createQuery("FindsAllByPublishStatus.countAllFilteredByPublishStatus()", nqb, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) return elasticSearchTemplate.count(query, javaClass) - */ + */ } + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createQueryBuilder"), DeprecationLevel.WARNING) fun buildQuery( pageable: Pageable, publishStatus: Set @@ -64,22 +72,24 @@ interface FindsAllByPublishStatus { return nsqb } - fun createStringQuery (msgPrefix: String, nsq: NativeSearchQuery, log: Logger): Query { - val queryStr = nsq.query.toString() - log.info(String.Companion.format("%s:\n%s", msgPrefix, queryStr)) + @Deprecated("", ReplaceWith("createQuery"), DeprecationLevel.WARNING) + fun createStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder, log: Logger): Query { + val queryStr = nqb.build().query.toString() + log.debug(String.Companion.format("%s:\n%s", msgPrefix, queryStr)) return StringQuery(queryStr) } - /* TODO: Upgrade to ElasticSearch v8.X - fun buildQuery( - pageable: Pageable, - publishStatus: Set - ): NativeQueryBuilder { + fun createQuery(msgPrefix: String, nqb: NativeQueryBuilder, log: Logger): Query { + val nq = nqb.build() + log.debug(String.Companion.format("%s:\n%s", msgPrefix, nq.query.toString())) + return nq; + } - val MATCH_ALL = matchAll().build()._toQuery() + fun createQueryBuilder(pageable: Pageable, publishStatus: Set): NativeQueryBuilder { + val MATCH_ALL = matchAll().build()._toQuery() val fieldValues = publishStatus .stream() - .map{ ps -> FieldValue.of(ps.name) } + .map { ps -> FieldValue.of(ps.name) } .collect(Collectors.toList()) var tqf: TermsQueryField = TermsQueryField.Builder().value(fieldValues).build() var terms = terms().field("publishStatus").terms(tqf).build()._toQuery() @@ -90,6 +100,5 @@ interface FindsAllByPublishStatus { .withQuery(MATCH_ALL) .withFilter(filter) } - */ } diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index de2be319f..04394fa02 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -12,6 +12,7 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits @@ -35,6 +36,7 @@ class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSea CustomJobCodeRepository { val log: Logger = LoggerFactory.getLogger(CustomJobCodeRepositoryImpl::class.java) + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith(""), DeprecationLevel.WARNING ) override fun typeAheadSearch(query: String): SearchHits { val disjunctionQuery = JobCodeQueries.multiPropertySearch(query) val nsq = NativeSearchQueryBuilder() @@ -56,11 +58,18 @@ class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSea return limitedPageable } + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createQuery"), DeprecationLevel.WARNING ) private fun createStringQuery (msgPrefix: String, nsq: NativeSearchQuery): Query { val queryStr = nsq.query.toString() - log.info(String.Companion.format("%s:\n%s", msgPrefix, queryStr)) + log.debug(String.Companion.format("%s:\n%s", msgPrefix, queryStr)) return StringQuery(queryStr) } + + private fun createQuery(msgPrefix: String, nqb: NativeQueryBuilder): Query { + val nq = nqb.build() + log.debug(String.Companion.format("%s:\n%s", msgPrefix, nq.query.toString())) + return nq; + } } object JobCodeQueries { diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt index c996d1e3d..44ff73a1d 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt @@ -3,16 +3,21 @@ package edu.wgu.osmt.keyword import edu.wgu.osmt.config.INDEX_KEYWORD_DOC import edu.wgu.osmt.config.SORT_INSENSITIVE import edu.wgu.osmt.elasticsearch.OffsetPageable +import edu.wgu.osmt.jobcode.CustomJobCodeRepositoryImpl import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.search.sort.SortBuilders import org.elasticsearch.search.sort.SortOrder +import org.slf4j.Logger +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates +import org.springframework.data.elasticsearch.core.query.Query import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -28,6 +33,7 @@ interface CustomKeywordRepository { class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchTemplate) : CustomKeywordRepository { + val log: Logger = LoggerFactory.getLogger(CustomJobCodeRepositoryImpl::class.java) override fun typeAheadSearch(query: String, type: KeywordTypeEnum): SearchHits { val limitedPageable: OffsetPageable val bq = QueryBuilders.boolQuery() @@ -60,8 +66,21 @@ class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSea QueryBuilders.matchPhraseQuery( Keyword::value.name, query ).boost(5f) ).minimumShouldMatch(1) } - val searchQuery = StringQuery(nsq.getQuery().toString()) - return elasticSearchTemplate.search(searchQuery, Keyword::class.java) + val query = createStringQuery("CustomKeywordRepositoryImpl.typeAheadSearch()", nsq) + return elasticSearchTemplate.search(query, Keyword::class.java) + } + + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createQuery"), DeprecationLevel.WARNING ) + private fun createStringQuery (msgPrefix: String, nsq: NativeSearchQuery): Query { + val queryStr = nsq.query.toString() + log.debug(String.Companion.format("%s:\n%s", msgPrefix, queryStr)) + return StringQuery(queryStr) + } + + private fun createQuery(msgPrefix: String, nqb: NativeQueryBuilder): Query { + val nq = nqb.build() + log.debug(String.Companion.format("%s:\n%s", msgPrefix, nq.query.toString())) + return nq; } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index 7e3c7729e..e22f48572 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -26,7 +26,6 @@ import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate -import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates @@ -331,8 +330,8 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear pageable: Pageable, collectionId: String? ): SearchHits { - val nsq: NativeSearchQuery = buildQuery(pageable, publishStatus, apiSearch, collectionId).build() - val query = createStringQuery("CustomRichSkillQueriesImpl.byApiSearch()", nsq, log) + val nqb: NativeSearchQueryBuilder = buildQuery(pageable, publishStatus, apiSearch, collectionId) + val query = createStringQuery("CustomRichSkillQueriesImpl.byApiSearch()", nqb, log) return elasticSearchTemplate.search(query, RichSkillDoc::class.java) } @@ -342,8 +341,8 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear pageable: Pageable, collectionId: String? ): Long { - val nsq: NativeSearchQuery = buildQuery(pageable, publishStatus, apiSearch, collectionId).build() - val query = createStringQuery("CustomRichSkillQueriesImpl.countByApiSearch()", nsq, log) + val nqb: NativeSearchQueryBuilder = buildQuery(pageable, publishStatus, apiSearch, collectionId) + val query = createStringQuery("CustomRichSkillQueriesImpl.countByApiSearch()", nqb, log) return elasticSearchTemplate.count(query, RichSkillDoc::class.java) } @@ -353,7 +352,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear apiSearch: ApiSearch, collectionId: String? ): NativeSearchQueryBuilder { - val nsq = NativeSearchQueryBuilder().withPageable(pageable) + val nsq: NativeSearchQueryBuilder = NativeSearchQueryBuilder().withPageable(pageable) val bq = boolQuery() nsq.withQuery(bq) @@ -467,11 +466,10 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear override fun findSimilar(apiSimilaritySearch: ApiSimilaritySearch): SearchHits { val limitedPageable = OffsetPageable(0, 10, null) - val nsq = NativeSearchQueryBuilder() + val nqb = NativeSearchQueryBuilder() .withPageable(limitedPageable) .withQuery( MatchPhraseQueryBuilder(RichSkillDoc::statement.name, apiSimilaritySearch.statement).slop(4)) - .build() - val query = createStringQuery("CustomRichSkillQueriesImpl.findSimilar()", nsq, log) + val query = createStringQuery("CustomRichSkillQueriesImpl.findSimilar()", nqb, log) return elasticSearchTemplate.search(query, RichSkillDoc::class.java) } } diff --git a/api/src/main/resources/config/application-dev.properties b/api/src/main/resources/config/application-dev.properties index baea913b6..2a5b30d9b 100644 --- a/api/src/main/resources/config/application-dev.properties +++ b/api/src/main/resources/config/application-dev.properties @@ -17,3 +17,4 @@ spring.flyway.enabled=false #logging.level.org.springframework.data.elasticsearch.client.WIRE=trace #logging.level.org.elasticsearch.client.RestClient=DEBUG #logging.level.org.springframework=DEBUG +logging.level.edu.wgu.osmt=DEBUG From 2e646aed53b46f070cc8e005334e9fc14e0909bb Mon Sep 17 00:00:00 2001 From: Huey Date: Tue, 26 Sep 2023 16:48:47 -0700 Subject: [PATCH 12/22] Added TODO notes regarding Elasticsearch upgrade related stories in backlog --- api/pom.xml | 4 ++++ .../main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt | 2 +- .../elasticsearch/ElasticsearchClientManager.kt | 1 + .../kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 13 +++++-------- .../kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt | 14 ++++++-------- .../edu/wgu/osmt/richskill/RichSkillEsRepo.kt | 6 +++--- .../kotlin/edu/wgu/osmt/security/SecurityConfig.kt | 3 +++ .../resources/config/application-dev.properties | 2 +- 8 files changed, 24 insertions(+), 21 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 030cde13c..839b27d1c 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -121,11 +121,15 @@ + + org.elasticsearch.client elasticsearch-rest-high-level-client 7.17.8 + + org.apache.logging.log4j log4j-api diff --git a/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt b/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt index 723020098..ca4bf1517 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt @@ -43,7 +43,7 @@ fun SchemaUtils.addMissingColumnsStatementsPublic(vararg tables: Table): List { val disjunctionQuery = JobCodeQueries.multiPropertySearch(query) - val nsq = NativeSearchQueryBuilder() + val nqb = NativeSearchQueryBuilder() .withPageable(createOffsetPageable(query)) .withQuery(disjunctionQuery) .withSort(SortBuilders.fieldSort("${JobCode::code.name}.keyword").order(SortOrder.ASC)) - .build() - val query = createStringQuery("CustomJobCodeRepositoryImpl.typeAheadSearch()", nsq) + val query = createStringQuery("CustomJobCodeRepositoryImpl.typeAheadSearch()", nqb) return elasticSearchTemplate.search(query, JobCode::class.java) } private fun createOffsetPageable(query: String): OffsetPageable { - val limitedPageable: OffsetPageable = if (query.isEmpty()) { + val limitedPageable = if (query.isEmpty()) { OffsetPageable(0, 10000, null) } else { OffsetPageable(0, 20, null) - } return limitedPageable } @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createQuery"), DeprecationLevel.WARNING ) - private fun createStringQuery (msgPrefix: String, nsq: NativeSearchQuery): Query { - val queryStr = nsq.query.toString() + fun createStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder): Query { + val queryStr = nqb.build().query.toString() log.debug(String.Companion.format("%s:\n%s", msgPrefix, queryStr)) return StringQuery(queryStr) } diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt index 44ff73a1d..7735fcb08 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt @@ -37,26 +37,24 @@ class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSea override fun typeAheadSearch(query: String, type: KeywordTypeEnum): SearchHits { val limitedPageable: OffsetPageable val bq = QueryBuilders.boolQuery() - val nsq: NativeSearchQuery + val nqb: NativeSearchQueryBuilder if(query.isEmpty()){ //retrieve all limitedPageable = OffsetPageable(0, 10000, null) - nsq = NativeSearchQueryBuilder() + nqb = NativeSearchQueryBuilder() .withPageable(limitedPageable) .withQuery(bq) .withSort(SortBuilders.fieldSort("${Keyword::value.name}$SORT_INSENSITIVE").order(SortOrder.ASC)) - .build() bq .must(QueryBuilders.termQuery(Keyword::type.name, type.name)) .should( QueryBuilders.matchAllQuery() ) } else { limitedPageable = OffsetPageable(0, 20, null) - nsq = NativeSearchQueryBuilder() + nqb = NativeSearchQueryBuilder() .withPageable(limitedPageable) .withQuery(bq) .withSort(SortBuilders.fieldSort("${Keyword::value.name}$SORT_INSENSITIVE").order(SortOrder.ASC)) - .build() bq .must(QueryBuilders.termQuery(Keyword::type.name, type.name)) .should( @@ -66,13 +64,13 @@ class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSea QueryBuilders.matchPhraseQuery( Keyword::value.name, query ).boost(5f) ).minimumShouldMatch(1) } - val query = createStringQuery("CustomKeywordRepositoryImpl.typeAheadSearch()", nsq) + val query = createStringQuery("CustomKeywordRepositoryImpl.typeAheadSearch()", nqb) return elasticSearchTemplate.search(query, Keyword::class.java) } @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createQuery"), DeprecationLevel.WARNING ) - private fun createStringQuery (msgPrefix: String, nsq: NativeSearchQuery): Query { - val queryStr = nsq.query.toString() + fun createStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder): Query { + val queryStr = nqb.build().query.toString() log.debug(String.Companion.format("%s:\n%s", msgPrefix, queryStr)) return StringQuery(queryStr) } diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index e22f48572..235449e8e 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -330,7 +330,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear pageable: Pageable, collectionId: String? ): SearchHits { - val nqb: NativeSearchQueryBuilder = buildQuery(pageable, publishStatus, apiSearch, collectionId) + val nqb = buildQuery(pageable, publishStatus, apiSearch, collectionId) val query = createStringQuery("CustomRichSkillQueriesImpl.byApiSearch()", nqb, log) return elasticSearchTemplate.search(query, RichSkillDoc::class.java) } @@ -341,7 +341,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear pageable: Pageable, collectionId: String? ): Long { - val nqb: NativeSearchQueryBuilder = buildQuery(pageable, publishStatus, apiSearch, collectionId) + val nqb = buildQuery(pageable, publishStatus, apiSearch, collectionId) val query = createStringQuery("CustomRichSkillQueriesImpl.countByApiSearch()", nqb, log) return elasticSearchTemplate.count(query, RichSkillDoc::class.java) } @@ -352,7 +352,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear apiSearch: ApiSearch, collectionId: String? ): NativeSearchQueryBuilder { - val nsq: NativeSearchQueryBuilder = NativeSearchQueryBuilder().withPageable(pageable) + val nsq = NativeSearchQueryBuilder().withPageable(pageable) val bq = boolQuery() nsq.withQuery(bq) diff --git a/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt b/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt index 296aa94b1..21f89057e 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt @@ -51,6 +51,7 @@ class SecurityConfig { .cors().and() .csrf().disable() .httpBasic().disable() +//TODO DMND-1780 // .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_AUDIT_LOG}", // "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_AUDIT_LOG}", // "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_AUDIT_LOG}").authenticated() @@ -115,6 +116,7 @@ class SecurityConfig { return http.build(); } + //TODO DMND-1780 fun configureForRoles(http: HttpSecurity) { val ADMIN = appConfig.roleAdmin val CURATOR = appConfig.roleCurator @@ -172,6 +174,7 @@ class SecurityConfig { .authorizeHttpRequests { auth -> auth.requestMatchers("/api/**").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) } } + //TODO DMND-1780 fun configureForNoRoles(http: HttpSecurity) { http .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", diff --git a/api/src/main/resources/config/application-dev.properties b/api/src/main/resources/config/application-dev.properties index 2a5b30d9b..e849b9bb2 100644 --- a/api/src/main/resources/config/application-dev.properties +++ b/api/src/main/resources/config/application-dev.properties @@ -11,7 +11,7 @@ app.security.cors.allowedOrigins=${app.baseUrl},${app.frontendUrl} management.endpoint.health.show-details=always -spring.flyway.enabled=false +spring.flyway.enabled=true # Common debuging log levels for OSMT development #logging.level.org.springframework.data.elasticsearch.client.WIRE=trace From b7b5987af91ff7ed49774a0294b689abf7070605 Mon Sep 17 00:00:00 2001 From: Huey Date: Wed, 27 Sep 2023 13:37:36 -0700 Subject: [PATCH 13/22] DMND-1790 Fix NPE and vague http 500 message when wrong Collection Id is entered for Collection Log request. --- .../osmt/collection/CollectionController.kt | 33 +++---------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionController.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionController.kt index 42d24f414..02e5d28d2 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionController.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionController.kt @@ -3,14 +3,7 @@ package edu.wgu.osmt.collection import edu.wgu.osmt.HasAllPaginated import edu.wgu.osmt.RoutePaths import edu.wgu.osmt.api.GeneralApiException -import edu.wgu.osmt.api.model.ApiCollection -import edu.wgu.osmt.api.model.ApiCollectionUpdate -import edu.wgu.osmt.api.model.ApiCollectionV2 -import edu.wgu.osmt.api.model.ApiSearch -import edu.wgu.osmt.api.model.ApiSearchV2 -import edu.wgu.osmt.api.model.ApiSkillListUpdate -import edu.wgu.osmt.api.model.ApiStringListUpdate -import edu.wgu.osmt.api.model.CollectionSortEnum +import edu.wgu.osmt.api.model.* import edu.wgu.osmt.auditlog.AuditLog import edu.wgu.osmt.auditlog.AuditLogRepository import edu.wgu.osmt.auditlog.AuditLogSortEnum @@ -20,34 +13,18 @@ import edu.wgu.osmt.db.PublishStatus import edu.wgu.osmt.elasticsearch.OffsetPageable import edu.wgu.osmt.richskill.RichSkillRepository import edu.wgu.osmt.security.OAuthHelper -import edu.wgu.osmt.task.AppliesToType -import edu.wgu.osmt.task.CsvTask -import edu.wgu.osmt.task.CsvTaskV2 -import edu.wgu.osmt.task.PublishTask -import edu.wgu.osmt.task.PublishTaskV2 -import edu.wgu.osmt.task.RemoveCollectionSkillsTask -import edu.wgu.osmt.task.Task -import edu.wgu.osmt.task.TaskMessageService -import edu.wgu.osmt.task.TaskResult -import edu.wgu.osmt.task.UpdateCollectionSkillsTask -import edu.wgu.osmt.task.XlsxTask +import edu.wgu.osmt.task.* import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.HttpEntity import org.springframework.http.HttpStatus +import org.springframework.http.HttpStatus.NOT_FOUND import org.springframework.http.MediaType import org.springframework.http.ResponseEntity import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.oauth2.jwt.Jwt import org.springframework.stereotype.Controller import org.springframework.transaction.annotation.Transactional -import org.springframework.web.bind.annotation.DeleteMapping -import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.PathVariable -import org.springframework.web.bind.annotation.PostMapping -import org.springframework.web.bind.annotation.RequestBody -import org.springframework.web.bind.annotation.RequestMapping -import org.springframework.web.bind.annotation.RequestParam -import org.springframework.web.bind.annotation.ResponseBody +import org.springframework.web.bind.annotation.* import org.springframework.web.server.ResponseStatusException import org.springframework.web.util.UriComponentsBuilder @@ -346,7 +323,7 @@ class CollectionController @Autowired constructor( @PathVariable uuid: String ): HttpEntity> { val pageable = OffsetPageable(0, Int.MAX_VALUE, AuditLogSortEnum.forValueOrDefault(AuditLogSortEnum.DateDesc.apiValue).sort) - val collection = collectionRepository.findByUUID(uuid) + val collection = collectionRepository.findByUUID(uuid) ?: throw ResponseStatusException(NOT_FOUND, "Collection with id $uuid not ready or not found") val sizedIterable = auditLogRepository.findByTableAndId(CollectionTable.tableName, entityId = collection!!.id.value, offsetPageable = pageable) return ResponseEntity.status(200).body(sizedIterable.toList().map { it.toModel() }) From fcf31703618a1e9403e0b91f6f3d319ee8ffb52f Mon Sep 17 00:00:00 2001 From: Huey Date: Wed, 27 Sep 2023 13:39:15 -0700 Subject: [PATCH 14/22] DMND-1791 Fix NPE and vague http 500 message when wrong Skill UIID is entered for Skill Log request. --- .../wgu/osmt/richskill/RichSkillController.kt | 42 +++---------------- 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillController.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillController.kt index 30e16ebcd..468ab5ad3 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillController.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillController.kt @@ -3,15 +3,7 @@ package edu.wgu.osmt.richskill import edu.wgu.osmt.HasAllPaginated import edu.wgu.osmt.RoutePaths import edu.wgu.osmt.api.GeneralApiException -import edu.wgu.osmt.api.model.ApiSearch -import edu.wgu.osmt.api.model.ApiSearchV2 -import edu.wgu.osmt.api.model.ApiSkill -import edu.wgu.osmt.api.model.ApiSkillUpdate -import edu.wgu.osmt.api.model.ApiSkillUpdateMapper -import edu.wgu.osmt.api.model.ApiSkillUpdateV2 -import edu.wgu.osmt.api.model.ApiSkillV2 -import edu.wgu.osmt.api.model.SkillSortEnum -import edu.wgu.osmt.api.model.SortOrder +import edu.wgu.osmt.api.model.* import edu.wgu.osmt.auditlog.AuditLog import edu.wgu.osmt.auditlog.AuditLogRepository import edu.wgu.osmt.auditlog.AuditLogSortEnum @@ -23,39 +15,17 @@ import edu.wgu.osmt.io.csv.RichSkillCsvExport import edu.wgu.osmt.io.csv.RichSkillCsvExportV2 import edu.wgu.osmt.keyword.KeywordDao import edu.wgu.osmt.security.OAuthHelper -import edu.wgu.osmt.task.AppliesToType -import edu.wgu.osmt.task.CreateSkillsTask -import edu.wgu.osmt.task.CreateSkillsTaskV2 -import edu.wgu.osmt.task.CsvTask -import edu.wgu.osmt.task.CsvTaskV2 -import edu.wgu.osmt.task.ExportSkillsToCsvTask -import edu.wgu.osmt.task.ExportSkillsToCsvTaskV2 -import edu.wgu.osmt.task.ExportSkillsToXlsxTask -import edu.wgu.osmt.task.PublishTask -import edu.wgu.osmt.task.PublishTaskV2 -import edu.wgu.osmt.task.Task -import edu.wgu.osmt.task.TaskMessageService -import edu.wgu.osmt.task.TaskResult -import edu.wgu.osmt.task.XlsxTask +import edu.wgu.osmt.task.* import org.apache.commons.lang3.StringUtils import org.springframework.beans.factory.annotation.Autowired import org.springframework.data.domain.Pageable -import org.springframework.http.HttpEntity -import org.springframework.http.HttpHeaders -import org.springframework.http.HttpStatus -import org.springframework.http.MediaType -import org.springframework.http.ResponseEntity +import org.springframework.http.* +import org.springframework.http.HttpStatus.NOT_FOUND import org.springframework.security.core.annotation.AuthenticationPrincipal import org.springframework.security.oauth2.jwt.Jwt import org.springframework.stereotype.Controller import org.springframework.transaction.annotation.Transactional -import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.PathVariable -import org.springframework.web.bind.annotation.PostMapping -import org.springframework.web.bind.annotation.RequestBody -import org.springframework.web.bind.annotation.RequestMapping -import org.springframework.web.bind.annotation.RequestParam -import org.springframework.web.bind.annotation.ResponseBody +import org.springframework.web.bind.annotation.* import org.springframework.web.server.ResponseStatusException import org.springframework.web.util.UriComponentsBuilder @@ -418,7 +388,7 @@ class RichSkillController @Autowired constructor( @PathVariable uuid: String ): HttpEntity> { val pageable = OffsetPageable(0, Int.MAX_VALUE, AuditLogSortEnum.forValueOrDefault(AuditLogSortEnum.DateDesc.apiValue).sort) - val skill = richSkillRepository.findByUUID(uuid) + val skill = richSkillRepository.findByUUID(uuid) ?: throw ResponseStatusException(NOT_FOUND, "Skill with id $uuid not ready or not found") val sizedIterable = auditLogRepository.findByTableAndId(RichSkillDescriptorTable.tableName, entityId = skill!!.id.value, offsetPageable = pageable) return ResponseEntity.status(200).body(sizedIterable.toList().map { it.toModel() }) From 6efe8faff7fa68a4b0d6724f540b8505be7b9577 Mon Sep 17 00:00:00 2001 From: shnooga Date: Fri, 6 Oct 2023 09:52:23 -0700 Subject: [PATCH 15/22] =?UTF-8?q?DMND-1779=20Utilizing=20application.prope?= =?UTF-8?q?rties=20to=20set=20the=20new=20Easticsearc=E2=80=A6=20(#455)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * DMND-1779 Utilizing application.properties to set the new Easticsearch RestClient in ElasticsearchClientManager.kt * Bug fix, using StringQuery(str) for old ES 7.x causes the the search template to ignore the filter property; fixed for FindsAllByPublishStatus but needs to update the more complex CollectionEsRepo and RichSkilsEsRepo. * Code clean up * More clean up * Update code to handle filter queries and not inadvertently ignore it as before * More cleanup * DMND-1793 Upgrade github work flow to use JDK17 * Created CustomKeywordRepositoryImpl.typeAheadSearchNu() to replace deprecated typeAheadSearch() as sample code for upgrading to ElasticSearch v8.7.X * Add/remove comments for clarifications --------- Co-authored-by: Huey --- .github/workflows/api-tests.yml | 4 +- .github/workflows/maven.yml | 4 +- .../wgu/osmt/collection/CollectionEsRepo.kt | 18 ++- .../ElasticsearchClientManager.kt | 43 +++++- .../edu/wgu/osmt/elasticsearch/EsConfig.kt | 6 +- .../elasticsearch/FindsAllByPublishStatus.kt | 144 ++++++++++-------- .../edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 24 +-- .../edu/wgu/osmt/keyword/KeywordEsRepo.kt | 68 +++++++-- .../edu/wgu/osmt/richskill/RichSkillEsRepo.kt | 61 ++++---- .../resources/config/application.properties | 2 + 10 files changed, 241 insertions(+), 133 deletions(-) diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 13e581d87..3746eed93 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -15,10 +15,10 @@ jobs: steps: - name: Checkout out branch uses: actions/checkout@v2 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v2 with: - java-version: '11' + java-version: '17' distribution: 'adopt' - name: Maven version run: mvn -version diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index b596fd51c..45605844f 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -15,10 +15,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v2 with: - java-version: '11' + java-version: '17' distribution: 'adopt' - name: Maven version run: mvn -version diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index fdb04c00b..2261cb731 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -27,6 +27,10 @@ import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories +/** + * This have been partially converted to use the ElasticSearch 8.7.X apis. Need to do full conversion to use + * the v8.7.x ES Java API client, https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html + */ interface CustomCollectionQueries : FindsAllByPublishStatus { val richSkillEsRepo: RichSkillEsRepo @@ -81,6 +85,9 @@ class CustomCollectionQueriesImpl @Autowired constructor( } } + /** + * TODO upgrade to ElasticSearch v8.7.x api style; see KeywordEsRepo.kt & FindsAllByPublishStatus.kt + */ override fun byApiSearch( apiSearch: ApiSearch, publishStatus: Set, @@ -88,6 +95,7 @@ class CustomCollectionQueriesImpl @Autowired constructor( ): SearchHits { val nqb = NativeSearchQueryBuilder().withPageable(Pageable.unpaged()) val bq = QueryBuilders.boolQuery() + //TODO Replace with FindsAllByPublishStatus.createTermsDslQuery(publishStatus.name, publishStatus.map { ps -> ps.toString() }) val filter = BoolQueryBuilder().must( QueryBuilders.termsQuery( RichSkillDoc::publishStatus.name, @@ -120,7 +128,7 @@ class CustomCollectionQueriesImpl @Autowired constructor( .withPageable(Pageable.unpaged()) .withFilter(filter) // search on collection specific properties - val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()1", nqb, log) + val query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()1", nqb, log) collectionMultiPropertyResults = elasticSearchTemplate .search(query, CollectionDoc::class.java) .searchHits @@ -135,7 +143,7 @@ class CustomCollectionQueriesImpl @Autowired constructor( .withQuery( simpleQueryStringQuery(apiSearch.advanced.collectionName).field("${CollectionDoc::name.name}.raw").defaultOperator(Operator.AND) ) .withPageable(Pageable.unpaged()) .withFilter(filter) - val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()2", nqb, log) + val query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()2", nqb, log) collectionMultiPropertyResults = elasticSearchTemplate .search( query, CollectionDoc::class.java ) .searchHits @@ -145,7 +153,7 @@ class CustomCollectionQueriesImpl @Autowired constructor( .withQuery( matchPhrasePrefixQuery( CollectionDoc::name.name, apiSearch.advanced.collectionName ) ) .withPageable(Pageable.unpaged()) .withFilter(filter) - val query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()3", nqb, log) + val query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()3", nqb, log) collectionMultiPropertyResults = elasticSearchTemplate .search( query, CollectionDoc::class.java ) .searchHits @@ -170,7 +178,7 @@ class CustomCollectionQueriesImpl @Autowired constructor( ) } - var query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch().innerHitCollectionUuids", nqb, log) + var query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch().innerHitCollectionUuids", nqb, log) val results = elasticSearchTemplate.search(query, RichSkillDoc::class.java) val innerHitCollectionUuids = @@ -181,7 +189,7 @@ class CustomCollectionQueriesImpl @Autowired constructor( .withQuery( QueryBuilders.termsQuery( "_id", (innerHitCollectionUuids + collectionMultiPropertyResults).distinct() ) ) .withFilter(filter) .withPageable(pageable) - query = createStringQuery("CustomCollectionQueriesImpl.byApiSearch()4", nqb2, log) + query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()4", nqb2, log) return elasticSearchTemplate.search(query, CollectionDoc::class.java) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt index 3562c800f..eef71f2b5 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt @@ -2,11 +2,13 @@ package edu.wgu.osmt.elasticsearch import co.elastic.clients.elasticsearch.ElasticsearchClient import co.elastic.clients.json.jackson.JacksonJsonpMapper -import co.elastic.clients.transport.ElasticsearchTransport import co.elastic.clients.transport.rest_client.RestClientTransport import org.apache.http.HttpHost +import org.apache.http.auth.AuthScope +import org.apache.http.auth.UsernamePasswordCredentials +import org.apache.http.client.CredentialsProvider +import org.apache.http.impl.client.BasicCredentialsProvider import org.elasticsearch.client.RestClient -import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -16,6 +18,7 @@ import org.springframework.data.convert.WritingConverter import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate import org.springframework.data.elasticsearch.core.convert.ElasticsearchCustomConversions import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories +import org.springframework.util.StringUtils import java.time.LocalDateTime import java.time.format.DateTimeFormatter import java.util.* @@ -30,12 +33,10 @@ class ElasticsearchClientManager { @Override @Bean fun elasticSearchClient(): ElasticsearchClient { - //TODO DMND-1779 - val builder = RestClient.builder(HttpHost("localhost", 9200, "http")) - //val httpClientConfigCallback: HttpClientConfigCallback = HttpClientConfigCallbackImpl() - //builder.setHttpClientConfigCallback(httpClientConfigCallback) - val restClient = builder.build() - val transport: ElasticsearchTransport = RestClientTransport(restClient, JacksonJsonpMapper()) + val transport = RestClientTransport( + createRestClient(), + JacksonJsonpMapper() + ) return ElasticsearchClient(transport) } @@ -81,4 +82,30 @@ class ElasticsearchClientManager { return UUID.fromString(source) } } + + private fun createRestClient(): RestClient { + val restClientBuilder = RestClient.builder(createHttpHost()) + val credentialsProvider = getCredentialsProvider() + + credentialsProvider?.let { + restClientBuilder.setHttpClientConfigCallback { b -> b.setDefaultCredentialsProvider(it) } + } + return restClientBuilder.build() + } + + private fun createHttpHost(): HttpHost { + val params = StringUtils.split(esConfig.uri, ":") + return HttpHost(params!![0], params[1].toInt()) + } + + private fun getCredentialsProvider(): CredentialsProvider? { + if (esConfig.username.isNullOrBlank() || esConfig.password.isNullOrBlank()) { + return null + } + + val credentialsProvider = BasicCredentialsProvider() + val credential = UsernamePasswordCredentials(esConfig.username, esConfig.password) + credentialsProvider.setCredentials(AuthScope.ANY, credential) + return credentialsProvider + } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/EsConfig.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/EsConfig.kt index c57c55ae5..c019b5e94 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/EsConfig.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/EsConfig.kt @@ -3,4 +3,8 @@ package edu.wgu.osmt.elasticsearch import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "es", ignoreInvalidFields = true) -data class EsConfig(val uri: String) +data class EsConfig( + val uri: String, + val username: String?, + val password: String? +) diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt index b177ec397..ef9df1ca4 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt @@ -1,103 +1,119 @@ package edu.wgu.osmt.elasticsearch +import co.elastic.clients.elasticsearch._types.FieldValue +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.* +import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField import edu.wgu.osmt.db.PublishStatus -import org.elasticsearch.index.query.BoolQueryBuilder -import org.elasticsearch.index.query.QueryBuilders +import edu.wgu.osmt.richskill.RichSkillDoc import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.data.domain.Pageable import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate +import org.springframework.data.elasticsearch.client.elc.NativeQuery +import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.query.Query import org.springframework.data.elasticsearch.core.query.StringQuery - -import org.springframework.data.elasticsearch.client.elc.* -import co.elastic.clients.elasticsearch._types.FieldValue -import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.* -import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField import java.util.stream.Collectors +/** + * This have been partially converted to use the ElasticSearch 8.7.X apis. Need to do full conversion to use + * the v8.7.x ES Java API client, https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html + */ interface FindsAllByPublishStatus { val elasticSearchTemplate: ElasticsearchTemplate val javaClass: Class fun findAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): SearchHits { - val nqb = buildQuery(pageable, publishStatus) - val query = createStringQuery( - "FindsAllByPublishStatus.findAllFilteredByPublishStatus()", - nqb, - LoggerFactory.getLogger(FindsAllByPublishStatus::class.java) - ) - return elasticSearchTemplate.search(query, javaClass) - - /* TODO: Uncomment to Upgrade to ElasticSearch 8.x apis val nqb = createQueryBuilder(pageable, publishStatus) - val query = createQuery("FindsAllByPublishStatus.countAllFilteredByPublishStatus()", nqb, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) + val query = createQuery("FindsAllByPublishStatus.findAllFilteredByPublishStatus()", nqb, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) return elasticSearchTemplate.search(query, javaClass) - */ } fun countAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): Long { - val nqb = buildQuery(pageable, publishStatus) - val query = createStringQuery( - "FindsAllByPublishStatus.countAllFilteredByPublishStatus()", - nqb, - LoggerFactory.getLogger(FindsAllByPublishStatus::class.java) - ) - return elasticSearchTemplate.count(query, javaClass) - - /* TODO: Uncomment to Upgrade to ElasticSearch 8.x apis val nqb = createQueryBuilder(pageable, publishStatus) val query = createQuery("FindsAllByPublishStatus.countAllFilteredByPublishStatus()", nqb, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) return elasticSearchTemplate.count(query, javaClass) - */ - } - - @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createQueryBuilder"), DeprecationLevel.WARNING) - fun buildQuery( - pageable: Pageable, - publishStatus: Set - ): NativeSearchQueryBuilder { - val nsqb = NativeSearchQueryBuilder().withPageable(pageable) - nsqb.withQuery(QueryBuilders.matchAllQuery()) - nsqb.withFilter( - BoolQueryBuilder().should( - QueryBuilders.termsQuery( - "publishStatus", - publishStatus.map { ps -> ps.toString() } - ) - ) - ) - return nsqb } - @Deprecated("", ReplaceWith("createQuery"), DeprecationLevel.WARNING) - fun createStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder, log: Logger): Query { - val queryStr = nqb.build().query.toString() - log.debug(String.Companion.format("%s:\n%s", msgPrefix, queryStr)) - return StringQuery(queryStr) + fun createQueryBuilder(pageable: Pageable, publishStatus: Set): NativeQueryBuilder { + val MATCH_ALL = matchAll().build()._toQuery() + var filterValues = publishStatus + .stream() + .map { ps -> ps.name} + .collect(Collectors.toList()) + var filter = createTermsDslQuery(false, RichSkillDoc::publishStatus.name, filterValues) + return NativeQueryBuilder() + .withPageable(pageable) + .withQuery(MATCH_ALL) + .withFilter(filter) } fun createQuery(msgPrefix: String, nqb: NativeQueryBuilder, log: Logger): Query { - val nq = nqb.build() - log.debug(String.Companion.format("%s:\n%s", msgPrefix, nq.query.toString())) - return nq; + val query = nqb.build() + log.debug(String.Companion.format("\n%s query:\n\t\t%s", msgPrefix, query.query.toString())) + log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, query.filter.toString())) + return query; } - fun createQueryBuilder(pageable: Pageable, publishStatus: Set): NativeQueryBuilder { - val MATCH_ALL = matchAll().build()._toQuery() - val fieldValues = publishStatus + /** + * Create a query_dsl.Query instance that ElasticSearchTemplate v8.x mandates for filtering. + */ + fun createTermsDslQuery(andFlag: Boolean, fieldName: String, filterValues: List): co.elastic.clients.elasticsearch._types.query_dsl.Query { + val values = filterValues .stream() - .map { ps -> FieldValue.of(ps.name) } + .map { FieldValue.of(it) } .collect(Collectors.toList()) - var tqf: TermsQueryField = TermsQueryField.Builder().value(fieldValues).build() - var terms = terms().field("publishStatus").terms(tqf).build()._toQuery() - var filter = bool().should(terms).build()._toQuery() + val tqf = TermsQueryField.Builder() + .value(values) + .build() + val terms = terms() + .field(fieldName) + .terms(tqf) + .build() + ._toQuery() + /* Short hand version https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html + val terms2 = terms { qb -> qb.field(fieldName).terms(tqf) } + return bool { qb -> if (andFlag) + qb.must(terms) + else + qb.should(terms) } + */ - return NativeQueryBuilder() - .withPageable(pageable) - .withQuery(MATCH_ALL) + return bool() + .let { if (andFlag) it.must(terms) + else it.should(terms) } + .build() + ._toQuery() + } + + fun createTermsDslQuery(fieldName: String, filterValues: List): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return createTermsDslQuery(true, fieldName, filterValues) + } + + @Deprecated("", ReplaceWith("convertToNativeQuery"), DeprecationLevel.WARNING) + fun convertToStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder, log: Logger): Query { + val query = nqb.build() + log.debug(String.Companion.format("\n%s query:\n\t\t%s", msgPrefix, query.query.toString())) + log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, query.filter.toString())) + //NOTE: this causes us to lose the filter query + return StringQuery(query.query.toString()) + } + + /** + * Stepping stone to 100% migration to ES v8.7.x apis; see KeywordEsRepo.kt + */ + fun convertToNativeQuery(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, nsqb: NativeSearchQueryBuilder, msgPrefix: String, log: Logger): Query { + val oldQuery = nsqb.build() + val nuQuery = NativeQuery.builder() .withFilter(filter) + .withQuery(StringQuery(oldQuery.query.toString())) + .withPageable(pageable) + .build() + log.debug(String.Companion.format("\n%s springDataQuery:\n\t\t%s", msgPrefix, (nuQuery.springDataQuery as StringQuery).source)) + log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, nuQuery.filter.toString())) + return nuQuery } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index e95feeb3d..23687e873 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -22,6 +22,10 @@ import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories +/** + * This have been partially converted to use the ElasticSearch 8.7.X apis. Need to do full conversion to use + * the v8.x ES Java API client, https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html + */ interface CustomJobCodeRepository { val elasticSearchTemplate: ElasticsearchTemplate fun typeAheadSearch(query: String): SearchHits @@ -35,6 +39,9 @@ class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSea CustomJobCodeRepository { val log: Logger = LoggerFactory.getLogger(CustomJobCodeRepositoryImpl::class.java) + /** + * TODO upgrade to ElasticSearch v8.7.x api style; see KeywordEsRepo.kt & FindsAllByPublishStatus.kt + */ @Deprecated("Upgrade to ES v8.x queries", ReplaceWith(""), DeprecationLevel.WARNING ) override fun typeAheadSearch(query: String): SearchHits { val disjunctionQuery = JobCodeQueries.multiPropertySearch(query) @@ -55,21 +62,18 @@ class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSea return limitedPageable } - @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createQuery"), DeprecationLevel.WARNING ) fun createStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder): Query { - val queryStr = nqb.build().query.toString() - log.debug(String.Companion.format("%s:\n%s", msgPrefix, queryStr)) - return StringQuery(queryStr) - } - - private fun createQuery(msgPrefix: String, nqb: NativeQueryBuilder): Query { - val nq = nqb.build() - log.debug(String.Companion.format("%s:\n%s", msgPrefix, nq.query.toString())) - return nq; + val query = nqb.build() + log.debug(String.Companion.format("\n%s query:\n\t\t%s", msgPrefix, query.query.toString())) + log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, query.filter.toString())) + //NOTE: this is causing us to lose the filter query + return StringQuery(query.query.toString()) } } +@Deprecated("Upgrade to ES v8.x queries", ReplaceWith("Replacement method"), DeprecationLevel.WARNING ) object JobCodeQueries { + //TODO Convert to ES v8.7.x apis and return the newer BoolQuery.Builder instance; see KeywordEsRep.kt fun multiPropertySearch(query: String, parentDocPath: String? = null): BoolQueryBuilder { val disjunctionQuery = disMaxQuery() val path = parentDocPath?.let { "${it}." } ?: "" diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt index f1f498fb7..1b7f285cf 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt @@ -1,5 +1,6 @@ package edu.wgu.osmt.keyword +import co.elastic.clients.elasticsearch._types.query_dsl.* import edu.wgu.osmt.config.INDEX_KEYWORD_DOC import edu.wgu.osmt.config.SORT_INSENSITIVE import edu.wgu.osmt.elasticsearch.OffsetPageable @@ -12,8 +13,7 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate -import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder -import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery +import org.springframework.data.elasticsearch.client.elc.NativeQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates @@ -31,15 +31,21 @@ interface CustomKeywordRepository { } } +/** + * This have been partially converted to use the ElasticSearch 8.7.X apis. For full conversion + * replace typeAheadSearch() with TypeAheadSearchNu() + */ class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchTemplate) : CustomKeywordRepository { val log: Logger = LoggerFactory.getLogger(CustomJobCodeRepositoryImpl::class.java) - override fun typeAheadSearch(query: String, type: KeywordTypeEnum): SearchHits { + + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("typeAheadSearchNu"), DeprecationLevel.WARNING ) + override fun typeAheadSearch(searchStr: String, type: KeywordTypeEnum): SearchHits { val limitedPageable: OffsetPageable val bq = QueryBuilders.boolQuery() val nqb: NativeSearchQueryBuilder - if(query.isEmpty()){ //retrieve all + if(searchStr.isEmpty()){ //retrieve all limitedPageable = OffsetPageable(0, 10000, null) nqb = NativeSearchQueryBuilder() .withPageable(limitedPageable) @@ -58,27 +64,59 @@ class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSea bq .must(QueryBuilders.termQuery(Keyword::type.name, type.name)) .should( - QueryBuilders.matchBoolPrefixQuery( Keyword::value.name, query ) + QueryBuilders.matchBoolPrefixQuery( Keyword::value.name, searchStr ) ) .should( - QueryBuilders.matchPhraseQuery( Keyword::value.name, query ).boost(5f) + QueryBuilders.matchPhraseQuery( Keyword::value.name, searchStr ).boost(5f) ).minimumShouldMatch(1) } + val query = createStringQuery("CustomKeywordRepositoryImpl.typeAheadSearch()", nqb) return elasticSearchTemplate.search(query, Keyword::class.java) } - @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createQuery"), DeprecationLevel.WARNING ) - fun createStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder): Query { - val queryStr = nqb.build().query.toString() - log.debug(String.Companion.format("%s:\n%s", msgPrefix, queryStr)) - return StringQuery(queryStr) + /** + * Uses the latest ES 8.7.x Java Client API + */ + fun typeAheadSearchNu(searchStr: String, type: KeywordTypeEnum): SearchHits { + val pageable: OffsetPageable + val criteria: co.elastic.clients.elasticsearch._types.query_dsl.Query + + if (searchStr.isEmpty()) { + pageable = OffsetPageable(0, 10000, null) + criteria = searchAll(type) + } else { + pageable = OffsetPageable(0, 20, null) + criteria = searchSpecific(searchStr, type) + } + log.debug(String.Companion.format("\ntypeAheadSearchNu query:\n\t\t%s", criteria.bool().toString())) + return elasticSearchTemplate.search( NativeQuery.builder() + .withPageable(pageable) + .withQuery(criteria).build(), Keyword::class.java ) } - private fun createQuery(msgPrefix: String, nqb: NativeQueryBuilder): Query { - val nq = nqb.build() - log.debug(String.Companion.format("%s:\n%s", msgPrefix, nq.query.toString())) - return nq; + fun searchAll(type: KeywordTypeEnum): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.bool { builder: BoolQuery.Builder -> + builder + .must(co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.term { qt: TermQuery.Builder -> qt.field(Keyword::type.name).value(type.name) } ) + .should(co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.matchAll { q : MatchAllQuery.Builder -> q } ) } + } + + fun searchSpecific(searchStr: String, type: KeywordTypeEnum): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.bool { builder: BoolQuery.Builder -> + builder + .must(co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.term { qt: TermQuery.Builder -> qt.field(Keyword::type.name).value(type.name) } ) + .should(co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.matchBoolPrefix { q : MatchBoolPrefixQuery.Builder -> q.field(Keyword::value.name).query(searchStr)} ) + .should(co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.matchPhrase { q : MatchPhraseQuery.Builder -> q.field(Keyword::value.name).query(searchStr)} ) + .minimumShouldMatch("1") } + } + + fun createStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder): Query { + val query = nqb.build() + log.debug(String.Companion.format("\n%s query:\n\t\t%s", msgPrefix, query.query.toString())) + log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, query.filter.toString())) + //NOTE: this is causing us to lose the filter query + return StringQuery(query.query.toString()) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index 091eee9bd..0f5008322 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -35,6 +35,10 @@ import org.springframework.security.oauth2.jwt.Jwt const val collectionsUuid = "collections.uuid" +/** + * This have been partially converted to use the ElasticSearch 8.7.X apis. Need to do full conversion to use + * the v8.7.x ES Java API client, https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html + */ interface CustomRichSkillQueries : FindsAllByPublishStatus { fun getUuidsFromApiSearch( apiSearch: ApiSearch, @@ -330,8 +334,13 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear pageable: Pageable, collectionId: String? ): SearchHits { - val nqb = buildQuery(pageable, publishStatus, apiSearch, collectionId) - val query = createStringQuery("CustomRichSkillQueriesImpl.byApiSearch()", nqb, log) + val query = convertToNativeQuery( + pageable, + createTermsDslQuery(RichSkillDoc::publishStatus.name, publishStatus.map { ps -> ps.toString() }), + buildQuery(publishStatus, apiSearch, collectionId), + "CustomRichSkillQueriesImpl.byApiSearch()", + log + ) return elasticSearchTemplate.search(query, RichSkillDoc::class.java) } @@ -341,30 +350,28 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear pageable: Pageable, collectionId: String? ): Long { - val nqb = buildQuery(pageable, publishStatus, apiSearch, collectionId) - val query = createStringQuery("CustomRichSkillQueriesImpl.countByApiSearch()", nqb, log) + val query = convertToNativeQuery( + pageable, + createTermsDslQuery(RichSkillDoc::publishStatus.name, publishStatus.map { ps -> ps.toString() }), + buildQuery(publishStatus, apiSearch, collectionId), + "CustomRichSkillQueriesImpl.countByApiSearch()", + log + ) return elasticSearchTemplate.count(query, RichSkillDoc::class.java) } - fun buildQuery( - pageable: Pageable, + /** + * TODO upgrade to ElasticSearch v8.7.x api style; see KeywordEsRepo.kt & FindsAllByPublishStatus.kt + */ + private fun buildQuery( publishStatus: Set, apiSearch: ApiSearch, collectionId: String? ): NativeSearchQueryBuilder { - val nsq = NativeSearchQueryBuilder().withPageable(pageable) + val nsqb = NativeSearchQueryBuilder() val bq = boolQuery() - nsq.withQuery(bq) - nsq.withFilter( - BoolQueryBuilder().must( - termsQuery( - RichSkillDoc::publishStatus.name, - publishStatus.map { ps -> ps.toString() } - ) - ) - ) - + nsqb.withQuery(bq) apiSearch.filtered?.let { generateBoolQueriesFromApiSearchWithFilters(bq, it, publishStatus) } // treat the presence of query property to mean multi field search with that term @@ -439,7 +446,8 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear var apiSearchUuids = apiSearch.uuids?.filterNotNull()?.filter { x: String? -> x != "" } if (!apiSearchUuids.isNullOrEmpty()) { - nsq.withFilter( + nsqb.withFilter( + //TODO Replace with FindsAllByPublishStatus.createTermsDslQuery(uuid.name, apiSearchUuids) BoolQueryBuilder().must( termsQuery( RichSkillDoc::uuid.name, @@ -452,8 +460,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear bq.must( nestedQuery( RichSkillDoc::collections.name, - boolQuery() - .must(matchQuery(collectionsUuid, collectionId)), + boolQuery().must(matchQuery(collectionsUuid, collectionId)), ScoreMode.Avg ) ) @@ -461,15 +468,17 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear } } - return nsq + return nsqb } override fun findSimilar(apiSimilaritySearch: ApiSimilaritySearch): SearchHits { - val limitedPageable = OffsetPageable(0, 10, null) - val nqb = NativeSearchQueryBuilder() - .withPageable(limitedPageable) - .withQuery( MatchPhraseQueryBuilder(RichSkillDoc::statement.name, apiSimilaritySearch.statement).slop(4)) - val query = createStringQuery("CustomRichSkillQueriesImpl.findSimilar()", nqb, log) + val query = convertToNativeQuery( + OffsetPageable(0, 10, null), + null, + NativeSearchQueryBuilder().withQuery( MatchPhraseQueryBuilder(RichSkillDoc::statement.name, apiSimilaritySearch.statement).slop(4)), + "CustomRichSkillQueriesImpl.findSimilar()", + log + ) return elasticSearchTemplate.search(query, RichSkillDoc::class.java) } } diff --git a/api/src/main/resources/config/application.properties b/api/src/main/resources/config/application.properties index f9bfaffe0..b1233e9fa 100644 --- a/api/src/main/resources/config/application.properties +++ b/api/src/main/resources/config/application.properties @@ -14,6 +14,8 @@ spring.datasource.url=${db.composedUrl} # Elasticsearch es.uri=${ELASTICSEARCH_URI:localhost:9200} +#es.username= +#es.password= # Redis redis.uri=${REDIS_URI:localhost:6379} From ef7dd5407a0f3cf3eaff26bef5495e120b8aaafb Mon Sep 17 00:00:00 2001 From: Jesus Bautista Gudino <103442937+jesusgudino@users.noreply.github.com> Date: Thu, 12 Oct 2023 14:55:16 -0600 Subject: [PATCH 16/22] Feature/dmnd 1780 security config (#459) * fix SecurityConfig.kt methods * fix http:// url ES clientmanager --- .../ElasticsearchClientManager.kt | 10 +- .../edu/wgu/osmt/security/SecurityConfig.kt | 285 +++++++++--------- 2 files changed, 155 insertions(+), 140 deletions(-) diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt index eef71f2b5..b47dba227 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticsearchClientManager.kt @@ -94,8 +94,14 @@ class ElasticsearchClientManager { } private fun createHttpHost(): HttpHost { - val params = StringUtils.split(esConfig.uri, ":") - return HttpHost(params!![0], params[1].toInt()) + val scheme = StringUtils.split(esConfig.uri, "://") + if(scheme.isNullOrEmpty()){ + val params = StringUtils.split(esConfig.uri, ":") + return HttpHost(params!![0], params[1].toInt()) + } else { + val params = StringUtils.split(scheme!![1], ":") + return HttpHost(params!![0], params[1].toInt(), scheme!![0]) + } } private fun getCredentialsProvider(): CredentialsProvider? { diff --git a/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt b/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt index 21f89057e..bba44aeb3 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt @@ -46,77 +46,78 @@ class SecurityConfig { lateinit var returnUnauthorized: ReturnUnauthorized @Bean - fun filterChain(http: HttpSecurity) : SecurityFilterChain { + fun securityFilterChain(http: HttpSecurity) : SecurityFilterChain { http .cors().and() .csrf().disable() .httpBasic().disable() -//TODO DMND-1780 -// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_AUDIT_LOG}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_AUDIT_LOG}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_AUDIT_LOG}").authenticated() -// } -// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_AUDIT_LOG}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_AUDIT_LOG}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_AUDIT_LOG}").authenticated() } -// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_SKILLS}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_SKILLS}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_SKILLS}").authenticated() } -// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_BATCH}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_BATCH}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_BATCH}").authenticated() } -// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_JOBCODES_PATH}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_JOBCODES_PATH}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_JOBCODES_PATH}").authenticated() } -// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_KEYWORDS_PATH}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_KEYWORDS_PATH}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_KEYWORDS_PATH}").authenticated() } -// -// // public search endpoints -// .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_SKILLS}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_SKILLS}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_SKILLS}").permitAll() } -// .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_COLLECTIONS}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_COLLECTIONS}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_COLLECTIONS}").permitAll() } -// -// // public canonical URL endpoints -// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_DETAIL}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_DETAIL}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_DETAIL}").permitAll() } -// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_DETAIL}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_DETAIL}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_DETAIL}").permitAll() } -// -// .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS}").permitAll() } -// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CSV}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CSV}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CSV}").permitAll() } -// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_TEXT}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_TEXT}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_TEXT}").permitAll() } -// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_XLSX}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_XLSX}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_XLSX}").permitAll() } -// .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_MEDIA}", -// "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_MEDIA}", -// "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_MEDIA}").permitAll() } -// + .authorizeHttpRequests { auth -> + auth + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_AUDIT_LOG}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_AUDIT_LOG}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_AUDIT_LOG}").authenticated() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_AUDIT_LOG}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_AUDIT_LOG}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_AUDIT_LOG}").authenticated() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_SKILLS}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_SKILLS}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_SKILLS}").authenticated() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_BATCH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_BATCH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_BATCH}").authenticated() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_JOBCODES_PATH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_JOBCODES_PATH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_JOBCODES_PATH}").authenticated() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_KEYWORDS_PATH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_KEYWORDS_PATH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_KEYWORDS_PATH}").authenticated() + + // public search endpoints + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_SKILLS}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_SKILLS}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_SKILLS}").permitAll() + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SEARCH_COLLECTIONS}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SEARCH_COLLECTIONS}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SEARCH_COLLECTIONS}").permitAll() + + // public canonical URL endpoints + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_DETAIL}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_DETAIL}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_DETAIL}").permitAll() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_DETAIL}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_DETAIL}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_DETAIL}").permitAll() + + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS}").permitAll() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CSV}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CSV}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CSV}").permitAll() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_TEXT}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_TEXT}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_TEXT}").permitAll() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_XLSX}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_XLSX}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_XLSX}").permitAll() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.TASK_DETAIL_MEDIA}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.TASK_DETAIL_MEDIA}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.TASK_DETAIL_MEDIA}").permitAll() + } + .exceptionHandling().authenticationEntryPoint(returnUnauthorized) .and().oauth2Login().successHandler(redirectToFrontend) .and().oauth2ResourceServer().jwt() -// -// if (appConfig.enableRoles) { -// configureForRoles(http) -// } else { -// configureForNoRoles(http) -// } + + if (appConfig.enableRoles) { + configureForRoles(http) + } else { + configureForNoRoles(http) + } + return http.build(); } - //TODO DMND-1780 fun configureForRoles(http: HttpSecurity) { val ADMIN = appConfig.roleAdmin val CURATOR = appConfig.roleCurator @@ -124,94 +125,102 @@ class SecurityConfig { val READ = appConfig.scopeRead if (appConfig.allowPublicLists) { - http - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").permitAll() } - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").permitAll() } + http.authorizeHttpRequests { auth -> + auth + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").permitAll() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").permitAll() + } } else { - http - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) } - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) } + http.authorizeHttpRequests { auth -> + auth + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) } + } - http - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_UPDATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_UPDATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) } - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_CREATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_CREATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_CREATE}").hasAnyAuthority(ADMIN, CURATOR) } - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_PUBLISH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_PUBLISH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_PUBLISH}").hasAnyAuthority(ADMIN) } + http.authorizeHttpRequests { auth -> + auth + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_UPDATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_UPDATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_CREATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_CREATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_CREATE}").hasAnyAuthority(ADMIN, CURATOR) + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_PUBLISH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_PUBLISH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_PUBLISH}").hasAnyAuthority(ADMIN) - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CREATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CREATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CREATE}").hasAnyAuthority(ADMIN, CURATOR) } - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_PUBLISH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_PUBLISH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_PUBLISH}").hasAnyAuthority(ADMIN) } - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_UPDATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_UPDATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) } - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS_UPDATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS_UPDATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) } - .authorizeHttpRequests { auth -> auth.requestMatchers(DELETE, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_REMOVE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_REMOVE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_REMOVE}").hasAnyAuthority(ADMIN) } - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.WORKSPACE_PATH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.WORKSPACE_PATH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.WORKSPACE_PATH}").hasAnyAuthority(ADMIN, CURATOR) } + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CREATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CREATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CREATE}").hasAnyAuthority(ADMIN, CURATOR) + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_PUBLISH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_PUBLISH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_PUBLISH}").hasAnyAuthority(ADMIN) + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_UPDATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_UPDATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS_UPDATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS_UPDATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS_UPDATE}").hasAnyAuthority(ADMIN, CURATOR) + .requestMatchers(DELETE, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_REMOVE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_REMOVE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_REMOVE}").hasAnyAuthority(ADMIN) + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.WORKSPACE_PATH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.WORKSPACE_PATH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.WORKSPACE_PATH}").hasAnyAuthority(ADMIN, CURATOR) - .authorizeHttpRequests { auth -> auth.requestMatchers("/api/**").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) } + .requestMatchers("/api/**").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) + .requestMatchers("/**").permitAll() + } } - //TODO DMND-1780 fun configureForNoRoles(http: HttpSecurity) { - http - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").permitAll() } - .authorizeHttpRequests { auth -> auth.requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").permitAll() } + http.authorizeHttpRequests { auth -> + auth + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").permitAll() + .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").permitAll() - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_UPDATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_UPDATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_UPDATE}").authenticated() } - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_CREATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_CREATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_CREATE}").authenticated() } - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_PUBLISH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_PUBLISH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_PUBLISH}").authenticated() } + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_UPDATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_UPDATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_UPDATE}").authenticated() + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_CREATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_CREATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_CREATE}").authenticated() + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILL_PUBLISH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILL_PUBLISH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILL_PUBLISH}").authenticated() - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CREATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CREATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CREATE}").authenticated() } - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_PUBLISH}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_PUBLISH}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_PUBLISH}").authenticated() } - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_UPDATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_UPDATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_UPDATE}").authenticated() } - .authorizeHttpRequests { auth -> auth.requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS_UPDATE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS_UPDATE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS_UPDATE}").authenticated() } - .authorizeHttpRequests { auth -> auth.requestMatchers(DELETE, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_REMOVE}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_REMOVE}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_REMOVE}").denyAll() } + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_CREATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_CREATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_CREATE}").authenticated() + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_PUBLISH}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_PUBLISH}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_PUBLISH}").authenticated() + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_UPDATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_UPDATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_UPDATE}").authenticated() + .requestMatchers(POST, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_SKILLS_UPDATE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_SKILLS_UPDATE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_SKILLS_UPDATE}").authenticated() + .requestMatchers(DELETE, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTION_REMOVE}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTION_REMOVE}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTION_REMOVE}").denyAll() - // fall-through - .authorizeHttpRequests { auth -> auth.requestMatchers("/api/**").permitAll() } + // fall-through + .requestMatchers("/**").permitAll() + } } @Bean From f1a5684c92d4f26ba17708fa588681a4e19cbb9c Mon Sep 17 00:00:00 2001 From: shnooga Date: Fri, 13 Oct 2023 16:39:12 -0700 Subject: [PATCH 17/22] Feature/dmnd 1782 db config (#460) * Replacing the buggy convertToStringQuery method because it is missing the PageAble property. * Adding methods for migration to ElasticSearch v8.7.X * Clean up * Clean up * Code refactor for readability and ease of upgrade to ElasticSearch v8.7.x * Code refactoring to facilitate future migration to Elastic Search v8.7.X API * DMND-1782 working --------- Co-authored-by: Huey --- .../wgu/osmt/collection/CollectionEsRepo.kt | 99 ++++----- .../kotlin/edu/wgu/osmt/db/ExposedHelper.kt | 44 ++-- .../elasticsearch/FindsAllByPublishStatus.kt | 97 ++++++--- .../edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 2 +- .../edu/wgu/osmt/richskill/RichSkillEsRepo.kt | 196 ++++++++++++++++-- 5 files changed, 311 insertions(+), 127 deletions(-) diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index 2261cb731..7d8c20e12 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -9,8 +9,6 @@ import edu.wgu.osmt.richskill.RichSkillDoc import edu.wgu.osmt.richskill.RichSkillEsRepo import org.apache.lucene.search.join.ScoreMode import org.elasticsearch.index.query.* -import org.elasticsearch.index.query.QueryBuilders.matchPhrasePrefixQuery -import org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired @@ -93,17 +91,17 @@ class CustomCollectionQueriesImpl @Autowired constructor( publishStatus: Set, pageable: Pageable ): SearchHits { - val nqb = NativeSearchQueryBuilder().withPageable(Pageable.unpaged()) + val nsqb1 = NativeSearchQueryBuilder().withPageable(Pageable.unpaged()) val bq = QueryBuilders.boolQuery() - //TODO Replace with FindsAllByPublishStatus.createTermsDslQuery(publishStatus.name, publishStatus.map { ps -> ps.toString() }) + val filterDslQuery = createTermsDslQuery(RichSkillDoc::publishStatus.name, publishStatus.map { ps -> ps.toString() }) val filter = BoolQueryBuilder().must( QueryBuilders.termsQuery( RichSkillDoc::publishStatus.name, publishStatus.map { ps -> ps.toString() } ) ) - nqb.withFilter(filter) - nqb.withQuery(bq) + nsqb1.withFilter(filter) + nsqb1.withQuery(bq) var collectionMultiPropertyResults: List = listOf() @@ -113,84 +111,57 @@ class CustomCollectionQueriesImpl @Autowired constructor( bq.should( BoolQueryBuilder() .must(richSkillEsRepo.richSkillPropertiesMultiMatch(apiSearch.query)) - .must( - QueryBuilders.nestedQuery( - RichSkillDoc::collections.name, - QueryBuilders.matchAllQuery(), - ScoreMode.Avg - ).innerHit(InnerHitBuilder()) - ) + .must(createNestedQueryBuilder()) ) bq.should(richSkillEsRepo.occupationQueries(apiSearch.query)) - val nqb = NativeSearchQueryBuilder() + val nsqb = NativeSearchQueryBuilder() .withQuery( collectionPropertiesMultiMatch(apiSearch.query) ) .withPageable(Pageable.unpaged()) .withFilter(filter) // search on collection specific properties - val query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()1", nqb, log) + val query = convertToNativeQuery(Pageable.unpaged(), filterDslQuery, nsqb, "CustomCollectionQueriesImpl.byApiSearch()1", log) collectionMultiPropertyResults = elasticSearchTemplate - .search(query, CollectionDoc::class.java) - .searchHits - .map { it.content.uuid } + .search(query, CollectionDoc::class.java) + .searchHits + .map { it.content.uuid } } else if (apiSearch.advanced != null) { richSkillEsRepo.generateBoolQueriesFromApiSearch(bq, apiSearch.advanced) if (!apiSearch.advanced.collectionName.isNullOrBlank()) { - if (apiSearch.advanced.collectionName.contains("\"")) { - val nqb = NativeSearchQueryBuilder() - .withQuery( simpleQueryStringQuery(apiSearch.advanced.collectionName).field("${CollectionDoc::name.name}.raw").defaultOperator(Operator.AND) ) - .withPageable(Pageable.unpaged()) - .withFilter(filter) - val query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()2", nqb, log) - collectionMultiPropertyResults = elasticSearchTemplate - .search( query, CollectionDoc::class.java ) - .searchHits - .map { it.content.uuid } - } else { - val nqb = NativeSearchQueryBuilder() - .withQuery( matchPhrasePrefixQuery( CollectionDoc::name.name, apiSearch.advanced.collectionName ) ) - .withPageable(Pageable.unpaged()) - .withFilter(filter) - val query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()3", nqb, log) - collectionMultiPropertyResults = elasticSearchTemplate - .search( query, CollectionDoc::class.java ) - .searchHits - .map { it.content.uuid } - } + collectionMultiPropertyResults = getCollectionUuids(pageable, filterDslQuery, apiSearch.advanced.collectionName ) } else { - bq.must( - QueryBuilders.nestedQuery( - RichSkillDoc::collections.name, - QueryBuilders.matchAllQuery(), - ScoreMode.Avg - ).innerHit(InnerHitBuilder()) - ) + bq.must(createNestedQueryBuilder()) } } else { // query nor advanced search was provided, return all collections - bq.must( - QueryBuilders.nestedQuery( - RichSkillDoc::collections.name, - QueryBuilders.matchAllQuery(), - ScoreMode.Avg - ).innerHit(InnerHitBuilder()) - ) + bq.must(createNestedQueryBuilder()) } - var query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch().innerHitCollectionUuids", nqb, log) - val results = elasticSearchTemplate.search(query, RichSkillDoc::class.java) + var query = convertToNativeQuery(Pageable.unpaged(), filterDslQuery, nsqb1, "CustomCollectionQueriesImpl.byApiSearch().innerHitCollectionUuids", log) + val innerHitCollectionUuids = elasticSearchTemplate + .search(query, RichSkillDoc::class.java) + .searchHits.mapNotNull { it.getInnerHits("collections")?.searchHits?.mapNotNull { it.content as CollectionDoc } } + .flatten() + .map { it.uuid } + .distinct() + return getCollectionFromUuids(pageable, filterDslQuery, (innerHitCollectionUuids + collectionMultiPropertyResults).distinct()) + } - val innerHitCollectionUuids = - results.searchHits.mapNotNull { it.getInnerHits("collections")?.searchHits?.mapNotNull { it.content as CollectionDoc } } - .flatten().map { it.uuid }.distinct() + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createNestQueryDslQuery"), DeprecationLevel.WARNING ) + private fun createNestedQueryBuilder(): NestedQueryBuilder { + return QueryBuilders.nestedQuery( + RichSkillDoc::collections.name, + QueryBuilders.matchAllQuery(), + ScoreMode.Avg + ).innerHit(InnerHitBuilder()) + } - val nqb2 = NativeSearchQueryBuilder() - .withQuery( QueryBuilders.termsQuery( "_id", (innerHitCollectionUuids + collectionMultiPropertyResults).distinct() ) ) - .withFilter(filter) - .withPageable(pageable) - query = convertToStringQuery("CustomCollectionQueriesImpl.byApiSearch()4", nqb2, log) - return elasticSearchTemplate.search(query, CollectionDoc::class.java) + private fun getCollectionUuids(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String) : List { + return if (collectionName.contains("\"")) + getCollectionUuidsFromComplexName(pageable, filter, collectionName) + else + getCollectionUuidsFromName(pageable, filter, collectionName) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt b/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt index ca4bf1517..a1e43d240 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/db/ExposedHelper.kt @@ -43,28 +43,28 @@ fun SchemaUtils.addMissingColumnsStatementsPublic(vararg tables: Table): List { .stream() .map { ps -> ps.name} .collect(Collectors.toList()) - var filter = createTermsDslQuery(false, RichSkillDoc::publishStatus.name, filterValues) + var filter = createTermsDslQuery( RichSkillDoc::publishStatus.name, filterValues, false) return NativeQueryBuilder() .withPageable(pageable) .withQuery(MATCH_ALL) @@ -59,9 +62,45 @@ interface FindsAllByPublishStatus { } /** - * Create a query_dsl.Query instance that ElasticSearchTemplate v8.x mandates for filtering. + * Stepping stone to 100% migration to ES v8.7.x apis; see KeywordEsRepo.kt */ - fun createTermsDslQuery(andFlag: Boolean, fieldName: String, filterValues: List): co.elastic.clients.elasticsearch._types.query_dsl.Query { + fun convertToNativeQuery(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, nsqb: NativeSearchQueryBuilder, msgPrefix: String, log: Logger): Query { + val oldQuery = nsqb.build() + val nuQuery = NativeQuery.builder() + .withFilter(filter) + .withQuery(StringQuery(oldQuery.query.toString())) + .withPageable(pageable) + .build() + log.debug(String.Companion.format("\n%s springDataQuery:\n\t\t%s", msgPrefix, (nuQuery.springDataQuery as StringQuery).source)) + log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, nuQuery.filter.toString())) + return nuQuery + } + + /* + * Below methods are all leveraging the latest ElasticSearch v8.7.X Java API + */ + fun createMatchPhrasePrefixDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return matchPhrasePrefix { qb -> qb.field(fieldName).query(searchStr).boost(boostVal) } + } + + fun createMatchBoolPrefixDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return matchBoolPrefix { qb -> qb.field(fieldName).query(searchStr).boost(boostVal) } + } + + fun createSimpleQueryDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return simpleQueryString { qb -> qb.fields(fieldName).query(searchStr).boost(boostVal).defaultOperator(Operator.And) } + } + + fun createNestedQueryDslQuery(path: String, scoreMode: ChildScoreMode, query: co.elastic.clients.elasticsearch._types.query_dsl.Query? = null, innerHits: InnerHits? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + query ?: matchAll { b-> b } + innerHits ?: InnerHits.Builder().build() + return nested { qb -> qb.path(path) + .scoreMode(ChildScoreMode.Avg) + .innerHits(innerHits) + .query(matchAll { b-> b }) } + } + + fun createTermsDslQuery(fieldName: String, filterValues: List, andFlag: Boolean = true): co.elastic.clients.elasticsearch._types.query_dsl.Query { val values = filterValues .stream() .map { FieldValue.of(it) } @@ -77,9 +116,9 @@ interface FindsAllByPublishStatus { /* Short hand version https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html val terms2 = terms { qb -> qb.field(fieldName).terms(tqf) } return bool { qb -> if (andFlag) - qb.must(terms) + qb.must(terms2) else - qb.should(terms) } + qb.should(terms2) } */ return bool() @@ -89,31 +128,39 @@ interface FindsAllByPublishStatus { ._toQuery() } - fun createTermsDslQuery(fieldName: String, filterValues: List): co.elastic.clients.elasticsearch._types.query_dsl.Query { - return createTermsDslQuery(true, fieldName, filterValues) + fun getCollectionUuidsFromComplexName(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String) : List { + val query = NativeQuery + .builder() + .withFilter(filter) + .withQuery(createSimpleQueryDslQuery("${CollectionDoc::name.name}.raw", collectionName)) + .withPageable(pageable) + .build() + return elasticSearchTemplate + .search( query, CollectionDoc::class.java ) + .searchHits + .map { it.content.uuid } } - @Deprecated("", ReplaceWith("convertToNativeQuery"), DeprecationLevel.WARNING) - fun convertToStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder, log: Logger): Query { - val query = nqb.build() - log.debug(String.Companion.format("\n%s query:\n\t\t%s", msgPrefix, query.query.toString())) - log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, query.filter.toString())) - //NOTE: this causes us to lose the filter query - return StringQuery(query.query.toString()) + fun getCollectionUuidsFromName(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String) : List { + val query = NativeQuery + .builder() + .withFilter(filter) + .withQuery(createMatchPhrasePrefixDslQuery(CollectionDoc::name.name, collectionName)) + .withPageable(pageable) + .build() + return elasticSearchTemplate + .search( query, CollectionDoc::class.java ) + .searchHits + .map { it.content.uuid } } - /** - * Stepping stone to 100% migration to ES v8.7.x apis; see KeywordEsRepo.kt - */ - fun convertToNativeQuery(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, nsqb: NativeSearchQueryBuilder, msgPrefix: String, log: Logger): Query { - val oldQuery = nsqb.build() - val nuQuery = NativeQuery.builder() + fun getCollectionFromUuids(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, uuids: List ): SearchHits { + val query = NativeQuery + .builder() .withFilter(filter) - .withQuery(StringQuery(oldQuery.query.toString())) + .withQuery(createTermsDslQuery("_id", uuids)) .withPageable(pageable) .build() - log.debug(String.Companion.format("\n%s springDataQuery:\n\t\t%s", msgPrefix, (nuQuery.springDataQuery as StringQuery).source)) - log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, nuQuery.filter.toString())) - return nuQuery + return elasticSearchTemplate.search(query, CollectionDoc::class.java) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index 23687e873..226408aca 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -71,7 +71,7 @@ class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSea } } -@Deprecated("Upgrade to ES v8.x queries", ReplaceWith("Replacement method"), DeprecationLevel.WARNING ) +@Deprecated("Upgrade to ES v8.x queries", ReplaceWith("JobCodeQueriesEx"), DeprecationLevel.WARNING ) object JobCodeQueries { //TODO Convert to ES v8.7.x apis and return the newer BoolQuery.Builder instance; see KeywordEsRep.kt fun multiPropertySearch(query: String, parentDocPath: String? = null): BoolQueryBuilder { diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index 0f5008322..997b7093a 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -1,10 +1,11 @@ package edu.wgu.osmt.richskill +import co.elastic.clients.elasticsearch._types.FieldValue +import co.elastic.clients.elasticsearch._types.query_dsl.Query +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.* +import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField import edu.wgu.osmt.PaginationDefaults -import edu.wgu.osmt.api.model.ApiAdvancedSearch -import edu.wgu.osmt.api.model.ApiFilteredSearch -import edu.wgu.osmt.api.model.ApiSearch -import edu.wgu.osmt.api.model.ApiSimilaritySearch +import edu.wgu.osmt.api.model.* import edu.wgu.osmt.config.INDEX_RICHSKILL_DOC import edu.wgu.osmt.config.QUOTED_SEARCH_REGEX_PATTERN import edu.wgu.osmt.db.PublishStatus @@ -32,6 +33,8 @@ import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories import org.springframework.security.oauth2.jwt.Jwt +import java.util.function.Consumer +import java.util.stream.Collectors const val collectionsUuid = "collections.uuid" @@ -102,6 +105,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear return uuids } + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("buildNestedQueriesNu"), DeprecationLevel.WARNING) override fun occupationQueries(query: String): NestedQueryBuilder { val jobCodePath = RichSkillDoc::jobCodes.name return QueryBuilders.nestedQuery( @@ -111,6 +115,21 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear ) } + /** + * ElasticSearch v8.7.X version + */ + fun occupationQueriesNu(query: String): Query? { + /* + val multiPropQuery = null + return co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.nested { + qb -> qb.path(RichSkillDoc::jobCodes.name) + .query(JobCodeQueries.multiPropertySearch(query, jobCodePath),) + .scoreMode( ChildScoreMode.Max)} + */ + return null; + } + + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("buildNestedQueriesNu"), DeprecationLevel.WARNING) private fun buildNestedQueries(path: String?=null, queryParams: List) : BoolQueryBuilder { val disjunctionQuery = disMaxQuery() val queries = ArrayList() @@ -128,8 +147,23 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear return boolQuery().must(existsQuery("$path.keyword")).must(disjunctionQuery) } + /** + * ElasticSearch v8.7.X version + */ + private fun buildNestedQueriesNu(path: String?=null, queryParams: List) : Query { + val prefixQueries = ArrayList() + queryParams.forEach(Consumer { s: String? -> + val q = prefix { qb -> qb.field( "$path.keyword").value(s) } + prefixQueries.add(q) + }) + + val disMaxQuery = disMax {qb -> qb.queries(prefixQueries)} + val existQuery = exists { qb -> qb.field("$path.keyword")} + return bool { qb -> qb.must(disMaxQuery).must(existQuery)} + } // Query clauses for Rich Skill properties + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("generateBoolQueriesFromApiSearchNu"), DeprecationLevel.WARNING) override fun generateBoolQueriesFromApiSearch(bq: BoolQueryBuilder, advancedQuery: ApiAdvancedSearch) { with(advancedQuery) { // boolQuery.must for logical AND @@ -199,8 +233,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear it.mapNotNull { it.name }.map { s -> if (s.contains("\"")) { bq.must( - simpleQueryStringQuery(s).field("${RichSkillDoc::certifications.name}.raw") - .defaultOperator(Operator.AND) + simpleQueryStringQuery(s).field("${RichSkillDoc::certifications.name}.raw").defaultOperator(Operator.AND) ) } else { bq.must(matchBoolPrefixQuery(RichSkillDoc::certifications.name, s)) @@ -234,7 +267,50 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear } } + /** + * ElasticSearch v8.7.X version + */ + fun generateBoolQueriesFromApiSearchNu(advancedQuery: ApiAdvancedSearch): Query { + with(advancedQuery) { + return bool { bq -> + skillName.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::name.name, it)) } + category.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::categories.name, it, QUOTED_SEARCH_REGEX_PATTERN)) } + author.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::authors.name, it)) } + skillStatement.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::statement.name, it)) } + keywords?.let { bq.must(createQueryFromStringList(RichSkillDoc::searchingKeywords.name, it)) } +//TODO implement this +// occupations.nullIfEmpty()?.let { bq.must(createQueryFromString(RichSkillDoc::name.name, it)) } + + standards?.let { bq.must(createQueryFromApiNameList(RichSkillDoc::standards.name, it)) } + certifications?.let { bq.must(createQueryFromApiNameList(RichSkillDoc::certifications.name, it)) } + employers?.let { bq.must(createQueryFromApiNameList(RichSkillDoc::employers.name, it)) } + alignments?.let { bq.must(createQueryFromApiNameList(RichSkillDoc::alignments.name, it)) } + } + } + } + + private fun createQueryFromString(fieldName: String, searchStr: String, regEx: String? = null): Query { + val isComplex = searchStr.contains("\"") || (regEx != null && searchStr.matches(Regex(regEx))) + return if (isComplex) + createSimpleQueryDslQuery(String.format("%s.raw", fieldName), searchStr) + else + createMatchBoolPrefixDslQuery(fieldName, searchStr) + } + + private fun createQueryFromStringList(fieldName: String, searchStrList: List): List { + return searchStrList + .stream() + .map { createQueryFromString(fieldName, it ?: "") } + .collect(Collectors.toList()) + } + + private fun createQueryFromApiNameList(fieldName: String, searchStrList: List): List { + return createQueryFromStringList(fieldName, searchStrList.map { it.name?: "" }) + } + + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("generateBoolQueriesFromApiSearchWithFiltersNu"), DeprecationLevel.WARNING) override fun generateBoolQueriesFromApiSearchWithFilters(bq: BoolQueryBuilder, filteredQuery: ApiFilteredSearch, publishStatus: Set) { + bq.must( termsQuery( RichSkillDoc::publishStatus.name, @@ -283,25 +359,76 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear } } + /** + * TODO Fix the NPE at the return. + fun generateBoolQueriesFromApiSearchWithFiltersNu(filteredQuery: ApiFilteredSearch, publishStatus: Set) : Query { + val values = publishStatus + .stream() + .map { FieldValue.of(it.toString()) } + .collect(Collectors.toList()) + val tqf = TermsQueryField.Builder() + .value(values) + .build() + val terms2 = terms { qb -> qb.field(RichSkillDoc::publishStatus.name).terms(tqf) } + val qb = bool().must(terms2) + + with(filteredQuery) { + categories?. let { qb.must(buildNestedQueriesNu(RichSkillDoc::categories.name, it)) } + keywords?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::searchingKeywords.name, keywords)) } + } + standards?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::standards.name, standards)) } + } + certifications?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::certifications.name, certifications)) } + } + alignments?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::alignments.name, alignments)) } + } + employers?. let { + it.mapNotNull { qb.must(generateTermsSetQueryBuilderNu(RichSkillDoc::employers.name, employers)) } + } + authors?. let { + qb.must(buildNestedQueriesNu(RichSkillDoc::authors.name, it)) + } + occupations?.let { + it.mapNotNull { value -> qb.must( occupationQueriesNu(value) ) } + } + } + val s = qb.build()._toQuery() + return s + } + */ + + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("generateTermsSetQueryBuilderNu"), DeprecationLevel.WARNING) private fun generateTermsSetQueryBuilder(fieldName: String, list: List): TermsSetQueryBuilder { return TermsSetQueryBuilder("$fieldName.keyword", list).setMinimumShouldMatchScript(Script(list.size.toString())) } + /** + * ElasticSearch v8.7.X version + */ + private fun generateTermsSetQueryBuilderNu(fieldName: String, list: List): Query { + val sb = co.elastic.clients.elasticsearch._types.Script.Builder().inline { il -> il.source(list.size.toString())} + return termsSet { + qb -> qb.field("$fieldName.keyword") + .terms(list) + .minimumShouldMatchScript(sb.build()) + } + } + + @Deprecated("ElasticSearch 7.X has been deprecated", ReplaceWith("richSkillPropertiesMultiMatchNu"), DeprecationLevel.WARNING) override fun richSkillPropertiesMultiMatch(query: String): BoolQueryBuilder { val isComplex = query.contains("\"") - val boolQuery = boolQuery() - val complexQueries = listOf( - simpleQueryStringQuery(query).field("${RichSkillDoc::name.name}.raw").boost(2.0f) - .defaultOperator(Operator.AND), + simpleQueryStringQuery(query).field("${RichSkillDoc::name.name}.raw").boost(2.0f) .defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::statement.name}.raw").defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::categories.name}.raw").defaultOperator(Operator.AND), - simpleQueryStringQuery(query).field("${RichSkillDoc::searchingKeywords.name}.raw") - .defaultOperator(Operator.AND), + simpleQueryStringQuery(query).field("${RichSkillDoc::searchingKeywords.name}.raw") .defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::standards.name}.raw").defaultOperator(Operator.AND), - simpleQueryStringQuery(query).field("${RichSkillDoc::certifications.name}.raw") - .defaultOperator(Operator.AND), + simpleQueryStringQuery(query).field("${RichSkillDoc::certifications.name}.raw") .defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::employers.name}.raw").defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::alignments.name}.raw").defaultOperator(Operator.AND), simpleQueryStringQuery(query).field("${RichSkillDoc::authors.name}.raw").defaultOperator(Operator.AND) @@ -324,10 +451,48 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear } else { queries.map { boolQuery.should(it) } } - return boolQuery } + /** + * ElasticSearch v8.7.X version + */ + fun richSkillPropertiesMultiMatchNu(searchStr: String): Query { + var queries = if (searchStr.contains("\"")) + createComplexMultiMatchQueries(searchStr) + else + createMultiMatchQueries(searchStr) + return bool {qb -> qb.should(queries) } + } + + private fun createComplexMultiMatchQueries(searchStr: String) : List{ + return listOf( + createSimpleQueryDslQuery("${RichSkillDoc::name.name}.raw", searchStr, 2.0f), + createSimpleQueryDslQuery("${RichSkillDoc::statement.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::categories.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::searchingKeywords.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::standards.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::certifications.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::employers.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::alignments.name}.raw", searchStr), + createSimpleQueryDslQuery("${RichSkillDoc::authors.name}.raw", searchStr) + ) + } + + private fun createMultiMatchQueries(searchStr: String) : List{ + return listOf( + createMatchPhrasePrefixDslQuery(RichSkillDoc::name.name, searchStr, 2.0f), + createMatchPhrasePrefixDslQuery(RichSkillDoc::statement.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::categories.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::searchingKeywords.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::standards.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::certifications.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::employers.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::alignments.name, searchStr), + createMatchPhrasePrefixDslQuery(RichSkillDoc::authors.name, searchStr) + ) + } + override fun byApiSearch( apiSearch: ApiSearch, publishStatus: Set, @@ -481,6 +646,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear ) return elasticSearchTemplate.search(query, RichSkillDoc::class.java) } + } From 7468bc9eda16d298dd78fcb62f02de021582fc37 Mon Sep 17 00:00:00 2001 From: Huey Date: Fri, 20 Oct 2023 08:30:31 -0700 Subject: [PATCH 18/22] Updated common.sh version checking to recent version numbers --- bin/lib/common.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/lib/common.sh b/bin/lib/common.sh index f88e929a3..eae9b1032 100755 --- a/bin/lib/common.sh +++ b/bin/lib/common.sh @@ -265,7 +265,7 @@ _validate_java_version() { echo echo_info "Checking Java..." # OSMT requires at least Java 11 - local -i req_java_major=11 + local -i req_java_major=17 local det_java_version local -i det_java_major @@ -317,10 +317,10 @@ _validate_osmt_dev_dependencies() { echo_info "Maven version: $(mvn --version)" echo - echo_info "OSMT development recommends NodeJS version v16.13.0 or greater. Maven uses an embedded copy of NodeJS v16.13.0 via frontend-maven-plugin." + echo_info "OSMT development recommends NodeJS version v21.0.0 or greater. Maven uses an embedded copy of NodeJS v16.13.0 via frontend-maven-plugin." echo_info "NodeJS version: $(node --version)" echo - echo_info "OSMT development recommends npm version 8.1.0 or greater. Maven uses an embedded copy of npm 8.1.0 via frontend-maven-plugin." + echo_info "OSMT development recommends npm version 10.2.0 or greater. Maven uses an embedded copy of npm 8.1.0 via frontend-maven-plugin." echo_info "npm version: $(npm --version)" if [[ "${is_dependency_valid}" -ne 0 ]]; then echo From 4cf884200e237091c317c4bcc2edb2da5032f8f7 Mon Sep 17 00:00:00 2001 From: Huey Date: Fri, 20 Oct 2023 10:05:25 -0700 Subject: [PATCH 19/22] Update common.sh to reflect LTS versions --- bin/lib/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/lib/common.sh b/bin/lib/common.sh index eae9b1032..4c4097e38 100755 --- a/bin/lib/common.sh +++ b/bin/lib/common.sh @@ -317,10 +317,10 @@ _validate_osmt_dev_dependencies() { echo_info "Maven version: $(mvn --version)" echo - echo_info "OSMT development recommends NodeJS version v21.0.0 or greater. Maven uses an embedded copy of NodeJS v16.13.0 via frontend-maven-plugin." + echo_info "OSMT development recommends NodeJS version v18.18.2 or greater. Maven uses an embedded copy of NodeJS v16.13.0 via frontend-maven-plugin." echo_info "NodeJS version: $(node --version)" echo - echo_info "OSMT development recommends npm version 10.2.0 or greater. Maven uses an embedded copy of npm 8.1.0 via frontend-maven-plugin." + echo_info "OSMT development recommends npm version 9.8.1 or greater. Maven uses an embedded copy of npm 8.1.0 via frontend-maven-plugin." echo_info "npm version: $(npm --version)" if [[ "${is_dependency_valid}" -ne 0 ]]; then echo From bf29a8cbe2422c646450c61feffdd2b83d0e954a Mon Sep 17 00:00:00 2001 From: Huey Date: Fri, 20 Oct 2023 16:43:02 -0700 Subject: [PATCH 20/22] DMND-1721 Junits are now all passing. --- .../wgu/osmt/collection/CollectionEsRepo.kt | 2 + .../elasticsearch/FindsAllByPublishStatus.kt | 89 +------------- .../wgu/osmt/elasticsearch/WguQueryHelper.kt | 116 ++++++++++++++++++ .../edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 14 +-- .../edu/wgu/osmt/keyword/KeywordEsRepo.kt | 19 +-- .../edu/wgu/osmt/richskill/RichSkillEsRepo.kt | 24 +++- .../edu/wgu/osmt/keyword/KeywordEsRepoTest.kt | 4 +- 7 files changed, 152 insertions(+), 116 deletions(-) create mode 100644 api/src/main/kotlin/edu/wgu/osmt/elasticsearch/WguQueryHelper.kt diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index 7d8c20e12..87f4a716b 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -5,6 +5,8 @@ import edu.wgu.osmt.api.model.ApiSearch import edu.wgu.osmt.config.INDEX_COLLECTION_DOC import edu.wgu.osmt.db.PublishStatus import edu.wgu.osmt.elasticsearch.FindsAllByPublishStatus +import edu.wgu.osmt.elasticsearch.WguQueryHelper.convertToNativeQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createTermsDslQuery import edu.wgu.osmt.richskill.RichSkillDoc import edu.wgu.osmt.richskill.RichSkillEsRepo import org.apache.lucene.search.join.ScoreMode diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt index 764ff099c..66ebd0159 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt @@ -1,24 +1,19 @@ package edu.wgu.osmt.elasticsearch -import co.elastic.clients.elasticsearch._types.FieldValue -import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode -import co.elastic.clients.elasticsearch._types.query_dsl.Operator -import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.* -import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField -import co.elastic.clients.elasticsearch.core.search.InnerHits +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.matchAll import edu.wgu.osmt.collection.CollectionDoc import edu.wgu.osmt.db.PublishStatus +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createMatchPhrasePrefixDslQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createSimpleQueryDslQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createTermsDslQuery import edu.wgu.osmt.richskill.RichSkillDoc -import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.data.domain.Pageable import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate import org.springframework.data.elasticsearch.client.elc.NativeQuery import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder -import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits -import org.springframework.data.elasticsearch.core.query.Query -import org.springframework.data.elasticsearch.core.query.StringQuery import java.util.stream.Collectors /** @@ -54,80 +49,6 @@ interface FindsAllByPublishStatus { .withFilter(filter) } - fun createQuery(msgPrefix: String, nqb: NativeQueryBuilder, log: Logger): Query { - val query = nqb.build() - log.debug(String.Companion.format("\n%s query:\n\t\t%s", msgPrefix, query.query.toString())) - log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, query.filter.toString())) - return query; - } - - /** - * Stepping stone to 100% migration to ES v8.7.x apis; see KeywordEsRepo.kt - */ - fun convertToNativeQuery(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, nsqb: NativeSearchQueryBuilder, msgPrefix: String, log: Logger): Query { - val oldQuery = nsqb.build() - val nuQuery = NativeQuery.builder() - .withFilter(filter) - .withQuery(StringQuery(oldQuery.query.toString())) - .withPageable(pageable) - .build() - log.debug(String.Companion.format("\n%s springDataQuery:\n\t\t%s", msgPrefix, (nuQuery.springDataQuery as StringQuery).source)) - log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, nuQuery.filter.toString())) - return nuQuery - } - - /* - * Below methods are all leveraging the latest ElasticSearch v8.7.X Java API - */ - fun createMatchPhrasePrefixDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { - return matchPhrasePrefix { qb -> qb.field(fieldName).query(searchStr).boost(boostVal) } - } - - fun createMatchBoolPrefixDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { - return matchBoolPrefix { qb -> qb.field(fieldName).query(searchStr).boost(boostVal) } - } - - fun createSimpleQueryDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { - return simpleQueryString { qb -> qb.fields(fieldName).query(searchStr).boost(boostVal).defaultOperator(Operator.And) } - } - - fun createNestedQueryDslQuery(path: String, scoreMode: ChildScoreMode, query: co.elastic.clients.elasticsearch._types.query_dsl.Query? = null, innerHits: InnerHits? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { - query ?: matchAll { b-> b } - innerHits ?: InnerHits.Builder().build() - return nested { qb -> qb.path(path) - .scoreMode(ChildScoreMode.Avg) - .innerHits(innerHits) - .query(matchAll { b-> b }) } - } - - fun createTermsDslQuery(fieldName: String, filterValues: List, andFlag: Boolean = true): co.elastic.clients.elasticsearch._types.query_dsl.Query { - val values = filterValues - .stream() - .map { FieldValue.of(it) } - .collect(Collectors.toList()) - val tqf = TermsQueryField.Builder() - .value(values) - .build() - val terms = terms() - .field(fieldName) - .terms(tqf) - .build() - ._toQuery() - /* Short hand version https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html - val terms2 = terms { qb -> qb.field(fieldName).terms(tqf) } - return bool { qb -> if (andFlag) - qb.must(terms2) - else - qb.should(terms2) } - */ - - return bool() - .let { if (andFlag) it.must(terms) - else it.should(terms) } - .build() - ._toQuery() - } - fun getCollectionUuidsFromComplexName(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String) : List { val query = NativeQuery .builder() diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/WguQueryHelper.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/WguQueryHelper.kt new file mode 100644 index 000000000..410865655 --- /dev/null +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/WguQueryHelper.kt @@ -0,0 +1,116 @@ +package edu.wgu.osmt.elasticsearch + +import co.elastic.clients.elasticsearch._types.FieldValue +import co.elastic.clients.elasticsearch._types.SortOptions +import co.elastic.clients.elasticsearch._types.SortOrder +import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode +import co.elastic.clients.elasticsearch._types.query_dsl.Operator +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders +import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField +import co.elastic.clients.elasticsearch.core.search.InnerHits +import org.slf4j.Logger +import org.springframework.data.domain.Pageable +import org.springframework.data.elasticsearch.client.elc.NativeQuery +import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder +import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder +import org.springframework.data.elasticsearch.core.query.Query +import org.springframework.data.elasticsearch.core.query.StringQuery +import java.util.stream.Collectors + + +/** + * Utility class for leveraging latest ElasticSearch v8.7.X Java API + */ +object WguQueryHelper { + /** + * Stepping stone to 100% migration to ES v8.7.x apis; see KeywordEsRepo.kt + */ + fun convertToNativeQuery(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, nsqb: NativeSearchQueryBuilder, msgPrefix: String, log: Logger): Query { + val oldQuery = nsqb.build() + + val nuQuery = NativeQuery.builder() + .withFilter(filter) + .withQuery(StringQuery(oldQuery.query.toString())) + .withPageable(pageable) +// .withSort(createSort("blah")) + .build() + log.debug(String.Companion.format("\n%s springDataQuery:\n\t\t%s", msgPrefix, (nuQuery.springDataQuery as StringQuery).source)) + log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, nuQuery.filter.toString())) + return nuQuery + } + + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createQuery"), DeprecationLevel.WARNING ) + fun createStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder, log: Logger): Query { + val query = nqb.build() + log.debug(String.Companion.format("\n%s query:\n\t\t%s", msgPrefix, query.query.toString())) + log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, query.filter.toString())) + //NOTE: this is causing us to lose the filter query + return StringQuery(query.query.toString()) + } + + fun createQuery(msgPrefix: String, nqb: NativeQueryBuilder, log: Logger): Query { + val query = nqb.build() + log.debug(String.Companion.format("\n%s query:\n\t\t%s", msgPrefix, query.query.toString())) + log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, query.filter.toString())) + return query; + } + + fun createMatchPhrasePrefixDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return QueryBuilders.matchPhrasePrefix { qb -> qb.field(fieldName).query(searchStr).boost(boostVal) } + } + + fun createMatchBoolPrefixDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return QueryBuilders.matchBoolPrefix { qb -> qb.field(fieldName).query(searchStr).boost(boostVal) } + } + + fun createSimpleQueryDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + return QueryBuilders.simpleQueryString { qb -> + qb.fields(fieldName).query(searchStr).boost(boostVal).defaultOperator(Operator.And) + } + } + + fun createNestedQueryDslQuery(path: String, scoreMode: ChildScoreMode, query: co.elastic.clients.elasticsearch._types.query_dsl.Query? = null, innerHits: InnerHits? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { + query ?: QueryBuilders.matchAll { b -> b } + innerHits ?: InnerHits.Builder().build() + return QueryBuilders.nested { qb -> + qb.path(path) + .scoreMode(ChildScoreMode.Avg) + .innerHits(innerHits) + .query(QueryBuilders.matchAll { b -> b }) + } + } + + fun createTermsDslQuery(fieldName: String, filterValues: List, andFlag: Boolean = true): co.elastic.clients.elasticsearch._types.query_dsl.Query { + val values = filterValues + .stream() + .map { FieldValue.of(it) } + .collect(Collectors.toList()) + val tqf = TermsQueryField.Builder() + .value(values) + .build() + val terms = QueryBuilders.terms() + .field(fieldName) + .terms(tqf) + .build() + ._toQuery() + /* Short hand version https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html + val terms2 = terms { qb -> qb.field(fieldName).terms(tqf) } + return bool { qb -> if (andFlag) + qb.must(terms2) + else + qb.should(terms2) } + */ + + return QueryBuilders.bool() + .let { + if (andFlag) it.must(terms) + else it.should(terms) } + .build() + ._toQuery() + } + + // TODO handle case sensitivity + fun createSort(fieldName: String, sortOrder: SortOrder = SortOrder.Asc) : SortOptions { + return SortOptions.Builder().field{f -> f.field(fieldName).order(sortOrder)}.build() + } +} diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index 226408aca..f2ccd9673 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -2,6 +2,7 @@ package edu.wgu.osmt.jobcode import edu.wgu.osmt.config.INDEX_JOBCODE_DOC import edu.wgu.osmt.elasticsearch.OffsetPageable +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createStringQuery import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.Operator import org.elasticsearch.index.query.QueryBuilders.* @@ -12,12 +13,9 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate -import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.core.query.Query -import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -49,7 +47,7 @@ class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSea .withPageable(createOffsetPageable(query)) .withQuery(disjunctionQuery) .withSort(SortBuilders.fieldSort("${JobCode::code.name}.keyword").order(SortOrder.ASC)) - val query = createStringQuery("CustomJobCodeRepositoryImpl.typeAheadSearch()", nqb) + val query = createStringQuery("CustomJobCodeRepositoryImpl.typeAheadSearch()", nqb, log) return elasticSearchTemplate.search(query, JobCode::class.java) } @@ -61,14 +59,6 @@ class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSea } return limitedPageable } - - fun createStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder): Query { - val query = nqb.build() - log.debug(String.Companion.format("\n%s query:\n\t\t%s", msgPrefix, query.query.toString())) - log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, query.filter.toString())) - //NOTE: this is causing us to lose the filter query - return StringQuery(query.query.toString()) - } } @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("JobCodeQueriesEx"), DeprecationLevel.WARNING ) diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt index 1b7f285cf..19e8dfc89 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt @@ -4,6 +4,7 @@ import co.elastic.clients.elasticsearch._types.query_dsl.* import edu.wgu.osmt.config.INDEX_KEYWORD_DOC import edu.wgu.osmt.config.SORT_INSENSITIVE import edu.wgu.osmt.elasticsearch.OffsetPageable +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createStringQuery import edu.wgu.osmt.jobcode.CustomJobCodeRepositoryImpl import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.search.sort.SortBuilders @@ -17,8 +18,6 @@ import org.springframework.data.elasticsearch.client.elc.NativeQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates -import org.springframework.data.elasticsearch.core.query.Query -import org.springframework.data.elasticsearch.core.query.StringQuery import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -71,7 +70,7 @@ class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSea ).minimumShouldMatch(1) } - val query = createStringQuery("CustomKeywordRepositoryImpl.typeAheadSearch()", nqb) + val query = createStringQuery("CustomKeywordRepositoryImpl.typeAheadSearch()", nqb, log) return elasticSearchTemplate.search(query, Keyword::class.java) } @@ -80,7 +79,7 @@ class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSea */ fun typeAheadSearchNu(searchStr: String, type: KeywordTypeEnum): SearchHits { val pageable: OffsetPageable - val criteria: co.elastic.clients.elasticsearch._types.query_dsl.Query + val criteria: Query if (searchStr.isEmpty()) { pageable = OffsetPageable(0, 10000, null) @@ -95,14 +94,14 @@ class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSea .withQuery(criteria).build(), Keyword::class.java ) } - fun searchAll(type: KeywordTypeEnum): co.elastic.clients.elasticsearch._types.query_dsl.Query { + fun searchAll(type: KeywordTypeEnum): Query { return co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.bool { builder: BoolQuery.Builder -> builder .must(co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.term { qt: TermQuery.Builder -> qt.field(Keyword::type.name).value(type.name) } ) .should(co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.matchAll { q : MatchAllQuery.Builder -> q } ) } } - fun searchSpecific(searchStr: String, type: KeywordTypeEnum): co.elastic.clients.elasticsearch._types.query_dsl.Query { + fun searchSpecific(searchStr: String, type: KeywordTypeEnum): Query { return co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.bool { builder: BoolQuery.Builder -> builder .must(co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.term { qt: TermQuery.Builder -> qt.field(Keyword::type.name).value(type.name) } ) @@ -110,14 +109,6 @@ class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSea .should(co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.matchPhrase { q : MatchPhraseQuery.Builder -> q.field(Keyword::value.name).query(searchStr)} ) .minimumShouldMatch("1") } } - - fun createStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder): Query { - val query = nqb.build() - log.debug(String.Companion.format("\n%s query:\n\t\t%s", msgPrefix, query.query.toString())) - log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, query.filter.toString())) - //NOTE: this is causing us to lose the filter query - return StringQuery(query.query.toString()) - } } @Configuration diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index 997b7093a..7bb657020 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -1,9 +1,7 @@ package edu.wgu.osmt.richskill -import co.elastic.clients.elasticsearch._types.FieldValue import co.elastic.clients.elasticsearch._types.query_dsl.Query import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.* -import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField import edu.wgu.osmt.PaginationDefaults import edu.wgu.osmt.api.model.* import edu.wgu.osmt.config.INDEX_RICHSKILL_DOC @@ -11,6 +9,11 @@ import edu.wgu.osmt.config.QUOTED_SEARCH_REGEX_PATTERN import edu.wgu.osmt.db.PublishStatus import edu.wgu.osmt.elasticsearch.FindsAllByPublishStatus import edu.wgu.osmt.elasticsearch.OffsetPageable +import edu.wgu.osmt.elasticsearch.WguQueryHelper.convertToNativeQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createMatchBoolPrefixDslQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createMatchPhrasePrefixDslQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createSimpleQueryDslQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createTermsDslQuery import edu.wgu.osmt.jobcode.JobCodeQueries import edu.wgu.osmt.nullIfEmpty import org.apache.commons.lang3.StringUtils @@ -501,7 +504,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear ): SearchHits { val query = convertToNativeQuery( pageable, - createTermsDslQuery(RichSkillDoc::publishStatus.name, publishStatus.map { ps -> ps.toString() }), + createFilter(apiSearch, publishStatus), buildQuery(publishStatus, apiSearch, collectionId), "CustomRichSkillQueriesImpl.byApiSearch()", log @@ -517,7 +520,7 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear ): Long { val query = convertToNativeQuery( pageable, - createTermsDslQuery(RichSkillDoc::publishStatus.name, publishStatus.map { ps -> ps.toString() }), + createFilter(apiSearch, publishStatus), buildQuery(publishStatus, apiSearch, collectionId), "CustomRichSkillQueriesImpl.countByApiSearch()", log @@ -525,6 +528,17 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear return elasticSearchTemplate.count(query, RichSkillDoc::class.java) } + private fun createFilter( apiSearch: ApiSearch, publishStatus: Set) : Query { + var fieldName = RichSkillDoc::publishStatus.name + var filterValues = publishStatus.map { ps -> ps.toString() } + + if ( !apiSearch.uuids.isNullOrEmpty() ) { + fieldName = RichSkillDoc::uuid.name + filterValues = apiSearch.uuids.filter { x: String? -> x != "" } + } + return createTermsDslQuery(fieldName, filterValues) + } + /** * TODO upgrade to ElasticSearch v8.7.x api style; see KeywordEsRepo.kt & FindsAllByPublishStatus.kt */ @@ -611,8 +625,8 @@ class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSear var apiSearchUuids = apiSearch.uuids?.filterNotNull()?.filter { x: String? -> x != "" } if (!apiSearchUuids.isNullOrEmpty()) { + // This is not needed & ignored by WguQueryHelper.convertToNativeQuery() nsqb.withFilter( - //TODO Replace with FindsAllByPublishStatus.createTermsDslQuery(uuid.name, apiSearchUuids) BoolQueryBuilder().must( termsQuery( RichSkillDoc::uuid.name, diff --git a/api/src/test/kotlin/edu/wgu/osmt/keyword/KeywordEsRepoTest.kt b/api/src/test/kotlin/edu/wgu/osmt/keyword/KeywordEsRepoTest.kt index 83b67c6e5..3ec936838 100644 --- a/api/src/test/kotlin/edu/wgu/osmt/keyword/KeywordEsRepoTest.kt +++ b/api/src/test/kotlin/edu/wgu/osmt/keyword/KeywordEsRepoTest.kt @@ -54,6 +54,8 @@ class KeywordEsRepoTest @Autowired constructor( assertThat(result3.searchHits.count()).isEqualTo(2) assertThat(result4.searchHits.count()).isEqualTo(1) assertThat(result4.searchHits.first().content.value).isEqualTo("Yellow") - assertThat(result5.searchHits).hasSize(56) + + // Pagination causes the searchHits.count to be only 10 + assertThat(result5.totalHits).isEqualTo(56) } } From 77ba71389a328ac3b9cf2a493b910454e25cb0c5 Mon Sep 17 00:00:00 2001 From: shnooga Date: Wed, 25 Oct 2023 10:49:59 -0700 Subject: [PATCH 21/22] JUnit fix and synch with develop (#463) * Upgrade testcontainers to 1.18.3 (#453) * Add additional logging for env file sourcing in GH Action builds (#457) * Use 'debug' Spring profile for logging properties to console (#461) * Update pathings to allow POST of skills/filter and GET of Categories & Workspace * Code refactor for maintainability --------- Co-authored-by: John Kallies <3021949+JohnKallies@users.noreply.github.com> Co-authored-by: Huey --- .../kotlin/edu/wgu/osmt/PropertyLogger.kt | 4 +- .../main/kotlin/edu/wgu/osmt/RoutePaths.kt | 1 + .../wgu/osmt/collection/CollectionEsRepo.kt | 6 +- .../elasticsearch/FindsAllByPublishStatus.kt | 76 +++++++------------ .../wgu/osmt/elasticsearch/WguQueryHelper.kt | 56 +++++++------- .../edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 4 +- .../edu/wgu/osmt/keyword/KeywordEsRepo.kt | 18 +++-- .../edu/wgu/osmt/security/SecurityConfig.kt | 42 +++++++--- .../osmt/richskill/RichSkillControllerTest.kt | 2 +- bin/lib/common.sh | 10 ++- 10 files changed, 115 insertions(+), 104 deletions(-) diff --git a/api/src/main/kotlin/edu/wgu/osmt/PropertyLogger.kt b/api/src/main/kotlin/edu/wgu/osmt/PropertyLogger.kt index 5f6590229..5b0299c72 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/PropertyLogger.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/PropertyLogger.kt @@ -15,7 +15,7 @@ import java.util.stream.StreamSupport @Component -@Profile("dev") +@Profile("debug") class PropertyLogger { @EventListener fun handleContextRefresh(event: ContextRefreshedEvent) { @@ -43,4 +43,4 @@ class PropertyLogger { companion object { private val LOGGER = LoggerFactory.getLogger(PropertyLogger::class.java) } -} \ No newline at end of file +} diff --git a/api/src/main/kotlin/edu/wgu/osmt/RoutePaths.kt b/api/src/main/kotlin/edu/wgu/osmt/RoutePaths.kt index 4bc7c9f75..f3c72a6f2 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/RoutePaths.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/RoutePaths.kt @@ -57,6 +57,7 @@ object RoutePaths { const val COLLECTION_REMOVE = "$COLLECTION_DETAIL/remove" const val WORKSPACE_PATH = "/workspace" + const val WORKSPACE_LIST = WORKSPACE_PATH private const val TASKS_PATH = "/results" const val TASK_DETAIL_TEXT = "$TASKS_PATH/text/{uuid}" diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index 87f4a716b..7b95ec0cb 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -147,7 +147,7 @@ class CustomCollectionQueriesImpl @Autowired constructor( .flatten() .map { it.uuid } .distinct() - return getCollectionFromUuids(pageable, filterDslQuery, (innerHitCollectionUuids + collectionMultiPropertyResults).distinct()) + return getCollectionFromUuids(pageable, filterDslQuery, (innerHitCollectionUuids + collectionMultiPropertyResults).distinct(), "CustomCollectionQueriesImpl.byApiSearch()2", log) } @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createNestQueryDslQuery"), DeprecationLevel.WARNING ) @@ -161,9 +161,9 @@ class CustomCollectionQueriesImpl @Autowired constructor( private fun getCollectionUuids(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String) : List { return if (collectionName.contains("\"")) - getCollectionUuidsFromComplexName(pageable, filter, collectionName) + getCollectionUuidsFromComplexName(pageable, filter, collectionName, "getCollectionUuids", log) else - getCollectionUuidsFromName(pageable, filter, collectionName) + getCollectionUuidsFromName(pageable, filter, collectionName, "getCollectionUuids", log) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt index 66ebd0159..71911ad3d 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/FindsAllByPublishStatus.kt @@ -4,84 +4,66 @@ import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders.matchAll import edu.wgu.osmt.collection.CollectionDoc import edu.wgu.osmt.db.PublishStatus import edu.wgu.osmt.elasticsearch.WguQueryHelper.createMatchPhrasePrefixDslQuery -import edu.wgu.osmt.elasticsearch.WguQueryHelper.createQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.createNativeQuery import edu.wgu.osmt.elasticsearch.WguQueryHelper.createSimpleQueryDslQuery import edu.wgu.osmt.elasticsearch.WguQueryHelper.createTermsDslQuery import edu.wgu.osmt.richskill.RichSkillDoc -import org.slf4j.LoggerFactory +import org.slf4j.Logger import org.springframework.data.domain.Pageable import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate import org.springframework.data.elasticsearch.client.elc.NativeQuery -import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import java.util.stream.Collectors -/** - * This have been partially converted to use the ElasticSearch 8.7.X apis. Need to do full conversion to use - * the v8.7.x ES Java API client, https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/8.10/searching.html - */ interface FindsAllByPublishStatus { val elasticSearchTemplate: ElasticsearchTemplate val javaClass: Class fun findAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): SearchHits { - val nqb = createQueryBuilder(pageable, publishStatus) - val query = createQuery("FindsAllByPublishStatus.findAllFilteredByPublishStatus()", nqb, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) - return elasticSearchTemplate.search(query, javaClass) + val nativeQuery = createMatchAllRichSkillNativeQuery(pageable, publishStatus) + return elasticSearchTemplate.search(nativeQuery, javaClass) } fun countAllFilteredByPublishStatus(publishStatus: Set, pageable: Pageable): Long { - val nqb = createQueryBuilder(pageable, publishStatus) - val query = createQuery("FindsAllByPublishStatus.countAllFilteredByPublishStatus()", nqb, LoggerFactory.getLogger(FindsAllByPublishStatus::class.java)) - return elasticSearchTemplate.count(query, javaClass) + val nativeQuery = createMatchAllRichSkillNativeQuery(pageable, publishStatus) + return elasticSearchTemplate.count(nativeQuery, javaClass) } - fun createQueryBuilder(pageable: Pageable, publishStatus: Set): NativeQueryBuilder { + fun createMatchAllRichSkillNativeQuery(pageable: Pageable, publishStatus: Set): NativeQuery { val MATCH_ALL = matchAll().build()._toQuery() var filterValues = publishStatus - .stream() - .map { ps -> ps.name} - .collect(Collectors.toList()) + .stream() + .map { ps -> ps.name} + .collect(Collectors.toList()) var filter = createTermsDslQuery( RichSkillDoc::publishStatus.name, filterValues, false) - return NativeQueryBuilder() - .withPageable(pageable) - .withQuery(MATCH_ALL) - .withFilter(filter) + + return createNativeQuery(pageable, filter, MATCH_ALL) } - fun getCollectionUuidsFromComplexName(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String) : List { - val query = NativeQuery - .builder() - .withFilter(filter) - .withQuery(createSimpleQueryDslQuery("${CollectionDoc::name.name}.raw", collectionName)) - .withPageable(pageable) - .build() + fun getCollectionUuidsFromComplexName(pageable: Pageable, dslFilter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String, msgPrefix: String, log: Logger) : List { + var dslQuery = createSimpleQueryDslQuery("${CollectionDoc::name.name}.raw", collectionName) + var nativeQuery = createNativeQuery(pageable, dslFilter, dslQuery, msgPrefix, log) + return elasticSearchTemplate - .search( query, CollectionDoc::class.java ) - .searchHits - .map { it.content.uuid } + .search( nativeQuery, CollectionDoc::class.java ) + .searchHits + .map { it.content.uuid } } - fun getCollectionUuidsFromName(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String) : List { - val query = NativeQuery - .builder() - .withFilter(filter) - .withQuery(createMatchPhrasePrefixDslQuery(CollectionDoc::name.name, collectionName)) - .withPageable(pageable) - .build() + fun getCollectionUuidsFromName(pageable: Pageable, dslFilter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, collectionName: String, msgPrefix: String, log: Logger) : List { + var dslQuery = createMatchPhrasePrefixDslQuery(CollectionDoc::name.name, collectionName) + var nativeQuery = createNativeQuery(pageable, dslFilter, dslQuery, msgPrefix, log) + return elasticSearchTemplate - .search( query, CollectionDoc::class.java ) + .search( nativeQuery, CollectionDoc::class.java ) .searchHits .map { it.content.uuid } } - fun getCollectionFromUuids(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, uuids: List ): SearchHits { - val query = NativeQuery - .builder() - .withFilter(filter) - .withQuery(createTermsDslQuery("_id", uuids)) - .withPageable(pageable) - .build() - return elasticSearchTemplate.search(query, CollectionDoc::class.java) + fun getCollectionFromUuids(pageable: Pageable, dslFilter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, uuids: List, msgPrefix: String, log: Logger ): SearchHits { + var dslQuery = createTermsDslQuery("_id", uuids) + var nativeQuery = createNativeQuery(pageable, dslFilter, dslQuery, msgPrefix, log) + + return elasticSearchTemplate.search(nativeQuery, CollectionDoc::class.java) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/WguQueryHelper.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/WguQueryHelper.kt index 410865655..3f2c16c0d 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/WguQueryHelper.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/WguQueryHelper.kt @@ -11,7 +11,6 @@ import co.elastic.clients.elasticsearch.core.search.InnerHits import org.slf4j.Logger import org.springframework.data.domain.Pageable import org.springframework.data.elasticsearch.client.elc.NativeQuery -import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.query.Query import org.springframework.data.elasticsearch.core.query.StringQuery @@ -22,39 +21,42 @@ import java.util.stream.Collectors * Utility class for leveraging latest ElasticSearch v8.7.X Java API */ object WguQueryHelper { - /** - * Stepping stone to 100% migration to ES v8.7.x apis; see KeywordEsRepo.kt - */ - fun convertToNativeQuery(pageable: Pageable, filter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, nsqb: NativeSearchQueryBuilder, msgPrefix: String, log: Logger): Query { - val oldQuery = nsqb.build() - - val nuQuery = NativeQuery.builder() - .withFilter(filter) - .withQuery(StringQuery(oldQuery.query.toString())) - .withPageable(pageable) -// .withSort(createSort("blah")) - .build() - log.debug(String.Companion.format("\n%s springDataQuery:\n\t\t%s", msgPrefix, (nuQuery.springDataQuery as StringQuery).source)) - log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, nuQuery.filter.toString())) - return nuQuery + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createNativeQuery"), DeprecationLevel.WARNING ) + fun convertToNativeQuery(pageable: Pageable, dslFilter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, nsqb: NativeSearchQueryBuilder, msgPrefix: String, log: Logger): Query { + val springDataQuery = StringQuery(nsqb.build().query.toString()) + return createNativeQuery(pageable, dslFilter, springDataQuery, msgPrefix, log) } - @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createQuery"), DeprecationLevel.WARNING ) - fun createStringQuery(msgPrefix: String, nqb: NativeSearchQueryBuilder, log: Logger): Query { - val query = nqb.build() - log.debug(String.Companion.format("\n%s query:\n\t\t%s", msgPrefix, query.query.toString())) - log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, query.filter.toString())) - //NOTE: this is causing us to lose the filter query - return StringQuery(query.query.toString()) + @Deprecated("Upgrade to ES v8.x queries", ReplaceWith("createNativeQuery"), DeprecationLevel.WARNING ) + fun createNativeQuery(pageable: Pageable, dslFilter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, springDataQuery: Query, msgPrefix: String? = null, log: Logger? = null): NativeQuery { + val query = NativeQuery + .builder() + .withFilter(dslFilter) + .withQuery(springDataQuery) + .withPageable(pageable) + .build() + log(query, msgPrefix, log) + return query; } - fun createQuery(msgPrefix: String, nqb: NativeQueryBuilder, log: Logger): Query { - val query = nqb.build() - log.debug(String.Companion.format("\n%s query:\n\t\t%s", msgPrefix, query.query.toString())) - log.debug(String.Companion.format("\n%s filter:\n\t\t%s", msgPrefix, query.filter.toString())) + fun createNativeQuery(pageable: Pageable, dslFilter: co.elastic.clients.elasticsearch._types.query_dsl.Query?, dslQuery: co.elastic.clients.elasticsearch._types.query_dsl.Query, msgPrefix: String? = null, log: Logger? = null): NativeQuery { + val query = NativeQuery + .builder() + .withFilter(dslFilter) + .withQuery(dslQuery) + .withPageable(pageable) +// .withSort(createSort("blah")) + .build() + log(query, msgPrefix, log) return query; } + private fun log(nativeQuery: NativeQuery, msgPrefix: String?, log: Logger?) { + if (nativeQuery.springDataQuery == null || msgPrefix == null || log == null) return + log.debug(String.Companion.format("\n%s springDataQuery:\n\t\t%s", msgPrefix, (nativeQuery.springDataQuery as StringQuery).source)) + log.debug(String.Companion.format("\n%s dslFilter:\n\t\t%s", msgPrefix, nativeQuery.filter.toString())) + } + fun createMatchPhrasePrefixDslQuery(fieldName: String, searchStr: String, boostVal : Float? = null): co.elastic.clients.elasticsearch._types.query_dsl.Query { return QueryBuilders.matchPhrasePrefix { qb -> qb.field(fieldName).query(searchStr).boost(boostVal) } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index f2ccd9673..2c21ed012 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -2,7 +2,7 @@ package edu.wgu.osmt.jobcode import edu.wgu.osmt.config.INDEX_JOBCODE_DOC import edu.wgu.osmt.elasticsearch.OffsetPageable -import edu.wgu.osmt.elasticsearch.WguQueryHelper.createStringQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper.convertToNativeQuery import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.Operator import org.elasticsearch.index.query.QueryBuilders.* @@ -47,7 +47,7 @@ class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSea .withPageable(createOffsetPageable(query)) .withQuery(disjunctionQuery) .withSort(SortBuilders.fieldSort("${JobCode::code.name}.keyword").order(SortOrder.ASC)) - val query = createStringQuery("CustomJobCodeRepositoryImpl.typeAheadSearch()", nqb, log) + val query = convertToNativeQuery(createOffsetPageable(query), null, nqb, "CustomJobCodeRepositoryImpl.typeAheadSearch()", log) return elasticSearchTemplate.search(query, JobCode::class.java) } diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt index 19e8dfc89..ca9670dae 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt @@ -4,7 +4,8 @@ import co.elastic.clients.elasticsearch._types.query_dsl.* import edu.wgu.osmt.config.INDEX_KEYWORD_DOC import edu.wgu.osmt.config.SORT_INSENSITIVE import edu.wgu.osmt.elasticsearch.OffsetPageable -import edu.wgu.osmt.elasticsearch.WguQueryHelper.createStringQuery +import edu.wgu.osmt.elasticsearch.WguQueryHelper +import edu.wgu.osmt.elasticsearch.WguQueryHelper.convertToNativeQuery import edu.wgu.osmt.jobcode.CustomJobCodeRepositoryImpl import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.search.sort.SortBuilders @@ -14,7 +15,6 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate -import org.springframework.data.elasticsearch.client.elc.NativeQuery import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.core.SearchHits import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates @@ -70,7 +70,7 @@ class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSea ).minimumShouldMatch(1) } - val query = createStringQuery("CustomKeywordRepositoryImpl.typeAheadSearch()", nqb, log) + val query = convertToNativeQuery( limitedPageable, null, nqb, "CustomKeywordRepositoryImpl.typeAheadSearch()", log ) return elasticSearchTemplate.search(query, Keyword::class.java) } @@ -88,10 +88,14 @@ class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSea pageable = OffsetPageable(0, 20, null) criteria = searchSpecific(searchStr, type) } - log.debug(String.Companion.format("\ntypeAheadSearchNu query:\n\t\t%s", criteria.bool().toString())) - return elasticSearchTemplate.search( NativeQuery.builder() - .withPageable(pageable) - .withQuery(criteria).build(), Keyword::class.java ) +// log.debug(String.Companion.format("\ntypeAheadSearchNu query:\n\t\t%s", criteria.bool().toString())) +// return elasticSearchTemplate.search( NativeQuery.builder() +// .withPageable(pageable) +// .withQuery(criteria) +// .build(), Keyword::class.java ) + + var nativeQuery = WguQueryHelper.createNativeQuery(pageable, null, criteria) + return elasticSearchTemplate.search(nativeQuery, Keyword::class.java) } fun searchAll(type: KeywordTypeEnum): Query { diff --git a/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt b/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt index bba44aeb3..f0e990d68 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/security/SecurityConfig.kt @@ -127,22 +127,40 @@ class SecurityConfig { if (appConfig.allowPublicLists) { http.authorizeHttpRequests { auth -> auth - .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").permitAll() - .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").permitAll() + .requestMatchers( POST, + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_FILTER}" + ).permitAll() + .requestMatchers( GET, + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.CATEGORY_LIST}", + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.WORKSPACE_LIST}", + + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}", + + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}") + .permitAll() } } else { http.authorizeHttpRequests { auth -> auth - .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) - .requestMatchers(GET, "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", - "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}").hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) + .requestMatchers( POST, + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_FILTER}") + .hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) + .requestMatchers( GET, + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.CATEGORY_LIST}", + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.WORKSPACE_LIST}", + + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.SKILLS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.SKILLS_LIST}", + + "${RoutePaths.API}${RoutePaths.API_V3}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.API_V2}${RoutePaths.COLLECTIONS_LIST}", + "${RoutePaths.API}${RoutePaths.UNVERSIONED}${RoutePaths.COLLECTIONS_LIST}") + .hasAnyAuthority(ADMIN, CURATOR, VIEW, READ) } } diff --git a/api/src/test/kotlin/edu/wgu/osmt/richskill/RichSkillControllerTest.kt b/api/src/test/kotlin/edu/wgu/osmt/richskill/RichSkillControllerTest.kt index 9c1f72f22..aa22ad259 100644 --- a/api/src/test/kotlin/edu/wgu/osmt/richskill/RichSkillControllerTest.kt +++ b/api/src/test/kotlin/edu/wgu/osmt/richskill/RichSkillControllerTest.kt @@ -83,7 +83,7 @@ internal class RichSkillControllerTest @Autowired constructor( richSkillEsRepo.saveAll(listOfSkills) // Act - val result = richSkillController.allPaginatedV2( + val result = richSkillController.allPaginated( UriComponentsBuilder.newInstance(), size, 0, diff --git a/bin/lib/common.sh b/bin/lib/common.sh index 4c4097e38..b50fecede 100755 --- a/bin/lib/common.sh +++ b/bin/lib/common.sh @@ -52,7 +52,8 @@ source_env_file() { source_env_file_unless_provided_oauth() { local env_file="${1}" - # gracefully bypass sourcing env file if these 4 OAUTH values are provided + # gracefully bypass sourcing env file if these 4 OAUTH_ values are provided, i.e. as secrets + # via build automation if [[ \ -n "${OAUTH_ISSUER}" && \ -n "${OAUTH_CLIENTID}" && \ @@ -63,22 +64,25 @@ source_env_file_unless_provided_oauth() { return 0 fi + echo_info "OAUTH_ values are not provided by environment variables. Sourcing ${env_file} env file." source_env_file "${env_file}" } source_env_file_unless_provided_okta() { local env_file="${1}" - # gracefully bypass sourcing env file if these 4 OAUTH values are provided + # gracefully bypass sourcing env file if these 3 OKTA_ values are provided, i.e. as secrets + # via build automation if [[ \ -n "${OKTA_URL}" && \ -n "${OKTA_USERNAME}" && \ -n "${OKTA_PASSWORD}" \ ]]; then - echo_info "Okta values are provided by environment variables. Not sourcing ${env_file} env file." + echo_info "OKTA_ values are provided by environment variables. Not sourcing ${env_file} env file." return 0 fi + echo_info "Okta values are not provided by environment variables. Sourcing ${env_file} env file." source_env_file "${env_file}" } From e120cf2a08992b07a8845e31abcd5ec4834add0d Mon Sep 17 00:00:00 2001 From: Huey Date: Mon, 18 Dec 2023 18:13:41 -0800 Subject: [PATCH 22/22] Update application-dev.properties --- api/src/main/resources/config/application-dev.properties | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/api/src/main/resources/config/application-dev.properties b/api/src/main/resources/config/application-dev.properties index e849b9bb2..5de384416 100644 --- a/api/src/main/resources/config/application-dev.properties +++ b/api/src/main/resources/config/application-dev.properties @@ -14,7 +14,10 @@ management.endpoint.health.show-details=always spring.flyway.enabled=true # Common debuging log levels for OSMT development -#logging.level.org.springframework.data.elasticsearch.client.WIRE=trace -#logging.level.org.elasticsearch.client.RestClient=DEBUG -#logging.level.org.springframework=DEBUG +logging.level.org.springframework.data.elasticsearch.client.WIRE=trace +logging.level.org.elasticsearch.client.RestClient=DEBUG +logging.level.org.springframework=DEBUG logging.level.edu.wgu.osmt=DEBUG + +app.enableRoles=false +app.allowPublicLists=false