From 52d651a691bfea33016072305327a92802e69c89 Mon Sep 17 00:00:00 2001 From: Christopher Raquet Date: Mon, 3 Mar 2025 15:58:02 +0100 Subject: [PATCH 1/8] Try to add OTEL --- build.gradle | 12 +++++ collector-config.yaml | 19 +++++++ config/application-default.properties | 18 ++++++- .../configuration/OpenTelemetryConfig.java | 52 +++++++++++++++++++ src/main/resources/log4j.properties | 12 ----- src/main/resources/log4j2.xml | 11 ++++ 6 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 collector-config.yaml create mode 100644 src/main/java/edu/kit/datamanager/pit/configuration/OpenTelemetryConfig.java delete mode 100644 src/main/resources/log4j.properties create mode 100644 src/main/resources/log4j2.xml diff --git a/build.gradle b/build.gradle index 06a36087..0dc862e2 100644 --- a/build.gradle +++ b/build.gradle @@ -51,6 +51,12 @@ ext { springDocVersion = '2.8.4' } +dependencyManagement { + imports { + mavenBom("io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom:2.13.3") + } +} + dependencies { // Due to the spring boot gradle plugin, we can omit versions in org.springframework.* // dependencies. It will automatically choose the fitting ones. @@ -89,6 +95,12 @@ dependencies { implementation("net.handle:handle-client:9.3.1") + implementation("io.opentelemetry.instrumentation:opentelemetry-spring-boot-starter") + implementation("io.opentelemetry.contrib:opentelemetry-samplers:1.44.0-alpha") +// implementation 'io.opentelemetry:opentelemetry-api:1.29.0' +// implementation 'io.opentelemetry:opentelemetry-sdk:1.29.0' +// implementation("io.opentelemetry:opentelemetry-exporter-zipkin:1.47.0") + testImplementation(platform('org.junit:junit-bom:5.11.4')) testImplementation('org.junit.jupiter:junit-jupiter') testImplementation('org.junit.jupiter:junit-jupiter-params') diff --git a/collector-config.yaml b/collector-config.yaml new file mode 100644 index 00000000..0ae18ced --- /dev/null +++ b/collector-config.yaml @@ -0,0 +1,19 @@ +receivers: + otlp: + protocols: + http: + endpoint: "0.0.0.0:4318" +exporters: + debug: + verbosity: detailed +service: + pipelines: + metrics: + receivers: [otlp] + exporters: [debug] + traces: + receivers: [otlp] + exporters: [debug] + logs: + receivers: [otlp] + exporters: [debug] diff --git a/config/application-default.properties b/config/application-default.properties index 1ee66713..9640f786 100644 --- a/config/application-default.properties +++ b/config/application-default.properties @@ -56,7 +56,7 @@ management.endpoints.web.exposure.include: * # overwhelming. #logging.level.root: ERROR #logging.level.edu.kit.datamanager.doip:TRACE -logging.level.edu.kit: WARN +logging.level.edu.kit: DEBUG #logging.level.org.springframework.transaction: TRACE logging.level.org.springframework: WARN logging.level.org.springframework.amqp: WARN @@ -153,7 +153,7 @@ pit.pidsystem.handle.baseURI = https://hdl.handle.net/ # - IN_MEMORY (default, sandboxed, non-permanent PIDs, for short testing / demonstration only), # - LOCAL (sandboxed, uses local database, no public PIDs!, for long term testing or special use-cases), # - HANDLE_PROTOCOL (recommended, for real FAIR Digital Objects), -pit.pidsystem.implementation = LOCAL +pit.pidsystem.implementation = IN_MEMORY # If you chose IN_MEMORY, no further configuration is required. # If you chose LOCAL, no further configuration is required. # If you chose HANDLE_PROTOCOL, you need to set up your prefix and its key/certificate: @@ -275,3 +275,17 @@ spring.datasource.password: secure_me # Do not change ddl-auto if you do not know what you are doing: # https://docs.spring.io/spring-boot/docs/1.1.0.M1/reference/html/howto-database-initialization.html spring.jpa.hibernate.ddl-auto: update + + +management.otlp.logging.endpoint=http://localhost:4318/v1/logs + +management.otlp.metrics.export.url=http://localhost:4318/v1/metrics + +management.tracing.sampling.probability=1 + +otel.instrumentation.log4j-appender.enabled=true + +#otel.propagators=tracecontext +#otel.resource.attributes.deployment.environment=dev +#otel.resource.attributes.service.name=cart +#otel.resource.attributes.service.namespace=shop \ No newline at end of file diff --git a/src/main/java/edu/kit/datamanager/pit/configuration/OpenTelemetryConfig.java b/src/main/java/edu/kit/datamanager/pit/configuration/OpenTelemetryConfig.java new file mode 100644 index 00000000..c601f99c --- /dev/null +++ b/src/main/java/edu/kit/datamanager/pit/configuration/OpenTelemetryConfig.java @@ -0,0 +1,52 @@ +package edu.kit.datamanager.pit.configuration; + +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.contrib.sampler.RuleBasedRoutingSampler; +import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; +import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; +import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.semconv.UrlAttributes; +import java.util.Collections; +import java.util.Map; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class OpenTelemetryConfig { + + @Bean + public AutoConfigurationCustomizerProvider otelCustomizer() { + return p -> + p.addSamplerCustomizer(this::configureSampler) + .addSpanExporterCustomizer(this::configureSpanExporter); + } + + /** suppress spans for actuator endpoints */ + private RuleBasedRoutingSampler configureSampler(Sampler fallback, ConfigProperties config) { + return RuleBasedRoutingSampler.builder(SpanKind.SERVER, fallback) + .drop(UrlAttributes.URL_PATH, "^/actuator") + .build(); + } + + /** + * Configuration for the OTLP exporter. This configuration will replace the default OTLP exporter, + * and will add a custom header to the requests. + */ + private SpanExporter configureSpanExporter(SpanExporter exporter, ConfigProperties config) { + if (exporter instanceof OtlpHttpSpanExporter) { + return ((OtlpHttpSpanExporter) exporter).toBuilder().setHeaders(this::headers).build(); + } + return exporter; + } + + private Map headers() { + return Collections.singletonMap("Authorization", "Bearer " + refreshToken()); + } + + private String refreshToken() { + // e.g. read the token from a kubernetes secret + return "token"; + } +} diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties deleted file mode 100644 index df0583fb..00000000 --- a/src/main/resources/log4j.properties +++ /dev/null @@ -1,12 +0,0 @@ -# Set root logger level to DEBUG and its only appender to A1. -log4j.rootLogger=WARN, A1 - -# A1 is set to be a ConsoleAppender. -log4j.appender.A1=org.apache.log4j.ConsoleAppender - -# A1 uses PatternLayout. -log4j.appender.A1.layout=org.apache.log4j.PatternLayout -log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n - -# https://www.cert.kit.edu/downloads/20211210_BSI_Warnung_Log4j.pdf -log4j2.formatMsgNoLookups=true diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml new file mode 100644 index 00000000..fa19b951 --- /dev/null +++ b/src/main/resources/log4j2.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file From a5095ce6c5fbee0bd2e1d0abae7191db04a15312 Mon Sep 17 00:00:00 2001 From: Maximilian Inckmann Date: Thu, 14 Aug 2025 19:41:14 +0200 Subject: [PATCH 2/8] added observability features Signed-off-by: Maximilian Inckmann --- config/application-default.properties | 210 +-- .../PidRecordElasticWrapper.java | 22 +- .../pit/pidgeneration/PidSuffix.java | 66 +- .../generators/PidSuffixGenLowerCase.java | 23 +- .../generators/PidSuffixGenPrefixed.java | 25 +- .../generators/PidSuffixGenUpperCase.java | 23 +- .../pit/pidsystem/IIdentifierSystem.java | 334 ++--- .../impl/InMemoryIdentifierSystem.java | 81 +- .../impl/handle/HandleProtocolAdapter.java | 115 +- .../pidsystem/impl/local/LocalPidSystem.java | 96 +- .../impl/EmbeddedStrictValidatorStrategy.java | 86 +- .../pit/pitservice/impl/TypingService.java | 328 ++--- .../datamanager/pit/resolver/Resolver.java | 38 +- .../pit/typeregistry/impl/TypeApi.java | 67 +- .../schema/DtrTestSchemaGenerator.java | 34 +- .../schema/SchemaSetGenerator.java | 29 +- .../schema/TypeApiSchemaGenerator.java | 31 +- .../converter/SimplePidRecordConverter.java | 51 +- .../pit/web/impl/TypingRESTResourceImpl.java | 1123 +++++++++-------- 19 files changed, 1644 insertions(+), 1138 deletions(-) diff --git a/config/application-default.properties b/config/application-default.properties index 9640f786..4df22235 100644 --- a/config/application-default.properties +++ b/config/application-default.properties @@ -11,82 +11,84 @@ # More specific properties are documented within this file. ### General Spring Boot Settings ### -# When to include "message" attribute in HTTP responses on uncatched exceptions. -server.error.include-message: always +# When to include the "message" attribute in HTTP responses on uncatched exceptions. +spring.application.name=typed-pid-maker +#spring.profiles.active=default +server.error.include-message=always +springdoc.api-docs.enabled=true +springdoc.swagger-ui.enabled=true springdoc.show-actuator=true # Do __not__ change these settings below: spring.main.allow-bean-definition-overriding=true -spring.data.rest.detection-strategy:annotated +spring.data.rest.detection-strategy=annotated ##################################################### + ########################### ### Port, SSL, Security ### ########################### - -server.port: 8090 +server.port=8090 #server.ssl.key-store: keystore.p12 #server.ssl.key-store-password: test123 #server.ssl.keyStoreType: PKCS12 #server.ssl.keyAlias: tomcat - -# Data transfer settings, e.g. transfer compression and multipart message size. +# Data transfer settings, e.g. transfer compression and multipart message size. # The properties max-file-size and max-request-size define the maximum size of files # transferred to and from the repository. Setting them to -1 removes all limits. -server.compression.enabled: false -spring.servlet.multipart.max-file-size: 100MB -spring.servlet.multipart.max-request-size: 100MB - -# *Generic* Spring Management Endpoint Settings. By default, the health endpoint will be +server.compression.enabled=false +spring.servlet.multipart.max-file-size=100MB +spring.servlet.multipart.max-request-size=100MB +# *Generic* Spring Management Endpoint Settings. By default, the health endpoint will be # enabled to apply service monitoring including detailed information. # Furthermore, all endpoints will be exposed to external access. If this is not desired, # just comment the property 'management.endpoints.web.exposure.include' in order to only # allow local access. -management.endpoint.health.access: unrestricted -management.endpoint.health.show-details: ALWAYS -management.endpoint.health.sensitive: false -management.endpoints.web.exposure.include: * +management.endpoint.health.access=unrestricted +management.endpoint.health.show-details=ALWAYS +#management.endpoint.health.sensitive=false +management.endpoints.web.exposure.include=* + ############### ### Logging ### ############### - # Logging Settings. Most logging of KIT DM is performed on TRACE level. However, if you # plan to enable logging with this granularity it is recommended to this only for # a selection of a few packages. Otherwise, the amount of logging information might be # overwhelming. #logging.level.root: ERROR #logging.level.edu.kit.datamanager.doip:TRACE -logging.level.edu.kit: DEBUG +logging.level.edu.kit=INFO #logging.level.org.springframework.transaction: TRACE -logging.level.org.springframework: WARN -logging.level.org.springframework.amqp: WARN +logging.level.org.springframework=INFO +logging.level.org.springframework.amqp=WARN #logging.level.com.zaxxer.hikari: ERROR -logging.level.edu.kit.datamanager.pit.cli: INFO +logging.level.edu.kit.datamanager.pit.cli=INFO +#logging.config=classpath:logback-spring.xml + ###################### ### Authentication ### ###################### - -# Enable/disable (default) authentication. If authentication is enabled, a separate +# Enable/disable (default) authentication. If authentication is enabled, a separate # Authentication Service should be used in order to obtain JSON Web Tokens holding # login information. The token has then to be provided within the Authentication header # of each HTTP request with a value of 'Bearer ' without quotes, replacing # be the token obtained from the authentication service. # A token needs a "username" in its payload. A minimal token therefore may look like this: # https://jwt.io/#debugger-io?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VybmFtZSI6InVzZXIifQ.pfZuRuxbj_izZlCnmotWHQuH00BJ35CbjpHILpuQU70 -repo.auth.enabled: false - +repo.auth.enabled=false # The jwtSecret is the mutual secret between all trusted services. This means, that if # authentication is enabled, the jwtSecret used by the Authentication Service to sign # issued JWTokens must be the same as the jwtSecret of the repository in order to # be able to validate the signature. By default, the secret should be selected randomly # and with a sufficient length. -repo.auth.jwtSecret: vkfvoswsohwrxgjaxipuiyyjgubggzdaqrcuupbugxtnalhiegkppdgjgwxsmvdb +repo.auth.jwtSecret=vkfvoswsohwrxgjaxipuiyyjgubggzdaqrcuupbugxtnalhiegkppdgjgwxsmvdb + ############################### ### Keycloak Authentication ### ############################### - spring.autoconfigure.exclude=org.keycloak.adapters.springboot.KeycloakAutoConfiguration,org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration #keycloakjwt.jwk-url=http://localhost:8080/auth/realms/myrealm/protocol/openid-connect/certs #keycloakjwt.resource=keycloak-angular @@ -94,36 +96,34 @@ spring.autoconfigure.exclude=org.keycloak.adapters.springboot.KeycloakAutoConfig ##keycloakjwt.connect-timeoutms=500 # optional ##keycloakjwt.read-timeoutms=500 # optional # -#keycloak.realm = myrealm -#keycloak.auth-server-url = http://localhost:8080/auth -#keycloak.resource = keycloak-angular +#keycloak.realm=myrealm +#keycloak.auth-server-url=http://localhost:8080/auth +#keycloak.resource=keycloak-angular + ############################################ ### Elastic Indexing and search endpoint ### ######## (requires Elasticsearch 8) ######## ############################################ - # enables search endpoint at /api/v1/search -repo.search.enabled: false -repo.search.index: * -management.health.elasticsearch.enabled: false - +repo.search.enabled=false +repo.search.index=* +management.health.elasticsearch.enabled=false # TO BE REMOVED! -repo.search.url: http://localhost:9200 +repo.search.url=http://localhost:9200 # Soon will be: #spring.elasticsearch.uris=http://localhost:9200 #spring.elasticsearch.username=user #spring.elasticsearch.password=secret #spring.elasticsearch.socket-timeout=10s - # Due to bug in spring cloud gateway # https://github.com/spring-cloud/spring-cloud-gateway/issues/3154 spring.cloud.gateway.proxy.sensitive=content-length + ################# ### Messaging ### ################# - # Enable (default)/disable messaging. The messaging functionality requires a RabbitMQ # server receiving and distributing the messages sent by this service. The server is # accessed via repo.messaging.hostname and repo.messaging.port @@ -140,48 +140,44 @@ repo.messaging.sender.exchange=record_events # E.g. if a resource has been created, the repository may has to perform additional # ingest steps. Therefore, special handlers can be added which will be executed at the # configured repo.schedule.rate if a new message has been received. -repo.schedule.rate:1000 +repo.schedule.rate=1000 + ####################################################### ##################### PIT Service ##################### ####################################################### # Standard resolver for Handle PIDs. Should usually stay like this. -pit.pidsystem.handle.baseURI = https://hdl.handle.net/ - +pit.pidsystem.handle.baseURI=https://hdl.handle.net/ ### Choosing and configuring the PID system ### # Available implementations: # - IN_MEMORY (default, sandboxed, non-permanent PIDs, for short testing / demonstration only), # - LOCAL (sandboxed, uses local database, no public PIDs!, for long term testing or special use-cases), # - HANDLE_PROTOCOL (recommended, for real FAIR Digital Objects), -pit.pidsystem.implementation = IN_MEMORY +pit.pidsystem.implementation=LOCAL # If you chose IN_MEMORY, no further configuration is required. # If you chose LOCAL, no further configuration is required. # If you chose HANDLE_PROTOCOL, you need to set up your prefix and its key/certificate: -#pit.pidsystem.handle-protocol.credentials.handleIdentifierPrefix = 21.T11981 -#pit.pidsystem.handle-protocol.credentials.userHandle = 21.T11981/USER01 -#pit.pidsystem.handle-protocol.credentials.privateKeyPath = test_prefix_data/21.T11981_USER01_300_privkey.bin - +#pit.pidsystem.handle-protocol.credentials.handleIdentifierPrefix=21.T11981 +#pit.pidsystem.handle-protocol.credentials.userHandle=21.T11981/USER01 +#pit.pidsystem.handle-protocol.credentials.privateKeyPath=test_prefix_data/21.T11981_USER01_300_privkey.bin # The handle system supports the redirection of web browsers to a URL. # If your records may have such a URL stored in an attribute, you can # list the attributes here. The first attribute to be found will have # its value copied to a handle specific attribute (with key "URL"), # enabling URL redirection. Only affects the handle system! -# Obligation: Optional (option missing = empty list) -pit.pidsystem.handle-protocol.handleRedirectAttributes = {'21.T11148/b8457812905b83046284'} - +# Obligation: Optional (option missing=empty list) +pit.pidsystem.handle-protocol.handleRedirectAttributes={'21.T11148/b8457812905b83046284'} ### Base URL for the DTR used. ### # Currently, we support the DTRs of GWDG/ePIC. -pit.typeregistry.baseURI = https://typeapi.lab.pidconsortium.net +pit.typeregistry.baseURI=https://typeapi.lab.pidconsortium.net # If the attribute(s) keys/types in your PID records are not being # recognized as such, please contact us. # As a workaround, add them to this list. -# pit.validation.profileKeys = {} - +# pit.validation.profileKeys={} ### As this service is a RESTful serice without GUI, CSRF protection is not required. ### -pit.security.enable-csrf: false +pit.security.enable-csrf=false ### You may define patterns here for services which are allowed for communication. (CORS) ### -pit.security.allowedOriginPattern: http*://localhost:[*] - +pit.security.allowedOriginPattern=http*://localhost:[*] ### Caching settings for validation ### # The maximum number of entries in the cache. # pit.typeregistry.cache.maxEntries:1000 @@ -189,12 +185,11 @@ pit.security.allowedOriginPattern: http*://localhost:[*] # The time in minutes after which Entries will expire, starting from the # last update. # pit.typeregistry.cache.lifetimeMinutes:10 - # Profiles may disallow additional attributes in the PID records. This # option may be used to override this behavior for this instance. # If set to false, it will behave as the profiles describe. # If set to true, additional attributes will always be allowed. -pit.validation.alwaysAllowAdditionalAttributes = true +pit.validation.alwaysAllowAdditionalAttributes=true ### DANGEROUS OPTIONS! Please read carefully! ######################################## # This will disable validation. It is only meant for testing and rare cases @@ -204,32 +199,29 @@ pit.validation.alwaysAllowAdditionalAttributes = true # pit.validation.strategy=none-debug ### DANGEROUS OPTIONS! Please read carefully! ######################################## + ####################################################### #################### PID GENERATOR #################### ####################################################### - # The PID generator to use for the suffix. Possible values: # "uuid4": generates a UUID v4 (random) PID suffix. # "hex-chunks": generates hex-chunks. Each chunk is four characters long. Example: 1D6C-152C-C9E0-C136-1509 -pit.pidgeneration.mode = uuid4 - +pit.pidgeneration.mode=uuid4 # A prefix for branding, in addition to the PID system prefix. # Structure: -# Example: branding-prefix = "my-project.", system-prefix = "21.T11981", suffix = "12345" -# => PID = "21.T11981/my-project.12345" +# Example: branding-prefix="my-project.", system-prefix="21.T11981", suffix="12345" +# => PID="21.T11981/my-project.12345" # -# pit.pidgeneration.branding-prefix = my-project. - +# pit.pidgeneration.branding-prefix=my-project. # Applies a casing on the PIDs after generation (see "mode" property). Possible values: # "lower": all characters are lower case # "upper": all characters are upper case # "unmodified": no casing is applied after generation. Result depends fully on the generator. -pit.pidgeneration.casing = lower - +pit.pidgeneration.casing=lower # Affects chunk-based generation modes (see pid.pidgeneration.mode) only. # Defines the number of chunks the generator should generate for each PID. # Default: 4 -# pit.pidgeneration.num-chunks = 4 +# pit.pidgeneration.num-chunks=4 ### DANGEROUS OPTIONS! Please read carefully! ######################################## # Please keep this option as a last resort vor special use-cases @@ -238,12 +230,13 @@ pit.pidgeneration.casing = lower # a gateway which will manage your custom PIDs. # NOTE! If you do not already include the configured prefix in the PID, it will be appended. # This means that you can not create PIDs with a suffix starting with the system prefix. -# Example: system prefix = "abc", suffix = abcdef -# => PID = "abc/def" (delimiter may depend on PID system) +# Example: system prefix="abc", suffix=abcdef +# => PID="abc/def" (delimiter may depend on PID system) # -# pit.pidgeneration.custom-client-pids-enabled = false +# pit.pidgeneration.custom-client-pids-enabled=false ### DANGEROUS OPTIONS! Please read carefully! ######################################## + ################################ ######## Database ############## ################################ @@ -252,40 +245,81 @@ pit.pidgeneration.casing = lower ### system is set to LOCAL ### ### - Required for messaging ### ################################ - # This database will always run, as it is also required for the messaging feature, # but for the messaging it is not required to be persistent. # But the service will also use this database to store known PIDs. # This can be used as a backup or documentation of all PIDs. # The following properties can (and should) be set. - # When to store PIDs in the local database ("known PIDs") -pit.storage.strategy: keep-resolved-and-modified +pit.storage.strategy=keep-resolved-and-modified #pit.storage.strategy: keep-resolved # The driver determines the database system to start. Other drivers are untested, but may work. -spring.datasource.driver-class-name: org.h2.Driver +spring.datasource.driver-class-name=org.h2.Driver # Next, please choose a location for the database file on your file system. # WARNING: If no url is being defined, an in-memory database is being used, # loosing all data on restart. # WARNING: Change the DB to be stored somewhere outside of /tmp! -spring.datasource.url: jdbc:h2:file:/tmp/database;MODE=LEGACY;NON_KEYWORDS=VALUE +spring.datasource.url=jdbc:h2:file:/tmp/database;MODE=LEGACY;NON_KEYWORDS=VALUE # Credentials for the database: -spring.datasource.username: typid -spring.datasource.password: secure_me +spring.datasource.username=typid +spring.datasource.password=secure_me # Do not change ddl-auto if you do not know what you are doing: # https://docs.spring.io/spring-boot/docs/1.1.0.M1/reference/html/howto-database-initialization.html -spring.jpa.hibernate.ddl-auto: update +spring.jpa.hibernate.ddl-auto=update -management.otlp.logging.endpoint=http://localhost:4318/v1/logs - +################################ +####### Observability ########## +################################ +# Generic OpenTelemetry Configuration +#management.endpoints.web.exposure.include=* +#management.endpoint.health.show-details=always +management.endpoint.prometheus.access=unrestricted +management.metrics.distribution.sla.http.server.requests=100ms,500ms,1000ms +management.metrics.export.defaults.step=15s +management.metrics.distribution.percentiles-histogram.http.server.requests=true +management.metrics.tags.service_name=${spring.application.name} +management.metrics.tags.environment=${spring.profiles.active} +management.prometheus.metrics.export.enabled=false +otel.java.global-autoconfigure.enabled=true +otel.instrumentation.micrometer.enabled=true +otel.service.name=${spring.application.name} + +# OpenTelemetry Metrics Configuration +otel.metrics.exporter=otlp +otel.exporter.otlp.endpoint=http://localhost:4318 +otel.exporter.otlp.protocol=http/protobuf +management.otlp.metrics.export.enabled=true +management.otlp.metrics.export.step=2s management.otlp.metrics.export.url=http://localhost:4318/v1/metrics -management.tracing.sampling.probability=1 - -otel.instrumentation.log4j-appender.enabled=true - -#otel.propagators=tracecontext -#otel.resource.attributes.deployment.environment=dev -#otel.resource.attributes.service.name=cart -#otel.resource.attributes.service.namespace=shop \ No newline at end of file +# OpenTelemetry Logging Configuration +management.otlp.logging.export.enabled=true +management.otlp.logging.endpoint=http://localhost:4318/v1/logs +#otel.instrumentation.logback-appender.experimental.capture-mdc-attributes=* +otel.instrumentation.logback-appender.experimental-log-attributes=true +otel.instrumentation.logback-appender.experimental.capture-code-attributes=true +otel.instrumentation.logback-appender.experimental.capture-marker-attribute=true +otel.instrumentation.logback-appender.experimental.capture-mdc-attributes=trace_id,span_id +#logging.pattern.level=%5p [${spring.application.name:},%X{trace_id:-},%X{span_id:-}] +logging.context.enabled=true + +# Tracing Configuration +management.tracing.sampling.probability=1.0 +management.otlp.tracing.endpoint=http://localhost:4318/v1/traces +management.httpexchanges.recording.enabled=true +management.tracing.baggage.correlation.enabled=true +management.tracing.opentelemetry.export.include-unsampled=true +management.observations.annotations.enabled=true +otel.instrumentation.http.client.emit-experimental-telemetry=true +otel.instrumentation.runtime-telemetry-java17.enabled=true +otel.instrumentation.spring-webmvc.enabled=true +otel.instrumentation.annotations.enabled=true +otel.instrumentation.http.client.capture-request-headers=true +otel.instrumentation.http.client.capture-response-headers=true +otel.instrumentation.http.client.experimental.redact-query-parameters=false +otel.instrumentation.jdbc.experimental.transaction.enabled=true +otel.resource.attributes.exclude=process.command_args,process.command_line +otel.propagators=tracecontext,baggage +otel.traces.sampler=parentbased_traceidratio +otel.traces.sampler.arg=1 \ No newline at end of file diff --git a/src/main/java/edu/kit/datamanager/pit/elasticsearch/PidRecordElasticWrapper.java b/src/main/java/edu/kit/datamanager/pit/elasticsearch/PidRecordElasticWrapper.java index 5f8ea750..1ee67d5a 100644 --- a/src/main/java/edu/kit/datamanager/pit/elasticsearch/PidRecordElasticWrapper.java +++ b/src/main/java/edu/kit/datamanager/pit/elasticsearch/PidRecordElasticWrapper.java @@ -1,5 +1,5 @@ /* - * Copyright 2023 Karlsruhe Institute of Technology. + * Copyright (c) 2025 Karlsruhe Institute of Technology. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,22 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + */ package edu.kit.datamanager.pit.elasticsearch; import edu.kit.datamanager.pit.domain.Operations; import edu.kit.datamanager.pit.domain.PIDRecord; import edu.kit.datamanager.pit.pidsystem.impl.local.PidDatabaseObject; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.WithSpan; import jakarta.persistence.ElementCollection; import jakarta.persistence.FetchType; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.data.annotation.Id; @@ -37,6 +32,10 @@ import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; +import java.io.IOException; +import java.util.*; + +@Observed @Document(indexName = "typedpidmaker") public class PidRecordElasticWrapper { @@ -55,8 +54,9 @@ public class PidRecordElasticWrapper { private Date lastUpdate; @Field(type = FieldType.Text) - private List read = new ArrayList<>(); + private final List read = new ArrayList<>(); + @WithSpan(kind = SpanKind.INTERNAL) public PidRecordElasticWrapper(PIDRecord pidRecord, Operations dateOperations) { pid = pidRecord.getPid(); PidDatabaseObject simple = new PidDatabaseObject(pidRecord); diff --git a/src/main/java/edu/kit/datamanager/pit/pidgeneration/PidSuffix.java b/src/main/java/edu/kit/datamanager/pit/pidgeneration/PidSuffix.java index df93bc5c..30da9583 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidgeneration/PidSuffix.java +++ b/src/main/java/edu/kit/datamanager/pit/pidgeneration/PidSuffix.java @@ -1,52 +1,74 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.pidgeneration; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.WithSpan; + /** * A thin wrapper around a suffix string. - * + *

* The purpose is to indicate that this string is missing the prefix part and is * not used as a PID accidentially. */ public class PidSuffix { - private String suffix; + private final String suffix; public PidSuffix(String suffix) { this.suffix = suffix; } + /** + * Ensures a string is prefixed with the given prefix. + *

+ * It makes sure the prefix is not added, if the string already starts with the + * prefix. + * + * @param maybeSuffix the string to prefix. + * @param prefix the prefix to add. + * @return the string with the prefix added. + */ + @WithSpan(kind = SpanKind.INTERNAL) + public static String asPrefixedChecked(String maybeSuffix, String prefix) { + if (!maybeSuffix.startsWith(prefix)) { + return prefix + maybeSuffix; + } else { + return maybeSuffix; + } + } + /** * Returns the suffix string. - * + * * @return the suffix without any prefix. */ + @WithSpan(kind = SpanKind.INTERNAL) public String get() { return suffix; } /** * Returns the suffix string with the given prefix prepended. - * + * * @param prefix the prefix to prepend. * @return the prefix + suffix. */ + @WithSpan(kind = SpanKind.INTERNAL) public String getWithPrefix(String prefix) { return prefix + suffix; } - - /** - * Ensures a string is prefixed with the given prefix. - * - * It makes sure the prefix is not added, if the string already starts with the - * prefix. - * - * @param maybeSuffix the string to prefix. - * @param prefix the prefix to add. - * @return the string with the prefix added. - */ - public static String asPrefixedChecked(String maybeSuffix, String prefix) { - if (!maybeSuffix.startsWith(prefix)) { - return prefix + maybeSuffix; - } else { - return maybeSuffix; - } - } } diff --git a/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenLowerCase.java b/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenLowerCase.java index 581a6bd1..2e30d936 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenLowerCase.java +++ b/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenLowerCase.java @@ -1,21 +1,42 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.pidgeneration.generators; import edu.kit.datamanager.pit.pidgeneration.PidSuffix; import edu.kit.datamanager.pit.pidgeneration.PidSuffixGenerator; +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.WithSpan; /** * Generates a PID suffix based on a contained generator and returns the result * in lower case. */ +@Observed public class PidSuffixGenLowerCase implements PidSuffixGenerator { -private PidSuffixGenerator generator; + private final PidSuffixGenerator generator; public PidSuffixGenLowerCase(PidSuffixGenerator generator) { this.generator = generator; } @Override + @WithSpan(kind = SpanKind.INTERNAL) public PidSuffix generate() { return new PidSuffix(this.generator.generate().get().toLowerCase()); } diff --git a/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenPrefixed.java b/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenPrefixed.java index 3d62d766..8b5c9c31 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenPrefixed.java +++ b/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenPrefixed.java @@ -1,16 +1,36 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.pidgeneration.generators; import edu.kit.datamanager.pit.pidgeneration.PidSuffix; import edu.kit.datamanager.pit.pidgeneration.PidSuffixGenerator; +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.WithSpan; /** * Generates a PID suffix based on a contained generator and returns the result * prefixed with a customizable string. */ +@Observed public class PidSuffixGenPrefixed implements PidSuffixGenerator { - private PidSuffixGenerator generator; - private String prefix; + private final PidSuffixGenerator generator; + private final String prefix; public PidSuffixGenPrefixed(PidSuffixGenerator generator, String prefix) { this.generator = generator; @@ -18,6 +38,7 @@ public PidSuffixGenPrefixed(PidSuffixGenerator generator, String prefix) { } @Override + @WithSpan(kind = SpanKind.INTERNAL) public PidSuffix generate() { String suffix = this.generator.generate().get().toUpperCase(); return new PidSuffix(this.prefix.concat(suffix)); diff --git a/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenUpperCase.java b/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenUpperCase.java index e32fc2b5..0d31cf6d 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenUpperCase.java +++ b/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenUpperCase.java @@ -1,21 +1,42 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.pidgeneration.generators; import edu.kit.datamanager.pit.pidgeneration.PidSuffix; import edu.kit.datamanager.pit.pidgeneration.PidSuffixGenerator; +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.WithSpan; /** * Generates a PID suffix based on a contained generator and returns the result * in upper case. */ +@Observed public class PidSuffixGenUpperCase implements PidSuffixGenerator { - private PidSuffixGenerator generator; + private final PidSuffixGenerator generator; public PidSuffixGenUpperCase(PidSuffixGenerator generator) { this.generator = generator; } @Override + @WithSpan(kind = SpanKind.INTERNAL) public PidSuffix generate() { return new PidSuffix(this.generator.generate().get().toUpperCase()); } diff --git a/src/main/java/edu/kit/datamanager/pit/pidsystem/IIdentifierSystem.java b/src/main/java/edu/kit/datamanager/pit/pidsystem/IIdentifierSystem.java index a12eece6..5233fd36 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidsystem/IIdentifierSystem.java +++ b/src/main/java/edu/kit/datamanager/pit/pidsystem/IIdentifierSystem.java @@ -1,167 +1,167 @@ -package edu.kit.datamanager.pit.pidsystem; - -import edu.kit.datamanager.pit.common.ExternalServiceException; -import edu.kit.datamanager.pit.common.InvalidConfigException; -import edu.kit.datamanager.pit.common.PidAlreadyExistsException; -import edu.kit.datamanager.pit.common.PidNotFoundException; -import edu.kit.datamanager.pit.common.RecordValidationException; -import edu.kit.datamanager.pit.domain.PIDRecord; -import edu.kit.datamanager.pit.pidgeneration.PidSuffix; - -import java.util.Collection; -import java.util.Optional; - -/** - * Main abstraction interface towards the identifier system containing - * registered identifiers and associated state information. - * - */ -public interface IIdentifierSystem { - - /** - * Returns the configured prefix of this PID system. - * - * If this system can create PIDs, the prefix is the one it uses to create PIDs. - * Otherwise, it does not return a prefix. - * - * @return the prefix this system uses to create PIDs, if it can create PIDs, - * empty otherwise. - */ - public Optional getPrefix(); - - /** - * Appends the given PID to the prefix, if possible. - * - * It may not be possible if no prefix is present, or if the PID already starts - * with a the prefix. The returnes String is then exaxtly the same. - * - * @param pid the PID to append to the prefix. - * @return the PID with the prefix appended, if possible. - * @throws InvalidConfigException if the system can n. - */ - public default String appendPrefixIfAbsent(String pid) throws InvalidConfigException { - Optional prefix = this.getPrefix(); - if (prefix.isPresent() && !pid.startsWith(prefix.get())) { - return new PidSuffix(pid).getWithPrefix(prefix.get()); - } else { - return pid; - } - } - - /** - * Checks whether the given PID is already registered. - * - * @param pid the PID to check. - * @return true, if the PID is registered, false otherwise. - * @throws ExternalServiceException on commonication errors or errors on other - * services. - */ - public boolean isPidRegistered(String pid) throws ExternalServiceException; - - /** - * Checks whether the given PID is already registered. - * - * Assumes the PID to be the configured prefix of the system combined with the - * given suffix. - * - * @param suffix the given suffix, which, appended to the configured prefix, - * forms the PID to check. - * @return true, if the PID is registered, false otherwise. - * @throws ExternalServiceException on commonication errors or errors on other - * services. - * @throws InvalidConfigException if there is no prefix configured to append to - * the suffix. - */ - public default boolean isPidRegistered(PidSuffix suffix) throws ExternalServiceException, InvalidConfigException { - String prefix = getPrefix().orElseThrow(() -> new InvalidConfigException("This system cannot create PIDs.")); - return isPidRegistered(suffix.getWithPrefix(prefix)); - } - - /** - * Queries all properties from the given PID, independent of types. - * - * @param pid the PID to query the properties from. - * @return a PID information record with its PID and attribute-value-pairs. The - * property names will be empty strings. Contains all property values - * present in the record of the given PID. - * @throws PidNotFoundException if the pid is not registered. - * @throws ExternalServiceException on commonication errors or errors on other - * services. - */ - public PIDRecord queryPid(String pid) throws PidNotFoundException, ExternalServiceException; - - /** - * Registers a new PID with given property values. The method takes the PID from - * the record and treats it as a suffix. - * - * The method must process the given PID using the - * {@link #registerPid(PIDRecord)} method. - * - * @param pidRecord contains the initial PID record. - * @return the PID that was assigned to the record. - * @throws PidAlreadyExistsException if the PID already exists - * @throws ExternalServiceException if an error occured in communication with - * other services. - * @throws RecordValidationException if record validation errors occurred. - */ - public default String registerPid(final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException, RecordValidationException { - if (pidRecord.getPid() == null) { - throw new RecordValidationException(pidRecord, "PID must not be null."); - } - if (pidRecord.getPid().isEmpty()) { - throw new RecordValidationException(pidRecord, "PID must not be empty."); - } - pidRecord.setPid( - appendPrefixIfAbsent(pidRecord.getPid()) - ); - return registerPidUnchecked(pidRecord); - } - - /** - * Registers the given record with its given PID, without applying any checks. - * Recommended to use {@link #registerPid(PIDRecord)} instead. - * - * As an implementor, you can assume the PID to be not null, valid, - * non-registered, and prefixed. - * - * @param pidRecord the record to register. - * @return the PID that was assigned to the record. - * @throws PidAlreadyExistsException if the PID already exists - * @throws ExternalServiceException if an error occured in communication with - * other services. - */ - public String registerPidUnchecked(final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException; - - /** - * Updates an existing record with the new given values. If the PID in the given - * record is not valid, it will return false. - * - * @param pidRecord Assumes an existing, valid PID inside this record. - * @return false if there was no existing, valid PID in this record. - * @throws PidNotFoundException if PID is not registered. - * @throws ExternalServiceException if an error occured in communication with - * other services. - * @throws RecordValidationException if record validation errors occurred. - */ - public boolean updatePid(PIDRecord pidRecord) throws PidNotFoundException, ExternalServiceException, RecordValidationException; - - /** - * Remove the given PID. - * - * Obviously, this method is only for testing purposes, since we should not - * delete persistent identifiers. - * - * @param pid the PID to delete. - * @return true if the identifier was deleted, false if it did not exist. - */ - public boolean deletePid(String pid) throws ExternalServiceException; - - /** - * Returns all PIDs which are registered for the configured prefix. - * - * The result may be very large, use carefully. - * - * @return all PIDs which are registered for the configured prefix. - */ - public Collection resolveAllPidsOfPrefix() throws ExternalServiceException, InvalidConfigException; -} +package edu.kit.datamanager.pit.pidsystem; + +import edu.kit.datamanager.pit.common.ExternalServiceException; +import edu.kit.datamanager.pit.common.InvalidConfigException; +import edu.kit.datamanager.pit.common.PidAlreadyExistsException; +import edu.kit.datamanager.pit.common.PidNotFoundException; +import edu.kit.datamanager.pit.common.RecordValidationException; +import edu.kit.datamanager.pit.domain.PIDRecord; +import edu.kit.datamanager.pit.pidgeneration.PidSuffix; + +import java.util.Collection; +import java.util.Optional; + +/** + * Main abstraction interface towards the identifier system containing + * registered identifiers and associated state information. + * + */ +public interface IIdentifierSystem { + + /** + * Returns the configured prefix of this PID system. + * + * If this system can create PIDs, the prefix is the one it uses to create PIDs. + * Otherwise, it does not return a prefix. + * + * @return the prefix this system uses to create PIDs, if it can create PIDs, + * empty otherwise. + */ + public Optional getPrefix(); + + /** + * Appends the given PID to the prefix, if possible. + * + * It may not be possible if no prefix is present, or if the PID already starts + * with a the prefix. The returnes String is then exaxtly the same. + * + * @param pid the PID to append to the prefix. + * @return the PID with the prefix appended, if possible. + * @throws InvalidConfigException if the system can n. + */ + public default String appendPrefixIfAbsent(String pid) throws InvalidConfigException { + Optional prefix = this.getPrefix(); + if (prefix.isPresent() && !pid.startsWith(prefix.get())) { + return new PidSuffix(pid).getWithPrefix(prefix.get()); + } else { + return pid; + } + } + + /** + * Checks whether the given PID is already registered. + * + * @param pid the PID to check. + * @return true, if the PID is registered, false otherwise. + * @throws ExternalServiceException on commonication errors or errors on other + * services. + */ + public boolean isPidRegistered(String pid) throws ExternalServiceException; + + /** + * Checks whether the given PID is already registered. + * + * Assumes the PID to be the configured prefix of the system combined with the + * given suffix. + * + * @param suffix the given suffix, which, appended to the configured prefix, + * forms the PID to check. + * @return true, if the PID is registered, false otherwise. + * @throws ExternalServiceException on commonication errors or errors on other + * services. + * @throws InvalidConfigException if there is no prefix configured to append to + * the suffix. + */ + public default boolean isPidRegistered(PidSuffix suffix) throws ExternalServiceException, InvalidConfigException { + String prefix = getPrefix().orElseThrow(() -> new InvalidConfigException("This system cannot create PIDs.")); + return isPidRegistered(suffix.getWithPrefix(prefix)); + } + + /** + * Queries all properties from the given PID, independent of types. + * + * @param pid the PID to query the properties from. + * @return a PID information record with its PID and attribute-value-pairs. The + * property names will be empty strings. Contains all property values + * present in the record of the given PID. + * @throws PidNotFoundException if the pid is not registered. + * @throws ExternalServiceException on commonication errors or errors on other + * services. + */ + public PIDRecord queryPid(String pid) throws PidNotFoundException, ExternalServiceException; + + /** + * Registers a new PID with given property values. The method takes the PID from + * the record and treats it as a suffix. + * + * The method must process the given PID using the + * {@link #registerPid(PIDRecord)} method. + * + * @param pidRecord contains the initial PID record. + * @return the PID that was assigned to the record. + * @throws PidAlreadyExistsException if the PID already exists + * @throws ExternalServiceException if an error occured in communication with + * other services. + * @throws RecordValidationException if record validation errors occurred. + */ + public default String registerPid(final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException, RecordValidationException { + if (pidRecord.getPid() == null) { + throw new RecordValidationException(pidRecord, "PID must not be null."); + } + if (pidRecord.getPid().isEmpty()) { + throw new RecordValidationException(pidRecord, "PID must not be empty."); + } + pidRecord.setPid( + appendPrefixIfAbsent(pidRecord.getPid()) + ); + return registerPidUnchecked(pidRecord); + } + + /** + * Registers the given record with its given PID, without applying any checks. + * Recommended to use {@link #registerPid(PIDRecord)} instead. + * + * As an implementor, you can assume the PID to be not null, valid, + * non-registered, and prefixed. + * + * @param pidRecord the record to register. + * @return the PID that was assigned to the record. + * @throws PidAlreadyExistsException if the PID already exists + * @throws ExternalServiceException if an error occured in communication with + * other services. + */ + public String registerPidUnchecked(final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException; + + /** + * Updates an existing record with the new given values. If the PID in the given + * record is not valid, it will return false. + * + * @param pidRecord Assumes an existing, valid PID inside this record. + * @return false if there was no existing, valid PID in this record. + * @throws PidNotFoundException if PID is not registered. + * @throws ExternalServiceException if an error occured in communication with + * other services. + * @throws RecordValidationException if record validation errors occurred. + */ + public boolean updatePid(PIDRecord pidRecord) throws PidNotFoundException, ExternalServiceException, RecordValidationException; + + /** + * Remove the given PID. + * + * Obviously, this method is only for testing purposes, since we should not + * delete persistent identifiers. + * + * @param pid the PID to delete. + * @return true if the identifier was deleted, false if it did not exist. + */ + public boolean deletePid(String pid) throws ExternalServiceException; + + /** + * Returns all PIDs which are registered for the configured prefix. + * + * The result may be very large, use carefully. + * + * @return all PIDs which are registered for the configured prefix. + */ + public Collection resolveAllPidsOfPrefix() throws ExternalServiceException, InvalidConfigException; +} diff --git a/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/InMemoryIdentifierSystem.java b/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/InMemoryIdentifierSystem.java index 87e180f4..33fb9c41 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/InMemoryIdentifierSystem.java +++ b/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/InMemoryIdentifierSystem.java @@ -1,25 +1,43 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.pidsystem.impl; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import edu.kit.datamanager.pit.common.ExternalServiceException; -import edu.kit.datamanager.pit.common.InvalidConfigException; -import edu.kit.datamanager.pit.common.PidAlreadyExistsException; -import edu.kit.datamanager.pit.common.PidNotFoundException; -import edu.kit.datamanager.pit.common.RecordValidationException; +import edu.kit.datamanager.pit.common.*; import edu.kit.datamanager.pit.configuration.ApplicationProperties; import edu.kit.datamanager.pit.domain.PIDRecord; import edu.kit.datamanager.pit.pidsystem.IIdentifierSystem; - +import io.micrometer.core.annotation.Counted; +import io.micrometer.core.annotation.Timed; +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.SpanAttribute; +import io.opentelemetry.instrumentation.annotations.WithSpan; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.AutoConfigureAfter; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + /** * A simple basis for demonstrations or tests of the service. PIDs will be * stored in a HashMap and not stored anywhere else. @@ -27,13 +45,14 @@ @Component @AutoConfigureAfter(value = ApplicationProperties.class) @ConditionalOnExpression( - "#{ '${pit.pidsystem.implementation}' eq T(edu.kit.datamanager.pit.configuration.ApplicationProperties.IdentifierSystemImpl).IN_MEMORY.name() }" + "#{ '${pit.pidsystem.implementation}' eq T(edu.kit.datamanager.pit.configuration.ApplicationProperties.IdentifierSystemImpl).IN_MEMORY.name() }" ) +@Observed public class InMemoryIdentifierSystem implements IIdentifierSystem { private static final Logger LOG = LoggerFactory.getLogger(InMemoryIdentifierSystem.class); private static final String PREFIX = "sandboxed/"; - private Map records = new HashMap<>(); + private final Map records = new HashMap<>(); public InMemoryIdentifierSystem() { LOG.warn("Using in-memory identifier system. REGISTERED PIDs ARE NOT STORED PERMANENTLY."); @@ -45,26 +64,40 @@ public Optional getPrefix() { } @Override - public boolean isPidRegistered(String pid) throws ExternalServiceException { - return this.records.containsKey(pid); + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "memory_system_is_pid_registered", description = "Time taken to check if PID is registered in memory system") + @Counted(value = "memory_system_is_pid_registered_total", description = "Total number of PID registration checks") + public boolean isPidRegistered(@SpanAttribute String pid) throws ExternalServiceException { + return this.records.containsKey(pid); } @Override - public PIDRecord queryPid(String pid) throws PidNotFoundException, ExternalServiceException { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "memory_system_query_pid", description = "Time taken to query PID from memory system") + @Counted(value = "memory_system_query_pid_total", description = "Total number of PID queries") + public PIDRecord queryPid(@SpanAttribute String pid) throws PidNotFoundException, ExternalServiceException { PIDRecord pidRecord = this.records.get(pid); - if (pidRecord == null) { throw new PidNotFoundException(pid); } + if (pidRecord == null) { + throw new PidNotFoundException(pid); + } return pidRecord; } - + @Override - public String registerPidUnchecked(final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "memory_system_register_pid", description = "Time taken to register PID in memory system") + @Counted(value = "memory_system_register_pid_total", description = "Total number of PID registrations") + public String registerPidUnchecked(@SpanAttribute final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { this.records.put(pidRecord.getPid(), pidRecord); LOG.debug("Registered record with PID: {}", pidRecord.getPid()); return pidRecord.getPid(); } @Override - public boolean updatePid(PIDRecord record) throws PidNotFoundException, ExternalServiceException, RecordValidationException { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "memory_system_update_pid", description = "Time taken to update PID in memory system") + @Counted(value = "memory_system_update_pid_total", description = "Total number of PID updates") + public boolean updatePid(@SpanAttribute PIDRecord record) throws PidNotFoundException, ExternalServiceException, RecordValidationException { if (this.records.containsKey(record.getPid())) { this.records.put(record.getPid(), record); return true; @@ -73,11 +106,17 @@ public boolean updatePid(PIDRecord record) throws PidNotFoundException, External } @Override - public boolean deletePid(String pid) { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "memory_system_delete_pid", description = "Time taken to delete PID from memory system") + @Counted(value = "memory_system_delete_pid_total", description = "Total number of PID deletion attempts") + public boolean deletePid(@SpanAttribute String pid) { throw new UnsupportedOperationException("Deleting PIDs is against the P in PID."); } @Override + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "memory_system_resolve_all_pids", description = "Time taken to resolve all PIDs from memory system") + @Counted(value = "memory_system_resolve_all_pids_total", description = "Total number of resolve all PIDs requests") public Collection resolveAllPidsOfPrefix() throws ExternalServiceException, InvalidConfigException { return this.records.keySet().stream().filter(pid -> pid.startsWith(PREFIX)).collect(Collectors.toSet()); } diff --git a/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/handle/HandleProtocolAdapter.java b/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/handle/HandleProtocolAdapter.java index 5898bc27..abf97a86 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/handle/HandleProtocolAdapter.java +++ b/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/handle/HandleProtocolAdapter.java @@ -1,44 +1,52 @@ -package edu.kit.datamanager.pit.pidsystem.impl.handle; +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -import java.io.IOException; -import java.security.PrivateKey; -import java.security.spec.InvalidKeySpecException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; +package edu.kit.datamanager.pit.pidsystem.impl.handle; +import edu.kit.datamanager.pit.common.*; +import edu.kit.datamanager.pit.configuration.HandleCredentials; +import edu.kit.datamanager.pit.configuration.HandleProtocolProperties; +import edu.kit.datamanager.pit.domain.PIDRecord; +import edu.kit.datamanager.pit.pidsystem.IIdentifierSystem; import edu.kit.datamanager.pit.recordModifiers.RecordModifier; +import io.micrometer.core.annotation.Counted; +import io.micrometer.core.annotation.Timed; +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.SpanAttribute; +import io.opentelemetry.instrumentation.annotations.WithSpan; import jakarta.annotation.PostConstruct; import jakarta.validation.constraints.NotNull; - +import net.handle.api.HSAdapter; +import net.handle.api.HSAdapterFactory; +import net.handle.apps.batch.BatchUtil; +import net.handle.hdllib.*; import org.apache.commons.lang3.stream.Streams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.stereotype.Component; -import edu.kit.datamanager.pit.common.ExternalServiceException; -import edu.kit.datamanager.pit.common.InvalidConfigException; -import edu.kit.datamanager.pit.common.PidAlreadyExistsException; -import edu.kit.datamanager.pit.common.PidNotFoundException; -import edu.kit.datamanager.pit.common.RecordValidationException; -import edu.kit.datamanager.pit.configuration.HandleCredentials; -import edu.kit.datamanager.pit.configuration.HandleProtocolProperties; -import edu.kit.datamanager.pit.domain.PIDRecord; -import edu.kit.datamanager.pit.pidsystem.IIdentifierSystem; -import net.handle.api.HSAdapter; -import net.handle.api.HSAdapterFactory; -import net.handle.apps.batch.BatchUtil; -import net.handle.hdllib.HandleException; -import net.handle.hdllib.HandleResolver; -import net.handle.hdllib.HandleValue; -import net.handle.hdllib.PublicKeyAuthenticationInfo; -import net.handle.hdllib.SiteInfo; -import net.handle.hdllib.Util; + +import java.io.IOException; +import java.security.PrivateKey; +import java.security.spec.InvalidKeySpecException; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Uses the official java library to interact with the handle system using the @@ -46,6 +54,7 @@ */ @Component @ConditionalOnBean(HandleProtocolProperties.class) +@Observed public class HandleProtocolAdapter implements IIdentifierSystem { private static final Logger LOG = LoggerFactory.getLogger(HandleProtocolAdapter.class); @@ -72,7 +81,7 @@ public HandleProtocolAdapter(HandleProtocolProperties props) { * We use this method with the @PostConstruct annotation to run it * after the constructor and after springs autowiring is done properly * to make sure that all properties are already autowired. - * + * * @throws HandleException if a handle system error occurs. * @throws InvalidConfigException if the configuration is invalid, e.g. a path * does not lead to a file. @@ -116,7 +125,10 @@ public Optional getPrefix() { } @Override - public boolean isPidRegistered(final String pid) throws ExternalServiceException { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "handle_system_is_pid_registered", description = "Time taken to check if PID is registered in Handle system") + @Counted(value = "handle_system_is_pid_registered_total", description = "Total number of PID registration checks") + public boolean isPidRegistered(@SpanAttribute final String pid) throws ExternalServiceException { HandleValue[] recordProperties; try { recordProperties = this.client.resolveHandle(pid, null, null); @@ -131,7 +143,10 @@ public boolean isPidRegistered(final String pid) throws ExternalServiceException } @Override - public PIDRecord queryPid(final String pid) throws PidNotFoundException, ExternalServiceException { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "handle_system_query_pid", description = "Time taken to query PID from Handle system") + @Counted(value = "handle_system_query_pid_total", description = "Total number of PID queries") + public PIDRecord queryPid(@SpanAttribute final String pid) throws PidNotFoundException, ExternalServiceException { Collection allValues = this.queryAllHandleValues(pid); if (allValues.isEmpty()) { return null; @@ -143,7 +158,9 @@ public PIDRecord queryPid(final String pid) throws PidNotFoundException, Externa } @NotNull - protected Collection queryAllHandleValues(final String pid) throws PidNotFoundException, ExternalServiceException { + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "handle_system_query_all_values", description = "Time taken to query all handle values") + protected Collection queryAllHandleValues(@SpanAttribute final String pid) throws PidNotFoundException, ExternalServiceException { try { HandleValue[] values = this.client.resolveHandle(pid, null, null); return Stream.of(values) @@ -158,7 +175,10 @@ protected Collection queryAllHandleValues(final String pid) throws } @Override - public String registerPidUnchecked(final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "handle_system_register_pid", description = "Time taken to register PID in Handle system") + @Counted(value = "handle_system_register_pid_total", description = "Total number of PID registrations") + public String registerPidUnchecked(@SpanAttribute final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { // Add admin value for configured user only // TODO add options to add additional adminValues e.g. for user lists? ArrayList admin = new ArrayList<>(); @@ -169,7 +189,7 @@ public String registerPidUnchecked(final PIDRecord pidRecord) throws PidAlreadyE } ArrayList futurePairs = HandleBehavior.handleValuesFrom(preparedRecord, Optional.of(admin)); - HandleValue[] futurePairsArray = futurePairs.toArray(new HandleValue[] {}); + HandleValue[] futurePairsArray = futurePairs.toArray(new HandleValue[]{}); try { this.client.createHandle(preparedRecord.getPid(), futurePairsArray); @@ -185,7 +205,10 @@ public String registerPidUnchecked(final PIDRecord pidRecord) throws PidAlreadyE } @Override - public boolean updatePid(final PIDRecord pidRecord) throws PidNotFoundException, ExternalServiceException, RecordValidationException { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "handle_system_update_pid", description = "Time taken to update PID in Handle system") + @Counted(value = "handle_system_update_pid_total", description = "Total number of PID updates") + public boolean updatePid(@SpanAttribute final PIDRecord pidRecord) throws PidNotFoundException, ExternalServiceException, RecordValidationException { if (!this.isValidPID(pidRecord.getPid())) { return false; } @@ -251,7 +274,10 @@ public boolean updatePid(final PIDRecord pidRecord) throws PidNotFoundException, } @Override - public boolean deletePid(final String pid) throws ExternalServiceException { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "handle_system_delete_pid", description = "Time taken to delete PID from Handle system") + @Counted(value = "handle_system_delete_pid_total", description = "Total number of PID deletions") + public boolean deletePid(@SpanAttribute final String pid) throws ExternalServiceException { try { this.client.deleteHandle(pid); } catch (HandleException e) { @@ -265,6 +291,9 @@ public boolean deletePid(final String pid) throws ExternalServiceException { } @Override + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "handle_system_resolve_all_pids", description = "Time taken to resolve all PIDs from Handle system") + @Counted(value = "handle_system_resolve_all_pids_total", description = "Total number of resolve all PIDs requests") public Collection resolveAllPidsOfPrefix() throws ExternalServiceException, InvalidConfigException { HandleCredentials handleCredentials = this.props.getCredentials(); if (handleCredentials == null) { @@ -325,13 +354,15 @@ public Collection resolveAllPidsOfPrefix() throws ExternalServiceExcepti /** * Returns true if the PID is valid according to the following criteria: * - PID is valid according to isIdentifierRegistered - * - If a generator prefix is set, the PID is expedted to have this prefix. - * + * - If a generator prefix is set, the PID is expeted to have this prefix. + * * @param pid the identifier / PID to check. * @return true if PID is registered (and if has the generatorPrefix, if it - * exists). + * exists). */ - protected boolean isValidPID(final String pid) { + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "handle_system_is_valid_pid", description = "Time taken to validate PID") + protected boolean isValidPID(@SpanAttribute final String pid) { boolean isAuthMode = this.props.getCredentials() != null; if (isAuthMode && !pid.startsWith(this.props.getCredentials().getHandleIdentifierPrefix())) { return false; diff --git a/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/local/LocalPidSystem.java b/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/local/LocalPidSystem.java index 8190eb13..e5ab6437 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/local/LocalPidSystem.java +++ b/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/local/LocalPidSystem.java @@ -1,18 +1,31 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.pidsystem.impl.local; -import java.util.Collection; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import edu.kit.datamanager.pit.common.ExternalServiceException; -import edu.kit.datamanager.pit.common.InvalidConfigException; -import edu.kit.datamanager.pit.common.PidAlreadyExistsException; -import edu.kit.datamanager.pit.common.PidNotFoundException; -import edu.kit.datamanager.pit.common.RecordValidationException; +import edu.kit.datamanager.pit.common.*; import edu.kit.datamanager.pit.configuration.ApplicationProperties; import edu.kit.datamanager.pit.domain.PIDRecord; import edu.kit.datamanager.pit.pidsystem.IIdentifierSystem; - +import io.micrometer.core.annotation.Counted; +import io.micrometer.core.annotation.Timed; +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.SpanAttribute; +import io.opentelemetry.instrumentation.annotations.WithSpan; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -21,18 +34,22 @@ import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; +import java.util.Collection; +import java.util.Optional; +import java.util.stream.Collectors; + /** * A system that stores PIDs on the local machine, in its configured database. - * + *

* Purpose: This local system is made for demonstrations, preparations or local * tests, but may also be used for other cases where PIDs should not be public * (yet). - * + *

* Note: This system has its own PID string format and we can not guarantee that * you'll be able to register your PIDs later in another system with the same * format. If you need this feature, feel free to open an issue on GitHub: * https://github.com/kit-data-manager/pit-service - * + *

* Configuration: The database configuration of this service is done via the * `spring.datasource.*` properties. There is no configuration that controls a * separate database only for this system. Consider the InMemoryIdentifierSystem @@ -41,36 +58,35 @@ @Component @AutoConfigureAfter(value = ApplicationProperties.class) @ConditionalOnExpression( - "#{ '${pit.pidsystem.implementation}' eq T(edu.kit.datamanager.pit.configuration.ApplicationProperties.IdentifierSystemImpl).LOCAL.name() }" + "#{ '${pit.pidsystem.implementation}' eq T(edu.kit.datamanager.pit.configuration.ApplicationProperties.IdentifierSystemImpl).LOCAL.name() }" ) @Transactional +@Observed public class LocalPidSystem implements IIdentifierSystem { private static final Logger LOG = LoggerFactory.getLogger(LocalPidSystem.class); - + private static final String PREFIX = "sandboxed/"; @Autowired private PidDatabaseObjectDao db; - private static final String PREFIX = "sandboxed/"; - public LocalPidSystem() { LOG.warn("Using local identifier system to store PIDs. REGISTERED PIDs ARE NOT PERMANENTLY OR PUBLICLY STORED."); } /** - * For testing only. Allows to inject the database access object afterwards. - * - * @param db the new DAO. + * For testing purposes. */ - protected void setDatabase(PidDatabaseObjectDao db) { - this.db = db; + protected PidDatabaseObjectDao getDatabase() { + return this.db; } /** - * For testing purposes. + * For testing only. Allows to inject the database access object afterwards. + * + * @param db the new DAO. */ - protected PidDatabaseObjectDao getDatabase() { - return this.db; + protected void setDatabase(PidDatabaseObjectDao db) { + this.db = db; } @Override @@ -79,18 +95,27 @@ public Optional getPrefix() { } @Override - public boolean isPidRegistered(String pid) throws ExternalServiceException { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "local_pidsystem_is_pid_registered", description = "Time taken to check if PID is registered in local system") + @Counted(value = "local_pidsystem_is_pid_registered_total", description = "Total number of PID registration checks") + public boolean isPidRegistered(@SpanAttribute String pid) throws ExternalServiceException { return this.db.existsById(pid); } @Override - public PIDRecord queryPid(String pid) throws PidNotFoundException, ExternalServiceException { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "local_pidsystem_query_pid", description = "Time taken to query PID from local system") + @Counted(value = "local_pidsystem_query_pid_total", description = "Total number of PID queries") + public PIDRecord queryPid(@SpanAttribute String pid) throws PidNotFoundException, ExternalServiceException { Optional dbo = this.db.findByPid(pid); return new PIDRecord(dbo.orElseThrow(() -> new PidNotFoundException(pid))); } - + @Override - public String registerPidUnchecked(final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "local_pidsystem_register_pid", description = "Time taken to register PID in local system") + @Counted(value = "local_pidsystem_register_pid_total", description = "Total number of PID registrations") + public String registerPidUnchecked(@SpanAttribute final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { if (this.db.existsById(pidRecord.getPid())) { throw new PidAlreadyExistsException(pidRecord.getPid()); } @@ -100,7 +125,10 @@ public String registerPidUnchecked(final PIDRecord pidRecord) throws PidAlreadyE } @Override - public boolean updatePid(PIDRecord rec) throws PidNotFoundException, ExternalServiceException, RecordValidationException { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "local_pidsystem_update_pid", description = "Time taken to update PID in local system") + @Counted(value = "local_pidsystem_update_pid_total", description = "Total number of PID updates") + public boolean updatePid(@SpanAttribute PIDRecord rec) throws PidNotFoundException, ExternalServiceException, RecordValidationException { if (this.db.existsById(rec.getPid())) { this.db.save(new PidDatabaseObject(rec)); return true; @@ -109,11 +137,17 @@ public boolean updatePid(PIDRecord rec) throws PidNotFoundException, ExternalSer } @Override - public boolean deletePid(String pid) { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "local_pidsystem_delete_pid", description = "Time taken to delete PID from local system") + @Counted(value = "local_pidsystem_delete_pid_total", description = "Total number of PID deletion attempts") + public boolean deletePid(@SpanAttribute String pid) { throw new UnsupportedOperationException("Deleting PIDs is against the P in PID."); } @Override + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "local_pidsystem_resolve_all_pids", description = "Time taken to resolve all PIDs from local system") + @Counted(value = "local_pidsystem_resolve_all_pids_total", description = "Total number of resolve all PIDs requests") public Collection resolveAllPidsOfPrefix() throws ExternalServiceException, InvalidConfigException { return this.db.findAll().parallelStream() .map(dbo -> dbo.getPid()) diff --git a/src/main/java/edu/kit/datamanager/pit/pitservice/impl/EmbeddedStrictValidatorStrategy.java b/src/main/java/edu/kit/datamanager/pit/pitservice/impl/EmbeddedStrictValidatorStrategy.java index 9d08d768..fee21cbd 100644 --- a/src/main/java/edu/kit/datamanager/pit/pitservice/impl/EmbeddedStrictValidatorStrategy.java +++ b/src/main/java/edu/kit/datamanager/pit/pitservice/impl/EmbeddedStrictValidatorStrategy.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.pitservice.impl; import edu.kit.datamanager.pit.common.ExternalServiceException; @@ -8,14 +24,21 @@ import edu.kit.datamanager.pit.pitservice.IValidationStrategy; import edu.kit.datamanager.pit.typeregistry.AttributeInfo; import edu.kit.datamanager.pit.typeregistry.ITypeRegistry; +import io.micrometer.core.annotation.Counted; +import io.micrometer.core.annotation.Timed; +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.SpanAttribute; +import io.opentelemetry.instrumentation.annotations.WithSpan; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.List; import java.util.Set; -import java.util.concurrent.*; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; /** * Validates a PID record using embedded profile(s). @@ -24,6 +47,7 @@ * - validates all available attributes * - fails if an attribute is not defined within the profile */ +@Observed public class EmbeddedStrictValidatorStrategy implements IValidationStrategy { private static final Logger LOG = LoggerFactory.getLogger(EmbeddedStrictValidatorStrategy.class); @@ -41,10 +65,36 @@ public EmbeddedStrictValidatorStrategy( this.alwaysAcceptAdditionalAttributes = config.isValidationAlwaysAllowAdditionalAttributes(); } + /** + * Checks Exceptions' causes for a RecordValidationExceptions, and throws them, if present. + *

+ * Usually used to avoid exposing exceptions related to futures. + * + * @param e the exception to unwrap. + */ + @WithSpan(kind = SpanKind.INTERNAL) + private static void unpackAsyncExceptions(@SpanAttribute PIDRecord pidRecord, Throwable e) { + final int MAX_LEVEL = 10; + Throwable cause = e; + + for (int level = 0; level <= MAX_LEVEL && cause != null; level++) { + cause = cause.getCause(); + if (cause instanceof RecordValidationException rve) { + throw rve; + } else if (cause instanceof TypeNotFoundException tnf) { + throw new RecordValidationException( + pidRecord, + "Type not found: %s".formatted(tnf.getMessage())); + } + } + } + @Override - public void validate(PIDRecord pidRecord) - throws RecordValidationException, ExternalServiceException - { + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "validation_embedded_strict", description = "Time taken for embedded strict validation") + @Counted(value = "validation_embedded_strict_total", description = "Total number of embedded strict validations") + public void validate(@SpanAttribute PIDRecord pidRecord) + throws RecordValidationException, ExternalServiceException { if (pidRecord.getPropertyIdentifiers().isEmpty()) { throw new RecordValidationException(pidRecord, "Record is empty!"); } @@ -98,26 +148,4 @@ public void validate(PIDRecord pidRecord) String.format("Validation task was cancelled for %s. Please report.", pidRecord.getPid())); } } - - /** - * Checks Exceptions' causes for a RecordValidationExceptions, and throws them, if present. - *

- * Usually used to avoid exposing exceptions related to futures. - * @param e the exception to unwrap. - */ - private static void unpackAsyncExceptions(PIDRecord pidRecord, Throwable e) { - final int MAX_LEVEL = 10; - Throwable cause = e; - - for (int level = 0; level <= MAX_LEVEL && cause != null; level++) { - cause = cause.getCause(); - if (cause instanceof RecordValidationException rve) { - throw rve; - } else if (cause instanceof TypeNotFoundException tnf) { - throw new RecordValidationException( - pidRecord, - "Type not found: %s".formatted(tnf.getMessage())); - } - } - } } diff --git a/src/main/java/edu/kit/datamanager/pit/pitservice/impl/TypingService.java b/src/main/java/edu/kit/datamanager/pit/pitservice/impl/TypingService.java index d9a2fc99..22fcc845 100644 --- a/src/main/java/edu/kit/datamanager/pit/pitservice/impl/TypingService.java +++ b/src/main/java/edu/kit/datamanager/pit/pitservice/impl/TypingService.java @@ -1,141 +1,187 @@ -package edu.kit.datamanager.pit.pitservice.impl; - -import edu.kit.datamanager.pit.common.InvalidConfigException; -import edu.kit.datamanager.pit.common.PidAlreadyExistsException; -import edu.kit.datamanager.pit.common.PidNotFoundException; -import edu.kit.datamanager.pit.common.RecordValidationException; - -import java.util.Collection; -import java.util.Optional; - -import edu.kit.datamanager.pit.pidsystem.IIdentifierSystem; -import edu.kit.datamanager.pit.typeregistry.AttributeInfo; -import edu.kit.datamanager.pit.typeregistry.ITypeRegistry; -import edu.kit.datamanager.pit.pitservice.ITypingService; -import edu.kit.datamanager.pit.pitservice.IValidationStrategy; -import edu.kit.datamanager.pit.common.ExternalServiceException; -import edu.kit.datamanager.pit.domain.Operations; -import edu.kit.datamanager.pit.domain.PIDRecord; - -import java.util.concurrent.CancellationException; -import java.util.concurrent.CompletionException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * Core implementation class that offers the combined higher-level services - * through a type registry and an identifier system. - * - */ -public class TypingService implements ITypingService { - - private static final Logger LOG = LoggerFactory.getLogger(TypingService.class); - private static final String LOG_MSG_TYPING_SERVICE_MISCONFIGURED = "Typing service misconfigured."; - private static final String LOG_MSG_QUERY_TYPE = "Querying for type with identifier {}."; - - protected final IIdentifierSystem identifierSystem; - protected final ITypeRegistry typeRegistry; - - /** - * A validation strategy. Will never be null. - * - * ApplicationProperties::defaultValidationStrategy there is always either a - * default strategy or a noop strategy assigned. Therefore, autowiring will - * always work. Assigning null is done to avoid warnings on constructor. - */ - @Autowired - protected IValidationStrategy defaultStrategy = null; - - public TypingService(IIdentifierSystem identifierSystem, ITypeRegistry typeRegistry) { - super(); - this.identifierSystem = identifierSystem; - this.typeRegistry = typeRegistry; - } - - @Override - public Optional getPrefix() { - return this.identifierSystem.getPrefix(); - } - - @Override - public void setValidationStrategy(IValidationStrategy strategy) { - this.defaultStrategy = strategy; - } - - @Override - public void validate(PIDRecord pidRecord) - throws RecordValidationException, ExternalServiceException { - this.defaultStrategy.validate(pidRecord); - } - - @Override - public boolean isPidRegistered(String pid) throws ExternalServiceException { - LOG.trace("Performing isIdentifierRegistered({}).", pid); - return identifierSystem.isPidRegistered(pid); - } - - @Override - public String registerPidUnchecked(final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { - LOG.trace("Performing registerPID({}).", pidRecord); - return identifierSystem.registerPidUnchecked(pidRecord); - } - - @Override - public boolean deletePid(String pid) throws ExternalServiceException { - LOG.trace("Performing deletePID({}).", pid); - return identifierSystem.deletePid(pid); - } - - @Override - public PIDRecord queryPid(String pid) throws PidNotFoundException, ExternalServiceException { - return queryPid(pid, false); - } - - public PIDRecord queryPid(String pid, boolean includePropertyNames) - throws PidNotFoundException, ExternalServiceException { - LOG.trace("Performing queryAllProperties({}, {}).", pid, includePropertyNames); - PIDRecord pidInfo = identifierSystem.queryPid(pid); - - if (includePropertyNames) { - enrichPIDInformationRecord(pidInfo); - } - return pidInfo; - } - - private void enrichPIDInformationRecord(PIDRecord pidInfo) { - // enrich record by querying type registry for all property definitions - // to get the property names - for (String typeIdentifier : pidInfo.getPropertyIdentifiers()) { - AttributeInfo attributeInfo; - try { - attributeInfo = this.typeRegistry.queryAttributeInfo(typeIdentifier).join(); - } catch (CompletionException | CancellationException ex) { - // TODO convert exceptions like in validation service. - throw new InvalidConfigException(LOG_MSG_TYPING_SERVICE_MISCONFIGURED); - } - - if (attributeInfo != null) { - pidInfo.setPropertyName(typeIdentifier, attributeInfo.name()); - } else { - pidInfo.setPropertyName(typeIdentifier, typeIdentifier); - } - } - } - - @Override - public boolean updatePid(PIDRecord pidRecord) throws PidNotFoundException, ExternalServiceException, RecordValidationException { - return this.identifierSystem.updatePid(pidRecord); - } - - @Override - public Collection resolveAllPidsOfPrefix() throws ExternalServiceException, InvalidConfigException { - return this.identifierSystem.resolveAllPidsOfPrefix(); - } - - public Operations getOperations() { - return new Operations(this.typeRegistry, this.identifierSystem); - } - -} +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package edu.kit.datamanager.pit.pitservice.impl; + +import edu.kit.datamanager.pit.common.*; +import edu.kit.datamanager.pit.domain.Operations; +import edu.kit.datamanager.pit.domain.PIDRecord; +import edu.kit.datamanager.pit.pidsystem.IIdentifierSystem; +import edu.kit.datamanager.pit.pitservice.ITypingService; +import edu.kit.datamanager.pit.pitservice.IValidationStrategy; +import edu.kit.datamanager.pit.typeregistry.AttributeInfo; +import edu.kit.datamanager.pit.typeregistry.ITypeRegistry; +import io.micrometer.core.annotation.Counted; +import io.micrometer.core.annotation.Timed; +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.SpanAttribute; +import io.opentelemetry.instrumentation.annotations.WithSpan; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Collection; +import java.util.Optional; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletionException; + +/** + * Core implementation class that offers the combined higher-level services + * through a type registry and an identifier system. + * + */ + + +@Observed +public class TypingService implements ITypingService { + + private static final Logger LOG = LoggerFactory.getLogger(TypingService.class); + private static final String LOG_MSG_TYPING_SERVICE_MISCONFIGURED = "Typing service misconfigured."; + private static final String LOG_MSG_QUERY_TYPE = "Querying for type with identifier {}."; + + protected final IIdentifierSystem identifierSystem; + protected final ITypeRegistry typeRegistry; + + /** + * A validation strategy. Will never be null. + * + * ApplicationProperties::defaultValidationStrategy there is always either a + * default strategy or a noop strategy assigned. Therefore, autowiring will + * always work. Assigning null is done to avoid warnings on constructor. + */ + @Autowired + protected IValidationStrategy defaultStrategy = null; + + public TypingService(IIdentifierSystem identifierSystem, ITypeRegistry typeRegistry) { + super(); + this.identifierSystem = identifierSystem; + this.typeRegistry = typeRegistry; + } + + @Override + public Optional getPrefix() { + return this.identifierSystem.getPrefix(); + } + + @Override + public void setValidationStrategy(IValidationStrategy strategy) { + this.defaultStrategy = strategy; + } + + @Override + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typing_service_validate", description = "Time taken to validate PID record") + @Counted(value = "typing_service_validate_total", description = "Total number of validations") + public void validate(@SpanAttribute PIDRecord pidRecord) + throws RecordValidationException, ExternalServiceException { + this.defaultStrategy.validate(pidRecord); + } + + @Override + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typing_service_is_pid_registered", description = "Time taken to check PID registration") + @Counted(value = "typing_service_is_pid_registered_total", description = "Total number of PID registration checks") + public boolean isPidRegistered(@SpanAttribute String pid) throws ExternalServiceException { + LOG.trace("Performing isIdentifierRegistered({}).", pid); + return identifierSystem.isPidRegistered(pid); + } + + @Override + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typing_service_register_pid", description = "Time taken to register PID") + @Counted(value = "typing_service_register_pid_total", description = "Total number of PID registrations") + public String registerPidUnchecked(@SpanAttribute final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { + LOG.trace("Performing registerPID({}).", pidRecord); + return identifierSystem.registerPidUnchecked(pidRecord); + } + + @Override + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typing_service_delete_pid", description = "Time taken to delete PID") + @Counted(value = "typing_service_delete_pid_total", description = "Total number of PID deletions") + public boolean deletePid(@SpanAttribute String pid) throws ExternalServiceException { + LOG.trace("Performing deletePID({}).", pid); + return identifierSystem.deletePid(pid); + } + + @Override + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typing_service_query_pid", description = "Time taken to query PID") + @Counted(value = "typing_service_query_pid_total", description = "Total number of PID queries") + public PIDRecord queryPid(@SpanAttribute String pid) throws PidNotFoundException, ExternalServiceException { + return queryPid(pid, false); + } + + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typing_service_query_pid_with_names", description = "Time taken to query PID with property names") + @Counted(value = "typing_service_query_pid_with_names_total", description = "Total number of PID queries with names") + public PIDRecord queryPid(@SpanAttribute String pid, @SpanAttribute boolean includePropertyNames) + throws PidNotFoundException, ExternalServiceException { + LOG.trace("Performing queryAllProperties({}, {}).", pid, includePropertyNames); + PIDRecord pidInfo = identifierSystem.queryPid(pid); + + if (includePropertyNames) { + enrichPIDInformationRecord(pidInfo); + } + return pidInfo; + } + + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typing_service_enrich_record", description = "Time taken to enrich PID record with property names") + private void enrichPIDInformationRecord(@SpanAttribute PIDRecord pidInfo) { + // enrich record by querying type registry for all property definitions + // to get the property names + for (String typeIdentifier : pidInfo.getPropertyIdentifiers()) { + AttributeInfo attributeInfo; + try { + attributeInfo = this.typeRegistry.queryAttributeInfo(typeIdentifier).join(); + } catch (CompletionException | CancellationException ex) { + // TODO convert exceptions like in validation service. + throw new InvalidConfigException(LOG_MSG_TYPING_SERVICE_MISCONFIGURED); + } + + if (attributeInfo != null) { + pidInfo.setPropertyName(typeIdentifier, attributeInfo.name()); + } else { + pidInfo.setPropertyName(typeIdentifier, typeIdentifier); + } + } + } + + @Override + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typing_service_update_pid", description = "Time taken to update PID record") + @Counted(value = "typing_service_update_pid_total", description = "Total number of PID updates") + public boolean updatePid(@SpanAttribute PIDRecord pidRecord) throws PidNotFoundException, ExternalServiceException, RecordValidationException { + return this.identifierSystem.updatePid(pidRecord); + } + + @Override + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typing_service_resolve_all_pids", description = "Time taken to resolve all PIDs of prefix") + @Counted(value = "typing_service_resolve_all_pids_total", description = "Total number of resolve all PIDs requests") + public Collection resolveAllPidsOfPrefix() throws ExternalServiceException, InvalidConfigException { + return this.identifierSystem.resolveAllPidsOfPrefix(); + } + + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typing_service_get_operations", description = "Time taken to get operations") + public Operations getOperations() { + return new Operations(this.typeRegistry, this.identifierSystem); + } + +} diff --git a/src/main/java/edu/kit/datamanager/pit/resolver/Resolver.java b/src/main/java/edu/kit/datamanager/pit/resolver/Resolver.java index 66ac9044..9bf86264 100644 --- a/src/main/java/edu/kit/datamanager/pit/resolver/Resolver.java +++ b/src/main/java/edu/kit/datamanager/pit/resolver/Resolver.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.resolver; import edu.kit.datamanager.pit.common.ExternalServiceException; @@ -5,6 +21,12 @@ import edu.kit.datamanager.pit.domain.PIDRecord; import edu.kit.datamanager.pit.pidsystem.impl.handle.HandleBehavior; import edu.kit.datamanager.pit.pitservice.ITypingService; +import io.micrometer.core.annotation.Counted; +import io.micrometer.core.annotation.Timed; +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.SpanAttribute; +import io.opentelemetry.instrumentation.annotations.WithSpan; import net.handle.api.HSAdapter; import net.handle.api.HSAdapterFactory; import net.handle.hdllib.HandleException; @@ -22,19 +44,20 @@ *

* - Handle System */ + +@Observed public class Resolver { + private static final String SERVICE_NAME_HANDLE = "Handle System (read-only access)"; /** * The configured system to which we usually have write access. * Only used if the prefix of the PID matches the prefix of this system. */ private final ITypingService identifierSystem; - /** * The client to the Handle System, used in read-only mode. * Used as a fallback, if the prefix is not the one of the configured system. */ private final HSAdapter client = HSAdapterFactory.newInstance(); - private static final String SERVICE_NAME_HANDLE = "Handle System (read-only access)"; public Resolver(ITypingService identifierSystem) { this.identifierSystem = identifierSystem; @@ -48,13 +71,16 @@ public Resolver(ITypingService identifierSystem) { * * @param pid the PID to resolve. * @return the PIDRecord associated with the PID. - * @throws PidNotFoundException if the PID could not be found in any system. + * @throws PidNotFoundException if the PID could not be found in any system. * @throws ExternalServiceException if there was an error with the communication to an external system. */ - public PIDRecord resolve(String pid) throws PidNotFoundException, ExternalServiceException { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "resolver_resolve_pid", description = "Time taken to resolve PID from any system") + @Counted(value = "resolver_resolve_pid_total", description = "Total number of PID resolutions") + public PIDRecord resolve(@SpanAttribute String pid) throws PidNotFoundException, ExternalServiceException { String prefix = Arrays.stream( - pid.split("/", 2) - ) + pid.split("/", 2) + ) .findFirst() .orElseThrow(() -> new PidNotFoundException(pid, "Could not find prefix in PID.")) + "/"; // needed because the prefix is always followed by a slash diff --git a/src/main/java/edu/kit/datamanager/pit/typeregistry/impl/TypeApi.java b/src/main/java/edu/kit/datamanager/pit/typeregistry/impl/TypeApi.java index 26a6c2b9..a813bd93 100644 --- a/src/main/java/edu/kit/datamanager/pit/typeregistry/impl/TypeApi.java +++ b/src/main/java/edu/kit/datamanager/pit/typeregistry/impl/TypeApi.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.typeregistry.impl; import com.fasterxml.jackson.databind.JsonNode; @@ -15,6 +31,12 @@ import edu.kit.datamanager.pit.typeregistry.RegisteredProfileAttribute; import edu.kit.datamanager.pit.typeregistry.schema.SchemaInfo; import edu.kit.datamanager.pit.typeregistry.schema.SchemaSetGenerator; +import io.micrometer.core.annotation.Counted; +import io.micrometer.core.annotation.Timed; +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.SpanAttribute; +import io.opentelemetry.instrumentation.annotations.WithSpan; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.client.ClientHttpResponse; @@ -33,6 +55,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.StreamSupport; +@Observed public class TypeApi implements ITypeRegistry { private static final Logger LOG = LoggerFactory.getLogger(TypeApi.class); @@ -57,7 +80,7 @@ public TypeApi(ApplicationProperties properties, SchemaSetGenerator schemaSetGen .baseUrl(baseUri) .requestInterceptor((request, body, execution) -> { long start = System.currentTimeMillis(); - ClientHttpResponse response = execution.execute(request, body); + ClientHttpResponse response = execution.execute(request, body); long timeSpan = System.currentTimeMillis() - start; boolean isLongRequest = timeSpan > Application.LONG_HTTP_REQUEST_THRESHOLD; if (isLongRequest) { @@ -89,7 +112,7 @@ public TypeApi(ApplicationProperties properties, SchemaSetGenerator schemaSetGen .refreshAfterWrite(Duration.ofMinutes(expireAfterWrite / 2)) .expireAfterWrite(expireAfterWrite, TimeUnit.MINUTES) .removalListener((key, value, cause) -> - LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause) + LOG.trace("Removing profile {} from profile cache. Cause: {}", key, cause) ) .buildAsync(attributePid -> { LOG.trace("Loading attribute {} to cache.", attributePid); @@ -97,7 +120,10 @@ public TypeApi(ApplicationProperties properties, SchemaSetGenerator schemaSetGen }); } - protected AttributeInfo queryAttribute(String attributePid) { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "typeregistry_query_attribute", description = "Time taken to query attribute from type registry") + @Counted(value = "typeregistry_query_attribute_total", description = "Total number of attribute queries") + protected AttributeInfo queryAttribute(@SpanAttribute String attributePid) { return http.get() .uri(uriBuilder -> uriBuilder .path(attributePid) @@ -116,18 +142,25 @@ protected AttributeInfo queryAttribute(String attributePid) { }); } - protected AttributeInfo extractAttributeInformation(String attributePid, JsonNode jsonNode) { + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typeregistry_extract_attribute", description = "Time taken to extract attribute information from JSON") + protected AttributeInfo extractAttributeInformation(@SpanAttribute String attributePid, JsonNode jsonNode) { String typeName = jsonNode.path("type").asText(); String name = jsonNode.path("name").asText(); Set schemas = this.querySchemas(attributePid); return new AttributeInfo(attributePid, name, typeName, schemas); } - protected Set querySchemas(String maybeSchemaPid) throws TypeNotFoundException, ExternalServiceException { + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typeregistry_query_schemas", description = "Time taken to query schemas") + protected Set querySchemas(@SpanAttribute String maybeSchemaPid) throws TypeNotFoundException, ExternalServiceException { return schemaSetGenerator.generateFor(maybeSchemaPid).join(); } - protected RegisteredProfile queryProfile(String maybeProfilePid) throws TypeNotFoundException, ExternalServiceException { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "typeregistry_query_profile", description = "Time taken to query profile from type registry") + @Counted(value = "typeregistry_query_profile_total", description = "Total number of profile queries") + protected RegisteredProfile queryProfile(@SpanAttribute String maybeProfilePid) throws TypeNotFoundException, ExternalServiceException { return http.get() .uri(uriBuilder -> uriBuilder .path(maybeProfilePid) @@ -144,7 +177,9 @@ protected RegisteredProfile queryProfile(String maybeProfilePid) throws TypeNotF }); } - protected RegisteredProfile extractProfileInformation(String profilePid, JsonNode typeApiResponse) + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typeregistry_extract_profile", description = "Time taken to extract profile information from JSON") + protected RegisteredProfile extractProfileInformation(@SpanAttribute String profilePid, JsonNode typeApiResponse) throws TypeNotFoundException, ExternalServiceException { List attributes = new ArrayList<>(); @@ -178,10 +213,10 @@ protected RegisteredProfile extractProfileInformation(String profilePid, JsonNod }); boolean additionalAttributesDtrTestStyle = StreamSupport.stream(typeApiResponse - .path("content") - .path("representationsAndSemantics") - .spliterator(), - true) + .path("content") + .path("representationsAndSemantics") + .spliterator(), + true) .filter(JsonNode::isObject) .filter(node -> node.path("expression").asText("").equals("Format")) .map(node -> node.path("subSchemaRelation").asText("").equals("allowAdditionalProperties")) @@ -198,12 +233,18 @@ protected RegisteredProfile extractProfileInformation(String profilePid, JsonNod } @Override - public CompletableFuture queryAttributeInfo(String attributePid) { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "typeregistry_query_attribute_info", description = "Time taken to get attribute info (with cache)") + @Counted(value = "typeregistry_query_attribute_info_total", description = "Total number of attribute info requests") + public CompletableFuture queryAttributeInfo(@SpanAttribute String attributePid) { return this.attributeCache.get(attributePid); } @Override - public CompletableFuture queryAsProfile(String profilePid) { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "typeregistry_query_as_profile", description = "Time taken to get profile (with cache)") + @Counted(value = "typeregistry_query_as_profile_total", description = "Total number of profile requests") + public CompletableFuture queryAsProfile(@SpanAttribute String profilePid) { return this.profileCache.get(profilePid); } diff --git a/src/main/java/edu/kit/datamanager/pit/typeregistry/schema/DtrTestSchemaGenerator.java b/src/main/java/edu/kit/datamanager/pit/typeregistry/schema/DtrTestSchemaGenerator.java index c01727ec..ab174d53 100644 --- a/src/main/java/edu/kit/datamanager/pit/typeregistry/schema/DtrTestSchemaGenerator.java +++ b/src/main/java/edu/kit/datamanager/pit/typeregistry/schema/DtrTestSchemaGenerator.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.typeregistry.schema; import edu.kit.datamanager.pit.Application; @@ -5,6 +21,12 @@ import edu.kit.datamanager.pit.common.InvalidConfigException; import edu.kit.datamanager.pit.common.TypeNotFoundException; import edu.kit.datamanager.pit.configuration.ApplicationProperties; +import io.micrometer.core.annotation.Counted; +import io.micrometer.core.annotation.Timed; +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.SpanAttribute; +import io.opentelemetry.instrumentation.annotations.WithSpan; import jakarta.validation.constraints.NotNull; import org.everit.json.schema.Schema; import org.everit.json.schema.loader.SchemaLoader; @@ -23,10 +45,11 @@ import java.net.URISyntaxException; import java.net.http.HttpClient; -public class DtrTestSchemaGenerator implements SchemaGenerator { - private static final Logger LOG = LoggerFactory.getLogger(DtrTestSchemaGenerator.class); +@Observed +public class DtrTestSchemaGenerator implements SchemaGenerator { protected static final String ORIGIN = "dtr-test"; + private static final Logger LOG = LoggerFactory.getLogger(DtrTestSchemaGenerator.class); protected final URI baseUrl; protected final RestClient http; @@ -44,7 +67,7 @@ public DtrTestSchemaGenerator(@NotNull ApplicationProperties props) { .requestFactory(new JdkClientHttpRequestFactory(httpClient)) .requestInterceptor((request, body, execution) -> { long start = System.currentTimeMillis(); - ClientHttpResponse response = execution.execute(request, body); + ClientHttpResponse response = execution.execute(request, body); long timeSpan = System.currentTimeMillis() - start; boolean isLongRequest = timeSpan > Application.LONG_HTTP_REQUEST_THRESHOLD; if (isLongRequest) { @@ -56,7 +79,10 @@ public DtrTestSchemaGenerator(@NotNull ApplicationProperties props) { } @Override - public SchemaInfo generateSchema(@NotNull String maybeTypePid) { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "dtr_test_schema_generate", description = "Time taken to generate schema from DTR Test") + @Counted(value = "dtr_test_schema_generate_total", description = "Total number of DTR Test schema generations") + public SchemaInfo generateSchema(@SpanAttribute @NotNull String maybeTypePid) { return this.http.get().uri(uriBuilder -> uriBuilder.pathSegment(maybeTypePid).build()) .exchange((request, response) -> { HttpStatusCode status = response.getStatusCode(); diff --git a/src/main/java/edu/kit/datamanager/pit/typeregistry/schema/SchemaSetGenerator.java b/src/main/java/edu/kit/datamanager/pit/typeregistry/schema/SchemaSetGenerator.java index 166fae87..b98e66d3 100644 --- a/src/main/java/edu/kit/datamanager/pit/typeregistry/schema/SchemaSetGenerator.java +++ b/src/main/java/edu/kit/datamanager/pit/typeregistry/schema/SchemaSetGenerator.java @@ -1,9 +1,31 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.typeregistry.schema; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; import edu.kit.datamanager.pit.Application; import edu.kit.datamanager.pit.configuration.ApplicationProperties; +import io.micrometer.core.annotation.Counted; +import io.micrometer.core.annotation.Timed; +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.SpanAttribute; +import io.opentelemetry.instrumentation.annotations.WithSpan; import java.time.Duration; import java.util.Set; @@ -11,6 +33,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; + +@Observed public class SchemaSetGenerator { protected final Set GENERATORS; protected final AsyncLoadingCache> CACHE; @@ -41,7 +65,10 @@ public SchemaSetGenerator(ApplicationProperties props) { * @param attributePid the PID of the attribute to generate schemas for. * @return a set of information about the generated schemas, including the schemas themselves, if generation succeeded. */ - public CompletableFuture> generateFor(final String attributePid) { + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "schema_generator_generate_for", description = "Time taken to generate schemas for attribute") + @Counted(value = "schema_generator_generate_for_total", description = "Total number of schema generation requests") + public CompletableFuture> generateFor(@SpanAttribute final String attributePid) { return this.CACHE.get(attributePid); } } diff --git a/src/main/java/edu/kit/datamanager/pit/typeregistry/schema/TypeApiSchemaGenerator.java b/src/main/java/edu/kit/datamanager/pit/typeregistry/schema/TypeApiSchemaGenerator.java index 7555a7a4..ea87a2e6 100644 --- a/src/main/java/edu/kit/datamanager/pit/typeregistry/schema/TypeApiSchemaGenerator.java +++ b/src/main/java/edu/kit/datamanager/pit/typeregistry/schema/TypeApiSchemaGenerator.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.typeregistry.schema; import edu.kit.datamanager.pit.Application; @@ -5,6 +21,12 @@ import edu.kit.datamanager.pit.common.InvalidConfigException; import edu.kit.datamanager.pit.common.TypeNotFoundException; import edu.kit.datamanager.pit.configuration.ApplicationProperties; +import io.micrometer.core.annotation.Counted; +import io.micrometer.core.annotation.Timed; +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.SpanAttribute; +import io.opentelemetry.instrumentation.annotations.WithSpan; import jakarta.validation.constraints.NotNull; import org.everit.json.schema.Schema; import org.everit.json.schema.loader.SchemaLoader; @@ -21,6 +43,8 @@ import java.net.URISyntaxException; import java.net.URL; + +@Observed public class TypeApiSchemaGenerator implements SchemaGenerator { private static final Logger LOG = LoggerFactory.getLogger(TypeApiSchemaGenerator.class); @@ -39,7 +63,7 @@ public TypeApiSchemaGenerator(@NotNull ApplicationProperties props) { .baseUrl(baseUri) .requestInterceptor((request, body, execution) -> { long start = System.currentTimeMillis(); - ClientHttpResponse response = execution.execute(request, body); + ClientHttpResponse response = execution.execute(request, body); long timeSpan = System.currentTimeMillis() - start; boolean isLongRequest = timeSpan > Application.LONG_HTTP_REQUEST_THRESHOLD; if (isLongRequest) { @@ -51,7 +75,10 @@ public TypeApiSchemaGenerator(@NotNull ApplicationProperties props) { } @Override - public SchemaInfo generateSchema(@NotNull String maybeTypePid) { + @WithSpan(kind = SpanKind.CLIENT) + @Timed(value = "typeapi_schema_generator_generate", description = "Time taken to generate schema from Type API") + @Counted(value = "typeapi_schema_generator_generate_total", description = "Total number of Type API schema generations") + public SchemaInfo generateSchema(@SpanAttribute @NotNull String maybeTypePid) { return http.get() .uri(uriBuilder -> uriBuilder .pathSegment("schema") diff --git a/src/main/java/edu/kit/datamanager/pit/web/converter/SimplePidRecordConverter.java b/src/main/java/edu/kit/datamanager/pit/web/converter/SimplePidRecordConverter.java index 8132f361..45bfea7a 100644 --- a/src/main/java/edu/kit/datamanager/pit/web/converter/SimplePidRecordConverter.java +++ b/src/main/java/edu/kit/datamanager/pit/web/converter/SimplePidRecordConverter.java @@ -1,13 +1,26 @@ -package edu.kit.datamanager.pit.web.converter; +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; +package edu.kit.datamanager.pit.web.converter; +import edu.kit.datamanager.pit.Application; +import edu.kit.datamanager.pit.domain.PIDRecord; +import edu.kit.datamanager.pit.domain.SimplePidRecord; +import io.micrometer.core.annotation.Counted; +import io.micrometer.observation.annotation.Observed; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpInputMessage; @@ -17,34 +30,38 @@ import org.springframework.http.converter.HttpMessageNotReadableException; import org.springframework.http.converter.HttpMessageNotWritableException; -import edu.kit.datamanager.pit.Application; -import edu.kit.datamanager.pit.domain.PIDRecord; -import edu.kit.datamanager.pit.domain.SimplePidRecord; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.stream.Collectors; /** * Converts to-and-from PIDRecord format when SimplePidRecord format is actually * expected. - * + *

* Do not use explicitly. Spring will use it, as explained below. - * + *

* The idea is that all handlers in the REST API simply expect a serialized * (marshalled) version of a PID record. This is not always true, though. The * PIDRecord representation is pretty complex. To offer a simpler format, * `SimplePidRecord` was introduced. To avoid larger modifications within the * code, and to not break the API, the simple format comes into play only when * its content type is being used. - * + *

* If the client wants to send in the simple format, it needs to set the * content-type header accordingly. Spring will then use this converter to * convert it directly into a PIDRecord instance, as the handler expects. This * way only one handler must be used for multiple formats. - * + *

* For accepting formats, it is the same. With the accept header, a client may * control which format it would like to receive. If it prefers to receive the * simple format and sets the header accordingly, instead of directly * serializing the PIDRecord, this class will be used. It first converts the * record into the simple class representation before serializing into JSON. */ +@Observed public class SimplePidRecordConverter implements HttpMessageConverter { private static final Logger LOGGER = LoggerFactory.getLogger(SimplePidRecordConverter.class); @@ -70,12 +87,13 @@ public boolean canWrite(Class arg0, MediaType arg1) { @Override public List getSupportedMediaTypes() { - return Arrays.asList( + return List.of( MediaType.valueOf(SimplePidRecord.CONTENT_TYPE) ); } @Override + @Counted(value = "simplePidRecordConverter.read.count", description = "Number of reads of SimplePidRecord") public PIDRecord read(Class arg0, HttpInputMessage arg1) throws IOException, HttpMessageNotReadableException { LOGGER.trace("Read simple message from client and convert to PIDRecord."); @@ -86,6 +104,7 @@ public PIDRecord read(Class arg0, HttpInputMessage arg1) } @Override + @Counted(value = "simplePidRecordConverter.write.count", description = "Number of writes of SimplePidRecord") public void write(PIDRecord arg0, MediaType arg1, HttpOutputMessage arg2) throws IOException, HttpMessageNotWritableException { LOGGER.trace("Write PIDRecord to simple format for client."); diff --git a/src/main/java/edu/kit/datamanager/pit/web/impl/TypingRESTResourceImpl.java b/src/main/java/edu/kit/datamanager/pit/web/impl/TypingRESTResourceImpl.java index d67c2975..4de71255 100644 --- a/src/main/java/edu/kit/datamanager/pit/web/impl/TypingRESTResourceImpl.java +++ b/src/main/java/edu/kit/datamanager/pit/web/impl/TypingRESTResourceImpl.java @@ -1,540 +1,583 @@ -/* - * Copyright (c) 2025 Karlsruhe Institute of Technology. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package edu.kit.datamanager.pit.web.impl; - -import edu.kit.datamanager.entities.messaging.PidRecordMessage; -import edu.kit.datamanager.exceptions.CustomInternalServerError; -import edu.kit.datamanager.pit.common.*; -import edu.kit.datamanager.pit.configuration.ApplicationProperties; -import edu.kit.datamanager.pit.configuration.PidGenerationProperties; -import edu.kit.datamanager.pit.domain.PIDRecord; -import edu.kit.datamanager.pit.elasticsearch.PidRecordElasticRepository; -import edu.kit.datamanager.pit.elasticsearch.PidRecordElasticWrapper; -import edu.kit.datamanager.pit.pidgeneration.PidSuffix; -import edu.kit.datamanager.pit.pidgeneration.PidSuffixGenerator; -import edu.kit.datamanager.pit.pidlog.KnownPid; -import edu.kit.datamanager.pit.pidlog.KnownPidsDao; -import edu.kit.datamanager.pit.pitservice.ITypingService; -import edu.kit.datamanager.pit.resolver.Resolver; -import edu.kit.datamanager.pit.web.ITypingRestResource; -import edu.kit.datamanager.pit.web.TabulatorPaginationFormat; -import edu.kit.datamanager.service.IMessagingService; -import edu.kit.datamanager.util.AuthenticationHelper; -import edu.kit.datamanager.util.ControllerUtils; -import io.swagger.v3.oas.annotations.media.Schema; -import jakarta.servlet.http.HttpServletResponse; -import org.apache.commons.lang3.stream.Streams; -import org.apache.http.client.cache.HeaderConstants; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageImpl; -import org.springframework.data.domain.Pageable; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.context.request.RequestAttributes; -import org.springframework.web.context.request.WebRequest; -import org.springframework.web.servlet.HandlerMapping; -import org.springframework.web.util.UriComponentsBuilder; - -import java.io.IOException; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.*; -import java.util.stream.Stream; - -@RestController -@RequestMapping(value = "/api/v1/pit") -@Schema(description = "PID Information Types API") -public class TypingRESTResourceImpl implements ITypingRestResource { - - private static final Logger LOG = LoggerFactory.getLogger(TypingRESTResourceImpl.class); - @Autowired - protected ITypingService typingService; - @Autowired - protected Resolver resolver; - @Autowired - private ApplicationProperties applicationProps; - @Autowired - private IMessagingService messagingService; - - @Autowired - private KnownPidsDao localPidStorage; - - @Autowired - private Optional elastic; - - @Autowired - private PidSuffixGenerator suffixGenerator; - - @Autowired - private PidGenerationProperties pidGenerationProperties; - - public TypingRESTResourceImpl() { - super(); - } - - @Override - public ResponseEntity> createPIDs( - List rec, - boolean dryrun, - WebRequest request, - HttpServletResponse response, - UriComponentsBuilder uriBuilder - ) throws IOException, RecordValidationException, ExternalServiceException { - Instant startTime = Instant.now(); - LOG.info("Creating PIDs for {} records.", rec.size()); - String prefix = this.typingService.getPrefix().orElseThrow(() -> new IOException("No prefix configured.")); - - // Generate a map between temporary (user-defined) PIDs and final PIDs (generated) - Map pidMappings = generatePIDMapping(rec, dryrun); - Instant mappingTime = Instant.now(); - - // Apply the mappings to the records and validate them - List validatedRecords = applyMappingsToRecordsAndValidate(rec, pidMappings, prefix); - Instant validationTime = Instant.now(); - - if (dryrun) { - // dryrun only does validation. Stop now and return as we would later on. - LOG.info("Time taken for dryrun: {} ms", ChronoUnit.MILLIS.between(startTime, validationTime)); - LOG.info("-- Time taken for mapping: {} ms", ChronoUnit.MILLIS.between(startTime, mappingTime)); - LOG.info("-- Time taken for validation: {} ms", ChronoUnit.MILLIS.between(mappingTime, validationTime)); - LOG.info("Dryrun finished. Returning validated records for {} records.", validatedRecords.size()); - return ResponseEntity.status(HttpStatus.OK).body(validatedRecords); - } - - List failedRecords = new ArrayList<>(); - // register the records - validatedRecords.forEach(pidRecord -> { - try { - // register the PID - String pid = this.typingService.registerPid(pidRecord); - pidRecord.setPid(pid); - - // store pid locally in accordance with the storage strategy - if (applicationProps.getStorageStrategy().storesModified()) { - storeLocally(pid, true); - } - - // distribute pid creation event to other services - PidRecordMessage message = PidRecordMessage.creation( - pid, - "", // TODO parameter is deprecated and will be removed soon. - AuthenticationHelper.getPrincipal(), - ControllerUtils.getLocalHostname()); - try { - this.messagingService.send(message); - } catch (Exception e) { - LOG.error("Could not notify messaging service about the following message: {}", message); - } - - // save the record to elastic - this.saveToElastic(pidRecord); - } catch (Exception e) { - LOG.error("Could not register PID for record {}. Error: {}", pidRecord, e.getMessage()); - failedRecords.add(pidRecord); - validatedRecords.remove(pidRecord); - } - }); - - Instant endTime = Instant.now(); - - // return the created records - LOG.info("Total time taken: {} ms", ChronoUnit.MILLIS.between(startTime, endTime)); - LOG.info("-- Time taken for mapping: {} ms", ChronoUnit.MILLIS.between(startTime, mappingTime)); - LOG.info("-- Time taken for validation: {} ms", ChronoUnit.MILLIS.between(mappingTime, validationTime)); - LOG.info("-- Time taken for registration: {} ms", ChronoUnit.MILLIS.between(validationTime, endTime)); - - if (!failedRecords.isEmpty()) { - for (PIDRecord successfulRecord : validatedRecords) { // rollback the successful records - try { - LOG.debug("Rolling back PID creation for record with PID {}.", successfulRecord.getPid()); - this.typingService.deletePid(successfulRecord.getPid()); - } catch (Exception e) { - LOG.error("Could not rollback PID creation for record with PID {}. Error: {}", successfulRecord.getPid(), e.getMessage()); - } - } - - LOG.info("Creation finished. Returning validated records for {} records. {} records failed to be created.", validatedRecords.size(), failedRecords.size()); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(failedRecords); - } else { - LOG.info("Creation finished. Returning validated records for {} records.", validatedRecords.size()); - return ResponseEntity.status(HttpStatus.CREATED).body(validatedRecords); - } - } - - /** - * This method generates a mapping between user-provided "fantasy" PIDs and real PIDs. - * - * @param rec the list of records produced by the user - * @param dryrun whether the operation is a dryrun or not - * @return a map between the user-provided PIDs (key) and the real PIDs (values) - * @throws IOException if the prefix is not configured - * @throws RecordValidationException if the same internal PID is used for multiple records - * @throws ExternalServiceException if the PID generation fails - */ - private Map generatePIDMapping(List rec, boolean dryrun) throws IOException, RecordValidationException, ExternalServiceException { - Map pidMappings = new HashMap<>(); - for (PIDRecord pidRecord : rec) { - String internalPID = pidRecord.getPid(); // the internal PID is the one given by the user - if (!internalPID.isBlank() && pidMappings.containsKey(internalPID)) { // check if the internal PID was already used - // This internal PID was already used by some other record in the same request. - throw new RecordValidationException(pidRecord, "The PID " + internalPID + " was used for multiple records in the same request."); - } - - pidRecord.setPid(""); // clear the PID field in the record - if (dryrun) { // if it is a dryrun, we set the PID to a temporary value - pidRecord.setPid("dryrun_" + pidMappings.size()); - } else { - setPid(pidRecord); // otherwise, we generate a real PID - } - pidMappings.put(internalPID, pidRecord.getPid()); // store the mapping between the internal and real PID - } - return pidMappings; - } - - /** - * This method applies the mappings between temporary PIDs and real PIDs to the records and validates them. - * - * @param rec the list of records produced by the user - * @param pidMappings the map between the user-provided PIDs (key) and the real PIDs (values) - * @param prefix the prefix to be used for the real PIDs - * @return the list of validated records - * @throws RecordValidationException as a possible validation outcome - * @throws ExternalServiceException as a possible validation outcome - */ - private List applyMappingsToRecordsAndValidate(List rec, Map pidMappings, String prefix) throws RecordValidationException, ExternalServiceException { - List validatedRecords = new ArrayList<>(); - for (PIDRecord pidRecord : rec) { - - // use this map to replace all temporary PIDs in the record values with their corresponding real PIDs - pidRecord.getEntries().values().stream() // get all values of the record - .flatMap(List::stream) // flatten the list of values - .filter(entry -> entry.getValue() != null) // Filter out null values - .filter(entry -> pidMappings.containsKey(entry.getValue())) // replace only if the value (aka. "fantasy PID") is a key in the map - .peek(entry -> LOG.debug("Found reference. Replacing {} with {}.", entry.getValue(), prefix + pidMappings.get(entry.getValue()))) // log the replacement - .forEach(entry -> entry.setValue(prefix + pidMappings.get(entry.getValue()))); // replace the value with the real PID according to the map - - // validate the record - this.typingService.validate(pidRecord); - - // store the record - validatedRecords.add(pidRecord); - LOG.debug("Record {} is valid.", pidRecord); - } - return validatedRecords; - } - - @Override - public ResponseEntity createPID( - PIDRecord pidRecord, - boolean dryrun, - - final WebRequest request, - final HttpServletResponse response, - final UriComponentsBuilder uriBuilder - ) throws IOException { - LOG.info("Creating PID"); - - if (dryrun) { - pidRecord.setPid("dryrun"); - } else { - setPid(pidRecord); - } - - this.typingService.validate(pidRecord); - - if (dryrun) { - // dryrun only does validation. Stop now and return as we would later on. - return ResponseEntity.status(HttpStatus.OK).eTag(quotedEtag(pidRecord)).body(pidRecord); - } - - String pid = this.typingService.registerPid(pidRecord); - pidRecord.setPid(pid); - - if (applicationProps.getStorageStrategy().storesModified()) { - storeLocally(pid, true); - } - PidRecordMessage message = PidRecordMessage.creation( - pid, - "", // TODO parameter is deprecated and will be removed soon. - AuthenticationHelper.getPrincipal(), - ControllerUtils.getLocalHostname()); - try { - this.messagingService.send(message); - } catch (Exception e) { - LOG.error("Could not notify messaging service about the following message: {}", message); - } - this.saveToElastic(pidRecord); - return ResponseEntity.status(HttpStatus.CREATED).eTag(quotedEtag(pidRecord)).body(pidRecord); - } - - private boolean hasPid(PIDRecord pidRecord) { - return pidRecord.getPid() != null && !pidRecord.getPid().isBlank(); - } - - private void setPid(PIDRecord pidRecord) throws IOException { - boolean hasCustomPid = hasPid(pidRecord); - boolean allowsCustomPids = pidGenerationProperties.isCustomClientPidsEnabled(); - - if (allowsCustomPids && hasCustomPid) { - // in this only case, we do not have to generate a PID - // but we have to check if the PID is already registered and return an error if so - String prefix = this.typingService.getPrefix() - .orElseThrow(() -> new InvalidConfigException("No prefix configured.")); - String maybeSuffix = pidRecord.getPid(); - String pid = PidSuffix.asPrefixedChecked(maybeSuffix, prefix); - boolean isRegisteredPid = this.typingService.isPidRegistered(pid); - if (isRegisteredPid) { - throw new PidAlreadyExistsException(pidRecord.getPid()); - } - } else { - // In all other (usual) cases, we have to generate a PID. - // We store only the suffix in the pid field. - // The registration at the PID service will preprend the prefix. - - Stream suffixStream = suffixGenerator.infiniteStream(); - Optional maybeSuffix = Streams.failableStream(suffixStream) - // With failable streams, we can throw exceptions. - .filter(suffix -> !this.typingService.isPidRegistered(suffix)) - .stream() // back to normal java streams - .findFirst(); // as the stream is infinite, we should always find a prefix. - PidSuffix suffix = maybeSuffix - .orElseThrow(() -> new ExternalServiceException("Could not generate PID suffix which did not exist yet.")); - pidRecord.setPid(suffix.get()); - } - } - - @Override - public ResponseEntity updatePID( - PIDRecord pidRecord, - boolean dryrun, - - final WebRequest request, - final HttpServletResponse response, - final UriComponentsBuilder uriBuilder - ) throws IOException { - // PID validation - String pid = getContentPathFromRequest("pid", request); - String pidInternal = pidRecord.getPid(); - if (hasPid(pidRecord) && !pid.equals(pidInternal)) { - throw new RecordValidationException( - pidRecord, - "Optional PID in record is given (%s), but it was not the same as the PID in the URL (%s). Ignore request, assuming this was not intended.".formatted(pidInternal, pid)); - } - - PIDRecord existingRecord = this.resolver.resolve(pid); - if (existingRecord == null) { - throw new PidNotFoundException(pid); - } - - // record validation - pidRecord.setPid(pid); - this.typingService.validate(pidRecord); - - // throws exception (HTTP 412) if check fails. - ControllerUtils.checkEtag(request, existingRecord); - - if (dryrun) { - // dryrun only does validation. Stop now and return as we would later on. - return ResponseEntity.ok().eTag(quotedEtag(pidRecord)).body(pidRecord); - } - - // update and send message - if (this.typingService.updatePid(pidRecord)) { - // store pid locally - if (applicationProps.getStorageStrategy().storesModified()) { - storeLocally(pidRecord.getPid(), true); - } - // distribute pid to other services - PidRecordMessage message = PidRecordMessage.update( - pid, - "", // TODO parameter is depricated and will be removed soon. - AuthenticationHelper.getPrincipal(), - ControllerUtils.getLocalHostname()); - this.messagingService.send(message); - this.saveToElastic(pidRecord); - return ResponseEntity.ok().eTag(quotedEtag(pidRecord)).body(pidRecord); - } else { - throw new PidNotFoundException(pid); - } - } - - /** - * Stores the PID in a local database. - * - * @param pid the PID - * @param update if true, updates the modified timestamp if it already exists. - * If it does not exist, it will be created with both timestamps - * (created and modified) being the same. - */ - private void storeLocally(String pid, boolean update) { - Instant now = Instant.now(); - Optional oldPid = localPidStorage.findByPid(pid); - if (oldPid.isEmpty()) { - localPidStorage.saveAndFlush(new KnownPid(pid, now, now)); - } else if (update) { - KnownPid newPid = oldPid.get(); - newPid.setModified(now); - localPidStorage.saveAndFlush(newPid); - } - } - - private String getContentPathFromRequest(String lastPathElement, WebRequest request) { - String requestedUri = (String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE, - RequestAttributes.SCOPE_REQUEST); - if (requestedUri == null) { - throw new CustomInternalServerError("Unable to obtain request URI."); - } - return requestedUri.substring(requestedUri.indexOf(lastPathElement + "/") + (lastPathElement + "/").length()); - } - - @Override - public ResponseEntity getRecord( - boolean validation, - - final WebRequest request, - final HttpServletResponse response, - final UriComponentsBuilder uriBuilder - ) throws IOException { - String pid = getContentPathFromRequest("pid", request); - PIDRecord pidRecord = this.resolver.resolve(pid); - if (applicationProps.getStorageStrategy().storesResolved()) { - storeLocally(pid, false); - } - this.saveToElastic(pidRecord); - if (validation) { - typingService.validate(pidRecord); - } - return ResponseEntity.ok().eTag(quotedEtag(pidRecord)).body(pidRecord); - } - - private void saveToElastic(PIDRecord rec) { - this.elastic.ifPresent( - database -> database.save( - new PidRecordElasticWrapper(rec, typingService.getOperations()) - ) - ); - } - - @Override - public ResponseEntity findByPid( - WebRequest request, - HttpServletResponse response, - UriComponentsBuilder uriBuilder - ) throws IOException { - String pid = getContentPathFromRequest("known-pid", request); - Optional known = this.localPidStorage.findByPid(pid); - if (known.isPresent()) { - return ResponseEntity.ok().body(known.get()); - } - return ResponseEntity.notFound().build(); - } - - public Page findAllPage( - Instant createdAfter, - Instant createdBefore, - Instant modifiedAfter, - Instant modifiedBefore, - Pageable pageable - ) { - final boolean queriesCreated = createdAfter != null || createdBefore != null; - final boolean queriesModified = modifiedAfter != null || modifiedBefore != null; - if (queriesCreated && createdAfter == null) { - createdAfter = Instant.EPOCH; - } - if (queriesCreated && createdBefore == null) { - createdBefore = Instant.now().plus(1, ChronoUnit.DAYS); - } - if (queriesModified && modifiedAfter == null) { - modifiedAfter = Instant.EPOCH; - } - if (queriesModified && modifiedBefore == null) { - modifiedBefore = Instant.now().plus(1, ChronoUnit.DAYS); - } - - Page resultCreatedTimestamp = Page.empty(); - Page resultModifiedTimestamp = Page.empty(); - if (queriesCreated) { - resultCreatedTimestamp = this.localPidStorage - .findDistinctPidsByCreatedBetween(createdAfter, createdBefore, pageable); - } - if (queriesModified) { - resultModifiedTimestamp = this.localPidStorage - .findDistinctPidsByModifiedBetween(modifiedAfter, modifiedBefore, pageable); - } - if (queriesCreated && queriesModified) { - final Page tmp = resultModifiedTimestamp; - final List intersection = resultCreatedTimestamp.filter((x) -> tmp.getContent().contains(x)).toList(); - return new PageImpl<>(intersection); - } else if (queriesCreated) { - return resultCreatedTimestamp; - } else if (queriesModified) { - return resultModifiedTimestamp; - } - return new PageImpl<>(this.localPidStorage.findAll()); - } - - @Override - public ResponseEntity> findAll( - Instant createdAfter, - Instant createdBefore, - Instant modifiedAfter, - Instant modifiedBefore, - Pageable pageable, - WebRequest request, - HttpServletResponse response, - UriComponentsBuilder uriBuilder) throws IOException { - Page page = this.findAllPage(createdAfter, createdBefore, modifiedAfter, modifiedBefore, pageable); - response.addHeader( - HeaderConstants.CONTENT_RANGE, - ControllerUtils.getContentRangeHeader( - page.getNumber(), - page.getSize(), - page.getTotalElements())); - return ResponseEntity.ok().body(page.getContent()); - } - - @Override - public ResponseEntity> findAllForTabular( - Instant createdAfter, - Instant createdBefore, - Instant modifiedAfter, - Instant modifiedBefore, - Pageable pageable, - WebRequest request, - HttpServletResponse response, - UriComponentsBuilder uriBuilder) throws IOException { - Page page = this.findAllPage(createdAfter, createdBefore, modifiedAfter, modifiedBefore, pageable); - response.addHeader( - HeaderConstants.CONTENT_RANGE, - ControllerUtils.getContentRangeHeader( - page.getNumber(), - page.getSize(), - page.getTotalElements())); - TabulatorPaginationFormat tabPage = new TabulatorPaginationFormat<>(page); - return ResponseEntity.ok().body(tabPage); - } - - private String quotedEtag(PIDRecord pidRecord) { - return String.format("\"%s\"", pidRecord.getEtag()); - } - -} +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package edu.kit.datamanager.pit.web.impl; + +import edu.kit.datamanager.entities.messaging.PidRecordMessage; +import edu.kit.datamanager.exceptions.CustomInternalServerError; +import edu.kit.datamanager.pit.common.*; +import edu.kit.datamanager.pit.configuration.ApplicationProperties; +import edu.kit.datamanager.pit.configuration.PidGenerationProperties; +import edu.kit.datamanager.pit.domain.PIDRecord; +import edu.kit.datamanager.pit.elasticsearch.PidRecordElasticRepository; +import edu.kit.datamanager.pit.elasticsearch.PidRecordElasticWrapper; +import edu.kit.datamanager.pit.pidgeneration.PidSuffix; +import edu.kit.datamanager.pit.pidgeneration.PidSuffixGenerator; +import edu.kit.datamanager.pit.pidlog.KnownPid; +import edu.kit.datamanager.pit.pidlog.KnownPidsDao; +import edu.kit.datamanager.pit.pitservice.ITypingService; +import edu.kit.datamanager.pit.resolver.Resolver; +import edu.kit.datamanager.pit.web.ITypingRestResource; +import edu.kit.datamanager.pit.web.TabulatorPaginationFormat; +import edu.kit.datamanager.service.IMessagingService; +import edu.kit.datamanager.util.AuthenticationHelper; +import edu.kit.datamanager.util.ControllerUtils; +import io.micrometer.core.annotation.Counted; +import io.micrometer.core.annotation.Timed; +import io.micrometer.observation.annotation.Observed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.SpanAttribute; +import io.opentelemetry.instrumentation.annotations.WithSpan; +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.stream.Streams; +import org.apache.http.client.cache.HeaderConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.context.request.RequestAttributes; +import org.springframework.web.context.request.WebRequest; +import org.springframework.web.servlet.HandlerMapping; +import org.springframework.web.util.UriComponentsBuilder; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Stream; + +@RestController +@RequestMapping(value = "/api/v1/pit") +@Schema(description = "PID Information Types API") +@Observed +public class TypingRESTResourceImpl implements ITypingRestResource { + + private static final Logger LOG = LoggerFactory.getLogger(TypingRESTResourceImpl.class); + @Autowired + protected ITypingService typingService; + @Autowired + protected Resolver resolver; + @Autowired + private ApplicationProperties applicationProps; + @Autowired + private IMessagingService messagingService; + + @Autowired + private KnownPidsDao localPidStorage; + + @Autowired + private Optional elastic; + + @Autowired + private PidSuffixGenerator suffixGenerator; + + @Autowired + private PidGenerationProperties pidGenerationProperties; + + public TypingRESTResourceImpl() { + super(); + } + + @Override + @WithSpan(kind = SpanKind.SERVER) + @Timed(value = "pit_create_pids", description = "Time taken to create multiple PID records") + @Counted(value = "pit_create_pids_total", description = "Total number of create PIDs requests") + public ResponseEntity> createPIDs( + @SpanAttribute List rec, + @SpanAttribute boolean dryrun, + WebRequest request, + HttpServletResponse response, + UriComponentsBuilder uriBuilder + ) throws IOException, RecordValidationException, ExternalServiceException { + Instant startTime = Instant.now(); + LOG.info("Creating PIDs for {} records.", rec.size()); + String prefix = this.typingService.getPrefix().orElseThrow(() -> new IOException("No prefix configured.")); + + // Generate a map between temporary (user-defined) PIDs and final PIDs (generated) + Map pidMappings = generatePIDMapping(rec, dryrun); + Instant mappingTime = Instant.now(); + + // Apply the mappings to the records and validate them + List validatedRecords = applyMappingsToRecordsAndValidate(rec, pidMappings, prefix); + Instant validationTime = Instant.now(); + + if (dryrun) { + // dryrun only does validation. Stop now and return as we would later on. + LOG.info("Time taken for dryrun: {} ms", ChronoUnit.MILLIS.between(startTime, validationTime)); + LOG.info("-- Time taken for mapping: {} ms", ChronoUnit.MILLIS.between(startTime, mappingTime)); + LOG.info("-- Time taken for validation: {} ms", ChronoUnit.MILLIS.between(mappingTime, validationTime)); + LOG.info("Dryrun finished. Returning validated records for {} records.", validatedRecords.size()); + return ResponseEntity.status(HttpStatus.OK).body(validatedRecords); + } + + List failedRecords = new ArrayList<>(); + // register the records + validatedRecords.forEach(pidRecord -> { + try { + // register the PID + String pid = this.typingService.registerPid(pidRecord); + pidRecord.setPid(pid); + + // store pid locally in accordance with the storage strategy + if (applicationProps.getStorageStrategy().storesModified()) { + storeLocally(pid, true); + } + + // distribute pid creation event to other services + PidRecordMessage message = PidRecordMessage.creation( + pid, + "", // TODO parameter is deprecated and will be removed soon. + AuthenticationHelper.getPrincipal(), + ControllerUtils.getLocalHostname()); + try { + this.messagingService.send(message); + } catch (Exception e) { + LOG.error("Could not notify messaging service about the following message: {}", message); + } + + // save the record to elastic + this.saveToElastic(pidRecord); + } catch (Exception e) { + LOG.error("Could not register PID for record {}. Error: {}", pidRecord, e.getMessage()); + failedRecords.add(pidRecord); + validatedRecords.remove(pidRecord); + } + }); + + Instant endTime = Instant.now(); + + // return the created records + LOG.info("Total time taken: {} ms", ChronoUnit.MILLIS.between(startTime, endTime)); + LOG.info("-- Time taken for mapping: {} ms", ChronoUnit.MILLIS.between(startTime, mappingTime)); + LOG.info("-- Time taken for validation: {} ms", ChronoUnit.MILLIS.between(mappingTime, validationTime)); + LOG.info("-- Time taken for registration: {} ms", ChronoUnit.MILLIS.between(validationTime, endTime)); + + if (!failedRecords.isEmpty()) { + for (PIDRecord successfulRecord : validatedRecords) { // rollback the successful records + try { + LOG.debug("Rolling back PID creation for record with PID {}.", successfulRecord.getPid()); + this.typingService.deletePid(successfulRecord.getPid()); + } catch (Exception e) { + LOG.error("Could not rollback PID creation for record with PID {}. Error: {}", successfulRecord.getPid(), e.getMessage()); + } + } + + LOG.info("Creation finished. Returning validated records for {} records. {} records failed to be created.", validatedRecords.size(), failedRecords.size()); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(failedRecords); + } else { + LOG.info("Creation finished. Returning validated records for {} records.", validatedRecords.size()); + return ResponseEntity.status(HttpStatus.CREATED).body(validatedRecords); + } + } + + /** + * This method generates a mapping between user-provided "fantasy" PIDs and real PIDs. + * + * @param rec the list of records produced by the user + * @param dryrun whether the operation is a dryrun or not + * @return a map between the user-provided PIDs (key) and the real PIDs (values) + * @throws IOException if the prefix is not configured + * @throws RecordValidationException if the same internal PID is used for multiple records + * @throws ExternalServiceException if the PID generation fails + */ + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "pit_generate_pid_mapping", description = "Time taken to generate PID mappings") + private Map generatePIDMapping(@SpanAttribute List rec, @SpanAttribute boolean dryrun) throws IOException, RecordValidationException, ExternalServiceException { + Map pidMappings = new HashMap<>(); + for (PIDRecord pidRecord : rec) { + String internalPID = pidRecord.getPid(); // the internal PID is the one given by the user + if (!internalPID.isBlank() && pidMappings.containsKey(internalPID)) { // check if the internal PID was already used + // This internal PID was already used by some other record in the same request. + throw new RecordValidationException(pidRecord, "The PID " + internalPID + " was used for multiple records in the same request."); + } + + pidRecord.setPid(""); // clear the PID field in the record + if (dryrun) { // if it is a dryrun, we set the PID to a temporary value + pidRecord.setPid("dryrun_" + pidMappings.size()); + } else { + setPid(pidRecord); // otherwise, we generate a real PID + } + pidMappings.put(internalPID, pidRecord.getPid()); // store the mapping between the internal and real PID + } + return pidMappings; + } + + /** + * This method applies the mappings between temporary PIDs and real PIDs to the records and validates them. + * + * @param rec the list of records produced by the user + * @param pidMappings the map between the user-provided PIDs (key) and the real PIDs (values) + * @param prefix the prefix to be used for the real PIDs + * @return the list of validated records + * @throws RecordValidationException as a possible validation outcome + * @throws ExternalServiceException as a possible validation outcome + */ + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "pit_apply_mappings_and_validate", description = "Time taken to apply mappings and validate records") + private List applyMappingsToRecordsAndValidate(@SpanAttribute List rec, Map pidMappings, @SpanAttribute String prefix) throws RecordValidationException, ExternalServiceException { + List validatedRecords = new ArrayList<>(); + for (PIDRecord pidRecord : rec) { + + // use this map to replace all temporary PIDs in the record values with their corresponding real PIDs + pidRecord.getEntries().values().stream() // get all values of the record + .flatMap(List::stream) // flatten the list of values + .filter(entry -> entry.getValue() != null) // Filter out null values + .filter(entry -> pidMappings.containsKey(entry.getValue())) // replace only if the value (aka. "fantasy PID") is a key in the map + .peek(entry -> LOG.debug("Found reference. Replacing {} with {}.", entry.getValue(), prefix + pidMappings.get(entry.getValue()))) // log the replacement + .forEach(entry -> entry.setValue(prefix + pidMappings.get(entry.getValue()))); // replace the value with the real PID according to the map + + // validate the record + this.typingService.validate(pidRecord); + + // store the record + validatedRecords.add(pidRecord); + LOG.debug("Record {} is valid.", pidRecord); + } + return validatedRecords; + } + + @Override + @WithSpan(kind = SpanKind.SERVER) + @Timed(value = "pit_create_pid", description = "Time taken to create a single PID record") + @Counted(value = "pit_create_pid_total", description = "Total number of create PID requests") + public ResponseEntity createPID( + @SpanAttribute PIDRecord pidRecord, + @SpanAttribute boolean dryrun, + + final WebRequest request, + final HttpServletResponse response, + final UriComponentsBuilder uriBuilder + ) throws IOException { + LOG.info("Creating PID"); + + if (dryrun) { + pidRecord.setPid("dryrun"); + } else { + setPid(pidRecord); + } + + this.typingService.validate(pidRecord); + + if (dryrun) { + // dryrun only does validation. Stop now and return as we would later on. + return ResponseEntity.status(HttpStatus.OK).eTag(quotedEtag(pidRecord)).body(pidRecord); + } + + String pid = this.typingService.registerPid(pidRecord); + pidRecord.setPid(pid); + + if (applicationProps.getStorageStrategy().storesModified()) { + storeLocally(pid, true); + } + PidRecordMessage message = PidRecordMessage.creation( + pid, + "", // TODO parameter is deprecated and will be removed soon. + AuthenticationHelper.getPrincipal(), + ControllerUtils.getLocalHostname()); + try { + this.messagingService.send(message); + } catch (Exception e) { + LOG.error("Could not notify messaging service about the following message: {}", message); + } + this.saveToElastic(pidRecord); + return ResponseEntity.status(HttpStatus.CREATED).eTag(quotedEtag(pidRecord)).body(pidRecord); + } + + @WithSpan(kind = SpanKind.INTERNAL) + private boolean hasPid(@SpanAttribute PIDRecord pidRecord) { + return pidRecord.getPid() != null && !pidRecord.getPid().isBlank(); + } + + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "pit_set_pid", description = "Time taken to set PID on record") + private void setPid(@SpanAttribute PIDRecord pidRecord) throws IOException { + boolean hasCustomPid = hasPid(pidRecord); + boolean allowsCustomPids = pidGenerationProperties.isCustomClientPidsEnabled(); + + if (allowsCustomPids && hasCustomPid) { + // in this only case, we do not have to generate a PID + // but we have to check if the PID is already registered and return an error if so + String prefix = this.typingService.getPrefix() + .orElseThrow(() -> new InvalidConfigException("No prefix configured.")); + String maybeSuffix = pidRecord.getPid(); + String pid = PidSuffix.asPrefixedChecked(maybeSuffix, prefix); + boolean isRegisteredPid = this.typingService.isPidRegistered(pid); + if (isRegisteredPid) { + throw new PidAlreadyExistsException(pidRecord.getPid()); + } + } else { + // In all other (usual) cases, we have to generate a PID. + // We store only the suffix in the pid field. + // The registration at the PID service will preprend the prefix. + + Stream suffixStream = suffixGenerator.infiniteStream(); + Optional maybeSuffix = Streams.failableStream(suffixStream) + // With failable streams, we can throw exceptions. + .filter(suffix -> !this.typingService.isPidRegistered(suffix)) + .stream() // back to normal java streams + .findFirst(); // as the stream is infinite, we should always find a prefix. + PidSuffix suffix = maybeSuffix + .orElseThrow(() -> new ExternalServiceException("Could not generate PID suffix which did not exist yet.")); + pidRecord.setPid(suffix.get()); + } + } + + @Override + @WithSpan(kind = SpanKind.SERVER) + @Timed(value = "pit_update_pid", description = "Time taken to update a PID record") + @Counted(value = "pit_update_pid_total", description = "Total number of update PID requests") + public ResponseEntity updatePID( + @SpanAttribute PIDRecord pidRecord, + @SpanAttribute boolean dryrun, + + final WebRequest request, + final HttpServletResponse response, + final UriComponentsBuilder uriBuilder + ) throws IOException { + // PID validation + String pid = getContentPathFromRequest("pid", request); + String pidInternal = pidRecord.getPid(); + if (hasPid(pidRecord) && !pid.equals(pidInternal)) { + throw new RecordValidationException( + pidRecord, + "Optional PID in record is given (%s), but it was not the same as the PID in the URL (%s). Ignore request, assuming this was not intended.".formatted(pidInternal, pid)); + } + + PIDRecord existingRecord = this.resolver.resolve(pid); + if (existingRecord == null) { + throw new PidNotFoundException(pid); + } + + // record validation + pidRecord.setPid(pid); + this.typingService.validate(pidRecord); + + // throws exception (HTTP 412) if check fails. + ControllerUtils.checkEtag(request, existingRecord); + + if (dryrun) { + // dryrun only does validation. Stop now and return as we would later on. + return ResponseEntity.ok().eTag(quotedEtag(pidRecord)).body(pidRecord); + } + + // update and send message + if (this.typingService.updatePid(pidRecord)) { + // store pid locally + if (applicationProps.getStorageStrategy().storesModified()) { + storeLocally(pidRecord.getPid(), true); + } + // distribute pid to other services + PidRecordMessage message = PidRecordMessage.update( + pid, + "", // TODO parameter is depricated and will be removed soon. + AuthenticationHelper.getPrincipal(), + ControllerUtils.getLocalHostname()); + this.messagingService.send(message); + this.saveToElastic(pidRecord); + return ResponseEntity.ok().eTag(quotedEtag(pidRecord)).body(pidRecord); + } else { + throw new PidNotFoundException(pid); + } + } + + /** + * Stores the PID in a local database. + * + * @param pid the PID + * @param update if true, updates the modified timestamp if it already exists. + * If it does not exist, it will be created with both timestamps + * (created and modified) being the same. + */ + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "pit_store_locally", description = "Time taken to store PID locally") + private void storeLocally(@SpanAttribute String pid, @SpanAttribute boolean update) { + Instant now = Instant.now(); + Optional oldPid = localPidStorage.findByPid(pid); + if (oldPid.isEmpty()) { + localPidStorage.saveAndFlush(new KnownPid(pid, now, now)); + } else if (update) { + KnownPid newPid = oldPid.get(); + newPid.setModified(now); + localPidStorage.saveAndFlush(newPid); + } + } + + @WithSpan(kind = SpanKind.INTERNAL) + private String getContentPathFromRequest(@SpanAttribute String lastPathElement, WebRequest request) { + String requestedUri = (String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE, + RequestAttributes.SCOPE_REQUEST); + if (requestedUri == null) { + throw new CustomInternalServerError("Unable to obtain request URI."); + } + return requestedUri.substring(requestedUri.indexOf(lastPathElement + "/") + (lastPathElement + "/").length()); + } + + @Override + @WithSpan(kind = SpanKind.SERVER) + @Timed(value = "pit_get_record", description = "Time taken to get a PID record") + @Counted(value = "pit_get_record_total", description = "Total number of get PID record requests") + public ResponseEntity getRecord( + @SpanAttribute boolean validation, + + final WebRequest request, + final HttpServletResponse response, + final UriComponentsBuilder uriBuilder + ) throws IOException { + String pid = getContentPathFromRequest("pid", request); + PIDRecord pidRecord = this.resolver.resolve(pid); + if (applicationProps.getStorageStrategy().storesResolved()) { + storeLocally(pid, false); + } + this.saveToElastic(pidRecord); + if (validation) { + typingService.validate(pidRecord); + } + return ResponseEntity.ok().eTag(quotedEtag(pidRecord)).body(pidRecord); + } + + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "pit_save_to_elastic", description = "Time taken to save record to Elasticsearch") + private void saveToElastic(@SpanAttribute PIDRecord rec) { + this.elastic.ifPresent( + database -> database.save( + new PidRecordElasticWrapper(rec, typingService.getOperations()) + ) + ); + } + + @Override + @WithSpan(kind = SpanKind.SERVER) + @Timed(value = "pit_find_by_pid", description = "Time taken to find a known PID") + @Counted(value = "pit_find_by_pid_total", description = "Total number of find by PID requests") + public ResponseEntity findByPid( + WebRequest request, + HttpServletResponse response, + UriComponentsBuilder uriBuilder + ) throws IOException { + String pid = getContentPathFromRequest("known-pid", request); + Optional known = this.localPidStorage.findByPid(pid); + if (known.isPresent()) { + return ResponseEntity.ok().body(known.get()); + } + return ResponseEntity.notFound().build(); + } + + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "pit_find_all_page", description = "Time taken to find paginated known PIDs") + public Page findAllPage( + @SpanAttribute Instant createdAfter, + @SpanAttribute Instant createdBefore, + @SpanAttribute Instant modifiedAfter, + @SpanAttribute Instant modifiedBefore, + Pageable pageable + ) { + final boolean queriesCreated = createdAfter != null || createdBefore != null; + final boolean queriesModified = modifiedAfter != null || modifiedBefore != null; + if (queriesCreated && createdAfter == null) { + createdAfter = Instant.EPOCH; + } + if (queriesCreated && createdBefore == null) { + createdBefore = Instant.now().plus(1, ChronoUnit.DAYS); + } + if (queriesModified && modifiedAfter == null) { + modifiedAfter = Instant.EPOCH; + } + if (queriesModified && modifiedBefore == null) { + modifiedBefore = Instant.now().plus(1, ChronoUnit.DAYS); + } + + Page resultCreatedTimestamp = Page.empty(); + Page resultModifiedTimestamp = Page.empty(); + if (queriesCreated) { + resultCreatedTimestamp = this.localPidStorage + .findDistinctPidsByCreatedBetween(createdAfter, createdBefore, pageable); + } + if (queriesModified) { + resultModifiedTimestamp = this.localPidStorage + .findDistinctPidsByModifiedBetween(modifiedAfter, modifiedBefore, pageable); + } + if (queriesCreated && queriesModified) { + final Page tmp = resultModifiedTimestamp; + final List intersection = resultCreatedTimestamp.filter((x) -> tmp.getContent().contains(x)).toList(); + return new PageImpl<>(intersection); + } else if (queriesCreated) { + return resultCreatedTimestamp; + } else if (queriesModified) { + return resultModifiedTimestamp; + } + return new PageImpl<>(this.localPidStorage.findAll()); + } + + @Override + @WithSpan(kind = SpanKind.SERVER) + @Timed(value = "pit_find_all", description = "Time taken to find all known PIDs") + @Counted(value = "pit_find_all_total", description = "Total number of find all PIDs requests") + public ResponseEntity> findAll( + @SpanAttribute Instant createdAfter, + @SpanAttribute Instant createdBefore, + @SpanAttribute Instant modifiedAfter, + @SpanAttribute Instant modifiedBefore, + Pageable pageable, + WebRequest request, + HttpServletResponse response, + UriComponentsBuilder uriBuilder) throws IOException { + Page page = this.findAllPage(createdAfter, createdBefore, modifiedAfter, modifiedBefore, pageable); + response.addHeader( + HeaderConstants.CONTENT_RANGE, + ControllerUtils.getContentRangeHeader( + page.getNumber(), + page.getSize(), + page.getTotalElements())); + return ResponseEntity.ok().body(page.getContent()); + } + + @Override + @WithSpan(kind = SpanKind.SERVER) + @Timed(value = "pit_find_all_tabular", description = "Time taken to find all known PIDs in tabular format") + @Counted(value = "pit_find_all_tabular_total", description = "Total number of find all PIDs tabular requests") + public ResponseEntity> findAllForTabular( + @SpanAttribute Instant createdAfter, + @SpanAttribute Instant createdBefore, + @SpanAttribute Instant modifiedAfter, + @SpanAttribute Instant modifiedBefore, + Pageable pageable, + WebRequest request, + HttpServletResponse response, + UriComponentsBuilder uriBuilder) throws IOException { + Page page = this.findAllPage(createdAfter, createdBefore, modifiedAfter, modifiedBefore, pageable); + response.addHeader( + HeaderConstants.CONTENT_RANGE, + ControllerUtils.getContentRangeHeader( + page.getNumber(), + page.getSize(), + page.getTotalElements())); + TabulatorPaginationFormat tabPage = new TabulatorPaginationFormat<>(page); + return ResponseEntity.ok().body(tabPage); + } + + @WithSpan(kind = SpanKind.INTERNAL) + private String quotedEtag(@SpanAttribute PIDRecord pidRecord) { + return String.format("\"%s\"", pidRecord.getEtag()); + } + +} From 5f0550f80e98533a310830577c55473619e9e8f9 Mon Sep 17 00:00:00 2001 From: Maximilian Inckmann Date: Fri, 15 Aug 2025 11:04:24 +0200 Subject: [PATCH 3/8] minor typos Signed-off-by: Maximilian Inckmann --- .../pit/elasticsearch/PidRecordElasticWrapper.java | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/pit/elasticsearch/PidRecordElasticWrapper.java b/src/main/java/edu/kit/datamanager/pit/elasticsearch/PidRecordElasticWrapper.java index 1ee67d5a..032442da 100644 --- a/src/main/java/edu/kit/datamanager/pit/elasticsearch/PidRecordElasticWrapper.java +++ b/src/main/java/edu/kit/datamanager/pit/elasticsearch/PidRecordElasticWrapper.java @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - */ package edu.kit.datamanager.pit.elasticsearch; import edu.kit.datamanager.pit.domain.Operations; @@ -40,22 +39,17 @@ public class PidRecordElasticWrapper { private static final Logger LOG = LoggerFactory.getLogger(PidRecordElasticWrapper.class); - + @Field(type = FieldType.Text) + private final List read = new ArrayList<>(); @Id private String pid; - @ElementCollection(fetch = FetchType.EAGER) private Map> attributes = new HashMap<>(); - @Field(type = FieldType.Date, format = DateFormat.basic_date_time) private Date created; - @Field(type = FieldType.Date, format = DateFormat.basic_date_time) private Date lastUpdate; - @Field(type = FieldType.Text) - private final List read = new ArrayList<>(); - @WithSpan(kind = SpanKind.INTERNAL) public PidRecordElasticWrapper(PIDRecord pidRecord, Operations dateOperations) { pid = pidRecord.getPid(); @@ -67,8 +61,7 @@ public PidRecordElasticWrapper(PIDRecord pidRecord, Operations dateOperations) { this.created = dateOperations.findDateCreated(pidRecord).orElse(null); this.lastUpdate = dateOperations.findDateModified(pidRecord).orElse(null); } catch (IOException e) { - LOG.error("Could not retrieve date from record (pid: " + pidRecord.getPid() + ").", e); - e.printStackTrace(); + LOG.error("Could not retrieve date from record (pid: {}).", pidRecord.getPid(), e); } } } From db2c2c87300c46af14886fa34581bfcba82f094d Mon Sep 17 00:00:00 2001 From: Maximilian Inckmann Date: Fri, 15 Aug 2025 15:06:39 +0200 Subject: [PATCH 4/8] optimized properties and dependencies Signed-off-by: Maximilian Inckmann --- build.gradle | 46 +++--- config/application-default.properties | 132 ++++++++++++++---- .../configuration/OpenTelemetryConfig.java | 52 ------- 3 files changed, 127 insertions(+), 103 deletions(-) delete mode 100644 src/main/java/edu/kit/datamanager/pit/configuration/OpenTelemetryConfig.java diff --git a/build.gradle b/build.gradle index 0dc862e2..873b8290 100644 --- a/build.gradle +++ b/build.gradle @@ -1,7 +1,7 @@ plugins { // Spring boot & dependency management: // https://docs.spring.io/spring-boot/docs/current/gradle-plugin/reference/htmlsingle/ - id 'org.springframework.boot' version '3.4.2' + id 'org.springframework.boot' version '3.5.4' // https://docs.spring.io/dependency-management-plugin/docs/current-SNAPSHOT/reference/html/ id "io.spring.dependency-management" version "1.1.7" // Lombok generates getter and setter and more. https://projectlombok.org/ @@ -48,12 +48,13 @@ repositories { } ext { - springDocVersion = '2.8.4' + springDocVersion = '2.8.9' + openTelemetryInstrumentationVersion = "2.18.1" } dependencyManagement { imports { - mavenBom("io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom:2.13.3") + mavenBom("io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom:${openTelemetryInstrumentationVersion}") } } @@ -88,21 +89,24 @@ dependencies { // More flexibility when (de-)serializing json: //implementation("com.monitorjbl:spring-json-view:1.1.0") - implementation("com.github.everit-org.json-schema:org.everit.json.schema:1.14.4") - + implementation("com.github.everit-org.json-schema:org.everit.json.schema:1.14.4") + implementation('org.apache.httpcomponents:httpclient:4.5.14') implementation('org.apache.httpcomponents:httpclient-cache:4.5.14') implementation("net.handle:handle-client:9.3.1") - implementation("io.opentelemetry.instrumentation:opentelemetry-spring-boot-starter") - implementation("io.opentelemetry.contrib:opentelemetry-samplers:1.44.0-alpha") -// implementation 'io.opentelemetry:opentelemetry-api:1.29.0' -// implementation 'io.opentelemetry:opentelemetry-sdk:1.29.0' -// implementation("io.opentelemetry:opentelemetry-exporter-zipkin:1.47.0") + /* Observability */ + implementation(platform("io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom:${openTelemetryInstrumentationVersion}")) + implementation "io.opentelemetry.instrumentation:opentelemetry-spring-boot-starter" + implementation "io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations" + implementation "io.opentelemetry.contrib:opentelemetry-samplers:1.48.0-alpha" + implementation "org.springframework.boot:spring-boot-starter-aop" + implementation "io.micrometer:micrometer-tracing-bridge-otel" + implementation "io.opentelemetry:opentelemetry-exporter-otlp" testImplementation(platform('org.junit:junit-bom:5.11.4')) - testImplementation('org.junit.jupiter:junit-jupiter') + testImplementation('org.junit.jupiter:junit-jupiter') testImplementation('org.junit.jupiter:junit-jupiter-params') testImplementation("org.springframework:spring-test") @@ -167,7 +171,7 @@ test { println "Tests will have verbose output" testLogging { // tests are never "up-to-date", always print everything - outputs.upToDateWhen {false} + outputs.upToDateWhen { false } // show stdio when tests are running showStandardStreams = true // for junit5 @@ -200,15 +204,15 @@ jacocoTestReport { afterEvaluate { //exclude some classes/package from code coverage report classDirectories.setFrom(files(classDirectories.files.collect { - fileTree(dir: it, exclude: [\ - 'edu/kit/datamanager/pit/configuration/**', \ - 'edu/kit/datamanager/pit/web/converter/**', \ - 'edu/kit/datamanager/pit/web/ExtendedErrorAttributes**', \ - 'edu/kit/datamanager/pit/web/UncontrolledExceptionHandler**', \ - 'edu/kit/datamanager/pit/common/**', \ - 'edu/kit/datamanager/pit/Application*' - ]) - })) + fileTree(dir: it, exclude: [\ + 'edu/kit/datamanager/pit/configuration/**', \ + 'edu/kit/datamanager/pit/web/converter/**', \ + 'edu/kit/datamanager/pit/web/ExtendedErrorAttributes**', \ + 'edu/kit/datamanager/pit/web/UncontrolledExceptionHandler**', \ + 'edu/kit/datamanager/pit/common/**', \ + 'edu/kit/datamanager/pit/Application*' + ]) + })) } } diff --git a/config/application-default.properties b/config/application-default.properties index 4df22235..a60fe751 100644 --- a/config/application-default.properties +++ b/config/application-default.properties @@ -271,55 +271,127 @@ spring.jpa.hibernate.ddl-auto=update ################################ ####### Observability ########## ################################ -# Generic OpenTelemetry Configuration -#management.endpoints.web.exposure.include=* -#management.endpoint.health.show-details=always + +### General OpenTelemetry Configuration ### +# Allows unrestricted access to Prometheus metrics endpoint management.endpoint.prometheus.access=unrestricted -management.metrics.distribution.sla.http.server.requests=100ms,500ms,1000ms -management.metrics.export.defaults.step=15s -management.metrics.distribution.percentiles-histogram.http.server.requests=true -management.metrics.tags.service_name=${spring.application.name} -management.metrics.tags.environment=${spring.profiles.active} -management.prometheus.metrics.export.enabled=false +# Enables automatic OpenTelemetry SDK configuration otel.java.global-autoconfigure.enabled=true +# Integrates OpenTelemetry with Micrometer metrics otel.instrumentation.micrometer.enabled=true +# Service name appearing in telemetry data otel.service.name=${spring.application.name} - -# OpenTelemetry Metrics Configuration -otel.metrics.exporter=otlp +# OpenTelemetry Collector endpoint URL otel.exporter.otlp.endpoint=http://localhost:4318 +# Protocol for telemetry data export (recommended for performance) otel.exporter.otlp.protocol=http/protobuf +# Context propagation formats (W3C standards) +otel.propagators=tracecontext,baggage +# Excludes process info from telemetry to reduce data volume +otel.resource.attributes.exclude=process.command_args,process.command_line + +### OpenTelemetry Metrics Configuration ### +# Export metrics via OTLP protocol +otel.metrics.exporter=otlp +# Enable OTLP metrics export through management endpoints management.otlp.metrics.export.enabled=true -management.otlp.metrics.export.step=2s +# Metrics endpoint URL management.otlp.metrics.export.url=http://localhost:4318/v1/metrics +# Metrics export interval (production-appropriate) +management.metrics.export.defaults.step=15s +# HTTP request duration histogram buckets for SLA monitoring +management.metrics.distribution.sla.http.server.requests=1ms,10ms,50ms,100ms,200ms,500ms,1s,2s,5s +# Enable percentile histograms for timer metrics +management.metrics.distribution.percentiles-histogram[timer]=true +# Timer metrics histogram buckets (database queries, method execution) +management.metrics.distribution.sla[timer]=0.1ms,0.5ms,1ms,10ms,50ms,100ms,200ms,500ms,1s,2s,5s +# Enable percentile histograms for HTTP request metrics +management.metrics.distribution.percentiles-histogram.http.server.requests=true +# Tag all metrics with service name for filtering +management.metrics.tags.service_name=${spring.application.name} +# Tag all metrics with environment for separation +management.metrics.tags.environment=${spring.profiles.active} +# Disable Prometheus export (using OTLP instead) +management.prometheus.metrics.export.enabled=false -# OpenTelemetry Logging Configuration +### OpenTelemetry Logging Configuration ### +# Enable log export via OTLP management.otlp.logging.export.enabled=true +# Logs endpoint URL management.otlp.logging.endpoint=http://localhost:4318/v1/logs -#otel.instrumentation.logback-appender.experimental.capture-mdc-attributes=* -otel.instrumentation.logback-appender.experimental-log-attributes=true -otel.instrumentation.logback-appender.experimental.capture-code-attributes=true -otel.instrumentation.logback-appender.experimental.capture-marker-attribute=true +# Capture trace/span IDs in logs for correlation otel.instrumentation.logback-appender.experimental.capture-mdc-attributes=trace_id,span_id -#logging.pattern.level=%5p [${spring.application.name:},%X{trace_id:-},%X{span_id:-}] +# Enable logging context propagation logging.context.enabled=true -# Tracing Configuration -management.tracing.sampling.probability=1.0 +### Tracing Configuration ### +# Production-safe sampling rate (10% of requests) +management.tracing.sampling.probability=0.1 +# Traces endpoint URL management.otlp.tracing.endpoint=http://localhost:4318/v1/traces +# Record HTTP exchanges for detailed analysis management.httpexchanges.recording.enabled=true +# Enable baggage correlation for distributed traces management.tracing.baggage.correlation.enabled=true -management.tracing.opentelemetry.export.include-unsampled=true +# Enable @Observed annotation support management.observations.annotations.enabled=true -otel.instrumentation.http.client.emit-experimental-telemetry=true +# Enable all built-in Spring Boot observations +management.observations.enable.all=true +# Enable JVM runtime metrics (GC, memory, threads) otel.instrumentation.runtime-telemetry-java17.enabled=true +# Enable Spring WebMVC request tracing otel.instrumentation.spring-webmvc.enabled=true +# Enable OpenTelemetry annotation support (@WithSpan) otel.instrumentation.annotations.enabled=true -otel.instrumentation.http.client.capture-request-headers=true -otel.instrumentation.http.client.capture-response-headers=true -otel.instrumentation.http.client.experimental.redact-query-parameters=false -otel.instrumentation.jdbc.experimental.transaction.enabled=true -otel.resource.attributes.exclude=process.command_args,process.command_line -otel.propagators=tracecontext,baggage +# Parent-based sampling with trace ID ratio otel.traces.sampler=parentbased_traceidratio -otel.traces.sampler.arg=1 \ No newline at end of file +# Sampling ratio argument (10%) +otel.traces.sampler.arg=0.1 +# Enable Spring Boot application lifecycle tracing +otel.instrumentation.spring-boot.enabled=true +# Enable Spring Data repository tracing +otel.instrumentation.spring-data.enabled=true +# Enable RabbitMQ message tracing +otel.instrumentation.spring-rabbit.enabled=true +# Enable Spring Security tracing +otel.instrumentation.spring-security.enabled=true + +### 5) Sensitive Information Configuration (COMMENTED OUT FOR PRODUCTION) ### +## WARNING: These capture potentially sensitive data - NOT for production +## Captures HTTP request headers (may contain auth tokens) +# otel.instrumentation.http.client.capture-request-headers=true +## Captures HTTP response headers (may contain sensitive data) +# otel.instrumentation.http.client.capture-response-headers=true +## Disables URL parameter redaction (may expose sensitive URLs) +# otel.instrumentation.http.client.experimental.redact-query-parameters=false + +### 6) Experimental/Verbose Configuration (COMMENTED OUT FOR PRODUCTION) ### +## WARNING: Experimental features or high overhead - NOT for production +## Too frequent export (high overhead) +# management.otlp.metrics.export.step=2s +## 100% sampling (performance impact) +# management.tracing.sampling.probability=1.0 +## Exports unsampled traces (storage cost) +# management.tracing.opentelemetry.export.include-unsampled=true +## 100% sampling argument +# otel.traces.sampler.arg=1 +## Verbose log attributes (experimental) +# otel.instrumentation.logback-appender.experimental-log-attributes=true +## Code location capture (experimental and expensive) +# otel.instrumentation.logback-appender.experimental.capture-code-attributes=true +## Log marker capture (experimental feature) +# otel.instrumentation.logback-appender.experimental.capture-marker-attribute=true +## Experimental HTTP telemetry (may be unstable) +# otel.instrumentation.http.client.emit-experimental-telemetry=true +## Experimental JDBC tracing (may impact performance) +# otel.instrumentation.jdbc.experimental.transaction.enabled=true +## Request parameter capture (may expose sensitive data) +# otel.instrumentation.spring-webmvc.experimental.capture-request-parameters=true +## Verbose controller telemetry (experimental) +# otel.instrumentation.spring-webmvc.experimental.capture-controller-telemetry=true +## View rendering telemetry (experimental and verbose) +# otel.instrumentation.spring-webmvc.experimental.capture-view-telemetry=true +## Hibernate span control (experimental, may miss important data) +# otel.instrumentation.hibernate.experimental.span-suppression-strategy=statement-only +## Verbose JPA query reporting (experimental, performance impact) +# otel.instrumentation.jpa.experimental.query-reporting=true \ No newline at end of file diff --git a/src/main/java/edu/kit/datamanager/pit/configuration/OpenTelemetryConfig.java b/src/main/java/edu/kit/datamanager/pit/configuration/OpenTelemetryConfig.java deleted file mode 100644 index c601f99c..00000000 --- a/src/main/java/edu/kit/datamanager/pit/configuration/OpenTelemetryConfig.java +++ /dev/null @@ -1,52 +0,0 @@ -package edu.kit.datamanager.pit.configuration; - -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.contrib.sampler.RuleBasedRoutingSampler; -import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; -import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; -import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; -import io.opentelemetry.sdk.trace.export.SpanExporter; -import io.opentelemetry.sdk.trace.samplers.Sampler; -import io.opentelemetry.semconv.UrlAttributes; -import java.util.Collections; -import java.util.Map; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class OpenTelemetryConfig { - - @Bean - public AutoConfigurationCustomizerProvider otelCustomizer() { - return p -> - p.addSamplerCustomizer(this::configureSampler) - .addSpanExporterCustomizer(this::configureSpanExporter); - } - - /** suppress spans for actuator endpoints */ - private RuleBasedRoutingSampler configureSampler(Sampler fallback, ConfigProperties config) { - return RuleBasedRoutingSampler.builder(SpanKind.SERVER, fallback) - .drop(UrlAttributes.URL_PATH, "^/actuator") - .build(); - } - - /** - * Configuration for the OTLP exporter. This configuration will replace the default OTLP exporter, - * and will add a custom header to the requests. - */ - private SpanExporter configureSpanExporter(SpanExporter exporter, ConfigProperties config) { - if (exporter instanceof OtlpHttpSpanExporter) { - return ((OtlpHttpSpanExporter) exporter).toBuilder().setHeaders(this::headers).build(); - } - return exporter; - } - - private Map headers() { - return Collections.singletonMap("Authorization", "Bearer " + refreshToken()); - } - - private String refreshToken() { - // e.g. read the token from a kubernetes secret - return "token"; - } -} From 55948bb06a1e1815e67bcaf5db7e9a9cc6c34020 Mon Sep 17 00:00:00 2001 From: Maximilian Inckmann Date: Fri, 15 Aug 2025 16:04:43 +0200 Subject: [PATCH 5/8] optimized properties and dependencies Signed-off-by: Maximilian Inckmann --- config/application-default.properties | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/application-default.properties b/config/application-default.properties index a60fe751..5582bd78 100644 --- a/config/application-default.properties +++ b/config/application-default.properties @@ -287,8 +287,6 @@ otel.exporter.otlp.endpoint=http://localhost:4318 otel.exporter.otlp.protocol=http/protobuf # Context propagation formats (W3C standards) otel.propagators=tracecontext,baggage -# Excludes process info from telemetry to reduce data volume -otel.resource.attributes.exclude=process.command_args,process.command_line ### OpenTelemetry Metrics Configuration ### # Export metrics via OTLP protocol From e8fa18e2ab6765b687e68dbdf98786536c4a9202 Mon Sep 17 00:00:00 2001 From: Maximilian Inckmann Date: Fri, 15 Aug 2025 18:24:12 +0200 Subject: [PATCH 6/8] bugfix due to merge failure Signed-off-by: Maximilian Inckmann --- .../pit/pidgeneration/PidSuffix.java | 6 - .../generators/PidSuffixGenLowerCase.java | 5 - .../generators/PidSuffixGenPrefixed.java | 5 - .../generators/PidSuffixGenUpperCase.java | 5 - .../pit/typeregistry/impl/TypeApi.java | 10 +- .../pit/web/impl/TypingRESTResourceImpl.java | 383 ++++++++++-------- 6 files changed, 209 insertions(+), 205 deletions(-) diff --git a/src/main/java/edu/kit/datamanager/pit/pidgeneration/PidSuffix.java b/src/main/java/edu/kit/datamanager/pit/pidgeneration/PidSuffix.java index 30da9583..2488a0a1 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidgeneration/PidSuffix.java +++ b/src/main/java/edu/kit/datamanager/pit/pidgeneration/PidSuffix.java @@ -16,9 +16,6 @@ package edu.kit.datamanager.pit.pidgeneration; -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.instrumentation.annotations.WithSpan; - /** * A thin wrapper around a suffix string. *

@@ -42,7 +39,6 @@ public PidSuffix(String suffix) { * @param prefix the prefix to add. * @return the string with the prefix added. */ - @WithSpan(kind = SpanKind.INTERNAL) public static String asPrefixedChecked(String maybeSuffix, String prefix) { if (!maybeSuffix.startsWith(prefix)) { return prefix + maybeSuffix; @@ -56,7 +52,6 @@ public static String asPrefixedChecked(String maybeSuffix, String prefix) { * * @return the suffix without any prefix. */ - @WithSpan(kind = SpanKind.INTERNAL) public String get() { return suffix; } @@ -67,7 +62,6 @@ public String get() { * @param prefix the prefix to prepend. * @return the prefix + suffix. */ - @WithSpan(kind = SpanKind.INTERNAL) public String getWithPrefix(String prefix) { return prefix + suffix; } diff --git a/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenLowerCase.java b/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenLowerCase.java index 2e30d936..d772befa 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenLowerCase.java +++ b/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenLowerCase.java @@ -18,15 +18,11 @@ import edu.kit.datamanager.pit.pidgeneration.PidSuffix; import edu.kit.datamanager.pit.pidgeneration.PidSuffixGenerator; -import io.micrometer.observation.annotation.Observed; -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.instrumentation.annotations.WithSpan; /** * Generates a PID suffix based on a contained generator and returns the result * in lower case. */ -@Observed public class PidSuffixGenLowerCase implements PidSuffixGenerator { private final PidSuffixGenerator generator; @@ -36,7 +32,6 @@ public PidSuffixGenLowerCase(PidSuffixGenerator generator) { } @Override - @WithSpan(kind = SpanKind.INTERNAL) public PidSuffix generate() { return new PidSuffix(this.generator.generate().get().toLowerCase()); } diff --git a/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenPrefixed.java b/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenPrefixed.java index 8b5c9c31..b5de4f3e 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenPrefixed.java +++ b/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenPrefixed.java @@ -18,15 +18,11 @@ import edu.kit.datamanager.pit.pidgeneration.PidSuffix; import edu.kit.datamanager.pit.pidgeneration.PidSuffixGenerator; -import io.micrometer.observation.annotation.Observed; -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.instrumentation.annotations.WithSpan; /** * Generates a PID suffix based on a contained generator and returns the result * prefixed with a customizable string. */ -@Observed public class PidSuffixGenPrefixed implements PidSuffixGenerator { private final PidSuffixGenerator generator; @@ -38,7 +34,6 @@ public PidSuffixGenPrefixed(PidSuffixGenerator generator, String prefix) { } @Override - @WithSpan(kind = SpanKind.INTERNAL) public PidSuffix generate() { String suffix = this.generator.generate().get().toUpperCase(); return new PidSuffix(this.prefix.concat(suffix)); diff --git a/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenUpperCase.java b/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenUpperCase.java index 0d31cf6d..df8cf440 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenUpperCase.java +++ b/src/main/java/edu/kit/datamanager/pit/pidgeneration/generators/PidSuffixGenUpperCase.java @@ -18,15 +18,11 @@ import edu.kit.datamanager.pit.pidgeneration.PidSuffix; import edu.kit.datamanager.pit.pidgeneration.PidSuffixGenerator; -import io.micrometer.observation.annotation.Observed; -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.instrumentation.annotations.WithSpan; /** * Generates a PID suffix based on a contained generator and returns the result * in upper case. */ -@Observed public class PidSuffixGenUpperCase implements PidSuffixGenerator { private final PidSuffixGenerator generator; @@ -36,7 +32,6 @@ public PidSuffixGenUpperCase(PidSuffixGenerator generator) { } @Override - @WithSpan(kind = SpanKind.INTERNAL) public PidSuffix generate() { return new PidSuffix(this.generator.generate().get().toUpperCase()); } diff --git a/src/main/java/edu/kit/datamanager/pit/typeregistry/impl/TypeApi.java b/src/main/java/edu/kit/datamanager/pit/typeregistry/impl/TypeApi.java index a813bd93..93045cbe 100644 --- a/src/main/java/edu/kit/datamanager/pit/typeregistry/impl/TypeApi.java +++ b/src/main/java/edu/kit/datamanager/pit/typeregistry/impl/TypeApi.java @@ -212,11 +212,11 @@ protected RegisteredProfile extractProfileInformation(@SpanAttribute String prof }); - boolean additionalAttributesDtrTestStyle = StreamSupport.stream(typeApiResponse - .path("content") - .path("representationsAndSemantics") - .spliterator(), - true) + boolean additionalAttributesDtrTestStyle = StreamSupport + .stream(typeApiResponse + .path("content") + .path("representationsAndSemantics") + .spliterator(), true) .filter(JsonNode::isObject) .filter(node -> node.path("expression").asText("").equals("Format")) .map(node -> node.path("subSchemaRelation").asText("").equals("allowAdditionalProperties")) diff --git a/src/main/java/edu/kit/datamanager/pit/web/impl/TypingRESTResourceImpl.java b/src/main/java/edu/kit/datamanager/pit/web/impl/TypingRESTResourceImpl.java index 4de71255..10fbe34c 100644 --- a/src/main/java/edu/kit/datamanager/pit/web/impl/TypingRESTResourceImpl.java +++ b/src/main/java/edu/kit/datamanager/pit/web/impl/TypingRESTResourceImpl.java @@ -30,6 +30,7 @@ import edu.kit.datamanager.pit.pidlog.KnownPidsDao; import edu.kit.datamanager.pit.pitservice.ITypingService; import edu.kit.datamanager.pit.resolver.Resolver; +import edu.kit.datamanager.pit.web.BatchRecordResponse; import edu.kit.datamanager.pit.web.ITypingRestResource; import edu.kit.datamanager.pit.web.TabulatorPaginationFormat; import edu.kit.datamanager.service.IMessagingService; @@ -41,19 +42,16 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.instrumentation.annotations.SpanAttribute; import io.opentelemetry.instrumentation.annotations.WithSpan; -import io.swagger.v3.oas.annotations.media.Schema; import jakarta.servlet.http.HttpServletResponse; import org.apache.commons.lang3.stream.Streams; import org.apache.http.client.cache.HeaderConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.RequestAttributes; import org.springframework.web.context.request.WebRequest; @@ -67,48 +65,55 @@ import java.util.stream.Stream; @RestController -@RequestMapping(value = "/api/v1/pit") -@Schema(description = "PID Information Types API") @Observed public class TypingRESTResourceImpl implements ITypingRestResource { private static final Logger LOG = LoggerFactory.getLogger(TypingRESTResourceImpl.class); - @Autowired - protected ITypingService typingService; - @Autowired - protected Resolver resolver; - @Autowired - private ApplicationProperties applicationProps; - @Autowired - private IMessagingService messagingService; - @Autowired - private KnownPidsDao localPidStorage; + private final ITypingService typingService; + private final Resolver resolver; + private final ApplicationProperties applicationProps; + private final IMessagingService messagingService; + private final KnownPidsDao localPidStorage; + private final Optional elastic; + private final PidSuffixGenerator suffixGenerator; + private final PidGenerationProperties pidGenerationProperties; - @Autowired - private Optional elastic; - - @Autowired - private PidSuffixGenerator suffixGenerator; - - @Autowired - private PidGenerationProperties pidGenerationProperties; - - public TypingRESTResourceImpl() { + public TypingRESTResourceImpl(ITypingService typingService, Resolver resolver, ApplicationProperties applicationProps, IMessagingService messagingService, KnownPidsDao localPidStorage, Optional elastic, PidSuffixGenerator suffixGenerator, PidGenerationProperties pidGenerationProperties) { super(); + this.typingService = typingService; + this.resolver = resolver; + this.applicationProps = applicationProps; + this.messagingService = messagingService; + this.localPidStorage = localPidStorage; + this.elastic = elastic; + this.suffixGenerator = suffixGenerator; + this.pidGenerationProperties = pidGenerationProperties; } @Override @WithSpan(kind = SpanKind.SERVER) @Timed(value = "pit_create_pids", description = "Time taken to create multiple PID records") @Counted(value = "pit_create_pids_total", description = "Total number of create PIDs requests") - public ResponseEntity> createPIDs( + public ResponseEntity createPIDs( @SpanAttribute List rec, @SpanAttribute boolean dryrun, WebRequest request, HttpServletResponse response, UriComponentsBuilder uriBuilder ) throws IOException, RecordValidationException, ExternalServiceException { + if (rec == null || rec.isEmpty()) { + LOG.warn("No records provided for PID creation."); + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(new BatchRecordResponse(Collections.emptyList(), Collections.emptyMap())); + } + if (rec.size() == 1) { + // If only one record is provided, we can use the single record creation method. + LOG.info("Only one record provided. Using single record creation method."); + var result = createPID(rec.getFirst(), dryrun, request, response, uriBuilder); + // Return the single record in a list + assert result.getBody() != null; + return ResponseEntity.status(result.getStatusCode()).headers(result.getHeaders()).body(new BatchRecordResponse(Collections.singletonList(result.getBody()), Collections.singletonMap(rec.getFirst().getPid(), result.getBody().getPid()))); + } Instant startTime = Instant.now(); LOG.info("Creating PIDs for {} records.", rec.size()); String prefix = this.typingService.getPrefix().orElseThrow(() -> new IOException("No prefix configured.")); @@ -127,10 +132,11 @@ public ResponseEntity> createPIDs( LOG.info("-- Time taken for mapping: {} ms", ChronoUnit.MILLIS.between(startTime, mappingTime)); LOG.info("-- Time taken for validation: {} ms", ChronoUnit.MILLIS.between(mappingTime, validationTime)); LOG.info("Dryrun finished. Returning validated records for {} records.", validatedRecords.size()); - return ResponseEntity.status(HttpStatus.OK).body(validatedRecords); + return ResponseEntity.status(HttpStatus.OK).body(new BatchRecordResponse(validatedRecords, pidMappings)); } List failedRecords = new ArrayList<>(); + List successfulRecords = new ArrayList<>(); // register the records validatedRecords.forEach(pidRecord -> { try { @@ -157,10 +163,11 @@ public ResponseEntity> createPIDs( // save the record to elastic this.saveToElastic(pidRecord); + successfulRecords.add(pidRecord); + LOG.debug("Successfully registered PID for record: {}", pidRecord); } catch (Exception e) { LOG.error("Could not register PID for record {}. Error: {}", pidRecord, e.getMessage()); failedRecords.add(pidRecord); - validatedRecords.remove(pidRecord); } }); @@ -173,87 +180,27 @@ public ResponseEntity> createPIDs( LOG.info("-- Time taken for registration: {} ms", ChronoUnit.MILLIS.between(validationTime, endTime)); if (!failedRecords.isEmpty()) { - for (PIDRecord successfulRecord : validatedRecords) { // rollback the successful records + List rollbackFailures = new ArrayList<>(); + for (PIDRecord successfulRecord : successfulRecords) { // rollback the successful records try { LOG.debug("Rolling back PID creation for record with PID {}.", successfulRecord.getPid()); this.typingService.deletePid(successfulRecord.getPid()); } catch (Exception e) { + rollbackFailures.add(successfulRecord.getPid()); LOG.error("Could not rollback PID creation for record with PID {}. Error: {}", successfulRecord.getPid(), e.getMessage()); } } - LOG.info("Creation finished. Returning validated records for {} records. {} records failed to be created.", validatedRecords.size(), failedRecords.size()); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(failedRecords); - } else { - LOG.info("Creation finished. Returning validated records for {} records.", validatedRecords.size()); - return ResponseEntity.status(HttpStatus.CREATED).body(validatedRecords); - } - } - - /** - * This method generates a mapping between user-provided "fantasy" PIDs and real PIDs. - * - * @param rec the list of records produced by the user - * @param dryrun whether the operation is a dryrun or not - * @return a map between the user-provided PIDs (key) and the real PIDs (values) - * @throws IOException if the prefix is not configured - * @throws RecordValidationException if the same internal PID is used for multiple records - * @throws ExternalServiceException if the PID generation fails - */ - @WithSpan(kind = SpanKind.INTERNAL) - @Timed(value = "pit_generate_pid_mapping", description = "Time taken to generate PID mappings") - private Map generatePIDMapping(@SpanAttribute List rec, @SpanAttribute boolean dryrun) throws IOException, RecordValidationException, ExternalServiceException { - Map pidMappings = new HashMap<>(); - for (PIDRecord pidRecord : rec) { - String internalPID = pidRecord.getPid(); // the internal PID is the one given by the user - if (!internalPID.isBlank() && pidMappings.containsKey(internalPID)) { // check if the internal PID was already used - // This internal PID was already used by some other record in the same request. - throw new RecordValidationException(pidRecord, "The PID " + internalPID + " was used for multiple records in the same request."); + if (!rollbackFailures.isEmpty()) { + LOG.error("Failed to rollback {} PIDs: {}", rollbackFailures.size(), rollbackFailures); } - pidRecord.setPid(""); // clear the PID field in the record - if (dryrun) { // if it is a dryrun, we set the PID to a temporary value - pidRecord.setPid("dryrun_" + pidMappings.size()); - } else { - setPid(pidRecord); // otherwise, we generate a real PID - } - pidMappings.put(internalPID, pidRecord.getPid()); // store the mapping between the internal and real PID - } - return pidMappings; - } - - /** - * This method applies the mappings between temporary PIDs and real PIDs to the records and validates them. - * - * @param rec the list of records produced by the user - * @param pidMappings the map between the user-provided PIDs (key) and the real PIDs (values) - * @param prefix the prefix to be used for the real PIDs - * @return the list of validated records - * @throws RecordValidationException as a possible validation outcome - * @throws ExternalServiceException as a possible validation outcome - */ - @WithSpan(kind = SpanKind.INTERNAL) - @Timed(value = "pit_apply_mappings_and_validate", description = "Time taken to apply mappings and validate records") - private List applyMappingsToRecordsAndValidate(@SpanAttribute List rec, Map pidMappings, @SpanAttribute String prefix) throws RecordValidationException, ExternalServiceException { - List validatedRecords = new ArrayList<>(); - for (PIDRecord pidRecord : rec) { - - // use this map to replace all temporary PIDs in the record values with their corresponding real PIDs - pidRecord.getEntries().values().stream() // get all values of the record - .flatMap(List::stream) // flatten the list of values - .filter(entry -> entry.getValue() != null) // Filter out null values - .filter(entry -> pidMappings.containsKey(entry.getValue())) // replace only if the value (aka. "fantasy PID") is a key in the map - .peek(entry -> LOG.debug("Found reference. Replacing {} with {}.", entry.getValue(), prefix + pidMappings.get(entry.getValue()))) // log the replacement - .forEach(entry -> entry.setValue(prefix + pidMappings.get(entry.getValue()))); // replace the value with the real PID according to the map - - // validate the record - this.typingService.validate(pidRecord); - - // store the record - validatedRecords.add(pidRecord); - LOG.debug("Record {} is valid.", pidRecord); + LOG.info("Creation finished. Returning validated records for {} records. {} records failed to be created.", validatedRecords.size(), failedRecords.size()); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(new BatchRecordResponse(failedRecords, pidMappings)); + } else { + LOG.info("Creation finished. Returning successfully validated and created records for {} records of {}.", successfulRecords.size(), validatedRecords.size()); + return ResponseEntity.status(HttpStatus.CREATED).body(new BatchRecordResponse(successfulRecords, pidMappings)); } - return validatedRecords; } @Override @@ -267,7 +214,7 @@ public ResponseEntity createPID( final WebRequest request, final HttpServletResponse response, final UriComponentsBuilder uriBuilder - ) throws IOException { + ) { LOG.info("Creating PID"); if (dryrun) { @@ -303,45 +250,6 @@ public ResponseEntity createPID( return ResponseEntity.status(HttpStatus.CREATED).eTag(quotedEtag(pidRecord)).body(pidRecord); } - @WithSpan(kind = SpanKind.INTERNAL) - private boolean hasPid(@SpanAttribute PIDRecord pidRecord) { - return pidRecord.getPid() != null && !pidRecord.getPid().isBlank(); - } - - @WithSpan(kind = SpanKind.INTERNAL) - @Timed(value = "pit_set_pid", description = "Time taken to set PID on record") - private void setPid(@SpanAttribute PIDRecord pidRecord) throws IOException { - boolean hasCustomPid = hasPid(pidRecord); - boolean allowsCustomPids = pidGenerationProperties.isCustomClientPidsEnabled(); - - if (allowsCustomPids && hasCustomPid) { - // in this only case, we do not have to generate a PID - // but we have to check if the PID is already registered and return an error if so - String prefix = this.typingService.getPrefix() - .orElseThrow(() -> new InvalidConfigException("No prefix configured.")); - String maybeSuffix = pidRecord.getPid(); - String pid = PidSuffix.asPrefixedChecked(maybeSuffix, prefix); - boolean isRegisteredPid = this.typingService.isPidRegistered(pid); - if (isRegisteredPid) { - throw new PidAlreadyExistsException(pidRecord.getPid()); - } - } else { - // In all other (usual) cases, we have to generate a PID. - // We store only the suffix in the pid field. - // The registration at the PID service will preprend the prefix. - - Stream suffixStream = suffixGenerator.infiniteStream(); - Optional maybeSuffix = Streams.failableStream(suffixStream) - // With failable streams, we can throw exceptions. - .filter(suffix -> !this.typingService.isPidRegistered(suffix)) - .stream() // back to normal java streams - .findFirst(); // as the stream is infinite, we should always find a prefix. - PidSuffix suffix = maybeSuffix - .orElseThrow(() -> new ExternalServiceException("Could not generate PID suffix which did not exist yet.")); - pidRecord.setPid(suffix.get()); - } - } - @Override @WithSpan(kind = SpanKind.SERVER) @Timed(value = "pit_update_pid", description = "Time taken to update a PID record") @@ -353,7 +261,7 @@ public ResponseEntity updatePID( final WebRequest request, final HttpServletResponse response, final UriComponentsBuilder uriBuilder - ) throws IOException { + ) { // PID validation String pid = getContentPathFromRequest("pid", request); String pidInternal = pidRecord.getPid(); @@ -400,6 +308,10 @@ public ResponseEntity updatePID( } } + private boolean hasPid(PIDRecord pidRecord) { + return pidRecord.getPid() != null && !pidRecord.getPid().isBlank(); + } + /** * Stores the PID in a local database. * @@ -410,7 +322,7 @@ public ResponseEntity updatePID( */ @WithSpan(kind = SpanKind.INTERNAL) @Timed(value = "pit_store_locally", description = "Time taken to store PID locally") - private void storeLocally(@SpanAttribute String pid, @SpanAttribute boolean update) { + private void storeLocally(String pid, boolean update) { Instant now = Instant.now(); Optional oldPid = localPidStorage.findByPid(pid); if (oldPid.isEmpty()) { @@ -422,6 +334,14 @@ private void storeLocally(@SpanAttribute String pid, @SpanAttribute boolean upda } } + /** + * Extracts and returns the content path from the incoming web request, based on the specified last path element. + * + * @param lastPathElement the last path element used to determine the content path + * @param request the incoming web request containing the requested URI and attributes + * @return the extracted content path from the request + * @throws CustomInternalServerError if the requested URI cannot be obtained from the web request + */ @WithSpan(kind = SpanKind.INTERNAL) private String getContentPathFromRequest(@SpanAttribute String lastPathElement, WebRequest request) { String requestedUri = (String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE, @@ -442,7 +362,7 @@ public ResponseEntity getRecord( final WebRequest request, final HttpServletResponse response, final UriComponentsBuilder uriBuilder - ) throws IOException { + ) { String pid = getContentPathFromRequest("pid", request); PIDRecord pidRecord = this.resolver.resolve(pid); if (applicationProps.getStorageStrategy().storesResolved()) { @@ -455,16 +375,6 @@ public ResponseEntity getRecord( return ResponseEntity.ok().eTag(quotedEtag(pidRecord)).body(pidRecord); } - @WithSpan(kind = SpanKind.INTERNAL) - @Timed(value = "pit_save_to_elastic", description = "Time taken to save record to Elasticsearch") - private void saveToElastic(@SpanAttribute PIDRecord rec) { - this.elastic.ifPresent( - database -> database.save( - new PidRecordElasticWrapper(rec, typingService.getOperations()) - ) - ); - } - @Override @WithSpan(kind = SpanKind.SERVER) @Timed(value = "pit_find_by_pid", description = "Time taken to find a known PID") @@ -476,14 +386,37 @@ public ResponseEntity findByPid( ) throws IOException { String pid = getContentPathFromRequest("known-pid", request); Optional known = this.localPidStorage.findByPid(pid); - if (known.isPresent()) { - return ResponseEntity.ok().body(known.get()); - } - return ResponseEntity.notFound().build(); + return known + .map(knownPid -> ResponseEntity.ok().body(knownPid)) + .orElseGet(() -> ResponseEntity.notFound().build()); + } + + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "pit_find_all", description = "Time taken to find known PIDs") + @Counted(value = "pit_find_all_total", description = "Total number of find all PIDs requests") + @Override + public ResponseEntity> findAll( + Instant createdAfter, + Instant createdBefore, + Instant modifiedAfter, + Instant modifiedBefore, + Pageable pageable, + WebRequest request, + HttpServletResponse response, + UriComponentsBuilder uriBuilder) { + Page page = this.findAllPage(createdAfter, createdBefore, modifiedAfter, modifiedBefore, pageable); + response.addHeader( + HeaderConstants.CONTENT_RANGE, + ControllerUtils.getContentRangeHeader( + page.getNumber(), + page.getSize(), + page.getTotalElements())); + return ResponseEntity.ok().body(page.getContent()); } @WithSpan(kind = SpanKind.INTERNAL) @Timed(value = "pit_find_all_page", description = "Time taken to find paginated known PIDs") + @Counted(value = "pit_find_all_page_total", description = "Total number of paginated find all PIDs requests") public Page findAllPage( @SpanAttribute Instant createdAfter, @SpanAttribute Instant createdBefore, @@ -528,29 +461,6 @@ public Page findAllPage( return new PageImpl<>(this.localPidStorage.findAll()); } - @Override - @WithSpan(kind = SpanKind.SERVER) - @Timed(value = "pit_find_all", description = "Time taken to find all known PIDs") - @Counted(value = "pit_find_all_total", description = "Total number of find all PIDs requests") - public ResponseEntity> findAll( - @SpanAttribute Instant createdAfter, - @SpanAttribute Instant createdBefore, - @SpanAttribute Instant modifiedAfter, - @SpanAttribute Instant modifiedBefore, - Pageable pageable, - WebRequest request, - HttpServletResponse response, - UriComponentsBuilder uriBuilder) throws IOException { - Page page = this.findAllPage(createdAfter, createdBefore, modifiedAfter, modifiedBefore, pageable); - response.addHeader( - HeaderConstants.CONTENT_RANGE, - ControllerUtils.getContentRangeHeader( - page.getNumber(), - page.getSize(), - page.getTotalElements())); - return ResponseEntity.ok().body(page.getContent()); - } - @Override @WithSpan(kind = SpanKind.SERVER) @Timed(value = "pit_find_all_tabular", description = "Time taken to find all known PIDs in tabular format") @@ -563,7 +473,7 @@ public ResponseEntity> findAllForTabular( Pageable pageable, WebRequest request, HttpServletResponse response, - UriComponentsBuilder uriBuilder) throws IOException { + UriComponentsBuilder uriBuilder) { Page page = this.findAllPage(createdAfter, createdBefore, modifiedAfter, modifiedBefore, pageable); response.addHeader( HeaderConstants.CONTENT_RANGE, @@ -575,9 +485,124 @@ public ResponseEntity> findAllForTabular( return ResponseEntity.ok().body(tabPage); } + /** + * Saves a PIDRecord to Elasticsearch if the Elasticsearch database is available. + * + * @param rec the PIDRecord object to be saved to Elasticsearch + */ + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "pit_save_to_elastic", description = "Time taken to save record to Elasticsearch") + private void saveToElastic(PIDRecord rec) { + this.elastic.ifPresent( + database -> database.save( + new PidRecordElasticWrapper(rec, typingService.getOperations()) + ) + ); + } + @WithSpan(kind = SpanKind.INTERNAL) private String quotedEtag(@SpanAttribute PIDRecord pidRecord) { return String.format("\"%s\"", pidRecord.getEtag()); } + /** + * This method generates a mapping between user-provided "fantasy" PIDs and real PIDs. + * + * @param rec the list of records produced by the user + * @param dryrun whether the operation is a dryrun or not + * @return a map between the user-provided PIDs (key) and the real PIDs (values) + * @throws RecordValidationException if the same internal PID is used for multiple records + * @throws ExternalServiceException if the PID generation fails + */ + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "pit_generate_pid_mapping", description = "Time taken to generate PID mappings") + private Map generatePIDMapping(@SpanAttribute List rec, @SpanAttribute boolean dryrun) throws RecordValidationException, ExternalServiceException { + Map pidMappings = new HashMap<>(); + for (PIDRecord pidRecord : rec) { + String internalPID = pidRecord.getPid(); // the internal PID is the one given by the user + if (internalPID == null) { + internalPID = ""; // if no PID was given, we set it to an empty string + } + if (!internalPID.isBlank() && pidMappings.containsKey(internalPID)) { // check if the internal PID was already used + // This internal PID was already used by some other record in the same request. + throw new RecordValidationException(pidRecord, "The PID " + internalPID + " was used for multiple records in the same request."); + } + + pidRecord.setPid(""); // clear the PID field in the record + if (dryrun) { // if it is a dryrun, we set the PID to a temporary value + pidRecord.setPid("dryrun_" + pidMappings.size()); + } else { + setPid(pidRecord); // otherwise, we generate a real PID + } + pidMappings.put(internalPID, pidRecord.getPid()); // store the mapping between the internal and real PID + } + return pidMappings; + } + + /** + * This method applies the mappings between temporary PIDs and real PIDs to the records and validates them. + * + * @param rec the list of records produced by the user + * @param pidMappings the map between the user-provided PIDs (key) and the real PIDs (values) + * @param prefix the prefix to be used for the real PIDs + * @return the list of validated records + * @throws RecordValidationException as a possible validation outcome + * @throws ExternalServiceException as a possible validation outcome + */ + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "pit_apply_mappings_and_validate", description = "Time taken to apply mappings and validate records") + private List applyMappingsToRecordsAndValidate(@SpanAttribute List rec, Map pidMappings, @SpanAttribute String prefix) throws RecordValidationException, ExternalServiceException { + List validatedRecords = new ArrayList<>(); + for (PIDRecord pidRecord : rec) { + + // use this map to replace all temporary PIDs in the record values with their corresponding real PIDs + pidRecord.getEntries().values().stream() // get all values of the record + .flatMap(List::stream) // flatten the list of values + .filter(entry -> entry.getValue() != null) // Filter out null values + .filter(entry -> pidMappings.containsKey(entry.getValue())) // replace only if the value (aka. "fantasy PID") is a key in the map + .peek(entry -> LOG.debug("Found reference. Replacing {} with {}.", entry.getValue(), prefix + pidMappings.get(entry.getValue()))) // log the replacement + .forEach(entry -> entry.setValue(prefix + pidMappings.get(entry.getValue()))); // replace the value with the real PID according to the map + + // validate the record + this.typingService.validate(pidRecord); + + // store the record + validatedRecords.add(pidRecord); + LOG.debug("Record {} is valid.", pidRecord); + } + return validatedRecords; + } + + private void setPid(PIDRecord pidRecord) { + boolean hasCustomPid = hasPid(pidRecord); + boolean allowsCustomPids = pidGenerationProperties.isCustomClientPidsEnabled(); + + if (allowsCustomPids && hasCustomPid) { + // in this only case, we do not have to generate a PID + // but we have to check if the PID is already registered and return an error if so + String prefix = this.typingService.getPrefix() + .orElseThrow(() -> new InvalidConfigException("No prefix configured.")); + String maybeSuffix = pidRecord.getPid(); + String pid = PidSuffix.asPrefixedChecked(maybeSuffix, prefix); + boolean isRegisteredPid = this.typingService.isPidRegistered(pid); + if (isRegisteredPid) { + throw new PidAlreadyExistsException(pidRecord.getPid()); + } + } else { + // In all other (usual) cases, we have to generate a PID. + // We store only the suffix in the pid field. + // The registration at the PID service will preprend the prefix. + + Stream suffixStream = suffixGenerator.infiniteStream(); + Optional maybeSuffix = Streams.failableStream(suffixStream) + // With failable streams, we can throw exceptions. + .filter(suffix -> !this.typingService.isPidRegistered(suffix)) + .stream() // back to normal java streams + .findFirst(); // as the stream is infinite, we should always find a prefix. + PidSuffix suffix = maybeSuffix + .orElseThrow(() -> new ExternalServiceException("Could not generate PID suffix which did not exist yet.")); + pidRecord.setPid(suffix.get()); + } + } + } From d091cbfe91c36e560e24df909172c6755f93a579 Mon Sep 17 00:00:00 2001 From: Maximilian Inckmann Date: Fri, 15 Aug 2025 18:48:45 +0200 Subject: [PATCH 7/8] minor improvements Signed-off-by: Maximilian Inckmann --- build.gradle | 20 ++++++++++---------- collector-config.yaml | 19 ------------------- config/application-default.properties | 8 ++++++-- 3 files changed, 16 insertions(+), 31 deletions(-) delete mode 100644 collector-config.yaml diff --git a/build.gradle b/build.gradle index 156a8369..fe55cf13 100644 --- a/build.gradle +++ b/build.gradle @@ -89,14 +89,14 @@ dependencies { implementation "org.springframework.data:spring-data-elasticsearch" // More flexibility when (de-)serializing json: - implementation(group: 'com.networknt', name: 'json-schema-validator', version: '1.5.7'); - + implementation(group: 'com.networknt', name: 'json-schema-validator', version: '1.5.7') + implementation('org.apache.httpcomponents:httpclient:4.5.14') implementation('org.apache.httpcomponents:httpclient-cache:4.5.14') implementation("net.handle:handle-client:9.3.2") - /* Observability */ + /* Observability using OpenTelemetry */ implementation(platform("io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom:${openTelemetryInstrumentationVersion}")) implementation "io.opentelemetry.instrumentation:opentelemetry-spring-boot-starter" implementation "io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations" @@ -106,7 +106,7 @@ dependencies { implementation "io.opentelemetry:opentelemetry-exporter-otlp" testImplementation(platform('org.junit:junit-bom:5.13.1')) - testImplementation('org.junit.jupiter:junit-jupiter') + testImplementation('org.junit.jupiter:junit-jupiter') testImplementation('org.junit.jupiter:junit-jupiter-params') testImplementation("org.springframework:spring-test") @@ -205,12 +205,12 @@ jacocoTestReport { //exclude some classes/package from code coverage report classDirectories.setFrom(files(classDirectories.files.collect { fileTree(dir: it, exclude: [\ - 'edu/kit/datamanager/pit/configuration/**', \ - 'edu/kit/datamanager/pit/web/converter/**', \ - 'edu/kit/datamanager/pit/web/ExtendedErrorAttributes**', \ - 'edu/kit/datamanager/pit/web/UncontrolledExceptionHandler**', \ - 'edu/kit/datamanager/pit/common/**', \ - 'edu/kit/datamanager/pit/Application*' + 'edu/kit/datamanager/pit/configuration/**', \ + 'edu/kit/datamanager/pit/web/converter/**', \ + 'edu/kit/datamanager/pit/web/ExtendedErrorAttributes**', \ + 'edu/kit/datamanager/pit/web/UncontrolledExceptionHandler**', \ + 'edu/kit/datamanager/pit/common/**', \ + 'edu/kit/datamanager/pit/Application*' ]) })) } diff --git a/collector-config.yaml b/collector-config.yaml deleted file mode 100644 index 0ae18ced..00000000 --- a/collector-config.yaml +++ /dev/null @@ -1,19 +0,0 @@ -receivers: - otlp: - protocols: - http: - endpoint: "0.0.0.0:4318" -exporters: - debug: - verbosity: detailed -service: - pipelines: - metrics: - receivers: [otlp] - exporters: [debug] - traces: - receivers: [otlp] - exporters: [debug] - logs: - receivers: [otlp] - exporters: [debug] diff --git a/config/application-default.properties b/config/application-default.properties index 090211ae..91037d73 100644 --- a/config/application-default.properties +++ b/config/application-default.properties @@ -340,7 +340,7 @@ management.otlp.logging.export.enabled=true # Logs endpoint URL management.otlp.logging.endpoint=http://localhost:4318/v1/logs # Capture trace/span IDs in logs for correlation -otel.instrumentation.logback-appender.experimental.capture-mdc-attributes=trace_id,span_id +otel.instrumentation.log4j-appender.experimental.capture-mdc-attributes=trace_id,span_id # Enable logging context propagation logging.context.enabled=true @@ -357,11 +357,15 @@ management.tracing.baggage.correlation.enabled=true management.observations.annotations.enabled=true # Enable all built-in Spring Boot observations management.observations.enable.all=true -# Enable JVM runtime metrics (GC, memory, threads) +# Enable runtime telemetry (JVM metrics) +otel.instrumentation.runtime-telemetry.enabled=true +# (optional: enable JFR/Java17-specific telemetry) otel.instrumentation.runtime-telemetry-java17.enabled=true # Enable Spring WebMVC request tracing otel.instrumentation.spring-webmvc.enabled=true # Enable OpenTelemetry annotation support (@WithSpan) +otel.instrumentation.opentelemetry-instrumentation-annotations.enabled=true +# Enable OpenTelemetry annotation support (@WithSpan) otel.instrumentation.annotations.enabled=true # Parent-based sampling with trace ID ratio otel.traces.sampler=parentbased_traceidratio From 33653cfdf18360ac8a1734bb5d76b2489885fe27 Mon Sep 17 00:00:00 2001 From: Maximilian Inckmann Date: Wed, 20 Aug 2025 18:47:07 +0200 Subject: [PATCH 8/8] added support for optional PII tracing through OpenTelemetry minor fixes Signed-off-by: Maximilian Inckmann --- config/application-default.properties | 2 + .../edu/kit/datamanager/pit/Application.java | 188 ++++---- .../configuration/ApplicationProperties.java | 432 +++++++++--------- .../pit/configuration/PIISpanAttribute.java | 48 ++ .../configuration/PIISpanAttributeAspect.java | 352 ++++++++++++++ .../impl/InMemoryIdentifierSystem.java | 12 +- .../pit/pidsystem/impl/handle/HandleDiff.java | 25 +- .../impl/handle/HandleProtocolAdapter.java | 16 +- .../pidsystem/impl/local/LocalPidSystem.java | 12 +- .../impl/EmbeddedStrictValidatorStrategy.java | 56 ++- .../pit/pitservice/impl/TypingService.java | 79 ++-- .../datamanager/pit/resolver/Resolver.java | 4 +- .../pit/typeregistry/AttributeInfo.java | 37 +- .../pit/typeregistry/RegisteredProfile.java | 36 +- .../RegisteredProfileAttribute.java | 19 + .../pit/web/ExtendedErrorAttributes.java | 39 +- .../pit/web/impl/TypingRESTResourceImpl.java | 23 +- 17 files changed, 935 insertions(+), 445 deletions(-) create mode 100644 src/main/java/edu/kit/datamanager/pit/configuration/PIISpanAttribute.java create mode 100644 src/main/java/edu/kit/datamanager/pit/configuration/PIISpanAttributeAspect.java diff --git a/config/application-default.properties b/config/application-default.properties index 91037d73..6a6e994a 100644 --- a/config/application-default.properties +++ b/config/application-default.properties @@ -388,6 +388,8 @@ otel.instrumentation.spring-security.enabled=true # otel.instrumentation.http.client.capture-response-headers=true ## Disables URL parameter redaction (may expose sensitive URLs) # otel.instrumentation.http.client.experimental.redact-query-parameters=false +## Include PII (Personally Identifiable Information) as trace attributes +pit.observability.includePiiInTraces=true ### 6) Experimental/Verbose Configuration (COMMENTED OUT FOR PRODUCTION) ### ## WARNING: Experimental features or high overhead - NOT for production diff --git a/src/main/java/edu/kit/datamanager/pit/Application.java b/src/main/java/edu/kit/datamanager/pit/Application.java index 056ec5ee..9a28e473 100644 --- a/src/main/java/edu/kit/datamanager/pit/Application.java +++ b/src/main/java/edu/kit/datamanager/pit/Application.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 Karlsruhe Institute of Technology. + * Copyright (c) 2025 Karlsruhe Institute of Technology. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; - import edu.kit.datamanager.pit.cli.CliTaskBootstrap; import edu.kit.datamanager.pit.cli.CliTaskWriteFile; import edu.kit.datamanager.pit.cli.ICliTask; @@ -36,19 +35,11 @@ import edu.kit.datamanager.pit.typeregistry.schema.SchemaSetGenerator; import edu.kit.datamanager.pit.web.converter.SimplePidRecordConverter; import edu.kit.datamanager.security.filter.KeycloakJwtProperties; - -import java.io.IOException; -import java.util.Objects; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.stream.Stream; - import org.apache.http.client.HttpClient; import org.apache.http.impl.client.cache.CacheConfig; import org.apache.http.impl.client.cache.CachingHttpClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.springframework.beans.factory.InjectionPoint; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -57,71 +48,52 @@ import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.EnableAspectJAutoProxy; import org.springframework.context.annotation.Scope; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; import org.springframework.scheduling.annotation.EnableScheduling; +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Stream; + @SpringBootApplication @EnableScheduling -@EntityScan({ "edu.kit.datamanager" }) +@EntityScan({"edu.kit.datamanager"}) // Required for "DAO" objects to work, needed for messaging service and database // mappings @EnableJpaRepositories("edu.kit.datamanager") // Detects services and components in datamanager dependencies (service-base and // repo-core) -@ComponentScan({ "edu.kit.datamanager" }) +@ComponentScan({"edu.kit.datamanager"}) +// Scans for components in the current package and sub-packages +@EnableAspectJAutoProxy(proxyTargetClass = true, exposeProxy = true) +// Scans for aspects in the current package and sub-packages (e.g. for PIISpanAttribute) public class Application { - private static final Logger LOG = LoggerFactory.getLogger(Application.class); - + /** + * This is a threshold considered very long for a http request. + * Usually used in logging context + */ + public static final long LONG_HTTP_REQUEST_THRESHOLD = 400; protected static final String CMD_BOOTSTRAP = "bootstrap"; protected static final String CMD_WRITE_FILE = "write-file"; protected static final String SOURCE_FROM_PREFIX = "all-pids-from-prefix"; protected static final String SOURCE_KNOWN_PIDS = "known-pids"; - + protected static final String ERROR_COMMUNICATION = "Communication error: {}"; protected static final String ERROR_CONFIGURATION = "Configuration error: {}"; - - /** - * This is a threshold considered very long for a http request. - * Usually used in logging context - */ - public static final long LONG_HTTP_REQUEST_THRESHOLD = 400; - - @Bean - @Scope("prototype") - public Logger logger(InjectionPoint injectionPoint) { - Class targetClass = injectionPoint.getMember().getDeclaringClass(); - return LoggerFactory.getLogger(targetClass.getCanonicalName()); - } + private static final Logger LOG = LoggerFactory.getLogger(Application.class); public static ExecutorService newExecutor() { return Executors.newThreadPerTaskExecutor(Executors.defaultThreadFactory()); } - @Bean - public SchemaSetGenerator schemaSetGenerator(ApplicationProperties props) { - return new SchemaSetGenerator(props); - } - - @Bean - public ITypeRegistry typeRegistry(ApplicationProperties props, SchemaSetGenerator schemaSetGenerator) { - return new TypeApi(props, schemaSetGenerator); - } - - @Bean - public Resolver resolver(ITypingService identifierSystem) { - return new Resolver(identifierSystem); - } - - @Bean - public ITypingService typingService(IIdentifierSystem identifierSystem, ITypeRegistry typeRegistry) { - return new TypingService(identifierSystem, typeRegistry); - } - @Bean(name = "OBJECT_MAPPER_BEAN") public static ObjectMapper jsonObjectMapper() { return Jackson2ObjectMapperBuilder.json() @@ -131,59 +103,21 @@ public static ObjectMapper jsonObjectMapper() { .build(); } - @Bean - public HttpClient httpClient() { - return CachingHttpClientBuilder - .create() - .setCacheConfig(cacheConfig()) - .build(); - } - - @Bean - public CacheConfig cacheConfig() { - return CacheConfig - .custom() - .setMaxObjectSize(500000) // 500KB - .setMaxCacheEntries(2000) - // Set this to false and a response with queryString - // will be cached when it is explicitly cacheable - // .setNeverCacheHTTP10ResponsesWithQueryString(false) - .build(); - } - - @Bean - @ConfigurationProperties("pit") - public ApplicationProperties applicationProperties() { - return new ApplicationProperties(); - } - - @Bean - // Reads keycloak related settings from properties.application. - public KeycloakJwtProperties properties() { - return new KeycloakJwtProperties(); - } - - @Bean - public HttpMessageConverter simplePidRecordConverter() { - return new SimplePidRecordConverter(); - } - public static void main(String[] args) { ConfigurableApplicationContext context = SpringApplication.run(Application.class, args); System.out.println("Spring is running!"); - final boolean cliArgsAmountValid = args != null && args.length != 0 && args.length >= 2; - + final boolean cliArgsAmountValid = args != null && args.length >= 2; + if (cliArgsAmountValid) { ICliTask task = null; Stream pidSource = null; - + if (Objects.equals(args[1], SOURCE_FROM_PREFIX)) { try { pidSource = PidSource.fromPrefix(context); } catch (IOException e) { - e.printStackTrace(); - LOG.error(ERROR_COMMUNICATION, e.getMessage()); + LOG.error(ERROR_COMMUNICATION, e); exitApp(context, 1); } } else if (Objects.equals(args[1], SOURCE_KNOWN_PIDS)) { @@ -207,20 +141,18 @@ public static void main(String[] args) { exitApp(context, 1); } } catch (InvalidConfigException e) { - e.printStackTrace(); - LOG.error(ERROR_CONFIGURATION, e.getMessage()); + LOG.error(ERROR_CONFIGURATION, e); exitApp(context, 1); } catch (IOException e) { - e.printStackTrace(); - LOG.error(ERROR_COMMUNICATION, e.getMessage()); + LOG.error(ERROR_COMMUNICATION, e); exitApp(context, 1); } } } private static void printUsage(String[] args) { - String firstArg = args[0].replaceAll("[\r\n]",""); - String secondArg = args[1].replaceAll("[\r\n]",""); + String firstArg = args[0].replaceAll("[\r\n]", ""); + String secondArg = args[1].replaceAll("[\r\n]", ""); LOG.error("Got commands: {} and {}", firstArg, secondArg); LOG.error("CLI usage incorrect. Usage:"); LOG.error("java -jar TypedPIDMaker.jar [ACTION] [SOURCE]"); @@ -245,4 +177,68 @@ private static void exitApp(ConfigurableApplicationContext context, int errCode) System.exit(errCode); } + @Bean + @Scope("prototype") + public Logger logger(InjectionPoint injectionPoint) { + Class targetClass = injectionPoint.getMember().getDeclaringClass(); + return LoggerFactory.getLogger(targetClass.getCanonicalName()); + } + + @Bean + public SchemaSetGenerator schemaSetGenerator(ApplicationProperties props) { + return new SchemaSetGenerator(props); + } + + @Bean + public ITypeRegistry typeRegistry(ApplicationProperties props, SchemaSetGenerator schemaSetGenerator) { + return new TypeApi(props, schemaSetGenerator); + } + + @Bean + public Resolver resolver(ITypingService identifierSystem) { + return new Resolver(identifierSystem); + } + + @Bean + public ITypingService typingService(IIdentifierSystem identifierSystem, ITypeRegistry typeRegistry) { + return new TypingService(identifierSystem, typeRegistry, applicationProperties()); + } + + @Bean + @ConfigurationProperties("pit") + public ApplicationProperties applicationProperties() { + return new ApplicationProperties(); + } + + @Bean + public HttpClient httpClient() { + return CachingHttpClientBuilder + .create() + .setCacheConfig(cacheConfig()) + .build(); + } + + @Bean + public CacheConfig cacheConfig() { + return CacheConfig + .custom() + .setMaxObjectSize(500000) // 500KB + .setMaxCacheEntries(2000) + // Set this to false and a response with queryString + // will be cached when it is explicitly cacheable + // .setNeverCacheHTTP10ResponsesWithQueryString(false) + .build(); + } + + @Bean + // Reads keycloak related settings from properties.application. + public KeycloakJwtProperties properties() { + return new KeycloakJwtProperties(); + } + + @Bean + public HttpMessageConverter simplePidRecordConverter() { + return new SimplePidRecordConverter(); + } + } diff --git a/src/main/java/edu/kit/datamanager/pit/configuration/ApplicationProperties.java b/src/main/java/edu/kit/datamanager/pit/configuration/ApplicationProperties.java index a9c422fa..2951c0ab 100644 --- a/src/main/java/edu/kit/datamanager/pit/configuration/ApplicationProperties.java +++ b/src/main/java/edu/kit/datamanager/pit/configuration/ApplicationProperties.java @@ -1,220 +1,212 @@ -/* - * Copyright 2018 Karlsruhe Institute of Technology. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package edu.kit.datamanager.pit.configuration; - -import edu.kit.datamanager.configuration.GenericApplicationProperties; -import edu.kit.datamanager.pit.pitservice.IValidationStrategy; -import edu.kit.datamanager.pit.pitservice.impl.EmbeddedStrictValidatorStrategy; -import edu.kit.datamanager.pit.pitservice.impl.NoValidationStrategy; - -import java.net.URL; -import java.util.List; -import java.util.Set; - -import edu.kit.datamanager.pit.typeregistry.ITypeRegistry; -import jakarta.validation.constraints.NotNull; - -import lombok.Getter; -import lombok.Setter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.validation.annotation.Validated; - -/** - * The main properties a user can give to this service using a - * application.properties file. - * - * Depending on the configuration, further configuration classes might be - * loaded, - * to give the user mode operions. - * - * Example: If "pit.pidsystem.implementation" is "HANDLE_PROTOCOL" is set, - * `HandleProtocolProperties` will be active. - * - * @author Andreas Pfeil - */ -@Configuration -@Validated -public class ApplicationProperties extends GenericApplicationProperties { - private static final Logger LOG = LoggerFactory.getLogger(ApplicationProperties.class); - - /** - * Internal default set of types which indicate that, when used as a key - * of an attribute, that the value of the attribute must be a profile. - * Used for profile detection in records. - */ - private static final Set KNOWN_PROFILE_KEYS = Set.of( - "21.T11148/076759916209e5d62bd5", - "21.T11969/bcc54a2a9ab5bf2a8f2c" - ); - - public enum IdentifierSystemImpl { - IN_MEMORY, - LOCAL, - HANDLE_PROTOCOL; - } - - @Value("${pit.pidsystem.implementation}") - @NotNull - private IdentifierSystemImpl identifierSystemImplementation; - - public enum ValidationStrategy { - EMBEDDED_STRICT, - NONE_DEBUG; - } - - @Value("${pit.validation.strategy:embedded-strict}") - @NotNull - private ValidationStrategy validationStrategy = ValidationStrategy.EMBEDDED_STRICT; - - @Bean - public IValidationStrategy defaultValidationStrategy(ITypeRegistry typeRegistry) { - IValidationStrategy defaultStrategy = new NoValidationStrategy(); - if (this.validationStrategy == ValidationStrategy.EMBEDDED_STRICT) { - defaultStrategy = new EmbeddedStrictValidatorStrategy(typeRegistry, this); - } - return defaultStrategy; - } - - public enum StorageStrategy { - // Only store PIDs which have been created or modified using this instance - KEEP_MODIFIED, - // Store created, modified or resolved PIDs. - KEEP_RESOLVED_AND_MODIFIED; - - public boolean storesModified() { - return this == StorageStrategy.KEEP_MODIFIED - || this == StorageStrategy.KEEP_RESOLVED_AND_MODIFIED; - } - - public boolean storesResolved() { - return this == StorageStrategy.KEEP_RESOLVED_AND_MODIFIED; - } - } - - @Value("${pit.storage.strategy:keep-modified}") - @NotNull - private StorageStrategy storageStrategy = StorageStrategy.KEEP_MODIFIED; - - // TODO Used by DTR implementation for resolving. Too unflexible in mid-term. - @Value("${pit.pidsystem.handle.baseURI}") - private URL handleBaseUri; - - @Value("${pit.typeregistry.baseURI}") - private URL typeRegistryUri; - - @Value("${pit.typeregistry.cache.maxEntries:1000}") - private int cacheMaxEntries; - - @Value("${pit.typeregistry.cache.lifetimeMinutes:10}") - private long cacheExpireAfterWriteLifetime; - - @Value("${pit.validation.profileKey:21.T11148/076759916209e5d62bd5}") - @Deprecated(forRemoval = true /*In Typed PID Maker 3.0.0*/) - private String profileKey; - - @Value("#{${pit.validation.profileKeys:{}}}") - @NotNull - protected List profileKeys = List.of(); - - @Getter - @Setter - @Value("${pit.validation.alwaysAllowAdditionalAttributes:true}") - private boolean validationAlwaysAllowAdditionalAttributes = true; - - public @NotNull Set getProfileKeys() { - Set allProfileKeys = new java.util.HashSet<>(Set.copyOf(KNOWN_PROFILE_KEYS)); - allProfileKeys.addAll(profileKeys); - allProfileKeys.add(this.getProfileKey()); - return allProfileKeys; - } - - public void setProfileKeys(@NotNull List profileKeys) { - this.profileKeys = profileKeys; - } - - public IdentifierSystemImpl getIdentifierSystemImplementation() { - return this.identifierSystemImplementation; - } - - public void setIdentifierSystemImplementation(IdentifierSystemImpl identifierSystemImplementation) { - this.identifierSystemImplementation = identifierSystemImplementation; - } - - public URL getHandleBaseUri() { - return this.handleBaseUri; - } - - public void setHandleBaseUri(URL handleBaseUri) { - this.handleBaseUri = handleBaseUri; - } - - public URL getTypeRegistryUri() { - return this.typeRegistryUri; - } - - public void setTypeRegistryUri(URL typeRegistryUri) { - this.typeRegistryUri = typeRegistryUri; - } - - @Deprecated(forRemoval = true) - public String getProfileKey() { - return this.profileKey; - } - - @Deprecated(forRemoval = true) - public void setProfileKey(String profileKey) { - this.profileKey = profileKey; - } - - public ValidationStrategy getValidationStrategy() { - return this.validationStrategy; - } - - public void setValidationStrategy(ValidationStrategy strategy) { - this.validationStrategy = strategy; - } - - public int getCacheMaxEntries() { - if (this.cacheMaxEntries <= 10) { - LOG.warn("Cache max entries is set to {} (low value)", this.cacheMaxEntries); - } - return this.cacheMaxEntries; - } - - public void setCacheMaxEntries(int cacheMaxEntries) { - this.cacheMaxEntries = cacheMaxEntries; - } - - public long getCacheExpireAfterWriteLifetime() { - return cacheExpireAfterWriteLifetime; - } - - public void setCacheExpireAfterWriteLifetime(long cacheExpireAfterWriteLifetime) { - this.cacheExpireAfterWriteLifetime = cacheExpireAfterWriteLifetime; - } - - public StorageStrategy getStorageStrategy() { - return storageStrategy; - } - - public void setStorageStrategy(StorageStrategy storageStrategy) { - this.storageStrategy = storageStrategy; - } -} +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.kit.datamanager.pit.configuration; + +import edu.kit.datamanager.configuration.GenericApplicationProperties; +import edu.kit.datamanager.pit.pitservice.IValidationStrategy; +import edu.kit.datamanager.pit.pitservice.impl.EmbeddedStrictValidatorStrategy; +import edu.kit.datamanager.pit.pitservice.impl.NoValidationStrategy; +import edu.kit.datamanager.pit.typeregistry.ITypeRegistry; +import jakarta.validation.constraints.NotNull; +import lombok.Getter; +import lombok.Setter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.validation.annotation.Validated; + +import java.net.URL; +import java.util.List; +import java.util.Set; + +/** + * The main properties a user can give to this service using a + * application.properties file. + *

+ * Depending on the configuration, further configuration classes might be + * loaded, + * to give the user mode operions. + *

+ * Example: If "pit.pidsystem.implementation" is "HANDLE_PROTOCOL" is set, + * `HandleProtocolProperties` will be active. + * + * @author Andreas Pfeil + */ +@Configuration +@Validated +public class ApplicationProperties extends GenericApplicationProperties { + private static final Logger LOG = LoggerFactory.getLogger(ApplicationProperties.class); + + /** + * Internal default set of types which indicate that, when used as a key + * of an attribute, that the value of the attribute must be a profile. + * Used for profile detection in records. + */ + private static final Set KNOWN_PROFILE_KEYS = Set.of( + "21.T11148/076759916209e5d62bd5", + "21.T11969/bcc54a2a9ab5bf2a8f2c" + ); + @Value("#{${pit.validation.profileKeys:{}}}") + @NotNull + protected List profileKeys = List.of(); + @Value("${pit.pidsystem.implementation}") + @NotNull + private IdentifierSystemImpl identifierSystemImplementation; + @Value("${pit.validation.strategy:embedded-strict}") + @NotNull + private ValidationStrategy validationStrategy = ValidationStrategy.EMBEDDED_STRICT; + @Value("${pit.storage.strategy:keep-modified}") + @NotNull + private StorageStrategy storageStrategy = StorageStrategy.KEEP_MODIFIED; + // TODO Used by DTR implementation for resolving. Too unflexible in mid-term. + @Value("${pit.pidsystem.handle.baseURI}") + private URL handleBaseUri; + @Value("${pit.typeregistry.baseURI}") + private URL typeRegistryUri; + @Value("${pit.typeregistry.cache.maxEntries:1000}") + private int cacheMaxEntries; + @Value("${pit.typeregistry.cache.lifetimeMinutes:10}") + private long cacheExpireAfterWriteLifetime; + @Value("${pit.validation.profileKey:21.T11148/076759916209e5d62bd5}") + @Deprecated(forRemoval = true /*In Typed PID Maker 3.0.0*/) + private String profileKey; + @Getter + @Setter + @Value("${pit.validation.alwaysAllowAdditionalAttributes:true}") + private boolean validationAlwaysAllowAdditionalAttributes = true; + @Getter + @Setter + @Value("${pit.observability.includePiiInTraces:false}") + private boolean includePiiInTraces = false; + + @Bean + public IValidationStrategy defaultValidationStrategy(ITypeRegistry typeRegistry) { + IValidationStrategy defaultStrategy = new NoValidationStrategy(); + if (this.validationStrategy == ValidationStrategy.EMBEDDED_STRICT) { + defaultStrategy = new EmbeddedStrictValidatorStrategy(typeRegistry, this); + } + return defaultStrategy; + } + + public @NotNull Set getProfileKeys() { + Set allProfileKeys = new java.util.HashSet<>(Set.copyOf(KNOWN_PROFILE_KEYS)); + allProfileKeys.addAll(profileKeys); + allProfileKeys.add(this.getProfileKey()); + return allProfileKeys; + } + + public void setProfileKeys(@NotNull List profileKeys) { + this.profileKeys = profileKeys; + } + + @Deprecated(forRemoval = true) + public String getProfileKey() { + return this.profileKey; + } + + @Deprecated(forRemoval = true) + public void setProfileKey(String profileKey) { + this.profileKey = profileKey; + } + + public IdentifierSystemImpl getIdentifierSystemImplementation() { + return this.identifierSystemImplementation; + } + + public void setIdentifierSystemImplementation(IdentifierSystemImpl identifierSystemImplementation) { + this.identifierSystemImplementation = identifierSystemImplementation; + } + + public URL getHandleBaseUri() { + return this.handleBaseUri; + } + + public void setHandleBaseUri(URL handleBaseUri) { + this.handleBaseUri = handleBaseUri; + } + + public URL getTypeRegistryUri() { + return this.typeRegistryUri; + } + + public void setTypeRegistryUri(URL typeRegistryUri) { + this.typeRegistryUri = typeRegistryUri; + } + + public ValidationStrategy getValidationStrategy() { + return this.validationStrategy; + } + + public void setValidationStrategy(ValidationStrategy strategy) { + this.validationStrategy = strategy; + } + + public int getCacheMaxEntries() { + if (this.cacheMaxEntries <= 10) { + LOG.warn("Cache max entries is set to {} (low value)", this.cacheMaxEntries); + } + return this.cacheMaxEntries; + } + + public void setCacheMaxEntries(int cacheMaxEntries) { + this.cacheMaxEntries = cacheMaxEntries; + } + + public long getCacheExpireAfterWriteLifetime() { + return cacheExpireAfterWriteLifetime; + } + + public void setCacheExpireAfterWriteLifetime(long cacheExpireAfterWriteLifetime) { + this.cacheExpireAfterWriteLifetime = cacheExpireAfterWriteLifetime; + } + + public StorageStrategy getStorageStrategy() { + return storageStrategy; + } + + public void setStorageStrategy(StorageStrategy storageStrategy) { + this.storageStrategy = storageStrategy; + } + + public enum IdentifierSystemImpl { + IN_MEMORY, + LOCAL, + HANDLE_PROTOCOL + } + + public enum ValidationStrategy { + EMBEDDED_STRICT, + NONE_DEBUG + } + + public enum StorageStrategy { + // Only store PIDs which have been created or modified using this instance + KEEP_MODIFIED, + // Store created, modified or resolved PIDs. + KEEP_RESOLVED_AND_MODIFIED; + + public boolean storesModified() { + return this == StorageStrategy.KEEP_MODIFIED + || this == StorageStrategy.KEEP_RESOLVED_AND_MODIFIED; + } + + public boolean storesResolved() { + return this == StorageStrategy.KEEP_RESOLVED_AND_MODIFIED; + } + } +} diff --git a/src/main/java/edu/kit/datamanager/pit/configuration/PIISpanAttribute.java b/src/main/java/edu/kit/datamanager/pit/configuration/PIISpanAttribute.java new file mode 100644 index 00000000..9a09b97e --- /dev/null +++ b/src/main/java/edu/kit/datamanager/pit/configuration/PIISpanAttribute.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.kit.datamanager.pit.configuration; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation to mark method parameters that contain PII (Personally Identifiable Information) + * for conditional inclusion in OpenTelemetry spans. + *

+ * This annotation works like @SpanAttribute but only processes the parameter + * when pit.observability.includePiiInTraces=true is configured. + *

+ * Usage: + * 

+ * public void myMethod(@PIISpanAttribute("pid") String pidValue,
+ *                      @PIISpanAttribute PIDRecord record) {
+ *     // method implementation
+ * }
+ *
+ */ +@Target(ElementType.PARAMETER) +@Retention(RetentionPolicy.RUNTIME) +public @interface PIISpanAttribute { + + /** + * The name of the span attribute. If not provided, the parameter name will be used. + * + * @return the span attribute name + */ + String value() default ""; +} diff --git a/src/main/java/edu/kit/datamanager/pit/configuration/PIISpanAttributeAspect.java b/src/main/java/edu/kit/datamanager/pit/configuration/PIISpanAttributeAspect.java new file mode 100644 index 00000000..7845d379 --- /dev/null +++ b/src/main/java/edu/kit/datamanager/pit/configuration/PIISpanAttributeAspect.java @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.kit.datamanager.pit.configuration; + +import io.opentelemetry.api.trace.Span; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.reflect.MethodSignature; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.core.DefaultParameterNameDiscoverer; +import org.springframework.core.ParameterNameDiscoverer; +import org.springframework.stereotype.Component; + +import java.lang.reflect.Method; +import java.lang.reflect.Parameter; + +/** + * Aspect-Oriented Programming (AOP) aspect that automatically intercepts method calls + * within the PIT service to extract and add Personally Identifiable Information (PII) + * data to OpenTelemetry distributed tracing spans. + * + *

This aspect provides enhanced observability for development and debugging purposes + * by capturing sensitive parameter values that are explicitly marked with the + * {@link PIISpanAttribute} annotation.

+ * + *

Key Features:

+ *
    + *
  • Conditional Activation: Only created when the configuration property + * {@code pit.observability.includePiiInTraces=true} is set
    • + *
  • Broad Interception: Intercepts ALL methods in the + * {@code edu.kit.datamanager.pit} package tree
    • + *
  • Selective Processing: Only processes methods that have parameters + * annotated with {@link PIISpanAttribute}
    • + *
  • OpenTelemetry Integration: Seamlessly integrates with the existing + * tracing infrastructure
    • + *
+ * + *

Security and Privacy Considerations:

+ *

⚠️ CRITICAL SECURITY WARNING: This aspect captures and exports + * potentially sensitive PII data to tracing systems. This functionality should + * NEVER be enabled in production environments and should be used + * with extreme caution in development environments.

+ * + *
    + *
  • PII data may include user IDs, email addresses, personal identifiers, etc.
    • + *
  • Traced data may be stored in external observability platforms
    • + *
  • Ensure compliance with privacy regulations (GDPR, CCPA, etc.)
    • + *
  • Consider data retention policies and access controls
    • + *
+ * + *

Performance Considerations:

+ *
    + *
  • Intercepts ALL method calls in the pit package (performance overhead)
    • + *
  • Uses reflection to inspect method parameters (additional CPU cost)
    • + *
  • Should be disabled in performance-critical production environments
    • + *
+ * + *

Usage Example:

+ * 
+ * {@code
+ * @Service
+ * public class UserService {
+ *     public User findUser(@PIISpanAttribute("userId") String userId) {
+ *         // This method call will be intercepted and userId will be added to the span
+ *         return userRepository.findById(userId);
+ *     }
+ * }
+ * }
+ *
+ * + * @see PIISpanAttribute + * @see io.opentelemetry.api.trace.Span + */ +@Aspect +@Component +@ConditionalOnProperty(name = "pit.observability.includePiiInTraces", havingValue = "true") +public class PIISpanAttributeAspect { + + private static final Logger LOG = LoggerFactory.getLogger(PIISpanAttributeAspect.class); + + /** + * Spring's parameter name discoverer used to retrieve parameter names from method signatures. + * This is essential when the {@link PIISpanAttribute} annotation doesn't specify a custom + * attribute name - we fall back to using the actual parameter name from the source code. + * + *

The DefaultParameterNameDiscoverer tries multiple strategies: + *

    + *
  • Uses debug information if available (compiled with -g flag)
    • + *
  • Falls back to ASM-based bytecode analysis
    • + *
  • Uses Java 8+ parameter names if compiled with -parameters flag
    • + *
+ */ + private final ParameterNameDiscoverer parameterNameDiscoverer = new DefaultParameterNameDiscoverer(); + + /** + * Constructor that logs the activation of PII tracing with appropriate warnings. + * + *

This constructor is only called when the Spring condition + * {@code pit.observability.includePiiInTraces=true} is met, ensuring that + * the aspect is only active when explicitly configured.

+ * + *

The constructor logs both informational and warning messages to ensure + * that the activation of PII tracing is clearly visible in application logs, + * helping to prevent accidental deployment to production environments.

+ */ + public PIISpanAttributeAspect() { + LOG.info("PIISpanAttributeAspect created - PII data will be included in traces"); + LOG.warn("WARNING: PII tracing is enabled! This should only be used in development environments."); + LOG.info("Aspect will intercept methods in package: edu.kit.datamanager.pit.*"); + LOG.info("Only methods with @PIISpanAttribute annotated parameters will have PII data extracted"); + } + + /** + * AspectJ around advice that intercepts ALL method calls within the PIT service + * package hierarchy to identify and process methods containing PII parameters. + * + *

This method uses a broad pointcut expression that matches every method execution + * in the {@code edu.kit.datamanager.pit} package and all its sub-packages. While this + * approach has performance implications, it ensures comprehensive coverage without + * requiring developers to explicitly mark classes or methods.

+ * + *

Execution Flow:

+ *
    + *
  1. Intercept method call before execution
    2. + *
  2. Perform quick scan of method parameters for {@link PIISpanAttribute} annotations
    3. + *
  3. If PII parameters found, extract and add them to the current OpenTelemetry span
    4. + *
  4. Proceed with original method execution
    5. + *
  5. Handle any errors gracefully without disrupting the original method
    6. + *
+ * + *

Performance Optimization:

+ *

To minimize performance impact, this method performs a quick preliminary check + * for PII annotations before proceeding with the more expensive span processing. + * Methods without PII parameters are processed with minimal overhead.

+ * + *

Error Handling:

+ *

Any exceptions during PII processing are caught and logged but do not interfere + * with the original method execution. This ensures that tracing issues don't break + * application functionality.

+ * + * @param joinPoint the AspectJ join point containing method signature and arguments + * @return the result of the original method execution + * @throws Throwable any exception thrown by the original method (PII processing exceptions are caught) + */ + @Around("execution(* edu.kit.datamanager.pit..*(..))") + public Object interceptAllMethods(ProceedingJoinPoint joinPoint) throws Throwable { + // Extract method information using AspectJ reflection capabilities + MethodSignature signature = (MethodSignature) joinPoint.getSignature(); + Method method = signature.getMethod(); + Parameter[] parameters = method.getParameters(); + + // Performance optimization: Quick scan for PII annotations before expensive processing + // This avoids the overhead of span processing for methods that don't have PII data + boolean hasPIIParams = false; + for (Parameter parameter : parameters) { + if (parameter.getAnnotation(PIISpanAttribute.class) != null) { + hasPIIParams = true; + break; // Early exit once we find the first PII parameter + } + } + + // Only process methods that actually have PII parameters + if (hasPIIParams) { + LOG.info("Found method with PII parameters: {}.{}", + method.getDeclaringClass().getSimpleName(), method.getName()); + + try { + LOG.info("Processing PII parameters for tracing"); + // Delegate to specialized method for span attribute processing + addPIIAttributesToCurrentSpan(joinPoint); + } catch (Exception e) { + // Critical: Ensure that PII processing errors don't break the application + // Log the error but continue with normal method execution + LOG.warn("Failed to add PII span attributes: {}", e.getMessage(), e); + } + } + + // Always proceed with the original method execution + // This is the core of the around advice - we must call proceed() to execute the original method + return joinPoint.proceed(); + } + + /** + * Core processing method that extracts PII data from method parameters and adds + * them as attributes to the current OpenTelemetry span. + * + *

This method performs the detailed work of: + *

    + *
  • Validating that a valid OpenTelemetry span context exists
    • + *
  • Iterating through method parameters to find PII annotations
    • + *
  • Extracting parameter values and converting them to string representations
    • + *
  • Adding the PII data as span attributes for observability
    • + *
+ * + *

OpenTelemetry Integration:

+ *

This method relies on OpenTelemetry's automatic span propagation through + * thread-local storage. The {@code Span.current()} call retrieves the active + * span from the current thread's context, which should have been created by + * OpenTelemetry's auto-instrumentation or manual span creation.

+ * + *

Parameter Name Resolution:

+ *

The method uses Spring's {@link ParameterNameDiscoverer} to resolve parameter + * names when the annotation doesn't specify a custom attribute name. This requires + * that the application be compiled with parameter name information (Java 8+ with + * -parameters flag or debug information with -g flag).

+ * + *

Data Safety:

+ *
    + *
  • Null parameter values are safely handled and logged
    • + *
  • Very long parameter values are truncated in log messages (but not in spans)
    • + *
  • All parameter values are converted to strings using {@code toString()}
    • + *
+ * + * @param joinPoint the AspectJ join point containing method signature and runtime arguments + */ + private void addPIIAttributesToCurrentSpan(ProceedingJoinPoint joinPoint) { + // Extract all necessary method information from the join point + MethodSignature signature = (MethodSignature) joinPoint.getSignature(); + Method method = signature.getMethod(); + Object[] args = joinPoint.getArgs(); // Runtime argument values + Parameter[] parameters = method.getParameters(); // Method parameter definitions + + // Attempt to discover parameter names for cases where annotation doesn't specify attribute name + // This uses reflection and bytecode analysis to retrieve the original parameter names + String[] parameterNames = parameterNameDiscoverer.getParameterNames(method); + + // Retrieve the current OpenTelemetry span from thread-local context + // This span should have been created by OpenTelemetry's auto-instrumentation + Span currentSpan = Span.current(); + if (currentSpan == null) { + LOG.warn("No current span available for PII attributes - OpenTelemetry may not be properly configured"); + return; + } + + // Validate that the span context is properly initialized and active + // An invalid span context indicates tracing infrastructure issues + if (!currentSpan.getSpanContext().isValid()) { + LOG.warn("Current span context is not valid - span may have been closed or not properly created"); + return; + } + + LOG.info("Current span: {}", currentSpan.getSpanContext().getSpanId()); + + // Process each parameter to look for PII annotations + int piiParamsProcessed = 0; + for (int i = 0; i < parameters.length && i < args.length; i++) { + Parameter parameter = parameters[i]; + PIISpanAttribute piiAnnotation = parameter.getAnnotation(PIISpanAttribute.class); + + if (piiAnnotation != null) { + if (args[i] != null) { + // Determine the span attribute name (from annotation or parameter name) + String attributeName = determineAttributeName(piiAnnotation, parameterNames, i); + + // Convert parameter value to string representation + // Note: This uses toString() which may not be suitable for all object types + String attributeValue = args[i].toString(); + + // Add the PII data to the OpenTelemetry span as a custom attribute + // This data will be included in distributed traces and exported to observability platforms + currentSpan.setAttribute(attributeName, attributeValue); + piiParamsProcessed++; + + // Log the addition with truncation for very long values to avoid log spam + LOG.debug("Successfully added PII span attribute: {} = {}", attributeName, + attributeValue.length() > 100 ? attributeValue.substring(0, 100) + "..." : attributeValue); + } else { + // Handle null parameter values gracefully - log but don't add to span + LOG.debug("PII parameter at index {} is null, skipping", i); + } + } + } + + // Summary logging to track PII processing activity + LOG.info("Total PII parameters processed: {} for method {}.{}", piiParamsProcessed, + method.getDeclaringClass().getSimpleName(), method.getName()); + } + + /** + * Determines the most appropriate span attribute name for a PII parameter using + * a fallback strategy that prioritizes explicit annotation values, then parameter + * names, and finally generates a generic name. + * + *

This method implements a three-tier naming strategy:

+ *
    + *
  1. Explicit Annotation Value: If the {@link PIISpanAttribute} + * annotation specifies a non-empty value, use it as the attribute name. + * This gives developers full control over span attribute naming.
    2. + *
  2. Parameter Name Discovery: If no explicit value is provided, + * attempt to use the actual parameter name from the method signature. + * This requires proper compilation settings to preserve parameter names.
    3. + *
  3. Generated Fallback: If parameter names are not available, + * generate a generic attribute name based on the parameter position.
    4. + *
+ * + *

Compilation Requirements for Parameter Names:

+ *

For the second tier to work properly, the application must be compiled with + * one of the following options:

+ *
    + *
  • Java 8+ with -parameters flag: Preserves parameter names in bytecode
    • + *
  • Debug information (-g flag): Includes variable names in debug info
    • + *
  • IDE default settings: Most IDEs enable parameter name preservation by default
    • + *
+ * + *

Attribute Naming Best Practices:

+ *
    + *
  • Use descriptive, consistent names for span attributes
    • + *
  • Consider namespace prefixes for application-specific attributes (e.g., "app.user.id")
    • + *
  • Avoid special characters that may cause issues in observability platforms
    • + *
  • Keep names reasonably short to minimize storage overhead
    • + *
+ * + * @param annotation the PII span attribute annotation that may contain an explicit name + * @param parameterNames array of parameter names discovered from the method signature, may be null + * @param parameterIndex zero-based index of the parameter in the method signature + * @return the determined span attribute name, never null or empty + */ + private String determineAttributeName(PIISpanAttribute annotation, String[] parameterNames, int parameterIndex) { + // First priority: Use explicit annotation value if provided + // This allows developers to specify meaningful, consistent attribute names + if (!annotation.value().isEmpty()) { + return annotation.value(); + } + + // Second priority: Use discovered parameter name if available + // This provides reasonable default names that match the source code + if (parameterNames != null && parameterIndex < parameterNames.length) { + return parameterNames[parameterIndex]; + } + + // Final fallback: Generate a generic but unique attribute name + // This ensures we always have a valid attribute name, even when parameter + // names are not available due to compilation settings + return "pii_arg" + parameterIndex; + } +} diff --git a/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/InMemoryIdentifierSystem.java b/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/InMemoryIdentifierSystem.java index 33fb9c41..6f2c154f 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/InMemoryIdentifierSystem.java +++ b/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/InMemoryIdentifierSystem.java @@ -18,13 +18,13 @@ import edu.kit.datamanager.pit.common.*; import edu.kit.datamanager.pit.configuration.ApplicationProperties; +import edu.kit.datamanager.pit.configuration.PIISpanAttribute; import edu.kit.datamanager.pit.domain.PIDRecord; import edu.kit.datamanager.pit.pidsystem.IIdentifierSystem; import io.micrometer.core.annotation.Counted; import io.micrometer.core.annotation.Timed; import io.micrometer.observation.annotation.Observed; import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.instrumentation.annotations.SpanAttribute; import io.opentelemetry.instrumentation.annotations.WithSpan; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,7 +67,7 @@ public Optional getPrefix() { @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "memory_system_is_pid_registered", description = "Time taken to check if PID is registered in memory system") @Counted(value = "memory_system_is_pid_registered_total", description = "Total number of PID registration checks") - public boolean isPidRegistered(@SpanAttribute String pid) throws ExternalServiceException { + public boolean isPidRegistered(@PIISpanAttribute String pid) throws ExternalServiceException { return this.records.containsKey(pid); } @@ -75,7 +75,7 @@ public boolean isPidRegistered(@SpanAttribute String pid) throws ExternalService @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "memory_system_query_pid", description = "Time taken to query PID from memory system") @Counted(value = "memory_system_query_pid_total", description = "Total number of PID queries") - public PIDRecord queryPid(@SpanAttribute String pid) throws PidNotFoundException, ExternalServiceException { + public PIDRecord queryPid(@PIISpanAttribute String pid) throws PidNotFoundException, ExternalServiceException { PIDRecord pidRecord = this.records.get(pid); if (pidRecord == null) { throw new PidNotFoundException(pid); @@ -87,7 +87,7 @@ public PIDRecord queryPid(@SpanAttribute String pid) throws PidNotFoundException @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "memory_system_register_pid", description = "Time taken to register PID in memory system") @Counted(value = "memory_system_register_pid_total", description = "Total number of PID registrations") - public String registerPidUnchecked(@SpanAttribute final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { + public String registerPidUnchecked(@PIISpanAttribute final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { this.records.put(pidRecord.getPid(), pidRecord); LOG.debug("Registered record with PID: {}", pidRecord.getPid()); return pidRecord.getPid(); @@ -97,7 +97,7 @@ public String registerPidUnchecked(@SpanAttribute final PIDRecord pidRecord) thr @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "memory_system_update_pid", description = "Time taken to update PID in memory system") @Counted(value = "memory_system_update_pid_total", description = "Total number of PID updates") - public boolean updatePid(@SpanAttribute PIDRecord record) throws PidNotFoundException, ExternalServiceException, RecordValidationException { + public boolean updatePid(@PIISpanAttribute PIDRecord record) throws PidNotFoundException, ExternalServiceException, RecordValidationException { if (this.records.containsKey(record.getPid())) { this.records.put(record.getPid(), record); return true; @@ -109,7 +109,7 @@ public boolean updatePid(@SpanAttribute PIDRecord record) throws PidNotFoundExce @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "memory_system_delete_pid", description = "Time taken to delete PID from memory system") @Counted(value = "memory_system_delete_pid_total", description = "Total number of PID deletion attempts") - public boolean deletePid(@SpanAttribute String pid) { + public boolean deletePid(@PIISpanAttribute String pid) { throw new UnsupportedOperationException("Deleting PIDs is against the P in PID."); } diff --git a/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/handle/HandleDiff.java b/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/handle/HandleDiff.java index 4d1a2e37..adbf7f16 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/handle/HandleDiff.java +++ b/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/handle/HandleDiff.java @@ -1,6 +1,24 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.pidsystem.impl.handle; import edu.kit.datamanager.pit.common.PidUpdateException; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.WithSpan; import net.handle.hdllib.HandleValue; import java.util.ArrayList; @@ -18,6 +36,7 @@ class HandleDiff { private final Collection toUpdate = new ArrayList<>(); private final Collection toRemove = new ArrayList<>(); + @WithSpan(kind = SpanKind.INTERNAL) HandleDiff( final Map recordOld, final Map recordNew @@ -69,14 +88,14 @@ class HandleDiff { } public HandleValue[] added() { - return this.toAdd.toArray(new HandleValue[] {}); + return this.toAdd.toArray(new HandleValue[]{}); } public HandleValue[] updated() { - return this.toUpdate.toArray(new HandleValue[] {}); + return this.toUpdate.toArray(new HandleValue[]{}); } public HandleValue[] removed() { - return this.toRemove.toArray(new HandleValue[] {}); + return this.toRemove.toArray(new HandleValue[]{}); } } \ No newline at end of file diff --git a/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/handle/HandleProtocolAdapter.java b/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/handle/HandleProtocolAdapter.java index f3f0fed8..895536a7 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/handle/HandleProtocolAdapter.java +++ b/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/handle/HandleProtocolAdapter.java @@ -19,6 +19,7 @@ import edu.kit.datamanager.pit.common.*; import edu.kit.datamanager.pit.configuration.HandleCredentials; import edu.kit.datamanager.pit.configuration.HandleProtocolProperties; +import edu.kit.datamanager.pit.configuration.PIISpanAttribute; import edu.kit.datamanager.pit.domain.PIDRecord; import edu.kit.datamanager.pit.pidsystem.IIdentifierSystem; import edu.kit.datamanager.pit.recordModifiers.RecordModifier; @@ -26,7 +27,6 @@ import io.micrometer.core.annotation.Timed; import io.micrometer.observation.annotation.Observed; import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.instrumentation.annotations.SpanAttribute; import io.opentelemetry.instrumentation.annotations.WithSpan; import jakarta.annotation.PostConstruct; import jakarta.validation.constraints.NotNull; @@ -128,7 +128,7 @@ public Optional getPrefix() { @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "handle_system_is_pid_registered", description = "Time taken to check if PID is registered in Handle system") @Counted(value = "handle_system_is_pid_registered_total", description = "Total number of PID registration checks") - public boolean isPidRegistered(@SpanAttribute final String pid) throws ExternalServiceException { + public boolean isPidRegistered(@PIISpanAttribute final String pid) throws ExternalServiceException { HandleValue[] recordProperties; try { recordProperties = this.client.resolveHandle(pid, null, null); @@ -146,7 +146,7 @@ public boolean isPidRegistered(@SpanAttribute final String pid) throws ExternalS @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "handle_system_query_pid", description = "Time taken to query PID from Handle system") @Counted(value = "handle_system_query_pid_total", description = "Total number of PID queries") - public PIDRecord queryPid(@SpanAttribute final String pid) throws PidNotFoundException, ExternalServiceException { + public PIDRecord queryPid(@PIISpanAttribute final String pid) throws PidNotFoundException, ExternalServiceException { Collection allValues = this.queryAllHandleValues(pid); if (allValues.isEmpty()) { return null; @@ -160,7 +160,7 @@ public PIDRecord queryPid(@SpanAttribute final String pid) throws PidNotFoundExc @NotNull @WithSpan(kind = SpanKind.INTERNAL) @Timed(value = "handle_system_query_all_values", description = "Time taken to query all handle values") - protected Collection queryAllHandleValues(@SpanAttribute final String pid) throws PidNotFoundException, ExternalServiceException { + protected Collection queryAllHandleValues(@PIISpanAttribute final String pid) throws PidNotFoundException, ExternalServiceException { try { HandleValue[] values = this.client.resolveHandle(pid, null, null); return Stream.of(values) @@ -178,7 +178,7 @@ protected Collection queryAllHandleValues(@SpanAttribute final Stri @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "handle_system_register_pid", description = "Time taken to register PID in Handle system") @Counted(value = "handle_system_register_pid_total", description = "Total number of PID registrations") - public String registerPidUnchecked(@SpanAttribute final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { + public String registerPidUnchecked(@PIISpanAttribute final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { // Add admin value for configured user only // TODO add options to add additional adminValues e.g. for user lists? ArrayList admin = new ArrayList<>(); @@ -208,7 +208,7 @@ public String registerPidUnchecked(@SpanAttribute final PIDRecord pidRecord) thr @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "handle_system_update_pid", description = "Time taken to update PID in Handle system") @Counted(value = "handle_system_update_pid_total", description = "Total number of PID updates") - public boolean updatePid(@SpanAttribute final PIDRecord pidRecord) throws PidNotFoundException, ExternalServiceException, RecordValidationException { + public boolean updatePid(@PIISpanAttribute final PIDRecord pidRecord) throws PidNotFoundException, ExternalServiceException, RecordValidationException { if (!this.isValidPID(pidRecord.getPid())) { return false; } @@ -277,7 +277,7 @@ public boolean updatePid(@SpanAttribute final PIDRecord pidRecord) throws PidNot @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "handle_system_delete_pid", description = "Time taken to delete PID from Handle system") @Counted(value = "handle_system_delete_pid_total", description = "Total number of PID deletions") - public boolean deletePid(@SpanAttribute final String pid) throws ExternalServiceException { + public boolean deletePid(@PIISpanAttribute final String pid) throws ExternalServiceException { try { this.client.deleteHandle(pid); } catch (HandleException e) { @@ -362,7 +362,7 @@ public Collection resolveAllPidsOfPrefix() throws ExternalServiceExcepti */ @WithSpan(kind = SpanKind.INTERNAL) @Timed(value = "handle_system_is_valid_pid", description = "Time taken to validate PID") - protected boolean isValidPID(@SpanAttribute final String pid) { + protected boolean isValidPID(@PIISpanAttribute final String pid) { boolean isAuthMode = this.props.getCredentials() != null; if (isAuthMode && !pid.startsWith(this.props.getCredentials().getHandleIdentifierPrefix())) { return false; diff --git a/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/local/LocalPidSystem.java b/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/local/LocalPidSystem.java index e5ab6437..de123f9c 100644 --- a/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/local/LocalPidSystem.java +++ b/src/main/java/edu/kit/datamanager/pit/pidsystem/impl/local/LocalPidSystem.java @@ -18,13 +18,13 @@ import edu.kit.datamanager.pit.common.*; import edu.kit.datamanager.pit.configuration.ApplicationProperties; +import edu.kit.datamanager.pit.configuration.PIISpanAttribute; import edu.kit.datamanager.pit.domain.PIDRecord; import edu.kit.datamanager.pit.pidsystem.IIdentifierSystem; import io.micrometer.core.annotation.Counted; import io.micrometer.core.annotation.Timed; import io.micrometer.observation.annotation.Observed; import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.instrumentation.annotations.SpanAttribute; import io.opentelemetry.instrumentation.annotations.WithSpan; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -98,7 +98,7 @@ public Optional getPrefix() { @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "local_pidsystem_is_pid_registered", description = "Time taken to check if PID is registered in local system") @Counted(value = "local_pidsystem_is_pid_registered_total", description = "Total number of PID registration checks") - public boolean isPidRegistered(@SpanAttribute String pid) throws ExternalServiceException { + public boolean isPidRegistered(@PIISpanAttribute String pid) throws ExternalServiceException { return this.db.existsById(pid); } @@ -106,7 +106,7 @@ public boolean isPidRegistered(@SpanAttribute String pid) throws ExternalService @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "local_pidsystem_query_pid", description = "Time taken to query PID from local system") @Counted(value = "local_pidsystem_query_pid_total", description = "Total number of PID queries") - public PIDRecord queryPid(@SpanAttribute String pid) throws PidNotFoundException, ExternalServiceException { + public PIDRecord queryPid(@PIISpanAttribute String pid) throws PidNotFoundException, ExternalServiceException { Optional dbo = this.db.findByPid(pid); return new PIDRecord(dbo.orElseThrow(() -> new PidNotFoundException(pid))); } @@ -115,7 +115,7 @@ public PIDRecord queryPid(@SpanAttribute String pid) throws PidNotFoundException @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "local_pidsystem_register_pid", description = "Time taken to register PID in local system") @Counted(value = "local_pidsystem_register_pid_total", description = "Total number of PID registrations") - public String registerPidUnchecked(@SpanAttribute final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { + public String registerPidUnchecked(@PIISpanAttribute final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { if (this.db.existsById(pidRecord.getPid())) { throw new PidAlreadyExistsException(pidRecord.getPid()); } @@ -128,7 +128,7 @@ public String registerPidUnchecked(@SpanAttribute final PIDRecord pidRecord) thr @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "local_pidsystem_update_pid", description = "Time taken to update PID in local system") @Counted(value = "local_pidsystem_update_pid_total", description = "Total number of PID updates") - public boolean updatePid(@SpanAttribute PIDRecord rec) throws PidNotFoundException, ExternalServiceException, RecordValidationException { + public boolean updatePid(@PIISpanAttribute PIDRecord rec) throws PidNotFoundException, ExternalServiceException, RecordValidationException { if (this.db.existsById(rec.getPid())) { this.db.save(new PidDatabaseObject(rec)); return true; @@ -140,7 +140,7 @@ public boolean updatePid(@SpanAttribute PIDRecord rec) throws PidNotFoundExcepti @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "local_pidsystem_delete_pid", description = "Time taken to delete PID from local system") @Counted(value = "local_pidsystem_delete_pid_total", description = "Total number of PID deletion attempts") - public boolean deletePid(@SpanAttribute String pid) { + public boolean deletePid(@PIISpanAttribute String pid) { throw new UnsupportedOperationException("Deleting PIDs is against the P in PID."); } diff --git a/src/main/java/edu/kit/datamanager/pit/pitservice/impl/EmbeddedStrictValidatorStrategy.java b/src/main/java/edu/kit/datamanager/pit/pitservice/impl/EmbeddedStrictValidatorStrategy.java index 3a8a35ae..ea412e37 100644 --- a/src/main/java/edu/kit/datamanager/pit/pitservice/impl/EmbeddedStrictValidatorStrategy.java +++ b/src/main/java/edu/kit/datamanager/pit/pitservice/impl/EmbeddedStrictValidatorStrategy.java @@ -20,6 +20,7 @@ import edu.kit.datamanager.pit.common.RecordValidationException; import edu.kit.datamanager.pit.common.TypeNotFoundException; import edu.kit.datamanager.pit.configuration.ApplicationProperties; +import edu.kit.datamanager.pit.configuration.PIISpanAttribute; import edu.kit.datamanager.pit.domain.PIDRecord; import edu.kit.datamanager.pit.pitservice.IValidationStrategy; import edu.kit.datamanager.pit.typeregistry.AttributeInfo; @@ -28,7 +29,6 @@ import io.micrometer.core.annotation.Timed; import io.micrometer.observation.annotation.Observed; import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.instrumentation.annotations.SpanAttribute; import io.opentelemetry.instrumentation.annotations.WithSpan; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,35 +65,11 @@ public EmbeddedStrictValidatorStrategy( this.alwaysAcceptAdditionalAttributes = config.isValidationAlwaysAllowAdditionalAttributes(); } - /** - * Checks Exceptions' causes for a RecordValidationExceptions, and throws them, if present. - *

- * Usually used to avoid exposing exceptions related to futures. - * - * @param e the exception to unwrap. - */ - @WithSpan(kind = SpanKind.INTERNAL) - private static void unpackAsyncExceptions(@SpanAttribute PIDRecord pidRecord, Throwable e) { - final int MAX_LEVEL = 10; - Throwable cause = e; - - for (int level = 0; level <= MAX_LEVEL && cause != null; level++) { - cause = cause.getCause(); - if (cause instanceof RecordValidationException rve) { - throw rve; - } else if (cause instanceof TypeNotFoundException tnf) { - throw new RecordValidationException( - pidRecord, - "Type not found: %s".formatted(tnf.getMessage())); - } - } - } - @Override @WithSpan(kind = SpanKind.INTERNAL) @Timed(value = "validation_embedded_strict", description = "Time taken for embedded strict validation") @Counted(value = "validation_embedded_strict_total", description = "Total number of embedded strict validations") - public void validate(@SpanAttribute PIDRecord pidRecord) + public void validate(@PIISpanAttribute PIDRecord pidRecord) throws RecordValidationException, ExternalServiceException { if (pidRecord.getPropertyIdentifiers().isEmpty()) { throw new RecordValidationException(pidRecord, "Record is empty!"); @@ -125,9 +101,7 @@ public void validate(@SpanAttribute PIDRecord pidRecord) CompletableFuture[] profileFutures = Arrays.stream(pidRecord.getPropertyValues(attributeInfo.pid())) .map(this.typeRegistry::queryAsProfile) .map(registeredProfileFuture -> registeredProfileFuture.thenAccept( - registeredProfile -> { - registeredProfile.validateAttributes(pidRecord, this.alwaysAcceptAdditionalAttributes); - })) + registeredProfile -> registeredProfile.validateAttributes(pidRecord, this.alwaysAcceptAdditionalAttributes))) .toArray(CompletableFuture[]::new); return CompletableFuture.allOf(profileFutures).thenApply(v -> attributeInfo); })) @@ -150,4 +124,28 @@ public void validate(@SpanAttribute PIDRecord pidRecord) String.format("Validation task was cancelled for %s. Please report.", pidRecord.getPid())); } } + + /** + * Checks Exceptions' causes for a RecordValidationExceptions, and throws them, if present. + *

+ * Usually used to avoid exposing exceptions related to futures. + * + * @param e the exception to unwrap. + */ + @WithSpan(kind = SpanKind.INTERNAL) + private static void unpackAsyncExceptions(@PIISpanAttribute PIDRecord pidRecord, Throwable e) { + final int MAX_LEVEL = 10; + Throwable cause = e; + + for (int level = 0; level <= MAX_LEVEL && cause != null; level++) { + cause = cause.getCause(); + if (cause instanceof RecordValidationException rve) { + throw rve; + } else if (cause instanceof TypeNotFoundException tnf) { + throw new RecordValidationException( + pidRecord, + "Type not found: %s".formatted(tnf.getMessage())); + } + } + } } diff --git a/src/main/java/edu/kit/datamanager/pit/pitservice/impl/TypingService.java b/src/main/java/edu/kit/datamanager/pit/pitservice/impl/TypingService.java index 22fcc845..294b2675 100644 --- a/src/main/java/edu/kit/datamanager/pit/pitservice/impl/TypingService.java +++ b/src/main/java/edu/kit/datamanager/pit/pitservice/impl/TypingService.java @@ -17,6 +17,8 @@ package edu.kit.datamanager.pit.pitservice.impl; import edu.kit.datamanager.pit.common.*; +import edu.kit.datamanager.pit.configuration.ApplicationProperties; +import edu.kit.datamanager.pit.configuration.PIISpanAttribute; import edu.kit.datamanager.pit.domain.Operations; import edu.kit.datamanager.pit.domain.PIDRecord; import edu.kit.datamanager.pit.pidsystem.IIdentifierSystem; @@ -32,7 +34,6 @@ import io.opentelemetry.instrumentation.annotations.WithSpan; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import java.util.Collection; import java.util.Optional; @@ -44,32 +45,28 @@ * through a type registry and an identifier system. * */ - - @Observed public class TypingService implements ITypingService { private static final Logger LOG = LoggerFactory.getLogger(TypingService.class); private static final String LOG_MSG_TYPING_SERVICE_MISCONFIGURED = "Typing service misconfigured."; - private static final String LOG_MSG_QUERY_TYPE = "Querying for type with identifier {}."; - protected final IIdentifierSystem identifierSystem; protected final ITypeRegistry typeRegistry; /** * A validation strategy. Will never be null. - * + *

* ApplicationProperties::defaultValidationStrategy there is always either a * default strategy or a noop strategy assigned. Therefore, autowiring will * always work. Assigning null is done to avoid warnings on constructor. */ - @Autowired - protected IValidationStrategy defaultStrategy = null; + protected IValidationStrategy defaultStrategy; - public TypingService(IIdentifierSystem identifierSystem, ITypeRegistry typeRegistry) { + public TypingService(IIdentifierSystem identifierSystem, ITypeRegistry typeRegistry, ApplicationProperties applicationProperties) { super(); this.identifierSystem = identifierSystem; this.typeRegistry = typeRegistry; + this.defaultStrategy = applicationProperties.defaultValidationStrategy(typeRegistry); } @Override @@ -77,59 +74,61 @@ public Optional getPrefix() { return this.identifierSystem.getPrefix(); } - @Override - public void setValidationStrategy(IValidationStrategy strategy) { - this.defaultStrategy = strategy; - } - - @Override - @WithSpan(kind = SpanKind.INTERNAL) - @Timed(value = "typing_service_validate", description = "Time taken to validate PID record") - @Counted(value = "typing_service_validate_total", description = "Total number of validations") - public void validate(@SpanAttribute PIDRecord pidRecord) - throws RecordValidationException, ExternalServiceException { - this.defaultStrategy.validate(pidRecord); - } - @Override @WithSpan(kind = SpanKind.INTERNAL) @Timed(value = "typing_service_is_pid_registered", description = "Time taken to check PID registration") @Counted(value = "typing_service_is_pid_registered_total", description = "Total number of PID registration checks") - public boolean isPidRegistered(@SpanAttribute String pid) throws ExternalServiceException { + public boolean isPidRegistered(@PIISpanAttribute String pid) throws ExternalServiceException { LOG.trace("Performing isIdentifierRegistered({}).", pid); return identifierSystem.isPidRegistered(pid); } + @Override + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typing_service_query_pid", description = "Time taken to query PID") + @Counted(value = "typing_service_query_pid_total", description = "Total number of PID queries") + public PIDRecord queryPid(@PIISpanAttribute String pid) throws PidNotFoundException, ExternalServiceException { + return queryPid(pid, false); + } + @Override @WithSpan(kind = SpanKind.INTERNAL) @Timed(value = "typing_service_register_pid", description = "Time taken to register PID") @Counted(value = "typing_service_register_pid_total", description = "Total number of PID registrations") - public String registerPidUnchecked(@SpanAttribute final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { + public String registerPidUnchecked(@PIISpanAttribute final PIDRecord pidRecord) throws PidAlreadyExistsException, ExternalServiceException { LOG.trace("Performing registerPID({}).", pidRecord); return identifierSystem.registerPidUnchecked(pidRecord); } + @Override + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "typing_service_update_pid", description = "Time taken to update PID record") + @Counted(value = "typing_service_update_pid_total", description = "Total number of PID updates") + public boolean updatePid(@PIISpanAttribute PIDRecord pidRecord) throws PidNotFoundException, ExternalServiceException, RecordValidationException { + return this.identifierSystem.updatePid(pidRecord); + } + @Override @WithSpan(kind = SpanKind.INTERNAL) @Timed(value = "typing_service_delete_pid", description = "Time taken to delete PID") @Counted(value = "typing_service_delete_pid_total", description = "Total number of PID deletions") - public boolean deletePid(@SpanAttribute String pid) throws ExternalServiceException { + public boolean deletePid(@PIISpanAttribute String pid) throws ExternalServiceException { LOG.trace("Performing deletePID({}).", pid); return identifierSystem.deletePid(pid); } @Override @WithSpan(kind = SpanKind.INTERNAL) - @Timed(value = "typing_service_query_pid", description = "Time taken to query PID") - @Counted(value = "typing_service_query_pid_total", description = "Total number of PID queries") - public PIDRecord queryPid(@SpanAttribute String pid) throws PidNotFoundException, ExternalServiceException { - return queryPid(pid, false); + @Timed(value = "typing_service_resolve_all_pids", description = "Time taken to resolve all PIDs of prefix") + @Counted(value = "typing_service_resolve_all_pids_total", description = "Total number of resolve all PIDs requests") + public Collection resolveAllPidsOfPrefix() throws ExternalServiceException, InvalidConfigException { + return this.identifierSystem.resolveAllPidsOfPrefix(); } @WithSpan(kind = SpanKind.INTERNAL) @Timed(value = "typing_service_query_pid_with_names", description = "Time taken to query PID with property names") @Counted(value = "typing_service_query_pid_with_names_total", description = "Total number of PID queries with names") - public PIDRecord queryPid(@SpanAttribute String pid, @SpanAttribute boolean includePropertyNames) + public PIDRecord queryPid(@PIISpanAttribute String pid, @SpanAttribute boolean includePropertyNames) throws PidNotFoundException, ExternalServiceException { LOG.trace("Performing queryAllProperties({}, {}).", pid, includePropertyNames); PIDRecord pidInfo = identifierSystem.queryPid(pid); @@ -142,7 +141,7 @@ public PIDRecord queryPid(@SpanAttribute String pid, @SpanAttribute boolean incl @WithSpan(kind = SpanKind.INTERNAL) @Timed(value = "typing_service_enrich_record", description = "Time taken to enrich PID record with property names") - private void enrichPIDInformationRecord(@SpanAttribute PIDRecord pidInfo) { + private void enrichPIDInformationRecord(@PIISpanAttribute PIDRecord pidInfo) { // enrich record by querying type registry for all property definitions // to get the property names for (String typeIdentifier : pidInfo.getPropertyIdentifiers()) { @@ -163,19 +162,17 @@ private void enrichPIDInformationRecord(@SpanAttribute PIDRecord pidInfo) { } @Override - @WithSpan(kind = SpanKind.INTERNAL) - @Timed(value = "typing_service_update_pid", description = "Time taken to update PID record") - @Counted(value = "typing_service_update_pid_total", description = "Total number of PID updates") - public boolean updatePid(@SpanAttribute PIDRecord pidRecord) throws PidNotFoundException, ExternalServiceException, RecordValidationException { - return this.identifierSystem.updatePid(pidRecord); + public void setValidationStrategy(IValidationStrategy strategy) { + this.defaultStrategy = strategy; } @Override @WithSpan(kind = SpanKind.INTERNAL) - @Timed(value = "typing_service_resolve_all_pids", description = "Time taken to resolve all PIDs of prefix") - @Counted(value = "typing_service_resolve_all_pids_total", description = "Total number of resolve all PIDs requests") - public Collection resolveAllPidsOfPrefix() throws ExternalServiceException, InvalidConfigException { - return this.identifierSystem.resolveAllPidsOfPrefix(); + @Timed(value = "typing_service_validate", description = "Time taken to validate PID record") + @Counted(value = "typing_service_validate_total", description = "Total number of validations") + public void validate(@PIISpanAttribute PIDRecord pidRecord) + throws RecordValidationException, ExternalServiceException { + this.defaultStrategy.validate(pidRecord); } @WithSpan(kind = SpanKind.INTERNAL) diff --git a/src/main/java/edu/kit/datamanager/pit/resolver/Resolver.java b/src/main/java/edu/kit/datamanager/pit/resolver/Resolver.java index 9bf86264..258ab578 100644 --- a/src/main/java/edu/kit/datamanager/pit/resolver/Resolver.java +++ b/src/main/java/edu/kit/datamanager/pit/resolver/Resolver.java @@ -18,6 +18,7 @@ import edu.kit.datamanager.pit.common.ExternalServiceException; import edu.kit.datamanager.pit.common.PidNotFoundException; +import edu.kit.datamanager.pit.configuration.PIISpanAttribute; import edu.kit.datamanager.pit.domain.PIDRecord; import edu.kit.datamanager.pit.pidsystem.impl.handle.HandleBehavior; import edu.kit.datamanager.pit.pitservice.ITypingService; @@ -25,7 +26,6 @@ import io.micrometer.core.annotation.Timed; import io.micrometer.observation.annotation.Observed; import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.instrumentation.annotations.SpanAttribute; import io.opentelemetry.instrumentation.annotations.WithSpan; import net.handle.api.HSAdapter; import net.handle.api.HSAdapterFactory; @@ -77,7 +77,7 @@ public Resolver(ITypingService identifierSystem) { @WithSpan(kind = SpanKind.CLIENT) @Timed(value = "resolver_resolve_pid", description = "Time taken to resolve PID from any system") @Counted(value = "resolver_resolve_pid_total", description = "Total number of PID resolutions") - public PIDRecord resolve(@SpanAttribute String pid) throws PidNotFoundException, ExternalServiceException { + public PIDRecord resolve(@PIISpanAttribute String pid) throws PidNotFoundException, ExternalServiceException { String prefix = Arrays.stream( pid.split("/", 2) ) diff --git a/src/main/java/edu/kit/datamanager/pit/typeregistry/AttributeInfo.java b/src/main/java/edu/kit/datamanager/pit/typeregistry/AttributeInfo.java index 2bfe77c1..0e89d560 100644 --- a/src/main/java/edu/kit/datamanager/pit/typeregistry/AttributeInfo.java +++ b/src/main/java/edu/kit/datamanager/pit/typeregistry/AttributeInfo.java @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.typeregistry; import com.fasterxml.jackson.core.JsonProcessingException; @@ -6,20 +22,24 @@ import com.networknt.schema.JsonSchema; import com.networknt.schema.ValidationMessage; import edu.kit.datamanager.pit.Application; +import edu.kit.datamanager.pit.configuration.PIISpanAttribute; import edu.kit.datamanager.pit.typeregistry.schema.SchemaInfo; +import io.micrometer.core.annotation.Counted; +import io.micrometer.core.annotation.Timed; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.WithSpan; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collection; -import java.util.Objects; import java.util.Set; /** - * @param pid the pid of this attribute - * @param name a human-readable name, defined in the DTR for this type. - * Note this is usually different from the name in a specific profile! - * @param typeName name of the schema type of this attribute in the DTR, - * e.g. "Profile", "InfoType", "Special-Info-Type", ... + * @param pid the pid of this attribute + * @param name a human-readable name, defined in the DTR for this type. + * Note this is usually different from the name in a specific profile! + * @param typeName name of the schema type of this attribute in the DTR, + * e.g. "Profile", "InfoType", "Special-Info-Type", ... * @param jsonSchema the json schema to validate a value of this attribute */ public record AttributeInfo( @@ -30,7 +50,10 @@ public record AttributeInfo( ) { private static final Logger log = LoggerFactory.getLogger(AttributeInfo.class); - public boolean validate(String value) { + @WithSpan(kind = SpanKind.INTERNAL) + @Counted(value = "validation_attribute", description = "Total number of attribute validations") + @Timed(value = "validation_attribute_time", description = "Time taken for attribute validation") + public boolean validate(@PIISpanAttribute String value) { return this.jsonSchema().stream() .filter(schemaInfo -> schemaInfo.error() == null) .filter(schemaInfo -> schemaInfo.schema() != null) diff --git a/src/main/java/edu/kit/datamanager/pit/typeregistry/RegisteredProfile.java b/src/main/java/edu/kit/datamanager/pit/typeregistry/RegisteredProfile.java index 661e1907..4419f8b1 100644 --- a/src/main/java/edu/kit/datamanager/pit/typeregistry/RegisteredProfile.java +++ b/src/main/java/edu/kit/datamanager/pit/typeregistry/RegisteredProfile.java @@ -1,8 +1,30 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.typeregistry; import edu.kit.datamanager.pit.common.RecordValidationException; +import edu.kit.datamanager.pit.configuration.PIISpanAttribute; import edu.kit.datamanager.pit.domain.ImmutableList; import edu.kit.datamanager.pit.domain.PIDRecord; +import io.micrometer.core.annotation.Timed; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.instrumentation.annotations.SpanAttribute; +import io.opentelemetry.instrumentation.annotations.WithSpan; import java.util.Objects; import java.util.Set; @@ -13,11 +35,17 @@ public record RegisteredProfile( boolean allowAdditionalAttributes, ImmutableList attributes ) { + @WithSpan(kind = SpanKind.INTERNAL) + @Timed(value = "validation_registered_profile", description = "Time taken for validation of a PID record against a registered profile") public void validateAttributes( - PIDRecord pidRecord, - boolean alwaysAllowAdditionalAttributes - ) throws RecordValidationException - { + @PIISpanAttribute PIDRecord pidRecord, + @SpanAttribute boolean alwaysAllowAdditionalAttributes + ) throws RecordValidationException { + Span.current() + .setAttribute("profile.pid", this.pid) + .setAttribute("profile.allowAdditionalAttributes", this.allowAdditionalAttributes) + .setAttribute("profile.alwaysAllowAdditionalAttributes", alwaysAllowAdditionalAttributes); + Set attributesNotDefinedInProfile = pidRecord.getPropertyIdentifiers().stream() .filter(recordKey -> attributes.items().stream().noneMatch( profileAttribute -> Objects.equals(profileAttribute.pid(), recordKey))) diff --git a/src/main/java/edu/kit/datamanager/pit/typeregistry/RegisteredProfileAttribute.java b/src/main/java/edu/kit/datamanager/pit/typeregistry/RegisteredProfileAttribute.java index 0469d296..117f02cc 100644 --- a/src/main/java/edu/kit/datamanager/pit/typeregistry/RegisteredProfileAttribute.java +++ b/src/main/java/edu/kit/datamanager/pit/typeregistry/RegisteredProfileAttribute.java @@ -1,18 +1,37 @@ +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package edu.kit.datamanager.pit.typeregistry; import edu.kit.datamanager.pit.domain.PIDRecord; +import io.micrometer.core.annotation.Counted; public record RegisteredProfileAttribute( String pid, boolean mandatory, boolean repeatable ) { + @Counted(value = "registered_profile_attribute_violations", description = "Count of violations of registered profile attributes") public boolean violatesMandatoryProperty(PIDRecord pidRecord) { boolean contains = pidRecord.getPropertyIdentifiers().contains(this.pid) && pidRecord.getPropertyValues(this.pid).length > 0; return this.mandatory && !contains; } + @Counted(value = "registered_profile_attribute_repeatable_violations", description = "Count of violations of repeatable properties in registered profile attributes") public boolean violatesRepeatableProperty(PIDRecord pidRecord) { boolean repeats = pidRecord.getPropertyValues(this.pid).length > 1; return !this.repeatable && repeats; diff --git a/src/main/java/edu/kit/datamanager/pit/web/ExtendedErrorAttributes.java b/src/main/java/edu/kit/datamanager/pit/web/ExtendedErrorAttributes.java index 17fa70b6..3213e2d2 100644 --- a/src/main/java/edu/kit/datamanager/pit/web/ExtendedErrorAttributes.java +++ b/src/main/java/edu/kit/datamanager/pit/web/ExtendedErrorAttributes.java @@ -1,31 +1,46 @@ -package edu.kit.datamanager.pit.web; +/* + * Copyright (c) 2025 Karlsruhe Institute of Technology. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -import java.util.Map; +package edu.kit.datamanager.pit.web; -import org.springframework.beans.factory.annotation.Autowired; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.kit.datamanager.pit.common.RecordValidationException; import org.springframework.boot.web.error.ErrorAttributeOptions; import org.springframework.boot.web.servlet.error.DefaultErrorAttributes; import org.springframework.stereotype.Component; import org.springframework.web.context.request.WebRequest; -import com.fasterxml.jackson.databind.ObjectMapper; - -import edu.kit.datamanager.pit.common.RecordValidationException; +import java.util.Map; @Component public class ExtendedErrorAttributes extends DefaultErrorAttributes { - @Autowired(required = true) - ObjectMapper objectMapperBean; + private final ObjectMapper objectMapperBean; + + public ExtendedErrorAttributes(ObjectMapper objectMapperBean) { + this.objectMapperBean = objectMapperBean; + } @Override public Map getErrorAttributes(WebRequest webRequest, ErrorAttributeOptions options) { - final Map errorAttributes = - super.getErrorAttributes(webRequest, options); + final Map errorAttributes = + super.getErrorAttributes(webRequest, options); final Throwable error = super.getError(webRequest); - if (error instanceof RecordValidationException) { - final RecordValidationException validationError = (RecordValidationException) error; + if (error instanceof RecordValidationException validationError) { try { errorAttributes.put("pid-record", objectMapperBean.writeValueAsString(validationError.getPidRecord())); } catch (Exception e) { diff --git a/src/main/java/edu/kit/datamanager/pit/web/impl/TypingRESTResourceImpl.java b/src/main/java/edu/kit/datamanager/pit/web/impl/TypingRESTResourceImpl.java index 10fbe34c..12a4c237 100644 --- a/src/main/java/edu/kit/datamanager/pit/web/impl/TypingRESTResourceImpl.java +++ b/src/main/java/edu/kit/datamanager/pit/web/impl/TypingRESTResourceImpl.java @@ -20,6 +20,7 @@ import edu.kit.datamanager.exceptions.CustomInternalServerError; import edu.kit.datamanager.pit.common.*; import edu.kit.datamanager.pit.configuration.ApplicationProperties; +import edu.kit.datamanager.pit.configuration.PIISpanAttribute; import edu.kit.datamanager.pit.configuration.PidGenerationProperties; import edu.kit.datamanager.pit.domain.PIDRecord; import edu.kit.datamanager.pit.elasticsearch.PidRecordElasticRepository; @@ -96,7 +97,7 @@ public TypingRESTResourceImpl(ITypingService typingService, Resolver resolver, A @Timed(value = "pit_create_pids", description = "Time taken to create multiple PID records") @Counted(value = "pit_create_pids_total", description = "Total number of create PIDs requests") public ResponseEntity createPIDs( - @SpanAttribute List rec, + @PIISpanAttribute List rec, @SpanAttribute boolean dryrun, WebRequest request, HttpServletResponse response, @@ -208,7 +209,7 @@ public ResponseEntity createPIDs( @Timed(value = "pit_create_pid", description = "Time taken to create a single PID record") @Counted(value = "pit_create_pid_total", description = "Total number of create PID requests") public ResponseEntity createPID( - @SpanAttribute PIDRecord pidRecord, + @PIISpanAttribute PIDRecord pidRecord, @SpanAttribute boolean dryrun, final WebRequest request, @@ -255,7 +256,7 @@ public ResponseEntity createPID( @Timed(value = "pit_update_pid", description = "Time taken to update a PID record") @Counted(value = "pit_update_pid_total", description = "Total number of update PID requests") public ResponseEntity updatePID( - @SpanAttribute PIDRecord pidRecord, + @PIISpanAttribute PIDRecord pidRecord, @SpanAttribute boolean dryrun, final WebRequest request, @@ -343,7 +344,7 @@ private void storeLocally(String pid, boolean update) { * @throws CustomInternalServerError if the requested URI cannot be obtained from the web request */ @WithSpan(kind = SpanKind.INTERNAL) - private String getContentPathFromRequest(@SpanAttribute String lastPathElement, WebRequest request) { + private String getContentPathFromRequest(String lastPathElement, WebRequest request) { String requestedUri = (String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE, RequestAttributes.SCOPE_REQUEST); if (requestedUri == null) { @@ -466,10 +467,10 @@ public Page findAllPage( @Timed(value = "pit_find_all_tabular", description = "Time taken to find all known PIDs in tabular format") @Counted(value = "pit_find_all_tabular_total", description = "Total number of find all PIDs tabular requests") public ResponseEntity> findAllForTabular( - @SpanAttribute Instant createdAfter, - @SpanAttribute Instant createdBefore, - @SpanAttribute Instant modifiedAfter, - @SpanAttribute Instant modifiedBefore, + Instant createdAfter, + Instant createdBefore, + Instant modifiedAfter, + Instant modifiedBefore, Pageable pageable, WebRequest request, HttpServletResponse response, @@ -501,7 +502,7 @@ private void saveToElastic(PIDRecord rec) { } @WithSpan(kind = SpanKind.INTERNAL) - private String quotedEtag(@SpanAttribute PIDRecord pidRecord) { + private String quotedEtag(@PIISpanAttribute PIDRecord pidRecord) { return String.format("\"%s\"", pidRecord.getEtag()); } @@ -516,7 +517,7 @@ private String quotedEtag(@SpanAttribute PIDRecord pidRecord) { */ @WithSpan(kind = SpanKind.INTERNAL) @Timed(value = "pit_generate_pid_mapping", description = "Time taken to generate PID mappings") - private Map generatePIDMapping(@SpanAttribute List rec, @SpanAttribute boolean dryrun) throws RecordValidationException, ExternalServiceException { + private Map generatePIDMapping(@PIISpanAttribute List rec, @SpanAttribute boolean dryrun) throws RecordValidationException, ExternalServiceException { Map pidMappings = new HashMap<>(); for (PIDRecord pidRecord : rec) { String internalPID = pidRecord.getPid(); // the internal PID is the one given by the user @@ -551,7 +552,7 @@ private Map generatePIDMapping(@SpanAttribute List re */ @WithSpan(kind = SpanKind.INTERNAL) @Timed(value = "pit_apply_mappings_and_validate", description = "Time taken to apply mappings and validate records") - private List applyMappingsToRecordsAndValidate(@SpanAttribute List rec, Map pidMappings, @SpanAttribute String prefix) throws RecordValidationException, ExternalServiceException { + private List applyMappingsToRecordsAndValidate(@PIISpanAttribute List rec, Map pidMappings, String prefix) throws RecordValidationException, ExternalServiceException { List validatedRecords = new ArrayList<>(); for (PIDRecord pidRecord : rec) {