Preparing for release 5.3.13

.gitlab-ci.yml

@@ -4,40 +4,85 @@ services:
 
 variables:
   DEPLOY_PATH: ~/gitlab
+  DOCKER_DRIVER: overlay2
+  DOCKER_TLS_CERTDIR: ""
+  DOCKER_TAG: $CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+  DOCKER_IMAGE_NAME: $CI_REGISTRY_IMAGE:$DOCKER_TAG
+  DOCKER_PULL_SECRET: docker-registry-secret
+  APP_NAME: $CI_PROJECT_NAME
+
 
 before_script:
   - echo $CI_BUILD_REF
   - echo $CI_PROJECT_DIR
   - apk update && apk add git
 
 stages:
+  - build
   - package
   - config
   - deploy
 
-maven-package:
+maven-package-webapps-core:
   image: ${CI_REGISTRY_IMAGE}/eclipse-temurin:17-jdk
-  stage: package
+  stage: build
   script:
-    - './mvnw -q deploy -P embl-ebi -s ci_settings.xml -DskipTests -Dmaven.source.skip=true'
-    - mkdir deployment
-    - cp webapps/core/target/webapps-core-*.war deployment/webapps-core.war
-    - cp webapps/core-v2/target/webapps-core-v2*.jar deployment/webapps-core-v2.jar
-    - cp agents/solr/target/agents-solr-*.jar deployment/agents-solr.jar
-    - cp agents/uploadworkers/target/agents-uploadworkers-*.jar deployment/agents-uploadworkers.jar
-    - cp pipelines/curation/target/pipelines-curation-*.jar deployment/pipelines-curation.jar
-    # - cp pipelines/ena/target/pipelines-ena-*.jar deployment/pipelines-ena.jar
-    # - cp pipelines/ncbi-ena-link/target/pipelines-ncbi-ena-link-*.jar deployment/pipelines-ncbi-ena-link.jar
-    - cp pipelines/sample-release/target/pipelines-sample-release-*.jar deployment/pipelines-sample-release.jar
-    - cp pipelines/sample-post-release-action/target/pipelines-sample-post-release-action*.jar deployment/pipelines-sample-post-release-action.jar
-    - cp pipelines/ncbi/target/pipelines-ncbi-*.jar deployment/pipelines-ncbi.jar
-    - cp pipelines/reindex/target/pipelines-reindex-*.jar deployment/pipelines-reindex.jar
-    - cp pipelines/sample-transformation-dtol/target/pipelines-sample-transformation-dtol-*.jar deployment/pipelines-sample-transformation-dtol.jar
+    - './mvnw -q deploy -pl webapps/core -am -P embl-ebi -s ci_settings.xml -DskipTests -Dmaven.source.skip=true'
   artifacts:
     paths:
-      - deployment
+      - webapps/core/target/webapps-core-*.war
+
+    # maven-package:
+    # image: ${CI_REGISTRY_IMAGE}/eclipse-temurin:17-jdk
+    # stage: build
+    # script:
+    #  - './mvnw -q deploy -P embl-ebi -s ci_settings.xml -DskipTests -Dmaven.source.skip=true'
+    #  - mkdir deployment
+    #  - cp webapps/core/target/webapps-core-*.war deployment/webapps-core.war
+    #  - cp webapps/core-v2/target/webapps-core-v2*.jar deployment/webapps-core-v2.jar
+    #  - cp agents/solr/target/agents-solr-*.jar deployment/agents-solr.jar
+    # - cp agents/uploadworkers/target/agents-uploadworkers-*.jar deployment/agents-uploadworkers.jar
+    #  - cp pipelines/curation/target/pipelines-curation-*.jar deployment/pipelines-curation.jar
+    # - cp pipelines/ena/target/pipelines-ena-*.jar deployment/pipelines-ena.jar
+    # - cp pipelines/ncbi-ena-link/target/pipelines-ncbi-ena-link-*.jar deployment/pipelines-ncbi-ena-link.jar
+  # - cp pipelines/sample-release/target/pipelines-sample-release-*.jar deployment/pipelines-sample-release.jar
+  # - cp pipelines/sample-post-release-action/target/pipelines-sample-post-release-action*.jar deployment/pipelines-sample-post-release-action.jar
+  # - cp pipelines/ncbi/target/pipelines-ncbi-*.jar deployment/pipelines-ncbi.jar
+  # - cp pipelines/reindex/target/pipelines-reindex-*.jar deployment/pipelines-reindex.jar
+  # - cp pipelines/sample-transformation-dtol/target/pipelines-sample-transformation-dtol-*.jar deployment/pipelines-sample-transformation-dtol.jar
+  # artifacts:
+  #  paths:
+  #    - deployment
 #  when: manual
 
+build_docker_image:
+  stage: package
+  image: docker:stable
+  services:
+    - docker:stable-dind
+  before_script:
+    - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
+  script:
+    - docker build --build-arg DOCKER_REGISTRY=${CI_REGISTRY_IMAGE} -t $DOCKER_IMAGE_NAME -f webapps/core/Dockerfile .
+    - docker push $DOCKER_IMAGE_NAME
+  after_script:
+    - docker logout ${CI_REGISTRY_IMAGE}
+
+deploy-dev-bsd-v1:
+  image: dtzar/helm-kubectl:3.11.0
+  stage: deploy
+  script:
+    - cd webapps/core
+    - sed -i "s|%DOCKER_IMAGE%|$DOCKER_IMAGE_NAME|g" core-deployment.yaml
+    - kubectl config set-cluster bsd-cluster --server="${K8_HL_SERVER}"
+    - kubectl config set clusters.bsd-cluster.certificate-authority-data ${K8_HL_CERTIFICATE_AUTHORITY_DATA}
+    - kubectl config set-credentials bsd-user --token="${K8_HL_CREDENTIALS}"
+    - kubectl config set-context bsd-context --cluster=bsd-cluster --user=bsd-user
+    - kubectl config use-context bsd-context
+    - kubectl apply -f core-deployment.yaml --namespace=biosamples-dev
+    - kubectl apply -f core-service.yaml --namespace=biosamples-dev
+  when: manual
+
 clone-config-dev:
   stage: config
   script:
@@ -127,7 +172,7 @@ deploy-production:
   parallel:
     matrix:
       - BSD_NODE_NAME: bsd_prod
-        BSD_HOST_NAME: [wp-p2m-40, wp-p2m-41, wp-p1m-40, wp-p1m-41]
+        BSD_HOST_NAME: [ wp-p2m-40, wp-p2m-41, wp-p1m-40, wp-p1m-41 ]
   environment:
     name: prod-${BSD_HOST_NAME}
     url: http://${BSD_HOST_NAME}:8081/biosamples/
@@ -144,7 +189,7 @@ deploy-production-solr:
   parallel:
     matrix:
       - BSD_NODE_NAME: bsd_prod
-        BSD_SOLR_HOST_NAME: [wp-p1m-42, wp-p2m-42]
+        BSD_SOLR_HOST_NAME: [ wp-p1m-42, wp-p2m-42 ]
   environment:
     name: prod-solr-${BSD_HOST_NAME}
     url: http://${BSD_SOLR_HOST_NAME}.ebi.ac.uk:8983/solr

RELEASE.adoc

@@ -3,6 +3,10 @@
 
 This pages contains links to release notes for BioSamples for version 4.0.0 and higher. This release represents a comprehensive overhaul and therefore previous release notes are no longer applicable.
 
+[[v5.3.13]]
+== v5.3.13 Release notes
+* Zenodo archive for the publication
+
 [[v5.3.12]]
 == v5.3.12 Release notes
 * EBI Survey banner

agents/pom.xml

@@ -9,7 +9,7 @@
 	<parent>
 		<groupId>uk.ac.ebi.biosamples</groupId>
 		<artifactId>biosamples</artifactId>
-		<version>5.3.12</version>
+		<version>5.3.13-SNAPSHOT</version>
 	</parent>
 
 	<modules>

agents/solr/pom.xml

@@ -7,37 +7,31 @@
 	<parent>
 		<groupId>uk.ac.ebi.biosamples</groupId>
 		<artifactId>biosamples</artifactId>
-		<version>5.3.12</version>
+		<version>5.3.13-SNAPSHOT</version>
 		<relativePath>../../</relativePath>
 	</parent>
 
 	<dependencies>
 		<dependency>
 			<groupId>uk.ac.ebi.biosamples</groupId>
 			<artifactId>properties</artifactId>
-			<version>5.3.12</version>
+			<version>5.3.13-SNAPSHOT</version>
 		</dependency>
 		<dependency>
 			<groupId>uk.ac.ebi.biosamples</groupId>
-			<artifactId>models-solr</artifactId>
-			<version>5.3.12</version>
-		</dependency>
-		<dependency>
-			<groupId>uk.ac.ebi.biosamples</groupId>
-			<artifactId>messaging</artifactId>
-			<version>5.3.12</version>
-		</dependency>
-		<dependency>
-			<groupId>uk.ac.ebi.biosamples</groupId>
-			<artifactId>utils-ols</artifactId>
-			<version>5.3.12</version>
+			<artifactId>core</artifactId>
+			<version>5.3.13-SNAPSHOT</version>
 		</dependency>
 
 		<dependency>
 			<groupId>org.springframework.hateoas</groupId>
 			<artifactId>spring-hateoas</artifactId>
 			<version>1.3.4</version>
 		</dependency>
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-web</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>com.github.ben-manes.caffeine</groupId>
 			<artifactId>caffeine</artifactId>

agents/solr/src/main/java/uk/ac/ebi/biosamples/solr/MessageHandlerSolr.java

@@ -19,10 +19,10 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 import org.springframework.stereotype.Service;
-import uk.ac.ebi.biosamples.MessageContent;
-import uk.ac.ebi.biosamples.Messaging;
-import uk.ac.ebi.biosamples.model.Attribute;
-import uk.ac.ebi.biosamples.model.Sample;
+import uk.ac.ebi.biosamples.core.model.Attribute;
+import uk.ac.ebi.biosamples.core.model.Sample;
+import uk.ac.ebi.biosamples.messaging.MessagingConstants;
+import uk.ac.ebi.biosamples.messaging.model.MessageContent;
 import uk.ac.ebi.biosamples.solr.model.SolrSample;
 import uk.ac.ebi.biosamples.solr.repo.SolrSampleRepository;
 import uk.ac.ebi.biosamples.solr.service.SampleToSolrSampleConverter;
@@ -51,14 +51,14 @@ public MessageHandlerSolr(
   }
 
   @RabbitListener(
-      queues = Messaging.INDEXING_QUEUE,
+      queues = MessagingConstants.INDEXING_QUEUE,
       containerFactory = "biosamplesAgentSolrContainerFactory")
   public void handleIndexing(final MessageContent messageContent) {
     handle(messageContent);
   }
 
   @RabbitListener(
-      queues = Messaging.REINDEXING_QUEUE,
+      queues = MessagingConstants.REINDEXING_QUEUE,
       containerFactory = "biosamplesAgentSolrContainerFactory")
   public void handleReindexing(final MessageContent messageContent) {
     handle(messageContent);

agents/solr/src/main/java/uk/ac/ebi/biosamples/solr/SolrRunner.java

@@ -17,8 +17,8 @@
 import org.springframework.boot.ApplicationRunner;
 import org.springframework.stereotype.Component;
 import uk.ac.ebi.biosamples.BioSamplesProperties;
-import uk.ac.ebi.biosamples.MessageUtils;
-import uk.ac.ebi.biosamples.Messaging;
+import uk.ac.ebi.biosamples.messaging.MessagingConstants;
+import uk.ac.ebi.biosamples.messaging.service.MessageUtils;
 
 @Component
 public class SolrRunner implements ApplicationRunner {
@@ -36,9 +36,9 @@ public void run(final ApplicationArguments args) throws Exception {
         || messageCount == null
         || messageCount > 0) {
       Thread.sleep(1000);
-      messageCount = messageUtils.getQueueCount(Messaging.INDEXING_QUEUE);
+      messageCount = messageUtils.getQueueCount(MessagingConstants.INDEXING_QUEUE);
 
-      log.trace("Messages remaining in " + Messaging.INDEXING_QUEUE + " " + messageCount);
+      log.trace("Messages remaining in " + MessagingConstants.INDEXING_QUEUE + " " + messageCount);
     }
   }
 }

agents/solr/src/test/java/uk/ac/ebi/biosamples/solr/MessageHandlerSolrTest.java

@@ -25,9 +25,9 @@
 import org.springframework.http.converter.HttpMessageConverter;
 import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
 import org.springframework.web.client.RestTemplate;
-import uk.ac.ebi.biosamples.model.Attribute;
-import uk.ac.ebi.biosamples.model.Sample;
-import uk.ac.ebi.biosamples.model.SampleStatus;
+import uk.ac.ebi.biosamples.core.model.Attribute;
+import uk.ac.ebi.biosamples.core.model.Sample;
+import uk.ac.ebi.biosamples.core.model.SampleStatus;
 
 public class MessageHandlerSolrTest {
 

agents/uploadworkers/pom.xml

@@ -1,22 +1,38 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-<modelVersion>4.0.0</modelVersion>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+		xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
+	<modelVersion>4.0.0</modelVersion>
 	<artifactId>agents-uploadworkers</artifactId>
 	<packaging>jar</packaging>
 
 	<parent>
 		<groupId>uk.ac.ebi.biosamples</groupId>
 		<artifactId>biosamples</artifactId>
-		<version>5.3.12</version>
+		<version>5.3.13-SNAPSHOT</version>
 		<relativePath>../../</relativePath>
 	</parent>
 
 	<dependencies>
+
+		<!-- Internal BioSamples Modules -->
 		<dependency>
 			<groupId>uk.ac.ebi.biosamples</groupId>
 			<artifactId>biosamples-spring-boot-starter</artifactId>
-			<version>5.3.12</version>
+			<version>5.3.13-SNAPSHOT</version>
 		</dependency>
+		<dependency>
+			<groupId>uk.ac.ebi.biosamples</groupId>
+			<artifactId>properties</artifactId>
+			<version>5.3.13-SNAPSHOT</version>
+		</dependency>
+		<dependency>
+			<groupId>uk.ac.ebi.biosamples</groupId>
+			<artifactId>core</artifactId>
+			<version>5.3.13-SNAPSHOT</version>
+		</dependency>
+
+		<!-- Spring Boot -->
 		<dependency>
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-starter-web</artifactId>
@@ -30,31 +46,14 @@
 			<artifactId>spring-hateoas</artifactId>
 			<version>1.3.4</version>
 		</dependency>
-		<dependency>
-			<groupId>uk.ac.ebi.biosamples</groupId>
-			<artifactId>properties</artifactId>
-			<version>5.3.12</version>
-		</dependency>
-		<dependency>
-			<groupId>uk.ac.ebi.biosamples</groupId>
-			<artifactId>utils-webapp</artifactId>
-			<version>5.3.12</version>
-		</dependency>
-		<dependency>
-			<groupId>uk.ac.ebi.biosamples</groupId>
-			<artifactId>messaging</artifactId>
-			<version>5.3.12</version>
-		</dependency>
-		<dependency>
-			<groupId>uk.ac.ebi.biosamples</groupId>
-			<artifactId>models-mongo</artifactId>
-			<version>5.3.12</version>
-		</dependency>
+
+		<!-- CSV Utility -->
 		<dependency>
 			<groupId>org.apache.commons</groupId>
 			<artifactId>commons-csv</artifactId>
 			<version>1.8</version>
 		</dependency>
+
 	</dependencies>
 
 	<build>
@@ -72,4 +71,5 @@
 			</plugin>
 		</plugins>
 	</build>
+
 </project>

agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/runner/FileUploadMessageQueueRunner.java

@@ -16,8 +16,8 @@
 import org.springframework.boot.ApplicationArguments;
 import org.springframework.boot.ApplicationRunner;
 import org.springframework.stereotype.Component;
-import uk.ac.ebi.biosamples.MessageUtils;
-import uk.ac.ebi.biosamples.Messaging;
+import uk.ac.ebi.biosamples.messaging.MessagingConstants;
+import uk.ac.ebi.biosamples.messaging.service.MessageUtils;
 
 @Component
 public class FileUploadMessageQueueRunner implements ApplicationRunner {
@@ -30,9 +30,9 @@ public void run(final ApplicationArguments args) {
     while (true) {
       log.trace(
           "Messages remaining in "
-              + Messaging.UPLOAD_QUEUE
+              + MessagingConstants.UPLOAD_QUEUE
               + " "
-              + messageUtils.getQueueCount(Messaging.UPLOAD_QUEUE));
+              + messageUtils.getQueueCount(MessagingConstants.UPLOAD_QUEUE));
     }
   }
 }

agents/uploadworkers/src/main/java/uk/ac/ebi/biosamples/submission/FileUploadSubmissionService.java

@@ -27,11 +27,11 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.stereotype.Service;
-import uk.ac.ebi.biosamples.Messaging;
 import uk.ac.ebi.biosamples.client.BioSamplesClient;
-import uk.ac.ebi.biosamples.exceptions.GlobalExceptions;
-import uk.ac.ebi.biosamples.model.Relationship;
-import uk.ac.ebi.biosamples.model.Sample;
+import uk.ac.ebi.biosamples.core.model.Relationship;
+import uk.ac.ebi.biosamples.core.model.Sample;
+import uk.ac.ebi.biosamples.exception.GlobalExceptions;
+import uk.ac.ebi.biosamples.messaging.MessagingConstants;
 import uk.ac.ebi.biosamples.model.SubmissionFile;
 import uk.ac.ebi.biosamples.mongo.model.MongoFileUpload;
 import uk.ac.ebi.biosamples.mongo.repository.MongoFileUploadRepository;
@@ -56,7 +56,7 @@ public class FileUploadSubmissionService {
   @Autowired MongoSampleRepository mongoSampleRepository;
 
   @RabbitListener(
-      queues = Messaging.UPLOAD_QUEUE,
+      queues = MessagingConstants.UPLOAD_QUEUE,
       containerFactory = "biosamplesFileUploadSubmissionContainerFactory")
   public void receiveMessageFromBioSamplesFileUploaderQueue(final String mongoFileId) {
     handleMessage(mongoFileId);