From f47d4d11ff5883e3444e29bd8e326882183055b6 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 21 Dec 2021 12:19:34 +0530 Subject: [PATCH 01/58] Added changes done in previous version. rel-3.6.0, rel-3.7.0, rel-3.9.0 --- .../templates/application.conf.j2 | 3 ++ .../definitions/content_definition.json | 4 ++- .../helm_charts/datapipeline_jobs/values.j2 | 2 +- .../src/main/config/auto-creator.properties | 4 +++ .../local.auto-creator.properties.properties | 6 ++++ .../main/config/publish-pipeline.properties | 5 +++ .../local.qrcode-image-generator.properties | 5 +++ .../config/qrcode-image-generator.properties | 5 +++ .../jobs/samza/util/CloudStorageUtil.java | 9 +++-- .../org/sunbird/learning/util/CloudStore.java | 11 ++++-- .../common/util/HttpDownloadUtility.java | 35 ++++++++++++++++++- .../mgr/impl/HTMLMimeTypeMgrImpl.java | 5 ++- .../operation/finalizer/ReviewFinalizer.java | 4 ++- platform-modules/pom.xml | 2 +- .../mgr/impl/FrameworkManagerImpl.java | 1 + platform-tools/spikes/content-tool/pom.xml | 2 +- .../content/tool/CloudStoreManager.java | 11 ++++-- 17 files changed, 100 insertions(+), 14 deletions(-) diff --git a/ansible/roles/learning-service/templates/application.conf.j2 b/ansible/roles/learning-service/templates/application.conf.j2 index e7466ac507..ea1ea930e5 100644 --- a/ansible/roles/learning-service/templates/application.conf.j2 +++ b/ansible/roles/learning-service/templates/application.conf.j2 @@ -250,3 +250,6 @@ content.tagging.property="subject,medium" # Search Service Config kp.search_service.base_url="{{ kp_search_service_base_url }}" + +collection.only_published_children=true +indexHtmlValidation.env=false diff --git a/docs/domain_model_v2/definitions/content_definition.json b/docs/domain_model_v2/definitions/content_definition.json index 8e0a97a6c8..c831063254 100644 --- a/docs/domain_model_v2/definitions/content_definition.json +++ b/docs/domain_model_v2/definitions/content_definition.json @@ -271,7 +271,9 @@ "audio/ogg", "audio/webm", "audio/x-wav", - "audio/wav" + "audio/wav", + "application/json", + "application/quiz" ], "defaultValue": "application/vnd.ekstep.ecml-archive", "renderingHints": "{ 'inputType': 'select', 'order': 9 }", diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 907b7213b4..e4ddabab92 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -357,7 +357,7 @@ search-indexer: dialcode.index.name = "dialcode" dailcodemetrics.index.name = "dialcodemetrics" restrict.metadata.objectTypes = [] - nested.fields = ["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "discussionForum", "provider", "osMetadata", "actions"] + nested.fields = ["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "discussionForum", "provider", "osMetadata", "actions", "taxonomyPaths_v2", "competencies_v3"] schema.definition_cache.expiry = {{ schema_definition_cache_expiry_in_sec }} restrict.objectTypes = ["EventSet", "EventSetImage", "Event", "EventImage", "Questionnaire", "Misconception", "FrameworkType", "Concept", "Misconception", "Language", "Reference", "Dimension", "Method", "Library", "Domain", "Api"] ignored.fields={{ search_indexer_ignored_fields | to_json }} diff --git a/platform-jobs/samza/auto-creator/src/main/config/auto-creator.properties b/platform-jobs/samza/auto-creator/src/main/config/auto-creator.properties index b3f049495b..e5666efbe1 100644 --- a/platform-jobs/samza/auto-creator/src/main/config/auto-creator.properties +++ b/platform-jobs/samza/auto-creator/src/main/config/auto-creator.properties @@ -85,5 +85,9 @@ aws_storage_key=__aws_access_key_id__ aws_storage_secret=__aws_secret_access_key__ aws_storage_container=__aws_storage_container__ +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ diff --git a/platform-jobs/samza/auto-creator/src/main/config/local.auto-creator.properties.properties b/platform-jobs/samza/auto-creator/src/main/config/local.auto-creator.properties.properties index 45783a38b0..9491fc6a2c 100644 --- a/platform-jobs/samza/auto-creator/src/main/config/local.auto-creator.properties.properties +++ b/platform-jobs/samza/auto-creator/src/main/config/local.auto-creator.properties.properties @@ -72,3 +72,9 @@ aws_storage_key=__aws_access_key_id__ aws_storage_secret=__aws_secret_access_key__ aws_storage_container=__aws_storage_container__ + +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ + diff --git a/platform-jobs/samza/publish-pipeline/src/main/config/publish-pipeline.properties b/platform-jobs/samza/publish-pipeline/src/main/config/publish-pipeline.properties index fc8c47972b..046e2e8a22 100644 --- a/platform-jobs/samza/publish-pipeline/src/main/config/publish-pipeline.properties +++ b/platform-jobs/samza/publish-pipeline/src/main/config/publish-pipeline.properties @@ -86,6 +86,11 @@ cloud_storage.media.folder=media cloud_storage.ecar.folder=ecar_files cloud_storage.upload.url.ttl=600 +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ + # Media download configuration content.media.base.url=__content_media_base_url__ diff --git a/platform-jobs/samza/qrcode-image-generator/src/main/config/local.qrcode-image-generator.properties b/platform-jobs/samza/qrcode-image-generator/src/main/config/local.qrcode-image-generator.properties index 55f84fae02..f025402c82 100644 --- a/platform-jobs/samza/qrcode-image-generator/src/main/config/local.qrcode-image-generator.properties +++ b/platform-jobs/samza/qrcode-image-generator/src/main/config/local.qrcode-image-generator.properties @@ -55,6 +55,11 @@ aws_storage_secret=__aws_secret_access_key__ aws_storage_container=__aws_storage_container__ cloud_upload_retry_count=3 +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ + # Cassandra connection details cassandra.lp.connection=localhost:9042 cassandra.lpa.connection=localhost:9042 diff --git a/platform-jobs/samza/qrcode-image-generator/src/main/config/qrcode-image-generator.properties b/platform-jobs/samza/qrcode-image-generator/src/main/config/qrcode-image-generator.properties index b5954fd766..b8b030df33 100644 --- a/platform-jobs/samza/qrcode-image-generator/src/main/config/qrcode-image-generator.properties +++ b/platform-jobs/samza/qrcode-image-generator/src/main/config/qrcode-image-generator.properties @@ -55,6 +55,11 @@ aws_storage_secret=__aws_secret_access_key__ aws_storage_container=__aws_storage_container__ cloud_upload_retry_count=__cloud_upload_retry_count__ +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ + # Cassandra connection details cassandra.lp.connection=__cassandra_lp_connection__ cassandra.lpa.connection=__cassandra_lpa_connection__ diff --git a/platform-jobs/samza/qrcode-image-generator/src/main/java/org/sunbird/jobs/samza/util/CloudStorageUtil.java b/platform-jobs/samza/qrcode-image-generator/src/main/java/org/sunbird/jobs/samza/util/CloudStorageUtil.java index 81493cae58..35abb3967f 100644 --- a/platform-jobs/samza/qrcode-image-generator/src/main/java/org/sunbird/jobs/samza/util/CloudStorageUtil.java +++ b/platform-jobs/samza/qrcode-image-generator/src/main/java/org/sunbird/jobs/samza/util/CloudStorageUtil.java @@ -26,11 +26,16 @@ public class CloudStorageUtil { if(StringUtils.equalsIgnoreCase(cloudStoreType, "azure")) { String storageKey = Platform.config.getString("azure_storage_key"); String storageSecret = Platform.config.getString("azure_storage_secret"); - storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret)); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.empty())); }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { String storageKey = Platform.config.getString("aws_storage_key"); String storageSecret = Platform.config.getString("aws_storage_secret"); - storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret)); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.empty())); + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + String storageKey = Platform.config.getString("cephs3_storage_key"); + String storageSecret = Platform.config.getString("cephs3_storage_secret"); + String endPoint = Platform.config.getString("cephs3_storage_endpoint"); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.apply(endPoint))); }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while initialising cloud storage"); } diff --git a/platform-modules/actors/src/main/java/org/sunbird/learning/util/CloudStore.java b/platform-modules/actors/src/main/java/org/sunbird/learning/util/CloudStore.java index 32643d61e8..942df9881a 100644 --- a/platform-modules/actors/src/main/java/org/sunbird/learning/util/CloudStore.java +++ b/platform-modules/actors/src/main/java/org/sunbird/learning/util/CloudStore.java @@ -28,11 +28,16 @@ public class CloudStore { if(StringUtils.equalsIgnoreCase(cloudStoreType, "azure")) { String storageKey = Platform.config.getString("azure_storage_key"); String storageSecret = Platform.config.getString("azure_storage_secret"); - storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret)); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.empty())); }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { String storageKey = Platform.config.getString("aws_storage_key"); String storageSecret = Platform.config.getString("aws_storage_secret"); - storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret)); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.empty())); + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + String storageKey = Platform.config.getString("cephs3_storage_key"); + String storageSecret = Platform.config.getString("cephs3_storage_secret"); + String endPoint = Platform.config.getString("cephs3_storage_endpoint"); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.apply(endPoint))); }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while initialising cloud storage"); } @@ -47,6 +52,8 @@ public static String getContainerName() { return Platform.config.getString("azure_storage_container"); }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { return S3PropertyReader.getProperty("aws_storage_container"); + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + return S3PropertyReader.getProperty("cephs3_storage_container"); }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while getting container name"); } diff --git a/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java b/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java index 6939f89057..bc29b0446b 100644 --- a/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java +++ b/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java @@ -24,7 +24,7 @@ public class HttpDownloadUtility { private static final int BUFFER_SIZE = 4096; - + /** * Downloads a file from a URL * @@ -34,6 +34,26 @@ public class HttpDownloadUtility { * path of the directory to save the file */ public static File downloadFile(String fileURL, String saveDir) { + try { + System.out.println("http utility called........."); + return download(fileURL, false); + } catch (Exception e) { + e.printStackTrace(); + TelemetryManager.error("Error! While Downloading File:"+ e.getMessage(), e); + } + TelemetryManager.warn("Something Went Wrong While Downloading the File '" + fileURL + "' returning 'null'. File url: "+ fileURL); + return null; + } + + /** + * Downloads a file from a URL + * + * @param fileURL + * HTTP URL of the file to be downloaded + * @param saveDir + * path of the directory to save the file + */ + public static File downloadFileOld(String fileURL, String saveDir) { HttpURLConnection httpConn = null; InputStream inputStream = null; FileOutputStream outputStream = null; @@ -158,5 +178,18 @@ public static String readFromUrl(String url) { TelemetryManager.log("Data read from url: " + sb.toString()); return sb.toString(); } + + public static File download(String artifactUrl, boolean extractFile) throws Exception { + if(StringUtils.isNotBlank(artifactUrl)){ + String localPath = "tmp/" + artifactUrl.trim() + File.separator; + String[] fileUrl = artifactUrl.split("/"); + String filename = fileUrl[fileUrl.length - 1]; + + File file = new File(localPath + filename); + FileUtils.copyURLToFile(new URL(artifactUrl), file); + return file; + } + return null; + } } \ No newline at end of file diff --git a/platform-modules/content-manager/src/main/java/org/sunbird/content/mimetype/mgr/impl/HTMLMimeTypeMgrImpl.java b/platform-modules/content-manager/src/main/java/org/sunbird/content/mimetype/mgr/impl/HTMLMimeTypeMgrImpl.java index f63ad30e7e..188e57bdae 100644 --- a/platform-modules/content-manager/src/main/java/org/sunbird/content/mimetype/mgr/impl/HTMLMimeTypeMgrImpl.java +++ b/platform-modules/content-manager/src/main/java/org/sunbird/content/mimetype/mgr/impl/HTMLMimeTypeMgrImpl.java @@ -5,6 +5,7 @@ import java.util.Map; import org.apache.commons.lang3.BooleanUtils; +import org.sunbird.common.Platform; import org.sunbird.common.dto.Response; import org.sunbird.common.exception.ResponseCode; import org.sunbird.content.common.ContentOperations; @@ -42,7 +43,9 @@ public class HTMLMimeTypeMgrImpl extends BaseMimeTypeManager implements IMimeTyp @Override public Response upload(String contentId, Node node, File uploadFile, boolean isAsync) { TelemetryManager.log("Calling Upload Content For Node ID: " + node.getIdentifier() + " Uploaded File :" + uploadFile); - if (hasGivenFile(uploadFile, "index.html")) { + Boolean indexHtmlValidation = (Platform.config.hasPath("indexHtmlValidation.env"))? Platform.config.getBoolean("indexHtmlValidation.env") : false; + Boolean flag = indexHtmlValidation ? hasGivenFile(uploadFile, "index.html") : true; + if (flag) { return uploadContentArtifact(contentId, node, uploadFile, false); } else { return ERROR(ContentErrorCodes.ERR_CONTENT_UPLOAD_FILE.name(), "Zip file doesn't have required files.", ResponseCode.CLIENT_ERROR); diff --git a/platform-modules/content-manager/src/main/java/org/sunbird/content/operation/finalizer/ReviewFinalizer.java b/platform-modules/content-manager/src/main/java/org/sunbird/content/operation/finalizer/ReviewFinalizer.java index 21111b918f..c2635bb900 100644 --- a/platform-modules/content-manager/src/main/java/org/sunbird/content/operation/finalizer/ReviewFinalizer.java +++ b/platform-modules/content-manager/src/main/java/org/sunbird/content/operation/finalizer/ReviewFinalizer.java @@ -270,7 +270,9 @@ private void validateResource(String collectionId){ notPublishedList.add(resource.getIdentifier()); } } - if(CollectionUtils.isNotEmpty(notPublishedList)){ + Boolean childPublishEnabled = Platform.config.hasPath("collection.only_published_children") ? + Platform.config.getBoolean("collection.only_published_children") : true; + if(!childPublishEnabled && CollectionUtils.isNotEmpty(notPublishedList)){ TelemetryManager.error("Collection: " + collectionId + " has unpublished resources:: " + notPublishedList); throw new ClientException("ERR_COLLECTION_WITH_UNPUBLISHED_RESOURCE", "Collection: " + collectionId + " has unpublished resources:: " + notPublishedList); } diff --git a/platform-modules/pom.xml b/platform-modules/pom.xml index 94cdc3d8ee..ad3a7e306b 100644 --- a/platform-modules/pom.xml +++ b/platform-modules/pom.xml @@ -19,7 +19,7 @@ 2.3.1 1.8 1.8 - 1.2.8 + 1.3.0-beta diff --git a/platform-modules/sunbird-framework/src/main/java/org/sunbird/framework/mgr/impl/FrameworkManagerImpl.java b/platform-modules/sunbird-framework/src/main/java/org/sunbird/framework/mgr/impl/FrameworkManagerImpl.java index 96caddf7cb..90bbb64430 100644 --- a/platform-modules/sunbird-framework/src/main/java/org/sunbird/framework/mgr/impl/FrameworkManagerImpl.java +++ b/platform-modules/sunbird-framework/src/main/java/org/sunbird/framework/mgr/impl/FrameworkManagerImpl.java @@ -201,6 +201,7 @@ public Response publishFramework(String frameworkId, String channelId) throws Ex } if (StringUtils.isNotBlank(frameworkId) && validateObject(frameworkId)) { generateFrameworkHierarchy(frameworkId); + FrameworkCache.delete(frameworkId); Response response = OK(); response.put(FrameworkEnum.publishStatus.name(), "Publish Operation for Framework Id '" + frameworkId + "' Started Successfully!"); diff --git a/platform-tools/spikes/content-tool/pom.xml b/platform-tools/spikes/content-tool/pom.xml index a485ae2693..22f0ceff9c 100644 --- a/platform-tools/spikes/content-tool/pom.xml +++ b/platform-tools/spikes/content-tool/pom.xml @@ -66,7 +66,7 @@ org.sunbird cloud-store-sdk - 1.2.5 + 1.3.0-beta diff --git a/platform-tools/spikes/content-tool/src/main/java/org/sunbird/content/tool/CloudStoreManager.java b/platform-tools/spikes/content-tool/src/main/java/org/sunbird/content/tool/CloudStoreManager.java index d4f0e23e60..34ac6cb41e 100644 --- a/platform-tools/spikes/content-tool/src/main/java/org/sunbird/content/tool/CloudStoreManager.java +++ b/platform-tools/spikes/content-tool/src/main/java/org/sunbird/content/tool/CloudStoreManager.java @@ -22,9 +22,10 @@ public class CloudStoreManager { protected String destStorageType = Platform.config.getString("destination.storage_type"); - protected BaseStorageService awsService = StorageServiceFactory.getStorageService(new StorageConfig("aws", Platform.config.getString("aws_storage_key"), Platform.config.getString("aws_storage_secret"))); - protected BaseStorageService azureService = StorageServiceFactory.getStorageService((new StorageConfig("azure", Platform.config.getString("azure_storage_key"), Platform.config.getString("azure_storage_secret")))); - private String cloudSrcBaseURL = Platform.config.getString("cloud.src.baseurl"); + protected BaseStorageService awsService = StorageServiceFactory.getStorageService(new StorageConfig("aws", Platform.config.getString("aws_storage_key"), Platform.config.getString("aws_storage_secret"), Option.empty())); + protected BaseStorageService azureService = StorageServiceFactory.getStorageService((new StorageConfig("azure", Platform.config.getString("azure_storage_key"), Platform.config.getString("azure_storage_secret"), Option.empty()))); + protected BaseStorageService cephService = StorageServiceFactory.getStorageService((new StorageConfig("cephs3", Platform.config.getString("cephs3_storage_key"), Platform.config.getString("cephs3_storage_secret"), Option.apply(Platform.config.getString("cephs3_storage_endpoint"))))); + private String cloudSrcBaseURL = Platform.config.getString("cloud.src.baseurl"); private String cloudDestBaseURL = Platform.config.getString("cloud.dest.baseurl"); @@ -239,6 +240,8 @@ public String getContainerName(String cloudStoreType) { return Platform.config.getString("azure_storage_container"); }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { return Platform.config.getString("aws_storage_container"); + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + return Platform.config.getString("cephs3_storage_container"); }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while getting container name"); } @@ -249,6 +252,8 @@ public BaseStorageService getcloudService(String cloudStoreType){ return azureService; }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { return awsService; + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + return cephService; }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while getting container name"); } From 0528c6cc1a9727d81a489d734f5582dae6e9c519 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 21 Dec 2021 12:33:48 +0530 Subject: [PATCH 02/58] Added missing import --- .../main/java/org/sunbird/common/util/HttpDownloadUtility.java | 1 + 1 file changed, 1 insertion(+) diff --git a/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java b/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java index bc29b0446b..1c75e1d204 100644 --- a/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java +++ b/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java @@ -11,6 +11,7 @@ import java.net.URL; import java.util.List; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.sunbird.common.Slug; import org.sunbird.telemetry.logger.TelemetryManager; From 47467d9d41faa70eebe44c4f770dc67b1e239e1b Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 21 Dec 2021 12:51:30 +0530 Subject: [PATCH 03/58] Updated mimeType definition --- .../domain_model_v2/definitions/content_image_definition.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/domain_model_v2/definitions/content_image_definition.json b/docs/domain_model_v2/definitions/content_image_definition.json index d4d6b09c65..4ab316951f 100644 --- a/docs/domain_model_v2/definitions/content_image_definition.json +++ b/docs/domain_model_v2/definitions/content_image_definition.json @@ -271,7 +271,9 @@ "audio/ogg", "audio/webm", "audio/x-wav", - "audio/wav" + "audio/wav", + "application/json", + "application/quiz" ], "defaultValue": "application/vnd.ekstep.ecml-archive", "renderingHints": "{ 'inputType': 'select', 'order': 9 }", From 3264987dc6b61dcba98b1f56db09f42f034784d0 Mon Sep 17 00:00:00 2001 From: Ankit Date: Mon, 10 Jan 2022 15:27:12 +0530 Subject: [PATCH 04/58] aws setup for cloudstore --- ansible/roles/learning-service/defaults/main.yml | 2 +- ansible/roles/lp-synctool-deploy/defaults/main.yml | 2 +- ansible/roles/samza-jobs/tasks/deploy.yml | 12 +++++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/ansible/roles/learning-service/defaults/main.yml b/ansible/roles/learning-service/defaults/main.yml index ced3460453..fbc7cb98e5 100644 --- a/ansible/roles/learning-service/defaults/main.yml +++ b/ansible/roles/learning-service/defaults/main.yml @@ -34,7 +34,7 @@ platform_api_url: "http://{{ groups['learning1'][0] }}:8080/learning-service" youtube_app_name: fetch-youtube-license -cloud_store: azure +cloud_store: aws azure_public_container: azure_account_name: diff --git a/ansible/roles/lp-synctool-deploy/defaults/main.yml b/ansible/roles/lp-synctool-deploy/defaults/main.yml index b9919c28d4..ded17cf57a 100644 --- a/ansible/roles/lp-synctool-deploy/defaults/main.yml +++ b/ansible/roles/lp-synctool-deploy/defaults/main.yml @@ -23,6 +23,6 @@ redis_port: 6379 graphevent_kafka_url: "{{groups['processing-cluster-zookeepers']|join(':9092,')}}:9092" search_lms_index_host: "{{ groups['core-es']|join(':9200,')}}:9200" -cloud_store: azure +cloud_store: aws azure_public_container: azure_account_name: \ No newline at end of file diff --git a/ansible/roles/samza-jobs/tasks/deploy.yml b/ansible/roles/samza-jobs/tasks/deploy.yml index 2b9598eb02..feb0b54876 100644 --- a/ansible/roles/samza-jobs/tasks/deploy.yml +++ b/ansible/roles/samza-jobs/tasks/deploy.yml @@ -71,7 +71,17 @@ replace: dest="{{item[1].stdout}}" regexp="{{item[0].key}}" replace="{{item[0].value}}" when: "{{item[1]|changed}}" with_nested: - - [{key: "__yarn_host__", value: "{{__yarn_host__}}"}, {key: "__yarn_port__", value: "{{__yarn_port__}}"}, {key: "__env__", value: "{{env}}" }, {key: "__env_name__", value: "{{env_name}}" }, {key: "__zookeepers__", value: "{{zookeepers}}"}, {key: "__kafka_brokers__", value: "{{kafka_brokers}}"}, {key: "__delayInMilliSeconds__", value: "{{delayInMilliSeconds}}" }, {key: "__retryTimeInMilliSeconds__", value: "{{retryTimeInMilliSeconds}}" }, {key: "__bypass_reverse_search__", value: "{{bypass_reverse_search}}" }, {key: "__retryBackoffBaseInSeconds__", value: "{{retry_backoff_base_in_seconds}}" }, {key: "__retryLimit__", value: "{{retry_limit}}" }, {key: "__retryLimitEnable__", value: "{{retry_limit_enable}}" }, {key: "__google_api_key__", value: "{{google_api_key}}" }, {key: "__searchServiceEndpoint__", value: "{{search_service_endpoint}}" }, {key: "__objectDenormalizationAdditionalConfig__", value: "{{object_denormalization_additional_config}}" },{key: "__audit_es_host__", value: "{{audit_es_host}}"}, {key: "__search_es_host__", value: "{{search_es_host}}"}, {key: "__redis_host__", value: "{{redis_host}}"}, {key: "__dp_redis_host__", value: "{{dp_redis_host}}"}, {key: "__redis_port__", value: "{{redis_port}}"}, {key: "__environment_id__", value: "{{environment_id}}"}, {key: "__graph_passport_key__", value: "{{graph_passport_key}}"}, {key: "__lp_bolt_url__", value: "{{lp_bolt_url}}"}, {key: "__lp_bolt_read_url__", value: "{{lp_bolt_read_url}}"}, {key: "__lp_bolt_write_url__", value: "{{lp_bolt_write_url}}"}, {key: "__other_bolt_url__", value: "{{other_bolt_url}}"}, {key: "__other_bolt_read_url__", value: "{{other_bolt_read_url}}"}, {key: "__other_bolt_write_url__", value: "{{other_bolt_write_url}}"}, {key: "__mw_shard_id__", value: "{{mw_shard_id}}"}, {key: "__lp_url__", value: "{{lp_url}}"}, {key: "__cloud_storage_config_environment__", value: "{{cloud_storage_config_environment}}"}, {key: "__google_vision_tagging__", value: "{{google_vision_tagging}}"}, {key: "__lp_tmpfile_location__", value: "{{lp_tmpfile_location}}"}, {key: "__esRouterAdditionalConfig__", value: "{{es_router_additional_config}}"},{key: "__esRouterSecondaryAdditionalConfig__", value: "{{es_router_additional_secondary_config}}"},{key: "__es_port__", value: "{{es_port}}"}, {key: "__keyspace_name__", value: "{{content_keyspace_name}}"}, {key: "__collection_fullecar_disable__", value: "{{collection_fullecar_disable}}"},{key: "__max_iteration_count_for_samza_job__", value: "{{max_iteration_count_for_samza_job}}"},{key: "__cloud_storage_type__", value: "{{cloud_store}}"},{key: "__azure_storage_key__", value: "{{sunbird_public_storage_account_name}}"},{key: "__azure_storage_secret__", value: "{{sunbird_public_storage_account_key}}"},{key: "__azure_storage_container__", value: "{{azure_public_container}}"},{key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__plugin_media_base_url__", value: "{{plugin_media_base_url}}"}, {key: "__installation_id__", value: "{{instance_name}}"}, {key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__hierarchy_keyspace_name__", value: "{{hierarchy_keyspace_name}}"}, {key: "__composite_search_indexer_container_count__", value: "{{composite_search_indexer_container_count}}"},{key: "__cassandra_lp_connection__", value: "{{lp_cassandra_connection}}"}, {key: "__cassandra_lpa_connection__", value: "{{dp_cassandra_connection}}"}, {key: "__streaming_mime_type__", value: "{{streaming_mime_type}}"}, {key: "__cassandra_sunbird_connection__", value: "{{core_cassandra_connection}}"}, {key: "__cloud_upload_retry_count__", value: "{{cloud_upload_retry_count}}"}, {key: "__compositesearch_index_name__", value: "{{compositesearch_index_name}}"},{key: "__publish_pipeline_container_count__", value: "{{publish_pipeline_container_count}}"},{key: "__yarn_container_memory_mb__", value: "{{publish_yarn_container_memory_mb}}"},{key: "__youtube_api_key__", value: "{{youtube_api_key}}"},{key: "__kp_learning_service_base_url__", value: "{{kp_learning_service_base_url}}"},{key: "__sunbird_installation__", value: "{{sunbird_platform_installation}}"}, {key: "__search_lms_es_host__", value: "{{search_lms_es_host}}"},{key: "__dial_image_storage_container__", value: "{{dial_image_storage_container}}"},{key: "__dial_base_url__", value: "{{dial_base_url}}"},{key: "__learner_service_base_url__", value: "{{learner_service_base_url}}"},{key: "__cert_service_base_url__", value: "{{cert_service_base_url}}"},{key: "__certificate_base_path__", value: "{{certificate_base_path}}"},{key: "__kp_content_service_base_url__", value: "{{kp_content_service_base_url}}"},{key: "__kp_print_service_base_url__", value: "{{kp_print_service_base_url}}"},{key: "__cert_reg_service_base_url__", value: "{{cert_reg_service_base_url}}"},{key: "__kp_search_service_base_url__", value: "{{kp_search_service_base_url}}"},{key: "__samza_coordinator_replication_factor__", value: "{{samza_coordinator_replication_factor}}"},{key: "__samza_checkpoint_replication_factor__", value: "{{samza_checkpoint_replication_factor}}"},{key: "__course_batch_updater_container_count__", value: "{{course_batch_updater_container_count}}"},{key: "__course_certificate_generator_container_count__", value: "{{course_certificate_generator_container_count}}"},{key: "__course_progress_batch_size__", value: "{{course_progress_batch_size}}"},{key: "__itemset_generate_pdf__", value: "{{itemset_generate_pdf}}"},{key: "__auto_creator_container_count__", value: "{{auto_creator_container_count}}"},{key: "__content_streaming_enabled__", value: "{{content_streaming_enabled}}"},{key: "__lms_service_base_url__", value: "{{lms_service_base_url}}"},{key: "__mvc_search_indexer_container_count__", value: "{{mvc_search_indexer_container_count}}"}, {key: "__search_es7_host__", value: "{{search_es7_host}}"} , {key: "__ml-keywordapi__", value: "{{mlworkbench}}"},{key: "__auto_creator_artifact_allowed_sources__", value: "{{auto_creator_artifact_allowed_sources}}"},{key: "__publish_pipeline_task_opts__", value: "{{publish_pipeline_task_opts}}"},{key: "__auto_creator_g_service_acct_cred__", value: "{{auto_creator_gservice_acct_cred}}"},{key: "__certificate_pre_processor_container_count__", value: "{{certificate_pre_processor_container_count}}"},{key: "__master_category_validation_enabled__", value: "{{master_category_validation_enabled}}"}] + - [{key: "__yarn_host__", value: "{{__yarn_host__}}"}, {key: "__yarn_port__", value: "{{__yarn_port__}}"}, {key: "__env__", value: "{{env}}" }, {key: "__env_name__", value: "{{env_name}}" }, {key: "__zookeepers__", value: "{{zookeepers}}"}, {key: "__kafka_brokers__", value: "{{kafka_brokers}}"}, {key: "__delayInMilliSeconds__", value: "{{delayInMilliSeconds}}" }, {key: "__retryTimeInMilliSeconds__", value: "{{retryTimeInMilliSeconds}}" }, {key: "__bypass_reverse_search__", value: "{{bypass_reverse_search}}" }, {key: "__retryBackoffBaseInSeconds__", value: "{{retry_backoff_base_in_seconds}}" }, {key: "__retryLimit__", value: "{{retry_limit}}" }, {key: "__retryLimitEnable__", value: "{{retry_limit_enable}}" }, {key: "__google_api_key__", value: "{{google_api_key}}" }, {key: "__searchServiceEndpoint__", value: "{{search_service_endpoint}}" }, {key: "__objectDenormalizationAdditionalConfig__", value: "{{object_denormalization_additional_config}}" },{key: "__audit_es_host__", value: "{{audit_es_host}}"}, {key: "__search_es_host__", value: "{{search_es_host}}"}, {key: "__redis_host__", value: "{{redis_host}}"}, {key: "__dp_redis_host__", value: "{{dp_redis_host}}"}, {key: "__redis_port__", value: "{{redis_port}}"}, {key: "__environment_id__", value: "{{environment_id}}"}, {key: "__graph_passport_key__", value: "{{graph_passport_key}}"}, {key: "__lp_bolt_url__", value: "{{lp_bolt_url}}"}, {key: "__lp_bolt_read_url__", value: "{{lp_bolt_read_url}}"}, {key: "__lp_bolt_write_url__", value: "{{lp_bolt_write_url}}"}, {key: "__other_bolt_url__", value: "{{other_bolt_url}}"}, {key: "__other_bolt_read_url__", value: "{{other_bolt_read_url}}"}, {key: "__other_bolt_write_url__", value: "{{other_bolt_write_url}}"}, {key: "__mw_shard_id__", value: "{{mw_shard_id}}"}, {key: "__lp_url__", value: "{{lp_url}}"}, {key: "__cloud_storage_config_environment__", value: "{{cloud_storage_config_environment}}"}, {key: "__google_vision_tagging__", value: "{{google_vision_tagging}}"}, {key: "__lp_tmpfile_location__", value: "{{lp_tmpfile_location}}"}, {key: "__esRouterAdditionalConfig__", value: "{{es_router_additional_config}}"},{key: "__esRouterSecondaryAdditionalConfig__", value: "{{es_router_additional_secondary_config}}"},{key: "__es_port__", value: "{{es_port}}"}, {key: "__keyspace_name__", value: "{{content_keyspace_name}}"}, {key: "__collection_fullecar_disable__", value: "{{collection_fullecar_disable}}"},{key: "__max_iteration_count_for_samza_job__", value: "{{max_iteration_count_for_samza_job}}"},{key: "__cloud_storage_type__", value: "{{cloud_store}}"},{key: "__aws_access_key_id__",value: "{{aws_storage_key}}"},{key: "__aws_secret_access_key__",value: "{{aws_storage_secret}}"},{key: "__aws_storage_container__",value: "{{aws_storage_container}}"},{key: "__azure_storage_key__", value: "{{sunbird_public_storage_account_name}}"},{key: "__azure_storage_secret__", value: "{{sunbird_public_storage_account_key}}"},{key: "__azure_storage_container__", value: "{{azure_public_container}}"},{key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__plugin_media_base_url__", value: "{{plugin_media_base_url}}"}, {key: "__installation_id__", value: "{{instance_name}}"}, {key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__hierarchy_keyspace_name__", value: "{{hierarchy_keyspace_name}}"}, {key: "__composite_search_indexer_container_count__", value: "{{composite_search_indexer_container_count}}"},{key: "__cassandra_lp_connection__", value: "{{lp_cassandra_connection}}"}, {key: "__cassandra_lpa_connection__", value: "{{dp_cassandra_connection}}"}, {key: "__streaming_mime_type__", value: "{{streaming_mime_type}}"}, {key: "__cassandra_sunbird_connection__", value: "{{core_cassandra_connection}}"}, {key: "__cloud_upload_retry_count__", value: "{{cloud_upload_retry_count}}"}, {key: "__compositesearch_index_name__", value: "{{compositesearch_index_name}}"},{key: "__publish_pipeline_container_count__", value: "{{publish_pipeline_container_count}}"},{key: "__yarn_container_memory_mb__", value: "{{publish_yarn_container_memory_mb}}"},{key: "__youtube_api_key__", value: "{{youtube_api_key}}"},{key: "__kp_learning_service_base_url__", value: "{{kp_learning_service_base_url}}"},{key: "__sunbird_installation__", value: "{{sunbird_platform_installation}}"}, {key: "__search_lms_es_host__", value: "{{search_lms_es_host}}"},{key: "__dial_image_storage_container__", value: "{{dial_image_storage_container}}"},{key: "__dial_base_url__", value: "{{dial_base_url}}"},{key: "__learner_service_base_url__", value: "{{learner_service_base_url}}"},{key: "__cert_service_base_url__", value: "{{cert_service_base_url}}"},{key: "__certificate_base_path__", value: "{{certificate_base_path}}"},{key: "__kp_content_service_base_url__", value: "{{kp_content_service_base_url}}"},{key: "__kp_print_service_base_url__", value: "{{kp_print_service_base_url}}"},{key: "__cert_reg_service_base_url__", value: "{{cert_reg_service_base_url}}"},{key: "__kp_search_service_base_url__", value: "{{kp_search_service_base_url}}"},{key: "__samza_coordinator_replication_factor__", value: "{{samza_coordinator_replication_factor}}"},{key: "__samza_checkpoint_replication_factor__", value: "{{samza_checkpoint_replication_factor}}"},{key: "__course_batch_updater_container_count__", value: "{{course_batch_updater_container_count}}"},{key: "__course_certificate_generator_container_count__", value: "{{course_certificate_generator_container_count}}"},{key: "__course_progress_batch_size__", value: "{{course_progress_batch_size}}"},{key: "__itemset_generate_pdf__", value: "{{itemset_generate_pdf}}"},{key: "__auto_creator_container_count__", value: "{{auto_creator_container_count}}"},{key: "__content_streaming_enabled__", value: "{{content_streaming_enabled}}"},{key: "__lms_service_base_url__", value: "{{lms_service_base_url}}"},{key: "__mvc_search_indexer_container_count__", value: "{{mvc_search_indexer_container_count}}"}, {key: "__search_es7_host__", value: "{{search_es7_host}}"} , {key: "__ml-keywordapi__", value: "{{mlworkbench}}"},{key: "__auto_creator_artifact_allowed_sources__", value: "{{auto_creator_artifact_allowed_sources}}"},{key: "__publish_pipeline_task_opts__", value: "{{publish_pipeline_task_opts}}"},{key: "__auto_creator_g_service_acct_cred__", value: "{{auto_creator_gservice_acct_cred}}"}] + - "{{ (config_files|default({})).results|default([]) }}" + + +@@ -101,4 +101,4 @@ + args: + chdir: "{{samza_jobs_dir}}/extract/" + +- file: path={{samza_jobs_dir}} owner=hduser group=hadoop state=directory recurse=yes +- file: path={{samza_jobs_dir}} owner=hduser group=hadoop state=directory recurse=yes + - "{{ (config_files|default({})).results|default([]) }}" From fc889ac4f88e259a248c7ea255ef61d9b0b856de Mon Sep 17 00:00:00 2001 From: Ankit Date: Mon, 10 Jan 2022 15:27:34 +0530 Subject: [PATCH 05/58] checkpoint disabled for flink --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index e4ddabab92..adc68e7f1f 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -73,7 +73,7 @@ base_config: | } job { env = "{{ env_name }}" - enable.distributed.checkpointing = true + enable.distributed.checkpointing = false statebackend { blob { storage { From d78dfcb52603e92a284acc67352882f36baa5c32 Mon Sep 17 00:00:00 2001 From: Ankit Date: Thu, 13 Jan 2022 13:50:13 +0530 Subject: [PATCH 06/58] flink job changes --- ansible/roles/learning-service/templates/application.conf.j2 | 3 +++ ansible/roles/lp-contenttool/templates/application.conf.j2 | 3 +++ ansible/roles/lp-synctool-deploy/templates/application.conf.j2 | 3 +++ kubernetes/helm_charts/datapipeline_jobs/values.j2 | 3 +++ 4 files changed, 12 insertions(+) diff --git a/ansible/roles/learning-service/templates/application.conf.j2 b/ansible/roles/learning-service/templates/application.conf.j2 index ea1ea930e5..d6a508ab63 100644 --- a/ansible/roles/learning-service/templates/application.conf.j2 +++ b/ansible/roles/learning-service/templates/application.conf.j2 @@ -199,6 +199,9 @@ cloud_storage_type="{{ cloud_store }}" azure_storage_key="{{sunbird_public_storage_account_name}}" azure_storage_secret="{{sunbird_public_storage_account_key}}" azure_storage_container="{{ azure_public_container }}" +aws_storage_key = "{{ aws_storage_key }}" +aws_storage_secret: "{{ aws_storage_secret }}" +aws_storage_container: "{{ aws_storage_container }}" installation.id="{{ instance_name }}" diff --git a/ansible/roles/lp-contenttool/templates/application.conf.j2 b/ansible/roles/lp-contenttool/templates/application.conf.j2 index 7444400ad2..97096a79e9 100644 --- a/ansible/roles/lp-contenttool/templates/application.conf.j2 +++ b/ansible/roles/lp-contenttool/templates/application.conf.j2 @@ -10,6 +10,9 @@ destination.storage_type="{{ contenttool.destination.storage_type }}" azure_storage_key="{{sunbird_public_storage_account_name}}" azure_storage_secret="{{sunbird_public_storage_account_key}}" azure_storage_container="{{ azure_public_container }}" +aws_storage_key = "{{ aws_storage_key }}" +aws_storage_secret: "{{ aws_storage_secret }}" +aws_storage_container: "{{ aws_storage_container }}" content.external_fields="body,stageIcons,screenshots" content.extract_mimetype="application/vnd.ekstep.h5p-archive,application/vnd.ekstep.ecml-archive,application/vnd.ekstep.html-archive" diff --git a/ansible/roles/lp-synctool-deploy/templates/application.conf.j2 b/ansible/roles/lp-synctool-deploy/templates/application.conf.j2 index 686acdda15..03e546831a 100644 --- a/ansible/roles/lp-synctool-deploy/templates/application.conf.j2 +++ b/ansible/roles/lp-synctool-deploy/templates/application.conf.j2 @@ -81,6 +81,9 @@ cloud_storage_type="{{ cloud_store }}" azure_storage_key="{{sunbird_public_storage_account_name}}" azure_storage_secret="{{sunbird_public_storage_account_key}}" azure_storage_container="{{ azure_public_container }}" +aws_storage_key = "{{ aws_storage_key }}" +aws_storage_secret: "{{ aws_storage_secret }}" +aws_storage_container: "{{ aws_storage_container }}" contentTypeToPrimaryCategory { ClassroomTeachingVideo: "Explanation Content" diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index adc68e7f1f..406c52c4cf 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -273,6 +273,9 @@ questionset-publish: azure_storage_key="{{ sunbird_public_storage_account_name }}" azure_storage_secret="{{ sunbird_public_storage_account_key }}" azure_storage_container="{{ azure_public_container }}" + aws_storage_key = "{{ aws_storage_key }}" + aws_storage_secret: "{{ aws_storage_secret }}" + aws_storage_container: "{{ aws_storage_container }}" master.category.validation.enabled ="{{ master_category_validation_enabled }}" From 28b12e7574585445bf29d599cfebe64e5e661d30 Mon Sep 17 00:00:00 2001 From: Haritest Date: Fri, 21 Jan 2022 18:14:24 +0530 Subject: [PATCH 07/58] Update values.j2 --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 406c52c4cf..3837549a35 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -360,7 +360,7 @@ search-indexer: dialcode.index.name = "dialcode" dailcodemetrics.index.name = "dialcodemetrics" restrict.metadata.objectTypes = [] - nested.fields = ["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "discussionForum", "provider", "osMetadata", "actions", "taxonomyPaths_v2", "competencies_v3"] + nested.fields = ["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "provider", "osMetadata", "actions", "taxonomyPaths_v2", "competencies_v3"] schema.definition_cache.expiry = {{ schema_definition_cache_expiry_in_sec }} restrict.objectTypes = ["EventSet", "EventSetImage", "Event", "EventImage", "Questionnaire", "Misconception", "FrameworkType", "Concept", "Misconception", "Language", "Reference", "Dimension", "Method", "Library", "Domain", "Api"] ignored.fields={{ search_indexer_ignored_fields | to_json }} From 224bead83177be12214e5d39be36886ef2fa863e Mon Sep 17 00:00:00 2001 From: Haritest Date: Fri, 21 Jan 2022 18:51:09 +0530 Subject: [PATCH 08/58] Update values.j2 --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 3837549a35..63ad21ec26 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -172,6 +172,7 @@ activity-aggregate-updater: jobmanager.memory.flink.size: {{ flink_job_names['activity-aggregate-updater'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['activity-aggregate-updater'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['activity-aggregate-updater'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 From f2e155dfe5df69f3119a88121f902ac2d4915dab Mon Sep 17 00:00:00 2001 From: Haritest Date: Fri, 21 Jan 2022 19:08:49 +0530 Subject: [PATCH 09/58] Update values.j2 --- .../helm_charts/datapipeline_jobs/values.j2 | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 63ad21ec26..d3729c12f4 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -205,6 +205,7 @@ relation-cache-updater: jobmanager.memory.flink.size: {{ flink_job_names['relation-cache-updater'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['relation-cache-updater'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['relation-cache-updater'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -244,6 +245,7 @@ post-publish-processor: jobmanager.memory.flink.size: {{ flink_job_names['post-publish-processor'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['post-publish-processor'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['post-publish-processor'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -284,6 +286,7 @@ questionset-publish: jobmanager.memory.flink.size: {{ flink_job_names['questionset-publish'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['questionset-publish'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['questionset-publish'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -338,6 +341,7 @@ video-stream-generator: jobmanager.memory.flink.size: {{ flink_job_names['video-stream-generator'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['video-stream-generator'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['video-stream-generator'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -370,6 +374,7 @@ search-indexer: jobmanager.memory.flink.size: {{ flink_job_names['search-indexer'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['search-indexer'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['search-indexer'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -416,6 +421,7 @@ enrolment-reconciliation: jobmanager.memory.flink.size: {{ flink_job_names['enrolment-reconciliation'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['enrolment-reconciliation'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['enrolment-reconciliation'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -463,6 +469,7 @@ asset-enrichment: jobmanager.memory.flink.size: {{ flink_job_names['asset-enrichment'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['asset-enrichment'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['asset-enrichment'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -487,6 +494,7 @@ audit-history-indexer: jobmanager.memory.flink.size: {{ flink_job_names['audit-history-indexer'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['audit-history-indexer'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['audit-history-indexer'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -527,6 +535,7 @@ auto-creator-v2: jobmanager.memory.flink.size: {{ flink_job_names['auto-creator-v2'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['auto-creator-v2'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['auto-creator-v2'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -557,6 +566,7 @@ audit-event-generator: jobmanager.memory.flink.size: {{ flink_job_names['audit-event-generator'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['audit-event-generator'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['audit-event-generator'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -587,6 +597,7 @@ metrics-data-transformer: jobmanager.memory.flink.size: {{ flink_job_names['metrics-data-transformer'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['metrics-data-transformer'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['metrics-data-transformer'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -635,6 +646,7 @@ collection-cert-pre-processor: jobmanager.memory.flink.size: {{ flink_job_names['collection-cert-pre-processor'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['collection-cert-pre-processor'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['collection-cert-pre-processor'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -673,6 +685,7 @@ collection-certificate-generator: jobmanager.memory.flink.size: {{ flink_job_names['collection-certificate-generator'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['collection-certificate-generator'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['collection-certificate-generator'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -710,6 +723,7 @@ mvc-indexer: jobmanager.memory.flink.size: {{ flink_job_names['mvc-indexer'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['mvc-indexer'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['mvc-indexer'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -772,6 +786,7 @@ content-publish: jobmanager.memory.flink.size: {{ flink_job_names['content-publish'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['content-publish'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['content-publish'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 From deee8be96fdacfe951c8c5effd8b923a051051df Mon Sep 17 00:00:00 2001 From: Haritest Date: Sat, 22 Jan 2022 18:22:20 +0530 Subject: [PATCH 10/58] updated cert_container_name --- ansible/inventory/env/group_vars/all.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml index 7d890ff110..8248dcbf36 100644 --- a/ansible/inventory/env/group_vars/all.yml +++ b/ansible/inventory/env/group_vars/all.yml @@ -106,9 +106,9 @@ cert_service_base_url: "http://{{private_ingressgateway_ip}}/cert" enc_service_base_url: "http://{{private_ingressgateway_ip}}/enc" certificate_base_path: "https://{{domain_name}}/certs" cert_domain_url: "{{proto}}://{{domain_name}}" -cert_container_name: "{{cert_env_prefix}}-e-credentials" +cert_container_name: "e-credentials" cert_cloud_storage_type: "azure" cert_azure_storage_secret: "{{sunbird_private_storage_account_key}}" cert_azure_storage_key: "{{sunbird_private_storage_account_name}}" -default_channel: "org.sunbird" \ No newline at end of file +default_channel: "org.sunbird" From f50da778eec482a85eb16dc4359944e984ddb081 Mon Sep 17 00:00:00 2001 From: Haritest Date: Fri, 4 Feb 2022 17:08:05 +0530 Subject: [PATCH 11/58] asset-enrichment aws values added --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index d3729c12f4..9aa85f906e 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -464,6 +464,9 @@ asset-enrichment: azure_storage_key="{{ sunbird_public_storage_account_name }}" azure_storage_secret="{{ sunbird_public_storage_account_key }}" azure_storage_container="{{ azure_public_container }}" + aws_storage_key="{{ aws_storage_key }}" + aws_storage_secret="{{ aws_storage_secret }}" + aws_storage_container="{{ aws_storage_container }}" flink-conf: |+ jobmanager.memory.flink.size: {{ flink_job_names['asset-enrichment'].jobmanager_memory }} From e84ce4e2167aa9be275fd846cde5cb7ec2bd32ed Mon Sep 17 00:00:00 2001 From: Haritest Date: Fri, 4 Feb 2022 17:11:14 +0530 Subject: [PATCH 12/58] Update values.j2 --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 9aa85f906e..b372deeaee 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -779,6 +779,9 @@ content-publish: azure_storage_key="{{ sunbird_public_storage_account_name }}" azure_storage_secret="{{ sunbird_public_storage_account_key }}" azure_storage_container="{{ azure_public_container }}" + aws_storage_key="{{ aws_storage_key }}" + aws_storage_secret="{{ aws_storage_secret }}" + aws_storage_container="{{ aws_storage_container }}" master.category.validation.enabled ="{{ master_category_validation_enabled }}" service { From 8ae592cc0b1a7c7e94073e3d0f2f4db0ab77539a Mon Sep 17 00:00:00 2001 From: Haritest Date: Sat, 5 Feb 2022 15:06:17 +0530 Subject: [PATCH 13/58] Update values.j2 --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index b372deeaee..7d7597517b 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -73,7 +73,7 @@ base_config: | } job { env = "{{ env_name }}" - enable.distributed.checkpointing = false + enable.distributed.checkpointing = true statebackend { blob { storage { From 256404a026b7a97566565fde706c223778be888e Mon Sep 17 00:00:00 2001 From: Haritest Date: Tue, 20 Dec 2022 10:40:52 +0530 Subject: [PATCH 14/58] S3 details added --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 7d7597517b..3037f30068 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -5,6 +5,10 @@ repository: {{flink_repository|default('knowledge-platform-jobs')}} image_tag: {{ image_tag }} azure_account: {{ azure_account }} azure_secret: {{ azure_secret }} +s3_access_key: {{ s3_storage_key }} +s3_secret_key: {{ s3_storage_secret }} +s3_endpoint: {{ s3_storage_endpoint }} +s3_path_style_access: {{ s3_path_style_access }} serviceMonitor: enabled: {{ service_monitor_enabled | lower}} @@ -77,12 +81,16 @@ base_config: | statebackend { blob { storage { - account = "{{ azure_account }}.blob.core.windows.net" + account = "{% if checkpoint_store_type == "azure" %}{{ azure_account }}.blob.core.windows.net{% elif checkpoint_store_type == "s3" %}{{ flink_dp_storage_container }}{% endif %}" container = "{{ flink_container_name }}" checkpointing.dir = "checkpoint" } } +{% if checkpoint_store_type == "azure" %} base.url = "wasbs://"${job.statebackend.blob.storage.container}"@"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.checkpointing.dir} +{% elif checkpoint_store_type == "s3" %} + base.url = "s3://"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.container}"/"${job.statebackend.blob.storage.checkpointing.dir} +{% endif %} } } task { From a504f470a8bf1fe19a644bde08ef1dd81980446b Mon Sep 17 00:00:00 2001 From: Haritest Date: Tue, 20 Dec 2022 11:05:01 +0530 Subject: [PATCH 15/58] Update values.j2 --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 3037f30068..f93bddae22 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -3,6 +3,7 @@ imagepullsecrets: {{ imagepullsecrets }} dockerhub: {{ dockerhub }} repository: {{flink_repository|default('knowledge-platform-jobs')}} image_tag: {{ image_tag }} +checkpoint_store_type: {{ checkpoint_store_type }} azure_account: {{ azure_account }} azure_secret: {{ azure_secret }} s3_access_key: {{ s3_storage_key }} From 3635f15938e69a48eba291077a31eb2f495b800d Mon Sep 17 00:00:00 2001 From: Haritest Date: Thu, 22 Dec 2022 16:26:56 +0530 Subject: [PATCH 16/58] Update flink_job_deployment.yaml --- .../templates/flink_job_deployment.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index ebd851a454..0fe6bb7263 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -110,7 +110,15 @@ spec: command: ["/opt/flink/bin/standalone-job.sh"] args: ["start-foreground", "--job-classname={{ .Values.job_classname }}", +{{- if eq .Values.checkpoint_store_type "azure" }} "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}", +{{- end }} +{{- if eq .Values.checkpoint_store_type "s3" }} + "-Ds3.access-key={{ .Values.s3_access_key }}", + "-Ds3.secret-key={{ .Values.s3_secret_key }}", + "-Ds3.endpoint={{ .Values.s3_endpoint }}", + "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", +{{- end }} "-Dweb.submit.enable=false", "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", "-Dmetrics.reporter.prom.port={{ .Values.jobmanager.prom_port }}", @@ -183,7 +191,15 @@ spec: workingDir: {{ .Values.taskmanager.flink_work_dir }} command: ["/opt/flink/bin/taskmanager.sh"] args: ["start-foreground", +{{- if eq .Values.checkpoint_store_type "azure" }} "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}", +{{- end }} +{{- if eq .Values.checkpoint_store_type "s3" }} + "-Ds3.access-key={{ .Values.s3_access_key }}", + "-Ds3.secret-key={{ .Values.s3_secret_key }}", + "-Ds3.endpoint={{ .Values.s3_endpoint }}", + "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", +{{- end }} "-Dweb.submit.enable=false", "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", "-Dmetrics.reporter.prom.host={{ .Release.Name }}-taskmanager", From a1a4a9fecc8475e3d2ddd98ff1610ea09a38a498 Mon Sep 17 00:00:00 2001 From: Haritest Date: Thu, 19 Jan 2023 15:50:42 +0530 Subject: [PATCH 17/58] service_monitor_enabled disabled --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 522ebf5649..e423c42e9b 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -277,7 +277,7 @@ mvc_indexer_ml_keyword_host: "{{ mlworkbench | default('') }}" mvc_indexer_ml_keyword_port: 3579 mvc_indexer_nested_fields: "trackable,credentials" -service_monitor_enabled: true +service_monitor_enabled: false ### controlling the flink jobs log level flink_jobs_console_log_level: INFO From 4c6eaea90b07ba00886b0a32f1f1b5b4341e6956 Mon Sep 17 00:00:00 2001 From: Haritest Date: Wed, 25 Jan 2023 09:40:36 +0530 Subject: [PATCH 18/58] Update flink_job_deployment.yaml --- .../datapipeline_jobs/templates/flink_job_deployment.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index 0fe6bb7263..7514dc721d 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -116,8 +116,6 @@ spec: {{- if eq .Values.checkpoint_store_type "s3" }} "-Ds3.access-key={{ .Values.s3_access_key }}", "-Ds3.secret-key={{ .Values.s3_secret_key }}", - "-Ds3.endpoint={{ .Values.s3_endpoint }}", - "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", {{- end }} "-Dweb.submit.enable=false", "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", From 60890e5a77ca2ff936ee6ac76a6292377fb0e732 Mon Sep 17 00:00:00 2001 From: Haritest Date: Mon, 6 Feb 2023 13:16:52 +0530 Subject: [PATCH 19/58] Removed the s3 endpoint parameters --- .../datapipeline_jobs/templates/flink_job_deployment.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index 7514dc721d..f13eea269b 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -195,8 +195,6 @@ spec: {{- if eq .Values.checkpoint_store_type "s3" }} "-Ds3.access-key={{ .Values.s3_access_key }}", "-Ds3.secret-key={{ .Values.s3_secret_key }}", - "-Ds3.endpoint={{ .Values.s3_endpoint }}", - "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", {{- end }} "-Dweb.submit.enable=false", "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", From 248b18ef820978666e739083fe055e83c3449c96 Mon Sep 17 00:00:00 2001 From: Haritest Date: Mon, 6 Feb 2023 14:32:42 +0530 Subject: [PATCH 20/58] updated 4.8.0 changes --- .../helm_charts/datapipeline_jobs/values.j2 | 126 +++++++++++++++++- 1 file changed, 124 insertions(+), 2 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index f93bddae22..9ca81e63aa 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -746,6 +746,7 @@ content-publish: kafka { input.topic = {{ env_name }}.publish.job.request post_publish.topic = {{ env_name }}.content.postpublish.request + mvc.topic = "{{ env_name }}.mvc.processor.job.request" error.topic = "{{ env_name }}.learning.events.failed" groupId = {{ env_name }}-content-publish-group } @@ -768,14 +769,52 @@ content-publish: keyspace = "{{ content_keyspace_name }}" table = "content_data" tmp_file_location = "/tmp" - objectType = ["Content", "ContentImage"] - mimeType = ["application/pdf", "video/avi", "video/mpeg", "video/quicktime", "video/3gpp", "video/mpeg", "video/mp4", "video/ogg", "video/webm", "application/vnd.ekstep.html-archive","application/vnd.ekstep.ecml-archive"] + objectType = ["Content", "ContentImage","Collection","CollectionImage"] + mimeType = ["application/pdf", "video/avi", "video/mpeg", "video/quicktime", "video/3gpp", "video/mpeg", "video/mp4", "video/ogg", "video/webm", "application/vnd.ekstep.html-archive","application/vnd.ekstep.ecml-archive","application/vnd.ekstep.content-collection" + "application/vnd.ekstep.ecml-archive", + "application/vnd.ekstep.html-archive", + "application/vnd.android.package-archive", + "application/vnd.ekstep.content-archive", + "application/octet-stream", + "application/json", + "application/javascript", + "application/xml", + "text/plain", + "text/html", + "application/survey", + "text/javascript", + "text/xml", + "text/css", + "image/jpeg", "image/jpg", "image/png", "image/tiff", "image/bmp", "image/gif", "image/svg+xml", + "image/x-quicktime", + "video/avi", "video/mpeg", "video/quicktime", "video/3gpp", "video/mpeg", "video/mp4", "video/ogg", "video/webm", + "video/msvideo", + "video/x-msvideo", + "video/x-qtc", + "video/x-mpeg", + "audio/mp3", "audio/mp4", "audio/mpeg", "audio/ogg", "audio/webm", "audio/x-wav", "audio/wav", + "audio/mpeg3", + "audio/x-mpeg-3", + "audio/vorbis", + "application/x-font-ttf", + "application/pdf", "application/epub", "application/msword", + "application/vnd.ekstep.h5p-archive", + "application/vnd.ekstep.plugin-archive", + "video/x-youtube", "video/youtube", + "text/x-url"] asset_download_duration = "60 seconds" stream { enabled = {{ content_stream_enabled | lower }} mimeType = ["video/mp4", "video/webm"] } artifact.size.for_online= {{ content_artifact_size_for_online }} + + downloadFiles { + spine = ["appIcon"] + full = ["appIcon", "grayScaleAppIcon", "artifactUrl", "itemSetPreviewUrl", "media"] + } + nested.fields = ["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "discussionForum", "provider", "osMetadata", "actions", "taxonomyPaths_v2", "competencies_v3", "test_competencies_v2","competencies_v2"] + restrict.objectTypes = ["EventSet", "Questionnaire", "Misconception", "FrameworkType", "EventSetImage", "EventImage"] } cloud_storage { folder { @@ -784,6 +823,86 @@ content-publish: } } + hierarchy { + keyspace = "{{ hierarchy_keyspace_name }}" + table = "content_hierarchy" + } + contentTypeToPrimaryCategory { + ClassroomTeachingVideo: "Explanation Content" + ConceptMap: "Learning Resource" + Course: "Course" + CuriosityQuestionSet: "Practice Question Set" + eTextBook: "eTextbook" + Event: "Event" + EventSet: "Event Set" + ExperientialResource: "Learning Resource" + ExplanationResource: "Explanation Content" + ExplanationVideo: "Explanation Content" + FocusSpot: "Teacher Resource" + LearningOutcomeDefinition: "Teacher Resource" + MarkingSchemeRubric: "Teacher Resource" + PedagogyFlow: "Teacher Resource" + PracticeQuestionSet: "Practice Question Set" + PracticeResource: "Practice Question Set" + SelfAssess: "Course Assessment" + TeachingMethod: "Teacher Resource" + TextBook: "Digital Textbook" + Collection: "Content Playlist" + ExplanationReadingMaterial: "Learning Resource" + LearningActivity: "Learning Resource" + LessonPlan: "Content Playlist" + LessonPlanResource: "Teacher Resource" + PreviousBoardExamPapers: "Learning Resource" + TVLesson: "Explanation Content" + OnboardingResource: "Learning Resource" + ReadingMaterial: "Learning Resource" + Template: "Template" + Asset: "Asset" + Plugin: "Plugin" + LessonPlanUnit: "Lesson Plan Unit" + CourseUnit: "Course Unit" + TextBookUnit: "Textbook Unit" + Asset: "Certificate Template" + } + contentTypeToPrimaryCategory { + ClassroomTeachingVideo: "Explanation Content" + ConceptMap: "Learning Resource" + Course: "Course" + CuriosityQuestionSet: "Practice Question Set" + eTextBook: "eTextbook" + ExperientialResource: "Learning Resource" + ExplanationResource: "Explanation Content" + ExplanationVideo: "Explanation Content" + FocusSpot: "Teacher Resource" + LearningOutcomeDefinition: "Teacher Resource" + MarkingSchemeRubric: "Teacher Resource" + PedagogyFlow: "Teacher Resource" + PracticeQuestionSet: "Practice Question Set" + PracticeResource: "Practice Question Set" + SelfAssess: "Course Assessment" + TeachingMethod: "Teacher Resource" + TextBook: "Digital Textbook" + Collection: "Content Playlist" + ExplanationReadingMaterial: "Learning Resource" + LearningActivity: "Learning Resource" + LessonPlan: "Content Playlist" + LessonPlanResource: "Teacher Resource" + PreviousBoardExamPapers: "Learning Resource" + TVLesson: "Explanation Content" + OnboardingResource: "Learning Resource" + ReadingMaterial: "Learning Resource" + Template: "Template" + Asset: "Asset" + Plugin: "Plugin" + LessonPlanUnit: "Lesson Plan Unit" + CourseUnit: "Course Unit" + TextBookUnit: "Textbook Unit" + Asset: "Certificate Template" + Course: "Program" + } + compositesearch.index.name = "compositesearch" + search.document.type = "cs" + cloud_storage_type="{{ cloud_store }}" azure_storage_key="{{ sunbird_public_storage_account_name }}" azure_storage_secret="{{ sunbird_public_storage_account_key }}" @@ -791,6 +910,9 @@ content-publish: aws_storage_key="{{ aws_storage_key }}" aws_storage_secret="{{ aws_storage_secret }}" aws_storage_container="{{ aws_storage_container }}" + + Course="Program" + Course="Mandatory Course Goal" master.category.validation.enabled ="{{ master_category_validation_enabled }}" service { From c4c467794fe9f43e019586a821dfc0b356e0955f Mon Sep 17 00:00:00 2001 From: Haritest Date: Mon, 6 Feb 2023 15:12:04 +0530 Subject: [PATCH 21/58] updated with 4.8.0 --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 9ca81e63aa..23e7c60e45 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -636,7 +636,9 @@ collection-cert-pre-processor: user_enrolments.table = "{{ middleware_user_enrolments_table }}" course_batch.table = "{{ middleware_course_batch_table }}" assessment_aggregator.table = "{{ middleware_assessment_aggregator_table }}" + user_activity_agg.table = "{{ middleware_user_activity_agg_table }}" } + cert_domain_url = "{{ cert_domain_url }}" user_read_api = "/private/user/v1/read" content_read_api = "/content/v3/read" From 404171377a1f3798efd3f1af425a3a782ab48793 Mon Sep 17 00:00:00 2001 From: Haritest Date: Mon, 6 Feb 2023 15:22:13 +0530 Subject: [PATCH 22/58] config update to 4.8.0 --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 23e7c60e45..22c35b96c4 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -683,6 +683,8 @@ collection-certificate-generator: keyspace = "{{ middleware_course_keyspace }}" user_enrolments.table = "{{ middleware_user_enrolments_table }}" course_batch.table = "{{ middleware_course_batch_table }}" + sbkeyspace = "{{ registry_sunbird_keyspace }}" + certreg.table ="{{ cert_registry_table }}" } cert_domain_url = "{{ cert_domain_url }}" cert_container_name = "{{ cert_container_name }}" @@ -693,8 +695,13 @@ collection-certificate-generator: certreg.basePath = "{{ cert_reg_service_base_url }}" learner.basePath = "{{ learner_service_base_url }}" enc.basePath = "{{ enc_service_base_url }}" + rc.basePath = "{{ cert_rc_base_url }}" + rc.entity = "{{ cert_rc_entity }}" } + enable.suppress.exception = {{ collection_certificate_generator_enable_suppress_exception | lower }} + enable.rc.certificate = false + flink-conf: |+ jobmanager.memory.flink.size: {{ flink_job_names['collection-certificate-generator'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['collection-certificate-generator'].taskmanager_memory }} From dbabbc12e871dc805a07c80e5b0eda465e0d3ad5 Mon Sep 17 00:00:00 2001 From: Haritest Date: Mon, 6 Feb 2023 15:27:44 +0530 Subject: [PATCH 23/58] Update main.yml --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index e423c42e9b..d05ef05a4e 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -290,3 +290,6 @@ content_stream_enabled: true content_artifact_size_for_online: 209715200 source_base_url: "{{proto}}://{{domain_name}}/api" + + +registry_sunbird_keyspace: "sunbird" From 0c17142e649a59ad83a434fa1ddd3e9f59791627 Mon Sep 17 00:00:00 2001 From: Haritest Date: Mon, 6 Feb 2023 15:35:07 +0530 Subject: [PATCH 24/58] Update main.yml --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index d05ef05a4e..8b2f440837 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -293,3 +293,4 @@ source_base_url: "{{proto}}://{{domain_name}}/api" registry_sunbird_keyspace: "sunbird" +cert_registry_table: "cert_registry" From 9029a808ae007f2369b321c7306d32c99e30e434 Mon Sep 17 00:00:00 2001 From: Haritest Date: Mon, 6 Feb 2023 16:27:14 +0530 Subject: [PATCH 25/58] Update all.yml --- ansible/inventory/env/group_vars/all.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml index 8248dcbf36..82132a0cae 100644 --- a/ansible/inventory/env/group_vars/all.yml +++ b/ansible/inventory/env/group_vars/all.yml @@ -112,3 +112,6 @@ cert_azure_storage_secret: "{{sunbird_private_storage_account_key}}" cert_azure_storage_key: "{{sunbird_private_storage_account_name}}" default_channel: "org.sunbird" + +cert_rc_base_url: "http://{{private_ingressgateway_ip}}/registry-service/api/v1" +cert_rc_entity: "TrainingCertificate" From aaca6872025301e7dbfb6cb740a8573b22905992 Mon Sep 17 00:00:00 2001 From: Haritest Date: Mon, 6 Feb 2023 16:30:50 +0530 Subject: [PATCH 26/58] Update main.yml --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 8b2f440837..370c9dc3c4 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -294,3 +294,5 @@ source_base_url: "{{proto}}://{{domain_name}}/api" registry_sunbird_keyspace: "sunbird" cert_registry_table: "cert_registry" +collection_certificate_generator_enable_suppress_exception: false +collection_certificate_generator_enable_rc_certificate: true From 8fb9707828aac0c78504281f9f0c12d4a9780e77 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Mon, 3 Apr 2023 15:03:00 +0530 Subject: [PATCH 27/58] new variableas added for cert-gen with s3 --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 22c35b96c4..4b5233ea8d 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -689,8 +689,10 @@ collection-certificate-generator: cert_domain_url = "{{ cert_domain_url }}" cert_container_name = "{{ cert_container_name }}" cert_cloud_storage_type = "{{ cert_cloud_storage_type }}" - cert_azure_storage_secret = "{{ cert_azure_storage_secret }}" - cert_azure_storage_key = "{{ cert_azure_storage_key }}" + cert_aws_storage_secret = "{{ cert_aws_storage_secret }}" + cert_aws_storage_key = "{{ cert_aws_storage_key }}" + + service { certreg.basePath = "{{ cert_reg_service_base_url }}" learner.basePath = "{{ learner_service_base_url }}" From 84de9072a6c7d97b5de166d3eca164f71501d700 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Thu, 9 May 2024 16:48:10 +0530 Subject: [PATCH 28/58] Update values.j2 --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 4b5233ea8d..740dfb0983 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -133,7 +133,7 @@ activity-aggregate-updater: output.audit.topic = {{ env_name }}.telemetry.raw output.failed.topic = {{ env_name }}.activity.agg.failed output.certissue.topic = {{ env_name }}.issue.certificate.request - groupId = {{ env_name }}-activity-aggregate-group + groupId = {{ env_name }}-activity-aggregate-group2 } task { window.shards = {{ activity_agg_window_shards }} From edc4d3aef6aa94afafd63509fbbff87858058483 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 14 May 2024 09:28:09 +0530 Subject: [PATCH 29/58] Update main.yml --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 370c9dc3c4..820f883c0a 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -10,7 +10,7 @@ jobmanager_heap_memory: 1024 jobmanager_ui_rest_port: 80 jobmanager_ui_tcp_port: 8081 jobmanager_ui_service: - type: LoadBalancer + type: ClusterIP annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" From c6cdb2a45ba81da1dc94b90b1907ba7e803cee49 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 14 May 2024 14:22:32 +0530 Subject: [PATCH 30/58] Update main.yml --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 820f883c0a..370c9dc3c4 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -10,7 +10,7 @@ jobmanager_heap_memory: 1024 jobmanager_ui_rest_port: 80 jobmanager_ui_tcp_port: 8081 jobmanager_ui_service: - type: ClusterIP + type: LoadBalancer annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" From 17e89e4992bc426da699ab7084ef551ee99a7f01 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 21 May 2024 19:05:48 +0530 Subject: [PATCH 31/58] Update main.yml --- .../ansible/roles/flink-jobs-deploy/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 370c9dc3c4..aeb2220fe8 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -1,4 +1,4 @@ -imagepullsecrets: sunbird-registry-secret +imagepullsecrets: devregistrysecret ### Job manager related vars jobmanager_rpc_port: 6123 @@ -9,8 +9,8 @@ jobmanager_prom_port: 9250 jobmanager_heap_memory: 1024 jobmanager_ui_rest_port: 80 jobmanager_ui_tcp_port: 8081 -jobmanager_ui_service: - type: LoadBalancer +jobmanager_ui_service: + type: ClusterIP annotations: service.beta.kubernetes.io/azure-load-balancer-internal: "true" From e76bbd1fd47dae601f64974ab231f0eac59b6dcb Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 21 May 2024 19:43:29 +0530 Subject: [PATCH 32/58] Update main.yml --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index aeb2220fe8..f9592523c4 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -1,4 +1,4 @@ -imagepullsecrets: devregistrysecret +imagepullsecrets: sunbird-registry-secret ### Job manager related vars jobmanager_rpc_port: 6123 From b261d011909bf2c79fdd3ecebcd78a4eba1faac6 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Mon, 27 May 2024 16:12:11 +0530 Subject: [PATCH 33/58] Update values.j2 for cert-email test --- .../helm_charts/datapipeline_jobs/values.j2 | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 740dfb0983..d16a92ea6d 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -750,7 +750,41 @@ mvc-indexer: parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 - +certificate-email-service: + certificate-email-service: |+ + kafkaBootstrapServers={{ kafka_servers_list }} + kafkaTopic=dev.issue.certificate.request + GroupId=dev-activity-aggregate-group + ###Elasticsearch configuration + es_hosts={{course_elasticsearch_host}} + es_ports={{elasticsearch_http_port}} + es_user_index={{ es_user_index }} + es_course_index={{ es_course_index }} + certUrl=https://{{ domain_name }}/apis/public/v8/publicCertificateFlinkv2/download + secretKey={{ cert_email_secret_key }} + emailHost={{ mail_server_host }} + emailPort={{ mail_server_port }} + mailFrom={{ sunbird_mail_server_from_email }} + mailPassword={{ mail_server_password }} + mailUserName={{ mail_server_username }} + nhsrc_abhyaas_api={{ nhsrc_abhyaas_api }} + nhsrc_apiKey={{ nhsrc_apiKey }} + get_cassandra_host={{ sunbird_cassandra_host }} + cassandra_ports={{ sunbird_cassandra_port }} + cassandra_key_space={{ keyspace_sunbird }} + #cassandra_enrol_table={{ cassandra_enrol_table }} + user_sso_bulkupload_v2_table={{ user_sso_bulkupload_v2_table }} + fresh_chat_bearer_key={{ fresh_chat_bearer_key }} + fresh_chat_baseURI=https://api.in.freshchat.com + fresh_chat_basePath_upload=/v2/images/upload + fresh_chat_basePath_whatsapp=/v2/outbound-messages/whatsapp + fresh_chat_from_phnumber={{ fresh_chat_from_phnumber }} + aws_accessKey={{ aws_accessKey }} + aws_secretKey={{ aws_secretKey }} + s3_bucketName={{ s3_bucketName }} + fresh_chat_msg_template={\"message_template\":{\"storage\":\"none\",\"template_name\":\"completion_certificate\",\"namespace\":\"60ecfaaf_855f_43b6_803c_45580bb195b5\",\"language\":{\"policy\":\"deterministic\",\"code\":\"en\"} + template_name=completion_certificate + template_namespace={{ template_namespace }} content-publish: content-publish: |+ include file("/data/flink/conf/base-config.conf") From bcd84c8f2b8afc0a1eb1c126ce666e4cce77a603 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Mon, 27 May 2024 16:41:47 +0530 Subject: [PATCH 34/58] Update main.yml --- .../ansible/roles/flink-jobs-deploy/defaults/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index f9592523c4..ea6438da8d 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -125,6 +125,13 @@ job_classname: "" ### class name vars flink_job_names: + certificate-email-service: + job_class_name: 'Main' + replica: 1 + jobmanager_memory: 1024m + taskmanager_memory: 1024m + taskslots: 1 + cpu_requests: 0.3 activity-aggregate-updater: job_class_name: 'org.sunbird.job.aggregate.task.ActivityAggregateUpdaterStreamTask' replica: 1 From f06a39d96171325b5c190977cd92ffa955254d29 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Mon, 27 May 2024 16:50:56 +0530 Subject: [PATCH 35/58] Update values.j2 --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index d16a92ea6d..a646ba64dd 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -785,6 +785,15 @@ certificate-email-service: fresh_chat_msg_template={\"message_template\":{\"storage\":\"none\",\"template_name\":\"completion_certificate\",\"namespace\":\"60ecfaaf_855f_43b6_803c_45580bb195b5\",\"language\":{\"policy\":\"deterministic\",\"code\":\"en\"} template_name=completion_certificate template_namespace={{ template_namespace }} + + flink-conf: |+ + jobmanager.memory.flink.size: {{ flink_job_names['certificate-email-service'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['certificate-email-service'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['certificate-email-service'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 content-publish: content-publish: |+ include file("/data/flink/conf/base-config.conf") From 8430546b052d4a1cef8ecd36485b4d4700ffce06 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Mon, 27 May 2024 17:56:09 +0530 Subject: [PATCH 36/58] Update flink_job_deployment.yaml --- .../datapipeline_jobs/templates/flink_job_deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index f13eea269b..a9fc349580 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -107,7 +107,7 @@ spec: image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" imagePullPolicy: Always workingDir: /opt/flink - command: ["/opt/flink/bin/standalone-job.sh"] + command: ["/opt/flink/bin/start-cluster.sh -Djobmanager.entry-class=Main.scala"] args: ["start-foreground", "--job-classname={{ .Values.job_classname }}", {{- if eq .Values.checkpoint_store_type "azure" }} From 4a924857538dd14027cba9ae29ba05f5ea69ac04 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Mon, 27 May 2024 17:56:52 +0530 Subject: [PATCH 37/58] Update flink_job_deployment.yaml --- .../datapipeline_jobs/templates/flink_job_deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index a9fc349580..f13eea269b 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -107,7 +107,7 @@ spec: image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" imagePullPolicy: Always workingDir: /opt/flink - command: ["/opt/flink/bin/start-cluster.sh -Djobmanager.entry-class=Main.scala"] + command: ["/opt/flink/bin/standalone-job.sh"] args: ["start-foreground", "--job-classname={{ .Values.job_classname }}", {{- if eq .Values.checkpoint_store_type "azure" }} From ce508e7b371962fb849aa3b6e1ce4bd76d26d066 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 27 May 2025 11:32:58 +0530 Subject: [PATCH 38/58] Update values.j2 --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index a646ba64dd..9133b8f61a 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -120,6 +120,7 @@ base_config: | } schema { basePath = "{{ kp_schema_base_path }}" + base_path= "{{ kp_schema_base_path }}" supportedVersion = { itemset = "2.0" } From 97387c6d6caacc0e61abca3ddb42f8b20a05ce38 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Wed, 19 Nov 2025 09:54:23 +0530 Subject: [PATCH 39/58] Activity aggrgator parallelism updated from 1 to 2 --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index ea6438da8d..57e5a8e1b1 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -40,7 +40,7 @@ producer_max_request_size: 1572864 ### Activity Aggregate job related vars activity_agg_consumer_parallelism: 1 activity_agg_dedup_parallelism: 1 -activity_agg_parallelism: 1 +activity_agg_parallelism: 2 enrolment_complete_parallelism: 1 middleware_consumption_table: "user_content_consumption" middleware_user_activity_agg_table: "user_activity_agg" From 22735cbc2e0bc6d3deb7dfb8dc6e9d7a78d1f894 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Wed, 19 Nov 2025 10:01:57 +0530 Subject: [PATCH 40/58] reverted activity parallelism from 2 to 1 and enrolment parllelism updated from 1 to 2 --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 57e5a8e1b1..da67a80fc5 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -40,8 +40,8 @@ producer_max_request_size: 1572864 ### Activity Aggregate job related vars activity_agg_consumer_parallelism: 1 activity_agg_dedup_parallelism: 1 -activity_agg_parallelism: 2 -enrolment_complete_parallelism: 1 +activity_agg_parallelism: 1 +enrolment_complete_parallelism: 2 middleware_consumption_table: "user_content_consumption" middleware_user_activity_agg_table: "user_activity_agg" activity_agg_checkpointing_interval: 300000 From 899ca24f3fb237c06df8f22e2a26b672492d68e2 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Wed, 19 Nov 2025 10:04:53 +0530 Subject: [PATCH 41/58] activity_agg_consumer_parallelism updated from 1 to 2 --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index da67a80fc5..55977bd3f1 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -38,7 +38,7 @@ restart_delay: 30000 # in milli-seconds producer_max_request_size: 1572864 ### Activity Aggregate job related vars -activity_agg_consumer_parallelism: 1 +activity_agg_consumer_parallelism: 2 activity_agg_dedup_parallelism: 1 activity_agg_parallelism: 1 enrolment_complete_parallelism: 2 From ab83d40eec38784093964c6407491729f99f449f Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Wed, 19 Nov 2025 10:10:27 +0530 Subject: [PATCH 42/58] Update main.yml --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 55977bd3f1..945e4484cd 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -39,8 +39,8 @@ producer_max_request_size: 1572864 ### Activity Aggregate job related vars activity_agg_consumer_parallelism: 2 -activity_agg_dedup_parallelism: 1 -activity_agg_parallelism: 1 +activity_agg_dedup_parallelism: 2 +activity_agg_parallelism: 2 enrolment_complete_parallelism: 2 middleware_consumption_table: "user_content_consumption" middleware_user_activity_agg_table: "user_activity_agg" From 8dd278ab355d6c35b60f2c8ea4722fbf054d99b7 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Wed, 19 Nov 2025 10:25:59 +0530 Subject: [PATCH 43/58] Update main.yml --- .../ansible/roles/flink-jobs-deploy/defaults/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 945e4484cd..ea6438da8d 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -38,10 +38,10 @@ restart_delay: 30000 # in milli-seconds producer_max_request_size: 1572864 ### Activity Aggregate job related vars -activity_agg_consumer_parallelism: 2 -activity_agg_dedup_parallelism: 2 -activity_agg_parallelism: 2 -enrolment_complete_parallelism: 2 +activity_agg_consumer_parallelism: 1 +activity_agg_dedup_parallelism: 1 +activity_agg_parallelism: 1 +enrolment_complete_parallelism: 1 middleware_consumption_table: "user_content_consumption" middleware_user_activity_agg_table: "user_activity_agg" activity_agg_checkpointing_interval: 300000 From 4e07dc91b128d23ad10c19cf04313946d5ba7df8 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:21:44 +0530 Subject: [PATCH 44/58] parallelism changes for activity aggrgator --- .../roles/flink-jobs-deploy/defaults/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index ea6438da8d..85fa2fe927 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -39,17 +39,17 @@ producer_max_request_size: 1572864 ### Activity Aggregate job related vars activity_agg_consumer_parallelism: 1 -activity_agg_dedup_parallelism: 1 -activity_agg_parallelism: 1 +activity_agg_dedup_parallelism: 2 +activity_agg_parallelism: 2 enrolment_complete_parallelism: 1 middleware_consumption_table: "user_content_consumption" middleware_user_activity_agg_table: "user_activity_agg" -activity_agg_checkpointing_interval: 300000 -activity_agg_checkpointing_pause_interval: 90000 +activity_agg_checkpointing_interval: 60000 +activity_agg_checkpointing_pause_interval: 30000 activity_agg_batch_interval: 60 activity_agg_batch_read_size: 1 activity_agg_batch_write_size: 10 -activity_agg_window_shards: 1000 +activity_agg_window_shards: 50 activity_agg_dedup_index: 13 activity_agg_dedup_expiry: 604800 activity_module_aggs_enabled: true @@ -134,7 +134,7 @@ flink_job_names: cpu_requests: 0.3 activity-aggregate-updater: job_class_name: 'org.sunbird.job.aggregate.task.ActivityAggregateUpdaterStreamTask' - replica: 1 + replica: 2 jobmanager_memory: 1024m taskmanager_memory: 1024m taskslots: 1 From 01a01c846883ea2c14be06d41d21b8a9d48da19b Mon Sep 17 00:00:00 2001 From: Anil Gupta Date: Mon, 24 Nov 2025 13:24:19 +0530 Subject: [PATCH 45/58] Add user-assessment-summary configuration --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 9133b8f61a..41a0e41ec0 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -639,10 +639,15 @@ collection-cert-pre-processor: assessment_aggregator.table = "{{ middleware_assessment_aggregator_table }}" user_activity_agg.table = "{{ middleware_user_activity_agg_table }}" } + user-assessment-summary { + keyspace = "sunbird" + table = "user_assessment_summary" + } cert_domain_url = "{{ cert_domain_url }}" user_read_api = "/private/user/v1/read" content_read_api = "/content/v3/read" + collection_hierarchy_read_api = "/content/v3/hierarchy" service { content.basePath = "{{ content_service_base_url }}" learner.basePath = "{{ learner_service_base_url }}" From 401f1d594c2fc839736bcf06f7f07f680163dfd2 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Wed, 3 Dec 2025 15:41:12 +0530 Subject: [PATCH 46/58] Update main.yml --- .../ansible/roles/flink-jobs-deploy/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 85fa2fe927..2e58d88b57 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -39,8 +39,8 @@ producer_max_request_size: 1572864 ### Activity Aggregate job related vars activity_agg_consumer_parallelism: 1 -activity_agg_dedup_parallelism: 2 -activity_agg_parallelism: 2 +activity_agg_dedup_parallelism: 3 +activity_agg_parallelism: 3 enrolment_complete_parallelism: 1 middleware_consumption_table: "user_content_consumption" middleware_user_activity_agg_table: "user_activity_agg" @@ -134,7 +134,7 @@ flink_job_names: cpu_requests: 0.3 activity-aggregate-updater: job_class_name: 'org.sunbird.job.aggregate.task.ActivityAggregateUpdaterStreamTask' - replica: 2 + replica: 3 jobmanager_memory: 1024m taskmanager_memory: 1024m taskslots: 1 From 48ec91692ef12268f46cad1b3720b18cff06ee1b Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Thu, 18 Dec 2025 13:36:46 +0530 Subject: [PATCH 47/58] Update values.j2 --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 41a0e41ec0..d69f485738 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -300,6 +300,9 @@ questionset-publish: parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 + metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter + metrics.reporter.prom.port: 9250-9260 + video-stream-generator: video-stream-generator: |+ From 9972099efbd268727174d5101d40804394485965 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Thu, 18 Dec 2025 16:56:37 +0530 Subject: [PATCH 48/58] Update main.yml --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 2e58d88b57..4c0f771997 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -39,8 +39,8 @@ producer_max_request_size: 1572864 ### Activity Aggregate job related vars activity_agg_consumer_parallelism: 1 -activity_agg_dedup_parallelism: 3 -activity_agg_parallelism: 3 +activity_agg_dedup_parallelism: 4 +activity_agg_parallelism: 4 enrolment_complete_parallelism: 1 middleware_consumption_table: "user_content_consumption" middleware_user_activity_agg_table: "user_activity_agg" From a5682e75bc6a873380731e57f1c67eca4f103796 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Thu, 18 Dec 2025 16:57:12 +0530 Subject: [PATCH 49/58] Update main.yml --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 4c0f771997..75901deda0 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -134,7 +134,7 @@ flink_job_names: cpu_requests: 0.3 activity-aggregate-updater: job_class_name: 'org.sunbird.job.aggregate.task.ActivityAggregateUpdaterStreamTask' - replica: 3 + replica: 4 jobmanager_memory: 1024m taskmanager_memory: 1024m taskslots: 1 From 507037b79548ebc4d85c914ccba542f620b29f70 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Mon, 29 Dec 2025 17:56:05 +0530 Subject: [PATCH 50/58] Update flink_job_deployment.yaml-test --- .../datapipeline_jobs/templates/flink_job_deployment.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index f13eea269b..d980be235a 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -156,6 +156,12 @@ kind: Deployment metadata: name: {{ .Release.Name }}-taskmanager namespace: {{ .Values.namespace }} +{{- if .Values.service.annotations }} +{{- with .Values.service.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +{{- end }} spec: replicas: {{ .Values.taskmanager.replicas }} selector: From a70e1d91cd0f92772fe6b28307bf722456564cfa Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Mon, 29 Dec 2025 18:07:53 +0530 Subject: [PATCH 51/58] Update flink_job_deployment.yaml --- .../templates/flink_job_deployment.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index d980be235a..017b944380 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -156,23 +156,22 @@ kind: Deployment metadata: name: {{ .Release.Name }}-taskmanager namespace: {{ .Values.namespace }} -{{- if .Values.service.annotations }} -{{- with .Values.service.annotations }} - annotations: -{{ toYaml . | indent 4 }} -{{- end }} -{{- end }} + spec: replicas: {{ .Values.taskmanager.replicas }} selector: matchLabels: app: flink component: {{ .Release.Name }}-taskmanager + template: metadata: labels: app: flink component: {{ .Release.Name }}-taskmanager + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: "{{ .Values.jobmanager.prom_port }}" spec: volumes: - name: flink-config-volume From a1bb473aa9835509fa417418530710a051ae32b8 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Mon, 29 Dec 2025 18:13:39 +0530 Subject: [PATCH 52/58] Update flink_job_deployment.yaml --- .../datapipeline_jobs/templates/flink_job_deployment.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index 017b944380..9a1caec95d 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -169,9 +169,7 @@ spec: labels: app: flink component: {{ .Release.Name }}-taskmanager - annotations: - prometheus.io/scrape: 'true' - prometheus.io/port: "{{ .Values.jobmanager.prom_port }}" + spec: volumes: - name: flink-config-volume From 8cd1b98a51fd6e0e2eb82120cf60f7d7381c8e8c Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 3 Feb 2026 12:12:49 +0530 Subject: [PATCH 53/58] parallelism increased 6 --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 75901deda0..3b6ab53304 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -39,8 +39,8 @@ producer_max_request_size: 1572864 ### Activity Aggregate job related vars activity_agg_consumer_parallelism: 1 -activity_agg_dedup_parallelism: 4 -activity_agg_parallelism: 4 +activity_agg_dedup_parallelism: 6 +activity_agg_parallelism: 6 enrolment_complete_parallelism: 1 middleware_consumption_table: "user_content_consumption" middleware_user_activity_agg_table: "user_activity_agg" From f072fb2c3f30b417718435db5855beab7a1b3401 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 3 Feb 2026 12:15:12 +0530 Subject: [PATCH 54/58] Update main.yml --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 3b6ab53304..4e6713fbe1 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -134,7 +134,7 @@ flink_job_names: cpu_requests: 0.3 activity-aggregate-updater: job_class_name: 'org.sunbird.job.aggregate.task.ActivityAggregateUpdaterStreamTask' - replica: 4 + replica: 6 jobmanager_memory: 1024m taskmanager_memory: 1024m taskslots: 1 From 107e00f53ee92ba58743eb0d8a49ebf059ec9a2b Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 3 Feb 2026 12:25:10 +0530 Subject: [PATCH 55/58] Update main.yml --- .../ansible/roles/flink-jobs-deploy/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 4e6713fbe1..75901deda0 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -39,8 +39,8 @@ producer_max_request_size: 1572864 ### Activity Aggregate job related vars activity_agg_consumer_parallelism: 1 -activity_agg_dedup_parallelism: 6 -activity_agg_parallelism: 6 +activity_agg_dedup_parallelism: 4 +activity_agg_parallelism: 4 enrolment_complete_parallelism: 1 middleware_consumption_table: "user_content_consumption" middleware_user_activity_agg_table: "user_activity_agg" @@ -134,7 +134,7 @@ flink_job_names: cpu_requests: 0.3 activity-aggregate-updater: job_class_name: 'org.sunbird.job.aggregate.task.ActivityAggregateUpdaterStreamTask' - replica: 6 + replica: 4 jobmanager_memory: 1024m taskmanager_memory: 1024m taskslots: 1 From e7aa02898a2ecfaccfec7f5e34b82f92c257081f Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 3 Feb 2026 12:31:38 +0530 Subject: [PATCH 56/58] Updated main.yml with new parallelism configurations --- .../ansible/roles/flink-jobs-deploy/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 75901deda0..4e6713fbe1 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -39,8 +39,8 @@ producer_max_request_size: 1572864 ### Activity Aggregate job related vars activity_agg_consumer_parallelism: 1 -activity_agg_dedup_parallelism: 4 -activity_agg_parallelism: 4 +activity_agg_dedup_parallelism: 6 +activity_agg_parallelism: 6 enrolment_complete_parallelism: 1 middleware_consumption_table: "user_content_consumption" middleware_user_activity_agg_table: "user_activity_agg" @@ -134,7 +134,7 @@ flink_job_names: cpu_requests: 0.3 activity-aggregate-updater: job_class_name: 'org.sunbird.job.aggregate.task.ActivityAggregateUpdaterStreamTask' - replica: 4 + replica: 6 jobmanager_memory: 1024m taskmanager_memory: 1024m taskslots: 1 From c605511c202be9fa1e9559bb58a659ff6b269c50 Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Tue, 3 Feb 2026 12:35:01 +0530 Subject: [PATCH 57/58] revrtedmain.yml --- .../ansible/roles/flink-jobs-deploy/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 4e6713fbe1..75901deda0 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -39,8 +39,8 @@ producer_max_request_size: 1572864 ### Activity Aggregate job related vars activity_agg_consumer_parallelism: 1 -activity_agg_dedup_parallelism: 6 -activity_agg_parallelism: 6 +activity_agg_dedup_parallelism: 4 +activity_agg_parallelism: 4 enrolment_complete_parallelism: 1 middleware_consumption_table: "user_content_consumption" middleware_user_activity_agg_table: "user_activity_agg" @@ -134,7 +134,7 @@ flink_job_names: cpu_requests: 0.3 activity-aggregate-updater: job_class_name: 'org.sunbird.job.aggregate.task.ActivityAggregateUpdaterStreamTask' - replica: 6 + replica: 4 jobmanager_memory: 1024m taskmanager_memory: 1024m taskslots: 1 From 954f7eb88d64783cb972246ba9ef3bdbdd70d6fd Mon Sep 17 00:00:00 2001 From: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Date: Mon, 9 Feb 2026 09:53:49 +0530 Subject: [PATCH 58/58] Update main.yml --- .../ansible/roles/flink-jobs-deploy/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 75901deda0..1548107ed5 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -44,12 +44,12 @@ activity_agg_parallelism: 4 enrolment_complete_parallelism: 1 middleware_consumption_table: "user_content_consumption" middleware_user_activity_agg_table: "user_activity_agg" -activity_agg_checkpointing_interval: 60000 -activity_agg_checkpointing_pause_interval: 30000 +activity_agg_checkpointing_interval: 180000 +activity_agg_checkpointing_pause_interval: 60000 activity_agg_batch_interval: 60 activity_agg_batch_read_size: 1 activity_agg_batch_write_size: 10 -activity_agg_window_shards: 50 +activity_agg_window_shards: 30 activity_agg_dedup_index: 13 activity_agg_dedup_expiry: 604800 activity_module_aggs_enabled: true