diff --git a/ansible/roles/lp-contenttool/templates/application.conf.j2 b/ansible/roles/lp-contenttool/templates/application.conf.j2 index 7444400ad2..97096a79e9 100644 --- a/ansible/roles/lp-contenttool/templates/application.conf.j2 +++ b/ansible/roles/lp-contenttool/templates/application.conf.j2 @@ -10,6 +10,9 @@ destination.storage_type="{{ contenttool.destination.storage_type }}" azure_storage_key="{{sunbird_public_storage_account_name}}" azure_storage_secret="{{sunbird_public_storage_account_key}}" azure_storage_container="{{ azure_public_container }}" +aws_storage_key = "{{ aws_storage_key }}" +aws_storage_secret: "{{ aws_storage_secret }}" +aws_storage_container: "{{ aws_storage_container }}" content.external_fields="body,stageIcons,screenshots" content.extract_mimetype="application/vnd.ekstep.h5p-archive,application/vnd.ekstep.ecml-archive,application/vnd.ekstep.html-archive" diff --git a/ansible/roles/lp-synctool-deploy/defaults/main.yml b/ansible/roles/lp-synctool-deploy/defaults/main.yml index b9919c28d4..ded17cf57a 100644 --- a/ansible/roles/lp-synctool-deploy/defaults/main.yml +++ b/ansible/roles/lp-synctool-deploy/defaults/main.yml @@ -23,6 +23,6 @@ redis_port: 6379 graphevent_kafka_url: "{{groups['processing-cluster-zookeepers']|join(':9092,')}}:9092" search_lms_index_host: "{{ groups['core-es']|join(':9200,')}}:9200" -cloud_store: azure +cloud_store: aws azure_public_container: azure_account_name: \ No newline at end of file diff --git a/ansible/roles/lp-synctool-deploy/templates/application.conf.j2 b/ansible/roles/lp-synctool-deploy/templates/application.conf.j2 index 686acdda15..03e546831a 100644 --- a/ansible/roles/lp-synctool-deploy/templates/application.conf.j2 +++ b/ansible/roles/lp-synctool-deploy/templates/application.conf.j2 @@ -81,6 +81,9 @@ cloud_storage_type="{{ cloud_store }}" azure_storage_key="{{sunbird_public_storage_account_name}}" azure_storage_secret="{{sunbird_public_storage_account_key}}" azure_storage_container="{{ azure_public_container }}" +aws_storage_key = "{{ aws_storage_key }}" +aws_storage_secret: "{{ aws_storage_secret }}" +aws_storage_container: "{{ aws_storage_container }}" contentTypeToPrimaryCategory { ClassroomTeachingVideo: "Explanation Content" diff --git a/ansible/roles/samza-jobs/tasks/deploy.yml b/ansible/roles/samza-jobs/tasks/deploy.yml index 2b9598eb02..feb0b54876 100644 --- a/ansible/roles/samza-jobs/tasks/deploy.yml +++ b/ansible/roles/samza-jobs/tasks/deploy.yml @@ -71,7 +71,17 @@ replace: dest="{{item[1].stdout}}" regexp="{{item[0].key}}" replace="{{item[0].value}}" when: "{{item[1]|changed}}" with_nested: - - [{key: "__yarn_host__", value: "{{__yarn_host__}}"}, {key: "__yarn_port__", value: "{{__yarn_port__}}"}, {key: "__env__", value: "{{env}}" }, {key: "__env_name__", value: "{{env_name}}" }, {key: "__zookeepers__", value: "{{zookeepers}}"}, {key: "__kafka_brokers__", value: "{{kafka_brokers}}"}, {key: "__delayInMilliSeconds__", value: "{{delayInMilliSeconds}}" }, {key: "__retryTimeInMilliSeconds__", value: "{{retryTimeInMilliSeconds}}" }, {key: "__bypass_reverse_search__", value: "{{bypass_reverse_search}}" }, {key: "__retryBackoffBaseInSeconds__", value: "{{retry_backoff_base_in_seconds}}" }, {key: "__retryLimit__", value: "{{retry_limit}}" }, {key: "__retryLimitEnable__", value: "{{retry_limit_enable}}" }, {key: "__google_api_key__", value: "{{google_api_key}}" }, {key: "__searchServiceEndpoint__", value: "{{search_service_endpoint}}" }, {key: "__objectDenormalizationAdditionalConfig__", value: "{{object_denormalization_additional_config}}" },{key: "__audit_es_host__", value: "{{audit_es_host}}"}, {key: "__search_es_host__", value: "{{search_es_host}}"}, {key: "__redis_host__", value: "{{redis_host}}"}, {key: "__dp_redis_host__", value: "{{dp_redis_host}}"}, {key: "__redis_port__", value: "{{redis_port}}"}, {key: "__environment_id__", value: "{{environment_id}}"}, {key: "__graph_passport_key__", value: "{{graph_passport_key}}"}, {key: "__lp_bolt_url__", value: "{{lp_bolt_url}}"}, {key: "__lp_bolt_read_url__", value: "{{lp_bolt_read_url}}"}, {key: "__lp_bolt_write_url__", value: "{{lp_bolt_write_url}}"}, {key: "__other_bolt_url__", value: "{{other_bolt_url}}"}, {key: "__other_bolt_read_url__", value: "{{other_bolt_read_url}}"}, {key: "__other_bolt_write_url__", value: "{{other_bolt_write_url}}"}, {key: "__mw_shard_id__", value: "{{mw_shard_id}}"}, {key: "__lp_url__", value: "{{lp_url}}"}, {key: "__cloud_storage_config_environment__", value: "{{cloud_storage_config_environment}}"}, {key: "__google_vision_tagging__", value: "{{google_vision_tagging}}"}, {key: "__lp_tmpfile_location__", value: "{{lp_tmpfile_location}}"}, {key: "__esRouterAdditionalConfig__", value: "{{es_router_additional_config}}"},{key: "__esRouterSecondaryAdditionalConfig__", value: "{{es_router_additional_secondary_config}}"},{key: "__es_port__", value: "{{es_port}}"}, {key: "__keyspace_name__", value: "{{content_keyspace_name}}"}, {key: "__collection_fullecar_disable__", value: "{{collection_fullecar_disable}}"},{key: "__max_iteration_count_for_samza_job__", value: "{{max_iteration_count_for_samza_job}}"},{key: "__cloud_storage_type__", value: "{{cloud_store}}"},{key: "__azure_storage_key__", value: "{{sunbird_public_storage_account_name}}"},{key: "__azure_storage_secret__", value: "{{sunbird_public_storage_account_key}}"},{key: "__azure_storage_container__", value: "{{azure_public_container}}"},{key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__plugin_media_base_url__", value: "{{plugin_media_base_url}}"}, {key: "__installation_id__", value: "{{instance_name}}"}, {key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__hierarchy_keyspace_name__", value: "{{hierarchy_keyspace_name}}"}, {key: "__composite_search_indexer_container_count__", value: "{{composite_search_indexer_container_count}}"},{key: "__cassandra_lp_connection__", value: "{{lp_cassandra_connection}}"}, {key: "__cassandra_lpa_connection__", value: "{{dp_cassandra_connection}}"}, {key: "__streaming_mime_type__", value: "{{streaming_mime_type}}"}, {key: "__cassandra_sunbird_connection__", value: "{{core_cassandra_connection}}"}, {key: "__cloud_upload_retry_count__", value: "{{cloud_upload_retry_count}}"}, {key: "__compositesearch_index_name__", value: "{{compositesearch_index_name}}"},{key: "__publish_pipeline_container_count__", value: "{{publish_pipeline_container_count}}"},{key: "__yarn_container_memory_mb__", value: "{{publish_yarn_container_memory_mb}}"},{key: "__youtube_api_key__", value: "{{youtube_api_key}}"},{key: "__kp_learning_service_base_url__", value: "{{kp_learning_service_base_url}}"},{key: "__sunbird_installation__", value: "{{sunbird_platform_installation}}"}, {key: "__search_lms_es_host__", value: "{{search_lms_es_host}}"},{key: "__dial_image_storage_container__", value: "{{dial_image_storage_container}}"},{key: "__dial_base_url__", value: "{{dial_base_url}}"},{key: "__learner_service_base_url__", value: "{{learner_service_base_url}}"},{key: "__cert_service_base_url__", value: "{{cert_service_base_url}}"},{key: "__certificate_base_path__", value: "{{certificate_base_path}}"},{key: "__kp_content_service_base_url__", value: "{{kp_content_service_base_url}}"},{key: "__kp_print_service_base_url__", value: "{{kp_print_service_base_url}}"},{key: "__cert_reg_service_base_url__", value: "{{cert_reg_service_base_url}}"},{key: "__kp_search_service_base_url__", value: "{{kp_search_service_base_url}}"},{key: "__samza_coordinator_replication_factor__", value: "{{samza_coordinator_replication_factor}}"},{key: "__samza_checkpoint_replication_factor__", value: "{{samza_checkpoint_replication_factor}}"},{key: "__course_batch_updater_container_count__", value: "{{course_batch_updater_container_count}}"},{key: "__course_certificate_generator_container_count__", value: "{{course_certificate_generator_container_count}}"},{key: "__course_progress_batch_size__", value: "{{course_progress_batch_size}}"},{key: "__itemset_generate_pdf__", value: "{{itemset_generate_pdf}}"},{key: "__auto_creator_container_count__", value: "{{auto_creator_container_count}}"},{key: "__content_streaming_enabled__", value: "{{content_streaming_enabled}}"},{key: "__lms_service_base_url__", value: "{{lms_service_base_url}}"},{key: "__mvc_search_indexer_container_count__", value: "{{mvc_search_indexer_container_count}}"}, {key: "__search_es7_host__", value: "{{search_es7_host}}"} , {key: "__ml-keywordapi__", value: "{{mlworkbench}}"},{key: "__auto_creator_artifact_allowed_sources__", value: "{{auto_creator_artifact_allowed_sources}}"},{key: "__publish_pipeline_task_opts__", value: "{{publish_pipeline_task_opts}}"},{key: "__auto_creator_g_service_acct_cred__", value: "{{auto_creator_gservice_acct_cred}}"},{key: "__certificate_pre_processor_container_count__", value: "{{certificate_pre_processor_container_count}}"},{key: "__master_category_validation_enabled__", value: "{{master_category_validation_enabled}}"}] + - [{key: "__yarn_host__", value: "{{__yarn_host__}}"}, {key: "__yarn_port__", value: "{{__yarn_port__}}"}, {key: "__env__", value: "{{env}}" }, {key: "__env_name__", value: "{{env_name}}" }, {key: "__zookeepers__", value: "{{zookeepers}}"}, {key: "__kafka_brokers__", value: "{{kafka_brokers}}"}, {key: "__delayInMilliSeconds__", value: "{{delayInMilliSeconds}}" }, {key: "__retryTimeInMilliSeconds__", value: "{{retryTimeInMilliSeconds}}" }, {key: "__bypass_reverse_search__", value: "{{bypass_reverse_search}}" }, {key: "__retryBackoffBaseInSeconds__", value: "{{retry_backoff_base_in_seconds}}" }, {key: "__retryLimit__", value: "{{retry_limit}}" }, {key: "__retryLimitEnable__", value: "{{retry_limit_enable}}" }, {key: "__google_api_key__", value: "{{google_api_key}}" }, {key: "__searchServiceEndpoint__", value: "{{search_service_endpoint}}" }, {key: "__objectDenormalizationAdditionalConfig__", value: "{{object_denormalization_additional_config}}" },{key: "__audit_es_host__", value: "{{audit_es_host}}"}, {key: "__search_es_host__", value: "{{search_es_host}}"}, {key: "__redis_host__", value: "{{redis_host}}"}, {key: "__dp_redis_host__", value: "{{dp_redis_host}}"}, {key: "__redis_port__", value: "{{redis_port}}"}, {key: "__environment_id__", value: "{{environment_id}}"}, {key: "__graph_passport_key__", value: "{{graph_passport_key}}"}, {key: "__lp_bolt_url__", value: "{{lp_bolt_url}}"}, {key: "__lp_bolt_read_url__", value: "{{lp_bolt_read_url}}"}, {key: "__lp_bolt_write_url__", value: "{{lp_bolt_write_url}}"}, {key: "__other_bolt_url__", value: "{{other_bolt_url}}"}, {key: "__other_bolt_read_url__", value: "{{other_bolt_read_url}}"}, {key: "__other_bolt_write_url__", value: "{{other_bolt_write_url}}"}, {key: "__mw_shard_id__", value: "{{mw_shard_id}}"}, {key: "__lp_url__", value: "{{lp_url}}"}, {key: "__cloud_storage_config_environment__", value: "{{cloud_storage_config_environment}}"}, {key: "__google_vision_tagging__", value: "{{google_vision_tagging}}"}, {key: "__lp_tmpfile_location__", value: "{{lp_tmpfile_location}}"}, {key: "__esRouterAdditionalConfig__", value: "{{es_router_additional_config}}"},{key: "__esRouterSecondaryAdditionalConfig__", value: "{{es_router_additional_secondary_config}}"},{key: "__es_port__", value: "{{es_port}}"}, {key: "__keyspace_name__", value: "{{content_keyspace_name}}"}, {key: "__collection_fullecar_disable__", value: "{{collection_fullecar_disable}}"},{key: "__max_iteration_count_for_samza_job__", value: "{{max_iteration_count_for_samza_job}}"},{key: "__cloud_storage_type__", value: "{{cloud_store}}"},{key: "__aws_access_key_id__",value: "{{aws_storage_key}}"},{key: "__aws_secret_access_key__",value: "{{aws_storage_secret}}"},{key: "__aws_storage_container__",value: "{{aws_storage_container}}"},{key: "__azure_storage_key__", value: "{{sunbird_public_storage_account_name}}"},{key: "__azure_storage_secret__", value: "{{sunbird_public_storage_account_key}}"},{key: "__azure_storage_container__", value: "{{azure_public_container}}"},{key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__plugin_media_base_url__", value: "{{plugin_media_base_url}}"}, {key: "__installation_id__", value: "{{instance_name}}"}, {key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__hierarchy_keyspace_name__", value: "{{hierarchy_keyspace_name}}"}, {key: "__composite_search_indexer_container_count__", value: "{{composite_search_indexer_container_count}}"},{key: "__cassandra_lp_connection__", value: "{{lp_cassandra_connection}}"}, {key: "__cassandra_lpa_connection__", value: "{{dp_cassandra_connection}}"}, {key: "__streaming_mime_type__", value: "{{streaming_mime_type}}"}, {key: "__cassandra_sunbird_connection__", value: "{{core_cassandra_connection}}"}, {key: "__cloud_upload_retry_count__", value: "{{cloud_upload_retry_count}}"}, {key: "__compositesearch_index_name__", value: "{{compositesearch_index_name}}"},{key: "__publish_pipeline_container_count__", value: "{{publish_pipeline_container_count}}"},{key: "__yarn_container_memory_mb__", value: "{{publish_yarn_container_memory_mb}}"},{key: "__youtube_api_key__", value: "{{youtube_api_key}}"},{key: "__kp_learning_service_base_url__", value: "{{kp_learning_service_base_url}}"},{key: "__sunbird_installation__", value: "{{sunbird_platform_installation}}"}, {key: "__search_lms_es_host__", value: "{{search_lms_es_host}}"},{key: "__dial_image_storage_container__", value: "{{dial_image_storage_container}}"},{key: "__dial_base_url__", value: "{{dial_base_url}}"},{key: "__learner_service_base_url__", value: "{{learner_service_base_url}}"},{key: "__cert_service_base_url__", value: "{{cert_service_base_url}}"},{key: "__certificate_base_path__", value: "{{certificate_base_path}}"},{key: "__kp_content_service_base_url__", value: "{{kp_content_service_base_url}}"},{key: "__kp_print_service_base_url__", value: "{{kp_print_service_base_url}}"},{key: "__cert_reg_service_base_url__", value: "{{cert_reg_service_base_url}}"},{key: "__kp_search_service_base_url__", value: "{{kp_search_service_base_url}}"},{key: "__samza_coordinator_replication_factor__", value: "{{samza_coordinator_replication_factor}}"},{key: "__samza_checkpoint_replication_factor__", value: "{{samza_checkpoint_replication_factor}}"},{key: "__course_batch_updater_container_count__", value: "{{course_batch_updater_container_count}}"},{key: "__course_certificate_generator_container_count__", value: "{{course_certificate_generator_container_count}}"},{key: "__course_progress_batch_size__", value: "{{course_progress_batch_size}}"},{key: "__itemset_generate_pdf__", value: "{{itemset_generate_pdf}}"},{key: "__auto_creator_container_count__", value: "{{auto_creator_container_count}}"},{key: "__content_streaming_enabled__", value: "{{content_streaming_enabled}}"},{key: "__lms_service_base_url__", value: "{{lms_service_base_url}}"},{key: "__mvc_search_indexer_container_count__", value: "{{mvc_search_indexer_container_count}}"}, {key: "__search_es7_host__", value: "{{search_es7_host}}"} , {key: "__ml-keywordapi__", value: "{{mlworkbench}}"},{key: "__auto_creator_artifact_allowed_sources__", value: "{{auto_creator_artifact_allowed_sources}}"},{key: "__publish_pipeline_task_opts__", value: "{{publish_pipeline_task_opts}}"},{key: "__auto_creator_g_service_acct_cred__", value: "{{auto_creator_gservice_acct_cred}}"}] + - "{{ (config_files|default({})).results|default([]) }}" + + +@@ -101,4 +101,4 @@ + args: + chdir: "{{samza_jobs_dir}}/extract/" + +- file: path={{samza_jobs_dir}} owner=hduser group=hadoop state=directory recurse=yes +- file: path={{samza_jobs_dir}} owner=hduser group=hadoop state=directory recurse=yes + - "{{ (config_files|default({})).results|default([]) }}" diff --git a/docs/domain_model_v2/definitions/content_definition.json b/docs/domain_model_v2/definitions/content_definition.json index 8e0a97a6c8..c831063254 100644 --- a/docs/domain_model_v2/definitions/content_definition.json +++ b/docs/domain_model_v2/definitions/content_definition.json @@ -271,7 +271,9 @@ "audio/ogg", "audio/webm", "audio/x-wav", - "audio/wav" + "audio/wav", + "application/json", + "application/quiz" ], "defaultValue": "application/vnd.ekstep.ecml-archive", "renderingHints": "{ 'inputType': 'select', 'order': 9 }", diff --git a/docs/domain_model_v2/definitions/content_image_definition.json b/docs/domain_model_v2/definitions/content_image_definition.json index d4d6b09c65..4ab316951f 100644 --- a/docs/domain_model_v2/definitions/content_image_definition.json +++ b/docs/domain_model_v2/definitions/content_image_definition.json @@ -271,7 +271,9 @@ "audio/ogg", "audio/webm", "audio/x-wav", - "audio/wav" + "audio/wav", + "application/json", + "application/quiz" ], "defaultValue": "application/vnd.ekstep.ecml-archive", "renderingHints": "{ 'inputType': 'select', 'order': 9 }", diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index f13eea269b..9a1caec95d 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -156,17 +156,20 @@ kind: Deployment metadata: name: {{ .Release.Name }}-taskmanager namespace: {{ .Values.namespace }} + spec: replicas: {{ .Values.taskmanager.replicas }} selector: matchLabels: app: flink component: {{ .Release.Name }}-taskmanager + template: metadata: labels: app: flink component: {{ .Release.Name }}-taskmanager + spec: volumes: - name: flink-config-volume diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 32b744c1df..8ca21f3df6 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -432,6 +432,7 @@ enrolment-reconciliation: jobmanager.memory.flink.size: {{ flink_job_names['enrolment-reconciliation'].jobmanager_memory }} taskmanager.memory.flink.size: {{ flink_job_names['enrolment-reconciliation'].taskmanager_memory }} taskmanager.numberOfTaskSlots: {{ flink_job_names['enrolment-reconciliation'].taskslots }} + env.java.opts: -Dlog4j2.formatMsgNoLookups=true parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 @@ -867,7 +868,7 @@ certificate-email-service: parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 - + content-publish: content-publish: |+ include file("/data/flink/conf/base-config.conf") @@ -966,7 +967,7 @@ content-publish: } nested.fields=["badgeAssertions", "targets", "badgeAssociations", "plugins", "me_totalTimeSpent", "me_totalPlaySessionCount", "me_totalTimeSpentInSec", "batches", "trackable", "credentials", "provider", "osMetadata", "actions", "transcripts", "accessibility","taxonomyPaths_v2", "competencies_v3", "test_competencies_v2","competencies_v2"] - restrict.objectTypes = ["EventSet", "Questionnaire", "Misconception", "FrameworkType", "EventSetImage", "EventImage"] + restrict.objectTypes = ["EventSet", "Questionnaire", "Misconception", "FrameworkType", "EventSetImage", "EventImage"] } cloud_storage { @@ -1064,7 +1065,7 @@ content-publish: aws_storage_key="{{ aws_storage_key }}" aws_storage_secret="{{ aws_storage_secret }}" aws_storage_container="{{ aws_storage_container }}" - + Course="Program" Course="Mandatory Course Goal" diff --git a/platform-jobs/samza/auto-creator/src/main/config/auto-creator.properties b/platform-jobs/samza/auto-creator/src/main/config/auto-creator.properties index b3f049495b..e5666efbe1 100644 --- a/platform-jobs/samza/auto-creator/src/main/config/auto-creator.properties +++ b/platform-jobs/samza/auto-creator/src/main/config/auto-creator.properties @@ -85,5 +85,9 @@ aws_storage_key=__aws_access_key_id__ aws_storage_secret=__aws_secret_access_key__ aws_storage_container=__aws_storage_container__ +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ diff --git a/platform-jobs/samza/auto-creator/src/main/config/local.auto-creator.properties.properties b/platform-jobs/samza/auto-creator/src/main/config/local.auto-creator.properties.properties index 45783a38b0..9491fc6a2c 100644 --- a/platform-jobs/samza/auto-creator/src/main/config/local.auto-creator.properties.properties +++ b/platform-jobs/samza/auto-creator/src/main/config/local.auto-creator.properties.properties @@ -72,3 +72,9 @@ aws_storage_key=__aws_access_key_id__ aws_storage_secret=__aws_secret_access_key__ aws_storage_container=__aws_storage_container__ + +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ + diff --git a/platform-jobs/samza/publish-pipeline/src/main/config/publish-pipeline.properties b/platform-jobs/samza/publish-pipeline/src/main/config/publish-pipeline.properties index fc8c47972b..046e2e8a22 100644 --- a/platform-jobs/samza/publish-pipeline/src/main/config/publish-pipeline.properties +++ b/platform-jobs/samza/publish-pipeline/src/main/config/publish-pipeline.properties @@ -86,6 +86,11 @@ cloud_storage.media.folder=media cloud_storage.ecar.folder=ecar_files cloud_storage.upload.url.ttl=600 +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ + # Media download configuration content.media.base.url=__content_media_base_url__ diff --git a/platform-jobs/samza/qrcode-image-generator/src/main/config/local.qrcode-image-generator.properties b/platform-jobs/samza/qrcode-image-generator/src/main/config/local.qrcode-image-generator.properties index 55f84fae02..f025402c82 100644 --- a/platform-jobs/samza/qrcode-image-generator/src/main/config/local.qrcode-image-generator.properties +++ b/platform-jobs/samza/qrcode-image-generator/src/main/config/local.qrcode-image-generator.properties @@ -55,6 +55,11 @@ aws_storage_secret=__aws_secret_access_key__ aws_storage_container=__aws_storage_container__ cloud_upload_retry_count=3 +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ + # Cassandra connection details cassandra.lp.connection=localhost:9042 cassandra.lpa.connection=localhost:9042 diff --git a/platform-jobs/samza/qrcode-image-generator/src/main/config/qrcode-image-generator.properties b/platform-jobs/samza/qrcode-image-generator/src/main/config/qrcode-image-generator.properties index b5954fd766..b8b030df33 100644 --- a/platform-jobs/samza/qrcode-image-generator/src/main/config/qrcode-image-generator.properties +++ b/platform-jobs/samza/qrcode-image-generator/src/main/config/qrcode-image-generator.properties @@ -55,6 +55,11 @@ aws_storage_secret=__aws_secret_access_key__ aws_storage_container=__aws_storage_container__ cloud_upload_retry_count=__cloud_upload_retry_count__ +cephs3_storage_key=__cephs3_access_key_id__ +cephs3_storage_secret=__cephs3_secret_access_key__ +cephs3_storage_container=__cephs3_storage_container__ +cephs3_storage_endpoint=__cephs3_storage_endpoint__ + # Cassandra connection details cassandra.lp.connection=__cassandra_lp_connection__ cassandra.lpa.connection=__cassandra_lpa_connection__ diff --git a/platform-jobs/samza/qrcode-image-generator/src/main/java/org/sunbird/jobs/samza/util/CloudStorageUtil.java b/platform-jobs/samza/qrcode-image-generator/src/main/java/org/sunbird/jobs/samza/util/CloudStorageUtil.java index 81493cae58..35abb3967f 100644 --- a/platform-jobs/samza/qrcode-image-generator/src/main/java/org/sunbird/jobs/samza/util/CloudStorageUtil.java +++ b/platform-jobs/samza/qrcode-image-generator/src/main/java/org/sunbird/jobs/samza/util/CloudStorageUtil.java @@ -26,11 +26,16 @@ public class CloudStorageUtil { if(StringUtils.equalsIgnoreCase(cloudStoreType, "azure")) { String storageKey = Platform.config.getString("azure_storage_key"); String storageSecret = Platform.config.getString("azure_storage_secret"); - storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret)); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.empty())); }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { String storageKey = Platform.config.getString("aws_storage_key"); String storageSecret = Platform.config.getString("aws_storage_secret"); - storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret)); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.empty())); + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + String storageKey = Platform.config.getString("cephs3_storage_key"); + String storageSecret = Platform.config.getString("cephs3_storage_secret"); + String endPoint = Platform.config.getString("cephs3_storage_endpoint"); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.apply(endPoint))); }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while initialising cloud storage"); } diff --git a/platform-modules/actors/src/main/java/org/sunbird/learning/util/CloudStore.java b/platform-modules/actors/src/main/java/org/sunbird/learning/util/CloudStore.java index 32643d61e8..942df9881a 100644 --- a/platform-modules/actors/src/main/java/org/sunbird/learning/util/CloudStore.java +++ b/platform-modules/actors/src/main/java/org/sunbird/learning/util/CloudStore.java @@ -28,11 +28,16 @@ public class CloudStore { if(StringUtils.equalsIgnoreCase(cloudStoreType, "azure")) { String storageKey = Platform.config.getString("azure_storage_key"); String storageSecret = Platform.config.getString("azure_storage_secret"); - storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret)); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.empty())); }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { String storageKey = Platform.config.getString("aws_storage_key"); String storageSecret = Platform.config.getString("aws_storage_secret"); - storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret)); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.empty())); + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + String storageKey = Platform.config.getString("cephs3_storage_key"); + String storageSecret = Platform.config.getString("cephs3_storage_secret"); + String endPoint = Platform.config.getString("cephs3_storage_endpoint"); + storageService = StorageServiceFactory.getStorageService(new StorageConfig(cloudStoreType, storageKey, storageSecret, Option.apply(endPoint))); }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while initialising cloud storage"); } @@ -47,6 +52,8 @@ public static String getContainerName() { return Platform.config.getString("azure_storage_container"); }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { return S3PropertyReader.getProperty("aws_storage_container"); + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + return S3PropertyReader.getProperty("cephs3_storage_container"); }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while getting container name"); } diff --git a/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java b/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java index 6939f89057..1c75e1d204 100644 --- a/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java +++ b/platform-modules/common/src/main/java/org/sunbird/common/util/HttpDownloadUtility.java @@ -11,6 +11,7 @@ import java.net.URL; import java.util.List; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.sunbird.common.Slug; import org.sunbird.telemetry.logger.TelemetryManager; @@ -24,7 +25,7 @@ public class HttpDownloadUtility { private static final int BUFFER_SIZE = 4096; - + /** * Downloads a file from a URL * @@ -34,6 +35,26 @@ public class HttpDownloadUtility { * path of the directory to save the file */ public static File downloadFile(String fileURL, String saveDir) { + try { + System.out.println("http utility called........."); + return download(fileURL, false); + } catch (Exception e) { + e.printStackTrace(); + TelemetryManager.error("Error! While Downloading File:"+ e.getMessage(), e); + } + TelemetryManager.warn("Something Went Wrong While Downloading the File '" + fileURL + "' returning 'null'. File url: "+ fileURL); + return null; + } + + /** + * Downloads a file from a URL + * + * @param fileURL + * HTTP URL of the file to be downloaded + * @param saveDir + * path of the directory to save the file + */ + public static File downloadFileOld(String fileURL, String saveDir) { HttpURLConnection httpConn = null; InputStream inputStream = null; FileOutputStream outputStream = null; @@ -158,5 +179,18 @@ public static String readFromUrl(String url) { TelemetryManager.log("Data read from url: " + sb.toString()); return sb.toString(); } + + public static File download(String artifactUrl, boolean extractFile) throws Exception { + if(StringUtils.isNotBlank(artifactUrl)){ + String localPath = "tmp/" + artifactUrl.trim() + File.separator; + String[] fileUrl = artifactUrl.split("/"); + String filename = fileUrl[fileUrl.length - 1]; + + File file = new File(localPath + filename); + FileUtils.copyURLToFile(new URL(artifactUrl), file); + return file; + } + return null; + } } \ No newline at end of file diff --git a/platform-modules/content-manager/src/main/java/org/sunbird/content/mimetype/mgr/impl/HTMLMimeTypeMgrImpl.java b/platform-modules/content-manager/src/main/java/org/sunbird/content/mimetype/mgr/impl/HTMLMimeTypeMgrImpl.java index f63ad30e7e..188e57bdae 100644 --- a/platform-modules/content-manager/src/main/java/org/sunbird/content/mimetype/mgr/impl/HTMLMimeTypeMgrImpl.java +++ b/platform-modules/content-manager/src/main/java/org/sunbird/content/mimetype/mgr/impl/HTMLMimeTypeMgrImpl.java @@ -5,6 +5,7 @@ import java.util.Map; import org.apache.commons.lang3.BooleanUtils; +import org.sunbird.common.Platform; import org.sunbird.common.dto.Response; import org.sunbird.common.exception.ResponseCode; import org.sunbird.content.common.ContentOperations; @@ -42,7 +43,9 @@ public class HTMLMimeTypeMgrImpl extends BaseMimeTypeManager implements IMimeTyp @Override public Response upload(String contentId, Node node, File uploadFile, boolean isAsync) { TelemetryManager.log("Calling Upload Content For Node ID: " + node.getIdentifier() + " Uploaded File :" + uploadFile); - if (hasGivenFile(uploadFile, "index.html")) { + Boolean indexHtmlValidation = (Platform.config.hasPath("indexHtmlValidation.env"))? Platform.config.getBoolean("indexHtmlValidation.env") : false; + Boolean flag = indexHtmlValidation ? hasGivenFile(uploadFile, "index.html") : true; + if (flag) { return uploadContentArtifact(contentId, node, uploadFile, false); } else { return ERROR(ContentErrorCodes.ERR_CONTENT_UPLOAD_FILE.name(), "Zip file doesn't have required files.", ResponseCode.CLIENT_ERROR); diff --git a/platform-modules/content-manager/src/main/java/org/sunbird/content/operation/finalizer/ReviewFinalizer.java b/platform-modules/content-manager/src/main/java/org/sunbird/content/operation/finalizer/ReviewFinalizer.java index 21111b918f..c2635bb900 100644 --- a/platform-modules/content-manager/src/main/java/org/sunbird/content/operation/finalizer/ReviewFinalizer.java +++ b/platform-modules/content-manager/src/main/java/org/sunbird/content/operation/finalizer/ReviewFinalizer.java @@ -270,7 +270,9 @@ private void validateResource(String collectionId){ notPublishedList.add(resource.getIdentifier()); } } - if(CollectionUtils.isNotEmpty(notPublishedList)){ + Boolean childPublishEnabled = Platform.config.hasPath("collection.only_published_children") ? + Platform.config.getBoolean("collection.only_published_children") : true; + if(!childPublishEnabled && CollectionUtils.isNotEmpty(notPublishedList)){ TelemetryManager.error("Collection: " + collectionId + " has unpublished resources:: " + notPublishedList); throw new ClientException("ERR_COLLECTION_WITH_UNPUBLISHED_RESOURCE", "Collection: " + collectionId + " has unpublished resources:: " + notPublishedList); } diff --git a/platform-modules/pom.xml b/platform-modules/pom.xml index 94cdc3d8ee..ad3a7e306b 100644 --- a/platform-modules/pom.xml +++ b/platform-modules/pom.xml @@ -19,7 +19,7 @@ 2.3.1 1.8 1.8 - 1.2.8 + 1.3.0-beta diff --git a/platform-modules/sunbird-framework/src/main/java/org/sunbird/framework/mgr/impl/FrameworkManagerImpl.java b/platform-modules/sunbird-framework/src/main/java/org/sunbird/framework/mgr/impl/FrameworkManagerImpl.java index 96caddf7cb..90bbb64430 100644 --- a/platform-modules/sunbird-framework/src/main/java/org/sunbird/framework/mgr/impl/FrameworkManagerImpl.java +++ b/platform-modules/sunbird-framework/src/main/java/org/sunbird/framework/mgr/impl/FrameworkManagerImpl.java @@ -201,6 +201,7 @@ public Response publishFramework(String frameworkId, String channelId) throws Ex } if (StringUtils.isNotBlank(frameworkId) && validateObject(frameworkId)) { generateFrameworkHierarchy(frameworkId); + FrameworkCache.delete(frameworkId); Response response = OK(); response.put(FrameworkEnum.publishStatus.name(), "Publish Operation for Framework Id '" + frameworkId + "' Started Successfully!"); diff --git a/platform-tools/spikes/content-tool/pom.xml b/platform-tools/spikes/content-tool/pom.xml index a485ae2693..22f0ceff9c 100644 --- a/platform-tools/spikes/content-tool/pom.xml +++ b/platform-tools/spikes/content-tool/pom.xml @@ -66,7 +66,7 @@ org.sunbird cloud-store-sdk - 1.2.5 + 1.3.0-beta diff --git a/platform-tools/spikes/content-tool/src/main/java/org/sunbird/content/tool/CloudStoreManager.java b/platform-tools/spikes/content-tool/src/main/java/org/sunbird/content/tool/CloudStoreManager.java index d4f0e23e60..34ac6cb41e 100644 --- a/platform-tools/spikes/content-tool/src/main/java/org/sunbird/content/tool/CloudStoreManager.java +++ b/platform-tools/spikes/content-tool/src/main/java/org/sunbird/content/tool/CloudStoreManager.java @@ -22,9 +22,10 @@ public class CloudStoreManager { protected String destStorageType = Platform.config.getString("destination.storage_type"); - protected BaseStorageService awsService = StorageServiceFactory.getStorageService(new StorageConfig("aws", Platform.config.getString("aws_storage_key"), Platform.config.getString("aws_storage_secret"))); - protected BaseStorageService azureService = StorageServiceFactory.getStorageService((new StorageConfig("azure", Platform.config.getString("azure_storage_key"), Platform.config.getString("azure_storage_secret")))); - private String cloudSrcBaseURL = Platform.config.getString("cloud.src.baseurl"); + protected BaseStorageService awsService = StorageServiceFactory.getStorageService(new StorageConfig("aws", Platform.config.getString("aws_storage_key"), Platform.config.getString("aws_storage_secret"), Option.empty())); + protected BaseStorageService azureService = StorageServiceFactory.getStorageService((new StorageConfig("azure", Platform.config.getString("azure_storage_key"), Platform.config.getString("azure_storage_secret"), Option.empty()))); + protected BaseStorageService cephService = StorageServiceFactory.getStorageService((new StorageConfig("cephs3", Platform.config.getString("cephs3_storage_key"), Platform.config.getString("cephs3_storage_secret"), Option.apply(Platform.config.getString("cephs3_storage_endpoint"))))); + private String cloudSrcBaseURL = Platform.config.getString("cloud.src.baseurl"); private String cloudDestBaseURL = Platform.config.getString("cloud.dest.baseurl"); @@ -239,6 +240,8 @@ public String getContainerName(String cloudStoreType) { return Platform.config.getString("azure_storage_container"); }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { return Platform.config.getString("aws_storage_container"); + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + return Platform.config.getString("cephs3_storage_container"); }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while getting container name"); } @@ -249,6 +252,8 @@ public BaseStorageService getcloudService(String cloudStoreType){ return azureService; }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "aws")) { return awsService; + }else if(StringUtils.equalsIgnoreCase(cloudStoreType, "cephs3")) { + return cephService; }else { throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while getting container name"); }