Update fetch-incident

.circleci/config.yml

@@ -1,4 +1,8 @@
 version: 2.1
+orbs:
+  python: circleci/python@<< pipeline.parameters.python-orb-version >>
+  node: circleci/node@<< pipeline.parameters.node-orb-version >>
+
 commands:
   set-instance-role-env-variable:
     description: "Sets INSTANCE_ROLE env variable into $BASH_ENV file"
@@ -40,7 +44,16 @@ parameters:
   gcs_market_bucket:
     type: string
     default: "marketplace-dist"
-
+  cache-version:
+    type: string
+    default: v1  # Change this parameter to clear cache.
+  python-orb-version:
+    type: string
+    default: "2.0.3" # disable-secrets-detection
+  node-orb-version:
+    type: string
+    default: "5.0.1" # disable-secrets-detection
+
 references:
   environment: &environment
     environment:
@@ -55,36 +68,48 @@ references:
       PULL_REQUEST_NUMBER: << pipeline.parameters.pr_number >>
       NIGHTLY_PARAMETER: << pipeline.parameters.nightly >>
       GCS_MARKET_BUCKET: << pipeline.parameters.gcs_market_bucket >>
+
+  install_build_dependencies: &install_build_dependencies
+    python/install-packages: 
+        pkg-manager: "poetry"
+        args: "--with ci"
+        cache-version: << pipeline.parameters.cache-version >>
+        pre-install-steps:
+          - run:
+              name: Check if pyproject.toml is consistent with poetry.lock
+              command: poetry lock --check
+
+  install_node_ci: &install_node_ci
+    node/install-packages:
+        cache-version: << pipeline.parameters.cache-version >>
+
 
   install_neo4j: &install_neo4j
     run:
       name: Install Neo4j
       command: |
-        # workaround for 402 Payment required issue: https://github.com/actions/runner-images/issues/1983
-        sudo rm -fv /etc/apt/sources.list.d/github_git-lfs.list
-        sudo rm -fv /etc/apt/sources.list.d/github_git-lfs.list.save
         wget -O - https://debian.neo4j.com/neotechnology.gpg.key | sudo apt-key add -
-        echo 'deb https://debian.neo4j.com stable 4.4' | sudo tee /etc/apt/sources.list.d/neo4j.list
+        echo 'deb https://debian.neo4j.com stable 5' | sudo tee /etc/apt/sources.list.d/neo4j.list
         sudo apt-get update
         apt list -a neo4j
         sudo apt-get install neo4j
         sudo chown -R circleci /var/log/neo4j
         sudo chown -R circleci /var/lib/neo4j
         sudo chown -R circleci /etc/neo4j
         mkdir -p /var/lib/neo4j/plugins
-        wget -O /var/lib/neo4j/plugins/apoc-4.4.0.8-all.jar https://github.com/neo4j-contrib/neo4j-apoc-procedures/releases/download/4.4.0.8/apoc-4.4.0.8-all.jar
+        wget -O /var/lib/neo4j/plugins/apoc-5.5.0-core.jar https://github.com/neo4j/apoc/releases/download/5.5.0/apoc-5.5.0-core.jar
         neo4j_conf_file="/etc/neo4j/neo4j.conf"
         sudo echo "dbms.security.procedures.unrestricted=apoc.*" >> $neo4j_conf_file
         sudo echo "dbms.security.procedures.allowlist=apoc.*" >> $neo4j_conf_file
         apoc_conf_file="/etc/neo4j/apoc.conf"
         sudo echo "apoc.export.file.enabled=true" > $apoc_conf_file
         sudo echo "apoc.import.file.enabled=true" >> $apoc_conf_file
         sudo echo "apoc.import.file.use_neo4j_config=true" >> $apoc_conf_file
-        neo4j-admin set-initial-password test
+        neo4j-admin dbms set-initial-password contentgraph
 
   container_config: &container_config
     docker:
-      - image: devdemisto/content-build:3.0.0.33772  # disable-secrets-detection
+      - image: devdemisto/content-build:3.0.0.49685  # disable-secrets-detection
         auth:
           username: $DOCKERHUB_USER
           password: $DOCKERHUB_PASSWORD
@@ -139,12 +164,6 @@ references:
         chmod +x ./Tests/scripts/*
         chmod +x ./Tests/Marketplace/*
 
-        echo "Checking if pyproject.toml is consistent with poetry.lock"
-        poetry lock --check
-
-        # we still need to install even if cached. if cached, `poetry` will handle it
-        echo "installing venv"
-        NO_HOOKS=1 .hooks/bootstrap
         source ./.venv/bin/activate
 
         # store in bash env so we load our venv in each step
@@ -159,10 +178,6 @@ references:
         npm --version
         demisto-sdk --version
 
-  restore_cache: &restore_cache
-    restore_cache:
-      key: virtualenv-venv-{{ checksum "pyproject.toml" }}-{{ checksum "poetry.lock" }}-{{ checksum "package-lock.json" }}
-
   remote_docker: &remote_docker
     setup_remote_docker:
       version: 20.10.6
@@ -303,13 +318,9 @@ jobs:
     <<: *environment
     steps:
       - checkout
-      - *restore_cache
+      - *install_build_dependencies
+      - *install_node_ci
       - *prepare_environment
-      - save_cache:
-          paths:
-            - .venv
-            - node_modules
-          key: virtualenv-venv-{{ checksum "pyproject.toml" }}-{{ checksum "poetry.lock" }}-{{ checksum "package-lock.json" }}
       - *get_contribution_pack
       - *persist_to_workspace
 
@@ -323,9 +334,10 @@ jobs:
     steps:
       - *attach_workspace
       - *remote_docker
-      - *restore_cache
-      - *prepare_environment
+      - *install_build_dependencies
+      - *install_node_ci
       - *install_neo4j
+      - *prepare_environment
       - *infrastructure_testing
       - *run_unit_testing_and_lint
       - *generate_coverage_reports
@@ -339,9 +351,10 @@ jobs:
     <<: *environment
     steps:
       - *attach_workspace
-      - *restore_cache
-      - *prepare_environment
+      - *install_build_dependencies
+      - *install_node_ci
       - *install_neo4j
+      - *prepare_environment
       - *secrets
       - *validate_files_and_yaml
       - run:
@@ -370,6 +383,8 @@ jobs:
 
             python3 Tests/Marketplace/validate_landing_page_sections.py -i $UNZIP_PATH
       - *store_artifacts
+      - store_artifacts:
+          path: $ARTIFACTS_FOLDER
 
 
 

.devcontainer/createCommand.sh

@@ -20,5 +20,5 @@ echo "PYTHONPATH=""$path"":$PYTHONPATH" >> .env
 echo "MYPYPATH=""$path"":$MYPYPATH" >> .env
 
 echo "Setting up content dependencies"
-poetry install --with native
+poetry install
 npm install

.devcontainer/devcontainer.json

@@ -1,4 +1,4 @@
-// Development container for Content. Poetry hash dba719d35b89ee660a81ae1d1b15c1d0dc8125df45af8cfba5d530e936341491
+// Development container for Content. Poetry hash 97b4446a28329c24352fcb41ca8570403da0ae3f7434121ab73b1ff83fdec395
 {
 	"name": "XSOAR Content",
 	"build": {

.github/CODEOWNERS

@@ -25,6 +25,8 @@
 /Tests/scripts/prepare_content_packs_for_testing.sh @yaakovpraisler
 /Utils/trigger_test_upload_flow.sh @yaakovpraisler
 /Utils/trigger_upload_packs_to_production.sh @yaakovpraisler
+/Utils/should_trigger_test_upload.sh @yaakovpraisler
+/Utils/test_upload_flow/* @yaakovpraisler
 
 # Test Collection
 /Tests/scripts/collect_tests @dorschw

.github/workflows/review-release-notes.yml

@@ -1,32 +1,38 @@
 name: Review Release Notes
 on: pull_request
+
+env:
+  CHANGED_FILES_DELIMITER: ";"
+
 jobs:
   release_notes_review:
     runs-on: ubuntu-latest
     if: github.repository == 'demisto/content'
     steps:
+
       - name: Checkout
         uses: actions/checkout@v3
         with:
           fetch-depth: 2
+
       - name: Get changed files
         id: changed-files
-        uses: tj-actions/changed-files@v18.7
+        uses: tj-actions/changed-files@v35.1.0
         with:
-          separator: ';'
-      - name: Setup Python
-        uses: actions/setup-python@v3
-        with:
-          python-version: '3.9'
-      - name: Setup Poetry
-        uses: Gr1N/setup-poetry@v7
-      - name: Install Python Dependencies
+          separator: ${{ env.CHANGED_FILES_DELIMITER }}
+          files: |
+            Packs/**/ReleaseNotes/**
+          since_last_remote_commit: false
+
+      - name: Install SDK
+        if: ${{ steps.changed-files.outputs.all_changed_files }}
         run: |
-          poetry install --with ci
+          python3 -m pip install demisto-sdk
+
       - name: Review Release Notes
+        if: ${{ steps.changed-files.outputs.all_changed_files }}
         run: |
-          changed_files="${{ steps.changed-files.outputs.all_changed_files }}"
-          delim=';'
-          echo "Starting docs review"
-          poetry run python Utils/github_workflow_scripts/run_docs_review.py --changed_files "$changed_files" --delimiter $delim
-          echo "Finished docs review successfully"
+          release_notes="${{ steps.changed-files.outputs.all_changed_files }}"
+          echo "Release notes found in PR: ${release_notes}. Starting docs review..."
+
+          Utils/github_workflow_scripts/run_docs_review.py --changed_files "$release_notes" --delimiter "${{ env.CHANGED_FILES_DELIMITER }}"

.gitignore

@@ -69,3 +69,7 @@ neo4j-data
 
 # MicroSocks (Potentially harmful)
 Packs/Whois/Integrations/Whois/test_data/microsocks
+
+# log files
+demisto_sdk_debug.log
+demisto_sdk_debug.log.*

.gitlab/ci/.gitlab-ci.yml

@@ -1,6 +1,6 @@
 
 default:
-  image: docker-io.art.code.pan.run/devdemisto/gitlab-content-ci:1.0.0.41284
+  image: docker-io.art.code.pan.run/devdemisto/gitlab-content-ci:1.0.0.49555
   artifacts:
     expire_in: 30 days
     paths:

.gitlab/ci/bucket-upload.yml

@@ -34,7 +34,7 @@
     - |
       if [[ $TEST_UPLOAD == "false" ]]; then
         section_start "Upload content graph CSVs to GCP" --collapsed
-        gsutil cp $ARTIFACTS_FOLDER/content_graph/$MARKETPLACE_VERSION.zip "gs://$GCS_MARKET_BUCKET_DEV/content_graph/$MARKETPLACE_VERSION.zip"
+        # gsutil cp $ARTIFACTS_FOLDER/content_graph/$MARKETPLACE_VERSION.zip "gs://$GCS_MARKET_BUCKET_DEV/content_graph/$MARKETPLACE_VERSION.zip"
         section_end "Upload content graph CSVs to GCP"
       fi
 
@@ -192,7 +192,7 @@ install-packs-in-server-master:
     ENV_RESULTS_PATH: "${ARTIFACTS_FOLDER_MPV2}/env_results.json"
     INSTANCE_CREATED: "true"
     TIME_TO_LIVE: ""
-    GCS_LOCKS_PATH: "gs://xsoar-ci-artifacts/content-locks-xsiam"
+    GCS_LOCKS_PATH: "content-locks/locks-xsiam-ga"
   extends:
     - .bucket-upload-rule
     - .default-job-settings
@@ -230,7 +230,7 @@ install-packs-in-server-master:
   after_script:
     - echo "Job finished, removing lock file"
     - gcloud auth activate-service-account --key-file="$GCS_ARTIFACTS_KEY" > auth.out 2>&1
-    - gsutil rm "gs://xsoar-ci-artifacts/content-locks-xsiam/*-lock-$CI_JOB_ID"
+    - gsutil rm "gs://xsoar-ci-artifacts/content-locks-xsiam/machines_locks/*-lock-$CI_JOB_ID"
     - echo "Finished removing lock file"
 
 
@@ -528,6 +528,7 @@ sync-buckets-between-projects:
     - .bucket-upload-rule
   stage: upload-to-marketplace
   needs: ["upload-packs-to-marketplace", "upload-packs-to-marketplace-v2", "upload-packs-to-xpanse-marketplace"]
+  when: always
   script:
     - |
       if [[ -z "$GCS_XSOAR_CONTENT_DEV_KEY" ]] || [[ -z "$GCS_XSOAR_CONTENT_PROD_KEY" ]]; then

.gitlab/ci/global.yml

@@ -169,7 +169,7 @@
     - *install_ssh_keys
     - section_start "Build Parameters"
     - set | grep -E "^NIGHTLY=|^INSTANCE_TESTS=|^SERVER_BRANCH_NAME=|^ARTIFACT_BUILD_NUM=|^DEMISTO_SDK_NIGHTLY=|^TIME_TO_LIVE=|^CONTRIB_BRANCH=|^FORCE_PACK_UPLOAD=|^PACKS_TO_UPLOAD=|^BUCKET_UPLOAD=|^STORAGE_BASE_PATH=|^OVERRIDE_ALL_PACKS=|^GCS_MARKET_BUCKET=|^GCS_MARKET_V2_BUCKET=|^GCS_MARKET_XPANSE_BUCKET=|^SLACK_CHANNEL=|^NVM_DIR=|^NODE_VERSION=|^PATH=|^ARTIFACTS_FOLDER=|^ENV_RESULTS_PATH=|^LAST_UPLOAD_COMMIT="
-    - neo4j-admin set-initial-password test
+    - neo4j-admin dbms set-initial-password contentgraph
     - neo4j start
     - python --version
     - python2 --version
@@ -262,7 +262,7 @@
           fi
         fi
         if [[ -f $ARTIFACTS_FOLDER/coverage_report/.coverage ]]; then
-          if [[ "$CI_PIPELINE_SOURCE" == "schedule" ||  -n "${NIGHTLY}" || -n "${BUCKET_UPLOAD}" || -n "${DEMISTO_SDK_NIGHTLY}" ]]; then
+          if [[ "$CI_PIPELINE_SOURCE" == "schedule" ||   -n "$SHOULD_LINT_ALL"  ||  -n "${NIGHTLY}" || -n "${BUCKET_UPLOAD}" || -n "${DEMISTO_SDK_NIGHTLY}" ]]; then
             demisto-sdk coverage-analyze -i $ARTIFACTS_FOLDER/coverage_report/.coverage --report-dir $ARTIFACTS_FOLDER/coverage_report --report-type all --allowed-coverage-degradation-percentage 100
             if [[ -n "${NIGHTLY}" && "$CI_COMMIT_BRANCH" == "master" ]]; then
               python3 Utils/upload_code_coverage_report.py --service_account $GCS_MARKET_KEY --source_file_name $ARTIFACTS_FOLDER/coverage_report/coverage.json --minimal_file_name $ARTIFACTS_FOLDER/coverage_report/coverage-min.json
@@ -297,8 +297,8 @@
     - section_end "Copy Tests To Artifact Folder"
     - section_start "Validate Files and Yaml"
     - |
-      if [[ -n $FORCE_BUCKET_UPLOAD || -n $BUCKET_UPLOAD ]] && [[ "$(echo "$GCS_MARKET_BUCKET" | tr '[:upper:]' '[:lower:]')" != "marketplace-dist" ]] && [[ $CI_COMMIT_BRANCH != "master" ]]; then
-        echo "Skipping the -Validate Files and Yaml- step when uploading to a test bucket."
+      if [[ -n $FORCE_BUCKET_UPLOAD ]]; then
+        echo "Skipping the -Validate Files and Yaml- step when force uploading to a bucket."
       else
         ./Tests/scripts/linters_runner.sh
         ./Tests/scripts/validate.sh