diff --git a/.github/ci/files/composer.json b/.github/ci/files/composer.json deleted file mode 100644 index 94074c67..00000000 --- a/.github/ci/files/composer.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "type": "project", - "config": { - "optimize-autoloader": true, - "sort-packages": true, - "discard-changes": true, - "process-timeout": 0, - "allow-plugins": { - "php-http/discovery": false, - "symfony/runtime": true - } - }, - "prefer-stable": true, - "minimum-stability": "dev", - "require": { - "pimcore/generic-data-index-bundle": "@dev" - }, - "require-dev": { - "codeception/codeception": "^5.3.2", - "codeception/phpunit-wrapper": "^9", - "codeception/module-asserts": "^2", - "codeception/module-symfony": "^3.1.1", - "phpunit/phpunit": "10.2.7" - }, - "autoload": { - "psr-4": { - "App\\": "src/", - "Pimcore\\Model\\DataObject\\": "var/classes/DataObject" - }, - "files": [ - "kernel/Kernel.php" - ] - }, - "extra": { - "symfony-assets-install": "relative" - }, - "repositories": { - "bundles": { - "type": "path", - "url": "./bundles/*/*", - "options": { - "symlink": true - } - } - } -} diff --git a/.github/ci/files/config/config.yaml b/.github/ci/files/config/config.yaml index 459522ee..f2c5c29e 100644 --- a/.github/ci/files/config/config.yaml +++ b/.github/ci/files/config/config.yaml @@ -1,3 +1,2 @@ imports: - - { resource: services.yaml } - { resource: system.yaml } \ No newline at end of file diff --git a/.github/ci/files/config/doctrine.yaml b/.github/ci/files/config/doctrine.yaml new file mode 100644 index 00000000..04b5bbc7 --- /dev/null +++ b/.github/ci/files/config/doctrine.yaml @@ -0,0 +1,12 @@ +doctrine: + orm: + entity_managers: + default: + connection: default + mappings: + PimcoreGenericDataIndexBundle: + type: attribute + is_bundle: true + dir: "src/Entity" + prefix: Pimcore\Bundle\GenericDataIndexBundle\Entity + alias: PimcoreGenericDataIndex diff --git a/.github/ci/files/config/doctrine_migrations.yaml b/.github/ci/files/config/doctrine_migrations.yaml new file mode 100644 index 00000000..df4fe00f --- /dev/null +++ b/.github/ci/files/config/doctrine_migrations.yaml @@ -0,0 +1,3 @@ +doctrine_migrations: + migrations_paths: + 'Pimcore\Bundle\GenericDataIndexBundle\Migrations': '@PimcoreGenericDataIndexBundle/src/Migrations' \ No newline at end of file diff --git a/.github/ci/files/config/packages/test/config.yaml b/.github/ci/files/config/packages/test/config.yaml index 1172b0a3..7421ea0f 100644 --- a/.github/ci/files/config/packages/test/config.yaml +++ b/.github/ci/files/config/packages/test/config.yaml @@ -27,6 +27,7 @@ doctrine: bit: boolean parameters: + secret: ThisTokenIsNotSoSecretChangeIt pimcore_test.db.dsn: '%env(PIMCORE_TEST_DB_DSN)%' env(PIMCORE_TEST_DB_DSN): ~ pimcore.encryption.secret: '%env(PIMCORE_ENCRYPTION_SECRET)%' @@ -40,9 +41,7 @@ pimcore_elasticsearch_client: es_clients: default: hosts: ['%env(PIMCORE_ELASTIC_SEARCH_HOST)%'] - username: 'elastic' - password: 'somethingsecret' - logger_channel: 'pimcore.elasicsearch' + logger_channel: 'pimcore.elasticsearch' ssl_verification: false pimcore: diff --git a/.github/ci/files/config/pimcore/config.yaml b/.github/ci/files/config/pimcore/config.yaml new file mode 100644 index 00000000..584a1dce --- /dev/null +++ b/.github/ci/files/config/pimcore/config.yaml @@ -0,0 +1,194 @@ +imports: + - { resource: messenger.yaml } + +pimcore_generic_data_index: + index_service: + search_settings: + list_page_size: 60 + list_max_filter_options: 500 + search_analyzer_attributes: + text: + fields: + analyzed_ngram: + type: text + analyzer: generic_data_index_ngram_analyzer + search_analyzer: generic_data_index_whitespace_analyzer + analyzed: + type: text + analyzer: standard + search_analyzer: generic_data_index_whitespace_analyzer + system_fields_settings: + general: + id: + type: long + elementType: + type: keyword + parentId: + type: long + creationDate: + type: date + modificationDate: + type: date + type: + type: keyword + key: + type: keyword + fields: + analyzed_ngram: + type: text + analyzer: generic_data_index_ngram_analyzer + search_analyzer: generic_data_index_whitespace_analyzer + analyzed: + type: text + analyzer: standard + search_analyzer: generic_data_index_whitespace_analyzer + sort: + type: keyword + normalizer: generic_data_index_sort_normalizer + path: + type: text + analyzer: generic_data_index_path_analyzer + fields: + keyword: + type: keyword + sort: + type: keyword + normalizer: generic_data_index_sort_normalizer + fullPath: + type: text + analyzer: generic_data_index_path_analyzer + fields: + keyword: + type: keyword + sort: + type: keyword + normalizer: generic_data_index_sort_normalizer + pathLevels: + type: nested + properties: + level: + type: integer + name: + type: keyword + pathLevel: + type: integer + tags: + type: integer + parentTags: + type: integer + thumbnail: + type: keyword + userOwner: + type: integer + userModification: + type: integer + lock: + type: keyword + hasWorkflowWithPermissions: + type: boolean + dependencies: + type: object + properties: + asset: + type: long + document: + type: long + object: + type: long + document: + published: + type: boolean + index: + type: integer + controller: + type: keyword + template: + type: keyword + contentMainDocumentId: + type: integer + supportsContentMain: + type: boolean + missingRequiredEditable: + type: boolean + staticGeneratorEnabled: + type: boolean + staticGeneratorLifetime: + type: integer + title: + type: keyword + description: + type: keyword + prettyUrl: + type: keyword + sourceId: + type: integer + propertiesFromSource: + type: boolean + childrenFromSource: + type: boolean + subject: + type: keyword + from: + type: keyword + replyTo: + type: keyword + to: + type: keyword + cc: + type: keyword + bcc: + type: keyword + internal: + type: integer + internalType: + type: keyword + direct: + type: keyword + linktype: + type: keyword + href: + type: keyword + data_object: + published: + type: boolean + classname: + type: keyword + fields: + sort: + type: keyword + normalizer: generic_data_index_sort_normalizer + classDefinitionIcon: + type: keyword + index: + type: integer + childrenSortBy: + type: keyword + childrenSortOrder: + type: keyword + asset: + mimetype: + type: keyword + fileSize: + type: long + thumbnail: + type: keyword + imageThumbnail: + type: keyword + width: + type: integer + height: + type: integer + duration: + type: float + text: + type: keyword + ignore_above: 256 + fields: + analyzed_ngram: + type: text + analyzer: generic_data_index_ngram_analyzer + search_analyzer: generic_data_index_whitespace_analyzer + analyzed: + type: text + analyzer: standard + search_analyzer: generic_data_index_whitespace_analyzer \ No newline at end of file diff --git a/.github/ci/files/config/pimcore/default_index_settings.yaml b/.github/ci/files/config/pimcore/default_index_settings.yaml new file mode 100644 index 00000000..968f5a2a --- /dev/null +++ b/.github/ci/files/config/pimcore/default_index_settings.yaml @@ -0,0 +1,48 @@ +index: + mapping: + nested_fields: + limit: 200 + total_fields: + limit: 100000 +number_of_shards: 1 +number_of_replicas: 0 +max_ngram_diff: 30 +max_result_window: 10000000 +analysis: + analyzer: + generic_data_index_ngram_analyzer: + tokenizer: generic_data_index_ngram_tokenzier + filter: + - lowercase + generic_data_index_whitespace_analyzer: + tokenizer: generic_data_index_whitespace_tokenzier + filter: + - lowercase + generic_data_index_path_analyzer: + tokenizer: generic_data_index_path_tokenizer + normalizer: + generic_data_index_sort_normalizer: + type: custom + filter: + - lowercase + generic_data_index_sort_truncate_normalizer: + type: custom + char_filter: + - generic_data_index_sort_truncate + filter: + - lowercase + tokenizer: + generic_data_index_ngram_tokenzier: + type: ngram + min_gram: 3 + max_gram: 25 + token_chars: [ letter, digit ] + generic_data_index_whitespace_tokenzier: + type: whitespace + generic_data_index_path_tokenizer: + type: "path_hierarchy" + char_filter: + generic_data_index_sort_truncate: + type: pattern_replace + pattern: ^(.{256})(.*)$ + replacement: $1 diff --git a/.github/ci/files/config/pimcore/messenger.yaml b/.github/ci/files/config/pimcore/messenger.yaml new file mode 100644 index 00000000..cc8b2fd4 --- /dev/null +++ b/.github/ci/files/config/pimcore/messenger.yaml @@ -0,0 +1,19 @@ +framework: + messenger: + enabled: true + transports: + pimcore_generic_data_index_queue: + dsn: 'doctrine://default?queue_name=pimcore_generic_data_index_queue' + failure_transport: pimcore_generic_data_index_failed + pimcore_generic_data_index_sync: 'sync://' + pimcore_generic_data_index_failed: 'doctrine://default?queue_name=pimcore_generic_data_index_failed' + routing: + Pimcore\Bundle\GenericDataIndexBundle\Message\IndexUpdateQueueMessage: pimcore_generic_data_index_queue + Pimcore\Bundle\GenericDataIndexBundle\Message\DispatchQueueMessagesMessage: pimcore_generic_data_index_queue + Pimcore\Bundle\GenericDataIndexBundle\Message\UpdateLanguageSettingsMessage: pimcore_generic_data_index_queue + Pimcore\Bundle\GenericDataIndexBundle\Message\UpdateClassMappingMessage: pimcore_generic_data_index_queue + Pimcore\Bundle\GenericDataIndexBundle\Message\EnqueueRelatedIdsMessage: pimcore_generic_data_index_queue + buses: + messenger.bus.pimcore-generic-data-index: + middleware: + - 'Pimcore\Bundle\GenericDataIndexBundle\Service\Messenger\Middleware\CollectGarbageMiddleware' diff --git a/.github/ci/files/config/pimcore/routing.yaml b/.github/ci/files/config/pimcore/routing.yaml new file mode 100644 index 00000000..fa7a8c9b --- /dev/null +++ b/.github/ci/files/config/pimcore/routing.yaml @@ -0,0 +1,6 @@ +pimcore_generic_data_index: + resource: "../../src/Controller/" + type: attribute + prefix: /pimcore_generic_data_index + options: + expose: true diff --git a/.github/ci/files/config/services.yaml b/.github/ci/files/config/services.yaml deleted file mode 100644 index afa4b7e5..00000000 --- a/.github/ci/files/config/services.yaml +++ /dev/null @@ -1,16 +0,0 @@ -parameters: - secret: ThisTokenIsNotSoSecretChangeIt - -services: - _defaults: - autowire: true - autoconfigure: true - public: false - - # - # INSTALLER - # - Pimcore\Bundle\GenericDataIndexBundle\Installer: - public: true - arguments: - $bundle: "@=service('kernel').getBundle('PimcoreGenericDataIndexBundle')" \ No newline at end of file diff --git a/.github/ci/files/config/services_test.yaml b/.github/ci/files/config/services_test.yaml index 82037b2f..bffc08a4 100644 --- a/.github/ci/files/config/services_test.yaml +++ b/.github/ci/files/config/services_test.yaml @@ -34,4 +34,4 @@ services: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Asset\Aggregation\FileSizeAggregationServiceInterface: class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Asset\Aggregation\FileSizeAggregationService - public: true \ No newline at end of file + public: true diff --git a/.github/ci/files/kernel/Kernel.php b/.github/ci/files/kernel/Kernel.php index 69921b00..bf19282e 100644 --- a/.github/ci/files/kernel/Kernel.php +++ b/.github/ci/files/kernel/Kernel.php @@ -1,19 +1,45 @@ getProjectDir() . '/{config}'; + $container->import($configDir . '/{packages}/*.{php,yaml}'); + $container->import($configDir . '/{packages}/' . $this->environment . '/*.{php,yaml}'); + + // Skip loading config/services.yaml — it is the bundle's own service config + // and is already loaded by PimcoreGenericDataIndexExtension::load(). + // Only load the environment-specific services file (e.g., services_test.yaml). + $container->import($configDir . '/{services}_' . $this->environment . '.yaml'); + } } diff --git a/.github/ci/scripts/setup-pimcore-environment-functional-tests.sh b/.github/ci/scripts/setup-pimcore-environment-functional-tests.sh deleted file mode 100755 index 00e1bd81..00000000 --- a/.github/ci/scripts/setup-pimcore-environment-functional-tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -.github/ci/scripts/setup-pimcore-environment.sh - -# Add Elasticsearch config if specified -if [ "$1" == "elasticsearch" ]; then - CONFIG_FILE_PATH="config/packages/test/config.yaml" - echo -e "\n# Added by functional test script\npimcore_generic_data_index:\n index_service:\n client_params:\n client_type: 'elasticsearch'" >> "$CONFIG_FILE_PATH" -fi - -cp .github/ci/files/composer.json . -cp bundles/pimcore/generic-data-index-bundle/codeception.dist.yml . \ No newline at end of file diff --git a/.github/ci/scripts/setup-pimcore-environment.sh b/.github/ci/scripts/setup-pimcore-environment.sh index ac32e84e..f10d688e 100755 --- a/.github/ci/scripts/setup-pimcore-environment.sh +++ b/.github/ci/scripts/setup-pimcore-environment.sh @@ -5,10 +5,24 @@ set -eu mkdir -p var/config mkdir -p bin -cp .github/ci/files/.env . cp -r .github/ci/files/config/. config cp -r .github/ci/files/templates/. templates cp -r .github/ci/files/bin/console bin/console chmod 755 bin/console cp -r .github/ci/files/kernel/. kernel -cp -r .github/ci/files/public/. public \ No newline at end of file +cp -r .github/ci/files/public/. public + +# Detect Elasticsearch: if the ES host is reachable, switch client_type to elasticsearch +if [ -n "${PIMCORE_ELASTIC_SEARCH_HOST:-}" ]; then + if curl -s --max-time 5 "http://${PIMCORE_ELASTIC_SEARCH_HOST}" > /dev/null 2>&1; then + echo "Elasticsearch detected at ${PIMCORE_ELASTIC_SEARCH_HOST}, switching client_type to elasticsearch" + cat >> config/packages/test/config.yaml <<'ESCONFIG' + +# Elasticsearch client type (auto-detected by setup script) +pimcore_generic_data_index: + index_service: + client_params: + client_type: 'elasticsearch' +ESCONFIG + fi +fi \ No newline at end of file diff --git a/.github/workflows/elastic-search-codeception.yaml b/.github/workflows/elastic-search-codeception.yaml index b1495507..f9affc52 100644 --- a/.github/workflows/elastic-search-codeception.yaml +++ b/.github/workflows/elastic-search-codeception.yaml @@ -1,9 +1,8 @@ name: "Codeception Tests with Elasticsearch" on: - # Enable Later. schedule: - - cron: '0 3 * * 1,3,5' + - cron: "0 3 * * 1,3,5" workflow_dispatch: pull_request: branches: @@ -14,110 +13,77 @@ on: branches: - "[0-9]+.[0-9]+" - "[0-9]+.x" - env: PIMCORE_PROJECT_ROOT: ${{ github.workspace }} - APP_ENV: test - PIMCORE_TEST: 1 - PIMCORE_TEST_DB_DSN: "mysql://root@127.0.0.1:33006/pimcore_test" - PIMCORE_ELASTIC_SEARCH_HOST: "localhost:5300" - CODECEPTION_BUNDLE_PATH: "bundles/pimcore/generic-data-index-bundle/" - PIMCORE_INSTANCE_IDENTIFIER: ${{ secrets.PIMCORE_CI_INSTANCE_IDENTIFIER }} - PIMCORE_ENCRYPTION_SECRET: ${{ secrets.PIMCORE_CI_ENCRYPTION_SECRET }} - PIMCORE_PRODUCT_KEY: ${{ secrets.PIMCORE_CI_PRODUCT_KEY }} - -jobs: - codeception-tests: - name: "Codeception tests" - runs-on: "ubuntu-latest" - continue-on-error: ${{ matrix.experimental }} - strategy: - matrix: - include: - - { php-version: "8.4", dependencies: "highest", pimcore_version: "12.x-dev as 12.99.9", experimental: true, search_engine: "elasticsearch"} - - services: - mariadb: - image: "mariadb:10.11" - ports: - - 33006:3306 - env: - MYSQL_ALLOW_EMPTY_PASSWORD: yes + PRIVATE_REPO: ${{ github.event.repository.private }} - elastic: - image: elasticsearch:8.5.3 - ports: - - 5300:9200 - env: - discovery.type: "single-node" - ES_JAVA_OPTS: "-Xms1g -Xmx1g" - xpack.security.enabled: "true" - xpack.security.authc.anonymous.roles: "superuser,kibana_admin,kibana_system,kibana_user" - ELASTIC_USERNAME: "elastic" - ELASTIC_PASSWORD: "somethingsecret" +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true +jobs: + setup-matrix: + runs-on: ubuntu-latest + outputs: + php_versions: ${{ steps.parse-php-versions.outputs.php_versions }} + matrix: ${{ steps.set-matrix.outputs.matrix }} + private_repo: ${{ env.PRIVATE_REPO }} steps: - - name: "Checkout code" - uses: "actions/checkout@v4" - with: - path: "bundles/pimcore/generic-data-index-bundle" - - - name: "Copy .github directory" - env: - REQUIRE_ADMIN_BUNDLE: "${{ matrix.require_admin_bundle }}" - run: | - cp -R bundles/pimcore/generic-data-index-bundle/.github .github - - - name: "Install PHP" - uses: "shivammathur/setup-php@v2" - with: - coverage: xdebug - ini-values: display_errors=On, display_startup_errors=On, error_reporting=32767 - php-version: "${{ matrix.php-version }}" - - - name: Verify MariaDB connection - run: | - cp .github/ci/files/.my.cnf ~/.my.cnf - while ! mysqladmin ping --silent; do - sleep 1 - done + - name: Checkout code + uses: actions/checkout@v4 - - name: "Setup Pimcore environment" - env: - REQUIRE_ADMIN_BUNDLE: "${{ matrix.require_admin_bundle }}" - run: | - mysql -e "CREATE DATABASE pimcore_test CHARSET=utf8mb4;" - cp -R bundles/pimcore/generic-data-index-bundle/tests tests - mkdir src - chmod 755 .github/ci/scripts/setup-pimcore-environment-functional-tests.sh - .github/ci/scripts/setup-pimcore-environment-functional-tests.sh "${{ matrix.search_engine }}" + - name: Checkout reusable workflow repo + uses: actions/checkout@v4 + with: + repository: pimcore/workflows-collection-public + ref: main + path: reusable-workflows - - name: "Wait for Elasticsearch to be up" + - name: Parse PHP versions from composer.json + id: parse-php-versions run: | - if [ "${{ matrix.search_engine }}" == "elasticsearch" ]; then - echo "Waiting for Elasticsearch to be ready..." - until curl -s -u elastic:somethingsecret http://localhost:5300/_cluster/health | grep -q '"status":"green"'; do - echo "Waiting... Elasticsearch is not yet available." - sleep 5 - done - echo "Elasticsearch is up and running!" + if [ -f composer.json ]; then + php_versions=$(jq -r '.require.php' composer.json | grep -oP '\d+\.\d+' | tr '\n' ',' | sed 's/,$//') + if [ -z "$php_versions" ]; then + echo "php_versions=default" >> "$GITHUB_OUTPUT" + else + echo "php_versions=$php_versions" >> "$GITHUB_OUTPUT" + fi else - echo "Using OpenSearch for testing, skipping Elasticsearch wait." - fi + exit 1 + fi - - name: "Update Pimcore version" - env: - PIMCORE_VERSION: "${{ matrix.pimcore_version }}" + - name: Set up matrix JSON + id: set-matrix run: | - if [ ! -z "$PIMCORE_VERSION" ]; then - composer require --no-update pimcore/pimcore:"${PIMCORE_VERSION}" - fi - - - name: "Install dependencies with Composer" - uses: "ramsey/composer-install@v2" - with: - dependency-versions: "${{ matrix.dependencies }}" + php_versions="${{ steps.parse-php-versions.outputs.php_versions }}" + MATRIX_JSON=$(cat reusable-workflows/codeception-tests-configuration/matrix-config.json) + FILTERED_MATRIX_JSON=$(echo "$MATRIX_JSON" | jq --arg php_versions "$php_versions" '{ matrix: [ .configs[] | select(.php_version == $php_versions) | .matrix[] ] }') + ENCODED_MATRIX_JSON=$(echo "$FILTERED_MATRIX_JSON" | jq -c .) + echo "matrix=$ENCODED_MATRIX_JSON" >> "$GITHUB_OUTPUT" - - name: "Run Codeception" - run: | - vendor/bin/codecept run -c . -vvv --xml --coverage-xml + codeception-tests: + needs: setup-matrix + strategy: + matrix: ${{ fromJson(needs.setup-matrix.outputs.matrix) }} + uses: pimcore/workflows-collection-public/.github/workflows/reusable-codeception-tests-centralized.yaml@main + with: + APP_ENV: test + PIMCORE_TEST: "1" + PRIVATE_REPO: "false" + PHP_VERSION: ${{ matrix.matrix.php-version }} + DATABASE: "mariadb:10.11" + DEPENDENCIES: ${{ matrix.matrix.dependencies }} + EXPERIMENTAL: ${{ matrix.matrix.experimental }} + PIMCORE_VERSION: ${{ matrix.matrix.pimcore_version }} + REQUIRE_ADMIN_BUNDLE: "false" + PIMCORE_ELASTIC_SEARCH_HOST: "39201" + PIMCORE_ELASTIC_SEARCH_VERSION: "8.5.3" + ENABLE_ELASTICSEARCH: true + COVERAGE: "xdebug" + secrets: + SSH_PRIVATE_KEY_PIMCORE_DEPLOYMENTS_USER: ${{ secrets.SSH_PRIVATE_KEY_PIMCORE_DEPLOYMENTS_USER }} + COMPOSER_PIMCORE_REPO_PACKAGIST_TOKEN: ${{ secrets.COMPOSER_PIMCORE_REPO_PACKAGIST_TOKEN }} + PIMCORE_INSTANCE_IDENTIFIER: ${{ secrets.PIMCORE_CI_INSTANCE_IDENTIFIER }} + PIMCORE_ENCRYPTION_SECRET: ${{ secrets.PIMCORE_CI_ENCRYPTION_SECRET }} + PIMCORE_PRODUCT_KEY: ${{ secrets.PIMCORE_CI_PRODUCT_KEY }} diff --git a/.github/workflows/open-search-codeception.yaml b/.github/workflows/open-search-codeception.yaml index 05c97135..febf535d 100644 --- a/.github/workflows/open-search-codeception.yaml +++ b/.github/workflows/open-search-codeception.yaml @@ -1,9 +1,8 @@ name: "Codeception Tests with OpenSearch" on: - # Enable Later. schedule: - - cron: '0 3 * * 1,3,5' + - cron: "0 3 * * 1,3,5" workflow_dispatch: pull_request: branches: @@ -17,97 +16,75 @@ on: env: PIMCORE_PROJECT_ROOT: ${{ github.workspace }} - APP_ENV: test - PIMCORE_TEST: 1 - PIMCORE_TEST_DB_DSN: "mysql://root@127.0.0.1:33006/pimcore_test" - PIMCORE_OPEN_SEARCH_HOST: "localhost:39200" - CODECEPTION_BUNDLE_PATH: "bundles/pimcore/generic-data-index-bundle/" - PIMCORE_INSTANCE_IDENTIFIER: ${{ secrets.PIMCORE_CI_INSTANCE_IDENTIFIER }} - PIMCORE_ENCRYPTION_SECRET: ${{ secrets.PIMCORE_CI_ENCRYPTION_SECRET }} - PIMCORE_PRODUCT_KEY: ${{ secrets.PIMCORE_CI_PRODUCT_KEY }} + PRIVATE_REPO: ${{ github.event.repository.private }} -jobs: - codeception-tests: - name: "Codeception tests" - runs-on: "ubuntu-latest" - continue-on-error: ${{ matrix.experimental }} - strategy: - matrix: - include: - - { php-version: "8.3", dependencies: "lowest", pimcore_version: "", experimental: false, search_engine: "openSearch" } - - { php-version: "8.4", dependencies: "highest", pimcore_version: "", experimental: false, search_engine: "openSearch"} - - { php-version: "8.4", dependencies: "highest", pimcore_version: "12.x-dev as 12.99.9", experimental: true, search_engine: "openSearch"} - - services: - mariadb: - image: "mariadb:10.11" - ports: - - 33006:3306 - env: - MYSQL_ALLOW_EMPTY_PASSWORD: yes - - opensearch: - image: opensearchproject/opensearch:2 - ports: - - 39200:9200 - env: - cluster.name: "opensearch-cluster" - node.name: "opensearch-node" - discovery.seed_hosts: "opensearch-node" - bootstrap.memory_lock: true - OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" - discovery.type: "single-node" - DISABLE_SECURITY_PLUGIN: true +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true +jobs: + setup-matrix: + runs-on: ubuntu-latest + outputs: + php_versions: ${{ steps.parse-php-versions.outputs.php_versions }} + matrix: ${{ steps.set-matrix.outputs.matrix }} + private_repo: ${{ env.PRIVATE_REPO }} steps: - - name: "Checkout code" - uses: "actions/checkout@v4" - with: - path: "bundles/pimcore/generic-data-index-bundle" + - name: Checkout code + uses: actions/checkout@v4 - - name: "Copy .github directory" - env: - REQUIRE_ADMIN_BUNDLE: "${{ matrix.require_admin_bundle }}" - run: | - cp -R bundles/pimcore/generic-data-index-bundle/.github .github + - name: Checkout reusable workflow repo + uses: actions/checkout@v4 + with: + repository: pimcore/workflows-collection-public + ref: main + path: reusable-workflows - - name: "Install PHP" - uses: "shivammathur/setup-php@v2" - with: - coverage: xdebug - ini-values: display_errors=On, display_startup_errors=On, error_reporting=32767 - php-version: "${{ matrix.php-version }}" - - - name: Verify MariaDB connection - run: | - cp .github/ci/files/.my.cnf ~/.my.cnf - while ! mysqladmin ping --silent; do - sleep 1 - done - - - name: "Setup Pimcore environment" - env: - REQUIRE_ADMIN_BUNDLE: "${{ matrix.require_admin_bundle }}" + - name: Parse PHP versions from composer.json + id: parse-php-versions run: | - mysql -e "CREATE DATABASE pimcore_test CHARSET=utf8mb4;" - cp -R bundles/pimcore/generic-data-index-bundle/tests tests - mkdir src - chmod 755 .github/ci/scripts/setup-pimcore-environment-functional-tests.sh - .github/ci/scripts/setup-pimcore-environment-functional-tests.sh "${{ matrix.search_engine }}" + if [ -f composer.json ]; then + php_versions=$(jq -r '.require.php' composer.json | grep -oP '\d+\.\d+' | tr '\n' ',' | sed 's/,$//') + if [ -z "$php_versions" ]; then + echo "php_versions=default" >> "$GITHUB_OUTPUT" + else + echo "php_versions=$php_versions" >> "$GITHUB_OUTPUT" + fi + else + exit 1 + fi - - name: "Update Pimcore version" - env: - PIMCORE_VERSION: "${{ matrix.pimcore_version }}" + - name: Set up matrix JSON + id: set-matrix run: | - if [ ! -z "$PIMCORE_VERSION" ]; then - composer require --no-update pimcore/pimcore:"${PIMCORE_VERSION}" - fi + php_versions="${{ steps.parse-php-versions.outputs.php_versions }}" + MATRIX_JSON=$(cat reusable-workflows/codeception-tests-configuration/matrix-config.json) + FILTERED_MATRIX_JSON=$(echo "$MATRIX_JSON" | jq --arg php_versions "$php_versions" '{ matrix: [ .configs[] | select(.php_version == $php_versions) | .matrix[] ] }') + ENCODED_MATRIX_JSON=$(echo "$FILTERED_MATRIX_JSON" | jq -c .) + echo "matrix=$ENCODED_MATRIX_JSON" >> "$GITHUB_OUTPUT" - - name: "Install dependencies with Composer" - uses: "ramsey/composer-install@v2" - with: - dependency-versions: "${{ matrix.dependencies }}" - - - name: "Run Codeception" - run: | - vendor/bin/codecept run -c . -vvv --xml --coverage-xml + codeception-tests: + needs: setup-matrix + strategy: + matrix: ${{ fromJson(needs.setup-matrix.outputs.matrix) }} + uses: pimcore/workflows-collection-public/.github/workflows/reusable-codeception-tests-centralized.yaml@main + with: + APP_ENV: test + PIMCORE_TEST: "1" + PRIVATE_REPO: "false" + PHP_VERSION: ${{ matrix.matrix.php-version }} + DATABASE: "mariadb:10.11" + DEPENDENCIES: ${{ matrix.matrix.dependencies }} + EXPERIMENTAL: ${{ matrix.matrix.experimental }} + PIMCORE_VERSION: ${{ matrix.matrix.pimcore_version }} + REQUIRE_ADMIN_BUNDLE: "false" + PIMCORE_OPEN_SEARCH_HOST: "39200" + PIMCORE_OPENSEARCH_VERSION: "2" + ENABLE_ELASTICSEARCH: false + COVERAGE: "xdebug" + secrets: + SSH_PRIVATE_KEY_PIMCORE_DEPLOYMENTS_USER: ${{ secrets.SSH_PRIVATE_KEY_PIMCORE_DEPLOYMENTS_USER }} + COMPOSER_PIMCORE_REPO_PACKAGIST_TOKEN: ${{ secrets.COMPOSER_PIMCORE_REPO_PACKAGIST_TOKEN }} + PIMCORE_INSTANCE_IDENTIFIER: ${{ secrets.PIMCORE_CI_INSTANCE_IDENTIFIER }} + PIMCORE_ENCRYPTION_SECRET: ${{ secrets.PIMCORE_CI_ENCRYPTION_SECRET }} + PIMCORE_PRODUCT_KEY: ${{ secrets.PIMCORE_CI_PRODUCT_KEY }} diff --git a/composer.json b/composer.json index 5e87062f..62dd22d9 100644 --- a/composer.json +++ b/composer.json @@ -31,6 +31,7 @@ "codeception/codeception": "^5.3.4", "codeception/module-asserts": "^3.0", "codeception/module-symfony": "^3.7.1", + "codeception/stub": "^4.1.4", "phpstan/phpstan": "1.12.15", "phpstan/phpstan-symfony": "^1.2.20", "phpunit/phpunit": "^10.5", @@ -47,7 +48,10 @@ "autoload-dev": { "psr-4": { "Pimcore\\Bundle\\GenericDataIndexBundle\\Tests\\": "tests/" - } + }, + "files": [ + "kernel/Kernel.php" + ] }, "extra": { "branch-alias": { diff --git a/config/services/search-index-adapter/default-search.yaml b/config/services/search-index-adapter/default-search.yaml index e9c88168..ff214d97 100644 --- a/config/services/search-index-adapter/default-search.yaml +++ b/config/services/search-index-adapter/default-search.yaml @@ -1,6 +1,6 @@ imports: - - { resource: 'services/search-index-adapter/default-search/modifiers/*' } - - { resource: 'services/search-index-adapter/default-search/pql-field-name-transformers.yaml' } + - { resource: "default-search/modifiers/*" } + - { resource: "default-search/pql-field-name-transformers.yaml" } services: _defaults: @@ -11,32 +11,32 @@ services: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\SearchIndexServiceInterface: class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\DefaultSearchService arguments: - $client: '@generic-data-index.search-client' + $client: "@generic-data-index.search-client" Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\IndexAliasServiceInterface: class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\IndexAliasService arguments: - $client: '@generic-data-index.search-client' + $client: "@generic-data-index.search-client" Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\SearchExecutionServiceInterface: class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\SearchExecutionService arguments: - $client: '@generic-data-index.search-client' + $client: "@generic-data-index.search-client" Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\BulkOperationServiceInterface: class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\BulkOperationService arguments: - $client: '@generic-data-index.search-client' + $client: "@generic-data-index.search-client" Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\PathServiceInterface: class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\PathService arguments: - $client: '@generic-data-index.search-client' + $client: "@generic-data-index.search-client" Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\ElementLockServiceInterface: class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\ElementLockService arguments: - $client: '@generic-data-index.search-client' + $client: "@generic-data-index.search-client" Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\Search\Pagination\PaginationInfoServiceInterface: class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Pagination\PaginationInfoService @@ -56,7 +56,7 @@ services: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\IndexStatsServiceInterface: class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\IndexStatsService arguments: - $client: '@generic-data-index.search-client' + $client: "@generic-data-index.search-client" Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\IndexMappingServiceInterface: class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\IndexMappingService @@ -73,12 +73,11 @@ services: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\QueryLanguage\SubQueriesProcessorInterface: class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\QueryLanguage\SubQueriesProcessor - Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\QueryLanguage\FieldNameValidator\: - resource: '../../../src/SearchIndexAdapter/DefaultSearch/QueryLanguage/FieldNameValidator' - tags: ['pimcore.generic_data_index.pql_field_name_validator'] + resource: "../../../src/SearchIndexAdapter/DefaultSearch/QueryLanguage/FieldNameValidator" + tags: ["pimcore.generic_data_index.pql_field_name_validator"] Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DataObject\IndexIconUpdateServiceInterface: class: Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\DataObject\IndexIconUpdateService arguments: - $client: '@generic-data-index.search-client' \ No newline at end of file + $client: "@generic-data-index.search-client" diff --git a/src/SearchIndexAdapter/DefaultSearch/PathService.php b/src/SearchIndexAdapter/DefaultSearch/PathService.php index b27632de..50e25c03 100644 --- a/src/SearchIndexAdapter/DefaultSearch/PathService.php +++ b/src/SearchIndexAdapter/DefaultSearch/PathService.php @@ -124,6 +124,7 @@ private function updatePath(string $indexName, string $currentPath, string $newP 'params' => [ 'currentPath' => $currentPath . '/', 'newPath' => $newPath . '/', + 'now' => date('c'), ], ], @@ -167,8 +168,11 @@ private function getScriptSource(): string } if(ctx._source.containsKey("custom_fields") && + ctx._source.custom_fields instanceof Map && ctx._source.custom_fields.containsKey("PortalEngineBundle") && + ctx._source.custom_fields.PortalEngineBundle instanceof Map && ctx._source.custom_fields.PortalEngineBundle.containsKey("system_fields") && + ctx._source.custom_fields.PortalEngineBundle.system_fields instanceof Map && ctx._source.custom_fields.PortalEngineBundle.system_fields.containsKey("thumbnail")) { def customFields = ctx._source.custom_fields.PortalEngineBundle.system_fields; if(customFields.thumbnail != null && customFields.thumbnail instanceof String) { @@ -210,7 +214,7 @@ private function getScriptSource(): string } ctx._source.system_fields.pathLevels = newLevels; } - ctx._source.system_fields.modificationDate = Instant.now().toString(); + ctx._source.system_fields.modificationDate = params.now; ctx._source.system_fields.checksum = 0'; } diff --git a/tests/Support/Helper/GenericDataIndex.php b/tests/Support/Helper/GenericDataIndex.php index b52bbba9..abe742f2 100644 --- a/tests/Support/Helper/GenericDataIndex.php +++ b/tests/Support/Helper/GenericDataIndex.php @@ -17,7 +17,6 @@ use Codeception\Lib\ModuleContainer; use Pimcore\Bundle\GenericDataIndexBundle\Installer; -use Pimcore\Bundle\GenericDataIndexBundle\Installer as GenericDataIndexInstaller; use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search\Modifier\Sort\TreeSortHandlers; use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexQueue\QueueMessagesDispatcher; use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexQueue\SynchronousProcessingRelatedIdsServiceInterface; @@ -75,7 +74,7 @@ public function _beforeSuite($settings = []): void $this->debug('[Generic Data Index] Running bundle installer'); $genericDataIndexInstaller = $pimcoreModule->getContainer()->get( - GenericDataIndexInstaller::class + Installer::class ); $genericDataIndexInstaller->install(); @@ -175,8 +174,13 @@ public function cleanupIndex() { /** @var SearchClientInterface $client */ $client = $this->getIndexSearchClient(); + + /** @var SearchIndexConfigServiceInterface $configService */ + $configService = $this->grabService(SearchIndexConfigServiceInterface::class); + $indexPrefix = $configService->getIndexPrefix(); + $client->deleteByQuery([ - 'index' => '*', + 'index' => $indexPrefix . '*', 'body' => [ 'query' => [ 'match_all' => (object)[],