[ENG-9703] Feature/add new es8 metrics

.docker-compose.env

@@ -8,6 +8,7 @@ API_DOMAIN=http://localhost:8000/
 ELASTIC_URI=192.168.168.167:9200
 ELASTIC6_URI=192.168.168.167:9201
 ELASTIC8_URI=http://192.168.168.167:9202
+ELASTIC8_USERNAME=elastic
 OSF_DB_HOST=192.168.168.167
 DB_HOST=192.168.168.167
 REDIS_HOST=redis://192.168.168.167:6379

.github/workflows/test-build.yml

@@ -37,7 +37,19 @@ jobs:
     permissions:
       checks: write
     services:
-      postgres:
+      elasticsearch8: &ES8_SERVICE
+        image: elasticsearch:8.19.14
+        ports:
+          - 9202:9200
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
+        options: >-
+          --health-cmd "curl -sf http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=30s"
+          --health-interval 10s
+          --health-timeout 30s
+          --health-retries 5
+      postgres: &POSTGRES_SERVICE
         image: postgres
         env:
           POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
@@ -54,6 +66,8 @@ jobs:
     - uses: ./.github/actions/start-build
     - name: Run tests
       run: poetry run python3 -m invoke test-ci-addons --junit
+      env:
+        ELASTIC8_URI: http://localhost:9202
     - name: Upload report
       if: (success() || failure())    # run this step even if previous step failed
       uses: ./.github/actions/gen-report
@@ -64,18 +78,7 @@ jobs:
     permissions:
       checks: write
     services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      postgres: *POSTGRES_SERVICE
     steps:
     - uses: actions/checkout@v2
     - uses: ./.github/actions/start-build
@@ -91,25 +94,17 @@ jobs:
     permissions:
       checks: write
     services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      elasticsearch8: *ES8_SERVICE
+      postgres: *POSTGRES_SERVICE
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/start-build
       - name: NVM & yarn install
         run: poetry run python3 -m invoke assets --dev
       - name: Run test
         run: poetry run python3 -m invoke test-ci-api1-and-js --junit
+        env:
+          ELASTIC8_URI: http://localhost:9202
       - name: Upload report
         if: (success() || failure())    # run this step even if previous step failed
         uses: ./.github/actions/gen-report
@@ -120,23 +115,15 @@ jobs:
     permissions:
       checks: write
     services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      elasticsearch8: *ES8_SERVICE
+      postgres: *POSTGRES_SERVICE
     steps:
     - uses: actions/checkout@v2
     - uses: ./.github/actions/start-build
     - name: Run tests
       run: poetry run python3 -m invoke test-ci-api2 --junit
+      env:
+        ELASTIC8_URI: http://localhost:9202
     - name: Upload report
       if: (success() || failure())    # run this step even if previous step failed
       uses: ./.github/actions/gen-report
@@ -147,19 +134,7 @@ jobs:
       checks: write
     needs: build-cache
     services:
-      postgres:
-        image: postgres
-
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      postgres: *POSTGRES_SERVICE
     steps:
     - uses: actions/checkout@v2
     - uses: ./.github/actions/start-build
@@ -175,19 +150,7 @@ jobs:
       checks: write
     needs: build-cache
     services:
-      postgres:
-        image: postgres
-
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      postgres: *POSTGRES_SERVICE
       mailhog:
         image: mailhog/mailhog
         ports:
@@ -208,19 +171,7 @@ jobs:
       checks: write
     needs: build-cache
     services:
-      postgres:
-        image: postgres
-
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      postgres: *POSTGRES_SERVICE
     steps:
     - uses: actions/checkout@v2
     - uses: ./.github/actions/start-build

api/base/settings/defaults.py

@@ -316,7 +316,6 @@
 HASHIDS_SALT = 'pinkhimalayan'
 
 # django-elasticsearch-metrics
-DJELME_AUTOSETUP = True
 DJELME_BACKENDS = {
     'osfmetrics_es6': {
         'elasticsearch_metrics.imps.elastic6': {
@@ -327,6 +326,12 @@
     'osfmetrics_es8': {
         'elasticsearch_metrics.imps.elastic8': {
             'hosts': osf_settings.ELASTIC8_URI,
+            'ca_certs': osf_settings.ELASTIC8_CERT_PATH,
+            'basic_auth': (
+                (osf_settings.ELASTIC8_USERNAME, osf_settings.ELASTIC8_SECRET)
+                if osf_settings.ELASTIC8_SECRET is not None
+                else None
+            ),
         },
     },
 }

api_tests/metrics/test_counted_usage.py

@@ -38,8 +38,9 @@ def assert_saved_with(mock_save, *, expected_doc_id=None, expected_attrs):
 
 @pytest.fixture
 def mock_save():
-    with mock.patch('elasticsearch6_dsl.Document.save', autospec=True) as mock_save:
-        yield mock_save
+    with mock.patch('elasticsearch_metrics.imps.elastic6.BaseMetric.check_index_template'):
+        with mock.patch('elasticsearch6_dsl.Document.save', autospec=True) as mock_save:
+            yield mock_save
 
 
 @pytest.mark.django_db

api_tests/metrics/test_registries_moderation_metrics.py

@@ -1,7 +1,5 @@
 import pytest
-from waffle.testutils import override_switch
 
-from osf import features
 from osf_tests.factories import RegistrationFactory, AuthUserFactory
 from osf.utils.workflows import RegistrationModerationStates, RegistrationModerationTriggers
 from osf.metrics import RegistriesModerationMetrics
@@ -17,11 +15,6 @@ class TestRegistrationModerationMetrics:
     def registration(self):
         return RegistrationFactory()
 
-    @pytest.fixture(autouse=True)
-    def enable_elasticsearch_metrics(self):
-        with override_switch(features.ELASTICSEARCH_METRICS, active=True):
-            yield
-
     @pytest.mark.es_metrics
     def test_record_transitions(self, registration):
         with capture_notifications():
@@ -50,11 +43,6 @@ class TestRegistrationModerationMetricsView:
     def registration(self):
         return RegistrationFactory()
 
-    @pytest.fixture(autouse=True)
-    def enable_elasticsearch_metrics(self):
-        with override_switch(features.ELASTICSEARCH_METRICS, active=True):
-            yield
-
     @pytest.fixture
     def user(self):
         user = AuthUserFactory()

conftest.py

@@ -5,18 +5,20 @@
 
 from django.db import transaction
 from elasticsearch6_dsl.connections import connections
-from website import settings as osf_settings
-from elasticsearch_metrics.tests._test_util import RealElasticTestCase
+from elasticsearch_metrics.tests.util import djelme_test_backends
 from faker import Factory
 import pytest
 import responses
 import xml.etree.ElementTree as ET
+from waffle.testutils import override_switch
 
 from api_tests.share import _utils as shtrove_test_utils
 from framework.celery_tasks import app as celery_app
 from osf.external.spam import tasks as spam_tasks
 from website import settings as website_settings
 from osf.management.commands.populate_notification_types import populate_notification_types
+from osf import features
+
 
 def pytest_configure(config):
     if not os.getenv('GITHUB_ACTIONS') == 'true':
@@ -146,19 +148,12 @@ def _es_metrics_marker(request):
         yield
         return
 
-    connections.create_connection(
-        alias='osfmetrics_es6',
-        hosts=osf_settings.ELASTIC6_URI,
-    )
-
-    class _Es6TestCase(RealElasticTestCase, autosetup_djelme_backends=True):
-        ...
-    es6_test_case = _Es6TestCase()
-    es6_test_case.setUp()
-    try:
+    with (
+        override_switch(features.ELASTICSEARCH_METRICS, active=True),
+        djelme_test_backends(),
+    ):
         yield
-    finally:
-        es6_test_case.tearDown()
+
 
 @pytest.fixture
 def mock_share_responses():

docker-compose.yml

@@ -79,15 +79,22 @@ services:
     stdin_open: true
 
   elasticsearch8:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.11
-    platform: linux/arm64
+    image: elasticsearch:8.19.14
     environment:
-      - xpack.security.enabled=false
       - discovery.type=single-node
+      - xpack.security.enabled=false
+      - ES_JAVA_OPTS=-Xms512m -Xmx512m  # reduce memory usage
+      - xpack.ml.enabled=false
     ports:
       - 9202:9200
     volumes:
       - elasticsearch8_data_vol:/usr/share/elasticsearch/data
+    healthcheck:
+      start_period: 15s
+      test: ["CMD", "curl", "-sf", "http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=30s"]
+      interval: 10s
+      timeout: 30s
+      retries: 5
     stdin_open: true
 
   postgres:

osf/metrics/__init__.py

@@ -17,6 +17,8 @@
     StorageAddonUsage,
     UserSummaryReport,
 )
+from . import es8_metrics
+
 
 DAILY_REPORTS = (
     DownloadCountReport,
@@ -36,4 +38,5 @@
     'PreprintView',
     'PreprintDownload',
     'RegistriesModerationMetrics',
+    'es8_metrics',
 )