Reading grib files poetry

.coveragerc

@@ -1,8 +1,7 @@
 [run]
 source =
-	engine/
-    visualize/
-    probtest.py
-	util/
+  engine
+  util
+  visualize
 omit =
-	tests/*
+  tests/*

.github/workflows/pre-commit.yml

@@ -20,18 +20,13 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.10.8
-    - uses: conda-incubator/setup-miniconda@v2
-      with:
-        miniconda-version: "latest"
-        channels: conda-forge
-        channel-priority: flexible
-        show-channel-urls: true
-    - name: Create env from pinned environment
+    - name: Install dependencies with pip
       run: |
-        conda env create --name dev_env --file requirements/environment.yml
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
     - name: Install pre-commit hooks
       run: |
-        conda run --name dev_env pre-commit install-hooks
+        pre-commit install-hooks
     - name: Run pre-commit hooks
       run: |
-        conda run --name dev_env pre-commit run --all-files
+        pre-commit run --all-files

.github/workflows/pytest.yml

@@ -14,23 +14,28 @@ jobs:
     defaults:
       run:
         shell: bash -l {0}
+
     steps:
     - uses: actions/checkout@v2
+
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
         python-version: 3.10.8
-    - uses: conda-incubator/setup-miniconda@v2
-      with:
-        miniconda-version: "latest"
-        channels: conda-forge
-        channel-priority: flexible
-        show-channel-urls: true
-    - name: Create env from pinned environment
+    - name: Install Poetry
+      run: |
+        curl -sSL https://install.python-poetry.org | python3 -
+    - name: Configure Poetry to use the right Python version
+      run: |
+        poetry env use python3.10
+    - name: Install dependencies with Poetry
+      run: |
+        poetry install
+    - name: Run setup_env for eccodes definition
       run: |
-        ./setup_env.sh -n probtest
-    - name: Run Pytest
+        poetry run python setup_env.py
+    - name: Run Pytest with Poetry
       env:
         TZ: Europe/Zurich
       run: |
-        conda run --name probtest pytest -v -s --cov --cov-report=term tests/
+        poetry run pytest -v -s --cov --cov-report=term tests/

.gitignore

@@ -1,3 +1,5 @@
+.coverage
+.env
 .idea*
 .vscode
 *pycache*
@@ -6,12 +8,9 @@
 *.png
 *.log
 *.nc
-.coverage
 coverage.xml
-miniconda
+resource
 
-# output from unittest
-test_stats.csv
-test_stats_csv.csv
-test_stats_csv.dat
+# output from pytest
 random_tolerance.csv
+random_tolerance_None.csv

.pre-commit-config.yaml

@@ -83,7 +83,7 @@ repos:
           description: Check for spelling errors
           language: system
           entry: codespell
-          exclude: tests/data/
+          exclude: ^(poetry.lock|tests/data/.*)$
           args: ['-L astroid,fof']
   -   repo: local
       hooks:

AUTHORS.md

@@ -6,6 +6,7 @@
 | Marek Jacob             | DWD                                                 |
 | Jonas Jucker            | C2SM                                                |
 | Annika Lauber           | C2SM                                                |
+| Andrea Leuthard         | MeteoSwiss                                          |
 | Giacomo Serafini        | MeteoSwiss                                          |
 | Mikael Stellio          | C2SM                                                |
 | Ben Weber               | MeteoSwiss                                          |

README.md

@@ -80,31 +80,45 @@ Visualize the performance database generated with `performance`.
 
 Even though probtest is used exclusively with ICON at the moment, it does not contain any information about the model or its directory structure. This makes it very flexible and applicable to any circumstance (e.g. usable by Buildbot, Jenkins and human users alike). However, it also requires a lot of information about the model and the data to be processed upon invocation. Since a typical probtest usage involves multiple commands (e.g. run-ensemble -> stats -> tolerance -> check) this leads to a lot of redundancy in the invocation. Therefore, probtest can read commonly used input variables (e.g. the model output directory, the experiment name, the name of the submit script, ...) from a configuration file in json format. To further ease the process, these configuration files can be created from templates using the `init` command. A template for ICON is contained in this repository in the `templates` subdirectory.
 
-### Setup conda
+### Setup probtest
 
-All requirements for using probtest can be easily installed with conda using the
-setup scripts.
+All requirements for using probtest can be easily installed with pip or poetry.
 
-For setting up conda you use
+#### Setup with pip
+The setup with pip is recommended for users.
+
+For setting up the venv you use
 ```console
-./setup_miniconda.sh -u
+python3.10 -m venv venv
+source venv/bin/activate
+pip install --upgrade pip
+pip install -r requirements.txt
 ```
-which will modify your `.bashrc`, or you can use
+To check that everything is setup correctly (note this requires
+[eccodes definition](#Setup-the-eccodes-definition)), you can do
 ```console
-./setup_miniconda.sh
-source miniconda/bin/activate
+pytest -v -s --cov --cov-report=term tests/
 ```
-which requires to source minconda.
 
-The pinned requirements can be installed by
+#### Setup with poetry
+The setup with poetry is recommended for developers and needed for changing or
+adding requirements.
+It is assumed you are having a poetry installation.
 ```console
-./setup_env.sh
+poetry env use python3.10
+poetry install
 ```
-The unpinned requirements and updating the environment can be done by
+To check that everything is setup correctly (note this requires
+[eccodes definition](#Setup-the-eccodes-definition)), you can do
 ```console
-./setup_env.sh -u -e
+poetry run pytest -v -s --cov --cov-report=term tests/
 ```
 
+#### Setup the eccodes definition
+For setting up the eccode definition, you can use
+```console
+python setup_env.py
+```
 
 ### The init command
 
@@ -138,7 +152,17 @@ here should refer to your experiment script:
 
 ```console
 cd icon-base-dir/reference-build
-python ../externals/probtest/probtest.py init --codebase-install $PWD --experiment-name exp_name --reference $PWD --file-id NetCDF "*atm_3d_ml*.nc" --file-id NetCDF "*atm_3d_il*.nc" --file-id NetCDF "*atm_3d_hl*.nc" --file-id NetCDF "*atm_3d_pl*.nc" --file-id latlon "*atm_2d_ll*.nc" --file-id meteogram "Meteogram*.nc"
+python ../externals/probtest/probtest.py init  \
+            --codebase-install $PWD  \
+            --experiment-name exp_name  \
+            --reference $PWD  \
+            --file-id NetCDF "*atm_3d_ml*.nc"  \
+            --file-id NetCDF "*atm_3d_il*.nc"  \
+            --file-id NetCDF "*atm_3d_hl*.nc"  \
+            --file-id NetCDF "*atm_3d_pl*.nc"  \
+            --file-id latlon "*atm_2d_ll*.nc"  \
+            --file-id meteogram "Meteogram*.nc"  \
+            --file-id GRIB "lfff0*z"
 ```
 You might need to update the used account in the json file.
 The perturbation amplitude may also need to be changed in the json file

engine/cdo_table.py

@@ -17,7 +17,7 @@
 from util.click_util import CommaSeperatedInts, cli_help
 from util.constants import cdo_bins
 from util.dataframe_ops import df_from_file_ids
-from util.file_system import file_names_from_pattern
+from util.file_system import get_file_names_from_pattern
 from util.log_handler import logger
 
 
@@ -139,21 +139,21 @@ def cdo_table(
     assert isinstance(file_specification, dict), "must be dict"
 
     # save original method and restore at the end of this module
-    dataframe_from_ncfile_orig = model_output_parser.dataframe_from_ncfile
+    dataframe_from_ncfile_orig = model_output_parser.create_statistics_dataframe
     # modify netcdf parse method:
-    model_output_parser.dataframe_from_ncfile = rel_diff_stats
+    model_output_parser.create_statistics_dataframe = rel_diff_stats
 
     # step 1: compute rel-diff netcdf files
     with tempfile.TemporaryDirectory() as tmpdir:
         for _, file_pattern in file_id:
-            ref_files, err = file_names_from_pattern(model_output_dir, file_pattern)
+            ref_files, err = get_file_names_from_pattern(model_output_dir, file_pattern)
             if err > 0:
                 logger.info(
                     "did not find any files for pattern %s. Continue.", file_pattern
                 )
                 continue
             ref_files.sort()
-            perturb_files, err = file_names_from_pattern(
+            perturb_files, err = get_file_names_from_pattern(
                 perturbed_model_output_dir.format(member_id=member_id), file_pattern
             )
             if err > 0:
@@ -202,4 +202,4 @@ def cdo_table(
 
         Path(cdo_table_file).parent.mkdir(parents=True, exist_ok=True)
         df.to_csv(cdo_table_file)
-        model_output_parser.dataframe_from_ncfile = dataframe_from_ncfile_orig
+        model_output_parser.create_statistics_dataframe = dataframe_from_ncfile_orig

engine/performance.py

@@ -15,7 +15,7 @@
 import click
 
 from util.click_util import cli_help
-from util.file_system import file_names_from_pattern
+from util.file_system import get_file_names_from_pattern
 from util.icon.extract_timings import read_logfile
 from util.log_handler import logger
 from util.tree import TREEFILE_TEMPLATE, TimingTree
@@ -36,7 +36,9 @@ def performance(timing_regex, timing_database, append_time):
     if timing_file_name_base == "":
         timing_file_name_base = "."
 
-    timing_file_name, err = file_names_from_pattern(timing_file_name_base, timing_regex)
+    timing_file_name, err = get_file_names_from_pattern(
+        timing_file_name_base, timing_regex
+    )
     if err > 0:
         logger.info("Did not find any files for regex %s", timing_regex)
         sys.exit(1)

jenkins/mergerequest

@@ -6,13 +6,43 @@ pipeline {
     }
 
     stages {
-        stage('Pytest') {
+        stage('Setup Python Environment') {
             steps {
-                sh'''
-                source $SCRATCH/project/icon/probtest/conda/miniconda/bin/activate probtest
-                pytest -v tests/*
+                sh '''
+                module use /mch-environment/v7/modules
+                module load python/3.10.4
+                python3.10 -m venv venv
+                source venv/bin/activate
+                pip install --upgrade pip
+                pip install -r requirements.txt
                 '''
             }
         }
+        stage('Run setup_env.py Script') {
+            steps {
+                sh '''
+                source venv/bin/activate
+                python setup_env.py
+                '''
+            }
+        }
+        stage('Run Pytest') {
+            steps {
+                sh '''
+                source venv/bin/activate
+                pytest -v -s --cov --cov-report=term tests/
+                '''
+            }
+        }
+    }
+    post {
+        always {
+            sh '''
+            deactivate || true
+            rm -rf venv
+            rm -rf .env
+            rm -rf resource
+            '''
+        }
     }
 }