diff --git a/.github/workflows/python-examples.yml b/.github/workflows/python-examples.yml index 162da608..ace4ae7e 100644 --- a/.github/workflows/python-examples.yml +++ b/.github/workflows/python-examples.yml @@ -1,97 +1,69 @@ +# .github/workflows/verify_examples.yml +# +# CI for python/examples/01-03 select_values scripts. +# Strategy: download the daily HallD SQLite snapshot so the examples +# run against real data with a proper schema. + name: Python examples verification on: push: - branches: - - '*' + branches: ['*'] pull_request: - branches: - - main + branches: [main] + +env: + # Single source of truth for the connection string + RCDB_CONNECTION: "sqlite:///${{ github.workspace }}/tmp/rcdb2.sqlite" jobs: run-examples: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9"] - # Don't fail the entire job if one example fails - fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - name: Set up Python + uses: actions/setup-python@v6 with: - python-version: ${{ matrix.python-version }} + python-version: "3.9" - - name: Install dependencies + - name: Install rcdb run: | python -m pip install --upgrade pip - # Install rcdb and dependencies from pyproject.toml cd $GITHUB_WORKSPACE/python pip install --editable . - # Install additional dependencies for examples - pip install jsonpickle + pip install jsonpickle # for writing object example - - name: Setup SQLite for testing - run: | - mkdir -p $GITHUB_WORKSPACE/tmp - touch $GITHUB_WORKSPACE/tmp/test.sqlite.db - echo "RCDB_CONNECTION=sqlite:///$GITHUB_WORKSPACE/tmp/test.sqlite.db" >> $GITHUB_ENV +# - name: Download HallD SQLite snapshot +# run: | +# mkdir -p $GITHUB_WORKSPACE/tmp +# # Future put db creation here - - name: Create database service (MySQL) - if: false # Disabled until we have a proper way to initialize the MySQL database - run: | - # This would be the place to set up an actual MySQL database if needed - # For now, we'll use SQLite for examples that accept a connection string parameter - echo "RCDB_MYSQL_CONNECTION=sqlite:///$GITHUB_WORKSPACE/tmp/test.sqlite.db" >> $GITHUB_ENV - - name: List example files - id: list-examples - run: | - cd $GITHUB_WORKSPACE/python/examples - echo "examples=$(ls -1 example_*.py | tr '\n' ' ')" >> $GITHUB_OUTPUT + # ── Run examples ────────────────────────────────────────────── + # Each example gets its own step so failures are visible per-script. + # continue-on-error keeps the workflow green while showing which + # script had problems (yellow badge instead of silent swallow). - - name: Run basic examples (with in-memory or SQLite DB) - run: | - cd $GITHUB_WORKSPACE/python/examples - echo "Running example_conditions_basic.py" - python example_conditions_basic.py - - echo "Running example_conditions_store_array.py" - python example_conditions_store_array.py - - echo "Running example_conditions_store_object.py" - python example_conditions_store_object.py - - echo "Running example_sqlalchemy_query.py" - python example_sqlalchemy_query.py - - echo "Running example_runs_by_date.py (with connection string)" - python example_runs_by_date.py $RCDB_CONNECTION || echo "Skipping - requires properly initialized database" - - - name: Run query examples (with offline mode) - run: | - cd $GITHUB_WORKSPACE/python/examples - echo "Running example_simple_queries.py" - python example_simple_queries.py $RCDB_CONNECTION || echo "Skipping - requires properly initialized database" - - # The following examples normally use the production database - # We'll try them with the test db, but expect some to fail gracefully - export RCDB_OFFLINE=true - echo "Running examples in offline mode (will show errors but not fail workflow)" - - echo "Trying example_cdc_gas_pressure.py" - python example_cdc_gas_pressure.py || echo "Skipping - requires production database" - - echo "Trying example_select_halld_values.py" - python example_select_halld_values.py || echo "Skipping - requires production database" - - echo "Trying example_select_values.py" - python example_select_values.py || echo "Skipping - requires production database" - - - name: Summary - run: | - echo "Examples testing completed. Check logs for any failures." - echo "Note: Some examples are expected to fail if they require a production database connection." \ No newline at end of file + - name: "Example: 01_select_values_simple" + run: python $GITHUB_WORKSPACE/python/examples/01_select_values_simple.py + # Script must read RCDB_CONNECTION env var (see updated examples) + + - name: "Example: 02_select_values_extended" + run: python $GITHUB_WORKSPACE/python/examples/02_select_values_extended.py + + - name: "Example: 03_select_values_custom_runs" + run: python $GITHUB_WORKSPACE/python/examples/03_select_values_custom_runs.py + + - name: "Example: 10_create_conditions_basic" + run: python $GITHUB_WORKSPACE/python/examples/10_create_conditions_basic.py + + - name: "Example: 11_crete_conditions_store_array" + run: python $GITHUB_WORKSPACE/python/examples/11_crete_conditions_store_array.py + + - name: "Example: 12_create_conditions_store_object" + run: python $GITHUB_WORKSPACE/python/examples/12_create_conditions_store_object.py + + - name: "Example: 90_advanced_sqlalchemy_query" + run: python $GITHUB_WORKSPACE/python/examples/90_advanced_sqlalchemy_query.py \ No newline at end of file diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index efb52e3b..5c64d900 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore index 442a2512..ad9a67fa 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,5 @@ sql/schema.mwb.bak reqvenv/ python/repomix-output.txt + +python/uv.lock diff --git a/docs/development/documentation.md b/docs/development/documentation.md index 965e078e..9e3af580 100644 --- a/docs/development/documentation.md +++ b/docs/development/documentation.md @@ -27,6 +27,9 @@ Here is a python one-liner that makes the site running at `localhost:3000` ```bash # assuming current working directoy is rcdb/docs python3 -m http.server 3000 + +# Using uv from rcdb repository root +uv --project ./python --directory docs run -m http.server 3000 ``` > On windows machines you may have to use `python` instead of `python3` diff --git a/python/examples/example_conditions_basic.py b/python/examples/10_create_conditions_basic.py similarity index 100% rename from python/examples/example_conditions_basic.py rename to python/examples/10_create_conditions_basic.py diff --git a/python/examples/example_conditions_store_array.py b/python/examples/11_crete_conditions_store_array.py similarity index 100% rename from python/examples/example_conditions_store_array.py rename to python/examples/11_crete_conditions_store_array.py diff --git a/python/examples/example_conditions_store_object.py b/python/examples/12_create_conditions_store_object.py similarity index 100% rename from python/examples/example_conditions_store_object.py rename to python/examples/12_create_conditions_store_object.py diff --git a/python/examples/example_sqlalchemy_query.py b/python/examples/90_advanced_sqlalchemy_query.py similarity index 100% rename from python/examples/example_sqlalchemy_query.py rename to python/examples/90_advanced_sqlalchemy_query.py diff --git a/python/examples/example_cdc_gas_pressure.py b/python/examples/example_cdc_gas_pressure.py deleted file mode 100644 index 0e0120a9..00000000 --- a/python/examples/example_cdc_gas_pressure.py +++ /dev/null @@ -1,12 +0,0 @@ -# import RCDB -from rcdb.provider import RCDBProvider - -# connect to DB -db = RCDBProvider("mysql://rcdb@hallddb.jlab.org/rcdb") - -# select values with query -table = db.select_values(['cdc_gas_pressure'], run_min=41512, run_max=41540) - -for row in table: - (run_number, cdc_gas_pressure) = tuple(row) - print(f"{run_number} {cdc_gas_pressure}") diff --git a/python/examples/example_runs_by_date.py b/python/examples/example_runs_by_date.py deleted file mode 100644 index 2e69fcda..00000000 --- a/python/examples/example_runs_by_date.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -Example shows how to select runs and some data (event_count) and show it by date - -The script result is something like: - -Selecting runs 10000-19999 from run period: '28 Jan 2016 - 24 Apr 2016 Commissioning, 12 GeV e-' -'2016-02-20' has 6 runs 10391 - 10399 - 10391 2016-02-20 07:29:04 84235408 - 10392 2016-02-20 10:33:11 51150478 - ... -'2016-02-21' has 10 runs 10412 - 10436 - 10412 2016-02-21 02:42:24 59020003 - 10414 2016-02-21 05:00:05 21921629 - ... -... - -Usage: - python example_runs_by_date.py - -""" -import argparse -import sys -from rcdb import RCDBProvider - - -def get_runs_by_date(db, query, run_min=0, run_max=2147483647): - """ - Returns dictionary of dates with all runs per date - - :param db: RCDB provider class (aka rcdb.RCDBProvider) - :param query: Run search query. All runs are selected if blank - :param run_min: Run to start search from - :param run_max: Run (included) to search up to - :return: (data_by_date, performance) where data_by_date in form {date: [(run_number, start_time, event_count),...]} - - :description: - - """ - - assert isinstance(db, RCDBProvider) - - # Select production runs with event_count > 0.5M - result_table = db.select_values(['run_start_time', 'event_count'], query, run_min, run_max) - - # construct data by date - data_by_date = {} - for row in result_table: - run, run_start_time, event_count = tuple(row) - run_date = run_start_time.date() - - # is there the key already? - if run_date not in data_by_date: - data_by_date[run_date] = [] - - # add data to corresponding date - data_by_date[run_date].append((run, run_start_time, event_count)) - return data_by_date, result_table.performance - - -if __name__ == "__main__": - print(sys.argv) - # Get connection string from arguments - parser = argparse.ArgumentParser(description="This example shows select runs and put them by dates") - parser.add_argument("connection_string", - nargs='?', - help="RCDB connection string mysql://rcdb@localhost/rcdb", - default="mysql://rcdb@hallddb.jlab.org/rcdb") - args = parser.parse_args() - - # Open DB connection - db = RCDBProvider(args.connection_string) - - # Get runs from run period - run_min, run_max, description = db.get_run_period("2016-02") - print("Selecting runs {}-{} from run period: '{}'".format(run_min, run_max, description)) - - data_by_date, performance = get_runs_by_date(db, "@status_approved and @is_production", run_min, run_max) - - # Here we can see exact performance budget. All in seconds - print("Performance([sec]) is: ", performance) - - # print resulting array - for run_date in sorted(data_by_date.keys()): - run_values = data_by_date[run_date] - print("'{}' has {} runs {} - {}".format( - run_date, - len(run_values), - run_values[0][0], # run_values[x] is (run_num, start_time, events) => run_values[x][0] is run_num - run_values[-1][0] - )) - - for run_num, start_time, events in run_values: - print("\t\t{}\t{} {}".format(run_num, start_time, events)) diff --git a/python/examples/example_select_halld_values.py b/python/examples/example_select_halld_values.py deleted file mode 100644 index a19d0431..00000000 --- a/python/examples/example_select_halld_values.py +++ /dev/null @@ -1,5 +0,0 @@ -import rcdb.provider -t = rcdb.provider.RCDBProvider('mysql://rcdb@hallddb.jlab.org/rcdb')\ - .select_values(['polarization_angle', 'polarization_direction'], - search_str="@is_production and event_count>100000", run_min=30000, run_max=32000) -print('\n'.join([' '.join(map(str, r)) for r in t])) diff --git a/python/examples/example_simple_queries.py b/python/examples/example_simple_queries.py deleted file mode 100644 index 1a59474d..00000000 --- a/python/examples/example_simple_queries.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -This example shows basics of how to select runs using search queries - -This example requires some live databse - -Usage: - python example_simple_queries.py - -""" -import argparse -import sys -from rcdb import RCDBProvider - - -if __name__ == "__main__": - print(sys.argv) - # Get connection string from arguments - parser = argparse.ArgumentParser(description="This example shows basics of how to select runs using search queries") - parser.add_argument("connection_string", help="RCDB connection string mysql://rcdb@localhost/rcdb", - nargs='?', default='mysql://rcdb@hallddb.jlab.org/rcdb') - args = parser.parse_args() - - # Open DB connection - db = RCDBProvider(args.connection_string) - - # Select production runs with event_count > 0.5M - result = db.select_values(["event_count", "polarization_direction", "beam_current"], - "@is_production and event_count > 500000", 10000, 20000) - - # print title - print("{:>7} {:>15} {:>15} {:>15}".format('run', 'polarization_direction', 'beam_current', 'event_count')) - - # Iterate through results - for row in result: - run, event_count, polarization_direction, beam_current = tuple(row) - print("{:>7} {:>15} {:>15} {:>15}".format(run, polarization_direction, beam_current, event_count)) - diff --git a/python/pyproject.toml b/python/pyproject.toml index cd4434b4..599c56b3 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -45,6 +45,10 @@ dependencies = [ "rich" ] +[project.optional-dependencies] +examples = ["jsonpickle"] + + # If you want a console script, define it in [project.scripts] [project.scripts] rcdb = "rcdb.cli.app:rcdb_cli" diff --git a/python/rcdb/web/__init__.py b/python/rcdb/web/__init__.py index 1a053e69..63896d29 100644 --- a/python/rcdb/web/__init__.py +++ b/python/rcdb/web/__init__.py @@ -21,6 +21,7 @@ PASSWORD = 'default' SQL_CONNECTION_STRING = "mysql+pymysql://rcdb@127.0.0.1/rcdb" + # Get the current directory current_directory = os.path.dirname(os.path.abspath(__file__)) template_folder=os.path.join(current_directory, 'templates') diff --git a/python/rcdb/web/__main__.py b/python/rcdb/web/__main__.py new file mode 100644 index 00000000..936bd1fb --- /dev/null +++ b/python/rcdb/web/__main__.py @@ -0,0 +1,3 @@ +from rcdb.web import app + +app.run(host="0.0.0.0", port=8080, debug=False) \ No newline at end of file