diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 44dbcd64d..000000000 --- a/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.pyc -.idea/ diff --git a/roles/telemetry_chargeback/defaults/main.yml b/roles/telemetry_chargeback/defaults/main.yml index 7db48d026..cd8bacc78 100644 --- a/roles/telemetry_chargeback/defaults/main.yml +++ b/roles/telemetry_chargeback/defaults/main.yml @@ -28,3 +28,6 @@ openstackpod: "openstackclient" # Time window settings lookback: 6 limit: 50 + +# List of test scenario files to run +cloudkitty_test_scenarios: [] diff --git a/roles/telemetry_chargeback/files/gen_synth_loki_data.py b/roles/telemetry_chargeback/files/gen_synth_loki_data.py index 0af796378..2c0c2f77a 100755 --- a/roles/telemetry_chargeback/files/gen_synth_loki_data.py +++ b/roles/telemetry_chargeback/files/gen_synth_loki_data.py @@ -48,7 +48,7 @@ def _get_value_for_step( logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', - datefmt='%Y-%m-%dT%H:%M:%S+00:00' + datefmt='%Y-%m-%dT%H:%M:%SZ' ) logger = logging.getLogger() @@ -62,11 +62,11 @@ def _format_timestamp(epoch_seconds: float, invalid_timestamp: str) -> str: invalid_timestamp (str): String to return for invalid timestamps. Returns: - str: The formatted datetime string (e.g., "2023-10-26T14:30:00+00:00"). + str: The formatted datetime string (e.g., "2023-10-26T14:30:00Z"). """ try: dt_object = datetime.fromtimestamp(epoch_seconds, tz=timezone.utc) - return dt_object.isoformat() + return dt_object.strftime('%Y-%m-%dT%H:%M:%SZ') except (ValueError, TypeError): logger.warning(f"Invalid epoch value provided: {epoch_seconds}") return invalid_timestamp diff --git a/roles/telemetry_chargeback/tasks/retrieve_loki_data.yml b/roles/telemetry_chargeback/tasks/retrieve_loki_data.yml new file mode 100644 index 000000000..e21115291 --- /dev/null +++ b/roles/telemetry_chargeback/tasks/retrieve_loki_data.yml @@ -0,0 +1,71 @@ +--- +- name: "Expected Count {{ scenario_name }}" + ansible.builtin.debug: + msg: "Input file has {{ synth_data_rates.data_log.log_count }} data entries that Loki has to return" + +# Query Loki +- name: "Retrieve Logs from Loki via API {{ scenario_name }}" + block: + - name: "Query Loki API" + ansible.builtin.uri: + url: "{{ loki_query_url }}?query={{ logql_query | urlencode }}&start={{ synth_data_rates.time.begin_step.nanosec }}&limit={{ limit }}" + method: GET + client_cert: "{{ cert_dir }}/tls.crt" + client_key: "{{ cert_dir }}/tls.key" + ca_path: "{{ cert_dir }}/ca.crt" + validate_certs: false + return_content: true + body_format: json + register: loki_response + # Wait condition + until: + - loki_response.status == 200 + - loki_response.json.status == 'success' + - loki_response.json.data.result | length > 0 + - (loki_response.json.data.result | map(attribute='values') | map('length') | sum) >= (synth_data_rates.data_log.log_count | int) + retries: 25 + delay: 60 + + - name: "Save Loki Data to JSON file" + ansible.builtin.copy: + content: "{{ loki_response.json | to_json }}" + dest: "{{ artifacts_dir_zuul }}/{{ scenario_name }}{{ cloudkitty_loki_data_suffix }}" + mode: '0644' + + # Validate + - name: "Verify Data Integrity {{ scenario_name }}" + vars: + actual_count: "{{ loki_response.json.data.result | map(attribute='values') | map('length') | sum }}" + ansible.builtin.assert: + that: + - loki_response.json.status == 'success' + - loki_response.json.data.result | length > 0 + - actual_count | int == (synth_data_rates.data_log.log_count | int) + fail_msg: >- + Query did not return all data entries. Expected + {{ synth_data_rates.data_log.log_count }} log entries, but Loki + only returned {{ actual_count }} + success_msg: "Query returned all data entries. Input file had {{ synth_data_rates.data_log.log_count }} entries and Loki returned {{ actual_count }}" + + rescue: + - name: "Debug failure" + ansible.builtin.debug: + msg: + - "Status: {{ loki_response.status | default('Unknown') }}" + - "Body: {{ loki_response.content | default('No Content') }}" + - "Msg: {{ loki_response.msg | default('Request failed') }}" + + # Failure + - name: "Report Retrieval Failure" + ansible.builtin.fail: + msg: "Retrieval Failed" + +- name: "Generate chargeback stats from Loki-retrieved data file: {{ scenario_name }}" + ansible.builtin.command: + cmd: > + python3 "{{ cloudkitty_summary_script }}" + -j "{{ artifacts_dir_zuul }}/{{ scenario_name }}{{ cloudkitty_loki_data_suffix }}" + -o "{{ artifacts_dir_zuul }}/{{ scenario_name }}{{ cloudkitty_loki_totals_metrics_suffix }}" + --debug "{{ cloudkitty_debug_dir }}" + register: synth_rating_info + changed_when: synth_rating_info.rc == 0 diff --git a/roles/telemetry_chargeback/vars/main.yml b/roles/telemetry_chargeback/vars/main.yml index 2054c1b5b..3917ac1f3 100644 --- a/roles/telemetry_chargeback/vars/main.yml +++ b/roles/telemetry_chargeback/vars/main.yml @@ -1,14 +1,13 @@ --- cloudkitty_synth_script: "{{ role_path }}/files/gen_synth_loki_data.py" cloudkitty_data_template: "{{ role_path }}/templates/loki_data_templ.j2" +cloudkitty_summary_script: "{{ role_path }}/files/gen_db_summary.py" + +# Legacy variables (deprecated, kept for compatibility) ck_data_config: "{{ role_path }}/files/test_static.yml" ck_output_file_local: "{{ artifacts_dir_zuul }}/loki_synth_data.json" ck_output_file_remote: "{{ logs_dir_zuul }}/gen_loki_synth_data.log" -# Scenario and script paths (using role_path) -cloudkitty_scenario_dir: "{{ role_path }}/files" -cloudkitty_summary_script: "{{ role_path }}/files/gen_db_summary.py" - # File naming conventions (internal standardization) cloudkitty_synth_data_suffix: "-synth_data.json" cloudkitty_loki_data_suffix: "-loki_data.json"