diff --git a/roles/telemetry_chargeback/README.md b/roles/telemetry_chargeback/README.md index 192b72a3d..ecc19bf35 100644 --- a/roles/telemetry_chargeback/README.md +++ b/roles/telemetry_chargeback/README.md @@ -2,10 +2,13 @@ telemetry_chargeback ========= The **`telemetry_chargeback`** role is designed to test the **RHOSO Cloudkitty** feature. These tests are specific to the Cloudkitty feature. Tests that are not specific to this feature (e.g., standard OpenStack deployment validation, basic networking) should be added to a common role. -The role performs two main functions: +The role performs four main functions: 1. **CloudKitty Validation** - Enables and configures the CloudKitty hashmap rating module, then validates its state. -2. **Synthetic Data Generation** - Generates synthetic Loki log data for testing chargeback scenarios using a Python script and Jinja2 template. +2. **Synthetic Data Generation** - Generates synthetic Loki log data for testing chargeback scenarios using a Python script and +Jinja2 template. +3. **Ingest data and Flush to Loki** - Ingests synthetic CloudKitty log data and Flush Loki Ingester Memory to Storage +4. **Retrieval of data** - Verifies retrieval of data from loki Requirements ------------ @@ -40,13 +43,13 @@ These variables are used internally by the role and typically do not need to be | Variable | Default Value | Description | |----------|---------------|-------------| -| `logs_dir_zuul` | `/home/zuul/ci-framework-data/logs` | Remote directory for log files. | | `artifacts_dir_zuul` | `/home/zuul/ci-framework-data/artifacts` | Directory for generated artifacts. | | `ck_synth_script` | `{{ role_path }}/files/gen_synth_loki_data.py` | Path to the synthetic data generation script. | | `ck_data_template` | `{{ role_path }}/template/loki_data_templ.j2` | Path to the Jinja2 template for Loki data format. | | `ck_data_config` | `{{ role_path }}/files/test_static.yml` | Path to the scenario configuration file. | | `ck_output_file_local` | `{{ artifacts_dir_zuul }}/loki_synth_data.json` | Local path for generated synthetic data. | -| `ck_output_file_remote` | `{{ logs_dir_zuul }}/gen_loki_synth_data.log` | Remote destination for synthetic data. | +| `ck_output_file_remote` | `{{ logs_dir }}/gen_loki_synth_data.log` | Remote destination for synthetic data. | +| `ck_loki_retrieve_file` | `{{ logs_dir }}/retrieve_loki_op.json` | Path where the retrieval of loki data is stored. | Scenario Configuration ---------------------- @@ -62,6 +65,28 @@ Dependencies ------------ This role has no direct hard dependencies on other Ansible roles. +This runs 6 taskfiles +--------------------- +```yaml +- name: "Validate Chargeback Feature" + ansible.builtin.include_tasks: "chargeback_tests.yml" + +- name: "Generate Synthetic Data" + ansible.builtin.include_tasks: "gen_synth_loki_data.yml" + +- name: "Setup Loki Environment" + ansible.builtin.include_tasks: "setup_loki_env.yml" + +- name: "Ingests Cloudkitty Data log" + ansible.builtin.include_tasks: "ingest_loki_data.yml" + +- name: "Flush Data to loki Storage" + ansible.builtin.include_tasks: "flush_loki_data.yml" + +- name: "Retrieve Data log from loki" + ansible.builtin.include_tasks: "retrieve_loki_data.yml" +``` + Example Playbook ---------------- ```yaml @@ -79,3 +104,4 @@ Author Information ------------------ Alex Yefimov, Red Hat +Muneesha Yadla, Red Hat diff --git a/roles/telemetry_chargeback/defaults/main.yml b/roles/telemetry_chargeback/defaults/main.yml index 64f07b7a1..c4a7c17e6 100644 --- a/roles/telemetry_chargeback/defaults/main.yml +++ b/roles/telemetry_chargeback/defaults/main.yml @@ -1,2 +1,25 @@ --- openstack_cmd: "openstack" + +# Cloudkitty certificates +gateway_client_cert_secret_name: "cert-cloudkitty-client-internal" +gateway_client_cert_dir: "{{ ansible_user_dir }}/ck-certs" + +ingester_client_cert_secret: "secret/cloudkitty-lokistack-gateway-client-http" +ingester_ca_configmap: "cm/cloudkitty-lokistack-ca-bundle" +ingester_client_cert_dir: "{{ ansible_user_dir }}/flush_certs" +osp_remote_cert_dir: "osp-certs" + +# Loki Ingest & Retrieve logs +ck_output_file_remote: "{{ logs_dir }}/gen_loki_synth_data.log" +ck_loki_retrieve_file: "{{ logs_dir }}/retrieve_loki_op.json" + +# LogQL Query +logql_query: "{{ loki_query | default('{service=\"cloudkitty\"}') }}" + +# vars +openstack_ns: "openstack" +openstackpod: "openstackclient" + +# Time window settings +lookback: 6 diff --git a/roles/telemetry_chargeback/tasks/flush_loki_data.yml b/roles/telemetry_chargeback/tasks/flush_loki_data.yml new file mode 100644 index 000000000..4946df5c7 --- /dev/null +++ b/roles/telemetry_chargeback/tasks/flush_loki_data.yml @@ -0,0 +1,52 @@ +--- +# Flush Loki Ingester Memory to Storage + +- name: Flush Execution inside openstack CLI + block: + # create dir + - name: Create directory inside openstack CLI + ansible.builtin.command: + cmd: "oc exec -n {{ openstack_ns }} {{ openstackpod }} -- mkdir -p {{ osp_remote_cert_dir }}" + changed_when: false + + # copy all certs + - name: Copy certificates to openstack CLI + ansible.builtin.command: + cmd: "oc cp {{ ingester_client_cert_dir }}/. {{ openstack_ns }}/{{ openstackpod }}:{{ osp_remote_cert_dir }}/" + changed_when: true + + # flush loki + - name: Trigger Flush + ansible.builtin.command: + cmd: > + oc exec -n {{ openstack_ns }} {{ openstackpod }} -- + curl -v -X POST {{ ingester_flush_url }} + --cert {{ osp_remote_cert_dir }}/tls.crt + --key {{ osp_remote_cert_dir }}/tls.key + --cacert {{ osp_remote_cert_dir }}/service-ca.crt + register: flush_response + changed_when: true + failed_when: flush_response.rc != 0 + + # Status + - name: Verify Flush Status + ansible.builtin.assert: + that: + - "'204' in flush_response.stderr or '200' in flush_response.stderr" + fail_msg: "Flush failed" + success_msg: "Ingester Memory Flushed successfully" + + rescue: + - name: Debug Failure Output + ansible.builtin.debug: + msg: + - "Failure" + - "Stdout: {{ flush_response.stdout }}" + - "Stderr: {{ flush_response.stderr }}" + + always: + # Cleanup + - name: Cleanup local certificates + ansible.builtin.file: + path: "{{ ingester_client_cert_dir }}" + state: absent diff --git a/roles/telemetry_chargeback/tasks/ingest_loki_data.yml b/roles/telemetry_chargeback/tasks/ingest_loki_data.yml new file mode 100644 index 000000000..82c91b33f --- /dev/null +++ b/roles/telemetry_chargeback/tasks/ingest_loki_data.yml @@ -0,0 +1,43 @@ +--- +# Ingest data log to Loki that is generated from gen_synth_loki_data.yml" + +# Push the json format data log to loki +- name: Ingest data log to Loki via API + block: + + - name: Read log file content + ansible.builtin.slurp: + src: "{{ ck_output_file_remote }}" + register: log_file_content + + - name: Push data to Loki + ansible.builtin.uri: + url: "{{ loki_push_url }}" + method: POST + body: "{{ log_file_content['content'] | b64decode | from_json }}" + body_format: json + client_cert: "{{ gateway_client_cert_dir }}/tls.crt" + client_key: "{{ gateway_client_cert_dir }}/tls.key" + validate_certs: false + status_code: 204 + return_content: yes + register: loki_response + ignore_errors: false + failed_when: loki_response.status != 204 + + # Success + - name: Confirm Success + ansible.builtin.debug: + msg: "Ingestion Successful!" + + rescue: + # Rescue block + - name: Debug failure + ansible.builtin.debug: + msg: "{{ loki_response.status }}" + + # Failure + - name: Report Ingestion Failure + ansible.builtin.fail: + msg: "Ingestion Failed" + ignore_errors: false diff --git a/roles/telemetry_chargeback/tasks/main.yml b/roles/telemetry_chargeback/tasks/main.yml index 98a94b233..ec33e2c21 100644 --- a/roles/telemetry_chargeback/tasks/main.yml +++ b/roles/telemetry_chargeback/tasks/main.yml @@ -1,6 +1,25 @@ --- -- name: "Validate Chargeback Feature" - ansible.builtin.include_tasks: "chargeback_tests.yml" +- name: Validate Chargeback Feature + ansible.builtin.include_tasks: + file: chargeback_tests.yml + +- name: Generate Synthetic Data + ansible.builtin.include_tasks: + file: gen_synth_loki_data.yml + +- name: Setup Loki Environment + ansible.builtin.include_tasks: + file: setup_loki_env.yml + +- name: Ingests Cloudkitty Data log + ansible.builtin.include_tasks: + file: ingest_loki_data.yml + +- name: Flush Data to Loki Storage + ansible.builtin.include_tasks: + file: flush_loki_data.yml + +- name: Retrieve Data log from Loki + ansible.builtin.include_tasks: + file: retrieve_loki_data.yml -- name: "Generate Synthetic Data" - ansible.builtin.include_tasks: "gen_synth_loki_data.yml" diff --git a/roles/telemetry_chargeback/tasks/retrieve_loki_data.yml b/roles/telemetry_chargeback/tasks/retrieve_loki_data.yml new file mode 100644 index 000000000..5c5b63ee2 --- /dev/null +++ b/roles/telemetry_chargeback/tasks/retrieve_loki_data.yml @@ -0,0 +1,89 @@ +--- +# Query Loki to retrieve data and output to retrieve_loki_op.json + +# Count the entries in the log file +- name: Read the json data log file to calculate the no. of entries + ansible.builtin.slurp: + src: "{{ ck_output_file_remote }}" + register: source_file_data + +- name: Set Expected Log Count + ansible.builtin.set_fact: + expected_log_count: "{{ (source_file_data['content'] | b64decode | from_json).streams | map(attribute='values') | map('length') | sum }}" + +- name: Expected Count + ansible.builtin.debug: + msg: "Input file has {{ expected_log_count }} data entries that Loki has to return" + +# Calculate Time +- name: Calculate Start Time in nanoseconds + ansible.builtin.command: date -d "{{ lookback }} days ago" +%s000000000 + register: nano_time + changed_when: false + +- name: Set Start Time + ansible.builtin.set_fact: + start_time: "{{ nano_time.stdout }}" + +- name: Display Query Parameters + ansible.builtin.debug: + msg: + - "Query: {{ logql_query }}" + - "Start Time: {{ start_time }}" + +# Query Loki +- name: Retrieve Logs from Loki via API + block: + - name: Query Loki API + ansible.builtin.uri: + url: "{{ loki_query_url }}?query={{ logql_query | urlencode }}&start={{ start_time }}" + method: GET + client_cert: "{{ gateway_client_cert_dir }}/tls.crt" + client_key: "{{ gateway_client_cert_dir }}/tls.key" + ca_path: "{{ gateway_client_cert_dir }}/ca.crt" + validate_certs: false + return_content: yes + body_format: json + register: loki_response + # Wait condition + until: + - loki_response.status == 200 + - loki_response.json.status == 'success' + - loki_response.json.data.result | length > 0 + - (loki_response.json.data.result | map(attribute='values') | map('length') | sum) >= expected_log_count|int + retries: 25 + delay: 60 + + # Save data + - name: Save Loki Data to JSON file + ansible.builtin.copy: + content: "{{ loki_response.json | to_nice_json }}" + dest: "{{ ck_loki_retrieve_file }}" + mode: '0644' + + # Validate + - name: Verify Data Integrity + vars: + actual_count: "{{ loki_response.json.data.result | map(attribute='values') | map('length') | sum }}" + ansible.builtin.assert: + that: + - loki_response.json.status == 'success' + - loki_response.json.data.result | length > 0 + - actual_count|int == expected_log_count|int + fail_msg: "Query did not return all data entries. Expected {{ expected_log_count }} log entries, but Loki only returned {{ actual_count }}" + success_msg: "Query returned all data entries. Input file had {{ expected_log_count }} entries and Loki returned {{ actual_count }}" + ignore_errors: false + + rescue: + - name: Debug failure + ansible.builtin.debug: + msg: + - "Status: {{ loki_response.status | default('Unknown') }}" + - "Body: {{ loki_response.content | default('No Content') }}" + - "Msg: {{ loki_response.msg | default('Request failed') }}" + + # Failure + - name: Report Retrieval Failure + ansible.builtin.fail: + msg: "Retrieval Failed" + ignore_errors: false diff --git a/roles/telemetry_chargeback/tasks/setup_loki_env.yml b/roles/telemetry_chargeback/tasks/setup_loki_env.yml new file mode 100644 index 000000000..0f81ace0a --- /dev/null +++ b/roles/telemetry_chargeback/tasks/setup_loki_env.yml @@ -0,0 +1,70 @@ +--- +# Setup Loki Environment + +# Dynamic URL's +- name: Get Loki Public Route Host + ansible.builtin.command: + cmd: oc get route cloudkitty-lokistack -n {{ openstack_ns }} -o jsonpath='{.spec.host}' + register: loki_route + changed_when: false + +- name: Set Loki URLs + ansible.builtin.set_fact: + # Base URL + loki_base_url: "https://{{ loki_route.stdout }}" + + # Internal Flush URL (Service DNS: https://..svc:3100/flush) + ingester_flush_url: "https://cloudkitty-lokistack-ingester-http.{{ openstack_ns }}.svc:3100/flush" + +- name: Set Derived Loki URLs + ansible.builtin.set_fact: + loki_push_url: "{{ loki_base_url }}/api/logs/v1/cloudkitty/loki/api/v1/push" + loki_query_url: "{{ loki_base_url }}/api/logs/v1/cloudkitty/loki/api/v1/query_range" + +- name: Debug URLs + ansible.builtin.debug: + msg: + - "Loki Route: {{ loki_base_url }}" + - "Push URL: {{ loki_push_url }}" + - "Flush URL: {{ ingester_flush_url }}" + - "Query URL: {{ loki_query_url }}" + +# Certs to Ingest & Retrieve data to/from Loki +- name: Ensure Local Certificate Directory Exists + ansible.builtin.file: + path: "{{ gateway_client_cert_dir }}" + state: directory + mode: '0755' + +- name: Extract Certificates from Openshift Secret + ansible.builtin.command: + cmd: > + oc extract secret/{{ gateway_client_cert_secret_name }} + --to={{ gateway_client_cert_dir }} + --confirm + -n {{ openstack_ns }} + changed_when: true + +# Certs to Flush data to Loki +- name: Create a directory to extract certificates + ansible.builtin.file: + path: "{{ ingester_client_cert_dir }}" + state: directory + mode: '0755' + +- name: Extract Client Certificates + ansible.builtin.command: + cmd: > + oc extract {{ ingester_client_cert_secret }} + --to={{ ingester_client_cert_dir }} + --confirm + -n {{ openstack_ns }} + changed_when: true + +- name: Extract CA Bundle + ansible.builtin.command: + cmd: "oc extract {{ ingester_ca_configmap }} + --to={{ ingester_client_cert_dir }} + --confirm + -n {{ openstack_ns }}" + changed_when: true diff --git a/roles/telemetry_chargeback/vars/main.yml b/roles/telemetry_chargeback/vars/main.yml index 1014a6a9e..aaae0c086 100644 --- a/roles/telemetry_chargeback/vars/main.yml +++ b/roles/telemetry_chargeback/vars/main.yml @@ -1,9 +1,7 @@ --- -logs_dir_zuul: "/home/zuul/ci-framework-data/logs" artifacts_dir_zuul: "/home/zuul/ci-framework-data/artifacts" ck_synth_script: "{{ role_path }}/files/gen_synth_loki_data.py" ck_data_template: "{{ role_path }}/template/loki_data_templ.j2" ck_data_config: "{{ role_path }}/files/test_static.yml" ck_output_file_local: "{{ artifacts_dir_zuul }}/loki_synth_data.json" -ck_output_file_remote: "{{ logs_dir_zuul }}/gen_loki_synth_data.log"