Add task files to push and retrieve ck data log to loki

roles/telemetry_chargeback/README.md

@@ -2,10 +2,13 @@ telemetry_chargeback
 =========
 The **`telemetry_chargeback`** role is designed to test the **RHOSO Cloudkitty** feature. These tests are specific to the Cloudkitty feature. Tests that are not specific to this feature (e.g., standard OpenStack deployment validation, basic networking) should be added to a common role.
 
-The role performs two main functions:
+The role performs four main functions:
 
 1. **CloudKitty Validation** - Enables and configures the CloudKitty hashmap rating module, then validates its state.
-2. **Synthetic Data Generation** - Generates synthetic Loki log data for testing chargeback scenarios using a Python script and Jinja2 template.
+2. **Synthetic Data Generation** - Generates synthetic Loki log data for testing chargeback scenarios using a Python script and 
+Jinja2 template.
+3. **Ingest data and Flush to Loki** - Ingests synthetic CloudKitty log data and Flush Loki Ingester Memory to Storage
+4. **Retrieval of data** - Verifies retrieval of data from loki
 
 Requirements
 ------------
@@ -40,13 +43,13 @@ These variables are used internally by the role and typically do not need to be
 
 | Variable | Default Value | Description |
 |----------|---------------|-------------|
-| `logs_dir_zuul` | `/home/zuul/ci-framework-data/logs` | Remote directory for log files. |
 | `artifacts_dir_zuul` | `/home/zuul/ci-framework-data/artifacts` | Directory for generated artifacts. |
 | `ck_synth_script` | `{{ role_path }}/files/gen_synth_loki_data.py` | Path to the synthetic data generation script. |
 | `ck_data_template` | `{{ role_path }}/template/loki_data_templ.j2` | Path to the Jinja2 template for Loki data format. |
 | `ck_data_config` | `{{ role_path }}/files/test_static.yml` | Path to the scenario configuration file. |
 | `ck_output_file_local` | `{{ artifacts_dir_zuul }}/loki_synth_data.json` | Local path for generated synthetic data. |
-| `ck_output_file_remote` | `{{ logs_dir_zuul }}/gen_loki_synth_data.log` | Remote destination for synthetic data. |
+| `ck_output_file_remote` | `{{ logs_dir }}/gen_loki_synth_data.log` | Remote destination for synthetic data. |
+| `ck_loki_retrieve_file` | `{{ logs_dir }}/retrieve_loki_op.json` | Path where the retrieval of loki data is stored. |
 
 Scenario Configuration
 ----------------------
@@ -62,6 +65,28 @@ Dependencies
 ------------
 This role has no direct hard dependencies on other Ansible roles.
 
+This runs 6 taskfiles
+---------------------
+```yaml
+- name: "Validate Chargeback Feature"
+  ansible.builtin.include_tasks: "chargeback_tests.yml"
+
+- name: "Generate Synthetic Data"
+  ansible.builtin.include_tasks: "gen_synth_loki_data.yml"
+
+- name: "Setup Loki Environment"
+  ansible.builtin.include_tasks: "setup_loki_env.yml"
+
+- name: "Ingests Cloudkitty Data log"
+  ansible.builtin.include_tasks: "ingest_loki_data.yml"
+
+- name: "Flush Data to loki Storage"
+  ansible.builtin.include_tasks: "flush_loki_data.yml"
+
+- name: "Retrieve Data log from loki"
+  ansible.builtin.include_tasks: "retrieve_loki_data.yml"
+```
+
 Example Playbook
 ----------------
 ```yaml
@@ -79,3 +104,4 @@ Author Information
 ------------------
 
 Alex Yefimov, Red Hat
+Muneesha Yadla, Red Hat

roles/telemetry_chargeback/defaults/main.yml

@@ -1,2 +1,25 @@
 ---
 openstack_cmd: "openstack"
+
+# Cloudkitty certificates
+gateway_client_cert_secret_name: "cert-cloudkitty-client-internal"
+gateway_client_cert_dir: "{{ ansible_user_dir }}/ck-certs"
+
+ingester_client_cert_secret: "secret/cloudkitty-lokistack-gateway-client-http"
+ingester_ca_configmap: "cm/cloudkitty-lokistack-ca-bundle"
+ingester_client_cert_dir: "{{ ansible_user_dir }}/flush_certs"
+osp_remote_cert_dir: "osp-certs"
+
+# Loki Ingest & Retrieve logs
+ck_output_file_remote: "{{ logs_dir }}/gen_loki_synth_data.log"
+ck_loki_retrieve_file: "{{ logs_dir }}/retrieve_loki_op.json"
+
+# LogQL Query
+logql_query: "{{ loki_query | default('{service=\"cloudkitty\"}') }}"
+
+# vars
+openstack_ns: "openstack"
+openstackpod: "openstackclient"
+
+# Time window settings
+lookback: 6

roles/telemetry_chargeback/tasks/flush_loki_data.yml

@@ -0,0 +1,52 @@
+---
+# Flush Loki Ingester Memory to Storage
+
+- name: Flush Execution inside openstack CLI
+  block:
+    # create dir 
+    - name: Create directory inside openstack CLI
+      ansible.builtin.command:
+        cmd: "oc exec -n {{ openstack_ns }} {{ openstackpod }} -- mkdir -p {{ osp_remote_cert_dir }}"
+      changed_when: false
+
+    # copy all certs
+    - name: Copy certificates to openstack CLI
+      ansible.builtin.command:
+        cmd: "oc cp {{ ingester_client_cert_dir }}/. {{ openstack_ns }}/{{ openstackpod }}:{{ osp_remote_cert_dir }}/"
+      changed_when: true
+
+    # flush loki
+    - name: Trigger Flush
+      ansible.builtin.command:
+        cmd: >
+          oc exec -n {{ openstack_ns }} {{ openstackpod }} --
+          curl -v -X POST {{ ingester_flush_url }}
+          --cert {{ osp_remote_cert_dir }}/tls.crt
+          --key {{ osp_remote_cert_dir }}/tls.key
+          --cacert {{ osp_remote_cert_dir }}/service-ca.crt
+      register: flush_response
+      changed_when: true
+      failed_when: flush_response.rc != 0
+
+    # Status
+    - name: Verify Flush Status
+      ansible.builtin.assert:
+        that:
+          - "'204' in flush_response.stderr or '200' in flush_response.stderr"
+        fail_msg: "Flush failed"
+        success_msg: "Ingester Memory Flushed successfully"
+
+  rescue:
+    - name: Debug Failure Output
+      ansible.builtin.debug:
+        msg: 
+          - "Failure"
+          - "Stdout: {{ flush_response.stdout }}"
+          - "Stderr: {{ flush_response.stderr }}"
+
+  always:
+    # Cleanup
+    - name: Cleanup local certificates
+      ansible.builtin.file:
+        path: "{{ ingester_client_cert_dir }}"
+        state: absent

roles/telemetry_chargeback/tasks/ingest_loki_data.yml

@@ -0,0 +1,43 @@
+---
+# Ingest data log to Loki that is generated from gen_synth_loki_data.yml"
+
+# Push the json format data log to loki
+- name: Ingest data log to Loki via API
+  block:
+
+    - name: Read log file content
+      ansible.builtin.slurp:
+        src: "{{ ck_output_file_remote }}"
+      register: log_file_content
+
+    - name: Push data to Loki
+      ansible.builtin.uri:
+        url: "{{ loki_push_url }}"
+        method: POST
+        body: "{{ log_file_content['content'] | b64decode | from_json }}"
+        body_format: json
+        client_cert: "{{ gateway_client_cert_dir }}/tls.crt"
+        client_key: "{{ gateway_client_cert_dir }}/tls.key"
+        validate_certs: false
+        status_code: 204
+        return_content: yes
+      register: loki_response
+      ignore_errors: false
+      failed_when: loki_response.status != 204
+
+    # Success
+    - name: Confirm Success
+      ansible.builtin.debug:
+        msg: "Ingestion Successful!"
+
+  rescue:
+    # Rescue block
+    - name: Debug failure
+      ansible.builtin.debug:
+        msg: "{{ loki_response.status }}"
+
+    # Failure
+    - name: Report Ingestion Failure
+      ansible.builtin.fail:
+        msg: "Ingestion Failed"
+      ignore_errors: false

roles/telemetry_chargeback/tasks/main.yml

@@ -1,6 +1,25 @@
 ---
-- name: "Validate Chargeback Feature"
-  ansible.builtin.include_tasks: "chargeback_tests.yml"
+- name: Validate Chargeback Feature
+  ansible.builtin.include_tasks: 
+    file: chargeback_tests.yml
+
+- name: Generate Synthetic Data
+  ansible.builtin.include_tasks: 
+    file: gen_synth_loki_data.yml
+
+- name: Setup Loki Environment
+  ansible.builtin.include_tasks: 
+    file: setup_loki_env.yml
+
+- name: Ingests Cloudkitty Data log
+  ansible.builtin.include_tasks: 
+    file: ingest_loki_data.yml
+
+- name: Flush Data to Loki Storage
+  ansible.builtin.include_tasks: 
+    file: flush_loki_data.yml
+
+- name: Retrieve Data log from Loki
+  ansible.builtin.include_tasks: 
+    file: retrieve_loki_data.yml
 
-- name: "Generate Synthetic Data"
-  ansible.builtin.include_tasks: "gen_synth_loki_data.yml"

roles/telemetry_chargeback/tasks/retrieve_loki_data.yml

@@ -0,0 +1,89 @@
+---
+# Query Loki to retrieve data and output to retrieve_loki_op.json
+
+# Count the entries in the log file
+- name: Read the json data log file to calculate the no. of entries
+  ansible.builtin.slurp:
+    src: "{{ ck_output_file_remote }}"
+  register: source_file_data
+
+- name: Set Expected Log Count
+  ansible.builtin.set_fact:
+    expected_log_count: "{{ (source_file_data['content'] | b64decode | from_json).streams | map(attribute='values') | map('length') | sum }}"
+
+- name: Expected Count
+  ansible.builtin.debug:
+    msg: "Input file has {{ expected_log_count }} data entries that Loki has to return"
+
+# Calculate Time
+- name: Calculate Start Time in nanoseconds
+  ansible.builtin.command: date -d "{{ lookback }} days ago" +%s000000000
+  register: nano_time
+  changed_when: false
+
+- name: Set Start Time
+  ansible.builtin.set_fact:
+    start_time: "{{ nano_time.stdout }}"
+
+- name: Display Query Parameters
+  ansible.builtin.debug:
+    msg: 
+      - "Query: {{ logql_query }}"
+      - "Start Time: {{ start_time }}"
+
+# Query Loki
+- name: Retrieve Logs from Loki via API
+  block:
+    - name: Query Loki API
+      ansible.builtin.uri:
+        url: "{{ loki_query_url }}?query={{ logql_query | urlencode }}&start={{ start_time }}"
+        method: GET
+        client_cert: "{{ gateway_client_cert_dir }}/tls.crt"
+        client_key: "{{ gateway_client_cert_dir }}/tls.key"
+        ca_path: "{{ gateway_client_cert_dir }}/ca.crt"
+        validate_certs: false
+        return_content: yes
+        body_format: json
+      register: loki_response
+      # Wait condition
+      until:
+        - loki_response.status == 200
+        - loki_response.json.status == 'success'
+        - loki_response.json.data.result | length > 0
+        - (loki_response.json.data.result | map(attribute='values') | map('length') | sum) >= expected_log_count|int
+      retries: 25
+      delay: 60
+
+    # Save data
+    - name: Save Loki Data to JSON file
+      ansible.builtin.copy:
+        content: "{{ loki_response.json | to_nice_json }}"
+        dest: "{{ ck_loki_retrieve_file }}"
+        mode: '0644'
+
+    # Validate
+    - name: Verify Data Integrity
+      vars:
+        actual_count: "{{ loki_response.json.data.result | map(attribute='values') | map('length') | sum }}"
+      ansible.builtin.assert:
+        that:
+          - loki_response.json.status == 'success'
+          - loki_response.json.data.result | length > 0
+          - actual_count|int == expected_log_count|int
+        fail_msg: "Query did not return all data entries. Expected {{ expected_log_count }} log entries, but Loki only returned {{ actual_count }}"
+        success_msg: "Query returned all data entries. Input file had {{ expected_log_count }} entries and Loki returned {{ actual_count }}"
+      ignore_errors: false
+
+  rescue:
+    - name: Debug failure
+      ansible.builtin.debug:
+        msg: 
+          - "Status: {{ loki_response.status | default('Unknown') }}"
+          - "Body: {{ loki_response.content | default('No Content') }}"
+          - "Msg: {{ loki_response.msg | default('Request failed') }}"
+
+    # Failure
+    - name: Report Retrieval Failure
+      ansible.builtin.fail:
+        msg: "Retrieval Failed"
+      ignore_errors: false

roles/telemetry_chargeback/tasks/setup_loki_env.yml

@@ -0,0 +1,70 @@
+---
+# Setup Loki Environment
+
+# Dynamic URL's
+- name: Get Loki Public Route Host
+  ansible.builtin.command:
+    cmd: oc get route cloudkitty-lokistack -n {{ openstack_ns }} -o jsonpath='{.spec.host}'
+  register: loki_route
+  changed_when: false
+
+- name: Set Loki URLs
+  ansible.builtin.set_fact:
+    # Base URL
+    loki_base_url: "https://{{ loki_route.stdout }}"
+
+    # Internal Flush URL (Service DNS: https://<service>.<openstack_ns>.svc:3100/flush)
+    ingester_flush_url: "https://cloudkitty-lokistack-ingester-http.{{ openstack_ns }}.svc:3100/flush"
+
+- name: Set Derived Loki URLs
+  ansible.builtin.set_fact:
+    loki_push_url: "{{ loki_base_url }}/api/logs/v1/cloudkitty/loki/api/v1/push"
+    loki_query_url: "{{ loki_base_url }}/api/logs/v1/cloudkitty/loki/api/v1/query_range"
+
+- name: Debug URLs
+  ansible.builtin.debug:
+    msg: 
+      - "Loki Route: {{ loki_base_url }}"
+      - "Push URL:   {{ loki_push_url }}"
+      - "Flush URL:  {{ ingester_flush_url }}"
+      - "Query URL:  {{ loki_query_url }}"
+
+# Certs to Ingest & Retrieve data to/from Loki
+- name: Ensure Local Certificate Directory Exists
+  ansible.builtin.file:
+    path: "{{ gateway_client_cert_dir }}"
+    state: directory
+    mode: '0755'
+
+- name: Extract Certificates from Openshift Secret
+  ansible.builtin.command:
+    cmd: >
+      oc extract secret/{{ gateway_client_cert_secret_name }}
+      --to={{ gateway_client_cert_dir }}
+      --confirm
+      -n {{ openstack_ns }}
+  changed_when: true
+
+# Certs to Flush data to Loki
+- name: Create a directory to extract certificates
+  ansible.builtin.file:
+    path: "{{ ingester_client_cert_dir }}"
+    state: directory
+    mode: '0755'
+
+- name: Extract Client Certificates
+  ansible.builtin.command:
+    cmd: > 
+      oc extract {{ ingester_client_cert_secret }}
+      --to={{ ingester_client_cert_dir }}
+      --confirm
+      -n {{ openstack_ns }}
+  changed_when: true
+
+- name: Extract CA Bundle
+  ansible.builtin.command:
+    cmd: "oc extract {{ ingester_ca_configmap }}
+    --to={{ ingester_client_cert_dir }}
+    --confirm
+    -n {{ openstack_ns }}"
+  changed_when: true

roles/telemetry_chargeback/vars/main.yml

@@ -1,9 +1,7 @@
 ---
-logs_dir_zuul: "/home/zuul/ci-framework-data/logs"
 artifacts_dir_zuul: "/home/zuul/ci-framework-data/artifacts"
 
 ck_synth_script: "{{ role_path }}/files/gen_synth_loki_data.py"
 ck_data_template: "{{ role_path }}/template/loki_data_templ.j2"
 ck_data_config: "{{ role_path }}/files/test_static.yml"
 ck_output_file_local: "{{ artifacts_dir_zuul }}/loki_synth_data.json"
-ck_output_file_remote: "{{ logs_dir_zuul }}/gen_loki_synth_data.log"