From e528d53a7a47cecf8917974bcc25809a93b0147d Mon Sep 17 00:00:00 2001
From: Dmitrii Golovanov <dmitrii.golovanov@intel.com>
Date: Fri, 13 Sep 2024 20:39:03 +0200
Subject: [PATCH] ci: footprint: Add data transform and upload to ELK

Extend `footprint-tracking` CI workflow with two more steps:

 * pack Memory Footprint data produced by `track.py` script
   into Twister JSON footprint reports (`twister-footprint.json).

 * upload Twister JSON footprint reports into ElasticSearch storage.

Signed-off-by: Dmitrii Golovanov <dmitrii.golovanov@intel.com>
---
 .github/workflows/footprint-tracking.yml | 29 ++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/.github/workflows/footprint-tracking.yml b/.github/workflows/footprint-tracking.yml
index 08b858af477..3868f6f1545 100644
--- a/.github/workflows/footprint-tracking.yml
+++ b/.github/workflows/footprint-tracking.yml
@@ -93,3 +93,32 @@ jobs:
           . .venv/bin/activate
           pip3 install awscli
           aws s3 sync  --quiet footprint_data/ s3://testing.zephyrproject.org/footprint_data/
+
+      - name: Transform Footprint data to Twister JSON reports
+        run: |
+          shopt -s globstar
+          export ZEPHYR_BASE=${PWD}
+          python3 ./scripts/footprint/pack_as_twister.py -vvv \
+            --plan ./scripts/footprint/plan.txt \
+            --test-name='name.feature' \
+            ./footprint_data/*/footprints/*/*/
+
+      - name: Upload to ElasticSearch
+        env:
+          ELASTICSEARCH_KEY: ${{ secrets.ELASTICSEARCH_KEY }}
+          ELASTICSEARCH_SERVER: "https://elasticsearch.zephyrproject.io:443"
+          ELASTICSEARCH_INDEX: ${{ vars.FOOTPRINT_TRACKING_INDEX }}
+        run: |
+          pip3 install -U elasticsearch
+          run_date=`date --iso-8601=minutes`
+          python3 ./scripts/ci/upload_test_results_es.py -r ${run_date} \
+            --flatten footprint \
+            --flatten-list-names "{'children':'name'}" \
+            --transform "{ 'footprint_name': '^(?P<footprint_area>([^\/]+\/){0,2})(?P<footprint_path>([^\/]*\/)*)(?P<footprint_symbol>[^\/]*)$' }" \
+            --run-id "${{ github.run_id }}" \
+            --run-attempt "${{ github.run_attempt }}" \
+            --run-workflow "footprint-tracking:${{ github.event_name }}" \
+            --run-branch "${{ github.ref_name }}" \
+            -i ${ELASTICSEARCH_INDEX} \
+            ./footprint_data/**/twister_footprint.json
+        #