[DEVOPS-18]: Add map file parser, mariadb inserter (flipperdevices#2732)

.github/workflows/build.yml

@@ -100,25 +100,31 @@ jobs:
           cp build/f7-firmware-*/firmware.elf map_analyser_files/firmware.elf
           cp ${{ github.event_path }} map_analyser_files/event.json
 
-      - name: 'Upload map analyser files to storage'
+      - name: 'Analyse map file'
         if: ${{ !github.event.pull_request.head.repo.fork }}
-        uses: prewk/s3-cp-action@v2
-        with:
-          aws_s3_endpoint: "${{ secrets.MAP_REPORT_AWS_ENDPOINT }}"
-          aws_access_key_id: "${{ secrets.MAP_REPORT_AWS_ACCESS_KEY }}"
-          aws_secret_access_key: "${{ secrets.MAP_REPORT_AWS_SECRET_KEY }}"
-          source: "./map_analyser_files/"
-          dest: "s3://${{ secrets.MAP_REPORT_AWS_BUCKET }}/${{steps.names.outputs.random_hash}}"
-          flags: "--recursive --acl public-read"
-
-      - name: 'Trigger map file reporter'
-        if: ${{ !github.event.pull_request.head.repo.fork }}
-        uses: peter-evans/repository-dispatch@v2
-        with:
-          repository: flipperdevices/flipper-map-reporter
-          token: ${{ secrets.REPOSITORY_DISPATCH_TOKEN }}
-          event-type: map-file-analyse
-          client-payload: '{"random_hash": "${{steps.names.outputs.random_hash}}", "event_type": "${{steps.names.outputs.event_type}}"}'
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          get_size()
+          {
+            SECTION="$1";
+            arm-none-eabi-size \
+              -A map_analyser_files/firmware.elf \
+              | grep "^$SECTION" | awk '{print $2}'
+          }
+          export BSS_SIZE="$(get_size ".bss")"
+          export TEXT_SIZE="$(get_size ".text")"
+          export RODATA_SIZE="$(get_size ".rodata")"
+          export DATA_SIZE="$(get_size ".data")"
+          export FREE_FLASH_SIZE="$(get_size ".free_flash")"
+          python3 -m pip install mariadb==1.1.6 cxxfilt==0.3.0
+          python3 scripts/map_parser.py map_analyser_files/firmware.elf.map map_analyser_files/firmware.elf.map.all
+          python3 scripts/map_mariadb_insert.py \
+              ${{ secrets.AMAP_MARIADB_USER }} \
+              ${{ secrets.AMAP_MARIADB_PASSWORD }} \
+              ${{ secrets.AMAP_MARIADB_HOST }} \
+              ${{ secrets.AMAP_MARIADB_PORT }} \
+              ${{ secrets.AMAP_MARIADB_DATABASE }} \
+              map_analyser_files/firmware.elf.map.all
 
       - name: 'Upload artifacts to update server'
         if: ${{ !github.event.pull_request.head.repo.fork }}

scripts/map_mariadb_insert.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+
+# Requiremets:
+#   mariadb==1.1.6
+
+from datetime import datetime
+import argparse
+import mariadb
+import sys
+import os
+
+
+def parseArgs():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("db_user", help="MariaDB user")
+    parser.add_argument("db_pass", help="MariaDB password")
+    parser.add_argument("db_host", help="MariaDB hostname")
+    parser.add_argument("db_port", type=int, help="MariaDB port")
+    parser.add_argument("db_name", help="MariaDB database")
+    parser.add_argument("report_file", help="Report file(.map.all)")
+    args = parser.parse_args()
+    return args
+
+
+def mariadbConnect(args):
+    try:
+        conn = mariadb.connect(
+            user=args.db_user,
+            password=args.db_pass,
+            host=args.db_host,
+            port=args.db_port,
+            database=args.db_name,
+        )
+    except mariadb.Error as e:
+        print(f"Error connecting to MariaDB: {e}")
+        sys.exit(1)
+    return conn
+
+
+def parseEnv():
+    outArr = []
+    outArr.append(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
+    outArr.append(os.getenv("COMMIT_HASH", default=None))
+    outArr.append(os.getenv("COMMIT_MSG", default=None))
+    outArr.append(os.getenv("BRANCH_NAME", default=None))
+    outArr.append(os.getenv("BSS_SIZE", default=None))
+    outArr.append(os.getenv("TEXT_SIZE", default=None))
+    outArr.append(os.getenv("RODATA_SIZE", default=None))
+    outArr.append(os.getenv("DATA_SIZE", default=None))
+    outArr.append(os.getenv("FREE_FLASH_SIZE", default=None))
+    outArr.append(os.getenv("PULL_ID", default=None))
+    outArr.append(os.getenv("PULL_NAME", default=None))
+    return outArr
+
+
+def createTables(cur, conn):
+    headerTable = "CREATE TABLE IF NOT EXISTS `header` ( \
+            `id` int(10) unsigned NOT NULL AUTO_INCREMENT, \
+            `datetime` datetime NOT NULL, \
+            `commit` varchar(40) NOT NULL, \
+            `commit_msg` text NOT NULL, \
+            `branch_name` text NOT NULL, \
+            `bss_size` int(10) unsigned NOT NULL, \
+            `text_size` int(10) unsigned NOT NULL, \
+            `rodata_size` int(10) unsigned NOT NULL, \
+            `data_size` int(10) unsigned NOT NULL, \
+            `free_flash_size` int(10) unsigned NOT NULL, \
+            `pullrequest_id` int(10) unsigned DEFAULT NULL, \
+            `pullrequest_name` text DEFAULT NULL, \
+            PRIMARY KEY (`id`), \
+            KEY `header_id_index` (`id`) )"
+    dataTable = "CREATE TABLE IF NOT EXISTS `data` ( \
+            `header_id` int(10) unsigned NOT NULL, \
+            `id` int(10) unsigned NOT NULL AUTO_INCREMENT, \
+            `section` text NOT NULL, \
+            `address` text NOT NULL, \
+            `size` int(10) unsigned NOT NULL, \
+            `name` text NOT NULL, \
+            `lib` text NOT NULL, \
+            `obj_name` text NOT NULL, \
+            PRIMARY KEY (`id`), \
+            KEY `data_id_index` (`id`), \
+            KEY `data_header_id_index` (`header_id`), \
+            CONSTRAINT `data_header_id_foreign` FOREIGN KEY (`header_id`) REFERENCES `header` (`id`) )"
+    cur.execute(headerTable)
+    cur.execute(dataTable)
+    conn.commit()
+
+
+def insertHeader(data, cur, conn):
+    query = "INSERT INTO `header` ( \
+            datetime, commit, commit_msg, branch_name, bss_size, text_size, \
+            rodata_size, data_size, free_flash_size, pullrequest_id, pullrequest_name) \
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+    cur.execute(query, data)
+    conn.commit()
+    return cur.lastrowid
+
+
+def parseFile(fileObj, headerID):
+    arr = []
+    fileLines = fileObj.readlines()
+    for line in fileLines:
+        lineArr = []
+        tempLineArr = line.split("\t")
+        lineArr.append(headerID)
+        lineArr.append(tempLineArr[0])  # section
+        lineArr.append(int(tempLineArr[2], 16))  # address hex
+        lineArr.append(int(tempLineArr[3]))  # size
+        lineArr.append(tempLineArr[4])  # name
+        lineArr.append(tempLineArr[5])  # lib
+        lineArr.append(tempLineArr[6])  # obj_name
+        arr.append(tuple(lineArr))
+    return arr
+
+
+def insertData(data, cur, conn):
+    query = "INSERT INTO `data` ( \
+            header_id, section, address, size, \
+            name, lib, obj_name) \
+            VALUES (?, ?, ?, ?, ? ,?, ?)"
+    cur.executemany(query, data)
+    conn.commit()
+
+
+def main():
+    args = parseArgs()
+    dbConn = mariadbConnect(args)
+    reportFile = open(args.report_file)
+    dbCurs = dbConn.cursor()
+    createTables(dbCurs, dbConn)
+    headerID = insertHeader(parseEnv(), dbCurs, dbConn)
+    insertData(parseFile(reportFile, headerID), dbCurs, dbConn)
+    reportFile.close()
+    dbCurs.close()
+
+
+if __name__ == "__main__":
+    main()