Move updater and unit tests to dockerized runner (#4028)

* extended unit_tests and changed to dockerized runner
* added branch to run units
* fixing unit-tests-output
* online output
* command not found fix
* added stm logging
* cleaned output
* Updated updater test to work on dockerized runner
* Test run for changed actions
* small refactor of run_unit_tests
* Final test of jobs
* Checked
* On-failure actions runs only on fail
* Set action trigger to pull request
* Bumped timeout a little
* Removed extra steps
* Removed stm monitor, as it's now part of docker-runner
* fix: testops without stm_monitoring
* fix: timeout extended


Co-authored-by: あく <alleteam@gmail.com>

.github/workflows/unit_tests.yml

@@ -5,54 +5,39 @@ on:
 env:
   TARGETS: f7
   DEFAULT_TARGET: f7
-  FBT_TOOLCHAIN_PATH: /opt
+  FBT_TOOLCHAIN_PATH: /opt/
   FBT_GIT_SUBMODULE_SHALLOW: 1
 
 jobs:
   run_units_on_bench:
-    runs-on: [self-hosted, FlipperZeroUnitTest]
+    runs-on: [ self-hosted, FlipperZeroTest ]
     steps:
-      - name: 'Wipe workspace'
-        run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
-
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           fetch-depth: 1
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: 'Get flipper from device manager (mock)'
-        id: device
-        run: |
-          echo "flipper=auto" >> $GITHUB_OUTPUT
-
       - name: 'Flash unit tests firmware'
         id: flashing
         if: success()
-        timeout-minutes: 10
-        run: |
-          ./fbt resources firmware_latest flash SWD_TRANSPORT_SERIAL=2A0906016415303030303032 LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1
-
-      - name: 'Wait for flipper and format ext'
-        id: format_ext
-        if: steps.flashing.outcome == 'success'
-        timeout-minutes: 5
+        timeout-minutes: 20
         run: |
           source scripts/toolchain/fbtenv.sh
-          python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=120 await_flipper 
-          python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} format_ext
+          ./fbt resources firmware_latest flash LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1
+
 
       - name: 'Copy assets and unit data, reboot and wait for flipper'
         id: copy
-        if: steps.format_ext.outcome == 'success'
+        if: steps.flashing.outcome == 'success'
         timeout-minutes: 7
         run: |
           source scripts/toolchain/fbtenv.sh
-          python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=15 await_flipper 
-          rm -rf build/latest/resources/dolphin
-          python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} -f send build/latest/resources /ext
-          python3 scripts/power.py -p ${{steps.device.outputs.flipper}} reboot
-          python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=15 await_flipper 
+          python3 scripts/testops.py -t=15 await_flipper
+          python3 scripts/storage.py -f send build/latest/resources /ext
+          python3 scripts/storage.py -f send /region_data /ext/.int/.region_data
+          python3 scripts/power.py reboot
+          python3 scripts/testops.py -t=30 await_flipper
 
       - name: 'Run units and validate results'
         id: run_units
@@ -60,9 +45,16 @@ jobs:
         timeout-minutes: 7
         run: |
           source scripts/toolchain/fbtenv.sh
-          python3 scripts/testops.py run_units -p ${{steps.device.outputs.flipper}}
+          python3 scripts/testops.py run_units
+
+      - name: 'Upload test results'
+        if: failure() && steps.flashing.outcome == 'success' && steps.run_units.outcome != 'skipped'
+        uses: actions/upload-artifact@v4
+        with:
+          name: unit-tests_output
+          path: unit_tests*.txt
 
       - name: 'Check GDB output'
         if: failure() && steps.flashing.outcome == 'success'
         run: |
-          ./fbt gdb_trace_all SWD_TRANSPORT_SERIAL=2A0906016415303030303032 LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1
+          ./fbt gdb_trace_all LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1

.github/workflows/updater_test.yml

@@ -1,39 +1,31 @@
 name: 'Updater test'
 on:
   pull_request:
+
 env:
   TARGETS: f7
   DEFAULT_TARGET: f7
-  FBT_TOOLCHAIN_PATH: /opt
+  FBT_TOOLCHAIN_PATH: /opt/
   FBT_GIT_SUBMODULE_SHALLOW: 1
 
 jobs:
   test_updater_on_bench:
-    runs-on: [self-hosted, FlipperZeroUpdaterTest]
+    runs-on: [self-hosted, FlipperZeroTest ]
     steps:
-      - name: 'Wipe workspace'
-        run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
-
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           fetch-depth: 1
-          submodules: false
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: 'Get flipper from device manager (mock)'
-        id: device
-        run: |
-          echo "flipper=auto" >> $GITHUB_OUTPUT
-          echo "stlink=0F020D026415303030303032" >> $GITHUB_OUTPUT
-
       - name: 'Flashing target firmware'
         id: first_full_flash
-        timeout-minutes: 10
+        timeout-minutes: 20
         run: |
           source scripts/toolchain/fbtenv.sh
-          ./fbt flash_usb_full PORT=${{steps.device.outputs.flipper}} FORCE=1
-          python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=180 await_flipper 
+          python3 scripts/testops.py -t=180 await_flipper
+          ./fbt flash_usb_full FORCE=1
+          
 
       - name: 'Validating updater'
         id: second_full_flash
@@ -41,33 +33,7 @@ jobs:
         if: success()
         run: |
           source scripts/toolchain/fbtenv.sh
-          ./fbt flash_usb PORT=${{steps.device.outputs.flipper}} FORCE=1
-          python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=180 await_flipper 
+          python3 scripts/testops.py -t=180 await_flipper
+          ./fbt flash_usb FORCE=1
+          python3 scripts/testops.py -t=180 await_flipper
 
-      - name: 'Get last release tag'
-        id: release_tag
-        if: failure()
-        run: |
-          echo "tag=$(git tag -l --sort=-version:refname | grep -v "rc\|RC" | head -1)" >> $GITHUB_OUTPUT
-
-      - name: 'Wipe workspace'
-        run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
-
-      - name: 'Checkout latest release'
-        uses: actions/checkout@v4
-        if: failure()
-        with:
-          fetch-depth: 1
-          ref: ${{ steps.release_tag.outputs.tag }}
-
-      - name: 'Flash last release'
-        if: failure()
-        run: |
-          ./fbt flash SWD_TRANSPORT_SERIAL=${{steps.device.outputs.stlink}} FORCE=1
-
-      - name: 'Wait for flipper and format ext'
-        if: failure()
-        run: |
-          source scripts/toolchain/fbtenv.sh
-          python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=180 await_flipper 
-          python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} format_ext

scripts/testops.py

@@ -1,8 +1,7 @@
 #!/usr/bin/env python3
-
 import re
-import sys
 import time
+from datetime import datetime
 from typing import Optional
 
 from flipper.app import App
@@ -11,7 +10,10 @@ from flipper.utils.cdc import resolve_port
 
 
 class Main(App):
-    # this is basic use without sub-commands, simply to reboot flipper / power it off, not meant as a full CLI wrapper
+    def __init__(self, no_exit=False):
+        super().__init__(no_exit)
+        self.test_results = None
+
     def init(self):
         self.parser.add_argument("-p", "--port", help="CDC Port", default="auto")
         self.parser.add_argument(
@@ -67,65 +69,109 @@ class Main(App):
         self.logger.info("Running unit tests")
         flipper.send("unit_tests" + "\r")
         self.logger.info("Waiting for unit tests to complete")
-        data = flipper.read.until(">: ")
-        self.logger.info("Parsing result")
-
-        lines = data.decode().split("\r\n")
-
-        tests_re = r"Failed tests: \d{0,}"
-        time_re = r"Consumed: \d{0,}"
-        leak_re = r"Leaked: \d{0,}"
-        status_re = r"Status: \w{3,}"
-
-        tests_pattern = re.compile(tests_re)
-        time_pattern = re.compile(time_re)
-        leak_pattern = re.compile(leak_re)
-        status_pattern = re.compile(status_re)
 
         tests, elapsed_time, leak, status = None, None, None, None
         total = 0
+        all_required_found = False
 
-        for line in lines:
+        full_output = []
+
+        tests_pattern = re.compile(r"Failed tests: \d{0,}")
+        time_pattern = re.compile(r"Consumed: \d{0,}")
+        leak_pattern = re.compile(r"Leaked: \d{0,}")
+        status_pattern = re.compile(r"Status: \w{3,}")
+
+        try:
+            while not all_required_found:
+                try:
+                    line = flipper.read.until("\r\n", cut_eol=True).decode()
                     self.logger.info(line)
+                    if "command not found," in line:
+                        self.logger.error(f"Command not found: {line}")
+                        return 1
+
                     if "()" in line:
                         total += 1
+                        self.logger.debug(f"Test completed: {line}")
 
                     if not tests:
-                tests = re.match(tests_pattern, line)
+                        tests = tests_pattern.match(line)
                     if not elapsed_time:
-                elapsed_time = re.match(time_pattern, line)
+                        elapsed_time = time_pattern.match(line)
                     if not leak:
-                leak = re.match(leak_pattern, line)
+                        leak = leak_pattern.match(line)
                     if not status:
-                status = re.match(status_pattern, line)
+                        status = status_pattern.match(line)
+
+                    pattern = re.compile(
+                        r"(\[-]|\[\\]|\[\|]|\[/-]|\[[^\]]*\]|\x1b\[\d+D)"
+                    )
+                    line_to_append = pattern.sub("", line)
+                    pattern = re.compile(r"\[3D[^\]]*")
+                    line_to_append = pattern.sub("", line_to_append)
+                    line_to_append = f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S,%f')} {line_to_append}"
+
+                    full_output.append(line_to_append)
+
+                    if tests and elapsed_time and leak and status:
+                        all_required_found = True
+                        try:
+                            remaining = flipper.read.until(">: ", cut_eol=True).decode()
+                            if remaining.strip():
+                                full_output.append(remaining)
+                        except:
+                            pass
+                        break
+
+                except Exception as e:
+                    self.logger.error(f"Error reading output: {e}")
+                    raise
 
             if None in (tests, elapsed_time, leak, status):
-            self.logger.error(
+                raise RuntimeError(
                     f"Failed to parse output: {tests} {elapsed_time} {leak} {status}"
                 )
-            sys.exit(1)
 
             leak = int(re.findall(r"[- ]\d+", leak.group(0))[0])
             status = re.findall(r"\w+", status.group(0))[1]
             tests = int(re.findall(r"\d+", tests.group(0))[0])
             elapsed_time = int(re.findall(r"\d+", elapsed_time.group(0))[0])
 
+            test_results = {
+                "full_output": "\n".join(full_output),
+                "total_tests": total,
+                "failed_tests": tests,
+                "elapsed_time_ms": elapsed_time,
+                "memory_leak_bytes": leak,
+                "status": status,
+            }
+
+            self.test_results = test_results
+
+            output_file = "unit_tests_output.txt"
+            with open(output_file, "w") as f:
+                f.write(test_results["full_output"])
+
+            print(
+                f"::notice:: Total tests: {total} Failed tests: {tests} Status: {status} Elapsed time: {elapsed_time / 1000} s Memory leak: {leak} bytes"
+            )
+
             if tests > 0 or status != "PASSED":
                 self.logger.error(f"Got {tests} failed tests.")
                 self.logger.error(f"Leaked (not failing on this stat): {leak}")
                 self.logger.error(f"Status: {status}")
                 self.logger.error(f"Time: {elapsed_time / 1000} seconds")
-            flipper.stop()
                 return 1
 
             self.logger.info(f"Leaked (not failing on this stat): {leak}")
             self.logger.info(
                 f"Tests ran successfully! Time elapsed {elapsed_time / 1000} seconds. Passed {total} tests."
             )
-
-        flipper.stop()
             return 0
 
+        finally:
+            flipper.stop()
+
 
 if __name__ == "__main__":
     Main()()