github: workflow improvements (#3032)

* github: compact build: status reporting step
* github: build: matrix strategy
* debugging
* github: added version_token to /uploadfiles request
* github: reworked main build flow
* github: suppressed non-zero cp status
* github: build: fixed comment lookup; experimental changes to apps build order
* github: removed summary step for compact builds; united map analyzer steps
* fbt: added get_apiversion target; moved ext apps processing logic to AppBuildset
* ufbt: added missing global
* fbt: Moved incompatible app list to firmware config output
* fbt: cleaner extapps processing
* github: build: added automation for SDK publishing

.github/workflows/build.yml

@@ -9,13 +9,15 @@ on:
   pull_request:
 
 env:
-  TARGETS: f7 f18
   DEFAULT_TARGET: f7
   FBT_TOOLCHAIN_PATH: /runner/_work
 
 jobs:
   main:
     runs-on: [self-hosted, FlipperZeroShell]
+    strategy:
+      matrix:
+        target: [f7, f18]    
     steps:
       - name: 'Wipe workspace'
         run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
@@ -33,68 +35,69 @@ jobs:
             TYPE="pull"
           elif [[ "${{ github.ref }}" == "refs/tags/"* ]]; then
             TYPE="tag"
+            echo 'FBT_BUILD_TYPE="DEBUG=0 COMPACT=1"' >> $GITHUB_ENV
           else
             TYPE="other"
           fi
           python3 scripts/get_env.py "--event_file=${{ github.event_path }}" "--type=$TYPE" || cat "${{ github.event_path }}"
           echo "event_type=$TYPE" >> $GITHUB_OUTPUT
+          echo "TARGET=${{ matrix.target }}" >> $GITHUB_ENV
+          echo "TARGET_HW=$(echo "${{ matrix.target }}" | sed 's/f//')" >> $GITHUB_ENV
 
-      - name: 'Check API versions'
+      - name: 'Check API versions for consistency between targets'
         run: |
           set -e
           N_API_HEADER_SIGNATURES=`ls -1 firmware/targets/f*/api_symbols.csv | xargs -I {} sh -c "head -n2 {} | md5sum" | sort -u | wc -l`
           if [ $N_API_HEADER_SIGNATURES != 1 ] ; then
             echo API versions aren\'t matching for available targets. Please update!
+            echo API versions are:
             head -n2 firmware/targets/f*/api_symbols.csv
             exit 1
           fi
 
-      - name: 'Make artifacts directory'
+      - name: 'Build the firmware and apps'
         run: |
-          rm -rf artifacts map_analyser_files
+          ./fbt TARGET_HW=$TARGET_HW $FBT_BUILD_TYPE copro_dist updater_package fap_dist
-          mkdir artifacts map_analyser_files
 
-      - name: 'Bundle scripts'
+      - name: 'Check for uncommitted changes'
-        if: ${{ !github.event.pull_request.head.repo.fork }}
         run: |
-          tar czpf "artifacts/flipper-z-any-scripts-${SUFFIX}.tgz" scripts
+          git diff --exit-code
 
-      - name: 'Build the firmware'
+      - name: 'Copy build output'
         run: |
           set -e
-          for TARGET in ${TARGETS}; do
+          rm -rf artifacts map_analyser_files || true
-            TARGET_HW="$(echo "${TARGET}" | sed 's/f//')"; \
+          mkdir artifacts map_analyser_files
-            ./fbt TARGET_HW=$TARGET_HW copro_dist updater_package \
+          cp dist/${TARGET}-*/* artifacts/ || true
-              ${{ startsWith(github.ref, 'refs/tags') && 'DEBUG=0 COMPACT=1' || '' }}
-            mv dist/${TARGET}-*/* artifacts/
           tar czpf "artifacts/flipper-z-${TARGET}-resources-${SUFFIX}.tgz" \
             -C assets resources
-            ./fbt TARGET_HW=$TARGET_HW fap_dist
           tar czpf "artifacts/flipper-z-${TARGET}-debugapps-${SUFFIX}.tgz" \
             -C dist/${TARGET}-*/apps/Debug .
           tar czpf "artifacts/flipper-z-${TARGET}-appsymbols-${SUFFIX}.tgz" \
             -C dist/${TARGET}-*/debug_elf .
-          done
 
-      - name: "Check for uncommitted changes"
+      - name: 'Copy universal artifacts'
-        run: |
+        if: ${{ !github.event.pull_request.head.repo.fork && matrix.target == env.DEFAULT_TARGET }}
-          git diff --exit-code
-
-      - name: 'Bundle core2 firmware'
-        if: ${{ !github.event.pull_request.head.repo.fork }}
         run: |
+          tar czpf "artifacts/flipper-z-any-scripts-${SUFFIX}.tgz" scripts
           cp build/core2_firmware.tgz "artifacts/flipper-z-any-core2_firmware-${SUFFIX}.tgz"
 
-      - name: 'Copy map analyser files'
+      - name: 'Upload artifacts to update server'
         if: ${{ !github.event.pull_request.head.repo.fork }}
+        run: |
+          FILES=$(for ARTIFACT in $(find artifacts -maxdepth 1 -not -type d); do echo "-F files=@${ARTIFACT}"; done)
+          curl --fail -L -H "Token: ${{ secrets.INDEXER_TOKEN }}" \
+              -F "branch=${BRANCH_NAME}" \
+              -F "version_token=${COMMIT_SHA}" \
+              ${FILES[@]} \
+              "${{ secrets.INDEXER_URL }}"/firmware/uploadfiles
+
+      - name: 'Copy & analyse map analyser files'
+        if: ${{ !github.event.pull_request.head.repo.fork && matrix.target == env.DEFAULT_TARGET }}
         run: |
           cp build/${DEFAULT_TARGET}-firmware-*/firmware.elf.map map_analyser_files/firmware.elf.map
           cp build/${DEFAULT_TARGET}-firmware-*/firmware.elf map_analyser_files/firmware.elf
           cp ${{ github.event_path }} map_analyser_files/event.json
-
-      - name: 'Analyse map file'
-        if: ${{ !github.event.pull_request.head.repo.fork }}
-        run: |
           source scripts/toolchain/fbtenv.sh
           get_size()
           {
@@ -118,33 +121,56 @@ jobs:
               ${{ secrets.AMAP_MARIADB_DATABASE }} \
               map_analyser_files/firmware.elf.map.all
 
-      - name: 'Upload artifacts to update server'
+      - name: 'Find previous comment'
-        if: ${{ !github.event.pull_request.head.repo.fork }}
+        if: ${{ !github.event.pull_request.head.repo.fork && matrix.target == env.DEFAULT_TARGET && github.event.pull_request }}
-        run: |
-          FILES=$(for CUR in $(ls artifacts/); do echo "-F files=@artifacts/$CUR"; done)
-          curl --fail -L -H "Token: ${{ secrets.INDEXER_TOKEN }}" \
-              -F "branch=${BRANCH_NAME}" \
-              ${FILES[@]} \
-              "${{ secrets.INDEXER_URL }}"/firmware/uploadfiles
-
-      - name: 'Find Previous Comment'
-        if: ${{ !github.event.pull_request.head.repo.fork && github.event.pull_request }}
         uses: peter-evans/find-comment@v2
-        id: fc
+        id: find-comment
         with:
           issue-number: ${{ github.event.pull_request.number }}
           comment-author: 'github-actions[bot]'
-          body-includes: 'Compiled firmware for commit'
+          body-includes: 'Compiled ${{ matrix.target }} firmware for commit'
 
       - name: 'Create or update comment'
-        if: ${{ !github.event.pull_request.head.repo.fork && github.event.pull_request}}
+        if: ${{ !github.event.pull_request.head.repo.fork && matrix.target == env.DEFAULT_TARGET && github.event.pull_request }}
         uses: peter-evans/create-or-update-comment@v3
         with:
-          comment-id: ${{ steps.fc.outputs.comment-id }}
+          comment-id: ${{ steps.find-comment.outputs.comment-id }}
           issue-number: ${{ github.event.pull_request.number }}
           body: |
-            **Compiled firmware for commit `${{steps.names.outputs.commit_sha}}`:**
+            **Compiled ${{ matrix.target }} firmware for commit `${{steps.names.outputs.commit_sha}}`:**
             - [📦 Update package](https://update.flipperzero.one/builds/firmware/${{steps.names.outputs.branch_name}}/flipper-z-${{steps.names.outputs.default_target}}-update-${{steps.names.outputs.suffix}}.tgz)
             - [📥 DFU file](https://update.flipperzero.one/builds/firmware/${{steps.names.outputs.branch_name}}/flipper-z-${{steps.names.outputs.default_target}}-full-${{steps.names.outputs.suffix}}.dfu)
             - [☁️ Web/App updater](https://lab.flipper.net/?url=https://update.flipperzero.one/builds/firmware/${{steps.names.outputs.branch_name}}/flipper-z-${{steps.names.outputs.default_target}}-update-${{steps.names.outputs.suffix}}.tgz&channel=${{steps.names.outputs.branch_name}}&version=${{steps.names.outputs.commit_sha}})
           edit-mode: replace
+
+      - name: 'Check if API version exists'
+        if: ${{ steps.names.outputs.event_type == 'tag' && matrix.target == env.DEFAULT_TARGET }}
+        run: |
+          FIRMWARE_API=$(./fbt TARGET_HW=$TARGET_HW get_apiversion)
+          curl -sX 'GET' \
+            '${{ secrets.CATALOG_URL }}/api/v0/0/sdk?length=200' \
+            -H 'Accept: application/json' > sdk_versions.json
+          if jq -r -e ".[] | select((.api == \"${FIRMWARE_API}\") and .target == \"f${TARGET_HW}\")" sdk_versions.json > found_sdk.json ; then
+            echo "API version $FIRMWARE_API already exists in catalog"
+            if [ $(jq -r -e ".released_at" found_sdk.json) != "null" ] ; then
+              echo "API version is already released"
+              exit 0
+            fi
+            if ! echo "$SUFFIX" | grep -q "-rc" ; then
+              SDK_ID=$(jq -r ._id found_sdk.json)
+              echo "Marking SDK $SDK_ID as released"
+              curl -X 'POST' \
+                "${{ secrets.CATALOG_URL }}/api/v0/0/sdk/${SDK_ID}/release" \
+                -H 'Accept: application/json' \
+                -H 'Authorization: Bearer ${{ secrets.CATALOG_API_TOKEN }}' \
+                -d ''
+            fi
+          else
+            echo "API version $FIRMWARE_API doesn't exist in catalog, adding"
+            curl -X 'POST' \
+              '${{ secrets.CATALOG_URL }}/api/v0/0/sdk' \
+              -H 'Accept: application/json' \
+              -H 'Authorization: Bearer ${{ secrets.CATALOG_API_TOKEN }}' \
+              -H 'Content-Type: application/json' \
+              -d "{\"name\": \"${SUFFIX}\", \"target\": \"f${TARGET_HW}\", \"api\": \"${FIRMWARE_API}\"}\"
+          fi

.github/workflows/build_compact.yml

@@ -73,3 +73,12 @@ jobs:
               popd
             done
           done
+
+## Uncomment this for a single job that will run only if all targets are built successfully          
+#  report-status:
+#    name: Report status
+#    needs: [compact]
+#    if: always() && !contains(needs.*.result, 'failure')
+#    runs-on: [self-hosted, FlipperZeroShell]
+#    steps:
+#      - run: echo "All good ✨" ;

firmware.scons

@@ -8,6 +8,9 @@ from fbt_extra.util import (
     link_elf_dir_as_latest,
 )
 
+from fbt.sdk.cache import LazySdkVersionLoader
+
+
 Import("ENV", "fw_build_meta")
 
 # Building initial C environment for libs
@@ -71,6 +74,8 @@ env = ENV.Clone(
     },
     FW_API_TABLE=None,
     _APP_ICONS=None,
+    APPS=_.split(",") if (_ := GetOption("extra_int_apps")) else [],
+    EXTRA_EXT_APPS=_.split(",") if (_ := GetOption("extra_ext_apps")) else [],
 )
 
 env.PreConfigureFwEnvionment()
@@ -125,9 +130,6 @@ if env["IS_BASE_FIRMWARE"]:
 else:
     fwenv.Append(APPS=["updater"])
 
-if extra_int_apps := GetOption("extra_int_apps"):
-    fwenv.Append(APPS=extra_int_apps.split(","))
-
 
 for app_dir, _ in fwenv["APPDIRS"]:
     app_dir_node = env.Dir("#").Dir(app_dir)
@@ -136,7 +138,6 @@ for app_dir, _ in fwenv["APPDIRS"]:
         if isinstance(entry, FS.Dir) and not str(entry).startswith("."):
             fwenv.LoadAppManifest(entry)
 
-
 fwenv.PrepareApplicationsBuild()
 
 # Build external apps + configure SDK
@@ -148,6 +149,11 @@ if env["IS_BASE_FIRMWARE"]:
     )
     fw_artifacts.append(fwenv["FW_EXTAPPS"].sdk_tree)
 
+    fwenv.Append(FBT_API_VERSION=LazySdkVersionLoader(fwenv.subst("$SDK_DEFINITION")))
+    fwenv.PhonyTarget(
+        "get_apiversion",
+        "@echo $( ${FBT_API_VERSION} $)",
+    )
 
 # Add preprocessor definitions for current set of apps
 fwenv.Append(

scripts/fbt/appmanifest.py

@@ -193,8 +193,19 @@ class AppManager:
             raise FlipperManifestException(f"Duplicate app declaration: {app.appid}")
         self.known_apps[app.appid] = app
 
-    def filter_apps(self, applist: List[str], hw_target: str):
+    def filter_apps(
-        return AppBuildset(self, applist, hw_target)
+        self,
+        *,
+        applist: List[str],
+        ext_applist: List[str],
+        hw_target: str,
+    ):
+        return AppBuildset(
+            self,
+            hw_target=hw_target,
+            appnames=applist,
+            extra_ext_appnames=ext_applist,
+        )
 
 
 class AppBuilderException(Exception):
@@ -211,6 +222,12 @@ class AppBuildset:
         FlipperAppType.SETTINGS,
         FlipperAppType.STARTUP,
     )
+    EXTERNAL_APP_TYPES = (
+        FlipperAppType.EXTERNAL,
+        FlipperAppType.MENUEXTERNAL,
+        FlipperAppType.PLUGIN,
+        FlipperAppType.DEBUG,
+    )
 
     @staticmethod
     def print_writer(message):
@@ -219,16 +236,21 @@ class AppBuildset:
     def __init__(
         self,
         appmgr: AppManager,
-        appnames: List[str],
         hw_target: str,
+        appnames: List[str],
+        *,
+        extra_ext_appnames: List[str],
         message_writer: Callable | None = None,
     ):
         self.appmgr = appmgr
         self.appnames = set(appnames)
+        self.incompatible_extapps, self.extapps = [], []
+        self._extra_ext_appnames = extra_ext_appnames
         self._orig_appnames = appnames
         self.hw_target = hw_target
         self._writer = message_writer if message_writer else self.print_writer
         self._process_deps()
+        self._process_ext_apps()
         self._check_conflicts()
         self._check_unsatisfied()  # unneeded?
         self._check_target_match()
@@ -271,6 +293,27 @@ class AppBuildset:
                 break
             self.appnames.update(provided)
 
+    def _process_ext_apps(self):
+        extapps = [
+            app
+            for apptype in self.EXTERNAL_APP_TYPES
+            for app in self.get_apps_of_type(apptype, True)
+        ]
+        extapps.extend(map(self.appmgr.get, self._extra_ext_appnames))
+
+        for app in extapps:
+            (
+                self.extapps
+                if app.supports_hardware_target(self.hw_target)
+                else self.incompatible_extapps
+            ).append(app)
+
+    def get_ext_apps(self):
+        return self.extapps
+
+    def get_incompatible_ext_apps(self):
+        return self.incompatible_extapps
+
     def _check_conflicts(self):
         conflicts = []
         for app in self.appnames:

scripts/fbt/sdk/cache.py

@@ -255,3 +255,18 @@ class SdkCache:
         self.sync_sets(self.sdk.headers, api.headers, False)
         self.sync_sets(self.sdk.functions, api.functions)
         self.sync_sets(self.sdk.variables, api.variables)
+
+
+class LazySdkVersionLoader:
+    def __init__(self, sdk_path: str):
+        self.sdk_path = sdk_path
+        self._version = None
+
+    @property
+    def version(self) -> SdkVersion:
+        if self._version is None:
+            self._version = SdkCache(self.sdk_path, load_version_only=True).version
+        return self._version
+
+    def __str__(self) -> str:
+        return str(self.version)

scripts/fbt_tools/fbt_apps.py

@@ -36,7 +36,9 @@ def LoadAppManifest(env, entry):
 def PrepareApplicationsBuild(env):
     try:
         appbuild = env["APPBUILD"] = env["APPMGR"].filter_apps(
-            env["APPS"], env.subst("f${TARGET_HW}")
+            applist=env["APPS"],
+            ext_applist=env["EXTRA_EXT_APPS"],
+            hw_target=env.subst("f${TARGET_HW}"),
         )
     except Exception as e:
         raise StopError(e)
@@ -56,6 +58,11 @@ def DumpApplicationConfig(target, source, env):
                 fg.green(f"{apptype.value}:\n\t"),
                 ", ".join(app.appid for app in app_sublist),
             )
+    if incompatible_ext_apps := env["APPBUILD"].get_incompatible_ext_apps():
+        print(
+            fg.blue("Incompatible apps (skipped):\n\t"),
+            ", ".join(app.appid for app in incompatible_ext_apps),
+        )
 
 
 def build_apps_c(target, source, env):

scripts/ufbt/SConstruct

@@ -107,6 +107,7 @@ env = core_env.Clone(
     SINGLEQUOTEFUNC=single_quote,
     ABSPATHGETTERFUNC=resolve_real_dir_node,
     APPS=[],
+    EXTRA_EXT_APPS=[],
     UFBT_API_VERSION=SdkCache(
         core_env.subst("$SDK_DEFINITION"), load_version_only=True
     ).version,

site_scons/extapps.scons

@@ -60,40 +60,12 @@ class FlipperExtAppBuildArtifacts:
     sdk_tree: NodeList = field(default_factory=NodeList)
 
 
-apps_to_build_as_faps = [
+for app in appenv["APPBUILD"].get_ext_apps():
-    FlipperAppType.PLUGIN,
-    FlipperAppType.EXTERNAL,
-    FlipperAppType.MENUEXTERNAL,
-    FlipperAppType.DEBUG,
-]
-
-known_extapps = [
-    app
-    for apptype in apps_to_build_as_faps
-    for app in appenv["APPBUILD"].get_apps_of_type(apptype, True)
-]
-
-# Ugly access to global option
-if extra_app_list := GetOption("extra_ext_apps"):
-    known_extapps.extend(map(appenv["APPMGR"].get, extra_app_list.split(",")))
-
-incompatible_apps = []
-for app in known_extapps:
-    if not app.supports_hardware_target(appenv.subst("f${TARGET_HW}")):
-        incompatible_apps.append(app)
-        continue
-
     appenv.BuildAppElf(app)
 
 extapps = FlipperExtAppBuildArtifacts()
 extapps.application_map = appenv["EXT_APPS"]
 
-if incompatible_apps:
-    warn(
-        WarningOnByDefault,
-        f"Skipping build of {len(incompatible_apps)} incompatible app(s): "
-        + ", ".join(f"'{app.name}' (id '{app.appid}')" for app in incompatible_apps),
-    )
 
 if appenv["FORCE"]:
     appenv.AlwaysBuild(