github: workflow improvements (#3032)

* github: compact build: status reporting step * github: build: matrix strategy * debugging * github: added version_token to /uploadfiles request * github: reworked main build flow * github: suppressed non-zero cp status * github: build: fixed comment lookup; experimental changes to apps build order * github: removed summary step for compact builds; united map analyzer steps * fbt: added get_apiversion target; moved ext apps processing logic to AppBuildset * ufbt: added missing global * fbt: Moved incompatible app list to firmware config output * fbt: cleaner extapps processing * github: build: added automation for SDK publishing
2023-09-05 14:49:39 +03:00 · 2023-09-05 14:49:39 +03:00 · 452e27b05e
commit 452e27b05e
parent 0b806c2360
8 changed files with 167 additions and 88 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -9,13 +9,15 @@ on:
  pull_request:

 env:
-  TARGETS: f7 f18
  DEFAULT_TARGET: f7
  FBT_TOOLCHAIN_PATH: /runner/_work

 jobs:
  main:
    runs-on: [self-hosted, FlipperZeroShell]
+    strategy:
+      matrix:
+        target: [f7, f18]    
    steps:
      - name: 'Wipe workspace'
        run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
@ -33,68 +35,69 @@ jobs:
            TYPE="pull"
          elif [[ "${{ github.ref }}" == "refs/tags/"* ]]; then
            TYPE="tag"
+            echo 'FBT_BUILD_TYPE="DEBUG=0 COMPACT=1"' >> $GITHUB_ENV
          else
            TYPE="other"
          fi
          python3 scripts/get_env.py "--event_file=${{ github.event_path }}" "--type=$TYPE" || cat "${{ github.event_path }}"
          echo "event_type=$TYPE" >> $GITHUB_OUTPUT
+          echo "TARGET=${{ matrix.target }}" >> $GITHUB_ENV
+          echo "TARGET_HW=$(echo "${{ matrix.target }}" | sed 's/f//')" >> $GITHUB_ENV

-      - name: 'Check API versions'
+      - name: 'Check API versions for consistency between targets'
        run: |
          set -e
          N_API_HEADER_SIGNATURES=`ls -1 firmware/targets/f*/api_symbols.csv | xargs -I {} sh -c "head -n2 {} | md5sum" | sort -u | wc -l`
          if [ $N_API_HEADER_SIGNATURES != 1 ] ; then
            echo API versions aren\'t matching for available targets. Please update!
+            echo API versions are:
            head -n2 firmware/targets/f*/api_symbols.csv
            exit 1
          fi

-      - name: 'Make artifacts directory'
+      - name: 'Build the firmware and apps'
        run: |
-          rm -rf artifacts map_analyser_files
-          mkdir artifacts map_analyser_files
+          ./fbt TARGET_HW=$TARGET_HW $FBT_BUILD_TYPE copro_dist updater_package fap_dist

-      - name: 'Bundle scripts'
-        if: ${{ !github.event.pull_request.head.repo.fork }}
-        run: |
-          tar czpf "artifacts/flipper-z-any-scripts-${SUFFIX}.tgz" scripts
-
-      - name: 'Build the firmware'
-        run: |
-          set -e
-          for TARGET in ${TARGETS}; do
-            TARGET_HW="$(echo "${TARGET}" | sed 's/f//')"; \
-            ./fbt TARGET_HW=$TARGET_HW copro_dist updater_package \
-              ${{ startsWith(github.ref, 'refs/tags') && 'DEBUG=0 COMPACT=1' || '' }}
-            mv dist/${TARGET}-*/* artifacts/
-            tar czpf "artifacts/flipper-z-${TARGET}-resources-${SUFFIX}.tgz" \
-              -C assets resources
-            ./fbt TARGET_HW=$TARGET_HW fap_dist
-            tar czpf "artifacts/flipper-z-${TARGET}-debugapps-${SUFFIX}.tgz" \
-              -C dist/${TARGET}-*/apps/Debug .
-            tar czpf "artifacts/flipper-z-${TARGET}-appsymbols-${SUFFIX}.tgz" \
-              -C dist/${TARGET}-*/debug_elf .
-          done
-
-      - name: "Check for uncommitted changes"
+      - name: 'Check for uncommitted changes'
        run: |
          git diff --exit-code

-      - name: 'Bundle core2 firmware'
-        if: ${{ !github.event.pull_request.head.repo.fork }}
+      - name: 'Copy build output'
        run: |
+          set -e
+          rm -rf artifacts map_analyser_files || true
+          mkdir artifacts map_analyser_files
+          cp dist/${TARGET}-*/* artifacts/ || true
+          tar czpf "artifacts/flipper-z-${TARGET}-resources-${SUFFIX}.tgz" \
+            -C assets resources
+          tar czpf "artifacts/flipper-z-${TARGET}-debugapps-${SUFFIX}.tgz" \
+            -C dist/${TARGET}-*/apps/Debug .
+          tar czpf "artifacts/flipper-z-${TARGET}-appsymbols-${SUFFIX}.tgz" \
+            -C dist/${TARGET}-*/debug_elf .
+
+      - name: 'Copy universal artifacts'
+        if: ${{ !github.event.pull_request.head.repo.fork && matrix.target == env.DEFAULT_TARGET }}
+        run: |
+          tar czpf "artifacts/flipper-z-any-scripts-${SUFFIX}.tgz" scripts
          cp build/core2_firmware.tgz "artifacts/flipper-z-any-core2_firmware-${SUFFIX}.tgz"

-      - name: 'Copy map analyser files'
+      - name: 'Upload artifacts to update server'
        if: ${{ !github.event.pull_request.head.repo.fork }}
+        run: |
+          FILES=$(for ARTIFACT in $(find artifacts -maxdepth 1 -not -type d); do echo "-F files=@${ARTIFACT}"; done)
+          curl --fail -L -H "Token: ${{ secrets.INDEXER_TOKEN }}" \
+              -F "branch=${BRANCH_NAME}" \
+              -F "version_token=${COMMIT_SHA}" \
+              ${FILES[@]} \
+              "${{ secrets.INDEXER_URL }}"/firmware/uploadfiles
+
+      - name: 'Copy & analyse map analyser files'
+        if: ${{ !github.event.pull_request.head.repo.fork && matrix.target == env.DEFAULT_TARGET }}
        run: |
          cp build/${DEFAULT_TARGET}-firmware-*/firmware.elf.map map_analyser_files/firmware.elf.map
          cp build/${DEFAULT_TARGET}-firmware-*/firmware.elf map_analyser_files/firmware.elf
          cp ${{ github.event_path }} map_analyser_files/event.json
-
-      - name: 'Analyse map file'
-        if: ${{ !github.event.pull_request.head.repo.fork }}
-        run: |
          source scripts/toolchain/fbtenv.sh
          get_size()
          {
@ -118,33 +121,56 @@ jobs:
              ${{ secrets.AMAP_MARIADB_DATABASE }} \
              map_analyser_files/firmware.elf.map.all

-      - name: 'Upload artifacts to update server'
-        if: ${{ !github.event.pull_request.head.repo.fork }}
-        run: |
-          FILES=$(for CUR in $(ls artifacts/); do echo "-F files=@artifacts/$CUR"; done)
-          curl --fail -L -H "Token: ${{ secrets.INDEXER_TOKEN }}" \
-              -F "branch=${BRANCH_NAME}" \
-              ${FILES[@]} \
-              "${{ secrets.INDEXER_URL }}"/firmware/uploadfiles
-
-      - name: 'Find Previous Comment'
-        if: ${{ !github.event.pull_request.head.repo.fork && github.event.pull_request }}
+      - name: 'Find previous comment'
+        if: ${{ !github.event.pull_request.head.repo.fork && matrix.target == env.DEFAULT_TARGET && github.event.pull_request }}
        uses: peter-evans/find-comment@v2
-        id: fc
+        id: find-comment
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: 'github-actions[bot]'
-          body-includes: 'Compiled firmware for commit'
+          body-includes: 'Compiled ${{ matrix.target }} firmware for commit'

      - name: 'Create or update comment'
-        if: ${{ !github.event.pull_request.head.repo.fork && github.event.pull_request}}
+        if: ${{ !github.event.pull_request.head.repo.fork && matrix.target == env.DEFAULT_TARGET && github.event.pull_request }}
        uses: peter-evans/create-or-update-comment@v3
        with:
-          comment-id: ${{ steps.fc.outputs.comment-id }}
+          comment-id: ${{ steps.find-comment.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body: |
-            **Compiled firmware for commit `${{steps.names.outputs.commit_sha}}`:**
+            **Compiled ${{ matrix.target }} firmware for commit `${{steps.names.outputs.commit_sha}}`:**
            - [📦 Update package](https://update.flipperzero.one/builds/firmware/${{steps.names.outputs.branch_name}}/flipper-z-${{steps.names.outputs.default_target}}-update-${{steps.names.outputs.suffix}}.tgz)
            - [📥 DFU file](https://update.flipperzero.one/builds/firmware/${{steps.names.outputs.branch_name}}/flipper-z-${{steps.names.outputs.default_target}}-full-${{steps.names.outputs.suffix}}.dfu)
            - [☁️ Web/App updater](https://lab.flipper.net/?url=https://update.flipperzero.one/builds/firmware/${{steps.names.outputs.branch_name}}/flipper-z-${{steps.names.outputs.default_target}}-update-${{steps.names.outputs.suffix}}.tgz&channel=${{steps.names.outputs.branch_name}}&version=${{steps.names.outputs.commit_sha}})
          edit-mode: replace
+
+      - name: 'Check if API version exists'
+        if: ${{ steps.names.outputs.event_type == 'tag' && matrix.target == env.DEFAULT_TARGET }}
+        run: |
+          FIRMWARE_API=$(./fbt TARGET_HW=$TARGET_HW get_apiversion)
+          curl -sX 'GET' \
+            '${{ secrets.CATALOG_URL }}/api/v0/0/sdk?length=200' \
+            -H 'Accept: application/json' > sdk_versions.json
+          if jq -r -e ".[] | select((.api == \"${FIRMWARE_API}\") and .target == \"f${TARGET_HW}\")" sdk_versions.json > found_sdk.json ; then
+            echo "API version $FIRMWARE_API already exists in catalog"
+            if [ $(jq -r -e ".released_at" found_sdk.json) != "null" ] ; then
+              echo "API version is already released"
+              exit 0
+            fi
+            if ! echo "$SUFFIX" | grep -q "-rc" ; then
+              SDK_ID=$(jq -r ._id found_sdk.json)
+              echo "Marking SDK $SDK_ID as released"
+              curl -X 'POST' \
+                "${{ secrets.CATALOG_URL }}/api/v0/0/sdk/${SDK_ID}/release" \
+                -H 'Accept: application/json' \
+                -H 'Authorization: Bearer ${{ secrets.CATALOG_API_TOKEN }}' \
+                -d ''
+            fi
+          else
+            echo "API version $FIRMWARE_API doesn't exist in catalog, adding"
+            curl -X 'POST' \
+              '${{ secrets.CATALOG_URL }}/api/v0/0/sdk' \
+              -H 'Accept: application/json' \
+              -H 'Authorization: Bearer ${{ secrets.CATALOG_API_TOKEN }}' \
+              -H 'Content-Type: application/json' \
+              -d "{\"name\": \"${SUFFIX}\", \"target\": \"f${TARGET_HW}\", \"api\": \"${FIRMWARE_API}\"}\"
+          fi
--- a/.github/workflows/build_compact.yml
+++ b/.github/workflows/build_compact.yml
@ -73,3 +73,12 @@ jobs:
              popd
            done
          done
+
+## Uncomment this for a single job that will run only if all targets are built successfully          
+#  report-status:
+#    name: Report status
+#    needs: [compact]
+#    if: always() && !contains(needs.*.result, 'failure')
+#    runs-on: [self-hosted, FlipperZeroShell]
+#    steps:
+#      - run: echo "All good ✨" ;
--- a/firmware.scons
+++ b/firmware.scons
@ -8,6 +8,9 @@ from fbt_extra.util import (
    link_elf_dir_as_latest,
 )

+from fbt.sdk.cache import LazySdkVersionLoader
+
+
 Import("ENV", "fw_build_meta")

 # Building initial C environment for libs
@ -71,6 +74,8 @@ env = ENV.Clone(
    },
    FW_API_TABLE=None,
    _APP_ICONS=None,
+    APPS=_.split(",") if (_ := GetOption("extra_int_apps")) else [],
+    EXTRA_EXT_APPS=_.split(",") if (_ := GetOption("extra_ext_apps")) else [],
 )

 env.PreConfigureFwEnvionment()
@ -125,9 +130,6 @@ if env["IS_BASE_FIRMWARE"]:
 else:
    fwenv.Append(APPS=["updater"])

-if extra_int_apps := GetOption("extra_int_apps"):
-    fwenv.Append(APPS=extra_int_apps.split(","))
-

 for app_dir, _ in fwenv["APPDIRS"]:
    app_dir_node = env.Dir("#").Dir(app_dir)
@ -136,7 +138,6 @@ for app_dir, _ in fwenv["APPDIRS"]:
        if isinstance(entry, FS.Dir) and not str(entry).startswith("."):
            fwenv.LoadAppManifest(entry)

-
 fwenv.PrepareApplicationsBuild()

 # Build external apps + configure SDK
@ -148,6 +149,11 @@ if env["IS_BASE_FIRMWARE"]:
    )
    fw_artifacts.append(fwenv["FW_EXTAPPS"].sdk_tree)

+    fwenv.Append(FBT_API_VERSION=LazySdkVersionLoader(fwenv.subst("$SDK_DEFINITION")))
+    fwenv.PhonyTarget(
+        "get_apiversion",
+        "@echo $( ${FBT_API_VERSION} $)",
+    )

 # Add preprocessor definitions for current set of apps
 fwenv.Append(
--- a/scripts/fbt/appmanifest.py
+++ b/scripts/fbt/appmanifest.py
@ -193,8 +193,19 @@ class AppManager:
            raise FlipperManifestException(f"Duplicate app declaration: {app.appid}")
        self.known_apps[app.appid] = app

-    def filter_apps(self, applist: List[str], hw_target: str):
-        return AppBuildset(self, applist, hw_target)
+    def filter_apps(
+        self,
+        *,
+        applist: List[str],
+        ext_applist: List[str],
+        hw_target: str,
+    ):
+        return AppBuildset(
+            self,
+            hw_target=hw_target,
+            appnames=applist,
+            extra_ext_appnames=ext_applist,
+        )


 class AppBuilderException(Exception):
@ -211,6 +222,12 @@ class AppBuildset:
        FlipperAppType.SETTINGS,
        FlipperAppType.STARTUP,
    )
+    EXTERNAL_APP_TYPES = (
+        FlipperAppType.EXTERNAL,
+        FlipperAppType.MENUEXTERNAL,
+        FlipperAppType.PLUGIN,
+        FlipperAppType.DEBUG,
+    )

    @staticmethod
    def print_writer(message):
@ -219,16 +236,21 @@ class AppBuildset:
    def __init__(
        self,
        appmgr: AppManager,
-        appnames: List[str],
        hw_target: str,
+        appnames: List[str],
+        *,
+        extra_ext_appnames: List[str],
        message_writer: Callable | None = None,
    ):
        self.appmgr = appmgr
        self.appnames = set(appnames)
+        self.incompatible_extapps, self.extapps = [], []
+        self._extra_ext_appnames = extra_ext_appnames
        self._orig_appnames = appnames
        self.hw_target = hw_target
        self._writer = message_writer if message_writer else self.print_writer
        self._process_deps()
+        self._process_ext_apps()
        self._check_conflicts()
        self._check_unsatisfied()  # unneeded?
        self._check_target_match()
@ -271,6 +293,27 @@ class AppBuildset:
                break
            self.appnames.update(provided)

+    def _process_ext_apps(self):
+        extapps = [
+            app
+            for apptype in self.EXTERNAL_APP_TYPES
+            for app in self.get_apps_of_type(apptype, True)
+        ]
+        extapps.extend(map(self.appmgr.get, self._extra_ext_appnames))
+
+        for app in extapps:
+            (
+                self.extapps
+                if app.supports_hardware_target(self.hw_target)
+                else self.incompatible_extapps
+            ).append(app)
+
+    def get_ext_apps(self):
+        return self.extapps
+
+    def get_incompatible_ext_apps(self):
+        return self.incompatible_extapps
+
    def _check_conflicts(self):
        conflicts = []
        for app in self.appnames:
--- a/scripts/fbt/sdk/cache.py
+++ b/scripts/fbt/sdk/cache.py
@ -255,3 +255,18 @@ class SdkCache:
        self.sync_sets(self.sdk.headers, api.headers, False)
        self.sync_sets(self.sdk.functions, api.functions)
        self.sync_sets(self.sdk.variables, api.variables)
+
+
+class LazySdkVersionLoader:
+    def __init__(self, sdk_path: str):
+        self.sdk_path = sdk_path
+        self._version = None
+
+    @property
+    def version(self) -> SdkVersion:
+        if self._version is None:
+            self._version = SdkCache(self.sdk_path, load_version_only=True).version
+        return self._version
+
+    def __str__(self) -> str:
+        return str(self.version)
--- a/scripts/fbt_tools/fbt_apps.py
+++ b/scripts/fbt_tools/fbt_apps.py
@ -36,7 +36,9 @@ def LoadAppManifest(env, entry):
 def PrepareApplicationsBuild(env):
    try:
        appbuild = env["APPBUILD"] = env["APPMGR"].filter_apps(
-            env["APPS"], env.subst("f${TARGET_HW}")
+            applist=env["APPS"],
+            ext_applist=env["EXTRA_EXT_APPS"],
+            hw_target=env.subst("f${TARGET_HW}"),
        )
    except Exception as e:
        raise StopError(e)
@ -56,6 +58,11 @@ def DumpApplicationConfig(target, source, env):
                fg.green(f"{apptype.value}:\n\t"),
                ", ".join(app.appid for app in app_sublist),
            )
+    if incompatible_ext_apps := env["APPBUILD"].get_incompatible_ext_apps():
+        print(
+            fg.blue("Incompatible apps (skipped):\n\t"),
+            ", ".join(app.appid for app in incompatible_ext_apps),
+        )


 def build_apps_c(target, source, env):
--- a/scripts/ufbt/SConstruct
+++ b/scripts/ufbt/SConstruct
@ -107,6 +107,7 @@ env = core_env.Clone(
    SINGLEQUOTEFUNC=single_quote,
    ABSPATHGETTERFUNC=resolve_real_dir_node,
    APPS=[],
+    EXTRA_EXT_APPS=[],
    UFBT_API_VERSION=SdkCache(
        core_env.subst("$SDK_DEFINITION"), load_version_only=True
    ).version,
--- a/site_scons/extapps.scons
+++ b/site_scons/extapps.scons
@ -60,40 +60,12 @@ class FlipperExtAppBuildArtifacts:
    sdk_tree: NodeList = field(default_factory=NodeList)


-apps_to_build_as_faps = [
-    FlipperAppType.PLUGIN,
-    FlipperAppType.EXTERNAL,
-    FlipperAppType.MENUEXTERNAL,
-    FlipperAppType.DEBUG,
-]
-
-known_extapps = [
-    app
-    for apptype in apps_to_build_as_faps
-    for app in appenv["APPBUILD"].get_apps_of_type(apptype, True)
-]
-
-# Ugly access to global option
-if extra_app_list := GetOption("extra_ext_apps"):
-    known_extapps.extend(map(appenv["APPMGR"].get, extra_app_list.split(",")))
-
-incompatible_apps = []
-for app in known_extapps:
-    if not app.supports_hardware_target(appenv.subst("f${TARGET_HW}")):
-        incompatible_apps.append(app)
-        continue
-
+for app in appenv["APPBUILD"].get_ext_apps():
    appenv.BuildAppElf(app)

 extapps = FlipperExtAppBuildArtifacts()
 extapps.application_map = appenv["EXT_APPS"]

-if incompatible_apps:
-    warn(
-        WarningOnByDefault,
-        f"Skipping build of {len(incompatible_apps)} incompatible app(s): "
-        + ", ".join(f"'{app.name}' (id '{app.appid}')" for app in incompatible_apps),
-    )

 if appenv["FORCE"]:
    appenv.AlwaysBuild(