diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 07b21971..510542c3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,13 +9,15 @@ on: pull_request: env: - TARGETS: f7 f18 DEFAULT_TARGET: f7 FBT_TOOLCHAIN_PATH: /runner/_work jobs: main: runs-on: [self-hosted, FlipperZeroShell] + strategy: + matrix: + target: [f7, f18] steps: - name: 'Wipe workspace' run: find ./ -mount -maxdepth 1 -exec rm -rf {} \; @@ -33,68 +35,69 @@ jobs: TYPE="pull" elif [[ "${{ github.ref }}" == "refs/tags/"* ]]; then TYPE="tag" + echo 'FBT_BUILD_TYPE="DEBUG=0 COMPACT=1"' >> $GITHUB_ENV else TYPE="other" fi python3 scripts/get_env.py "--event_file=${{ github.event_path }}" "--type=$TYPE" || cat "${{ github.event_path }}" echo "event_type=$TYPE" >> $GITHUB_OUTPUT + echo "TARGET=${{ matrix.target }}" >> $GITHUB_ENV + echo "TARGET_HW=$(echo "${{ matrix.target }}" | sed 's/f//')" >> $GITHUB_ENV - - name: 'Check API versions' + - name: 'Check API versions for consistency between targets' run: | set -e N_API_HEADER_SIGNATURES=`ls -1 firmware/targets/f*/api_symbols.csv | xargs -I {} sh -c "head -n2 {} | md5sum" | sort -u | wc -l` if [ $N_API_HEADER_SIGNATURES != 1 ] ; then echo API versions aren\'t matching for available targets. Please update! + echo API versions are: head -n2 firmware/targets/f*/api_symbols.csv exit 1 fi - - name: 'Make artifacts directory' + - name: 'Build the firmware and apps' run: | - rm -rf artifacts map_analyser_files - mkdir artifacts map_analyser_files + ./fbt TARGET_HW=$TARGET_HW $FBT_BUILD_TYPE copro_dist updater_package fap_dist - - name: 'Bundle scripts' - if: ${{ !github.event.pull_request.head.repo.fork }} - run: | - tar czpf "artifacts/flipper-z-any-scripts-${SUFFIX}.tgz" scripts - - - name: 'Build the firmware' - run: | - set -e - for TARGET in ${TARGETS}; do - TARGET_HW="$(echo "${TARGET}" | sed 's/f//')"; \ - ./fbt TARGET_HW=$TARGET_HW copro_dist updater_package \ - ${{ startsWith(github.ref, 'refs/tags') && 'DEBUG=0 COMPACT=1' || '' }} - mv dist/${TARGET}-*/* artifacts/ - tar czpf "artifacts/flipper-z-${TARGET}-resources-${SUFFIX}.tgz" \ - -C assets resources - ./fbt TARGET_HW=$TARGET_HW fap_dist - tar czpf "artifacts/flipper-z-${TARGET}-debugapps-${SUFFIX}.tgz" \ - -C dist/${TARGET}-*/apps/Debug . - tar czpf "artifacts/flipper-z-${TARGET}-appsymbols-${SUFFIX}.tgz" \ - -C dist/${TARGET}-*/debug_elf . - done - - - name: "Check for uncommitted changes" + - name: 'Check for uncommitted changes' run: | git diff --exit-code - - name: 'Bundle core2 firmware' - if: ${{ !github.event.pull_request.head.repo.fork }} + - name: 'Copy build output' run: | + set -e + rm -rf artifacts map_analyser_files || true + mkdir artifacts map_analyser_files + cp dist/${TARGET}-*/* artifacts/ || true + tar czpf "artifacts/flipper-z-${TARGET}-resources-${SUFFIX}.tgz" \ + -C assets resources + tar czpf "artifacts/flipper-z-${TARGET}-debugapps-${SUFFIX}.tgz" \ + -C dist/${TARGET}-*/apps/Debug . + tar czpf "artifacts/flipper-z-${TARGET}-appsymbols-${SUFFIX}.tgz" \ + -C dist/${TARGET}-*/debug_elf . + + - name: 'Copy universal artifacts' + if: ${{ !github.event.pull_request.head.repo.fork && matrix.target == env.DEFAULT_TARGET }} + run: | + tar czpf "artifacts/flipper-z-any-scripts-${SUFFIX}.tgz" scripts cp build/core2_firmware.tgz "artifacts/flipper-z-any-core2_firmware-${SUFFIX}.tgz" - - name: 'Copy map analyser files' + - name: 'Upload artifacts to update server' if: ${{ !github.event.pull_request.head.repo.fork }} + run: | + FILES=$(for ARTIFACT in $(find artifacts -maxdepth 1 -not -type d); do echo "-F files=@${ARTIFACT}"; done) + curl --fail -L -H "Token: ${{ secrets.INDEXER_TOKEN }}" \ + -F "branch=${BRANCH_NAME}" \ + -F "version_token=${COMMIT_SHA}" \ + ${FILES[@]} \ + "${{ secrets.INDEXER_URL }}"/firmware/uploadfiles + + - name: 'Copy & analyse map analyser files' + if: ${{ !github.event.pull_request.head.repo.fork && matrix.target == env.DEFAULT_TARGET }} run: | cp build/${DEFAULT_TARGET}-firmware-*/firmware.elf.map map_analyser_files/firmware.elf.map cp build/${DEFAULT_TARGET}-firmware-*/firmware.elf map_analyser_files/firmware.elf cp ${{ github.event_path }} map_analyser_files/event.json - - - name: 'Analyse map file' - if: ${{ !github.event.pull_request.head.repo.fork }} - run: | source scripts/toolchain/fbtenv.sh get_size() { @@ -118,33 +121,56 @@ jobs: ${{ secrets.AMAP_MARIADB_DATABASE }} \ map_analyser_files/firmware.elf.map.all - - name: 'Upload artifacts to update server' - if: ${{ !github.event.pull_request.head.repo.fork }} - run: | - FILES=$(for CUR in $(ls artifacts/); do echo "-F files=@artifacts/$CUR"; done) - curl --fail -L -H "Token: ${{ secrets.INDEXER_TOKEN }}" \ - -F "branch=${BRANCH_NAME}" \ - ${FILES[@]} \ - "${{ secrets.INDEXER_URL }}"/firmware/uploadfiles - - - name: 'Find Previous Comment' - if: ${{ !github.event.pull_request.head.repo.fork && github.event.pull_request }} + - name: 'Find previous comment' + if: ${{ !github.event.pull_request.head.repo.fork && matrix.target == env.DEFAULT_TARGET && github.event.pull_request }} uses: peter-evans/find-comment@v2 - id: fc + id: find-comment with: issue-number: ${{ github.event.pull_request.number }} comment-author: 'github-actions[bot]' - body-includes: 'Compiled firmware for commit' + body-includes: 'Compiled ${{ matrix.target }} firmware for commit' - name: 'Create or update comment' - if: ${{ !github.event.pull_request.head.repo.fork && github.event.pull_request}} + if: ${{ !github.event.pull_request.head.repo.fork && matrix.target == env.DEFAULT_TARGET && github.event.pull_request }} uses: peter-evans/create-or-update-comment@v3 with: - comment-id: ${{ steps.fc.outputs.comment-id }} + comment-id: ${{ steps.find-comment.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} body: | - **Compiled firmware for commit `${{steps.names.outputs.commit_sha}}`:** + **Compiled ${{ matrix.target }} firmware for commit `${{steps.names.outputs.commit_sha}}`:** - [📦 Update package](https://update.flipperzero.one/builds/firmware/${{steps.names.outputs.branch_name}}/flipper-z-${{steps.names.outputs.default_target}}-update-${{steps.names.outputs.suffix}}.tgz) - [📥 DFU file](https://update.flipperzero.one/builds/firmware/${{steps.names.outputs.branch_name}}/flipper-z-${{steps.names.outputs.default_target}}-full-${{steps.names.outputs.suffix}}.dfu) - [☁️ Web/App updater](https://lab.flipper.net/?url=https://update.flipperzero.one/builds/firmware/${{steps.names.outputs.branch_name}}/flipper-z-${{steps.names.outputs.default_target}}-update-${{steps.names.outputs.suffix}}.tgz&channel=${{steps.names.outputs.branch_name}}&version=${{steps.names.outputs.commit_sha}}) edit-mode: replace + + - name: 'Check if API version exists' + if: ${{ steps.names.outputs.event_type == 'tag' && matrix.target == env.DEFAULT_TARGET }} + run: | + FIRMWARE_API=$(./fbt TARGET_HW=$TARGET_HW get_apiversion) + curl -sX 'GET' \ + '${{ secrets.CATALOG_URL }}/api/v0/0/sdk?length=200' \ + -H 'Accept: application/json' > sdk_versions.json + if jq -r -e ".[] | select((.api == \"${FIRMWARE_API}\") and .target == \"f${TARGET_HW}\")" sdk_versions.json > found_sdk.json ; then + echo "API version $FIRMWARE_API already exists in catalog" + if [ $(jq -r -e ".released_at" found_sdk.json) != "null" ] ; then + echo "API version is already released" + exit 0 + fi + if ! echo "$SUFFIX" | grep -q "-rc" ; then + SDK_ID=$(jq -r ._id found_sdk.json) + echo "Marking SDK $SDK_ID as released" + curl -X 'POST' \ + "${{ secrets.CATALOG_URL }}/api/v0/0/sdk/${SDK_ID}/release" \ + -H 'Accept: application/json' \ + -H 'Authorization: Bearer ${{ secrets.CATALOG_API_TOKEN }}' \ + -d '' + fi + else + echo "API version $FIRMWARE_API doesn't exist in catalog, adding" + curl -X 'POST' \ + '${{ secrets.CATALOG_URL }}/api/v0/0/sdk' \ + -H 'Accept: application/json' \ + -H 'Authorization: Bearer ${{ secrets.CATALOG_API_TOKEN }}' \ + -H 'Content-Type: application/json' \ + -d "{\"name\": \"${SUFFIX}\", \"target\": \"f${TARGET_HW}\", \"api\": \"${FIRMWARE_API}\"}\" + fi diff --git a/.github/workflows/build_compact.yml b/.github/workflows/build_compact.yml index 705888a4..37ebc0ca 100644 --- a/.github/workflows/build_compact.yml +++ b/.github/workflows/build_compact.yml @@ -73,3 +73,12 @@ jobs: popd done done + +## Uncomment this for a single job that will run only if all targets are built successfully +# report-status: +# name: Report status +# needs: [compact] +# if: always() && !contains(needs.*.result, 'failure') +# runs-on: [self-hosted, FlipperZeroShell] +# steps: +# - run: echo "All good ✨" ; diff --git a/firmware.scons b/firmware.scons index a91b7cdf..2a82db37 100644 --- a/firmware.scons +++ b/firmware.scons @@ -8,6 +8,9 @@ from fbt_extra.util import ( link_elf_dir_as_latest, ) +from fbt.sdk.cache import LazySdkVersionLoader + + Import("ENV", "fw_build_meta") # Building initial C environment for libs @@ -71,6 +74,8 @@ env = ENV.Clone( }, FW_API_TABLE=None, _APP_ICONS=None, + APPS=_.split(",") if (_ := GetOption("extra_int_apps")) else [], + EXTRA_EXT_APPS=_.split(",") if (_ := GetOption("extra_ext_apps")) else [], ) env.PreConfigureFwEnvionment() @@ -125,9 +130,6 @@ if env["IS_BASE_FIRMWARE"]: else: fwenv.Append(APPS=["updater"]) -if extra_int_apps := GetOption("extra_int_apps"): - fwenv.Append(APPS=extra_int_apps.split(",")) - for app_dir, _ in fwenv["APPDIRS"]: app_dir_node = env.Dir("#").Dir(app_dir) @@ -136,7 +138,6 @@ for app_dir, _ in fwenv["APPDIRS"]: if isinstance(entry, FS.Dir) and not str(entry).startswith("."): fwenv.LoadAppManifest(entry) - fwenv.PrepareApplicationsBuild() # Build external apps + configure SDK @@ -148,6 +149,11 @@ if env["IS_BASE_FIRMWARE"]: ) fw_artifacts.append(fwenv["FW_EXTAPPS"].sdk_tree) + fwenv.Append(FBT_API_VERSION=LazySdkVersionLoader(fwenv.subst("$SDK_DEFINITION"))) + fwenv.PhonyTarget( + "get_apiversion", + "@echo $( ${FBT_API_VERSION} $)", + ) # Add preprocessor definitions for current set of apps fwenv.Append( diff --git a/scripts/fbt/appmanifest.py b/scripts/fbt/appmanifest.py index b8b8a8d6..7bb8e40b 100644 --- a/scripts/fbt/appmanifest.py +++ b/scripts/fbt/appmanifest.py @@ -193,8 +193,19 @@ class AppManager: raise FlipperManifestException(f"Duplicate app declaration: {app.appid}") self.known_apps[app.appid] = app - def filter_apps(self, applist: List[str], hw_target: str): - return AppBuildset(self, applist, hw_target) + def filter_apps( + self, + *, + applist: List[str], + ext_applist: List[str], + hw_target: str, + ): + return AppBuildset( + self, + hw_target=hw_target, + appnames=applist, + extra_ext_appnames=ext_applist, + ) class AppBuilderException(Exception): @@ -211,6 +222,12 @@ class AppBuildset: FlipperAppType.SETTINGS, FlipperAppType.STARTUP, ) + EXTERNAL_APP_TYPES = ( + FlipperAppType.EXTERNAL, + FlipperAppType.MENUEXTERNAL, + FlipperAppType.PLUGIN, + FlipperAppType.DEBUG, + ) @staticmethod def print_writer(message): @@ -219,16 +236,21 @@ class AppBuildset: def __init__( self, appmgr: AppManager, - appnames: List[str], hw_target: str, + appnames: List[str], + *, + extra_ext_appnames: List[str], message_writer: Callable | None = None, ): self.appmgr = appmgr self.appnames = set(appnames) + self.incompatible_extapps, self.extapps = [], [] + self._extra_ext_appnames = extra_ext_appnames self._orig_appnames = appnames self.hw_target = hw_target self._writer = message_writer if message_writer else self.print_writer self._process_deps() + self._process_ext_apps() self._check_conflicts() self._check_unsatisfied() # unneeded? self._check_target_match() @@ -271,6 +293,27 @@ class AppBuildset: break self.appnames.update(provided) + def _process_ext_apps(self): + extapps = [ + app + for apptype in self.EXTERNAL_APP_TYPES + for app in self.get_apps_of_type(apptype, True) + ] + extapps.extend(map(self.appmgr.get, self._extra_ext_appnames)) + + for app in extapps: + ( + self.extapps + if app.supports_hardware_target(self.hw_target) + else self.incompatible_extapps + ).append(app) + + def get_ext_apps(self): + return self.extapps + + def get_incompatible_ext_apps(self): + return self.incompatible_extapps + def _check_conflicts(self): conflicts = [] for app in self.appnames: diff --git a/scripts/fbt/sdk/cache.py b/scripts/fbt/sdk/cache.py index 074cac6b..e3d93a31 100644 --- a/scripts/fbt/sdk/cache.py +++ b/scripts/fbt/sdk/cache.py @@ -255,3 +255,18 @@ class SdkCache: self.sync_sets(self.sdk.headers, api.headers, False) self.sync_sets(self.sdk.functions, api.functions) self.sync_sets(self.sdk.variables, api.variables) + + +class LazySdkVersionLoader: + def __init__(self, sdk_path: str): + self.sdk_path = sdk_path + self._version = None + + @property + def version(self) -> SdkVersion: + if self._version is None: + self._version = SdkCache(self.sdk_path, load_version_only=True).version + return self._version + + def __str__(self) -> str: + return str(self.version) diff --git a/scripts/fbt_tools/fbt_apps.py b/scripts/fbt_tools/fbt_apps.py index cbb3bf72..9a071805 100644 --- a/scripts/fbt_tools/fbt_apps.py +++ b/scripts/fbt_tools/fbt_apps.py @@ -36,7 +36,9 @@ def LoadAppManifest(env, entry): def PrepareApplicationsBuild(env): try: appbuild = env["APPBUILD"] = env["APPMGR"].filter_apps( - env["APPS"], env.subst("f${TARGET_HW}") + applist=env["APPS"], + ext_applist=env["EXTRA_EXT_APPS"], + hw_target=env.subst("f${TARGET_HW}"), ) except Exception as e: raise StopError(e) @@ -56,6 +58,11 @@ def DumpApplicationConfig(target, source, env): fg.green(f"{apptype.value}:\n\t"), ", ".join(app.appid for app in app_sublist), ) + if incompatible_ext_apps := env["APPBUILD"].get_incompatible_ext_apps(): + print( + fg.blue("Incompatible apps (skipped):\n\t"), + ", ".join(app.appid for app in incompatible_ext_apps), + ) def build_apps_c(target, source, env): diff --git a/scripts/ufbt/SConstruct b/scripts/ufbt/SConstruct index 1630135c..98e6b638 100644 --- a/scripts/ufbt/SConstruct +++ b/scripts/ufbt/SConstruct @@ -107,6 +107,7 @@ env = core_env.Clone( SINGLEQUOTEFUNC=single_quote, ABSPATHGETTERFUNC=resolve_real_dir_node, APPS=[], + EXTRA_EXT_APPS=[], UFBT_API_VERSION=SdkCache( core_env.subst("$SDK_DEFINITION"), load_version_only=True ).version, diff --git a/site_scons/extapps.scons b/site_scons/extapps.scons index 97b7ac09..f9227ed3 100644 --- a/site_scons/extapps.scons +++ b/site_scons/extapps.scons @@ -60,40 +60,12 @@ class FlipperExtAppBuildArtifacts: sdk_tree: NodeList = field(default_factory=NodeList) -apps_to_build_as_faps = [ - FlipperAppType.PLUGIN, - FlipperAppType.EXTERNAL, - FlipperAppType.MENUEXTERNAL, - FlipperAppType.DEBUG, -] - -known_extapps = [ - app - for apptype in apps_to_build_as_faps - for app in appenv["APPBUILD"].get_apps_of_type(apptype, True) -] - -# Ugly access to global option -if extra_app_list := GetOption("extra_ext_apps"): - known_extapps.extend(map(appenv["APPMGR"].get, extra_app_list.split(","))) - -incompatible_apps = [] -for app in known_extapps: - if not app.supports_hardware_target(appenv.subst("f${TARGET_HW}")): - incompatible_apps.append(app) - continue - +for app in appenv["APPBUILD"].get_ext_apps(): appenv.BuildAppElf(app) extapps = FlipperExtAppBuildArtifacts() extapps.application_map = appenv["EXT_APPS"] -if incompatible_apps: - warn( - WarningOnByDefault, - f"Skipping build of {len(incompatible_apps)} incompatible app(s): " - + ", ".join(f"'{app.name}' (id '{app.appid}')" for app in incompatible_apps), - ) if appenv["FORCE"]: appenv.AlwaysBuild(