[FL-2832] fbt: more fixes & improvements (#1854)

* github: bundling debug folder with scripts; docs: fixes & updates; fbt: added FAP_EXAMPLES variable to enable building example apps. Disabled by default. fbt: added TERM to list of proxied environment variables
* fbt: better help output; disabled implicit_deps_unchanged; added color to import validator reports
* fbt: moved debug configuration to separate tool
* fbt: proper dependency tracker for SDK source file; renamed linker script for external apps
* fbt: fixed debug elf path
* fbt: packaging sdk archive
* scripts: fixed sconsdist.py
* fbt: reworked sdk packing; docs: updates
* docs: info on cli target; linter fixes
* fbt: moved main code to scripts folder
* scripts: packing update into .tgz
* fbt, scripts: reworked copro_dist to build .tgz
* scripts: fixed naming for archived updater package
* Scripts: fix ぐるぐる回る

Co-authored-by: Aleksandr Kutuzov <alleteam@gmail.com>

.github/workflows/build.yml

@@ -56,14 +56,14 @@ jobs:
       - name: 'Bundle scripts'
         if: ${{ !github.event.pull_request.head.repo.fork }}
         run: |
-          tar czpf artifacts/flipper-z-any-scripts-${SUFFIX}.tgz scripts
+          tar czpf artifacts/flipper-z-any-scripts-${SUFFIX}.tgz scripts debug
 
       - name: 'Build the firmware'
         run: |
           set -e
           for TARGET in ${TARGETS}; do
             FBT_TOOLCHAIN_PATH=/runner/_work ./fbt TARGET_HW="$(echo "${TARGET}" | sed 's/f//')" \
-                updater_package ${{ startsWith(github.ref, 'refs/tags') && 'DEBUG=0 COMPACT=1' || '' }}
+                copro_dist updater_package ${{ startsWith(github.ref, 'refs/tags') && 'DEBUG=0 COMPACT=1' || '' }}
           done
 
       - name: 'Move upload files'
@@ -74,17 +74,6 @@ jobs:
             mv dist/${TARGET}-*/* artifacts/
           done
 
-      - name: 'Bundle self-update package'
-        if: ${{ !github.event.pull_request.head.repo.fork }}
-        run: |
-          set -e
-          for UPDATEBUNDLE in artifacts/*/; do
-            BUNDLE_NAME="$(echo "$UPDATEBUNDLE" | cut -d'/' -f2)"
-            echo Packaging "${BUNDLE_NAME}"
-            tar czpf "artifacts/flipper-z-${BUNDLE_NAME}.tgz" -C artifacts "${BUNDLE_NAME}"
-            rm -rf "artifacts/${BUNDLE_NAME}"
-          done
-
       - name: "Check for uncommitted changes"
         run: |
           git diff --exit-code
@@ -97,8 +86,7 @@ jobs:
       - name: 'Bundle core2 firmware'
         if: ${{ !github.event.pull_request.head.repo.fork }}
         run: |
-          FBT_TOOLCHAIN_PATH=/runner/_work ./fbt copro_dist
+          cp build/core2_firmware.tgz "artifacts/flipper-z-any-core2_firmware-${SUFFIX}.tgz"
-          tar czpf "artifacts/flipper-z-any-core2_firmware-${SUFFIX}.tgz" -C assets core2_firmware
 
       - name: 'Copy .map file'
         run: |

SConstruct

@@ -7,7 +7,6 @@
 # construction of certain targets behind command-line options.
 
 import os
-import subprocess
 
 DefaultEnvironment(tools=[])
 
@@ -15,17 +14,22 @@ EnsurePythonVersion(3, 8)
 
 # Progress(["OwO\r", "owo\r", "uwu\r", "owo\r"], interval=15)
 
-
 # This environment is created only for loading options & validating file/dir existence
 fbt_variables = SConscript("site_scons/commandline.scons")
-cmd_environment = Environment(tools=[], variables=fbt_variables)
+cmd_environment = Environment(
-Help(fbt_variables.GenerateHelpText(cmd_environment))
+    toolpath=["#/scripts/fbt_tools"],
+    tools=[
+        ("fbt_help", {"vars": fbt_variables}),
+    ],
+    variables=fbt_variables,
+)
 
 # Building basic environment - tools, utility methods, cross-compilation
 # settings, gcc flags for Cortex-M4, basic builders and more
 coreenv = SConscript(
     "site_scons/environ.scons",
     exports={"VAR_ENV": cmd_environment},
+    toolpath=["#/scripts/fbt_tools"],
 )
 SConscript("site_scons/cc.scons", exports={"ENV": coreenv})
 
@@ -35,41 +39,13 @@ coreenv["ROOT_DIR"] = Dir(".")
 
 # Create a separate "dist" environment and add construction envs to it
 distenv = coreenv.Clone(
-    tools=["fbt_dist", "openocd", "blackmagic", "jflash"],
+    tools=[
-    OPENOCD_GDB_PIPE=[
+        "fbt_dist",
-        "|openocd -c 'gdb_port pipe; log_output debug/openocd.log' ${[SINGLEQUOTEFUNC(OPENOCD_OPTS)]}"
+        "fbt_debugopts",
+        "openocd",
+        "blackmagic",
+        "jflash",
     ],
-    GDBOPTS_BASE=[
-        "-ex",
-        "target extended-remote ${GDBREMOTE}",
-        "-ex",
-        "set confirm off",
-        "-ex",
-        "set pagination off",
-    ],
-    GDBOPTS_BLACKMAGIC=[
-        "-ex",
-        "monitor swdp_scan",
-        "-ex",
-        "monitor debug_bmp enable",
-        "-ex",
-        "attach 1",
-        "-ex",
-        "set mem inaccessible-by-default off",
-    ],
-    GDBPYOPTS=[
-        "-ex",
-        "source debug/FreeRTOS/FreeRTOS.py",
-        "-ex",
-        "source debug/flipperapps.py",
-        "-ex",
-        "source debug/PyCortexMDebug/PyCortexMDebug.py",
-        "-ex",
-        "svd_load ${SVD_FILE}",
-        "-ex",
-        "compare-sections",
-    ],
-    JFLASHPROJECT="${ROOT_DIR.abspath}/debug/fw.jflash",
     ENV=os.environ,
 )
 
@@ -166,7 +142,7 @@ basic_dist = distenv.DistCommand("fw_dist", distenv["DIST_DEPENDS"])
 distenv.Default(basic_dist)
 
 dist_dir = distenv.GetProjetDirName()
-plugin_dist = [
+fap_dist = [
     distenv.Install(
         f"#/dist/{dist_dir}/apps/debug_elf",
         firmware_env["FW_EXTAPPS"]["debug"].values(),
@@ -176,9 +152,9 @@ plugin_dist = [
         for dist_entry in firmware_env["FW_EXTAPPS"]["dist"].values()
     ),
 ]
-Depends(plugin_dist, firmware_env["FW_EXTAPPS"]["validators"].values())
+Depends(fap_dist, firmware_env["FW_EXTAPPS"]["validators"].values())
-Alias("plugin_dist", plugin_dist)
+Alias("fap_dist", fap_dist)
-# distenv.Default(plugin_dist)
+# distenv.Default(fap_dist)
 
 plugin_resources_dist = list(
     distenv.Install(f"#/assets/resources/apps/{dist_entry[0]}", dist_entry[1])
@@ -189,9 +165,10 @@ distenv.Depends(firmware_env["FW_RESOURCES"], plugin_resources_dist)
 
 # Target for bundling core2 package for qFlipper
 copro_dist = distenv.CoproBuilder(
-    distenv.Dir("assets/core2_firmware"),
+    "#/build/core2_firmware.tgz",
     [],
 )
+distenv.AlwaysBuild(copro_dist)
 distenv.Alias("copro_dist", copro_dist)
 
 firmware_flash = distenv.AddOpenOCDFlashTarget(firmware_env)

applications/examples/example_images/example_images.c

@@ -4,6 +4,8 @@
 #include <gui/gui.h>
 #include <input/input.h>
 
+/* Magic happens here -- this file is generated by fbt.
+ * Just set fap_icon_assets in application.fam and #include {APPID}_icons.h */
 #include "example_images_icons.h"
 
 typedef struct {

documentation/AppManifests.md

@@ -30,7 +30,7 @@ Only 2 parameters are mandatory: ***appid*** and ***apptype***, others are optio
 | METAPACKAGE  | Does not define any code to be run, used for declaring dependencies and application bundles |
 
 * **name**: Name that is displayed in menus.
-* **entry_point**: C function to be used as application's entry point.
+* **entry_point**: C function to be used as application's entry point. Note that C++ function names are mangled, so you need to wrap them in `extern "C"` in order to use them as entry points.
 * **flags**: Internal flags for system apps. Do not use.
 * **cdefines**: C preprocessor definitions to declare globally for other apps when current application is included in active build configuration.
 * **requires**: List of application IDs to also include in build configuration, when current application is referenced in list of applications to build.
@@ -55,7 +55,7 @@ The following parameters are used only for [FAPs](./AppsOnSDCard.md):
 * **fap_author**: string, may be empty. Application's author.
 * **fap_weburl**: string, may be empty. Application's homepage.
 * **fap_icon_assets**: string. If present, defines a folder name to be used for gathering image assets for this application. These images will be preprocessed and built alongside the application. See [FAP assets](./AppsOnSDCard.md#fap-assets) for details.
-* **fap_extbuild**: provides support for parts of application sources to be build by external tools. Contains a list of `ExtFile(path="file name", command="shell command")` definitions. **`fbt`** will run the specified command for each file in the list.
+* **fap_extbuild**: provides support for parts of application sources to be built by external tools. Contains a list of `ExtFile(path="file name", command="shell command")` definitions. **`fbt`** will run the specified command for each file in the list.
 Note that commands are executed at the firmware root folder's root, and all intermediate files must be placed in a application's temporary build folder. For that, you can use pattern expansion by **`fbt`**: `${FAP_WORK_DIR}` will be replaced with the path to the application's temporary build folder, and `${FAP_SRC_DIR}` will be replaced with the path to the application's source folder. You can also use other variables defined internally by **`fbt`**. 
 
 Example for building an app from Rust sources:

documentation/AppsOnSDCard.md

@@ -2,7 +2,7 @@
 
 [fbt](./fbt.md) has support for building applications as FAP files. FAP are essentially .elf executables with extra metadata and resources bundled in.
 
-FAPs are built with `faps` **`fbt`** target. They can also be deployed to `dist` folder with `plugin_dist` **`fbt`** target.
+FAPs are built with `faps` target. They can also be deployed to `dist` folder with `fap_dist` target.
 
 FAPs do not depend on being run on a specific firmware version. Compatibility is determined by the FAP's metadata, which includes the required [API version](#api-versioning).
 
@@ -15,7 +15,7 @@ To build your application as a FAP, just create a folder with your app's source
 
  * To build your application, run `./fbt fap_{APPID}`, where APPID is your application's ID in its manifest.
  * To build your app, then upload it over USB & run it on Flipper, use `./fbt launch_app APPSRC=applications/path/to/app`. This command is configured in default [VSCode profile](../.vscode/ReadMe.md) as "Launch App on Flipper" build action (Ctrl+Shift+B menu).
- * To build all FAPs, run `./fbt plugin_dist`.
+ * To build all FAPs, run `./fbt faps` or `./fbt fap_dist`.
 
 
 ## FAP assets

documentation/fbt.md

@@ -43,7 +43,7 @@ To run cleanup (think of `make clean`) for specified targets, add `-c` option.
 ### High-level (what you most likely need)
  
 - `fw_dist` - build & publish firmware to `dist` folder. This is a default target, when no other are specified
-- `plugin_dist` - build external plugins & publish to `dist` folder  
+- `fap_dist` - build external plugins & publish to `dist` folder  
 - `updater_package`, `updater_minpackage` - build self-update package. Minimal version only inclues firmware's DFU file; full version also includes radio stack & resources for SD card
 - `copro_dist` - bundle Core2 FUS+stack binaries for qFlipper
 - `flash` - flash attached device with OpenOCD over ST-Link
@@ -56,6 +56,7 @@ To run cleanup (think of `make clean`) for specified targets, add `-c` option.
 - `get_blackmagic` - output blackmagic address in gdb remote format. Useful for IDE integration
 - `lint`, `format` - run clang-format on C source code to check and reformat it according to `.clang-format` specs
 - `lint_py`, `format_py` - run [black](https://black.readthedocs.io/en/stable/index.html) on Python source code, build system files & application manifests 
+- `cli` - start Flipper CLI session over USB
 
 ### Firmware targets
 

firmware.scons

@@ -3,7 +3,7 @@ Import("ENV", "fw_build_meta")
 from SCons.Errors import UserError
 import itertools
 
-from fbt.util import (
+from fbt_extra.util import (
     should_gen_cdb_and_link_dir,
     link_elf_dir_as_latest,
 )
@@ -141,6 +141,10 @@ else:
 if extra_int_apps := GetOption("extra_int_apps"):
     fwenv.Append(APPS=extra_int_apps.split(","))
 
+
+if fwenv["FAP_EXAMPLES"]:
+    fwenv.Append(APPDIRS=[("applications/examples", False)])
+
 fwenv.LoadApplicationManifests()
 fwenv.PrepareApplicationsBuild()
 
@@ -316,10 +320,13 @@ if fwenv["IS_BASE_FIRMWARE"]:
             "-D__inline__=inline",
         ],
     )
-    Depends(sdk_source, (fwenv["SDK_HEADERS"], fwenv["FW_ASSETS_HEADERS"]))
+    # Depends(sdk_source, (fwenv["SDK_HEADERS"], fwenv["FW_ASSETS_HEADERS"]))
+    Depends(sdk_source, fwenv.ProcessSdkDepends("sdk_origin.d"))
 
-    sdk_tree = fwenv.SDKTree("sdk/sdk.opts", "sdk_origin")
+    fwenv["SDK_DIR"] = fwenv.Dir("sdk")
-    AlwaysBuild(sdk_tree)
+    sdk_tree = fwenv.SDKTree(fwenv["SDK_DIR"], "sdk_origin")
+    fw_artifacts.append(sdk_tree)
+    # AlwaysBuild(sdk_tree)
     Alias("sdk_tree", sdk_tree)
 
     sdk_apicheck = fwenv.SDKSymUpdater(fwenv.subst("$SDK_DEFINITION"), "sdk_origin")
@@ -329,7 +336,7 @@ if fwenv["IS_BASE_FIRMWARE"]:
     Alias("sdk_check", sdk_apicheck)
 
     sdk_apisyms = fwenv.SDKSymGenerator(
-        "assets/compiled/symbols.h", fwenv.subst("$SDK_DEFINITION")
+        "assets/compiled/symbols.h", fwenv["SDK_DEFINITION"]
     )
     Alias("api_syms", sdk_apisyms)
 

scripts/fbt/util.py

@@ -41,25 +41,3 @@ def link_dir(target_path, source_path, is_windows):
 
 def single_quote(arg_list):
     return " ".join(f"'{arg}'" if " " in arg else str(arg) for arg in arg_list)
-
-
-def link_elf_dir_as_latest(env, elf_node):
-    elf_dir = elf_node.Dir(".")
-    latest_dir = env.Dir("#build/latest")
-    print(f"Setting {elf_dir} as latest built dir (./build/latest/)")
-    return link_dir(latest_dir.abspath, elf_dir.abspath, env["PLATFORM"] == "win32")
-
-
-def should_gen_cdb_and_link_dir(env, requested_targets):
-    explicitly_building_updater = False
-    # Hacky way to check if updater-related targets were requested
-    for build_target in requested_targets:
-        if "updater" in str(build_target):
-            explicitly_building_updater = True
-
-    is_updater = not env["IS_BASE_FIRMWARE"]
-    # If updater is explicitly requested, link to the latest updater
-    # Otherwise, link to firmware
-    return (is_updater and explicitly_building_updater) or (
-        not is_updater and not explicitly_building_updater
-    )

scripts/fbt_tools/fbt_debugopts.py

@@ -0,0 +1,41 @@
+def generate(env, **kw):
+    env.SetDefault(
+        OPENOCD_GDB_PIPE=[
+            "|openocd -c 'gdb_port pipe; log_output debug/openocd.log' ${[SINGLEQUOTEFUNC(OPENOCD_OPTS)]}"
+        ],
+        GDBOPTS_BASE=[
+            "-ex",
+            "target extended-remote ${GDBREMOTE}",
+            "-ex",
+            "set confirm off",
+            "-ex",
+            "set pagination off",
+        ],
+        GDBOPTS_BLACKMAGIC=[
+            "-ex",
+            "monitor swdp_scan",
+            "-ex",
+            "monitor debug_bmp enable",
+            "-ex",
+            "attach 1",
+            "-ex",
+            "set mem inaccessible-by-default off",
+        ],
+        GDBPYOPTS=[
+            "-ex",
+            "source debug/FreeRTOS/FreeRTOS.py",
+            "-ex",
+            "source debug/flipperapps.py",
+            "-ex",
+            "source debug/PyCortexMDebug/PyCortexMDebug.py",
+            "-ex",
+            "svd_load ${SVD_FILE}",
+            "-ex",
+            "compare-sections",
+        ],
+        JFLASHPROJECT="${ROOT_DIR.abspath}/debug/fw.jflash",
+    )
+
+
+def exists(env):
+    return True

scripts/fbt_tools/fbt_dist.py

@@ -136,7 +136,6 @@ def generate(env):
             "CoproBuilder": Builder(
                 action=Action(
                     [
-                        Mkdir("$TARGET"),
                         '${PYTHON3} "${ROOT_DIR.abspath}/scripts/assets.py" '
                         "copro ${COPRO_CUBE_DIR} "
                         "${TARGET} ${COPRO_MCU_FAMILY} "
@@ -145,7 +144,7 @@ def generate(env):
                         '--stack_file="${COPRO_STACK_BIN}" '
                         "--stack_addr=${COPRO_STACK_ADDR} ",
                     ],
-                    "",
+                    "\tCOPRO\t${TARGET}",
                 )
             ),
         }

scripts/fbt_tools/fbt_extapps.py

@@ -6,12 +6,10 @@ import SCons.Warnings
 import os
 import pathlib
 from fbt.elfmanifest import assemble_manifest_data
-from fbt.appmanifest import FlipperManifestException
+from fbt.appmanifest import FlipperApplication, FlipperManifestException
 from fbt.sdk import SdkCache
 import itertools
 
-from site_scons.fbt.appmanifest import FlipperApplication
-
 
 def BuildAppElf(env, app):
     ext_apps_work_dir = env.subst("$EXT_APPS_WORK_DIR")
@@ -111,7 +109,7 @@ def BuildAppElf(env, app):
     )
 
     app_elf_raw = app_env.Program(
-        os.path.join(app_work_dir, f"{app.appid}_d"),
+        os.path.join(ext_apps_work_dir, f"{app.appid}_d"),
         app_sources,
         APP_ENTRY=app.entry_point,
     )
@@ -180,7 +178,7 @@ def validate_app_imports(target, source, env):
     if unresolved_syms:
         SCons.Warnings.warn(
             SCons.Warnings.LinkWarning,
-            f"{source[0].path}: app won't run. Unresolved symbols: {unresolved_syms}",
+            f"\033[93m{source[0].path}: app won't run. Unresolved symbols: \033[95m{unresolved_syms}\033[0m",
         )
 
 

scripts/fbt_tools/fbt_help.py

@@ -0,0 +1,44 @@
+targets_help = """Configuration variables:
+"""
+
+tail_help = """
+
+TASKS:
+Building:
+    firmware_all, fw_dist:
+        Build firmware; create distribution package
+    faps, fap_dist:
+        Build all FAP apps
+    fap_{APPID}, launch_app APPSRC={APPID}:
+        Build FAP app with appid={APPID}; upload & start it over USB
+
+Flashing & debugging:
+    flash, flash_blackmagic, jflash:
+        Flash firmware to target using debug probe
+    flash_usb, flash_usb_full: 
+        Install firmware using self-update package
+    debug, debug_other, blackmagic: 
+        Start GDB
+
+Other:
+    cli:
+        Open a Flipper CLI session over USB
+    firmware_cdb, updater_cdb:
+        Generate сompilation_database.json
+    lint, lint_py:
+        run linters
+    format, format_py:
+        run code formatters
+
+For more targets & info, see documentation/fbt.md
+"""
+
+
+def generate(env, **kw):
+    vars = kw["vars"]
+    basic_help = vars.GenerateHelpText(env)
+    env.Help(targets_help + basic_help + tail_help)
+
+
+def exists(env):
+    return True

scripts/fbt_tools/fbt_sdk.py

@@ -9,10 +9,32 @@ from SCons.Util import LogicalLines
 import os.path
 import posixpath
 import pathlib
+import json
 
 from fbt.sdk import SdkCollector, SdkCache
 
 
+def ProcessSdkDepends(env, filename):
+    try:
+        with open(filename, "r") as fin:
+            lines = LogicalLines(fin).readlines()
+    except IOError:
+        return []
+
+    _, depends = lines[0].split(":", 1)
+    depends = depends.split()
+    depends.pop(0)  # remove the .c file
+    depends = list(
+        # Don't create dependency on non-existing files
+        # (e.g. when they were renamed since last build)
+        filter(
+            lambda file: file.exists(),
+            (env.File(f"#{path}") for path in depends),
+        )
+    )
+    return depends
+
+
 def prebuild_sdk_emitter(target, source, env):
     target.append(env.ChangeFileExtension(target[0], ".d"))
     target.append(env.ChangeFileExtension(target[0], ".i.c"))
@@ -25,6 +47,25 @@ def prebuild_sdk_create_origin_file(target, source, env):
         sdk_c.write("\n".join(f"#include <{h.path}>" for h in env["SDK_HEADERS"]))
 
 
+class SdkMeta:
+    def __init__(self, env):
+        self.env = env
+
+    def save_to(self, json_manifest_path: str):
+        meta_contents = {
+            "sdk_symbols": self.env["SDK_DEFINITION"].name,
+            "cc_args": self._wrap_scons_vars("$CCFLAGS $_CCCOMCOM"),
+            "cpp_args": self._wrap_scons_vars("$CXXFLAGS $CCFLAGS $_CCCOMCOM"),
+            "linker_args": self._wrap_scons_vars("$LINKFLAGS"),
+        }
+        with open(json_manifest_path, "wt") as f:
+            json.dump(meta_contents, f, indent=4)
+
+    def _wrap_scons_vars(self, vars: str):
+        expanded_vars = self.env.subst(vars, target=Entry("dummy"))
+        return expanded_vars.replace("\\", "/")
+
+
 class SdkTreeBuilder:
     def __init__(self, env, target, source) -> None:
         self.env = env
@@ -34,8 +75,9 @@ class SdkTreeBuilder:
         self.header_depends = []
         self.header_dirs = []
 
-        self.target_sdk_dir = env.subst("f${TARGET_HW}_sdk")
+        self.target_sdk_dir_name = env.subst("f${TARGET_HW}_sdk")
-        self.sdk_deploy_dir = target[0].Dir(self.target_sdk_dir)
+        self.sdk_root_dir = target[0].Dir(".")
+        self.sdk_deploy_dir = self.sdk_root_dir.Dir(self.target_sdk_dir_name)
 
     def _parse_sdk_depends(self):
         deps_file = self.source[0]
@@ -50,7 +92,7 @@ class SdkTreeBuilder:
             )
 
     def _generate_sdk_meta(self):
-        filtered_paths = [self.target_sdk_dir]
+        filtered_paths = [self.target_sdk_dir_name]
         full_fw_paths = list(
             map(
                 os.path.normpath,
@@ -62,17 +104,18 @@ class SdkTreeBuilder:
         for dir in full_fw_paths:
             if dir in sdk_dirs:
                 filtered_paths.append(
-                    posixpath.normpath(posixpath.join(self.target_sdk_dir, dir))
+                    posixpath.normpath(posixpath.join(self.target_sdk_dir_name, dir))
                 )
 
         sdk_env = self.env.Clone()
         sdk_env.Replace(CPPPATH=filtered_paths)
-        with open(self.target[0].path, "wt") as f:
+        meta = SdkMeta(sdk_env)
-            cmdline_options = sdk_env.subst(
+        meta.save_to(self.target[0].path)
-                "$CCFLAGS $_CCCOMCOM", target=Entry("dummy")
+
-            )
+    def emitter(self, target, source, env):
-            f.write(cmdline_options.replace("\\", "/"))
+        target_folder = target[0]
-            f.write("\n")
+        target = [target_folder.File("sdk.opts")]
+        return target, source
 
     def _create_deploy_commands(self):
         dirs_to_create = set(
@@ -81,13 +124,17 @@ class SdkTreeBuilder:
         actions = [
             Delete(self.sdk_deploy_dir),
             Mkdir(self.sdk_deploy_dir),
+            Copy(
+                self.sdk_root_dir,
+                self.env["SDK_DEFINITION"],
+            ),
         ]
         actions += [Mkdir(d) for d in dirs_to_create]
 
         actions += [
-            Copy(
+            Action(
-                self.sdk_deploy_dir.File(h).path,
+                Copy(self.sdk_deploy_dir.File(h).path, h),
-                h,
+                # f"Copy {h} to {self.sdk_deploy_dir}",
             )
             for h in self.header_depends
         ]
@@ -108,6 +155,11 @@ def deploy_sdk_tree(target, source, env, for_signature):
     return sdk_tree.generate_actions()
 
 
+def deploy_sdk_tree_emitter(target, source, env):
+    sdk_tree = SdkTreeBuilder(env, target, source)
+    return sdk_tree.emitter(target, source, env)
+
+
 def gen_sdk_data(sdk_cache: SdkCache):
     api_def = []
     api_def.extend(
@@ -165,6 +217,7 @@ def generate_sdk_symbols(source, target, env):
 
 
 def generate(env, **kw):
+    env.AddMethod(ProcessSdkDepends)
     env.Append(
         BUILDERS={
             "SDKPrebuilder": Builder(
@@ -183,6 +236,7 @@ def generate(env, **kw):
             ),
             "SDKTree": Builder(
                 generator=deploy_sdk_tree,
+                emitter=deploy_sdk_tree_emitter,
                 src_suffix=".d",
             ),
             "SDKSymUpdater": Builder(

scripts/flipper/assets/copro.py

@@ -1,10 +1,9 @@
 import logging
-import datetime
-import shutil
 import json
-from os.path import basename
+from io import BytesIO
-
+import tarfile
 import xml.etree.ElementTree as ET
+
 from flipper.utils import *
 from flipper.assets.coprobin import CoproBinary, get_stack_type
 
@@ -51,20 +50,19 @@ class Copro:
             raise Exception(f"Unsupported cube version")
         self.version = cube_version
 
+    @staticmethod
+    def _getFileName(name):
+        return os.path.join("core2_firmware", name)
+
     def addFile(self, array, filename, **kwargs):
         source_file = os.path.join(self.mcu_copro, filename)
-        destination_file = os.path.join(self.output_dir, filename)
+        self.output_tar.add(source_file, arcname=self._getFileName(filename))
-        shutil.copyfile(source_file, destination_file)
+        array.append({"name": filename, "sha256": file_sha256(source_file), **kwargs})
-        array.append(
+
-            {"name": filename, "sha256": file_sha256(destination_file), **kwargs}
+    def bundle(self, output_file, stack_file_name, stack_type, stack_addr=None):
-        )
+        self.output_tar = tarfile.open(output_file, "w:gz")
 
-    def bundle(self, output_dir, stack_file_name, stack_type, stack_addr=None):
-        if not os.path.isdir(output_dir):
-            raise Exception(f'"{output_dir}" doesn\'t exists')
-        self.output_dir = output_dir
         stack_file = os.path.join(self.mcu_copro, stack_file_name)
-        manifest_file = os.path.join(self.output_dir, "Manifest.json")
         # Form Manifest
         manifest = dict(MANIFEST_TEMPLATE)
         manifest["manifest"]["timestamp"] = timestamp()
@@ -105,6 +103,10 @@ class Copro:
             stack_file_name,
             address=f"0x{stack_addr:X}",
         )
-        # Save manifest to
+
-        with open(manifest_file, "w", newline="\n") as file:
+        # Save manifest
-            json.dump(manifest, file)
+        manifest_data = json.dumps(manifest, indent=4).encode("utf-8")
+        info = tarfile.TarInfo(self._getFileName("Manifest.json"))
+        info.size = len(manifest_data)
+        self.output_tar.addfile(info, BytesIO(manifest_data))
+        self.output_tar.close()

scripts/guruguru.py

@@ -17,7 +17,7 @@ class Main(App):
     async def rebuild(self, line):
         self.clearConsole()
         self.logger.info(f"Triggered by: {line}")
-        proc = await asyncio.create_subprocess_exec("make")
+        proc = await asyncio.create_subprocess_exec("./fbt")
         await proc.wait()
         await asyncio.sleep(1)
         self.is_building = False

scripts/sconsdist.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 
 from flipper.app import App
-from os.path import join, exists
+from os.path import join, exists, relpath
-from os import makedirs
+from os import makedirs, walk
 from update import Main as UpdateMain
 import shutil
+import zipfile
+import tarfile
 
 
 class ProjectDir:
@@ -17,6 +19,8 @@ class ProjectDir:
 
 
 class Main(App):
+    DIST_FILE_PREFIX = "flipper-z-"
+
     def init(self):
         self.subparsers = self.parser.add_subparsers(help="sub-command help")
 
@@ -45,9 +49,13 @@ class Main(App):
     def get_project_filename(self, project, filetype):
         #  Temporary fix
         project_name = project.project
-        if project_name == "firmware" and filetype != "elf":
+        if project_name == "firmware":
-            project_name = "full"
+            if filetype == "zip":
-        return f"flipper-z-{self.target}-{project_name}-{self.args.suffix}.{filetype}"
+                project_name = "sdk"
+            elif filetype != "elf":
+                project_name = "full"
+
+        return f"{self.DIST_FILE_PREFIX}{self.target}-{project_name}-{self.args.suffix}.{filetype}"
 
     def get_dist_filepath(self, filename):
         return join(self.output_dir_path, filename)
@@ -56,10 +64,28 @@ class Main(App):
         obj_directory = join("build", project.dir)
 
         for filetype in ("elf", "bin", "dfu", "json"):
-            shutil.copyfile(
+            if exists(src_file := join(obj_directory, f"{project.project}.{filetype}")):
-                join(obj_directory, f"{project.project}.{filetype}"),
+                shutil.copyfile(
-                self.get_dist_filepath(self.get_project_filename(project, filetype)),
+                    src_file,
-            )
+                    self.get_dist_filepath(
+                        self.get_project_filename(project, filetype)
+                    ),
+                )
+            if exists(sdk_folder := join(obj_directory, "sdk")):
+                with zipfile.ZipFile(
+                    self.get_dist_filepath(self.get_project_filename(project, "zip")),
+                    "w",
+                    zipfile.ZIP_DEFLATED,
+                ) as zf:
+                    for root, dirs, files in walk(sdk_folder):
+                        for file in files:
+                            zf.write(
+                                join(root, file),
+                                relpath(
+                                    join(root, file),
+                                    sdk_folder,
+                                ),
+                            )
 
     def copy(self):
         self.projects = dict(
@@ -103,9 +129,8 @@ class Main(App):
         )
 
         if self.args.version:
-            bundle_dir = join(
+            bundle_dir_name = f"{self.target}-update-{self.args.suffix}"
-                self.output_dir_path, f"{self.target}-update-{self.args.suffix}"
+            bundle_dir = join(self.output_dir_path, bundle_dir_name)
-            )
             bundle_args = [
                 "generate",
                 "-d",
@@ -131,10 +156,24 @@ class Main(App):
                     )
                 )
             bundle_args.extend(self.other_args)
-            self.logger.info(
+
-                f"Use this directory to self-update your Flipper:\n\t{bundle_dir}"
+            if (bundle_result := UpdateMain(no_exit=True)(bundle_args)) == 0:
-            )
+                self.logger.info(
-            return UpdateMain(no_exit=True)(bundle_args)
+                    f"Use this directory to self-update your Flipper:\n\t{bundle_dir}"
+                )
+
+                # Create tgz archive
+                with tarfile.open(
+                    join(
+                        self.output_dir_path,
+                        f"{self.DIST_FILE_PREFIX}{bundle_dir_name}.tgz",
+                    ),
+                    "w:gz",
+                    compresslevel=9,
+                ) as tar:
+                    tar.add(bundle_dir, arcname=bundle_dir_name)
+
+            return bundle_result
 
         return 0
 

site_scons/commandline.scons

@@ -81,46 +81,41 @@ vars.AddVariables(
         help="Enable debug tools to be built",
         default=False,
     ),
-)
+    BoolVariable(
-
+        "FAP_EXAMPLES",
-vars.Add(
+        help="Enable example applications to be built",
-    "DIST_SUFFIX",
+        default=False,
-    help="Suffix for binaries in build output for dist targets",
+    ),
-    default="local",
+    (
-)
+        "DIST_SUFFIX",
-
+        "Suffix for binaries in build output for dist targets",
-vars.Add(
+        "local",
-    "UPDATE_VERSION_STRING",
+    ),
-    help="Version string for updater package",
+    (
-    default="${DIST_SUFFIX}",
+        "UPDATE_VERSION_STRING",
-)
+        "Version string for updater package",
-
+        "${DIST_SUFFIX}",
-
+    ),
-vars.Add(
+    (
-    "COPRO_CUBE_VERSION",
+        "COPRO_CUBE_VERSION",
-    help="Cube version",
+        "Cube version",
-    default="",
+        "",
-)
+    ),
-
+    (
-vars.Add(
+        "COPRO_STACK_ADDR",
-    "COPRO_STACK_ADDR",
+        "Core2 Firmware address",
-    help="Core2 Firmware address",
+        "0",
-    default="0",
+    ),
-)
+    (
-
+        "COPRO_STACK_BIN",
-vars.Add(
+        "Core2 Firmware file name",
-    "COPRO_STACK_BIN",
+        "",
-    help="Core2 Firmware file name",
+    ),
-    default="",
+    (
-)
+        "COPRO_DISCLAIMER",
-
+        "Value to pass to bundling script to confirm dangerous operations",
-vars.Add(
+        "",
-    "COPRO_DISCLAIMER",
+    ),
-    help="Value to pass to bundling script to confirm dangerous operations",
-    default="",
-)
-
-vars.AddVariables(
     PathVariable(
         "COPRO_OB_DATA",
         help="Path to OB reference data",
@@ -161,86 +156,75 @@ vars.AddVariables(
         validator=PathVariable.PathAccept,
         default="",
     ),
-)
+    (
-
+        "FBT_TOOLCHAIN_VERSIONS",
-vars.Add(
+        "Whitelisted toolchain versions (leave empty for no check)",
-    "FBT_TOOLCHAIN_VERSIONS",
+        tuple(),
-    help="Whitelisted toolchain versions (leave empty for no check)",
+    ),
-    default=tuple(),
+    (
-)
+        "OPENOCD_OPTS",
-
+        "Options to pass to OpenOCD",
-vars.Add(
+        "",
-    "OPENOCD_OPTS",
+    ),
-    help="Options to pass to OpenOCD",
+    (
-    default="",
+        "BLACKMAGIC",
-)
+        "Blackmagic probe location",
-
+        "auto",
-vars.Add(
+    ),
-    "BLACKMAGIC",
+    (
-    help="Blackmagic probe location",
+        "UPDATE_SPLASH",
-    default="auto",
+        "Directory name with slideshow frames to render after installing update package",
-)
+        "update_default",
-
+    ),
-vars.Add(
+    (
-    "UPDATE_SPLASH",
+        "LOADER_AUTOSTART",
-    help="Directory name with slideshow frames to render after installing update package",
+        "Application name to automatically run on Flipper boot",
-    default="update_default",
+        "",
-)
+    ),
-
+    (
-vars.Add(
+        "FIRMWARE_APPS",
-    "LOADER_AUTOSTART",
+        "Map of (configuration_name->application_list)",
-    help="Application name to automatically run on Flipper boot",
+        {
-    default="",
+            "default": (
-)
+                # Svc
-
+                "basic_services",
-
+                # Apps
-vars.Add(
+                "main_apps",
-    "FIRMWARE_APPS",
+                "system_apps",
-    help="Map of (configuration_name->application_list)",
+                # Settings
-    default={
+                "settings_apps",
-        "default": (
+                # Plugins
-            # Svc
+                # "basic_plugins",
-            "basic_services",
+                # Debug
-            # Apps
+                # "debug_apps",
-            "main_apps",
+            )
-            "system_apps",
+        },
-            # Settings
+    ),
-            "settings_apps",
+    (
-            # Plugins
+        "FIRMWARE_APP_SET",
-            # "basic_plugins",
+        "Application set to use from FIRMWARE_APPS",
-            # Debug
+        "default",
-            # "debug_apps",
+    ),
-        )
+    (
-    },
+        "APPSRC",
-)
+        "Application source directory for app to build & upload",
-
+        "",
-vars.Add(
+    ),
-    "FIRMWARE_APP_SET",
+    # List of tuples (directory, add_to_global_include_path)
-    help="Application set to use from FIRMWARE_APPS",
+    (
-    default="default",
+        "APPDIRS",
-)
+        "Directories to search for firmware components & external apps",
-
+        [
-vars.Add(
+            ("applications", False),
-    "APPSRC",
+            ("applications/services", True),
-    help="Application source directory for app to build & upload",
+            ("applications/main", True),
-    default="",
+            ("applications/settings", False),
-)
+            ("applications/system", False),
-
+            ("applications/debug", False),
-# List of tuples (directory, add_to_global_include_path)
+            ("applications/plugins", False),
-vars.Add(
+            ("applications_user", False),
-    "APPDIRS",
+        ],
-    help="Directories to search for firmware components & external apps",
+    ),
-    default=[
-        ("applications", False),
-        ("applications/services", True),
-        ("applications/main", True),
-        ("applications/settings", False),
-        ("applications/system", False),
-        ("applications/debug", False),
-        ("applications/plugins", False),
-        ("applications/examples", False),
-        ("applications_user", False),
-    ],
 )
 
 Return("vars")

site_scons/environ.scons

@@ -1,6 +1,5 @@
-import SCons
 from SCons.Platform import TempFileMunge
-from fbt import util
+from fbt.util import tempfile_arg_esc_func, single_quote, wrap_tempfile
 
 import os
 import multiprocessing
@@ -13,14 +12,18 @@ forward_os_env = {
 }
 # Proxying CI environment to child processes & scripts
 variables_to_forward = [
+    # CI/CD variables
     "WORKFLOW_BRANCH_OR_TAG",
     "DIST_SUFFIX",
+    # Python & other tools
     "HOME",
     "APPDATA",
     "PYTHONHOME",
     "PYTHONNOUSERSITE",
     "TMP",
     "TEMP",
+    # Colors for tools
+    "TERM",
 ]
 if proxy_env := GetOption("proxy_env"):
     variables_to_forward.extend(proxy_env.split(","))
@@ -79,7 +82,7 @@ if not coreenv["VERBOSE"]:
 SetOption("num_jobs", multiprocessing.cpu_count())
 # Avoiding re-scan of all sources on every startup
 SetOption("implicit_cache", True)
-SetOption("implicit_deps_unchanged", True)
+# SetOption("implicit_deps_unchanged", True)
 # More aggressive caching
 SetOption("max_drift", 1)
 # Random task queue - to discover isses with build logic faster
@@ -87,10 +90,10 @@ SetOption("max_drift", 1)
 
 
 # Setting up temp file parameters - to overcome command line length limits
-coreenv["TEMPFILEARGESCFUNC"] = util.tempfile_arg_esc_func
+coreenv["TEMPFILEARGESCFUNC"] = tempfile_arg_esc_func
-util.wrap_tempfile(coreenv, "LINKCOM")
+wrap_tempfile(coreenv, "LINKCOM")
-util.wrap_tempfile(coreenv, "ARCOM")
+wrap_tempfile(coreenv, "ARCOM")
 
-coreenv["SINGLEQUOTEFUNC"] = util.single_quote
+coreenv["SINGLEQUOTEFUNC"] = single_quote
 
 Return("coreenv")

site_scons/extapps.scons

@@ -21,7 +21,7 @@ appenv = ENV.Clone(
 )
 
 appenv.Replace(
-    LINKER_SCRIPT="application-ext",
+    LINKER_SCRIPT="application_ext",
 )
 
 appenv.AppendUnique(
@@ -106,6 +106,7 @@ appenv.PhonyTarget("firmware_extapps", appenv.Action(legacy_app_build_stub, None
 
 
 Alias("faps", extapps["compact"].values())
+Alias("faps", extapps["validators"].values())
 
 if appsrc := appenv.subst("$APPSRC"):
     app_manifest, fap_file, app_validator = appenv.GetExtAppFromPath(appsrc)

site_scons/fbt_extra/util.py

@@ -0,0 +1,23 @@
+from fbt.util import link_dir
+
+
+def link_elf_dir_as_latest(env, elf_node):
+    elf_dir = elf_node.Dir(".")
+    latest_dir = env.Dir("#build/latest")
+    print(f"Setting {elf_dir} as latest built dir (./build/latest/)")
+    return link_dir(latest_dir.abspath, elf_dir.abspath, env["PLATFORM"] == "win32")
+
+
+def should_gen_cdb_and_link_dir(env, requested_targets):
+    explicitly_building_updater = False
+    # Hacky way to check if updater-related targets were requested
+    for build_target in requested_targets:
+        if "updater" in str(build_target):
+            explicitly_building_updater = True
+
+    is_updater = not env["IS_BASE_FIRMWARE"]
+    # If updater is explicitly requested, link to the latest updater
+    # Otherwise, link to firmware
+    return (is_updater and explicitly_building_updater) or (
+        not is_updater and not explicitly_building_updater
+    )