From 1891d54baf962b954b1541b8109bf842aed88273 Mon Sep 17 00:00:00 2001 From: hedger Date: Thu, 21 Sep 2023 15:56:00 +0300 Subject: [PATCH] [FL-3600] Added `fal_embedded` parameter for PLUGIN apps (#3083) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fbt, ufbt: added `fal_embedded` parameter for PLIGIN apps, to embed them into .fap * fbt: fixed dependency settings for assets * fbt: extapps: Removed unneeded casts * fbt: extapps: code simplification * fbt: fal_embedded: fixed dependency relations Co-authored-by: あく --- documentation/AppManifests.md | 1 + firmware.scons | 2 +- scripts/fbt/appmanifest.py | 13 ++++ scripts/fbt/fapassets.py | 28 +++++---- scripts/fbt_tools/fbt_extapps.py | 105 ++++++++++++++++++------------- 5 files changed, 93 insertions(+), 56 deletions(-) diff --git a/documentation/AppManifests.md b/documentation/AppManifests.md index b48a6b8e..72c15ad4 100644 --- a/documentation/AppManifests.md +++ b/documentation/AppManifests.md @@ -56,6 +56,7 @@ The following parameters are used only for [FAPs](./AppsOnSDCard.md): - **fap_weburl**: string, may be empty. Application's homepage. - **fap_icon_assets**: string. If present, it defines a folder name to be used for gathering image assets for this application. These images will be preprocessed and built alongside the application. See [FAP assets](./AppsOnSDCard.md#fap-assets) for details. - **fap_extbuild**: provides support for parts of application sources to be built by external tools. Contains a list of `ExtFile(path="file name", command="shell command")` definitions. **`fbt`** will run the specified command for each file in the list. +- **fal_embedded**: boolean, default `False`. Applies only to PLUGIN type. If `True`, the plugin will be embedded into host application's .fap file as a resource and extracted to `apps_assets/APPID` folder on its start. This allows plugins to be distributed as a part of the host application. Note that commands are executed at the firmware root folder, and all intermediate files must be placed in an application's temporary build folder. For that, you can use pattern expansion by **`fbt`**: `${FAP_WORK_DIR}` will be replaced with the path to the application's temporary build folder, and `${FAP_SRC_DIR}` will be replaced with the path to the application's source folder. You can also use other variables defined internally by **`fbt`**. diff --git a/firmware.scons b/firmware.scons index 65782270..82f775d7 100644 --- a/firmware.scons +++ b/firmware.scons @@ -143,7 +143,7 @@ fwenv.PrepareApplicationsBuild() # Build external apps + configure SDK if env["IS_BASE_FIRMWARE"]: - fwenv.SetDefault(FBT_FAP_DEBUG_ELF_ROOT="${BUILD_DIR}/.extapps") + fwenv.SetDefault(FBT_FAP_DEBUG_ELF_ROOT=fwenv["BUILD_DIR"].Dir(".extapps")) fwenv["FW_EXTAPPS"] = SConscript( "site_scons/extapps.scons", exports={"ENV": fwenv}, diff --git a/scripts/fbt/appmanifest.py b/scripts/fbt/appmanifest.py index 7bb8e40b..ff49707b 100644 --- a/scripts/fbt/appmanifest.py +++ b/scripts/fbt/appmanifest.py @@ -79,11 +79,19 @@ class FlipperApplication: fap_extbuild: List[ExternallyBuiltFile] = field(default_factory=list) fap_private_libs: List[Library] = field(default_factory=list) fap_file_assets: Optional[str] = None + fal_embedded: bool = False # Internally used by fbt _appmanager: Optional["AppManager"] = None _appdir: Optional[object] = None _apppath: Optional[str] = None _plugins: List["FlipperApplication"] = field(default_factory=list) + _assets_dirs: List[object] = field(default_factory=list) + _section_fapmeta: Optional[object] = None + _section_fapfileassets: Optional[object] = None + + @property + def embeds_plugins(self): + return any(plugin.fal_embedded for plugin in self._plugins) def supports_hardware_target(self, target: str): return target in self.targets or "all" in self.targets @@ -137,6 +145,11 @@ class AppManager: raise FlipperManifestException( f"Plugin {kw.get('appid')} must have 'requires' in manifest" ) + else: + if kw.get("fal_embedded"): + raise FlipperManifestException( + f"App {kw.get('appid')} cannot have fal_embedded set" + ) # Harmless - cdefines for external apps are meaningless # if apptype == FlipperAppType.EXTERNAL and kw.get("cdefines"): # raise FlipperManifestException( diff --git a/scripts/fbt/fapassets.py b/scripts/fbt/fapassets.py index 9902fd79..cd65ad20 100644 --- a/scripts/fbt/fapassets.py +++ b/scripts/fbt/fapassets.py @@ -1,7 +1,7 @@ import hashlib import os import struct -from typing import TypedDict +from typing import TypedDict, List class File(TypedDict): @@ -32,20 +32,19 @@ class FileBundler: u8[] file_content """ - def __init__(self, directory_path: str): - self.directory_path = directory_path - self.file_list: list[File] = [] - self.directory_list: list[Dir] = [] - self._gather() + def __init__(self, assets_dirs: List[object]): + self.src_dirs = list(assets_dirs) - def _gather(self): - for root, dirs, files in os.walk(self.directory_path): + def _gather(self, directory_path: str): + if not os.path.isdir(directory_path): + raise Exception(f"Assets directory {directory_path} does not exist") + for root, dirs, files in os.walk(directory_path): for file_info in files: file_path = os.path.join(root, file_info) file_size = os.path.getsize(file_path) self.file_list.append( { - "path": os.path.relpath(file_path, self.directory_path), + "path": os.path.relpath(file_path, directory_path), "size": file_size, "content_path": file_path, } @@ -57,15 +56,20 @@ class FileBundler: # os.path.getsize(os.path.join(dir_path, f)) for f in os.listdir(dir_path) # ) self.directory_list.append( - { - "path": os.path.relpath(dir_path, self.directory_path), - } + {"path": os.path.relpath(dir_path, directory_path)} ) self.file_list.sort(key=lambda f: f["path"]) self.directory_list.sort(key=lambda d: d["path"]) + def _process_src_dirs(self): + self.file_list: list[File] = [] + self.directory_list: list[Dir] = [] + for directory_path in self.src_dirs: + self._gather(directory_path) + def export(self, target_path: str): + self._process_src_dirs() self._md5_hash = hashlib.md5() with open(target_path, "wb") as f: # Write header magic and version diff --git a/scripts/fbt_tools/fbt_extapps.py b/scripts/fbt_tools/fbt_extapps.py index 6059628f..aa6354c9 100644 --- a/scripts/fbt_tools/fbt_extapps.py +++ b/scripts/fbt_tools/fbt_extapps.py @@ -3,7 +3,7 @@ import os import pathlib import shutil from dataclasses import dataclass, field -from typing import Optional, Dict, List +from typing import Dict, List, Optional import SCons.Warnings from ansi.color import fg @@ -32,11 +32,15 @@ class FlipperExternalAppInfo: class AppBuilder: + @staticmethod + def get_app_work_dir(env, app): + return env["EXT_APPS_WORK_DIR"].Dir(app.appid) + def __init__(self, env, app): self.fw_env = env self.app = app - self.ext_apps_work_dir = env.subst("$EXT_APPS_WORK_DIR") - self.app_work_dir = os.path.join(self.ext_apps_work_dir, self.app.appid) + self.ext_apps_work_dir = env["EXT_APPS_WORK_DIR"] + self.app_work_dir = self.get_app_work_dir(env, app) self.app_alias = f"fap_{self.app.appid}" self.externally_built_files = [] self.private_libs = [] @@ -83,9 +87,9 @@ class AppBuilder: return fap_icons = self.app_env.CompileIcons( - self.app_env.Dir(self.app_work_dir), + self.app_work_dir, self.app._appdir.Dir(self.app.fap_icon_assets), - icon_bundle_name=f"{self.app.fap_icon_assets_symbol if self.app.fap_icon_assets_symbol else self.app.appid }_icons", + icon_bundle_name=f"{self.app.fap_icon_assets_symbol or self.app.appid }_icons", ) self.app_env.Alias("_fap_icons", fap_icons) self.fw_env.Append(_APP_ICONS=[fap_icons]) @@ -95,7 +99,7 @@ class AppBuilder: self.private_libs.append(self._build_private_lib(lib_def)) def _build_private_lib(self, lib_def): - lib_src_root_path = os.path.join(self.app_work_dir, "lib", lib_def.name) + lib_src_root_path = self.app_work_dir.Dir("lib").Dir(lib_def.name) self.app_env.AppendUnique( CPPPATH=list( self.app_env.Dir(lib_src_root_path) @@ -119,9 +123,7 @@ class AppBuilder: private_lib_env = self.app_env.Clone() private_lib_env.AppendUnique( - CCFLAGS=[ - *lib_def.cflags, - ], + CCFLAGS=lib_def.cflags, CPPDEFINES=lib_def.cdefines, CPPPATH=list( map( @@ -132,14 +134,17 @@ class AppBuilder: ) return private_lib_env.StaticLibrary( - os.path.join(self.app_work_dir, lib_def.name), + self.app_work_dir.File(lib_def.name), lib_sources, ) def _build_app(self): + if self.app.fap_file_assets: + self.app._assets_dirs = [self.app._appdir.Dir(self.app.fap_file_assets)] + self.app_env.Append( LIBS=[*self.app.fap_libs, *self.private_libs], - CPPPATH=[self.app_env.Dir(self.app_work_dir), self.app._appdir], + CPPPATH=[self.app_work_dir, self.app._appdir], ) app_sources = list( @@ -155,32 +160,46 @@ class AppBuilder: app_artifacts = FlipperExternalAppInfo(self.app) app_artifacts.debug = self.app_env.Program( - os.path.join(self.ext_apps_work_dir, f"{self.app.appid}_d"), + self.ext_apps_work_dir.File(f"{self.app.appid}_d.elf"), app_sources, APP_ENTRY=self.app.entry_point, )[0] app_artifacts.compact = self.app_env.EmbedAppMetadata( - os.path.join(self.ext_apps_work_dir, self.app.appid), + self.ext_apps_work_dir.File(f"{self.app.appid}.fap"), app_artifacts.debug, APP=self.app, )[0] + if self.app.embeds_plugins: + self.app._assets_dirs.append(self.app_work_dir.Dir("assets")) + app_artifacts.validator = self.app_env.ValidateAppImports( app_artifacts.compact )[0] if self.app.apptype == FlipperAppType.PLUGIN: for parent_app_id in self.app.requires: - fal_path = ( - f"apps_data/{parent_app_id}/plugins/{app_artifacts.compact.name}" - ) - deployable = True - # If it's a plugin for a non-deployable app, don't include it in the resources - if parent_app := self.app._appmanager.get(parent_app_id): - if not parent_app.is_default_deployable: - deployable = False - app_artifacts.dist_entries.append((deployable, fal_path)) + if self.app.fal_embedded: + parent_app = self.app._appmanager.get(parent_app_id) + if not parent_app: + raise UserError( + f"Embedded plugin {self.app.appid} requires unknown app {parent_app_id}" + ) + self.app_env.Install( + target=self.get_app_work_dir(self.app_env, parent_app) + .Dir("assets") + .Dir("plugins"), + source=app_artifacts.compact, + ) + else: + fal_path = f"apps_data/{parent_app_id}/plugins/{app_artifacts.compact.name}" + deployable = True + # If it's a plugin for a non-deployable app, don't include it in the resources + if parent_app := self.app._appmanager.get(parent_app_id): + if not parent_app.is_default_deployable: + deployable = False + app_artifacts.dist_entries.append((deployable, fal_path)) else: fap_path = f"apps/{self.app.fap_category}/{app_artifacts.compact.name}" app_artifacts.dist_entries.append( @@ -194,7 +213,7 @@ class AppBuilder: # Extra things to clean up along with the app self.app_env.Clean( app_artifacts.debug, - [*self.externally_built_files, self.app_env.Dir(self.app_work_dir)], + [*self.externally_built_files, self.app_work_dir], ) # Create listing of the app @@ -219,13 +238,10 @@ class AppBuilder: ) # Add dependencies on file assets - if self.app.fap_file_assets: + for assets_dir in self.app._assets_dirs: self.app_env.Depends( app_artifacts.compact, - self.app_env.GlobRecursive( - "*", - self.app._appdir.Dir(self.app.fap_file_assets), - ), + (assets_dir, self.app_env.GlobRecursive("*", assets_dir)), ) # Always run the validator for the app's binary when building the app @@ -344,25 +360,26 @@ def embed_app_metadata_emitter(target, source, env): if app.apptype == FlipperAppType.PLUGIN: target[0].name = target[0].name.replace(".fap", ".fal") - target.append(env.File(source[0].abspath + _FAP_META_SECTION)) + app_work_dir = AppBuilder.get_app_work_dir(env, app) + app._section_fapmeta = app_work_dir.File(_FAP_META_SECTION) + target.append(app._section_fapmeta) - if app.fap_file_assets: - target.append(env.File(source[0].abspath + _FAP_FILEASSETS_SECTION)) + # At this point, we haven't added dir with embedded plugins to _assets_dirs yet + if app._assets_dirs or app.embeds_plugins: + app._section_fapfileassets = app_work_dir.File(_FAP_FILEASSETS_SECTION) + target.append(app._section_fapfileassets) return (target, source) -def prepare_app_files(target, source, env): +def prepare_app_file_assets(target, source, env): files_section_node = next( filter(lambda t: t.name.endswith(_FAP_FILEASSETS_SECTION), target) ) - app = env["APP"] - directory = env.Dir(app._apppath).Dir(app.fap_file_assets) - if not directory.exists(): - raise UserError(f"File asset directory {directory} does not exist") - - bundler = FileBundler(directory.abspath) + bundler = FileBundler( + list(env.Dir(asset_dir).abspath for asset_dir in env["APP"]._assets_dirs) + ) bundler.export(files_section_node.abspath) @@ -376,12 +393,14 @@ def generate_embed_app_metadata_actions(source, target, env, for_signature): objcopy_str = ( "${OBJCOPY} " "--remove-section .ARM.attributes " - "--add-section ${_FAP_META_SECTION}=${SOURCE}${_FAP_META_SECTION} " + "--add-section ${_FAP_META_SECTION}=${APP._section_fapmeta} " ) - if app.fap_file_assets: - actions.append(Action(prepare_app_files, "$APPFILE_COMSTR")) - objcopy_str += "--add-section ${_FAP_FILEASSETS_SECTION}=${SOURCE}${_FAP_FILEASSETS_SECTION} " + if app._section_fapfileassets: + actions.append(Action(prepare_app_file_assets, "$APPFILE_COMSTR")) + objcopy_str += ( + "--add-section ${_FAP_FILEASSETS_SECTION}=${APP._section_fapfileassets} " + ) objcopy_str += ( "--set-section-flags ${_FAP_META_SECTION}=contents,noload,readonly,data " @@ -470,7 +489,7 @@ def AddAppBuildTarget(env, appname, build_target_name): def generate(env, **kw): env.SetDefault( - EXT_APPS_WORK_DIR="${FBT_FAP_DEBUG_ELF_ROOT}", + EXT_APPS_WORK_DIR=env.Dir(env["FBT_FAP_DEBUG_ELF_ROOT"]), APP_RUN_SCRIPT="${FBT_SCRIPT_DIR}/runfap.py", ) if not env["VERBOSE"]: