From f8b532f06317298566b730a4c6bc17dd6ee35f4e Mon Sep 17 00:00:00 2001 From: hedger Date: Wed, 28 Sep 2022 21:13:12 +0400 Subject: [PATCH] [FL-2831] Resources cleanup in updater (#1796) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * updater: remove files from existing resources Manifest file before deploying new resources * toolbox: tar: single file extraction API Co-authored-by: あく --- .../system/updater/util/update_task.h | 2 +- .../updater/util/update_task_worker_backup.c | 41 ++++++- .../updater/util/update_task_worker_flasher.c | 2 +- firmware/targets/f7/api_symbols.csv | 1 + .../targets/f7/furi_hal/furi_hal_cortex.c | 3 + lib/toolbox/tar/tar_archive.c | 100 ++++++++------- lib/toolbox/tar/tar_archive.h | 5 + lib/update_util/resources/manifest.c | 115 ++++++++++++++++++ lib/update_util/resources/manifest.h | 58 +++++++++ 9 files changed, 281 insertions(+), 46 deletions(-) create mode 100644 lib/update_util/resources/manifest.c create mode 100644 lib/update_util/resources/manifest.h diff --git a/applications/system/updater/util/update_task.h b/applications/system/updater/util/update_task.h index 1f291556..ad8c5085 100644 --- a/applications/system/updater/util/update_task.h +++ b/applications/system/updater/util/update_task.h @@ -10,7 +10,7 @@ extern "C" { #include #include -#define UPDATE_DELAY_OPERATION_OK 300 +#define UPDATE_DELAY_OPERATION_OK 10 #define UPDATE_DELAY_OPERATION_ERROR INT_MAX typedef enum { diff --git a/applications/system/updater/util/update_task_worker_backup.c b/applications/system/updater/util/update_task_worker_backup.c index 09e45953..046be372 100644 --- a/applications/system/updater/util/update_task_worker_backup.c +++ b/applications/system/updater/util/update_task_worker_backup.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -50,10 +51,46 @@ static bool update_task_resource_unpack_cb(const char* name, bool is_directory, update_task_set_progress( unpack_progress->update_task, UpdateTaskStageProgress, - unpack_progress->processed_files * 100 / (unpack_progress->total_files + 1)); + /* For this stage, last 70% of progress = extraction */ + 30 + (unpack_progress->processed_files * 70) / (unpack_progress->total_files + 1)); return true; } +static void + update_task_cleanup_resources(UpdateTask* update_task, uint32_t n_approx_file_entries) { + ResourceManifestReader* manifest_reader = resource_manifest_reader_alloc(update_task->storage); + do { + FURI_LOG_I(TAG, "Cleaning up old manifest"); + if(!resource_manifest_reader_open(manifest_reader, EXT_PATH("Manifest"))) { + FURI_LOG_W(TAG, "No existing manifest"); + break; + } + + /* We got # of entries in TAR file. Approx 1/4th is dir entries, we skip them */ + n_approx_file_entries = n_approx_file_entries * 3 / 4 + 1; + uint32_t n_processed_files = 0; + + ResourceManifestEntry* entry_ptr = NULL; + while((entry_ptr = resource_manifest_reader_next(manifest_reader))) { + if(entry_ptr->type == ResourceManifestEntryTypeFile) { + update_task_set_progress( + update_task, + UpdateTaskStageProgress, + /* For this stage, first 30% of progress = cleanup */ + (n_processed_files++ * 30) / (n_approx_file_entries + 1)); + + string_t file_path; + string_init(file_path); + path_concat(STORAGE_EXT_PATH_PREFIX, string_get_cstr(entry_ptr->name), file_path); + FURI_LOG_D(TAG, "Removing %s", string_get_cstr(file_path)); + storage_simply_remove(update_task->storage, string_get_cstr(file_path)); + string_clear(file_path); + } + } + } while(false); + resource_manifest_reader_free(manifest_reader); +} + static bool update_task_post_update(UpdateTask* update_task) { bool success = false; @@ -88,6 +125,8 @@ static bool update_task_post_update(UpdateTask* update_task) { progress.total_files = tar_archive_get_entries_count(archive); if(progress.total_files > 0) { + update_task_cleanup_resources(update_task, progress.total_files); + CHECK_RESULT(tar_archive_unpack_to(archive, STORAGE_EXT_PATH_PREFIX, NULL)); } } diff --git a/applications/system/updater/util/update_task_worker_flasher.c b/applications/system/updater/util/update_task_worker_flasher.c index 7b598c50..b235d001 100644 --- a/applications/system/updater/util/update_task_worker_flasher.c +++ b/applications/system/updater/util/update_task_worker_flasher.c @@ -308,7 +308,7 @@ bool update_task_validate_optionbytes(UpdateTask* update_task) { } } } else { - FURI_LOG_I( + FURI_LOG_D( TAG, "OB MATCH: #%d: real %08X == %08X (exp.)", idx, diff --git a/firmware/targets/f7/api_symbols.csv b/firmware/targets/f7/api_symbols.csv index 18550984..2cbdae77 100644 --- a/firmware/targets/f7/api_symbols.csv +++ b/firmware/targets/f7/api_symbols.csv @@ -2269,6 +2269,7 @@ Function,+,tar_archive_get_entries_count,int32_t,TarArchive* Function,+,tar_archive_open,_Bool,"TarArchive*, const char*, TarOpenMode" Function,+,tar_archive_set_file_callback,void,"TarArchive*, tar_unpack_file_cb, void*" Function,+,tar_archive_store_data,_Bool,"TarArchive*, const char*, const uint8_t*, const int32_t" +Function,+,tar_archive_unpack_file,_Bool,"TarArchive*, const char*, const char*" Function,+,tar_archive_unpack_to,_Bool,"TarArchive*, const char*, Storage_name_converter" Function,-,tempnam,char*,"const char*, const char*" Function,+,text_box_alloc,TextBox*, diff --git a/firmware/targets/f7/furi_hal/furi_hal_cortex.c b/firmware/targets/f7/furi_hal/furi_hal_cortex.c index 2b4ea6e9..c9c8400a 100644 --- a/firmware/targets/f7/furi_hal/furi_hal_cortex.c +++ b/firmware/targets/f7/furi_hal/furi_hal_cortex.c @@ -6,6 +6,9 @@ void furi_hal_cortex_init_early() { CoreDebug->DEMCR |= CoreDebug_DEMCR_TRCENA_Msk; DWT->CTRL |= DWT_CTRL_CYCCNTENA_Msk; DWT->CYCCNT = 0U; + + /* Enable instruction prefetch */ + SET_BIT(FLASH->ACR, FLASH_ACR_PRFTEN); } void furi_hal_cortex_delay_us(uint32_t microseconds) { diff --git a/lib/toolbox/tar/tar_archive.c b/lib/toolbox/tar/tar_archive.c index f51d6231..1a542b2e 100644 --- a/lib/toolbox/tar/tar_archive.c +++ b/lib/toolbox/tar/tar_archive.c @@ -168,7 +168,44 @@ typedef struct { Storage_name_converter converter; } TarArchiveDirectoryOpParams; +static bool archive_extract_current_file(TarArchive* archive, const char* dst_path) { + mtar_t* tar = &archive->tar; + File* out_file = storage_file_alloc(archive->storage); + uint8_t* readbuf = malloc(FILE_BLOCK_SIZE); + + bool success = true; + uint8_t n_tries = FILE_OPEN_NTRIES; + do { + while(n_tries-- > 0) { + if(storage_file_open(out_file, dst_path, FSAM_WRITE, FSOM_CREATE_ALWAYS)) { + break; + } + FURI_LOG_W(TAG, "Failed to open '%s', reties: %d", dst_path, n_tries); + storage_file_close(out_file); + furi_delay_ms(FILE_OPEN_RETRY_DELAY); + } + + if(!storage_file_is_open(out_file)) { + success = false; + break; + } + + while(!mtar_eof_data(tar)) { + int32_t readcnt = mtar_read_data(tar, readbuf, FILE_BLOCK_SIZE); + if(!readcnt || !storage_file_write(out_file, readbuf, readcnt)) { + success = false; + break; + } + } + } while(false); + storage_file_free(out_file); + free(readbuf); + + return success; +} + static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header, void* param) { + UNUSED(tar); TarArchiveDirectoryOpParams* op_params = param; TarArchive* archive = op_params->archive; @@ -199,58 +236,22 @@ static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header, return 0; } - string_init(full_extracted_fname); + FURI_LOG_D(TAG, "Extracting %d bytes to '%s'", header->size, header->name); string_t converted_fname; string_init_set(converted_fname, header->name); if(op_params->converter) { op_params->converter(converted_fname); } + + string_init(full_extracted_fname); path_concat(op_params->work_dir, string_get_cstr(converted_fname), full_extracted_fname); + + bool success = archive_extract_current_file(archive, string_get_cstr(full_extracted_fname)); + string_clear(converted_fname); - - FURI_LOG_D(TAG, "Extracting %d bytes to '%s'", header->size, header->name); - File* out_file = storage_file_alloc(archive->storage); - uint8_t* readbuf = malloc(FILE_BLOCK_SIZE); - - bool failed = false; - uint8_t n_tries = FILE_OPEN_NTRIES; - do { - while(n_tries-- > 0) { - if(storage_file_open( - out_file, - string_get_cstr(full_extracted_fname), - FSAM_WRITE, - FSOM_CREATE_ALWAYS)) { - break; - } - FURI_LOG_W( - TAG, - "Failed to open '%s', reties: %d", - string_get_cstr(full_extracted_fname), - n_tries); - storage_file_close(out_file); - furi_delay_ms(FILE_OPEN_RETRY_DELAY); - } - - if(!storage_file_is_open(out_file)) { - failed = true; - break; - } - - while(!mtar_eof_data(tar)) { - int32_t readcnt = mtar_read_data(tar, readbuf, FILE_BLOCK_SIZE); - if(!readcnt || !storage_file_write(out_file, readbuf, readcnt)) { - failed = true; - break; - } - } - } while(false); - - storage_file_free(out_file); - free(readbuf); string_clear(full_extracted_fname); - return failed ? -1 : 0; + return success ? 0 : -1; } bool tar_archive_unpack_to( @@ -369,3 +370,16 @@ bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const ch storage_file_free(directory); return success; } + +bool tar_archive_unpack_file( + TarArchive* archive, + const char* archive_fname, + const char* destination) { + furi_assert(archive); + furi_assert(archive_fname); + furi_assert(destination); + if(mtar_find(&archive->tar, archive_fname) != MTAR_ESUCCESS) { + return false; + } + return archive_extract_current_file(archive, destination); +} \ No newline at end of file diff --git a/lib/toolbox/tar/tar_archive.h b/lib/toolbox/tar/tar_archive.h index 88cb3dd4..ceaf82ee 100644 --- a/lib/toolbox/tar/tar_archive.h +++ b/lib/toolbox/tar/tar_archive.h @@ -41,6 +41,11 @@ bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const ch int32_t tar_archive_get_entries_count(TarArchive* archive); +bool tar_archive_unpack_file( + TarArchive* archive, + const char* archive_fname, + const char* destination); + /* Optional per-entry callback on unpacking - return false to skip entry */ typedef bool (*tar_unpack_file_cb)(const char* name, bool is_directory, void* context); diff --git a/lib/update_util/resources/manifest.c b/lib/update_util/resources/manifest.c new file mode 100644 index 00000000..4f8a7d1a --- /dev/null +++ b/lib/update_util/resources/manifest.c @@ -0,0 +1,115 @@ +#include "manifest.h" + +#include +#include + +struct ResourceManifestReader { + Storage* storage; + Stream* stream; + string_t linebuf; + ResourceManifestEntry entry; +}; + +ResourceManifestReader* resource_manifest_reader_alloc(Storage* storage) { + ResourceManifestReader* resource_manifest = + (ResourceManifestReader*)malloc(sizeof(ResourceManifestReader)); + resource_manifest->storage = storage; + resource_manifest->stream = buffered_file_stream_alloc(resource_manifest->storage); + memset(&resource_manifest->entry, 0, sizeof(ResourceManifestEntry)); + string_init(resource_manifest->entry.name); + string_init(resource_manifest->linebuf); + return resource_manifest; +} + +void resource_manifest_reader_free(ResourceManifestReader* resource_manifest) { + furi_assert(resource_manifest); + + string_clear(resource_manifest->linebuf); + string_clear(resource_manifest->entry.name); + buffered_file_stream_close(resource_manifest->stream); + stream_free(resource_manifest->stream); + free(resource_manifest); +} + +bool resource_manifest_reader_open(ResourceManifestReader* resource_manifest, const char* filename) { + furi_assert(resource_manifest); + + return buffered_file_stream_open( + resource_manifest->stream, filename, FSAM_READ, FSOM_OPEN_EXISTING); +} + +/* Read entries in format of + * F::: + * D: + */ +ResourceManifestEntry* resource_manifest_reader_next(ResourceManifestReader* resource_manifest) { + furi_assert(resource_manifest); + + string_reset(resource_manifest->entry.name); + resource_manifest->entry.type = ResourceManifestEntryTypeUnknown; + resource_manifest->entry.size = 0; + memset(resource_manifest->entry.hash, 0, sizeof(resource_manifest->entry.hash)); + + do { + if(!stream_read_line(resource_manifest->stream, resource_manifest->linebuf)) { + return NULL; + } + + /* Trim end of line */ + string_strim(resource_manifest->linebuf); + + char type_code = string_get_char(resource_manifest->linebuf, 0); + switch(type_code) { + case 'F': + resource_manifest->entry.type = ResourceManifestEntryTypeFile; + break; + case 'D': + resource_manifest->entry.type = ResourceManifestEntryTypeDirectory; + break; + default: /* Skip other entries - version, timestamp, etc */ + continue; + }; + + if(resource_manifest->entry.type == ResourceManifestEntryTypeFile) { + /* Parse file entry + F::: */ + + /* Remove entry type code */ + string_right(resource_manifest->linebuf, 2); + + if(string_search_char(resource_manifest->linebuf, ':') != + sizeof(resource_manifest->entry.hash) * 2) { + /* Invalid hash */ + continue; + } + + /* Read hash */ + hex_chars_to_uint8( + string_get_cstr(resource_manifest->linebuf), resource_manifest->entry.hash); + + /* Remove hash */ + string_right( + resource_manifest->linebuf, sizeof(resource_manifest->entry.hash) * 2 + 1); + + resource_manifest->entry.size = atoi(string_get_cstr(resource_manifest->linebuf)); + + /* Remove size */ + size_t offs = string_search_char(resource_manifest->linebuf, ':'); + string_right(resource_manifest->linebuf, offs + 1); + + string_set(resource_manifest->entry.name, resource_manifest->linebuf); + } else if(resource_manifest->entry.type == ResourceManifestEntryTypeDirectory) { + /* Parse directory entry + D: */ + + /* Remove entry type code */ + string_right(resource_manifest->linebuf, 2); + + string_set(resource_manifest->entry.name, resource_manifest->linebuf); + } + + return &resource_manifest->entry; + } while(true); + + return NULL; +} diff --git a/lib/update_util/resources/manifest.h b/lib/update_util/resources/manifest.h new file mode 100644 index 00000000..092b7bad --- /dev/null +++ b/lib/update_util/resources/manifest.h @@ -0,0 +1,58 @@ +#pragma once + +#include + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + ResourceManifestEntryTypeUnknown = 0, + ResourceManifestEntryTypeDirectory, + ResourceManifestEntryTypeFile, +} ResourceManifestEntryType; + +typedef struct { + ResourceManifestEntryType type; + string_t name; + uint32_t size; + uint8_t hash[16]; +} ResourceManifestEntry; + +typedef struct ResourceManifestReader ResourceManifestReader; + +/** + * @brief Initialize resource manifest reader + * @param storage Storage API pointer + * @return allocated object + */ +ResourceManifestReader* resource_manifest_reader_alloc(Storage* storage); + +/** + * @brief Release resource manifest reader + * @param resource_manifest allocated object + */ +void resource_manifest_reader_free(ResourceManifestReader* resource_manifest); + +/** + * @brief Initialize resource manifest reader iteration + * @param resource_manifest allocated object + * @param filename manifest file name + * @return true if file opened + */ +bool resource_manifest_reader_open(ResourceManifestReader* resource_manifest, const char* filename); + +/** + * @brief Read next file/dir entry from manifest + * @param resource_manifest allocated object + * @return entry or NULL if end of file + */ +ResourceManifestEntry* resource_manifest_reader_next(ResourceManifestReader* resource_manifest); + +#ifdef __cplusplus +} // extern "C" +#endif \ No newline at end of file