mirrors/flipperzero-firmware
1
Fork 0
mirror of https://github.com/flipperdevices/flipperzero-firmware.git synced 2025-12-12 04:41:26 +04:00
Code Issues Activity

Updater: resource compression (#3716)

Browse Source 
* toolbox: compress: moved decompressor implementation to separate func
* toolbox: compress: callback-based api; cli: storage unpack command
* toolbox: compress: separate r/w contexts for stream api
* targets: f18: sync API
* compress: naming fixes & cleanup
* toolbox: compress: using hs buffer size for stream buffers
* toolbox: tar: heatshrink stream mode
* toolbox: compress: docs & small cleanup
* toolbox: tar: header support for .hs; updater: now uses .hs for resources; .hs.tar: now rewindable
* toolbox: compress: fixed hs stream tail handling
* updater: reworked progress for resources cleanup; rebalanced stage weights
* updater: single-pass decompression; scripts: print resources compression ratio
* updater: fixed warnings
* toolbox: tar: doxygen
* docs: update
* docs: info or tarhs format; scripts: added standalone compression/decompression tool for heatshrink-formatted streams
* scripts: tarhs: fixed parameter handling
* cli: storage extract command; toolbox: tar: guess type based on extension
* unit_tests: added test for streamed raw hs decompressor `compress_decode_streamed`
* unit_tests: compress: added extraction test for .tar.hs
* rpc: autodetect compressed archives
* scripts: minor cleanup of common parts
* scripts: update: now using in-memory intermediate tar stream
* scripts: added hs.py wrapper for heatshrink-related ops (single object and directory-as-tar compression)
* scripts: naming fixes
* Toolbox: export compress_config_heatshrink_default as const symbol
* Toolbox: fix various types naming
* Toolbox: more of types naming fixes
* Toolbox: use size_t in compress io callbacks and structures
* UnitTests: update to match new compress API
* Toolbox: proper path_extract_extension usage

Co-authored-by: あく <alleteam@gmail.com>
This commit is contained in:
hedger
2024-06-30 13:38:48 +03:00
committed by GitHub
parent a881816673
commit fcbcb6b5a8
25 changed files with 1339 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
applications/system/updater/util/update_task.c
View File

@@ -9,6 +9,8 @@
#include <update_util/lfs_backup.h>
#include <update_util/update_operation.h>
#define TAG "UpdWorker"
static const char* update_task_stage_descr[] = {
[UpdateTaskStageProgress] = "...",
[UpdateTaskStageReadManifest] = "Loading update manifest",
@@ -23,7 +25,9 @@ static const char* update_task_stage_descr[] = {
[UpdateTaskStageOBValidation] = "Validating opt. bytes",
[UpdateTaskStageLfsBackup] = "Backing up LFS",
[UpdateTaskStageLfsRestore] = "Restoring LFS",
[UpdateTaskStageResourcesUpdate] = "Updating resources",
[UpdateTaskStageResourcesFileCleanup] = "Cleaning up files",
[UpdateTaskStageResourcesDirCleanup] = "Cleaning up directories",
[UpdateTaskStageResourcesFileUnpack] = "Extracting resources",
[UpdateTaskStageSplashscreenInstall] = "Installing splashscreen",
[UpdateTaskStageCompleted] = "Restarting...",
[UpdateTaskStageError] = "Error",
@@ -196,7 +200,19 @@ static const struct {
.descr = "LFS I/O error",
},
{
.stage = UpdateTaskStageResourcesUpdate,
.stage = UpdateTaskStageResourcesFileCleanup,
.percent_min = 0,
.percent_max = 100,
.descr = "SD card I/O error",
},
{
.stage = UpdateTaskStageResourcesDirCleanup,
.percent_min = 0,
.percent_max = 100,
.descr = "SD card I/O error",
},
{
.stage = UpdateTaskStageResourcesFileUnpack,
.percent_min = 0,
.percent_max = 100,
.descr = "SD card I/O error",
@@ -230,20 +246,22 @@ static const UpdateTaskStageGroupMap update_task_stage_progress[] = {
[UpdateTaskStageLfsBackup] = STAGE_DEF(UpdateTaskStageGroupPreUpdate, 5),
[UpdateTaskStageRadioImageValidate] = STAGE_DEF(UpdateTaskStageGroupRadio, 15),
[UpdateTaskStageRadioErase] = STAGE_DEF(UpdateTaskStageGroupRadio, 35),
[UpdateTaskStageRadioWrite] = STAGE_DEF(UpdateTaskStageGroupRadio, 60),
[UpdateTaskStageRadioErase] = STAGE_DEF(UpdateTaskStageGroupRadio, 25),
[UpdateTaskStageRadioWrite] = STAGE_DEF(UpdateTaskStageGroupRadio, 40),
[UpdateTaskStageRadioInstall] = STAGE_DEF(UpdateTaskStageGroupRadio, 30),
[UpdateTaskStageRadioBusy] = STAGE_DEF(UpdateTaskStageGroupRadio, 5),
[UpdateTaskStageOBValidation] = STAGE_DEF(UpdateTaskStageGroupOptionBytes, 2),
[UpdateTaskStageValidateDFUImage] = STAGE_DEF(UpdateTaskStageGroupFirmware, 30),
[UpdateTaskStageFlashWrite] = STAGE_DEF(UpdateTaskStageGroupFirmware, 150),
[UpdateTaskStageFlashValidate] = STAGE_DEF(UpdateTaskStageGroupFirmware, 15),
[UpdateTaskStageValidateDFUImage] = STAGE_DEF(UpdateTaskStageGroupFirmware, 33),
[UpdateTaskStageFlashWrite] = STAGE_DEF(UpdateTaskStageGroupFirmware, 100),
[UpdateTaskStageFlashValidate] = STAGE_DEF(UpdateTaskStageGroupFirmware, 20),
[UpdateTaskStageLfsRestore] = STAGE_DEF(UpdateTaskStageGroupPostUpdate, 5),
[UpdateTaskStageResourcesUpdate] = STAGE_DEF(UpdateTaskStageGroupResources, 255),
[UpdateTaskStageResourcesFileCleanup] = STAGE_DEF(UpdateTaskStageGroupResources, 100),
[UpdateTaskStageResourcesDirCleanup] = STAGE_DEF(UpdateTaskStageGroupResources, 50),
[UpdateTaskStageResourcesFileUnpack] = STAGE_DEF(UpdateTaskStageGroupResources, 255),
[UpdateTaskStageSplashscreenInstall] = STAGE_DEF(UpdateTaskStageGroupSplashscreen, 5),
[UpdateTaskStageCompleted] = STAGE_DEF(UpdateTaskStageGroupMisc, 1),
@@ -288,6 +306,7 @@ static void update_task_calc_completed_stages(UpdateTask* update_task) {
void update_task_set_progress(UpdateTask* update_task, UpdateTaskStage stage, uint8_t progress) {
if(stage != UpdateTaskStageProgress) {
FURI_LOG_I(TAG, "Stage %d, progress %d", stage, progress);
/* do not override more specific error states */
if((stage >= UpdateTaskStageError) && (update_task->state.stage >= UpdateTaskStageError)) {
return;

4
applications/system/updater/util/update_task.h
View File

@@ -31,7 +31,9 @@ typedef enum {
UpdateTaskStageFlashValidate,
UpdateTaskStageLfsRestore,
UpdateTaskStageResourcesUpdate,
UpdateTaskStageResourcesFileCleanup,
UpdateTaskStageResourcesDirCleanup,
UpdateTaskStageResourcesFileUnpack,
UpdateTaskStageSplashscreenInstall,
UpdateTaskStageCompleted,

73
applications/system/updater/util/update_task_worker_backup.c
View File

@@ -35,36 +35,23 @@ static bool update_task_pre_update(UpdateTask* update_task) {
furi_string_free(backup_file_path);
return success;
}
typedef enum {
UpdateTaskResourcesWeightsFileCleanup = 20,
UpdateTaskResourcesWeightsDirCleanup = 20,
UpdateTaskResourcesWeightsFileUnpack = 60,
} UpdateTaskResourcesWeights;
#define UPDATE_TASK_RESOURCES_FILE_TO_TOTAL_PERCENT 90
typedef struct {
UpdateTask* update_task;
int32_t total_files, processed_files;
TarArchive* archive;
} TarUnpackProgress;
static bool update_task_resource_unpack_cb(const char* name, bool is_directory, void* context) {
UNUSED(name);
UNUSED(is_directory);
TarUnpackProgress* unpack_progress = context;
unpack_progress->processed_files++;
int32_t progress = 0, total = 0;
tar_archive_get_read_progress(unpack_progress->archive, &progress, &total);
update_task_set_progress(
unpack_progress->update_task,
UpdateTaskStageProgress,
/* For this stage, last progress segment = extraction */
(UpdateTaskResourcesWeightsFileCleanup + UpdateTaskResourcesWeightsDirCleanup) +
(unpack_progress->processed_files * UpdateTaskResourcesWeightsFileUnpack) /
(unpack_progress->total_files + 1));
unpack_progress->update_task, UpdateTaskStageProgress, (progress * 100) / (total + 1));
return true;
}
static void update_task_cleanup_resources(UpdateTask* update_task, const uint32_t n_tar_entries) {
static void update_task_cleanup_resources(UpdateTask* update_task) {
ResourceManifestReader* manifest_reader = resource_manifest_reader_alloc(update_task->storage);
do {
FURI_LOG_D(TAG, "Cleaning up old manifest");
@@ -73,20 +60,26 @@ static void update_task_cleanup_resources(UpdateTask* update_task, const uint32_
break;
}
const uint32_t n_approx_file_entries =
n_tar_entries * UPDATE_TASK_RESOURCES_FILE_TO_TOTAL_PERCENT / 100 + 1;
uint32_t n_dir_entries = 1;
ResourceManifestEntry* entry_ptr = NULL;
uint32_t n_processed_entries = 0;
/* Iterate over manifest and calculate entries count */
uint32_t n_file_entries = 1, n_dir_entries = 1;
while((entry_ptr = resource_manifest_reader_next(manifest_reader))) {
if(entry_ptr->type == ResourceManifestEntryTypeFile) {
n_file_entries++;
} else if(entry_ptr->type == ResourceManifestEntryTypeDirectory) {
n_dir_entries++;
}
}
resource_manifest_rewind(manifest_reader);
update_task_set_progress(update_task, UpdateTaskStageResourcesFileCleanup, 0);
uint32_t n_processed_file_entries = 0;
while((entry_ptr = resource_manifest_reader_next(manifest_reader))) {
if(entry_ptr->type == ResourceManifestEntryTypeFile) {
update_task_set_progress(
update_task,
UpdateTaskStageProgress,
/* For this stage, first pass = old manifest's file cleanup */
(n_processed_entries++ * UpdateTaskResourcesWeightsFileCleanup) /
n_approx_file_entries);
(n_processed_file_entries++ * 100) / n_file_entries);
FuriString* file_path = furi_string_alloc();
path_concat(
@@ -108,16 +101,14 @@ static void update_task_cleanup_resources(UpdateTask* update_task, const uint32_
}
}
n_processed_entries = 0;
update_task_set_progress(update_task, UpdateTaskStageResourcesDirCleanup, 0);
uint32_t n_processed_dir_entries = 0;
while((entry_ptr = resource_manifest_reader_previous(manifest_reader))) {
if(entry_ptr->type == ResourceManifestEntryTypeDirectory) {
update_task_set_progress(
update_task,
UpdateTaskStageProgress,
/* For this stage, second 10% of progress = cleanup directories */
UpdateTaskResourcesWeightsFileCleanup +
(n_processed_entries++ * UpdateTaskResourcesWeightsDirCleanup) /
n_dir_entries);
(n_processed_dir_entries++ * 100) / n_dir_entries);
FuriString* folder_path = furi_string_alloc();
@@ -166,26 +157,22 @@ static bool update_task_post_update(UpdateTask* update_task) {
if(update_task->state.groups & UpdateTaskStageGroupResources) {
TarUnpackProgress progress = {
.update_task = update_task,
.total_files = 0,
.processed_files = 0,
.archive = archive,
};
update_task_set_progress(update_task, UpdateTaskStageResourcesUpdate, 0);
path_concat(
furi_string_get_cstr(update_task->update_path),
furi_string_get_cstr(update_task->manifest->resource_bundle),
file_path);
CHECK_RESULT(tar_archive_open(
archive, furi_string_get_cstr(file_path), TarOpenModeReadHeatshrink));
update_task_cleanup_resources(update_task);
update_task_set_progress(update_task, UpdateTaskStageResourcesFileUnpack, 0);
tar_archive_set_file_callback(archive, update_task_resource_unpack_cb, &progress);
CHECK_RESULT(
tar_archive_open(archive, furi_string_get_cstr(file_path), TAR_OPEN_MODE_READ));
progress.total_files = tar_archive_get_entries_count(archive);
if(progress.total_files > 0) {
update_task_cleanup_resources(update_task, progress.total_files);
CHECK_RESULT(tar_archive_unpack_to(archive, STORAGE_EXT_PATH_PREFIX, NULL));
}
CHECK_RESULT(tar_archive_unpack_to(archive, STORAGE_EXT_PATH_PREFIX, NULL));
}
if(update_task->state.groups & UpdateTaskStageGroupSplashscreen) {