mirrors/unleashed-firmware
1
Fork 0
mirror of https://github.com/DarkFlippers/unleashed-firmware.git synced 2025-12-12 04:34:43 +04:00
Code Issues Activity

Updater: resource compression (#3716)

Browse Source 
* toolbox: compress: moved decompressor implementation to separate func
* toolbox: compress: callback-based api; cli: storage unpack command
* toolbox: compress: separate r/w contexts for stream api
* targets: f18: sync API
* compress: naming fixes & cleanup
* toolbox: compress: using hs buffer size for stream buffers
* toolbox: tar: heatshrink stream mode
* toolbox: compress: docs & small cleanup
* toolbox: tar: header support for .hs; updater: now uses .hs for resources; .hs.tar: now rewindable
* toolbox: compress: fixed hs stream tail handling
* updater: reworked progress for resources cleanup; rebalanced stage weights
* updater: single-pass decompression; scripts: print resources compression ratio
* updater: fixed warnings
* toolbox: tar: doxygen
* docs: update
* docs: info or tarhs format; scripts: added standalone compression/decompression tool for heatshrink-formatted streams
* scripts: tarhs: fixed parameter handling
* cli: storage extract command; toolbox: tar: guess type based on extension
* unit_tests: added test for streamed raw hs decompressor `compress_decode_streamed`
* unit_tests: compress: added extraction test for .tar.hs
* rpc: autodetect compressed archives
* scripts: minor cleanup of common parts
* scripts: update: now using in-memory intermediate tar stream
* scripts: added hs.py wrapper for heatshrink-related ops (single object and directory-as-tar compression)
* scripts: naming fixes
* Toolbox: export compress_config_heatshrink_default as const symbol
* Toolbox: fix various types naming
* Toolbox: more of types naming fixes
* Toolbox: use size_t in compress io callbacks and structures
* UnitTests: update to match new compress API
* Toolbox: proper path_extract_extension usage

Co-authored-by: あく <alleteam@gmail.com>
This commit is contained in:
hedger
2024-06-30 13:38:48 +03:00
committed by GitHub
parent a881816673
commit fcbcb6b5a8
25 changed files with 1339 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
lib/toolbox/tar/tar_archive.c
View File

@@ -4,6 +4,7 @@
#include <storage/storage.h>
#include <furi.h>
#include <toolbox/path.h>
#include <toolbox/compress.h>
#define TAG "TarArch"
#define MAX_NAME_LEN 255
@@ -12,14 +13,29 @@
#define FILE_OPEN_NTRIES 10
#define FILE_OPEN_RETRY_DELAY 25
TarOpenMode tar_archive_get_mode_for_path(const char* path) {
char ext[8];
FuriString* path_str = furi_string_alloc_set_str(path);
path_extract_extension(path_str, ext, sizeof(ext));
furi_string_free(path_str);
if(strcmp(ext, ".ths") == 0) {
return TarOpenModeReadHeatshrink;
} else {
return TarOpenModeRead;
}
}
typedef struct TarArchive {
Storage* storage;
File* stream;
mtar_t tar;
tar_unpack_file_cb unpack_cb;
void* unpack_cb_context;
} TarArchive;
/* API WRAPPER */
/* Plain file backend - uncompressed, supports read and write */
static int mtar_storage_file_write(void* stream, const void* data, unsigned size) {
uint16_t bytes_written = storage_file_write(stream, data, size);
return (bytes_written == size) ? bytes_written : MTAR_EWRITEFAIL;
@@ -38,7 +54,6 @@ static int mtar_storage_file_seek(void* stream, unsigned offset) {
static int mtar_storage_file_close(void* stream) {
if(stream) {
storage_file_close(stream);
storage_file_free(stream);
}
return MTAR_ESUCCESS;
}
@@ -50,41 +65,133 @@ const struct mtar_ops filesystem_ops = {
.close = mtar_storage_file_close,
};
/* Heatshrink stream backend - compressed, read-only */
typedef struct {
CompressConfigHeatshrink heatshrink_config;
File* stream;
CompressStreamDecoder* decoder;
} HeatshrinkStream;
/* HSDS 'heatshrink data stream' header magic */
static const uint32_t HEATSHRINK_MAGIC = 0x53445348;
typedef struct {
uint32_t magic;
uint8_t version;
uint8_t window_sz2;
uint8_t lookahead_sz2;
} FURI_PACKED HeatshrinkStreamHeader;
_Static_assert(sizeof(HeatshrinkStreamHeader) == 7, "Invalid HeatshrinkStreamHeader size");
static int mtar_heatshrink_file_close(void* stream) {
HeatshrinkStream* hs_stream = stream;
if(hs_stream) {
if(hs_stream->decoder) {
compress_stream_decoder_free(hs_stream->decoder);
}
storage_file_close(hs_stream->stream);
storage_file_free(hs_stream->stream);
free(hs_stream);
}
return MTAR_ESUCCESS;
}
static int mtar_heatshrink_file_read(void* stream, void* data, unsigned size) {
HeatshrinkStream* hs_stream = stream;
bool read_success = compress_stream_decoder_read(hs_stream->decoder, data, size);
return read_success ? (int)size : MTAR_EREADFAIL;
}
static int mtar_heatshrink_file_seek(void* stream, unsigned offset) {
HeatshrinkStream* hs_stream = stream;
bool success = false;
if(offset == 0) {
success = storage_file_seek(hs_stream->stream, sizeof(HeatshrinkStreamHeader), true) &&
compress_stream_decoder_rewind(hs_stream->decoder);
} else {
success = compress_stream_decoder_seek(hs_stream->decoder, offset);
}
return success ? MTAR_ESUCCESS : MTAR_ESEEKFAIL;
}
const struct mtar_ops heatshrink_ops = {
.read = mtar_heatshrink_file_read,
.write = NULL, // not supported
.seek = mtar_heatshrink_file_seek,
.close = mtar_heatshrink_file_close,
};
//////////////////////////////////////////////////////////////////////////
TarArchive* tar_archive_alloc(Storage* storage) {
furi_check(storage);
TarArchive* archive = malloc(sizeof(TarArchive));
archive->storage = storage;
archive->stream = storage_file_alloc(archive->storage);
archive->unpack_cb = NULL;
return archive;
}
static int32_t file_read_cb(void* context, uint8_t* buffer, size_t buffer_size) {
File* file = context;
return storage_file_read(file, buffer, buffer_size);
}
bool tar_archive_open(TarArchive* archive, const char* path, TarOpenMode mode) {
furi_check(archive);
FS_AccessMode access_mode;
FS_OpenMode open_mode;
bool compressed = false;
int mtar_access = 0;
switch(mode) {
case TAR_OPEN_MODE_READ:
case TarOpenModeRead:
mtar_access = MTAR_READ;
access_mode = FSAM_READ;
open_mode = FSOM_OPEN_EXISTING;
break;
case TAR_OPEN_MODE_WRITE:
case TarOpenModeWrite:
mtar_access = MTAR_WRITE;
access_mode = FSAM_WRITE;
open_mode = FSOM_CREATE_ALWAYS;
break;
case TarOpenModeReadHeatshrink:
mtar_access = MTAR_READ;
access_mode = FSAM_READ;
open_mode = FSOM_OPEN_EXISTING;
compressed = true;
break;
default:
return false;
}
File* stream = storage_file_alloc(archive->storage);
File* stream = archive->stream;
if(!storage_file_open(stream, path, access_mode, open_mode)) {
storage_file_free(stream);
return false;
}
mtar_init(&archive->tar, mtar_access, &filesystem_ops, stream);
if(compressed) {
/* Read and validate stream header */
HeatshrinkStreamHeader header;
if(storage_file_read(stream, &header, sizeof(HeatshrinkStreamHeader)) !=
sizeof(HeatshrinkStreamHeader) ||
header.magic != HEATSHRINK_MAGIC) {
storage_file_close(stream);
return false;
}
HeatshrinkStream* hs_stream = malloc(sizeof(HeatshrinkStream));
hs_stream->stream = stream;
hs_stream->heatshrink_config.window_sz2 = header.window_sz2;
hs_stream->heatshrink_config.lookahead_sz2 = header.lookahead_sz2;
hs_stream->heatshrink_config.input_buffer_sz = FILE_BLOCK_SIZE;
hs_stream->decoder = compress_stream_decoder_alloc(
CompressTypeHeatshrink, &hs_stream->heatshrink_config, file_read_cb, stream);
mtar_init(&archive->tar, mtar_access, &heatshrink_ops, hs_stream);
} else {
mtar_init(&archive->tar, mtar_access, &filesystem_ops, stream);
}
return true;
}
@@ -94,6 +201,7 @@ void tar_archive_free(TarArchive* archive) {
if(mtar_is_open(&archive->tar)) {
mtar_close(&archive->tar);
}
storage_file_free(archive->stream);
free(archive);
}
@@ -121,6 +229,21 @@ int32_t tar_archive_get_entries_count(TarArchive* archive) {
return counter;
}
bool tar_archive_get_read_progress(TarArchive* archive, int32_t* processed, int32_t* total) {
furi_check(archive);
if(mtar_access_mode(&archive->tar) != MTAR_READ) {
return false;
}
if(processed) {
*processed = storage_file_tell(archive->stream);
}
if(total) {
*total = storage_file_size(archive->stream);
}
return true;
}
bool tar_archive_dir_add_element(TarArchive* archive, const char* dirpath) {
furi_check(archive);
return (mtar_write_dir_header(&archive->tar, dirpath) == MTAR_ESUCCESS);
@@ -258,7 +381,7 @@ static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header,
furi_string_free(converted_fname);
furi_string_free(full_extracted_fname);
return success ? 0 : -1;
return success ? 0 : MTAR_EFAILURE;
}
bool tar_archive_unpack_to(

143
lib/toolbox/tar/tar_archive.h
View File

@@ -12,62 +12,197 @@ typedef struct TarArchive TarArchive;
typedef struct Storage Storage;
/** Tar archive open mode
*/
typedef enum {
TAR_OPEN_MODE_READ = 'r',
TAR_OPEN_MODE_WRITE = 'w',
TAR_OPEN_MODE_STDOUT = 's' /* to be implemented */
TarOpenModeRead = 'r',
TarOpenModeWrite = 'w',
/* read-only heatshrink compressed tar */
TarOpenModeReadHeatshrink = 'h',
} TarOpenMode;
/** Get expected open mode for archive at the path.
* Used for automatic mode detection based on the file extension.
*
* @param[in] path Path to the archive
*
* @return open mode from TarOpenMode enum
*/
TarOpenMode tar_archive_get_mode_for_path(const char* path);
/** Tar archive constructor
*
* @param storage Storage API pointer
*
* @return allocated object
*/
TarArchive* tar_archive_alloc(Storage* storage);
/** Open tar archive
*
* @param archive Tar archive object
* @param[in] path Path to the tar archive
* @param mode Open mode
*
* @return true if successful
*/
bool tar_archive_open(TarArchive* archive, const char* path, TarOpenMode mode);
/** Tar archive destructor
*
* @param archive Tar archive object
*/
void tar_archive_free(TarArchive* archive);
/* High-level API - assumes archive is open */
/** Unpack tar archive to destination
*
* @param archive Tar archive object. Must be opened in read mode
* @param[in] destination Destination path
* @param converter Storage name converter
*
* @return true if successful
*/
bool tar_archive_unpack_to(
TarArchive* archive,
const char* destination,
Storage_name_converter converter);
/** Add file to tar archive
*
* @param archive Tar archive object. Must be opened in write mode
* @param[in] fs_file_path Path to the file on the filesystem
* @param[in] archive_fname Name of the file in the archive
* @param file_size Size of the file
*
* @return true if successful
*/
bool tar_archive_add_file(
TarArchive* archive,
const char* fs_file_path,
const char* archive_fname,
const int32_t file_size);
/** Add directory to tar archive
*
* @param archive Tar archive object. Must be opened in write mode
* @param fs_full_path Path to the directory on the filesystem
* @param path_prefix Prefix to add to the directory name in the archive
*
* @return true if successful
*/
bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const char* path_prefix);
/** Get number of entries in the archive
*
* @param archive Tar archive object
*
* @return number of entries. -1 on error
*/
int32_t tar_archive_get_entries_count(TarArchive* archive);
/** Get read progress
*
* @param archive Tar archive object. Must be opened in read mode
* @param[in] processed Number of processed entries
* @param[in] total Total number of entries
*
* @return true if successful
*/
bool tar_archive_get_read_progress(TarArchive* archive, int32_t* processed, int32_t* total);
/** Unpack single file from tar archive
*
* @param archive Tar archive object. Must be opened in read mode
* @param[in] archive_fname Name of the file in the archive
* @param[in] destination Destination path
*
* @return true if successful
*/
bool tar_archive_unpack_file(
TarArchive* archive,
const char* archive_fname,
const char* destination);
/* Optional per-entry callback on unpacking - return false to skip entry */
/** Optional per-entry callback on unpacking
* @param name Name of the file or directory
* @param is_directory True if the entry is a directory
* @param[in] context User context
* @return true to process the entry, false to skip
*/
typedef bool (*tar_unpack_file_cb)(const char* name, bool is_directory, void* context);
/** Set per-entry callback on unpacking
* @param archive Tar archive object
* @param callback Callback function
* @param[in] context User context
*/
void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callback, void* context);
/* Low-level API */
/** Add tar archive directory header
*
* @param archive Tar archive object. Must be opened in write mode
* @param[in] dirpath Path to the directory
*
* @return true if successful
*/
bool tar_archive_dir_add_element(TarArchive* archive, const char* dirpath);
/** Add tar archive file header
*
* @param archive Tar archive object. Must be opened in write mode
* @param[in] path Path to the file
* @param data_len Size of the file
*
* @return true if successful
*/
bool tar_archive_file_add_header(TarArchive* archive, const char* path, const int32_t data_len);
/** Add tar archive file data block
*
* @param archive Tar archive object. Must be opened in write mode
* @param[in] data_block Data block
* @param block_len Size of the data block
*
* @return true if successful
*/
bool tar_archive_file_add_data_block(
TarArchive* archive,
const uint8_t* data_block,
const int32_t block_len);
/** Finalize tar archive file
*
* @param archive Tar archive object. Must be opened in write mode
*
* @return true if successful
*/
bool tar_archive_file_finalize(TarArchive* archive);
/** Store data in tar archive
*
* @param archive Tar archive object. Must be opened in write mode
* @param[in] path Path to the file
* @param[in] data Data to store
* @param data_len Size of the data
*
* @return true if successful
*/
bool tar_archive_store_data(
TarArchive* archive,
const char* path,
const uint8_t* data,
const int32_t data_len);
/** Finalize tar archive
*
* @param archive Tar archive object. Must be opened in write mode
*
* @return true if successful
*/
bool tar_archive_finalize(TarArchive* archive);
#ifdef __cplusplus