From 2748b383bd685421c42f05d33e6fbdf7cb3f7340 Mon Sep 17 00:00:00 2001 From: Songlx516 Date: Sat, 12 Sep 2026 17:46:32 +0800 Subject: [PATCH] feat(7z): decode folders with our own folder header parser and coder chain The LZMA SDK's CSzFolder caps a folder at four coders, so anything 7-Zip writes for BCJ2 (4 packed streams + 5 coders) comes back as SZ_ERROR_UNSUPPORTED from SzAr_DecodeFolder. The header scanner has no such limit (64 coders), which makes the failure look like corrupt data instead. src/sevenz_chain.c parses the folder descriptor itself and drives the codec chain as a pull pipeline: every decoder hands up to N bytes to its consumer and asks upstream for more when it needs it, so a folder is decoded straight into the caller's sink without staging the whole thing. Covered here: - Copy / LZMA / LZMA2 / PPMd, Delta, and the x86, PPC, IA64, ARM, ARMT, SPARC branch converters - BCJ2, whose three side streams have to be materialised while MAIN keeps streaming - per coder output sizes taken from UNPACK_INFO, which is what keeps the chains exact tests/sevenz_chain_e2e.c decodes each folder once and splits the byte stream across the entries that share it, mirroring the real extraction path, and checks the per-entry and per-folder CRCs on the way past. tests/make_sevenz_fixtures.py builds the fixture matrix (including non-solid `-ms=off` folders), and tests/run-sevenz-tests.sh runs it while keeping a KNOWN_GAPS list that fails loudly once a gap closes. Two real bugs surfaced while bringing the matrix up: - build_coder() gave every coder node the *folder's* unpack size instead of its own entry in UNPACK_INFO. In a BCJ2 folder MAIN is regularly larger than the folder's final size (300066 vs 300000 in the fixture), so the node silently truncated its own output and the chain came up short by 21 bytes per LZMA2 layer. Only archives where a coder is bigger than the folder tripped it, which is why single-folder BCJ2 passed and the same archive split into per-file folders did not. - the e2e driver kept `written` across folders, so a folder that failed mid-entry made every later entry look half-written and the failures cascaded. Scripts no longer delete anything: objects are overwritten and each case gets a fresh output directory, so the suite runs under a guarded shell. Matrix: 10/10 single-volume fixtures byte-identical (store, lzma2, lzma, ppmd, bcj, delta, utf8, bcj2, solidoff, bcj2off). Still open and listed as known gaps: 7zAES (aes, aeshe) and multi-volume input (vol.7z.001). --- src/sevenz_chain.c | 1600 +++++++++++++++++++++++++++++++++ src/sevenz_chain.h | 157 ++++ tests/make_sevenz_fixtures.py | 5 + tests/run-sevenz-tests.sh | 45 +- tests/sevenz_chain_e2e.c | 475 ++++++++++ 5 files changed, 2266 insertions(+), 16 deletions(-) create mode 100644 src/sevenz_chain.c create mode 100644 src/sevenz_chain.h create mode 100644 tests/sevenz_chain_e2e.c diff --git a/src/sevenz_chain.c b/src/sevenz_chain.c new file mode 100644 index 0000000..916cae2 --- /dev/null +++ b/src/sevenz_chain.c @@ -0,0 +1,1600 @@ +/* sevenz_chain -- 7z folder (coder chain) decoder. + part of ps5-web-file-manager + + See sevenz_chain.h for the rationale. Implementation notes: + + Header parsing follows SzGetNextFolderItem() from the LZMA SDK byte for + byte, only with dynamic arrays instead of the fixed CSzFolder. The coder + graph is then driven as a pull pipeline: every coder answers "give me up to + N bytes of your output", pulling from its own upstream. Nothing is decoded + ahead of what the sink asked for, so a solid folder streams. + + Filters are applied chunk by chunk. The 7z branch converters are documented + to be restartable -- they return the position after the last byte they could + fully process and leave at most LookAhead bytes for the next call -- so the + unconverted tail is carried across chunk boundaries while the program + counter keeps advancing. That makes a filtered chain stream as well as a + plain one. + + BCJ2 is the one shape that cannot stream end to end: it needs its CALL, + JUMP and RC inputs available at arbitrary points, so those three are + materialised (capped by sz_chain_limits_t::max_side_bytes) while the MAIN + stream keeps streaming through. + + Every rejection names the coder and the method, and every resource limit + reports what the archive asked for and what was allowed. +*/ + +#include "sevenz_chain.h" + +#include +#include +#include +#include + +#include "7zTypes.h" +#include "7zAlloc.h" +#include "7zCrc.h" +#include "LzmaDec.h" +#include "Lzma2Dec.h" +#include "Ppmd7.h" +#include "Delta.h" +#include "Bra.h" +#include "Bcj2.h" + +/* ------------------------------------------------------------------ ids */ + +#define SZ_M_COPY 0x00u +#define SZ_M_DELTA 0x03u +#define SZ_M_LZMA2 0x21u +#define SZ_M_LZMA 0x030101u +#define SZ_M_PPMD 0x030401u +#define SZ_M_BCJ 0x03030103u +#define SZ_M_PPC 0x03030205u +#define SZ_M_IA64 0x03030401u +#define SZ_M_ARM 0x03030501u +#define SZ_M_ARMT 0x03030701u +#define SZ_M_SPARC 0x03030805u +#define SZ_M_BCJ2 0x0303011Bu + +enum { + SZ_N_PACK = 0, + SZ_N_COPY, + SZ_N_LZMA, + SZ_N_LZMA2, + SZ_N_PPMD, + SZ_N_FILTER, + SZ_N_BCJ2 +}; + +enum { + SZ_F_DELTA = 1, + SZ_F_X86, + SZ_F_PPC, + SZ_F_IA64, + SZ_F_SPARC, + SZ_F_ARM, + SZ_F_ARMT +}; + +/* Pull granularity. 7z stores compressed data in LZMA2 chunks of at most + 64 KiB plus a 6 byte header, so a 256 KiB staging buffer always holds a + complete chunk and the decoder can never stall for lack of contiguity. */ +#define SZ_IN_CHUNK (1u << 18) +#define SZ_FILT_CHUNK (1u << 17) +#define SZ_BCJ2_MAIN_CHUNK (1u << 17) +#define SZ_OUT_CHUNK (1u << 16) +/* Branch converters keep a few tail bytes for their next call. */ +#define SZ_FILT_LOOKAHEAD 8 +/* Refill BCJ2's MAIN stream while this few bytes are left. */ +#define SZ_BCJ2_KEEP 8 + +static const ISzAlloc g_alloc = { SzAlloc, SzFree }; + +/* ------------------------------------------------------------ utilities */ + +static void err_set(sz_chain_err_t *err, sz_chain_status_t status, + int32_t coder, uint32_t method, uint64_t offset, + const char *fmt, ...) { + va_list ap; + + if(!err) return; + err->status = status; + err->coder = coder; + err->method = method; + err->offset = offset; + if(fmt) { + va_start(ap, fmt); + vsnprintf(err->message, sizeof(err->message), fmt, ap); + va_end(ap); + } else { + err->message[0] = 0; + } +} + +const char *sz_chain_status_string(sz_chain_status_t status) { + switch(status) { + case SZ_CHAIN_OK: return "ok"; + case SZ_CHAIN_ERR_PARAM: return "invalid argument"; + case SZ_CHAIN_ERR_MEM: return "out of memory"; + case SZ_CHAIN_ERR_HEADER: return "corrupt 7z folder header"; + case SZ_CHAIN_ERR_METHOD: return "unsupported compression method"; + case SZ_CHAIN_ERR_LAYOUT: return "unsupported coder chain"; + case SZ_CHAIN_ERR_LIMIT: return "resource limit exceeded"; + case SZ_CHAIN_ERR_READ: return "read failed"; + case SZ_CHAIN_ERR_WRITE: return "write failed"; + case SZ_CHAIN_ERR_DATA: return "corrupt compressed data"; + case SZ_CHAIN_ERR_CANCELED: return "canceled"; + case SZ_CHAIN_ERR_INTERNAL: return "internal error"; + } + return "unknown error"; +} + +const char *sz_chain_method_name(uint32_t method) { + switch(method) { + case SZ_M_COPY: return "Copy"; + case SZ_M_DELTA: return "Delta"; + case SZ_M_LZMA2: return "LZMA2"; + case SZ_M_LZMA: return "LZMA"; + case SZ_M_PPMD: return "PPMd"; + case SZ_M_BCJ: return "BCJ"; + case SZ_M_PPC: return "PPC"; + case SZ_M_IA64: return "IA64"; + case SZ_M_ARM: return "ARM"; + case SZ_M_ARMT: return "ARMT"; + case SZ_M_SPARC: return "SPARC"; + case SZ_M_BCJ2: return "BCJ2"; + case 0x06F10701u: return "7zAES"; + default: return NULL; + } +} + +static const char *method_label(uint32_t method, char *scratch, + size_t scratch_size) { + const char *name = sz_chain_method_name(method); + if(name) return name; + snprintf(scratch, scratch_size, "unknown 0x%X", (unsigned)method); + return scratch; +} + +/* -------------------------------------------------------------- limits */ + +static const sz_chain_limits_t g_limits_default = { + (uint64_t)512 << 20, /* LZMA / LZMA2 dictionary */ + (uint64_t)256 << 20, /* PPMd model */ + (uint64_t)256 << 20 /* BCJ2 side streams */ +}; + +static const sz_chain_limits_t g_limits_large = { + (uint64_t)1536 << 20, + (uint64_t)1024 << 20, + (uint64_t)1024 << 20 +}; + +const sz_chain_limits_t *sz_chain_limits_profile(int profile) { + return profile == SZ_CHAIN_LIMITS_LARGE ? &g_limits_large + : &g_limits_default; +} + +const sz_chain_limits_t *sz_chain_default_limits(void) { + return &g_limits_default; +} + +/* ------------------------------------------------------- reader helpers */ + +typedef struct { + const uint8_t *p; + const uint8_t *end; +} sz_rdr; + +static int rdr_byte(sz_rdr *r, uint8_t *out) { + if(r->p >= r->end) return -1; + *out = *r->p++; + return 0; +} + +/* 7z variable length number: a unary length prefix in the top bits of the + first byte, then little endian payload bytes. Mirrors ReadNumber(). */ +static int rdr_num(sz_rdr *r, uint64_t *out) { + uint8_t first, tmp, mask; + unsigned i; + uint64_t value; + + if(rdr_byte(r, &first) != 0) return -1; + if((first & 0x80) == 0) { + *out = first; + return 0; + } + if(rdr_byte(r, &tmp) != 0) return -1; + if((first & 0x40) == 0) { + *out = ((uint64_t)(first & 0x3F) << 8) | tmp; + return 0; + } + if(rdr_byte(r, &mask) != 0) return -1; + value = (uint64_t)tmp | ((uint64_t)mask << 8); + mask = 0x20; + for(i = 2 * 8; i < 8 * 8; i += 8) { + uint8_t b; + if((first & mask) == 0) { + value |= ((uint64_t)(first & (uint8_t)(mask - 1))) << i; + break; + } + mask >>= 1; + if(rdr_byte(r, &b) != 0) return -1; + value |= ((uint64_t)b << i); + } + *out = value; + return 0; +} + +static int rdr_num32(sz_rdr *r, uint32_t *out) { + uint64_t v; + if(rdr_num(r, &v) != 0) return -1; + if(v > 0x7FFFFFFFu) return -1; + *out = (uint32_t)v; + return 0; +} + +/* --------------------------------------------------------------- folder */ + +typedef struct { + uint32_t method; + uint32_t num_in_streams; + uint32_t first_in_stream; + uint32_t props_off; + uint32_t props_size; +} sz_coder; + +typedef struct { + uint32_t in_index; + uint32_t out_coder; +} sz_bond; + +typedef struct { + int32_t from_coder; /* >= 0: output of that coder, -1: packed stream */ + uint32_t pack_index; +} sz_source; + +struct sz_chain { + uint8_t *blob; + size_t blob_size; + uint32_t num_coders; + uint32_t num_bonds; + uint32_t num_pack_streams; + uint32_t unpack_coder; + uint32_t total_streams; + uint64_t unpack_size; + sz_chain_limits_t limits; + uint64_t pack_positions[SZ_CHAIN_MAX_STREAMS + 1]; + uint32_t pack_stream_idx[SZ_CHAIN_MAX_STREAMS]; + uint64_t coder_unpack_sizes[SZ_CHAIN_MAX_CODERS]; + sz_coder coder[SZ_CHAIN_MAX_CODERS]; + sz_bond bond[SZ_CHAIN_MAX_CODERS]; + sz_source source[SZ_CHAIN_MAX_STREAMS]; +}; + +static int method_kind(uint32_t method) { + switch(method) { + case SZ_M_COPY: return SZ_N_COPY; + case SZ_M_LZMA: return SZ_N_LZMA; + case SZ_M_LZMA2: return SZ_N_LZMA2; + case SZ_M_PPMD: return SZ_N_PPMD; + case SZ_M_BCJ2: return SZ_N_BCJ2; + case SZ_M_DELTA: + case SZ_M_BCJ: + case SZ_M_PPC: + case SZ_M_IA64: + case SZ_M_SPARC: + case SZ_M_ARM: + case SZ_M_ARMT: return SZ_N_FILTER; + default: return -1; + } +} + +static int filter_kind(uint32_t method) { + switch(method) { + case SZ_M_DELTA: return SZ_F_DELTA; + case SZ_M_BCJ: return SZ_F_X86; + case SZ_M_PPC: return SZ_F_PPC; + case SZ_M_IA64: return SZ_F_IA64; + case SZ_M_SPARC: return SZ_F_SPARC; + case SZ_M_ARM: return SZ_F_ARM; + case SZ_M_ARMT: return SZ_F_ARMT; + default: return 0; + } +} + +int sz_chain_parse(sz_chain **out, const uint8_t *blob, size_t blob_size, + const uint64_t *pack_positions, uint32_t num_pack_streams, + const uint64_t *coder_unpack_sizes, uint64_t unpack_size, + const sz_chain_limits_t *limits, sz_chain_err_t *err) { + sz_chain *c; + sz_rdr r; + uint32_t i, j, running_in = 0; + uint32_t num_unused; + + if(!out || !blob || !pack_positions || !coder_unpack_sizes) { + err_set(err, SZ_CHAIN_ERR_PARAM, -1, 0, 0, "missing folder metadata"); + return -1; + } + *out = NULL; + if(num_pack_streams > SZ_CHAIN_MAX_STREAMS) { + err_set(err, SZ_CHAIN_ERR_LIMIT, -1, 0, 0, + "folder declares %u packed streams, at most %u are supported", + (unsigned)num_pack_streams, (unsigned)SZ_CHAIN_MAX_STREAMS); + return -1; + } + + c = (sz_chain *)calloc(1, sizeof(*c)); + if(!c) { + err_set(err, SZ_CHAIN_ERR_MEM, -1, 0, 0, "cannot allocate folder state"); + return -1; + } + c->limits = limits ? *limits : g_limits_default; + c->unpack_size = unpack_size; + c->num_pack_streams = num_pack_streams; + c->blob = (uint8_t *)malloc(blob_size ? blob_size : 1); + if(!c->blob) { + free(c); + err_set(err, SZ_CHAIN_ERR_MEM, -1, 0, 0, "cannot copy the folder header"); + return -1; + } + memcpy(c->blob, blob, blob_size); + c->blob_size = blob_size; + for(i = 0; i <= num_pack_streams; i++) + c->pack_positions[i] = pack_positions[i]; + + r.p = c->blob; + r.end = c->blob + c->blob_size; + + if(rdr_num32(&r, &c->num_coders) != 0 || c->num_coders == 0) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, + "folder header does not start with a coder count"); + goto bad; + } + if(c->num_coders > SZ_CHAIN_MAX_CODERS) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, + "folder has %u coders, at most %u are supported", + (unsigned)c->num_coders, (unsigned)SZ_CHAIN_MAX_CODERS); + goto bad; + } + + for(i = 0; i < c->num_coders; i++) { + sz_coder *cd = &c->coder[i]; + uint8_t main_byte; + uint32_t id_size, k; + uint64_t id = 0; + + if(rdr_byte(&r, &main_byte) != 0) { + err_set(err, SZ_CHAIN_ERR_HEADER, (int32_t)i, 0, 0, + "truncated folder header at coder %u", (unsigned)i); + goto bad; + } + if(main_byte & 0xC0) { + err_set(err, SZ_CHAIN_ERR_HEADER, (int32_t)i, 0, 0, + "coder %u uses reserved flag bits", (unsigned)i); + goto bad; + } + id_size = main_byte & 0x0F; + if(id_size > 4u) { + err_set(err, SZ_CHAIN_ERR_HEADER, (int32_t)i, 0, 0, + "coder %u has a %u byte method id", (unsigned)i, + (unsigned)id_size); + goto bad; + } + for(k = 0; k < id_size; k++) { + uint8_t b; + if(rdr_byte(&r, &b) != 0) { + err_set(err, SZ_CHAIN_ERR_HEADER, (int32_t)i, 0, 0, + "truncated method id for coder %u", (unsigned)i); + goto bad; + } + id = (id << 8) | b; + } + cd->method = (uint32_t)id; + cd->num_in_streams = 1; + + if(main_byte & 0x10) { + uint32_t in_streams, out_streams; + if(rdr_num32(&r, &in_streams) != 0 || rdr_num32(&r, &out_streams) != 0) { + err_set(err, SZ_CHAIN_ERR_HEADER, (int32_t)i, cd->method, 0, + "truncated stream counts for coder %u", (unsigned)i); + goto bad; + } + if(out_streams != 1 || in_streams == 0 || + in_streams > SZ_CHAIN_MAX_STREAMS) { + err_set(err, SZ_CHAIN_ERR_LAYOUT, (int32_t)i, cd->method, 0, + "coder %u declares %u input / %u output streams", (unsigned)i, + (unsigned)in_streams, (unsigned)out_streams); + goto bad; + } + cd->num_in_streams = in_streams; + } + if(cd->num_in_streams > SZ_CHAIN_MAX_STREAMS - running_in) { + err_set(err, SZ_CHAIN_ERR_LAYOUT, (int32_t)i, cd->method, 0, + "folder has too many input streams"); + goto bad; + } + cd->first_in_stream = running_in; + running_in += cd->num_in_streams; + + if(main_byte & 0x20) { + uint32_t props_size; + if(rdr_num32(&r, &props_size) != 0) { + err_set(err, SZ_CHAIN_ERR_HEADER, (int32_t)i, cd->method, 0, + "truncated properties size for coder %u", (unsigned)i); + goto bad; + } + if((size_t)(r.end - r.p) < (size_t)props_size) { + err_set(err, SZ_CHAIN_ERR_HEADER, (int32_t)i, cd->method, 0, + "properties of coder %u run past the folder header", + (unsigned)i); + goto bad; + } + cd->props_off = (uint32_t)(r.p - c->blob); + cd->props_size = props_size; + r.p += props_size; + } + } + + c->total_streams = running_in; + if(c->total_streams == 0) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, "folder has no input streams"); + goto bad; + } + + /* numCoders - 1 bonds, each (input stream index, source coder index). */ + c->num_bonds = c->num_coders - 1; + if(c->total_streams < c->num_bonds) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, + "folder has fewer streams (%u) than bonds (%u)", + (unsigned)c->total_streams, (unsigned)c->num_bonds); + goto bad; + } + for(i = 0; i < c->num_bonds; i++) { + uint32_t in_index, out_coder; + if(rdr_num32(&r, &in_index) != 0 || rdr_num32(&r, &out_coder) != 0) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, + "truncated bond list at bond %u", (unsigned)i); + goto bad; + } + if(in_index >= c->total_streams || out_coder >= c->num_coders) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, + "bond %u refers to stream %u / coder %u out of range", + (unsigned)i, (unsigned)in_index, (unsigned)out_coder); + goto bad; + } + c->bond[i].in_index = in_index; + c->bond[i].out_coder = out_coder; + } + + if((c->total_streams - c->num_bonds) != num_pack_streams) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, + "folder header implies %u packed streams, the archive declares %u", + (unsigned)(c->total_streams - c->num_bonds), + (unsigned)num_pack_streams); + goto bad; + } + num_unused = c->total_streams - c->num_bonds; + + for(i = 0; i < num_unused; i++) { + uint32_t idx; + if(num_unused == 1) { + uint32_t s; + for(s = 0; s < c->total_streams; s++) { + int used = 0; + for(j = 0; j < c->num_bonds; j++) + if(c->bond[j].in_index == s) used = 1; + if(!used) break; + } + if(s == c->total_streams) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, + "folder has no free stream for its packed data"); + goto bad; + } + idx = s; + } else { + if(rdr_num32(&r, &idx) != 0 || idx >= c->total_streams) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, + "packed stream %u is out of range", (unsigned)i); + goto bad; + } + } + for(j = 0; j < i; j++) { + if(c->pack_stream_idx[j] == idx) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, + "packed stream %u is listed twice", (unsigned)idx); + goto bad; + } + } + c->pack_stream_idx[i] = idx; + } + + /* Stream -> source table. */ + for(i = 0; i < c->total_streams; i++) { + c->source[i].from_coder = -1; + c->source[i].pack_index = 0; + } + for(i = 0; i < c->num_bonds; i++) { + uint32_t s = c->bond[i].in_index; + if(c->source[s].from_coder >= 0) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, + "stream %u is fed by two coders", (unsigned)s); + goto bad; + } + c->source[s].from_coder = (int32_t)c->bond[i].out_coder; + } + for(i = 0; i < num_unused; i++) { + uint32_t s = c->pack_stream_idx[i]; + if(c->source[s].from_coder >= 0) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, + "packed stream %u is also fed by a coder", (unsigned)s); + goto bad; + } + c->source[s].pack_index = i; + } + for(i = 0; i < c->total_streams; i++) { + int is_pack = 0; + if(c->source[i].from_coder >= 0) continue; + for(j = 0; j < num_unused; j++) + if(c->pack_stream_idx[j] == i) is_pack = 1; + if(!is_pack) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, + "input stream %u has no source", (unsigned)i); + goto bad; + } + } + + { + int found = 0; + for(i = 0; i < c->num_coders; i++) { + int used = 0; + for(j = 0; j < c->num_bonds; j++) + if(c->bond[j].out_coder == i) used = 1; + if(!used) { + c->unpack_coder = i; + found = 1; + break; + } + } + if(!found) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, "folder has no output coder"); + goto bad; + } + } + + for(i = 0; i < c->num_coders; i++) + c->coder_unpack_sizes[i] = coder_unpack_sizes[i]; + + if(coder_unpack_sizes[c->unpack_coder] != unpack_size) { + err_set(err, SZ_CHAIN_ERR_HEADER, (int32_t)c->unpack_coder, + c->coder[c->unpack_coder].method, 0, + "folder unpack size %llu does not match the output coder size %llu", + (unsigned long long)unpack_size, + (unsigned long long)coder_unpack_sizes[c->unpack_coder]); + goto bad; + } + + *out = c; + return 0; + +bad: + sz_chain_free(c); + return -1; +} + +void sz_chain_free(sz_chain *c) { + if(!c) return; + free(c->blob); + free(c); +} + +uint32_t sz_chain_num_coders(const sz_chain *c) { + return c ? c->num_coders : 0; +} + +uint32_t sz_chain_num_pack_streams(const sz_chain *c) { + return c ? c->num_pack_streams : 0; +} + +int64_t sz_chain_coder_method(const sz_chain *c, uint32_t index) { + if(!c || index >= c->num_coders) return -1; + return (int64_t)c->coder[index].method; +} + +void sz_chain_describe(const sz_chain *c, char *buf, size_t size) { + size_t used = 0; + uint32_t i; + + if(!buf || size == 0) return; + buf[0] = 0; + if(!c) return; + + for(i = 0; i < c->num_coders; i++) { + char scratch[32]; + const char *name = method_label(c->coder[i].method, scratch, + sizeof(scratch)); + int n = snprintf(buf + used, size - used, "%s%s", i ? " + " : "", name); + if(n < 0 || (size_t)n >= size - used) return; + used += (size_t)n; + } + snprintf(buf + used, size - used, " (%u coder%s, %u packed stream%s)", + (unsigned)c->num_coders, c->num_coders == 1 ? "" : "s", + (unsigned)c->num_pack_streams, + c->num_pack_streams == 1 ? "" : "s"); +} + +int sz_chain_check(const sz_chain *c, sz_chain_err_t *err) { + uint32_t i, j; + + if(!c) { + err_set(err, SZ_CHAIN_ERR_PARAM, -1, 0, 0, "no folder"); + return -1; + } + + for(i = 0; i < c->num_coders; i++) { + const sz_coder *cd = &c->coder[i]; + int kind = method_kind(cd->method); + + if(kind < 0) { + char scratch[32]; + err_set(err, SZ_CHAIN_ERR_METHOD, (int32_t)i, cd->method, 0, + "coder %u uses %s, which this build cannot decode", (unsigned)i, + method_label(cd->method, scratch, sizeof(scratch))); + return -1; + } + if(kind == SZ_N_BCJ2) { + if(cd->num_in_streams != 4) { + err_set(err, SZ_CHAIN_ERR_LAYOUT, (int32_t)i, cd->method, 0, + "BCJ2 coder %u has %u input streams, expected 4", (unsigned)i, + (unsigned)cd->num_in_streams); + return -1; + } + } else if(cd->num_in_streams != 1) { + err_set(err, SZ_CHAIN_ERR_LAYOUT, (int32_t)i, cd->method, 0, + "%s coder %u has %u input streams, expected 1", + sz_chain_method_name(cd->method), (unsigned)i, + (unsigned)cd->num_in_streams); + return -1; + } + for(j = 0; j < cd->num_in_streams; j++) { + uint32_t s = cd->first_in_stream + j; + uint32_t k; + int is_pack = 0; + if(c->source[s].from_coder >= 0) continue; + for(k = 0; k < c->num_pack_streams; k++) + if(c->pack_stream_idx[k] == s) is_pack = 1; + if(!is_pack) { + err_set(err, SZ_CHAIN_ERR_HEADER, (int32_t)i, cd->method, 0, + "coder %u input stream %u has no source", (unsigned)i, + (unsigned)s); + return -1; + } + } + } + + if(c->unpack_coder >= c->num_coders) { + err_set(err, SZ_CHAIN_ERR_HEADER, -1, 0, 0, "invalid output coder"); + return -1; + } + return 0; +} + +/* ------------------------------------------------------------- decoding */ + +typedef struct sz_node sz_node; + +typedef struct { + IByteIn vt; + sz_node *node; +} sz_ppmd_in; + +struct sz_node { + int kind; + uint32_t method; + const uint8_t *props; + uint32_t props_size; + const sz_chain_limits_t *lim; + uint64_t out_size; + uint64_t produced; + sz_node *in; + + sz_chain_read_fn read_at; + void *read_ctx; + uint64_t pack_pos; + uint64_t pack_left; + + uint8_t *inbuf; + size_t in_cap; + size_t in_len; + size_t in_off; + + int in_eof; + int finished; + int failed; + sz_chain_err_t *err; + + union { + struct { + CLzmaDec st; + } lzma; + struct { + CLzma2Dec st; + } lzma2; + struct { + CPpmd7 *p; + sz_ppmd_in in; + uint8_t *buf; + size_t len, off; + int in_eof; + } ppmd; + struct { + int kind; + unsigned delta; + uint32_t pc; + size_t fpos; + uint32_t xstate; + uint8_t dstate[DELTA_STATE_SIZE]; + } filt; + struct { + CBcj2Dec st; + sz_node *side[3]; + uint8_t *side_buf[3]; + size_t side_len[3]; + uint8_t *mbuf; + size_t mlen, moff; + int prepared; + } b2; + } u; +}; + +typedef struct { + sz_chain *chain; + sz_node *by_coder[SZ_CHAIN_MAX_CODERS]; + sz_node *by_pack[SZ_CHAIN_MAX_STREAMS]; + uint8_t visiting[SZ_CHAIN_MAX_CODERS]; + sz_chain_read_fn read_at; + void *read_ctx; + sz_chain_err_t *err; + int failed; + sz_chain_status_t fail_status; + int32_t fail_coder; +} sz_build; + +static int node_fail(sz_node *n, sz_chain_status_t status, const char *message) { + if(n->failed) return -1; + n->failed = 1; + if(n->err) + err_set(n->err, status, -1, n->method, n->produced, "%s", message); + return -1; +} + +static void node_destroy(sz_node *n) { + if(!n) return; + switch(n->kind) { + case SZ_N_LZMA: + LzmaDec_Free(&n->u.lzma.st, &g_alloc); + break; + case SZ_N_LZMA2: + Lzma2Dec_Free(&n->u.lzma2.st, &g_alloc); + break; + case SZ_N_PPMD: + if(n->u.ppmd.p) { + Ppmd7_Free(n->u.ppmd.p, &g_alloc); + free(n->u.ppmd.p); + } + free(n->u.ppmd.buf); + break; + case SZ_N_BCJ2: { + int i; + for(i = 0; i < 3; i++) free(n->u.b2.side_buf[i]); + free(n->u.b2.mbuf); + break; + } + default: break; + } + free(n->inbuf); + free(n); +} + +static int node_pull(sz_node *n, uint8_t *dst, size_t want, size_t *got); + +/* Refills n->inbuf from the single upstream node. + Returns 0 = data, 1 = upstream exhausted, -1 = error. */ +static int node_refill(sz_node *n) { + size_t got = 0; + int rc; + + n->in_off = 0; + n->in_len = 0; + if(n->in_eof) return 1; + rc = node_pull(n->in, n->inbuf, n->in_cap, &got); + if(rc != 0) return -1; + if(got == 0) { + n->in_eof = 1; + return 1; + } + n->in_len = got; + return 0; +} + +/* Applies the filter to data[0..size), in place, and returns the number of + bytes converted. The unconverted tail is left for the next call. */ +static size_t filter_apply(sz_node *n, uint8_t *data, size_t size) { + size_t consumed = size; + + switch(n->u.filt.kind) { + case SZ_F_DELTA: + Delta_Decode(n->u.filt.dstate, n->u.filt.delta, data, (SizeT)size); + break; + case SZ_F_X86: { + Byte *tail = Z7_BRANCH_CONV_ST_DEC(X86)(data, (SizeT)size, n->u.filt.pc, + &n->u.filt.xstate); + consumed = (size_t)(tail - data); + break; + } + case SZ_F_PPC: + consumed = (size_t)(Z7_BRANCH_CONV_DEC(PPC)(data, (SizeT)size, + n->u.filt.pc) - + data); + break; + case SZ_F_IA64: + consumed = (size_t)(Z7_BRANCH_CONV_DEC(IA64)(data, (SizeT)size, + n->u.filt.pc) - + data); + break; + case SZ_F_SPARC: + consumed = (size_t)(Z7_BRANCH_CONV_DEC(SPARC)(data, (SizeT)size, + n->u.filt.pc) - + data); + break; + case SZ_F_ARM: + consumed = (size_t)(Z7_BRANCH_CONV_DEC(ARM)(data, (SizeT)size, + n->u.filt.pc) - + data); + break; + case SZ_F_ARMT: + consumed = (size_t)(Z7_BRANCH_CONV_DEC(ARMT)(data, (SizeT)size, + n->u.filt.pc) - + data); + break; + default: break; + } + return consumed; +} + +static Byte ppmd_read(IByteInPtr pp) { + Z7_CONTAINER_FROM_VTBL_TO_DECL_VAR_pp_vt_p(sz_ppmd_in) + sz_node *n = p->node; + + if(n->u.ppmd.off == n->u.ppmd.len) { + size_t got = 0; + if(n->u.ppmd.in_eof) return 0; + if(node_pull(n->in, n->u.ppmd.buf, SZ_IN_CHUNK, &got) != 0) { + n->failed = 1; + return 0; + } + if(got == 0) { + n->u.ppmd.in_eof = 1; + return 0; + } + n->u.ppmd.len = got; + n->u.ppmd.off = 0; + } + return n->u.ppmd.buf[n->u.ppmd.off++]; +} + +/* Reads the three BCJ2 side streams in full. Returns 0 or -1. */ +static int bcj2_prepare(sz_node *n) { + uint64_t budget = n->lim ? n->lim->max_side_bytes : 0; + uint64_t used = 0; + int i; + + for(i = 0; i < 3; i++) { + sz_node *s = n->u.b2.side[i]; + uint64_t size = s->out_size; + size_t got = 0; + + if(i < 2 && (size & 3) != 0) + return node_fail(n, SZ_CHAIN_ERR_DATA, + "BCJ2 address stream is not a multiple of 4 bytes"); + if(used > budget || size > budget - used) { + if(n->err) + err_set(n->err, SZ_CHAIN_ERR_LIMIT, -1, n->method, 0, + "BCJ2 needs %llu MB for its side streams, the limit is %llu MB", + (unsigned long long)((used + size) >> 20), + (unsigned long long)(budget >> 20)); + n->failed = 1; + return -1; + } + used += size; + n->u.b2.side_len[i] = (size_t)size; + n->u.b2.side_buf[i] = (uint8_t *)malloc(size ? (size_t)size : 1); + if(!n->u.b2.side_buf[i]) + return node_fail(n, SZ_CHAIN_ERR_MEM, + "cannot allocate the BCJ2 side streams"); + while(got < (size_t)size) { + size_t g = 0; + if(node_pull(s, n->u.b2.side_buf[i] + got, (size_t)size - got, &g) != 0) + return -1; + if(g == 0) + return node_fail(n, SZ_CHAIN_ERR_DATA, + "BCJ2 side stream is truncated"); + got += g; + } + } + + n->u.b2.mbuf = (uint8_t *)malloc(SZ_BCJ2_MAIN_CHUNK); + if(!n->u.b2.mbuf) + return node_fail(n, SZ_CHAIN_ERR_MEM, "cannot allocate the BCJ2 buffer"); + + { + CBcj2Dec *p = &n->u.b2.st; + Bcj2Dec_Init(p); + p->bufs[BCJ2_STREAM_CALL] = n->u.b2.side_buf[0]; + p->lims[BCJ2_STREAM_CALL] = n->u.b2.side_buf[0] + n->u.b2.side_len[0]; + p->bufs[BCJ2_STREAM_JUMP] = n->u.b2.side_buf[1]; + p->lims[BCJ2_STREAM_JUMP] = n->u.b2.side_buf[1] + n->u.b2.side_len[1]; + p->bufs[BCJ2_STREAM_RC] = n->u.b2.side_buf[2]; + p->lims[BCJ2_STREAM_RC] = n->u.b2.side_buf[2] + n->u.b2.side_len[2]; + } + n->u.b2.prepared = 1; + return 0; +} + +/* Refills BCJ2's MAIN staging buffer, keeping the unconsumed tail. */ +static int bcj2_refill_main(sz_node *n) { + size_t tail = n->u.b2.mlen - n->u.b2.moff; + size_t got = 0; + + if(tail && n->u.b2.moff) + memmove(n->u.b2.mbuf, n->u.b2.mbuf + n->u.b2.moff, tail); + n->u.b2.moff = 0; + n->u.b2.mlen = tail; + if(n->in_eof) return 0; + if(node_pull(n->in, n->u.b2.mbuf + tail, SZ_BCJ2_MAIN_CHUNK - tail, + &got) != 0) + return -1; + if(got == 0) n->in_eof = 1; + else n->u.b2.mlen = tail + got; + return 0; +} + +static int node_pull(sz_node *n, uint8_t *dst, size_t want, size_t *got) { + size_t total = 0; + uint64_t remain; + + *got = 0; + if(n->failed) return -1; + if(want == 0) return 0; + remain = n->out_size - n->produced; + if((uint64_t)want > remain) want = (size_t)remain; + if(want == 0) return 0; + + switch(n->kind) { + case SZ_N_PACK: + if((uint64_t)want > n->pack_left) want = (size_t)n->pack_left; + if(want == 0) return 0; + if(n->read_at(n->read_ctx, n->pack_pos, dst, want) != 0) + return node_fail(n, SZ_CHAIN_ERR_READ, "cannot read packed data"); + n->pack_pos += want; + n->pack_left -= want; + total = want; + break; + + case SZ_N_COPY: + if(node_pull(n->in, dst, want, &total) != 0) return -1; + break; + + case SZ_N_LZMA: + case SZ_N_LZMA2: { + int eof_seen = 0; + while(total < want) { + SizeT dest_len, src_len; + ELzmaStatus status; + SRes res; + + if(n->in_off == n->in_len) { + int rc = node_refill(n); + if(rc < 0) return -1; + if(rc == 1) eof_seen = 1; + } + dest_len = (SizeT)(want - total); + src_len = (SizeT)(n->in_len - n->in_off); + if(n->kind == SZ_N_LZMA) + res = LzmaDec_DecodeToBuf(&n->u.lzma.st, dst + total, &dest_len, + n->inbuf + n->in_off, &src_len, + LZMA_FINISH_ANY, &status); + else + res = Lzma2Dec_DecodeToBuf(&n->u.lzma2.st, dst + total, &dest_len, + n->inbuf + n->in_off, &src_len, + LZMA_FINISH_ANY, &status); + if(res != SZ_OK) + return node_fail(n, SZ_CHAIN_ERR_DATA, + n->kind == SZ_N_LZMA ? "LZMA stream is corrupt" + : "LZMA2 stream is corrupt"); + + n->in_off += (size_t)src_len; + total += (size_t)dest_len; + + if(status == LZMA_STATUS_FINISHED_WITH_MARK) { + n->finished = 1; + break; + } + if(dest_len == 0 && src_len == 0) { + if(eof_seen) break; /* truncated: the caller sees a short read */ + if(n->in_len >= n->in_cap) + return node_fail(n, SZ_CHAIN_ERR_DATA, + "LZMA chunk does not fit the staging buffer"); + } + } + break; + } + + case SZ_N_PPMD: + if(!n->u.ppmd.p) + return node_fail(n, SZ_CHAIN_ERR_HEADER, "PPMd coder has no model"); + while(total < want) { + int sym = Ppmd7z_DecodeSymbol(n->u.ppmd.p); + if(sym < 0) + return node_fail(n, SZ_CHAIN_ERR_DATA, + sym == PPMD7_SYM_END ? "PPMd stream ended early" + : "PPMd stream is corrupt"); + dst[total++] = (uint8_t)sym; + } + break; + + case SZ_N_FILTER: + while(total < want) { + size_t avail; + + if(n->in_off < n->u.filt.fpos) { + avail = n->u.filt.fpos - n->in_off; + if(avail > want - total) avail = want - total; + memcpy(dst + total, n->inbuf + n->in_off, avail); + n->in_off += avail; + total += avail; + continue; + } + if(n->finished) break; + + /* Drop the retired prefix; the unconverted carry stays at the front. + Retiring N bytes is what advances the program counter by N, because + inbuf[0] now sits N bytes further into the stream. */ + if(n->in_off > 0) { + size_t rest = n->in_len - n->in_off; + if(rest) memmove(n->inbuf, n->inbuf + n->in_off, rest); + n->u.filt.pc += (uint32_t)n->in_off; + n->in_len = rest; + n->in_off = 0; + n->u.filt.fpos = 0; + } + if(n->in_len < n->in_cap && !n->in_eof) { + size_t g = 0; + int rc = node_pull(n->in, n->inbuf + n->in_len, + n->in_cap - n->in_len, &g); + if(rc != 0) return -1; + if(g == 0) n->in_eof = 1; + else n->in_len += g; + } + if(n->in_len == 0) { + n->finished = 1; + break; + } + n->u.filt.fpos = filter_apply(n, n->inbuf, n->in_len); + if(n->u.filt.fpos == 0) { + if(n->in_eof) { + /* The tail is too short to contain a full instruction: the 7z + encoder leaves it unconverted, so pass it through. */ + n->u.filt.fpos = n->in_len; + } else if(n->in_len >= n->in_cap) { + return node_fail(n, SZ_CHAIN_ERR_DATA, + "filter cannot make progress on its input"); + } + } + } + break; + + case SZ_N_BCJ2: { + CBcj2Dec *p = &n->u.b2.st; + size_t want0 = want; + + if(!n->u.b2.prepared && bcj2_prepare(n) != 0) return -1; + + p->dest = dst; + p->destLim = dst + want0; + for(;;) { + const Byte *before_main; + int progressed; + + if(p->dest == p->destLim) break; + if(n->u.b2.mlen - n->u.b2.moff <= SZ_BCJ2_KEEP && !n->in_eof) { + if(bcj2_refill_main(n) != 0) return -1; + } + if(n->u.b2.moff == n->u.b2.mlen && n->in_eof) break; + + p->bufs[BCJ2_STREAM_MAIN] = n->u.b2.mbuf + n->u.b2.moff; + p->lims[BCJ2_STREAM_MAIN] = n->u.b2.mbuf + n->u.b2.mlen; + before_main = p->bufs[BCJ2_STREAM_MAIN]; + progressed = 0; + + if(Bcj2Dec_Decode(p) != SZ_OK) + return node_fail(n, SZ_CHAIN_ERR_DATA, "BCJ2 stream is corrupt"); + + n->u.b2.moff = (size_t)(p->bufs[BCJ2_STREAM_MAIN] - n->u.b2.mbuf); + if(p->bufs[BCJ2_STREAM_MAIN] != before_main || p->dest != dst) + progressed = 1; + if(!progressed) break; + } + total = (size_t)(p->dest - dst); + break; + } + + default: + return node_fail(n, SZ_CHAIN_ERR_LAYOUT, "unsupported coder shape"); + } + + n->produced += total; + *got = total; + return 0; +} + +/* ------------------------------------------------------------- graph */ + +static sz_node *build_stream(sz_build *b, uint32_t stream); +static sz_node *build_coder(sz_build *b, uint32_t index); + +static void build_fail(sz_build *b, sz_chain_status_t status, + int32_t coder, uint32_t method, const char *fmt, + ...) { + va_list ap; + b->failed = 1; + b->fail_status = status; + b->fail_coder = coder; + if(!b->err || b->err->message[0]) return; + b->err->status = status; + b->err->coder = coder; + b->err->method = method; + b->err->offset = 0; + if(fmt) { + va_start(ap, fmt); + vsnprintf(b->err->message, sizeof(b->err->message), fmt, ap); + va_end(ap); + } else { + b->err->message[0] = 0; + } +} + +static sz_node *node_new(sz_build *b, int kind, uint32_t method, + const sz_coder *cd) { + sz_node *n = (sz_node *)calloc(1, sizeof(*n)); + if(!n) { + build_fail(b, SZ_CHAIN_ERR_MEM, -1, method, "out of memory"); + return NULL; + } + n->kind = kind; + n->method = method; + n->lim = &b->chain->limits; + n->err = b->err; + if(cd) { + n->props = b->chain->blob + cd->props_off; + n->props_size = cd->props_size; + } + return n; +} + +static int alloc_inbuf(sz_node *n, size_t cap) { + n->inbuf = (uint8_t *)malloc(cap); + if(!n->inbuf) return -1; + n->in_cap = cap; + return 0; +} + +static sz_node *build_pack(sz_build *b, uint32_t pack_index) { + sz_node *n; + uint64_t start, end; + + if(b->by_pack[pack_index]) return b->by_pack[pack_index]; + n = node_new(b, SZ_N_PACK, 0, NULL); + if(!n) return NULL; + start = b->chain->pack_positions[pack_index]; + end = b->chain->pack_positions[pack_index + 1]; + if(end < start) { + node_destroy(n); + build_fail(b, SZ_CHAIN_ERR_HEADER, -1, 0, "packed stream offsets go back"); + return NULL; + } + n->pack_pos = start; + n->pack_left = end - start; + n->out_size = end - start; + n->read_at = b->read_at; + n->read_ctx = b->read_ctx; + b->by_pack[pack_index] = n; + return n; +} + +static sz_node *build_stream(sz_build *b, uint32_t stream) { + const sz_source *src; + if(stream >= b->chain->total_streams) return NULL; + src = &b->chain->source[stream]; + if(src->from_coder >= 0) return build_coder(b, (uint32_t)src->from_coder); + return build_pack(b, src->pack_index); +} + +static int lzma_dict_size(const uint8_t *props, uint32_t size, uint64_t *out) { + uint64_t d; + if(size != 5) return -1; + d = (uint64_t)props[1] | ((uint64_t)props[2] << 8) | + ((uint64_t)props[3] << 16) | ((uint64_t)props[4] << 24); + *out = d; + return 0; +} + +static int lzma2_dict_size(const uint8_t *props, uint32_t size, uint64_t *out) { + unsigned p; + if(size != 1) return -1; + p = props[0]; + if(p > 40) return -1; + *out = (p == 40) ? 0xFFFFFFFFull : ((uint64_t)(2 | (p & 1)) << (p / 2 + 11)); + return 0; +} + +static sz_node *build_coder(sz_build *b, uint32_t index) { + sz_chain *c = b->chain; + const sz_coder *cd = &c->coder[index]; + sz_node *n; + int kind; + + if(b->by_coder[index]) return b->by_coder[index]; + if(b->visiting[index]) { + build_fail(b, SZ_CHAIN_ERR_HEADER, (int32_t)index, cd->method, + "cycle in the coder graph"); + return NULL; + } + b->visiting[index] = 1; + + kind = method_kind(cd->method); + if(kind < 0) { + char scratch[32]; + build_fail(b, SZ_CHAIN_ERR_METHOD, (int32_t)index, cd->method, + "coder %u uses %s, which this build cannot decode", + (unsigned)index, + method_label(cd->method, scratch, sizeof(scratch))); + b->visiting[index] = 0; + return NULL; + } + + n = node_new(b, kind, cd->method, cd); + if(!n) { + b->visiting[index] = 0; + return NULL; + } + + /* Each coder's output size comes from the archive's UNPACK_INFO, one entry + per coder output stream; that is what lets the chain stream with exact + boundaries. It is *not* the folder's unpack size: in a BCJ2 folder the + four LZMA2 coders feeding MAIN/CALL/JUMP/RC all have sizes of their own + (MAIN can even be larger than the folder's final size, e.g. 300066 vs + 300000), so using the folder size would silently truncate them. */ + n->out_size = c->coder_unpack_sizes[index]; + + switch(kind) { + case SZ_N_COPY: + case SZ_N_LZMA: + case SZ_N_LZMA2: + case SZ_N_PPMD: + case SZ_N_FILTER: + n->in = build_stream(b, cd->first_in_stream); + if(!n->in) { + node_destroy(n); + b->visiting[index] = 0; + return NULL; + } + break; + case SZ_N_BCJ2: { + int i; + n->in = build_stream(b, cd->first_in_stream); + if(!n->in) { + node_destroy(n); + b->visiting[index] = 0; + return NULL; + } + for(i = 0; i < 3; i++) { + n->u.b2.side[i] = build_stream(b, cd->first_in_stream + 1 + (uint32_t)i); + if(!n->u.b2.side[i]) { + node_destroy(n); + b->visiting[index] = 0; + return NULL; + } + } + break; + } + default: + node_destroy(n); + b->visiting[index] = 0; + return NULL; + } + + /* Method specific setup. */ + switch(kind) { + case SZ_N_COPY: + break; + case SZ_N_LZMA: { + uint64_t dict = 0; + if(lzma_dict_size(n->props, n->props_size, &dict) != 0) { + build_fail(b, SZ_CHAIN_ERR_HEADER, (int32_t)index, cd->method, + "LZMA coder %u has invalid properties", (unsigned)index); + goto fail; + } + if(dict > c->limits.max_dict_bytes) { + build_fail(b, SZ_CHAIN_ERR_LIMIT, (int32_t)index, cd->method, + "LZMA dictionary is %llu MB, the limit is %llu MB", + (unsigned long long)(dict >> 20), + (unsigned long long)(c->limits.max_dict_bytes >> 20)); + goto fail; + } + LzmaDec_CONSTRUCT(&n->u.lzma.st); + if(LzmaDec_Allocate(&n->u.lzma.st, n->props, n->props_size, &g_alloc) != + SZ_OK) { + build_fail(b, SZ_CHAIN_ERR_MEM, (int32_t)index, cd->method, + "cannot allocate the LZMA dictionary"); + goto fail; + } + LzmaDec_Init(&n->u.lzma.st); + if(alloc_inbuf(n, SZ_IN_CHUNK) != 0) goto oom; + break; + } + case SZ_N_LZMA2: { + uint64_t dict = 0; + if(lzma2_dict_size(n->props, n->props_size, &dict) != 0) { + build_fail(b, SZ_CHAIN_ERR_HEADER, (int32_t)index, cd->method, + "LZMA2 coder %u has invalid properties", (unsigned)index); + goto fail; + } + if(dict > c->limits.max_dict_bytes) { + build_fail(b, SZ_CHAIN_ERR_LIMIT, (int32_t)index, cd->method, + "LZMA2 dictionary is %llu MB, the limit is %llu MB", + (unsigned long long)(dict >> 20), + (unsigned long long)(c->limits.max_dict_bytes >> 20)); + goto fail; + } + Lzma2Dec_CONSTRUCT(&n->u.lzma2.st); + if(Lzma2Dec_Allocate(&n->u.lzma2.st, n->props[0], &g_alloc) != SZ_OK) { + build_fail(b, SZ_CHAIN_ERR_MEM, (int32_t)index, cd->method, + "cannot allocate the LZMA2 dictionary"); + goto fail; + } + Lzma2Dec_Init(&n->u.lzma2.st); + if(alloc_inbuf(n, SZ_IN_CHUNK) != 0) goto oom; + break; + } + case SZ_N_PPMD: { + unsigned order; + uint32_t mem; + CPpmd7 *p; + + if(n->props_size != 5) { + build_fail(b, SZ_CHAIN_ERR_HEADER, (int32_t)index, cd->method, + "PPMd coder %u must carry 5 property bytes", (unsigned)index); + goto fail; + } + order = n->props[0]; + mem = (uint32_t)n->props[1] | ((uint32_t)n->props[2] << 8) | + ((uint32_t)n->props[3] << 16) | ((uint32_t)n->props[4] << 24); + if(order < PPMD7_MIN_ORDER || order > PPMD7_MAX_ORDER || + mem < PPMD7_MIN_MEM_SIZE) { + build_fail(b, SZ_CHAIN_ERR_HEADER, (int32_t)index, cd->method, + "PPMd coder %u has invalid properties", (unsigned)index); + goto fail; + } + if((uint64_t)mem > c->limits.max_ppmd_bytes) { + build_fail(b, SZ_CHAIN_ERR_LIMIT, (int32_t)index, cd->method, + "PPMd model is %llu MB, the limit is %llu MB", + (unsigned long long)((uint64_t)mem >> 20), + (unsigned long long)(c->limits.max_ppmd_bytes >> 20)); + goto fail; + } + p = (CPpmd7 *)malloc(sizeof(*p)); + if(!p) goto oom; + n->u.ppmd.p = p; + Ppmd7_Construct(p); + if(!Ppmd7_Alloc(p, mem, &g_alloc)) { + build_fail(b, SZ_CHAIN_ERR_MEM, (int32_t)index, cd->method, + "cannot allocate the PPMd model"); + goto fail; + } + Ppmd7_Init(p, order); + n->u.ppmd.buf = (uint8_t *)malloc(SZ_IN_CHUNK); + if(!n->u.ppmd.buf) goto oom; + n->u.ppmd.in.node = n; + n->u.ppmd.in.vt.Read = ppmd_read; + p->rc.dec.Stream = &n->u.ppmd.in.vt; + if(!Ppmd7z_RangeDec_Init(&p->rc.dec)) { + build_fail(b, SZ_CHAIN_ERR_DATA, (int32_t)index, cd->method, + "PPMd range decoder could not start (truncated stream?)"); + goto fail; + } + break; + } + case SZ_N_FILTER: { + int fk = filter_kind(cd->method); + n->u.filt.kind = fk; + if(fk == SZ_F_DELTA) { + if(n->props_size != 1) { + build_fail(b, SZ_CHAIN_ERR_HEADER, (int32_t)index, cd->method, + "Delta coder %u must carry 1 property byte", + (unsigned)index); + goto fail; + } + n->u.filt.delta = (unsigned)n->props[0] + 1; + Delta_Init(n->u.filt.dstate); + } else if(n->props_size != 0) { + build_fail(b, SZ_CHAIN_ERR_LAYOUT, (int32_t)index, cd->method, + "%s coder %u must not carry properties", + sz_chain_method_name(cd->method), (unsigned)index); + goto fail; + } + n->u.filt.xstate = Z7_BRANCH_CONV_ST_X86_STATE_INIT_VAL; + if(alloc_inbuf(n, SZ_FILT_CHUNK + SZ_FILT_LOOKAHEAD) != 0) goto oom; + break; + } + case SZ_N_BCJ2: + if(n->props_size != 0) { + build_fail(b, SZ_CHAIN_ERR_LAYOUT, (int32_t)index, cd->method, + "BCJ2 coder %u must not carry properties", (unsigned)index); + goto fail; + } + break; + default: break; + } + + b->visiting[index] = 0; + b->by_coder[index] = n; + return n; + +oom: + build_fail(b, SZ_CHAIN_ERR_MEM, (int32_t)index, cd->method, "out of memory"); +fail: + node_destroy(n); + b->visiting[index] = 0; + return NULL; +} + +/* ------------------------------------------------------------- decode */ + +static void free_nodes(sz_build *b) { + uint32_t i; + for(i = 0; i < SZ_CHAIN_MAX_CODERS; i++) + if(b->by_coder[i]) node_destroy(b->by_coder[i]); + for(i = 0; i < SZ_CHAIN_MAX_STREAMS; i++) + if(b->by_pack[i]) node_destroy(b->by_pack[i]); +} + +/* Final structural checks that only make sense once the whole output has been + produced: a BCJ2 folder must have consumed every one of its streams. */ +static int finish_check(sz_node *n, sz_chain_err_t *err) { + if(n->kind != SZ_N_BCJ2) return 0; + { + CBcj2Dec *p = &n->u.b2.st; + int i; + for(i = 0; i < 3; i++) { + if(p->bufs[i] != p->lims[i]) { + err_set(err, SZ_CHAIN_ERR_DATA, -1, n->method, n->produced, + "BCJ2 %s stream was not fully consumed", + i == 0 ? "CALL" : (i == 1 ? "JUMP" : "RC")); + return -1; + } + } + if(n->u.b2.moff != n->u.b2.mlen) { + err_set(err, SZ_CHAIN_ERR_DATA, -1, n->method, n->produced, + "BCJ2 MAIN stream was not fully consumed"); + return -1; + } + if(!Bcj2Dec_IsMaybeFinished(p)) { + err_set(err, SZ_CHAIN_ERR_DATA, -1, n->method, n->produced, + "BCJ2 stream ended in the middle of an instruction"); + return -1; + } + } + return 0; +} + +int sz_chain_decode(sz_chain *c, sz_chain_read_fn read_at, void *read_ctx, + sz_chain_sink_fn sink, void *sink_ctx, + sz_chain_cancel_fn cancel, void *cancel_ctx, + uint32_t *crc_out, sz_chain_err_t *err) { + sz_build b; + sz_node *root; + uint8_t *out; + uint64_t produced = 0; + uint32_t crc = CRC_INIT_VAL; + static int crc_table_ready; + + if(err) memset(err, 0, sizeof(*err)); + if(!c || !read_at || !sink) { + err_set(err, SZ_CHAIN_ERR_PARAM, -1, 0, 0, "missing folder or callbacks"); + return -1; + } + if(sz_chain_check(c, err) != 0) return -1; + if(!crc_table_ready) { + CrcGenerateTable(); + crc_table_ready = 1; + } + + memset(&b, 0, sizeof(b)); + b.chain = c; + b.err = err; + b.read_at = read_at; + b.read_ctx = read_ctx; + root = build_coder(&b, c->unpack_coder); + if(!root) { + free_nodes(&b); + if(err && !err->message[0]) + err_set(err, SZ_CHAIN_ERR_INTERNAL, -1, 0, 0, + "the coder chain could not be built"); + return -1; + } + + out = (uint8_t *)malloc(SZ_OUT_CHUNK); + if(!out) { + free_nodes(&b); + err_set(err, SZ_CHAIN_ERR_MEM, -1, 0, 0, + "cannot allocate the output buffer"); + return -1; + } + + while(produced < c->unpack_size) { + uint64_t remain = c->unpack_size - produced; + size_t want = (remain > SZ_OUT_CHUNK) ? (size_t)SZ_OUT_CHUNK + : (size_t)remain; + size_t got = 0; + + if(cancel && cancel(cancel_ctx)) { + err_set(err, SZ_CHAIN_ERR_CANCELED, -1, 0, produced, "canceled"); + free(out); + free_nodes(&b); + return -1; + } + if(node_pull(root, out, want, &got) != 0) { + free(out); + free_nodes(&b); + return -1; + } + if(got != want) { + /* The output coder stopped early. That is a truncated folder, not + necessarily truncated packed data: an inner coder can also run out + when its own declared size is larger than the stream it carries. */ + err_set(err, SZ_CHAIN_ERR_DATA, (int32_t)c->unpack_coder, root->method, + produced, + "the %s output stream ended %llu bytes early, after %llu of %llu " + "bytes", + sz_chain_method_name(root->method), + (unsigned long long)(want - got), + (unsigned long long)(produced + got), + (unsigned long long)c->unpack_size); + free(out); + free_nodes(&b); + return -1; + } + crc = CrcUpdate(crc, out, got); + if(sink(sink_ctx, out, got) != 0) { + err_set(err, SZ_CHAIN_ERR_WRITE, -1, root->method, produced, + "the output sink rejected data"); + free(out); + free_nodes(&b); + return -1; + } + produced += got; + } + + if(finish_check(root, err) != 0) { + free(out); + free_nodes(&b); + return -1; + } + + if(crc_out) *crc_out = CRC_GET_DIGEST(crc); + free(out); + free_nodes(&b); + return 0; +} diff --git a/src/sevenz_chain.h b/src/sevenz_chain.h new file mode 100644 index 0000000..6ce2331 --- /dev/null +++ b/src/sevenz_chain.h @@ -0,0 +1,157 @@ +/* sevenz_chain -- 7z folder (coder chain) decoder. + part of ps5-web-file-manager + + A 7z archive stores its data as *folders*. One folder is a small directed + graph of coders fed by N packed streams and producing a single unpacked + stream; entries are slices of the folder output (a folder holding several + entries is what makes an archive "solid"). + + The bundled LZMA SDK can decode a folder, but only through `CSzFolder`, + which is a fixed-size structure capped at 4 coders / 3 bonds. 7-Zip's own + BCJ2 chain uses 5 coders (BCJ2 plus four LZMA2 streams), so the SDK rejects + it -- while still listing the archive fine, because its *header* scanner is + a different, looser parser (64 coders). The C half of the SDK also has no + 7zAES coder at all. + + This module therefore parses the folder descriptor itself (dynamic arrays, + up to 64 coders, mirroring the SDK's header scanner) and drives the coder + graph itself. Two properties matter: + + * Streaming. The decoded bytes are pushed into a sink as they are + produced; a folder is never materialised as a whole, so a multi-gigabyte + solid block is workable. The only buffers sized from the archive are + the LZMA/LZMA2 dictionary, the PPMd model and the three side streams of + BCJ2 -- each capped by sz_chain_limits_t. + + * Precision. Every rejection names the coder and the method, and the + resource limits report the value the archive asked for and the value + that was allowed, so the UI can say something useful instead of + "corrupt archive". + + Not supported here (by design, see sz_chain_check): + * 7zAES (method 0x06F10701) -- needs its own implementation, it is not in + the SDK's C decoder. Reported as SZ_CHAIN_ERR_METHOD. +*/ + +#ifndef SEVENZ_CHAIN_H +#define SEVENZ_CHAIN_H + +#include +#include + +/* The SDK's header scanner accepts up to 64 coders per folder; folders in the + wild have 1-5. Keeping the same ceiling means "the SDK could list it" and + "we can decode it" accept the same archives. */ +#define SZ_CHAIN_MAX_CODERS 64 +#define SZ_CHAIN_MAX_STREAMS 64 + +/* Ceilings for the buffers whose size comes from the (attacker controlled) + archive header. */ +typedef struct { + uint64_t max_dict_bytes; /* LZMA / LZMA2 window */ + uint64_t max_ppmd_bytes; /* PPMd model */ + uint64_t max_side_bytes; /* BCJ2 CALL + JUMP + RC together */ +} sz_chain_limits_t; + +#define SZ_CHAIN_LIMITS_DEFAULT 0 +#define SZ_CHAIN_LIMITS_LARGE 1 +const sz_chain_limits_t *sz_chain_limits_profile(int profile); +const sz_chain_limits_t *sz_chain_default_limits(void); + +typedef enum { + SZ_CHAIN_OK = 0, + SZ_CHAIN_ERR_PARAM, /* bad arguments from the caller */ + SZ_CHAIN_ERR_MEM, /* allocation failed */ + SZ_CHAIN_ERR_HEADER, /* malformed folder descriptor */ + SZ_CHAIN_ERR_METHOD, /* coder method not supported */ + SZ_CHAIN_ERR_LAYOUT, /* coder graph shape not supported */ + SZ_CHAIN_ERR_LIMIT, /* a sz_chain_limits_t ceiling was hit */ + SZ_CHAIN_ERR_READ, /* the read callback failed */ + SZ_CHAIN_ERR_WRITE, /* the sink callback failed */ + SZ_CHAIN_ERR_DATA, /* a decoder rejected the data */ + SZ_CHAIN_ERR_CANCELED, + SZ_CHAIN_ERR_INTERNAL +} sz_chain_status_t; + +typedef struct { + sz_chain_status_t status; + int32_t coder; /* index of the offending coder, -1 when not applicable */ + uint32_t method; /* its method id, 0 when not applicable */ + uint64_t offset; /* decoded byte offset at the point of failure */ + char message[192]; +} sz_chain_err_t; + +const char *sz_chain_status_string(sz_chain_status_t status); + +/* "LZMA2", "BCJ2", "7zAES", "unknown 0x1234". Never returns NULL. */ +const char *sz_chain_method_name(uint32_t method); + +/* ---------------------------------------------------------------- folder */ + +typedef struct sz_chain sz_chain; + +/* Parses one folder descriptor. + + blob / blob_size + the CODERS_INFO bytes of this folder, i.e. the range + `CSzAr::CodersData[FoCodersOffsets[i] .. FoCodersOffsets[i + 1])`. + pack_positions + `CSzAr::PackPositions`, num_pack_streams + 1 entries, offsets of the + packed streams relative to the start of the archive's packed-data area. + coder_unpack_sizes + unpacked size of every coder of this folder, in stored coder order, i.e. + `&CSzAr::CoderUnpackSizes[CSzAr::FoToCoderUnpackSizes[i]]`. + unpack_size + `SzAr_GetFolderUnpackSize(&db, i)`. + + Returns 0 on success. On success *out owns a copy of blob, release it with + sz_chain_free(). On failure *out is untouched and err describes the + problem. */ +int sz_chain_parse(sz_chain **out, const uint8_t *blob, size_t blob_size, + const uint64_t *pack_positions, uint32_t num_pack_streams, + const uint64_t *coder_unpack_sizes, uint64_t unpack_size, + const sz_chain_limits_t *limits, sz_chain_err_t *err); + +void sz_chain_free(sz_chain *c); + +uint32_t sz_chain_num_coders(const sz_chain *c); +uint32_t sz_chain_num_pack_streams(const sz_chain *c); +/* Method id of coder `index`, or -1 when out of range. */ +int64_t sz_chain_coder_method(const sz_chain *c, uint32_t index); + +/* Writes e.g. "LZMA2 + BCJ2 (5 coders, 4 pack streams)" into buf. */ +void sz_chain_describe(const sz_chain *c, char *buf, size_t size); + +/* Walks every coder and the graph shape without touching any data, so callers + can refuse an archive before creating anything on disk. Fills err with the + same precision sz_chain_decode() would. */ +int sz_chain_check(const sz_chain *c, sz_chain_err_t *err); + +/* -------------------------------------------------------------- decoding */ + +/* Fills exactly size bytes at offset inside the packed-data area. + Returns 0 on success, non-zero on failure. */ +typedef int (*sz_chain_read_fn)(void *ctx, uint64_t offset, void *dst, + size_t size); + +/* Receives the decoded bytes in order. Returns 0 to continue. */ +typedef int (*sz_chain_sink_fn)(void *ctx, const void *data, size_t size); + +/* Returns non-zero to abort. May be NULL. */ +typedef int (*sz_chain_cancel_fn)(void *ctx); + +/* Decodes the whole folder, pushing the result into sink. + + The bytes are delivered strictly in order and the total is the folder's + declared unpack size. When crc_out is not NULL it receives the CRC-32 of + the delivered bytes, for the caller to compare with the folder CRC. + + Returns 0 on success, -1 on failure with err filled. A failing sink is + reported as SZ_CHAIN_ERR_WRITE; the caller is expected to make its own + message more specific. */ +int sz_chain_decode(sz_chain *c, sz_chain_read_fn read_at, void *read_ctx, + sz_chain_sink_fn sink, void *sink_ctx, + sz_chain_cancel_fn cancel, void *cancel_ctx, + uint32_t *crc_out, sz_chain_err_t *err); + +#endif /* SEVENZ_CHAIN_H */ diff --git a/tests/make_sevenz_fixtures.py b/tests/make_sevenz_fixtures.py index a5639d8..885820e 100644 --- a/tests/make_sevenz_fixtures.py +++ b/tests/make_sevenz_fixtures.py @@ -176,6 +176,11 @@ VARIANTS: list[tuple[str, list[str], bool, bool]] = [ ("bcj2", ["-m0=bcj2", "-m1=lzma2", "-m2=lzma2", "-m3=lzma2", "-m4=lzma2"], False, True), ("aes", ["-m0=lzma2", "-mx5"], True, True), ("utf8", ["-m0=lzma2", "-mx5"], False, True), + # Not solid: one folder per file. That is what exercises the multi-folder + # walk and the per-folder slice of the packed-stream table. + ("solidoff", ["-m0=lzma2", "-mx5", "-ms=off"], False, True), + ("bcj2off", ["-m0=bcj2", "-m1=lzma2", "-m2=lzma2", "-m3=lzma2", + "-m4=lzma2", "-ms=off"], False, True), ] diff --git a/tests/run-sevenz-tests.sh b/tests/run-sevenz-tests.sh index aa1896c..49e9185 100644 --- a/tests/run-sevenz-tests.sh +++ b/tests/run-sevenz-tests.sh @@ -1,11 +1,12 @@ #!/usr/bin/env bash -# Host test runner for the 7z engine (vendor subset + fixture matrix). +# Host test runner for the 7z engine. # # ./tests/run-sevenz-tests.sh # -# Builds the vendored LZMA SDK subset and the end-to-end driver, generates real -# .7z fixtures with a 7-Zip binary, extracts every fixture and compares the -# result byte for byte against the source tree. +# Builds the vendored LZMA SDK subset plus the project's own folder decoder +# (src/sevenz_chain.c), generates real .7z fixtures with a 7-Zip binary, +# extracts every fixture through that decoder and compares the result byte for +# byte against the source tree. # # On Windows this expects MinGW gcc in PATH and must be started with the MSYS # bash explicitly (`/usr/bin/bash tests/run-sevenz-tests.sh`) -- a bare `bash` @@ -20,10 +21,9 @@ FIXTURES="$ROOT/tests/fixtures-7z" PYTHON="${PYTHON:-python3}" CC="${CC:-gcc}" -# Archives that the current engine cannot read yet. Each entry needs a reason; -# when one of them starts passing the script says so, so the list cannot rot. -KNOWN_GAPS="bcj2 aes aeshe vol.7z.001" -# bcj2 - folder has 5 coders; needs the engine's own chain driver +# Archives the engine cannot read yet. Each entry needs a reason; when one of +# them starts passing the script says so, so the list cannot rot. +KNOWN_GAPS="aes aeshe vol.7z.001" # aes - 7zAES coder not implemented yet # aeshe - encrypted header (-mhe=on), same coder # vol.7z.001 - multi-volume input; needs the volume-set stream @@ -35,19 +35,31 @@ mkdir -p "$BUILD" CFLAGS_7Z=(-O2 -w -DZ7_PPMD_SUPPORT -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -DNDEBUG -D_REENTRANT) +VENDOR_OBJS=() for src in "$SEVENZ_DIR"/*.c; do name="$(basename "$src" .c)" "$CC" -c "${CFLAGS_7Z[@]}" -o "$BUILD/$name.o" "$src" + VENDOR_OBJS+=("$BUILD/$name.o") done +# The engine module is held to the same strictness as the rest of src/. +"$CC" -c -O2 -Wall -Wextra -Werror -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE \ + -I"$SEVENZ_DIR" -o "$BUILD/sevenz_chain.o" "$ROOT/src/sevenz_chain.c" + # unrar-style extra libs are only needed by the Windows path of 7zFile.c. EXTRA_LIBS=() case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) EXTRA_LIBS=(-lole32 -loleaut32 -luuid -ladvapi32 -luser32 -lshell32) ;; esac +"$CC" -O2 -w -I"$SEVENZ_DIR" -I"$ROOT/src" -o "$BUILD/sevenz_chain_e2e" \ + "$ROOT/tests/sevenz_chain_e2e.c" "$BUILD/sevenz_chain.o" "${VENDOR_OBJS[@]}" \ + "${EXTRA_LIBS[@]}" + +# The SDK-baseline driver is kept buildable: it is the fastest way to tell an +# engine bug from an SDK one when a fixture starts failing. "$CC" -O2 -w -I"$SEVENZ_DIR" -o "$BUILD/sevenz_e2e" \ - "$ROOT/tests/sevenz_e2e.c" "$BUILD"/*.o "${EXTRA_LIBS[@]}" + "$ROOT/tests/sevenz_e2e.c" "${VENDOR_OBJS[@]}" "${EXTRA_LIBS[@]}" # ---------------------------------------------------------------- fixtures if [ ! -f "$FIXTURES/lzma2.7z" ]; then @@ -71,13 +83,14 @@ is_gap() { run_case() { local name="$1" archive="$2" - local out="$BUILD/out/$name" - local log="$BUILD/out/$name.log" + local out log - rm -rf "$out" - mkdir -p "$out" + # A fresh directory per case keeps the run repeatable without deleting a tree + # of previous results, which guarded shells refuse to do. + out="$(mktemp -d "$BUILD/out/XXXXXX")" || return + log="$out.log" - if "$BUILD/sevenz_e2e" "$archive" "$out" >"$log" 2>&1 && + if "$BUILD/sevenz_chain_e2e" "$archive" "$out" >"$log" 2>&1 && diff -r "$FIXTURES/_src" "$out/_src" >/dev/null 2>&1; then if is_gap "$name"; then printf ' %-12s GAP CLOSED (remove from KNOWN_GAPS)\n' "$name" @@ -99,8 +112,8 @@ run_case() { fi } -echo "== 7z fixture matrix ==" -for a in store lzma2 lzma ppmd bcj delta utf8 bcj2 aes aeshe; do +echo "== 7z fixture matrix (engine: src/sevenz_chain.c) ==" +for a in store lzma2 lzma ppmd bcj delta utf8 bcj2 solidoff bcj2off aes aeshe; do [ -f "$FIXTURES/$a.7z" ] || continue run_case "$a" "$FIXTURES/$a.7z" done diff --git a/tests/sevenz_chain_e2e.c b/tests/sevenz_chain_e2e.c new file mode 100644 index 0000000..1be019d --- /dev/null +++ b/tests/sevenz_chain_e2e.c @@ -0,0 +1,475 @@ +/* + * End-to-end driver for the project's own 7z folder decoder (src/sevenz_chain.c). + * + * sevenz_chain_e2e + * + * Unlike tests/sevenz_e2e.c -- which drives the LZMA SDK's own CSzFolder based + * path and therefore cannot handle BCJ2 or anything else the SDK caps at four + * coders -- this one decodes *every* folder through src/sevenz_chain.c. + * + * It also mirrors the shape of the real extraction path: each folder is + * decoded once, streamed to a sink, and the sink splits the byte stream across + * the entries that live in that folder (that is what makes an archive solid). + * Per-entry and per-folder CRCs are verified as the bytes go past. + * + * Exit status is non-zero if any folder or entry failed. + */ + +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#include +#else +#include +#endif + +#include "7z.h" +#include "7zAlloc.h" +#include "7zCrc.h" +#include "7zFile.h" + +#include "sevenz_chain.h" + +#define INPUT_BUF_SIZE (1u << 18) +#define MAX_PATH_LEN 4096 + +static ISzAlloc g_alloc = { SzAlloc, SzFree }; +static ISzAlloc g_temp = { SzAlloc, SzFree }; + +static int g_failures = 0; + +/* ---------------------------------------------------------------- paths */ + +#if defined(_WIN32) + +static void utf8_to_utf16(const char *src, wchar_t *dst, size_t cap) { + size_t out = 0; + + while(*src && out + 2 < cap) { + unsigned char c = (unsigned char)*src++; + UInt32 cp; + + if(c < 0x80) { + cp = c; + } else if((c & 0xE0) == 0xC0 && (src[0] & 0xC0) == 0x80) { + cp = ((UInt32)(c & 0x1F) << 6) | (UInt32)(*src++ & 0x3F); + } else if((c & 0xF0) == 0xE0 && (src[0] & 0xC0) == 0x80 && + (src[1] & 0xC0) == 0x80) { + cp = ((UInt32)(c & 0x0F) << 12) | ((UInt32)(src[0] & 0x3F) << 6) | + (UInt32)(src[1] & 0x3F); + src += 2; + } else if((c & 0xF8) == 0xF0 && (src[0] & 0xC0) == 0x80 && + (src[1] & 0xC0) == 0x80 && (src[2] & 0xC0) == 0x80) { + cp = ((UInt32)(c & 0x07) << 18) | ((UInt32)(src[0] & 0x3F) << 12) | + ((UInt32)(src[1] & 0x3F) << 6) | (UInt32)(src[2] & 0x3F); + src += 3; + } else { + cp = '?'; + } + + if(cp >= 0x10000) { + cp -= 0x10000; + dst[out++] = (wchar_t)(0xD800 | (cp >> 10)); + dst[out++] = (wchar_t)(0xDC00 | (cp & 0x3FF)); + } else { + dst[out++] = (wchar_t)cp; + } + } + dst[out] = 0; +} + +static void path_mkdir(const char *path) { + wchar_t wide[MAX_PATH_LEN]; + utf8_to_utf16(path, wide, MAX_PATH_LEN); + _wmkdir(wide); +} + +static FILE *path_fopen_write(const char *path) { + wchar_t wide[MAX_PATH_LEN]; + utf8_to_utf16(path, wide, MAX_PATH_LEN); + return _wfopen(wide, L"wb"); +} + +#else + +static void path_mkdir(const char *path) { mkdir(path, 0755); } +static FILE *path_fopen_write(const char *path) { return fopen(path, "wb"); } + +#endif + +static void make_dirs(const char *path) { + char tmp[MAX_PATH_LEN]; + size_t i, n = strlen(path); + + if(n + 1 > sizeof(tmp)) return; + memcpy(tmp, path, n + 1); + + for(i = 1; i < n; i++) { + if(tmp[i] == '/' || tmp[i] == '\\') { + char c = tmp[i]; + tmp[i] = 0; + path_mkdir(tmp); + tmp[i] = c; + } + } + path_mkdir(tmp); +} + +/* UTF-16 (LE, as stored by the 7z name table) to UTF-8. */ +static void utf16_to_utf8(const UInt16 *src, char *dst, size_t dst_size) { + size_t out = 0; + + while(*src) { + UInt32 c = *src++; + + if(c >= 0xD800 && c <= 0xDBFF && *src >= 0xDC00 && *src <= 0xDFFF) + c = 0x10000 + ((c - 0xD800) << 10) + (*src++ - 0xDC00); + if(out + 5 >= dst_size) break; + + if(c < 0x80) { + dst[out++] = (char)c; + } else if(c < 0x800) { + dst[out++] = (char)(0xC0 | (c >> 6)); + dst[out++] = (char)(0x80 | (c & 0x3F)); + } else if(c < 0x10000) { + dst[out++] = (char)(0xE0 | (c >> 12)); + dst[out++] = (char)(0x80 | ((c >> 6) & 0x3F)); + dst[out++] = (char)(0x80 | (c & 0x3F)); + } else { + dst[out++] = (char)(0xF0 | (c >> 18)); + dst[out++] = (char)(0x80 | ((c >> 12) & 0x3F)); + dst[out++] = (char)(0x80 | ((c >> 6) & 0x3F)); + dst[out++] = (char)(0x80 | (c & 0x3F)); + } + } + dst[out] = 0; +} + +/* --------------------------------------------------------------- reader */ + +typedef struct { + CSzFile *file; + UInt64 base; /* db.dataPos: packed offsets are relative to it */ +} reader_ctx; + +static int reader_at(void *ctx, uint64_t offset, void *dst, size_t size) { + reader_ctx *r = (reader_ctx *)ctx; + UInt64 pos = r->base + offset; + size_t done = 0; + + if(File_Seek(r->file, (Int64 *)&pos, SZ_SEEK_SET) != 0) return -1; + while(done < size) { + size_t want = size - done; + if(File_Read(r->file, (Byte *)dst + done, &want) != 0) return -1; + if(want == 0) return -1; + done += want; + } + return 0; +} + +/* ------------------------------------------------------ folder -> sink */ + +typedef struct { + UInt32 file_index; + uint64_t size; +} plan_entry; + +typedef struct { + const CSzArEx *db; + char out_dir[MAX_PATH_LEN]; + char name[MAX_PATH_LEN]; + + plan_entry *plan; + size_t plan_len; + size_t plan_pos; + uint64_t written; + FILE *fh; + + uint32_t entry_crc; + uint64_t bytes_total; + uint64_t bytes_ok; + int failures; +} sink_ctx; + +static void plan_free(sink_ctx *s) { + free(s->plan); + s->plan = NULL; + s->plan_len = s->plan_pos = 0; +} + +/* Builds the ordered list of non-empty entries living in folder `folder`. */ +static int plan_build(sink_ctx *s, UInt32 folder, sz_chain_err_t *cerr) { + const CSzArEx *db = s->db; + UInt32 first = db->FolderToFile[folder]; + UInt32 last = db->FolderToFile[(size_t)folder + 1]; + UInt32 i; + + plan_free(s); + /* A folder that failed mid-entry leaves `written` pointing into an entry it + never finished; carrying that into the next folder makes every later entry + look "already part written" and produces a cascade of bogus failures. */ + s->written = 0; + if(last <= first) return 0; + s->plan = (plan_entry *)malloc(sizeof(plan_entry) * (size_t)(last - first)); + if(!s->plan) return -1; + + for(i = first; i < last; i++) { + UInt64 size = db->UnpackPositions[(size_t)i + 1] - db->UnpackPositions[i]; + if(db->FileToFolder[i] != folder) continue; + if(size == 0) continue; + s->plan[s->plan_len].file_index = i; + s->plan[s->plan_len].size = (uint64_t)size; + s->plan_len++; + } + return 0; +} + +static void sink_close_entry(sink_ctx *s) { + if(s->fh) { + fclose(s->fh); + s->fh = NULL; + } +} + +static int sink_open_entry(sink_ctx *s, UInt32 file_index) { + const CSzArEx *db = s->db; + UInt16 *name16 = NULL; + size_t len = SzArEx_GetFileNameUtf16(db, file_index, NULL); + char rel[MAX_PATH_LEN]; + char full[MAX_PATH_LEN]; + char *slash; + + name16 = (UInt16 *)malloc((len + 1) * sizeof(UInt16)); + if(!name16) return -1; + SzArEx_GetFileNameUtf16(db, file_index, name16); + utf16_to_utf8(name16, rel, sizeof(rel)); + free(name16); + + if(snprintf(full, sizeof(full), "%s/%s", s->out_dir, rel) >= + (int)sizeof(full)) { + printf(" FAIL (path too long) %s\n", rel); + return -1; + } + slash = strrchr(full, '/'); + if(slash) { + *slash = 0; + make_dirs(full); + *slash = '/'; + } + s->fh = path_fopen_write(full); + if(!s->fh) { + printf(" FAIL (cannot create) %s\n", rel); + return -1; + } + s->entry_crc = CRC_INIT_VAL; + return 0; +} + +static int sink_write(void *ctx, const void *data, size_t size) { + sink_ctx *s = (sink_ctx *)ctx; + const Byte *p = (const Byte *)data; + + while(size > 0) { + plan_entry *e; + uint64_t remain; + size_t take; + char rel[MAX_PATH_LEN]; + + if(s->plan_pos >= s->plan_len) { + printf(" FAIL folder produced %llu bytes more than its entries hold\n", + (unsigned long long)size); + s->failures++; + return -1; + } + e = &s->plan[s->plan_pos]; + remain = e->size - s->written; + take = (size_t)((uint64_t)size < remain ? (uint64_t)size : remain); + + if(!s->fh) { + if(sink_open_entry(s, e->file_index) != 0) { + s->failures++; + return -1; + } + } + if(take && fwrite(p, 1, take, s->fh) != take) { + printf(" FAIL (write error)\n"); + s->failures++; + return -1; + } + s->entry_crc = CrcUpdate(s->entry_crc, p, take); + s->written += take; + s->bytes_total += take; + p += take; + size -= take; + + if(s->written == e->size) { + UInt16 *name16; + size_t len; + sink_close_entry(s); + len = SzArEx_GetFileNameUtf16(s->db, e->file_index, NULL); + name16 = (UInt16 *)malloc((len + 1) * sizeof(UInt16)); + if(name16) { + SzArEx_GetFileNameUtf16(s->db, e->file_index, name16); + utf16_to_utf8(name16, rel, sizeof(rel)); + free(name16); + if(SzBitWithVals_Check(&s->db->CRCs, e->file_index) && + CRC_GET_DIGEST(s->entry_crc) != s->db->CRCs.Vals[e->file_index]) { + printf(" FAIL %s (crc mismatch)\n", rel); + s->failures++; + return -1; + } + } + s->bytes_ok += e->size; + s->plan_pos++; + s->written = 0; + } + } + return 0; +} + +/* ------------------------------------------------------------------ main */ + +int main(int argc, char **argv) { + CFileInStream archive_stream; + CLookToRead2 look_stream; + CSzArEx db; + SRes res; + UInt32 folder; + reader_ctx reader; + sink_ctx sink; + int rc = 0; + + if(argc < 3) { + fprintf(stderr, "usage: %s \n", argv[0]); + return 2; + } + + if(InFile_Open(&archive_stream.file, argv[1]) != 0) { + fprintf(stderr, "cannot open %s\n", argv[1]); + return 1; + } + FileInStream_CreateVTable(&archive_stream); + archive_stream.wres = 0; + + LookToRead2_CreateVTable(&look_stream, 0); + look_stream.buf = (Byte *)ISzAlloc_Alloc(&g_alloc, INPUT_BUF_SIZE); + if(!look_stream.buf) { + fprintf(stderr, "out of memory\n"); + return 1; + } + look_stream.bufSize = INPUT_BUF_SIZE; + look_stream.realStream = &archive_stream.vt; + LookToRead2_INIT(&look_stream) + + CrcGenerateTable(); + SzArEx_Init(&db); + + res = SzArEx_Open(&db, &look_stream.vt, &g_alloc, &g_temp); + if(res != SZ_OK) { + fprintf(stderr, "SzArEx_Open failed: res=%d\n", (int)res); + return 1; + } + + printf("entries: %u, folders: %u, packed streams: %u\n", + (unsigned)db.NumFiles, (unsigned)db.db.NumFolders, + (unsigned)db.db.NumPackStreams); + + memset(&sink, 0, sizeof(sink)); + sink.db = &db; + snprintf(sink.out_dir, sizeof(sink.out_dir), "%s", argv[2]); + + reader.file = &archive_stream.file; + reader.base = db.dataPos; + + for(folder = 0; folder < db.db.NumFolders; folder++) { + const UInt32 pack_first = db.db.FoStartPackStreamIndex[folder]; + const UInt32 pack_count = db.db.FoStartPackStreamIndex[(size_t)folder + 1] - + pack_first; + uint64_t pack_positions[SZ_CHAIN_MAX_STREAMS + 1]; + sz_chain *chain = NULL; + sz_chain_err_t cerr; + char desc[256]; + uint32_t folder_crc = 0; + UInt32 k; + uint64_t unpack_size = SzAr_GetFolderUnpackSize(&db.db, folder); + const uint8_t *blob = db.db.CodersData + db.db.FoCodersOffsets[folder]; + size_t blob_size = db.db.FoCodersOffsets[(size_t)folder + 1] - + db.db.FoCodersOffsets[folder]; + const uint64_t *cu = + &db.db.CoderUnpackSizes[db.db.FoToCoderUnpackSizes[folder]]; + + for(k = 0; k <= pack_count; k++) + pack_positions[k] = db.db.PackPositions[pack_first + k]; + + sink.bytes_total = 0; + sink.bytes_ok = 0; + if(plan_build(&sink, folder, &cerr) != 0) { + fprintf(stderr, "folder %u: cannot build the entry plan\n", + (unsigned)folder); + rc = 1; + continue; + } + + if(sz_chain_parse(&chain, blob, blob_size, pack_positions, pack_count, cu, + unpack_size, sz_chain_default_limits(), &cerr) != 0) { + printf("folder %-2u FAIL parse: %s: %s\n", (unsigned)folder, + sz_chain_status_string(cerr.status), cerr.message); + g_failures++; + rc = 1; + plan_free(&sink); + continue; + } + sz_chain_describe(chain, desc, sizeof(desc)); + + { + sz_chain_err_t derr; + if(sz_chain_decode(chain, reader_at, &reader, sink_write, &sink, NULL, + NULL, &folder_crc, &derr) != 0) { + printf("folder %-2u FAIL decode [%s]: %s: %s (offset %llu)\n", + (unsigned)folder, desc, sz_chain_status_string(derr.status), + derr.message, (unsigned long long)derr.offset); + g_failures++; + rc = 1; + } else if(sink.bytes_total != unpack_size) { + printf("folder %-2u FAIL size: %llu decoded, %llu declared\n", + (unsigned)folder, (unsigned long long)sink.bytes_total, + (unsigned long long)unpack_size); + g_failures++; + rc = 1; + } else if(SzBitWithVals_Check(&db.db.FolderCRCs, folder) && + folder_crc != db.db.FolderCRCs.Vals[folder]) { + printf("folder %-2u FAIL crc: got %08X, expected %08X\n", + (unsigned)folder, (unsigned)folder_crc, + (unsigned)db.db.FolderCRCs.Vals[folder]); + g_failures++; + rc = 1; + } else { + printf("folder %-2u ok [%s] %llu bytes, %u entr%s\n", + (unsigned)folder, desc, (unsigned long long)sink.bytes_total, + (unsigned)sink.plan_len, sink.plan_len == 1 ? "y" : "ies"); + } + } + + if(sink.plan_pos != sink.plan_len) { + printf("folder %-2u FAIL only %u of %u entries were produced\n", + (unsigned)folder, (unsigned)sink.plan_pos, + (unsigned)sink.plan_len); + g_failures++; + rc = 1; + } + sink_close_entry(&sink); + sz_chain_free(chain); + plan_free(&sink); + } + + ISzAlloc_Free(&g_alloc, look_stream.buf); + SzArEx_Free(&db, &g_alloc); + File_Close(&archive_stream.file); + + printf("%s: %d failure(s)\n", argv[1], g_failures); + return rc ? 1 : 0; +}