feat(7z): read multi-volume 7z sets as one stream

A split 7z (`name.7z.001`, `.002`, ...) only contains the first slice of the
archive, so SzArEx_Open() followed the header offsets off the end of the file
and gave up with SZ_ERROR_INPUT_EOF -- the set was unusable even though the
file listing of each part looked fine.

src/sevenz_volstream.c backs ISeekInStream with the ordered part list, so the
SDK sees one continuous archive and never learns it was split. Nothing is
merged on disk: a 160 GiB set would otherwise need a second 160 GiB scratch
copy. A 7z split is a plain byte split, so offsets stored inside the archive
are already absolute and the Stream/Seek callbacks stay trivial.

The ordered list comes from zipx_volume, which already understands
`name.7z.001` naming, so an incomplete set fails before any decoding starts
and names the missing part. Two further checks report a set that could not
have worked anyway: a lone first volume, and a part whose size differs from
the earlier ones (7-Zip cuts equal sized parts and lets only the last one be
short).

To reuse the split-layout constants without dragging the ZIP stream header
into the 7z build, ZIPX_VOL_MODE_* moved to zipx_volume.h: they describe the
set, and every consumer of zipx_volume_t needs them.

The e2e driver now opens its input through the volume stream, so the packed
data reads go through it too.

Matrix: all 10 single-volume fixtures plus vol.7z.001 are byte-identical
(13 passed, 0 failed). Remaining known gaps: 7zAES (aes, aeshe).
Regression: ZIP 108 / RAR 27 checks, 0 failures.
This commit is contained in:
Songlx516 committed 2026-09-12 17:53:17 +08:00
1 parent 2748b383bd
commit 9b2f5a07c3
7 files changed
+462 -23

No files matched your search

+356
View File
@@ -0,0 +1,356 @@
/* sevenz_volstream -- see sevenz_volstream.h for what this does and why. */
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#if defined(_WIN32)
#include <wchar.h>
#endif
#include "7zFile.h"
#include "sevenz_volstream.h"
#include "zipx_volume.h"
struct sevenz_volstream {
ISeekInStream vt;
zipx_volume_t vol; /* owns the ordered paths */
CSzFile *file; /* one per part */
uint64_t *start; /* count + 1 prefix offsets into the logical archive */
uint32_t count;
uint32_t open_count; /* how many entries of `file` were opened */
uint64_t pos; /* current offset in the logical archive */
uint32_t cur; /* part `pos` currently sits in, to skip redundant seeks */
uint64_t cur_off; /* file offset within that part */
char name[256]; /* stem of the set, for messages */
};
/* ---------------------------------------------------------------- helpers */
static char *err_printf(const char *fmt, ...) {
va_list ap;
char buf[512];
char *out;
va_start(ap, fmt);
vsnprintf(buf, sizeof(buf), fmt, ap);
va_end(ap);
out = (char *)malloc(strlen(buf) + 1);
if(out) memcpy(out, buf, strlen(buf) + 1);
return out;
}
static const char *file_base(const char *path) {
const char *slash = strrchr(path, '/');
const char *back = strrchr(path, '\\');
if(back && (!slash || back > slash)) slash = back;
return slash ? slash + 1 : path;
}
#if defined(_WIN32)
/* The SDK opens through CreateFileA otherwise, which cannot see non-ASCII
entry names. */
static void utf8_to_utf16(const char *src, WCHAR *dst, size_t cap) {
size_t out = 0;
while(*src && out + 2 < cap) {
unsigned char c = (unsigned char)*src++;
UInt32 cp;
if(c < 0x80) {
cp = c;
} else if((c & 0xE0) == 0xC0 && (src[0] & 0xC0) == 0x80) {
cp = ((UInt32)(c & 0x1F) << 6) | (UInt32)(*src++ & 0x3F);
} else if((c & 0xF0) == 0xE0 && (src[0] & 0xC0) == 0x80 &&
(src[1] & 0xC0) == 0x80) {
cp = ((UInt32)(c & 0x0F) << 12) | ((UInt32)(src[0] & 0x3F) << 6) |
(UInt32)(src[1] & 0x3F);
src += 2;
} else if((c & 0xF8) == 0xF0 && (src[0] & 0xC0) == 0x80 &&
(src[1] & 0xC0) == 0x80 && (src[2] & 0xC0) == 0x80) {
cp = ((UInt32)(c & 0x07) << 18) | ((UInt32)(src[0] & 0x3F) << 12) |
((UInt32)(src[1] & 0x3F) << 6) | (UInt32)(src[2] & 0x3F);
src += 3;
} else {
cp = '?';
}
if(cp >= 0x10000) {
cp -= 0x10000;
dst[out++] = (WCHAR)(0xD800 | (cp >> 10));
dst[out++] = (WCHAR)(0xDC00 | (cp & 0x3FF));
} else {
dst[out++] = (WCHAR)cp;
}
}
dst[out] = 0;
}
static int open_part(CSzFile *file, const char *path) {
WCHAR wide[4096];
utf8_to_utf16(path, wide, sizeof(wide) / sizeof(wide[0]));
return InFile_OpenW(file, wide) == 0 ? 0 : -1;
}
#else
static int open_part(CSzFile *file, const char *path) {
return InFile_Open(file, path) == 0 ? 0 : -1;
}
#endif
/* ----------------------------------------------------------------- stream */
static uint32_t part_at(const sevenz_volstream *v, uint64_t pos) {
uint32_t i;
for(i = 0; i < v->count; i++) {
if(pos < v->start[i + 1]) return i;
}
return v->count;
}
static SRes vol_read(const ISeekInStream *p, void *buf, size_t *size) {
sevenz_volstream *v = (sevenz_volstream *)p;
uint8_t *dst = (uint8_t *)buf;
size_t want = *size;
size_t got = 0;
*size = 0;
while(got < want) {
uint32_t i = part_at(v, v->pos);
uint64_t avail, off;
size_t take;
if(i >= v->count) break; /* end of the set: a short read means EOF */
off = v->pos - v->start[i];
avail = (v->start[i + 1] - v->start[i]) - off;
take = (size_t)(avail < (uint64_t)(want - got) ? avail
: (uint64_t)(want - got));
if(take == 0) break;
if(v->cur != i || v->cur_off != off) {
Int64 seek = (Int64)off;
if(File_Seek(&v->file[i], &seek, SZ_SEEK_SET) != 0) return SZ_ERROR_READ;
v->cur = i;
v->cur_off = off;
}
{
size_t part_got = take;
if(File_Read(&v->file[i], dst + got, &part_got) != 0) return SZ_ERROR_READ;
if(part_got == 0) break;
got += part_got;
v->pos += part_got;
v->cur_off += part_got;
if(part_got < take) break;
}
}
*size = got;
return SZ_OK;
}
static SRes vol_seek(const ISeekInStream *p, Int64 *pos, ESzSeek origin) {
sevenz_volstream *v = (sevenz_volstream *)p;
uint64_t total = v->start[v->count];
uint64_t target;
switch(origin) {
case SZ_SEEK_SET:
if(*pos < 0) return SZ_ERROR_PARAM;
target = (uint64_t)*pos;
break;
case SZ_SEEK_CUR:
if(*pos < 0) {
UInt64 back = (UInt64)(-*pos);
if(back > v->pos) return SZ_ERROR_PARAM;
target = v->pos - back;
} else {
target = v->pos + (UInt64)*pos;
}
break;
case SZ_SEEK_END:
if(*pos < 0) {
UInt64 back = (UInt64)(-*pos);
if(back > total) return SZ_ERROR_PARAM;
target = total - back;
} else {
target = total + (UInt64)*pos;
}
break;
default:
return SZ_ERROR_PARAM;
}
if(target > total) target = total;
v->pos = target;
*pos = (Int64)target;
return SZ_OK;
}
/* -------------------------------------------------------------------- open */
int sevenz_volstream_open(sevenz_volstream **out, const char *path, int *is_set,
char **err) {
sevenz_volstream *v;
char *vol_err = NULL;
int detected;
uint32_t i;
if(out) *out = NULL;
if(is_set) *is_set = 0;
if(err) *err = NULL;
if(!out || !path) {
if(err) *err = err_printf("no archive path given");
return -1;
}
v = (sevenz_volstream *)calloc(1, sizeof(*v));
if(!v) {
if(err) *err = err_printf("out of memory");
return -1;
}
/* One call does both jobs: it either reports "ordinary file" and clears the
struct, or fills in the ordered part list. A -1 here already carries the
message the user needs (a hole in the numbering names the missing part). */
detected = zipx_volume_detect(path, &v->vol, &vol_err);
if(detected < 0) {
if(err) {
*err = vol_err ? vol_err
: err_printf("'%s' cannot be read", file_base(path));
} else {
free(vol_err);
}
zipx_volume_free(&v->vol);
free(v);
return -1;
}
if(detected == 0) {
v->vol.paths = (char **)malloc(sizeof(char *));
if(v->vol.paths) v->vol.paths[0] = (char *)malloc(strlen(path) + 1);
if(!v->vol.paths || !v->vol.paths[0]) {
free(v->vol.paths);
free(v);
if(err) *err = err_printf("out of memory");
return -1;
}
memcpy(v->vol.paths[0], path, strlen(path) + 1);
v->vol.count = 1;
v->vol.mode = ZIPX_VOL_MODE_CONCAT;
v->vol.is_set = 0;
} else if(is_set) {
*is_set = 1;
}
snprintf(v->name, sizeof(v->name), "%s", file_base(path));
v->count = (uint32_t)v->vol.count;
v->file = (CSzFile *)calloc(v->count, sizeof(CSzFile));
v->start = (uint64_t *)calloc((size_t)v->count + 1, sizeof(uint64_t));
if(!v->file || !v->start) {
if(err) *err = err_printf("out of memory");
goto fail;
}
for(i = 0; i < v->count; i++) {
UInt64 length = 0;
File_Construct(&v->file[i]);
v->open_count = i + 1; /* File_Close() ignores a never-opened handle */
if(open_part(&v->file[i], v->vol.paths[i]) != 0) {
if(err) {
*err = err_printf("cannot open volume '%s' of '%s'",
file_base(v->vol.paths[i]), v->name);
}
goto fail;
}
if(File_GetLength(&v->file[i], &length) != 0) {
if(err) {
*err = err_printf("cannot measure volume '%s' of '%s'",
file_base(v->vol.paths[i]), v->name);
}
goto fail;
}
v->start[i + 1] = v->start[i] + length;
}
if(v->start[v->count] == 0) {
if(err) *err = err_printf("'%s' is empty", v->name);
goto fail;
}
/* 7-Zip cuts equal sized parts and lets only the last one be short. A part
of a different size in the middle means the set is damaged or was mixed
with another one, and decoding would fail much later with a message that
points nowhere useful. */
for(i = 0; i + 1 < v->count; i++) {
uint64_t size = v->start[i + 1] - v->start[i];
if(size != v->start[1]) {
if(err) {
*err = err_printf("volume '%s' of '%s' is %llu bytes, but the earlier "
"volumes are %llu bytes: the set is not a clean split",
file_base(v->vol.paths[i]), v->name,
(unsigned long long)size,
(unsigned long long)v->start[1]);
}
goto fail;
}
}
v->vt.Read = vol_read;
v->vt.Seek = vol_seek;
*out = v;
return 0;
fail:
sevenz_volstream_free(v);
return -1;
}
ISeekInStream *sevenz_volstream_stream(sevenz_volstream *v) {
return v ? &v->vt : NULL;
}
uint32_t sevenz_volstream_count(const sevenz_volstream *v) {
return v ? v->count : 0;
}
uint64_t sevenz_volstream_size(const sevenz_volstream *v) {
return v ? v->start[v->count] : 0;
}
const char *sevenz_volstream_describe(const sevenz_volstream *v, char *buf,
unsigned size) {
if(!buf || size == 0) return buf;
if(!v) {
snprintf(buf, size, "no archive");
} else if(v->count <= 1) {
snprintf(buf, size, "%s", v->name);
} else {
snprintf(buf, size, "%s (%u volumes)", v->name, (unsigned)v->count);
}
return buf;
}
void sevenz_volstream_free(sevenz_volstream *v) {
uint32_t i;
if(!v) return;
for(i = 0; i < v->open_count; i++) File_Close(&v->file[i]);
free(v->file);
free(v->start);
zipx_volume_free(&v->vol);
free(v);
}
+59
View File
@@ -0,0 +1,59 @@
/* sevenz_volstream -- present a multi-file 7z volume set as one stream.
part of ps5-web-file-manager
A split 7z is a plain byte split: `name.7z.001`, `name.7z.002`, ... are
consecutive slices of one archive, so byte N of the logical archive is byte
N of the concatenation and every offset stored inside the stream header is
already absolute. Nothing has to be merged on disk -- a 160 GiB set would
otherwise need a second 160 GiB scratch copy.
The LZMA SDK reads through ISeekInStream, so this module implements that
interface over the ordered part list produced by zipx_volume. The ordered
list is what makes a set with a hole in it fail loudly instead of decoding
garbage: zipx_volume names the missing part. */
#ifndef SEVENZ_VOLSTREAM_H
#define SEVENZ_VOLSTREAM_H
#include <stdint.h>
#include "7zTypes.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct sevenz_volstream sevenz_volstream;
/* Opens `path` together with every volume of the set it belongs to and exposes
them as one seekable byte stream. `path` may be any member of the set; an
ordinary single-file archive is the degenerate one-file case.
Returns 0 on success, with *is_set set to 1 when the path was part of a
multi-file set (non-NULL only). Returns -1 on failure and, when `err` is
non-NULL, stores a malloc'd message the caller must free -- an incomplete
set reports the missing volume by name. */
int sevenz_volstream_open(sevenz_volstream **out, const char *path, int *is_set,
char **err);
/* The stream to hand to SzArEx_Open(); valid until sevenz_volstream_free(). */
ISeekInStream *sevenz_volstream_stream(sevenz_volstream *v);
/* Number of files backing the stream (1 for an ordinary archive). */
uint32_t sevenz_volstream_count(const sevenz_volstream *v);
/* Size of the whole logical archive. */
uint64_t sevenz_volstream_size(const sevenz_volstream *v);
/* Human readable description, e.g. "name.7z (3 volumes)"; writes into buf and
returns buf. */
const char *sevenz_volstream_describe(const sevenz_volstream *v, char *buf,
unsigned size);
void sevenz_volstream_free(sevenz_volstream *v);
#ifdef __cplusplus
}
#endif
#endif
+2 -3
View File
@@ -25,13 +25,12 @@
#include <stdint.h>
#include "zipx_volume.h"
#ifdef __cplusplus
extern "C" {
#endif
#define ZIPX_VOL_MODE_CONCAT 0
#define ZIPX_VOL_MODE_DISK 1
/* Creates a stream handle; pass it to mz_stream_open() afterwards. */
void *zipx_volstream_create(int32_t mode);
-1
View File
@@ -9,7 +9,6 @@
#include <strings.h>
#include <sys/stat.h>
#include "zipx_volstream.h"
#include "zipx_volume.h"
typedef struct {
+8
View File
@@ -22,6 +22,14 @@ extern "C" {
#define ZIPX_VOL_MAX_PARTS 512
/* How the set is split. A byte split (`name.zip.001`, `name.7z.001`,
`name.part1.zip`) is the concatenation of its parts with absolute offsets;
zip split disks (`name.z01` + `name.zip`) store per-disk offsets instead.
The constants live here because they describe the *set*, and every consumer
of zipx_volume_t needs them. */
#define ZIPX_VOL_MODE_CONCAT 0
#define ZIPX_VOL_MODE_DISK 1
typedef struct {
char **paths; /* ordered part paths, owned by this struct */
int count;
+15 -5
View File
@@ -23,10 +23,9 @@ CC="${CC:-gcc}"
# Archives the engine cannot read yet. Each entry needs a reason; when one of
# them starts passing the script says so, so the list cannot rot.
KNOWN_GAPS="aes aeshe vol.7z.001"
KNOWN_GAPS="aes aeshe"
# aes - 7zAES coder not implemented yet
# aeshe - encrypted header (-mhe=on), same coder
# vol.7z.001 - multi-volume input; needs the volume-set stream
find "$BUILD" -maxdepth 1 -type f \( -name '*.o' -o -name '*.exe' \) -delete 2>/dev/null || true
mkdir -p "$BUILD"
@@ -42,9 +41,20 @@ for src in "$SEVENZ_DIR"/*.c; do
VENDOR_OBJS+=("$BUILD/$name.o")
done
# The engine module is held to the same strictness as the rest of src/.
# The engine modules are held to the same strictness as the rest of src/.
# sevenz_volstream reuses the ZIP side's volume-set detector, so zipx_volume
# is built here as well.
"$CC" -c -O2 -Wall -Wextra -Werror -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE \
-I"$SEVENZ_DIR" -o "$BUILD/sevenz_chain.o" "$ROOT/src/sevenz_chain.c"
-I"$SEVENZ_DIR" -I"$ROOT/src" -o "$BUILD/sevenz_chain.o" \
"$ROOT/src/sevenz_chain.c"
"$CC" -c -O2 -Wall -Wextra -Werror -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE \
-I"$SEVENZ_DIR" -I"$ROOT/src" -o "$BUILD/sevenz_volstream.o" \
"$ROOT/src/sevenz_volstream.c"
"$CC" -c -O2 -Wall -Wextra -Werror -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE \
-I"$ROOT/src" -o "$BUILD/zipx_volume.o" "$ROOT/src/zipx_volume.c"
ENGINE_OBJS=("$BUILD/sevenz_chain.o" "$BUILD/sevenz_volstream.o"
"$BUILD/zipx_volume.o")
# unrar-style extra libs are only needed by the Windows path of 7zFile.c.
EXTRA_LIBS=()
@@ -53,7 +63,7 @@ case "$(uname -s)" in
esac
"$CC" -O2 -w -I"$SEVENZ_DIR" -I"$ROOT/src" -o "$BUILD/sevenz_chain_e2e" \
"$ROOT/tests/sevenz_chain_e2e.c" "$BUILD/sevenz_chain.o" "${VENDOR_OBJS[@]}" \
"$ROOT/tests/sevenz_chain_e2e.c" "${ENGINE_OBJS[@]}" "${VENDOR_OBJS[@]}" \
"${EXTRA_LIBS[@]}"
# The SDK-baseline driver is kept buildable: it is the fastest way to tell an
+22 -14
View File
@@ -33,6 +33,7 @@
#include "7zFile.h"
#include "sevenz_chain.h"
#include "sevenz_volstream.h"
#define INPUT_BUF_SIZE (1u << 18)
#define MAX_PATH_LEN 4096
@@ -152,19 +153,19 @@ static void utf16_to_utf8(const UInt16 *src, char *dst, size_t dst_size) {
/* --------------------------------------------------------------- reader */
typedef struct {
CSzFile *file;
ISeekInStream *stream;
UInt64 base; /* db.dataPos: packed offsets are relative to it */
} reader_ctx;
static int reader_at(void *ctx, uint64_t offset, void *dst, size_t size) {
reader_ctx *r = (reader_ctx *)ctx;
UInt64 pos = r->base + offset;
Int64 pos = (Int64)(r->base + offset);
size_t done = 0;
if(File_Seek(r->file, (Int64 *)&pos, SZ_SEEK_SET) != 0) return -1;
if(r->stream->Seek(r->stream, &pos, SZ_SEEK_SET) != SZ_OK) return -1;
while(done < size) {
size_t want = size - done;
if(File_Read(r->file, (Byte *)dst + done, &want) != 0) return -1;
if(r->stream->Read(r->stream, (Byte *)dst + done, &want) != SZ_OK) return -1;
if(want == 0) return -1;
done += want;
}
@@ -334,13 +335,16 @@ static int sink_write(void *ctx, const void *data, size_t size) {
/* ------------------------------------------------------------------ main */
int main(int argc, char **argv) {
CFileInStream archive_stream;
CLookToRead2 look_stream;
CSzArEx db;
SRes res;
UInt32 folder;
reader_ctx reader;
sink_ctx sink;
sevenz_volstream *vol = NULL;
char *vol_err = NULL;
char vol_desc[512];
int is_set = 0;
int rc = 0;
if(argc < 3) {
@@ -348,21 +352,22 @@ int main(int argc, char **argv) {
return 2;
}
if(InFile_Open(&archive_stream.file, argv[1]) != 0) {
fprintf(stderr, "cannot open %s\n", argv[1]);
if(sevenz_volstream_open(&vol, argv[1], &is_set, &vol_err) != 0) {
fprintf(stderr, "%s\n", vol_err ? vol_err : "cannot open the archive");
free(vol_err);
return 1;
}
FileInStream_CreateVTable(&archive_stream);
archive_stream.wres = 0;
sevenz_volstream_describe(vol, vol_desc, sizeof(vol_desc));
LookToRead2_CreateVTable(&look_stream, 0);
look_stream.buf = (Byte *)ISzAlloc_Alloc(&g_alloc, INPUT_BUF_SIZE);
if(!look_stream.buf) {
fprintf(stderr, "out of memory\n");
sevenz_volstream_free(vol);
return 1;
}
look_stream.bufSize = INPUT_BUF_SIZE;
look_stream.realStream = &archive_stream.vt;
look_stream.realStream = sevenz_volstream_stream(vol);
LookToRead2_INIT(&look_stream)
CrcGenerateTable();
@@ -370,11 +375,14 @@ int main(int argc, char **argv) {
res = SzArEx_Open(&db, &look_stream.vt, &g_alloc, &g_temp);
if(res != SZ_OK) {
fprintf(stderr, "SzArEx_Open failed: res=%d\n", (int)res);
fprintf(stderr, "%s: cannot read the 7z header (res=%d)\n", vol_desc,
(int)res);
sevenz_volstream_free(vol);
return 1;
}
printf("entries: %u, folders: %u, packed streams: %u\n",
printf("archive: %s, %llu bytes, entries: %u, folders: %u, packed streams: %u\n",
vol_desc, (unsigned long long)sevenz_volstream_size(vol),
(unsigned)db.NumFiles, (unsigned)db.db.NumFolders,
(unsigned)db.db.NumPackStreams);
@@ -382,7 +390,7 @@ int main(int argc, char **argv) {
sink.db = &db;
snprintf(sink.out_dir, sizeof(sink.out_dir), "%s", argv[2]);
reader.file = &archive_stream.file;
reader.stream = sevenz_volstream_stream(vol);
reader.base = db.dataPos;
for(folder = 0; folder < db.db.NumFolders; folder++) {
@@ -468,7 +476,7 @@ int main(int argc, char **argv) {
ISzAlloc_Free(&g_alloc, look_stream.buf);
SzArEx_Free(&db, &g_alloc);
File_Close(&archive_stream.file);
sevenz_volstream_free(vol);
printf("%s: %d failure(s)\n", argv[1], g_failures);
return rc ? 1 : 0;