From 6bb957f2cab44f207d7d3c8e84d0910d9c9f723a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Sat, 17 Oct 2026 16:51:30 -0300 Subject: [PATCH] feat(vfs): add runtime libarchive support and eefm extraction - Add a read-only ArchiveProvider with optional runtime-loaded libarchive 3.x - Stream archive entries with bounded indexes and backing-file revision checks - Expose available formats and filters without a build-time dependency - Extend eefm archive browsing and add extraction context-menu actions - Merge extraction directories while requiring confirmation before overwriting files - Add archive, extraction, naming and transfer-conflict regression tests --- docs/articles/vfs.md | 77 ++ src/modules/vfs/include/eepp/vfs.hpp | 1 + src/modules/vfs/include/eepp/vfs/options.hpp | 2 + .../eepp/vfs/providers/archiveprovider.hpp | 83 +++ .../include/eepp/vfs/virtualfilesystem.hpp | 3 +- .../vfs/src/eepp/vfs/archive/archiveinput.cpp | 111 +++ .../vfs/src/eepp/vfs/archive/archiveinput.hpp | 48 ++ .../vfs/src/eepp/vfs/archive/libarchive.cpp | 293 ++++++++ .../vfs/src/eepp/vfs/archive/libarchive.hpp | 137 ++++ .../src/eepp/vfs/detail/archivelocation.hpp | 108 +++ .../eepp/vfs/providers/archiveprovider.cpp | 697 ++++++++++++++++++ .../src/eepp/vfs/providers/zipprovider.cpp | 103 +-- .../vfs/src/eepp/vfs/transferengine.cpp | 3 +- .../vfs/src/eepp/vfs/virtualfilesystem.cpp | 2 + .../unit_tests/archiveprovider_tests.cpp | 553 ++++++++++++++ src/tests/unit_tests/vfs_tests.cpp | 48 +- src/tests/unit_tests/zipprovider_tests.cpp | 46 ++ src/tools/eefm/fileoperations.cpp | 96 ++- src/tools/eefm/fileoperations.hpp | 6 +- src/tools/eefm/location.cpp | 61 +- src/tools/eefm/location.hpp | 7 +- 21 files changed, 2352 insertions(+), 133 deletions(-) create mode 100644 src/modules/vfs/include/eepp/vfs/providers/archiveprovider.hpp create mode 100644 src/modules/vfs/src/eepp/vfs/archive/archiveinput.cpp create mode 100644 src/modules/vfs/src/eepp/vfs/archive/archiveinput.hpp create mode 100644 src/modules/vfs/src/eepp/vfs/archive/libarchive.cpp create mode 100644 src/modules/vfs/src/eepp/vfs/archive/libarchive.hpp create mode 100644 src/modules/vfs/src/eepp/vfs/detail/archivelocation.hpp create mode 100644 src/modules/vfs/src/eepp/vfs/providers/archiveprovider.cpp create mode 100644 src/tests/unit_tests/archiveprovider_tests.cpp diff --git a/docs/articles/vfs.md b/docs/articles/vfs.md index 924f3c586..65cda6a52 100644 --- a/docs/articles/vfs.md +++ b/docs/articles/vfs.md @@ -9,6 +9,7 @@ for local files, packed resources and remote servers: | --- | --- | --- | | `file://` | `LocalProvider` | Native filesystem (POSIX and Win32). | | `zip://` | `ZipProvider` | Read-only browsing and streaming reads of individual local ZIP archives. | +| `archive://` | `ArchiveProvider` | Read-only local archives through optional runtime-loaded libarchive 3.x. | | `res://` | `ResourceProvider` | Read-only view of every open `Pack` (`System::PackFileIndex`). | | `sftp://` | `SFTPProvider` | Built by default (`--without-vfs-sftp` disables it), libssh2 based, honors `~/.ssh/config`. | | `mem://` | `MemoryProvider` | In-memory filesystem, mainly for tests. | @@ -171,6 +172,11 @@ when a destination appears while the copy is running. The `Overwrite` policy req Creating a file with `Fail` or `Skip` requires `Rename`. Moving a symlink through the copy fallback is unsupported; native same-filesystem renames still move the link itself. +`MergeDirectories` merges existing destination directories but reports `AlreadyExists` for files, +including files that appear during the copy. It supports copying an archive root into an existing +folder without overwriting files. The default `Fail` policy also rejects existing directories; +`Overwrite` and `Skip` merge directories and apply their respective policies to files. + Native renames also preserve an existing destination when safe replacement is unavailable. POSIX systems without an atomic no-replace primitive return `Unsupported` for no-overwrite renames, rather than using an existence check followed by a replacing rename. @@ -344,6 +350,77 @@ callers sharing an archive across threads must serialize access. `open()` never file; writing starts explicitly with `create()`. The default constructor retains resource-pack registration for the engine and other existing pack consumers. +## Runtime archives (`archive://`) + +`ArchiveProvider` is registered by default but loads libarchive only when used, on an I/O worker. +It requires a compatible libarchive **3.x** shared library, with no libarchive development headers +or build-time link dependency. Missing libraries, unsupported ABI versions and missing required +symbols return `Unsupported`; the rest of the VFS, including bundled ZIP support, remains available. +Emscripten and iOS have no runtime backend. + +```cpp +URI backing = URI::fromFSPath( "/home/me/downloads/source.tar.gz" ); +URI root = ArchiveProvider::makeURI( backing ); +vfs.listAll( root ); +vfs.copy( root.withAppendedSegment( "source" ), URI::fromFSPath( "/home/me/source" ) ); + +auto provider = std::dynamic_pointer_cast( vfs.provider( "archive" ) ); +provider->support().then( ui, []( VFSResult&& support ) { + // version, formats and compression filters from the installed runtime library. +} ); +``` + +Locations use `archive:///path/source.tar.gz!/member`, with the same bang escaping, member path +rules and archive-root navigation as `zip://`. Only local `file://` backing archives are accepted. +An optional constructor `libraryPath` selects an exact library; otherwise the provider tries the +platform's usual libarchive names (including `/usr/lib/libarchive.2.dylib` on macOS). + +Supported readers are enabled individually: TAR (including GNU and PAX variants), CPIO, ar, 7z, +RAR/RAR5, ISO9660, CAB, LHA, XAR and ZIP, when supplied by the loaded library. Compression filters +include gzip, bzip2, xz, LZMA, lzip, zstd and lz4 when built into that library. `support()` reports +the enabled set. External decompression programs are never enabled. Recognition does not guarantee +every entry can be decoded: encrypted or unsupported entry data and detected corruption report +errors when read. Integrity checks follow the installed libarchive; there is no extra checksum pass. +In particular, libarchive 3.8.2's gzip reader does not verify the gzip trailer CRC. + +`eefm` recognizes `.tar`, `.tar.gz`, `.tgz`, other common compressed TAR suffixes, `.7z`, `.rar`, +`.iso`, `.cab`, `.cpio`, `.ar`, `.deb`, `.xar` and `.lha`/`.lzh` as browsing candidates. Extensions +are case-insensitive; the archive contents determine whether opening succeeds. A `.deb` opens its +outer ar container; nested archives currently need to be copied out before browsing. ZIP-based +formats continue to use the bundled `ZipProvider`. + +Right-click a single recognized local archive in `eefm` for the **Extract** submenu. The first +action extracts into a folder named after the archive (for example `package.tar.gz` into +`package/`); **Extract Here** uses its containing folder. **Extract To...** accepts a destination +folder path or URI, including writable SFTP locations. Extraction uses VFS transfers with progress, +cancellation and confirmation before overwriting existing files. Hidden entries are included. +It follows VFS copy semantics: links to files are copied as contents, while directory symlinks and +unsupported special entries are skipped and reported as a partial transfer. Links are not recreated. + +The provider is read-only; moves out fail before copying because source removal is unavailable. +Indexing scans headers and skips entry data without extracting it into memory. Compressed archives +may require decompression during indexing. Each opened file has an independent streaming reader; +opening a late entry or seeking backwards can replay the archive prefix, so repeated random reads +can be expensive. For plain and gzip TAR, an explicit file close returns one idle reader to its +index: opening later entries can continue forward, avoiding repeated prefix decompression during +sequential directory copies. Other codecs and formats are released on close, so large decoder +windows or solid blocks are not retained by the index. `Seek` is supported, `RandomAccess` is not. +Idle readers close their native files; reuse reopens at the saved input position and checks the +backing revision again. Cached decoder state therefore does not block deleting or replacing the +archive on Windows. Data is not cached as extracted files. + +Symbolic and hard links resolve only inside the archive, with a 40-link limit; native filesystem +targets are never followed. Device nodes and FIFOs can be listed but cannot be opened as files. +Unsafe member names are excluded using the same checks as ZIP. Indexes have a 262,144-entry limit +and a conservative 64 MiB metadata budget, including implicit directories and stored paths, with +at most eight cached indexes. These limits cover the provider's index, not libarchive's own decoder +working memory. Open handles can retain older indexes. + +Backing-file revisions invalidate cached indexes for new operations. Existing readers retain their +opened file and library; reopening for a backward seek after replacement reports `ChangedExternally`. +In-place modifications may cause read errors. Network mounts use the local scheduler's isolated +lanes and buffer-safe timeouts, and archive changes can be watched through the VFS polling backend. + ## Resources (`res://`) `System::PackFileIndex` (formerly `System::VirtualFileSystem`) indexes the files of every open `Pack`. `res:///fonts/a.ttf` maps to the pack path diff --git a/src/modules/vfs/include/eepp/vfs.hpp b/src/modules/vfs/include/eepp/vfs.hpp index 0ef053f1a..caafc0dae 100644 --- a/src/modules/vfs/include/eepp/vfs.hpp +++ b/src/modules/vfs/include/eepp/vfs.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/src/modules/vfs/include/eepp/vfs/options.hpp b/src/modules/vfs/include/eepp/vfs/options.hpp index 5e3dfd694..f4723df36 100644 --- a/src/modules/vfs/include/eepp/vfs/options.hpp +++ b/src/modules/vfs/include/eepp/vfs/options.hpp @@ -100,6 +100,8 @@ enum class OverwritePolicy : Uint8 { Overwrite, /** Leave existing destination files untouched and continue. */ Skip, + /** Merge existing directories, but fail with AlreadyExists for existing files. */ + MergeDirectories, }; struct EE_VFS_API CopyOptions { diff --git a/src/modules/vfs/include/eepp/vfs/providers/archiveprovider.hpp b/src/modules/vfs/include/eepp/vfs/providers/archiveprovider.hpp new file mode 100644 index 000000000..4026580dd --- /dev/null +++ b/src/modules/vfs/include/eepp/vfs/providers/archiveprovider.hpp @@ -0,0 +1,83 @@ +#ifndef EE_VFS_PROVIDERS_ARCHIVEPROVIDER_HPP +#define EE_VFS_PROVIDERS_ARCHIVEPROVIDER_HPP + +#include +#include +#include +#include +#include +#include + +namespace EE { namespace VFS { + +class ArchiveProviderImpl; + +/** Runtime capabilities of the installed libarchive 3.x library. Only built-in decoders are + * enabled: the provider never delegates decompression to external commands. */ +struct EE_VFS_API ArchiveSupport { + std::string version; + std::vector formats; + std::vector filters; +}; + +/** Read-only archive:// filesystem backed by optional runtime-loaded libarchive 3.x. + * No libarchive headers or link dependency are required. Locations have the form + * archive:///path/archive.tar.gz!/member (use makeURI()). Local archives only. + * Indexing and reads run through LocalIOScheduler, including mount isolation and safe timeouts. + * Entry data streams with bounded buffers. Backward seeks reopen and replay the archive; + * RandomAccess is not advertised. Readers retain the runtime library until they are closed. + * Missing/incompatible libraries return Unsupported. Emscripten has no runtime backend. */ +class EE_VFS_API ArchiveProvider : public VFSProvider { + public: + /** libraryPath optionally selects an exact runtime library; empty uses platform defaults. + * Loading happens lazily on an I/O thread, never in the constructor. */ + explicit ArchiveProvider( std::shared_ptr scheduler, + std::string libraryPath = {} ); + + ~ArchiveProvider() override; + + /** Builds an archive location. member is an unescaped, '/'-separated path inside it. Only + * file:// backing archives are currently supported; other inputs return an empty URI. + * Root is "/". */ + static URI makeURI( const URI& archive, std::string_view member = "/" ); + + /** Decodes and validates the backing file:// URI of an archive:// location. */ + static VFSResult archiveURI( const URI& uri ); + + /** Validates an archive location and returns its normalized path within the archive. */ + static VFSResult memberPath( const URI& uri ); + + std::string_view scheme() const override { return "archive"; } + + /** Reports the actual installed library's capabilities, or Unsupported when unavailable. */ + VFSTask support( CancellationToken token = {} ); + + ProviderCapabilities capabilities( const URI& uri ) const override; + + VFSTask stat( const URI& uri, const StatOptions& options, + CancellationToken token ) override; + + VFSTask openDirectory( const URI& uri, const DirectoryOpenOptions& options, + CancellationToken token ) override; + + VFSTask open( const URI& uri, const OpenOptions& options, + CancellationToken token ) override; + + VFSTask mkdir( const URI& uri, const MkDirOptions& options, + CancellationToken token ) override; + + VFSTask remove( const URI& uri, const RemoveOptions& options, + CancellationToken token ) override; + + VFSTask rename( const URI& from, const URI& to, const RenameOptions& options, + CancellationToken token ) override; + + VFSTask canonicalize( const URI& uri, CancellationToken token ) override; + + private: + std::shared_ptr mImpl; +}; + +}} // namespace EE::VFS + +#endif diff --git a/src/modules/vfs/include/eepp/vfs/virtualfilesystem.hpp b/src/modules/vfs/include/eepp/vfs/virtualfilesystem.hpp index be1c751b6..4872f0872 100644 --- a/src/modules/vfs/include/eepp/vfs/virtualfilesystem.hpp +++ b/src/modules/vfs/include/eepp/vfs/virtualfilesystem.hpp @@ -46,7 +46,8 @@ class EE_VFS_API VirtualFileSystem { * counts as not responding: the call fails with TimedOut, and so do further calls until * the server answers again (see LocalIOScheduler). */ System::Time networkMountTimeout{ System::Seconds( 15 ) }; - /** Register file://, zip:// and res:// (and sftp:// when built with SFTP support). */ + /** Register file://, zip://, archive:// and res:// (and sftp:// when built with SFTP + * support). */ bool registerDefaultProviders{ true }; }; diff --git a/src/modules/vfs/src/eepp/vfs/archive/archiveinput.cpp b/src/modules/vfs/src/eepp/vfs/archive/archiveinput.cpp new file mode 100644 index 000000000..377c91e5f --- /dev/null +++ b/src/modules/vfs/src/eepp/vfs/archive/archiveinput.cpp @@ -0,0 +1,111 @@ +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 64 +#endif +#include "archiveinput.hpp" +#include +#include +#include +#if EE_PLATFORM == EE_PLATFORM_WIN +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#include +#endif + +namespace EE { namespace VFS { namespace detail { + +ArchiveInput::ArchiveInput( const std::string& path ) : mPath( path ) { + if ( !openFile() ) + return; +#if EE_PLATFORM == EE_PLATFORM_WIN + int result = _fseeki64( mFile, 0, SEEK_END ); +#else + int result = ::fseeko( mFile, 0, SEEK_END ); +#endif + mSize = result == 0 ? tell() : -1; + if ( mSize < 0 || seek( 0 ) < 0 ) { + std::fclose( mFile ); + mFile = nullptr; + } +} + +bool ArchiveInput::openFile() { +#if EE_PLATFORM == EE_PLATFORM_WIN + HANDLE handle = + CreateFileW( String::fromUtf8( mPath ).toWideString().c_str(), GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, nullptr ); + if ( handle == INVALID_HANDLE_VALUE ) + return false; + int fd = _open_osfhandle( reinterpret_cast( handle ), _O_RDONLY | _O_BINARY ); + if ( fd < 0 ) { + CloseHandle( handle ); + return false; + } + mFile = _fdopen( fd, "rb" ); + if ( !mFile ) + _close( fd ); +#else + mFile = System::FileSystem::fopenUtf8( mPath, "rb" ); +#endif + return mFile != nullptr; +} + +bool ArchiveInput::suspend() { + if ( !mFile ) + return false; + mPosition = tell(); + int result = std::fclose( mFile ); + mFile = nullptr; + return result == 0 && mPosition >= 0; +} + +bool ArchiveInput::resume() { + if ( mFile ) + return true; + if ( !openFile() ) + return false; + if ( seek( mPosition ) < 0 ) { + std::fclose( mFile ); + mFile = nullptr; + return false; + } + return true; +} + +ArchiveInput::~ArchiveInput() { + if ( mFile ) + std::fclose( mFile ); +} + +std::intptr_t ArchiveInput::read( char* data, size_t size ) { + const size_t got = std::fread( data, 1, size, mFile ); + return got == 0 && std::ferror( mFile ) ? -1 : static_cast( got ); +} + +Int64 ArchiveInput::seek( Int64 position ) { + if ( position < 0 || !mFile ) + return -1; +#if EE_PLATFORM == EE_PLATFORM_WIN + return _fseeki64( mFile, position, SEEK_SET ) == 0 ? position : -1; +#else + if ( static_cast( position ) > + static_cast( std::numeric_limits::max() ) ) + return -1; + return ::fseeko( mFile, static_cast( position ), SEEK_SET ) == 0 ? position : -1; +#endif +} + +Int64 ArchiveInput::tell() const { + if ( !mFile ) + return -1; +#if EE_PLATFORM == EE_PLATFORM_WIN + return _ftelli64( mFile ); +#else + return ::ftello( mFile ); +#endif +} + +}}} // namespace EE::VFS::detail diff --git a/src/modules/vfs/src/eepp/vfs/archive/archiveinput.hpp b/src/modules/vfs/src/eepp/vfs/archive/archiveinput.hpp new file mode 100644 index 000000000..af513eb39 --- /dev/null +++ b/src/modules/vfs/src/eepp/vfs/archive/archiveinput.hpp @@ -0,0 +1,48 @@ +#ifndef EE_VFS_ARCHIVE_ARCHIVEINPUT_HPP +#define EE_VFS_ARCHIVE_ARCHIVEINPUT_HPP + +#include +#include +#include +#include + +namespace EE { namespace VFS { namespace detail { + +/** Archive input needs checked 64-bit seeks and, on Windows, sharing that allows replacement + * while an active or idle reader keeps the old file open. It never exposes a CRT file to + * libarchive. */ +class EE_VFS_API ArchiveInput { + public: + explicit ArchiveInput( const std::string& path ); + + ~ArchiveInput(); + + ArchiveInput( const ArchiveInput& ) = delete; + + ArchiveInput& operator=( const ArchiveInput& ) = delete; + + bool isOpen() const { return mFile != nullptr; } + + std::intptr_t read( char* data, size_t size ); + + Int64 seek( Int64 position ); + + Int64 tell() const; + + Int64 size() const { return mSize; } + + bool suspend(); + + bool resume(); + + private: + bool openFile(); + + std::string mPath; + std::FILE* mFile = nullptr; + Int64 mSize = 0; + Int64 mPosition = 0; +}; + +}}} // namespace EE::VFS::detail +#endif diff --git a/src/modules/vfs/src/eepp/vfs/archive/libarchive.cpp b/src/modules/vfs/src/eepp/vfs/archive/libarchive.cpp new file mode 100644 index 000000000..fbeb3ddd0 --- /dev/null +++ b/src/modules/vfs/src/eepp/vfs/archive/libarchive.cpp @@ -0,0 +1,293 @@ +#include "libarchive.hpp" +#include +#include +#include +#include + +namespace EE { namespace VFS { namespace detail { + +namespace { +constexpr std::array Formats = { "tar", "cpio", "ar", "7zip", "rar", "rar5", + "iso9660", "cab", "lha", "xar", "zip" }; +constexpr std::array Filters = { "none", "gzip", "bzip2", "xz", "lzma", "lzip", + "zstd", "lz4", "compress", "rpm", "uu" }; +} // namespace + +VFSResult> LibArchive::load( const std::string& path, const URI& uri ) { +#if EE_PLATFORM == EE_PLATFORM_EMSCRIPTEN || EE_PLATFORM == EE_PLATFORM_IOS + (void)path; +#else + auto library = std::make_shared(); + if ( !path.empty() ) { + library->mLibrary = System::Sys::loadObject( path ); + } else { +#if EE_PLATFORM == EE_PLATFORM_WIN + const char* names[] = { "archive.dll", "libarchive-13.dll", "libarchive.dll" }; +#elif EE_PLATFORM == EE_PLATFORM_MACOS + const char* names[] = { "/usr/lib/libarchive.2.dylib", "libarchive.13.dylib", + "libarchive.dylib" }; +#else + const char* names[] = { "libarchive.so.13", "libarchive.so" }; +#endif + for ( const char* name : names ) { + library->mLibrary = System::Sys::loadObject( name ); + if ( library->mLibrary ) + break; + } + } + if ( library->mLibrary && library->bind() ) + return Ok( std::move( library ) ); +#endif + return Err( VFSError( VFSErrorCode::Unsupported, uri, + "A compatible libarchive 3.x runtime library is not available" ) ); +} + +LibArchive::~LibArchive() { +#if EE_PLATFORM != EE_PLATFORM_EMSCRIPTEN && EE_PLATFORM != EE_PLATFORM_IOS + if ( mLibrary ) + System::Sys::unloadObject( mLibrary ); +#endif +} + +bool LibArchive::bind() { +#if EE_PLATFORM == EE_PLATFORM_EMSCRIPTEN || EE_PLATFORM == EE_PLATFORM_IOS + return false; +#else + auto bind = [this]( auto& fn, const std::string& name ) { + fn = reinterpret_cast>( + System::Sys::loadFunction( mLibrary, name ) ); + return fn != nullptr; + }; + if ( !bind( mVersion, "archive_version_number" ) || mVersion() < 3000000 || + mVersion() >= 4000000 ) + return false; +#define EE_ARCHIVE_BIND( member, symbol ) \ + if ( !bind( member, symbol ) ) \ + return false + EE_ARCHIVE_BIND( mVersionString, "archive_version_string" ); + EE_ARCHIVE_BIND( readNew, "archive_read_new" ); + EE_ARCHIVE_BIND( readFree, "archive_read_free" ); + EE_ARCHIVE_BIND( readOpen, "archive_read_open2" ); + EE_ARCHIVE_BIND( setSeek, "archive_read_set_seek_callback" ); + EE_ARCHIVE_BIND( nextHeader, "archive_read_next_header" ); + EE_ARCHIVE_BIND( readData, "archive_read_data" ); + EE_ARCHIVE_BIND( errorString, "archive_error_string" ); + EE_ARCHIVE_BIND( errorNumber, "archive_errno" ); + EE_ARCHIVE_BIND( formatCode, "archive_format" ); + EE_ARCHIVE_BIND( filterCount, "archive_filter_count" ); + EE_ARCHIVE_BIND( filterCode, "archive_filter_code" ); + EE_ARCHIVE_BIND( entryPath, "archive_entry_pathname_utf8" ); + EE_ARCHIVE_BIND( entryLink, "archive_entry_symlink_utf8" ); + EE_ARCHIVE_BIND( entryHardLink, "archive_entry_hardlink_utf8" ); + EE_ARCHIVE_BIND( entrySize, "archive_entry_size" ); + EE_ARCHIVE_BIND( entrySizeSet, "archive_entry_size_is_set" ); + EE_ARCHIVE_BIND( entryType, "archive_entry_filetype" ); + EE_ARCHIVE_BIND( entryPermissions, "archive_entry_perm" ); + EE_ARCHIVE_BIND( entryTime, "archive_entry_mtime" ); + EE_ARCHIVE_BIND( entryTimeSet, "archive_entry_mtime_is_set" ); +#undef EE_ARCHIVE_BIND + // Probe each decoder on its own disposable reader. A WARN can register an external-command + // fallback, so that reader must never be used to open untrusted input. + auto probe = [&]( auto& functions, const auto& names, const char* prefix ) { + for ( size_t i = 0; i < names.size(); ++i ) { + Enable enable = nullptr; + if ( !bind( enable, std::string( prefix ) + names[i] ) ) + continue; + archive* reader = readNew(); + if ( !reader ) + continue; + if ( enable( reader ) == 0 ) + functions[i] = enable; + readFree( reader ); + } + }; + probe( mFormats, Formats, "archive_read_support_format_" ); + probe( mFilters, Filters, "archive_read_support_filter_" ); + return true; +#endif +} + +ArchiveSupport LibArchive::support() const { + ArchiveSupport support; + support.version = mVersionString(); + for ( size_t i = 0; i < Formats.size(); ++i ) { + if ( mFormats[i] ) + support.formats.emplace_back( Formats[i] ); + } + for ( size_t i = 0; i < Filters.size(); ++i ) { + if ( mFilters[i] ) + support.filters.emplace_back( Filters[i] ); + } + return support; +} + +VFSResult LibArchive::configure( archive* reader, const URI& uri ) const { + if ( !reader ) + return Err( VFSError( VFSErrorCode::IO, uri, "Could not allocate archive reader" ) ); + for ( auto enable : mFormats ) { + if ( enable && enable( reader ) != 0 ) + return Err( error( reader, uri ) ); + } + for ( auto enable : mFilters ) { + if ( enable && enable( reader ) != 0 ) + return Err( error( reader, uri ) ); + } + return Ok(); +} + +VFSError LibArchive::error( archive* reader, const URI& uri ) const { + const char* message = errorString( reader ); + int number = errorNumber( reader ); + return VFSError( number == EACCES || number == EPERM ? VFSErrorCode::PermissionDenied + : VFSErrorCode::IO, + uri, message ? message : "Could not read archive", number ); +} + +LibArchiveReader::LibArchiveReader( std::shared_ptr library, const std::string& path, + URI uri ) : + mLibrary( std::move( library ) ), mFile( path ), mUri( std::move( uri ) ) {} + +LibArchiveReader::~LibArchiveReader() { + if ( mReader ) + mLibrary->readFree( mReader ); +} + +VFSResult LibArchiveReader::open( CancellationToken token ) { + mToken = std::move( token ); + if ( mToken.isCancellationRequested() ) + return Err( cancelledError( mUri ) ); + if ( !mFile.isOpen() ) + return Err( VFSError( VFSErrorCode::IO, mUri, "Could not open archive backing file" ) ); + mReader = mLibrary->readNew(); + auto configured = mLibrary->configure( mReader, mUri ); + if ( !configured ) + return configured; + if ( mLibrary->setSeek( mReader, &seekCallback ) != 0 || + mLibrary->readOpen( mReader, this, nullptr, &readCallback, &skipCallback, nullptr ) != 0 ) + return Err( error() ); + return Ok(); +} + +VFSResult LibArchiveReader::next( CancellationToken token ) { + mToken = std::move( token ); + if ( mToken.isCancellationRequested() ) + return Err( cancelledError( mUri ) ); + int status = mLibrary->nextHeader( mReader, &mEntry ); + if ( status == 1 ) + return Ok( false ); + // WARN can mean an entry was only partially decoded. Never silently publish that index. + if ( status != 0 ) + return Err( error() ); + return Ok( true ); +} + +VFSResult LibArchiveReader::read( std::span data, CancellationToken token ) { + mToken = std::move( token ); + if ( mToken.isCancellationRequested() ) + return Err( cancelledError( mUri ) ); + auto size = mLibrary->readData( mReader, data.data(), data.size() ); + if ( size < 0 ) + return Err( error() ); + return Ok( static_cast( size ) ); +} + +bool LibArchiveReader::canReuse() const { + if ( ( mLibrary->formatCode( mReader ) & 0xff0000 ) != 0x30000 ) + return false; + for ( int i = 0; i < mLibrary->filterCount( mReader ); ++i ) { + const int code = mLibrary->filterCode( mReader, i ); + if ( code != 0 && code != 1 ) + return false; + } + return true; +} + +VFSResult LibArchiveReader::resume( CancellationToken token ) { + mToken = std::move( token ); + if ( mToken.isCancellationRequested() ) + return Err( cancelledError( mUri ) ); + if ( !mFile.resume() ) + return Err( VFSError( VFSErrorCode::IO, mUri, "Could not reopen archive input" ) ); + return Ok(); +} + +VFSResult LibArchiveReader::info() const { + VFSFileInfo info; + // POSIX archive type values, independent of the host's stat constants. + switch ( mLibrary->entryType( mEntry ) ) { + case 0100000: + info.type = FileType::Regular; + break; + case 0040000: + info.type = FileType::Directory; + break; + case 0120000: + info.type = FileType::Symlink; + break; + case 0010000: + info.type = FileType::FIFO; + break; + case 0020000: + info.type = FileType::CharacterDevice; + break; + case 0060000: + info.type = FileType::BlockDevice; + break; + default: + break; + } + if ( const char* link = mLibrary->entryLink( mEntry ) ) { + if ( std::char_traits::length( link ) > 4096 ) + return Err( + VFSError( VFSErrorCode::InvalidOperation, mUri, "Archive link is too long" ) ); + info.symlinkTarget = link; + } + if ( mLibrary->entrySizeSet( mEntry ) && mLibrary->entrySize( mEntry ) >= 0 ) + info.setSize( static_cast( mLibrary->entrySize( mEntry ) ) ); + if ( mLibrary->entryTimeSet( mEntry ) ) + info.setModifiedTime( mLibrary->entryTime( mEntry ) ); + info.setPermissions( mLibrary->entryPermissions( mEntry ) & 07777 ); + return Ok( std::move( info ) ); +} + +VFSError LibArchiveReader::error() const { + return mToken.isCancellationRequested() ? cancelledError( mUri ) + : mLibrary->error( mReader, mUri ); +} + +LibArchive::Count LibArchiveReader::readCallback( archive*, void* context, const void** buffer ) { + auto& self = *static_cast( context ); + if ( self.mToken.isCancellationRequested() ) + return -1; + *buffer = self.mBuffer.data(); + return self.mFile.read( self.mBuffer.data(), self.mBuffer.size() ); +} + +LibArchive::Offset LibArchiveReader::skipCallback( archive*, void* context, + LibArchive::Offset amount ) { + auto& self = *static_cast( context ); + if ( self.mToken.isCancellationRequested() || amount <= 0 ) + return 0; + const auto position = self.mFile.tell(); + const auto size = self.mFile.size(); + if ( position < 0 || size < position ) + return 0; + const auto skip = std::min( amount, size - position ); + return self.mFile.seek( position + skip ) == position + skip ? skip : 0; +} + +LibArchive::Offset LibArchiveReader::seekCallback( archive*, void* context, + LibArchive::Offset offset, int whence ) { + auto& self = *static_cast( context ); + if ( self.mToken.isCancellationRequested() ) + return -30; + Int64 base = whence == SEEK_SET ? 0 + : whence == SEEK_CUR ? self.mFile.tell() + : self.mFile.size(); + if ( base < 0 || ( offset < 0 && offset < -base ) || + ( offset > 0 && offset > std::numeric_limits::max() - base ) ) + return -25; + return self.mFile.seek( base + offset ); +} + +}}} // namespace EE::VFS::detail diff --git a/src/modules/vfs/src/eepp/vfs/archive/libarchive.hpp b/src/modules/vfs/src/eepp/vfs/archive/libarchive.hpp new file mode 100644 index 000000000..62c56eaf6 --- /dev/null +++ b/src/modules/vfs/src/eepp/vfs/archive/libarchive.hpp @@ -0,0 +1,137 @@ +#ifndef EE_VFS_ARCHIVE_LIBARCHIVE_HPP +#define EE_VFS_ARCHIVE_LIBARCHIVE_HPP + +#include "archiveinput.hpp" +#include +#include +#include +#include +#if EE_PLATFORM != EE_PLATFORM_WIN +#include +#endif + +struct archive; +struct archive_entry; + +namespace EE { namespace VFS { namespace detail { + +/** Private subset of libarchive 3.x's C ABI. No struct layout is imported. Reader callbacks use + * eepp's UTF-8 file I/O and cancellation; all native resources die before the loader reference. */ +class EE_VFS_API LibArchive { + public: +#if EE_PLATFORM == EE_PLATFORM_WIN && !defined( _WIN64 ) + using Count = long; +#else + using Count = std::intptr_t; +#endif + using Offset = std::int64_t; +#if EE_PLATFORM == EE_PLATFORM_WIN + using Mode = unsigned short; +#else + using Mode = mode_t; +#endif + using Enable = int ( * )( archive* ); + using Read = Count ( * )( archive*, void*, const void** ); + using Skip = Offset ( * )( archive*, void*, Offset ); + using Seek = Offset ( * )( archive*, void*, Offset, int ); + + static VFSResult> load( const std::string& path, const URI& uri ); + + LibArchive() = default; + + ~LibArchive(); + + LibArchive( const LibArchive& ) = delete; + + LibArchive& operator=( const LibArchive& ) = delete; + + ArchiveSupport support() const; + + VFSResult configure( archive* reader, const URI& uri ) const; + + VFSError error( archive* reader, const URI& uri ) const; + + archive* ( *readNew )() = nullptr; + int ( *readFree )( archive* ) = nullptr; + int ( *readOpen )( archive*, void*, int ( * )( archive*, void* ), Read, Skip, + int ( * )( archive*, void* ) ) = nullptr; + int ( *setSeek )( archive*, Seek ) = nullptr; + int ( *nextHeader )( archive*, archive_entry** ) = nullptr; + Count ( *readData )( archive*, void*, size_t ) = nullptr; + const char* ( *errorString )( archive* ) = nullptr; + int ( *errorNumber )( archive* ) = nullptr; + int ( *formatCode )( archive* ) = nullptr; + int ( *filterCount )( archive* ) = nullptr; + int ( *filterCode )( archive*, int ) = nullptr; + const char* ( *entryPath )( archive_entry* ) = nullptr; + const char* ( *entryLink )( archive_entry* ) = nullptr; + const char* ( *entryHardLink )( archive_entry* ) = nullptr; + Offset ( *entrySize )( archive_entry* ) = nullptr; + int ( *entrySizeSet )( archive_entry* ) = nullptr; + Mode ( *entryType )( archive_entry* ) = nullptr; + Mode ( *entryPermissions )( archive_entry* ) = nullptr; + std::time_t ( *entryTime )( archive_entry* ) = nullptr; + int ( *entryTimeSet )( archive_entry* ) = nullptr; + + private: + bool bind(); + + void* mLibrary = nullptr; + int ( *mVersion )() = nullptr; + const char* ( *mVersionString )() = nullptr; + std::array mFormats{}; + std::array mFilters{}; +}; + +class EE_VFS_API LibArchiveReader { + public: + LibArchiveReader( std::shared_ptr library, const std::string& path, URI uri ); + + ~LibArchiveReader(); + + LibArchiveReader( const LibArchiveReader& ) = delete; + + LibArchiveReader& operator=( const LibArchiveReader& ) = delete; + + VFSResult open( CancellationToken token ); + + VFSResult next( CancellationToken token ); + + VFSResult read( std::span data, CancellationToken token ); + + void setURI( const URI& uri ) { mUri = uri; } + + /** Only retain the bounded decoders for plain/gzip TAR after a file closes. */ + bool canReuse() const; + + bool suspend() { return mFile.suspend(); } + + VFSResult resume( CancellationToken token ); + + const char* path() const { return mLibrary->entryPath( mEntry ); } + + const char* hardLink() const { return mLibrary->entryHardLink( mEntry ); } + + VFSResult info() const; + + private: + static LibArchive::Count readCallback( archive*, void* self, const void** buffer ); + + static LibArchive::Offset skipCallback( archive*, void* self, LibArchive::Offset amount ); + + static LibArchive::Offset seekCallback( archive*, void* self, LibArchive::Offset offset, + int whence ); + + VFSError error() const; + + std::shared_ptr mLibrary; + ArchiveInput mFile; + URI mUri; + CancellationToken mToken; + archive* mReader = nullptr; + archive_entry* mEntry = nullptr; + std::array mBuffer; +}; + +}}} // namespace EE::VFS::detail +#endif diff --git a/src/modules/vfs/src/eepp/vfs/detail/archivelocation.hpp b/src/modules/vfs/src/eepp/vfs/detail/archivelocation.hpp new file mode 100644 index 000000000..7859a8ffa --- /dev/null +++ b/src/modules/vfs/src/eepp/vfs/detail/archivelocation.hpp @@ -0,0 +1,108 @@ +#ifndef EE_VFS_DETAIL_ARCHIVELOCATION_HPP +#define EE_VFS_DETAIL_ARCHIVELOCATION_HPP + +#include + +namespace EE { namespace VFS { namespace detail { + +/** Parsed archive identity and normalized path within it. */ +class ArchiveLocation { + public: + URI archive; + std::string member; + + /** Archive paths are untrusted. Reject names that could leave a copy destination or turn + * into native Windows paths/streams; do not alias unsafe entries to safe names. */ + static bool safeEntry( std::string_view name ) { + if ( name.empty() || name.size() > 4096 || name.front() == '/' || + name.find_first_of( std::string_view( "\\:\0", 3 ) ) != std::string_view::npos ) + return false; + size_t start = 0; + while ( start < name.size() ) { + size_t end = name.find( '/', start ); + if ( end == std::string_view::npos ) + end = name.size(); + auto part = name.substr( start, end - start ); + // Windows trims trailing dots/spaces; an all-dot/space segment can become a parent. + if ( part.empty() || part.find_first_not_of( ". " ) == std::string_view::npos ) + return false; + start = end + 1; + } + return true; + } + + static VFSResult parse( const URI& uri, std::string_view scheme ) { + if ( uri.getScheme() != scheme || uri.getAuthority() != uri.getHost() || + !uri.getFragment().empty() || uri.getPath().find( '\0' ) != std::string::npos ) + return Err( VFSError( VFSErrorCode::InvalidURI, uri, "Invalid archive location" ) ); + if ( !uri.getRawQuery().empty() ) + return Err( + VFSError( VFSErrorCode::Unsupported, uri, "Unsupported archive backing URI" ) ); + const auto& path = uri.getPath(); + if ( path.empty() || path.front() != '/' ) + return Err( + VFSError( VFSErrorCode::InvalidURI, uri, "archive path must be absolute" ) ); + std::string backing; + backing.reserve( path.size() ); + size_t boundary = std::string::npos; + for ( size_t i = 0; i < path.size(); ++i ) { + if ( path[i] == '!' ) { + if ( i + 1 < path.size() && path[i + 1] == '!' ) { + backing += '!'; + ++i; + continue; + } + if ( i + 1 == path.size() || path[i + 1] == '/' ) { + boundary = i + 1; + break; + } + } + backing += path[i]; + } + if ( boundary == std::string::npos || backing.empty() || backing.back() == '/' ) + return Err( + VFSError( VFSErrorCode::InvalidURI, uri, "archive path needs archive!/member" ) ); + ArchiveLocation result; + result.archive.setScheme( "file" ); + result.archive.setHost( uri.getHost() ); + result.archive.setRawPath( std::move( backing ) ); + URI member; + member.setScheme( std::string( scheme ) ); + member.setRawPath( boundary == path.size() ? "/" : path.substr( boundary ) ); + member.normalize(); + result.member = member.getPath(); + return Ok( std::move( result ) ); + } + static URI make( std::string_view scheme, const URI& archive, std::string_view member ) { + if ( archive.getScheme() != "file" || !archive.getRawQuery().empty() || + !archive.getFragment().empty() || archive.getAuthority() != archive.getHost() || + archive.getPath().empty() || archive.getPath().front() != '/' || + archive.getPath().find( '\0' ) != std::string::npos ) + return {}; + URI uri; + uri.setScheme( std::string( scheme ) ); + uri.setHost( archive.getHost() ); + std::string path; + path.reserve( archive.getPath().size() + member.size() + 2 ); + // Escape literal bangs in the backing path, so even a directory named "packages!" is + // unambiguous. Only the first unescaped "!/" separates the archive from its member. + for ( char c : archive.getPath() ) { + path += c; + if ( c == '!' ) + path += '!'; + } + URI entry; + entry.setScheme( std::string( scheme ) ); + entry.setRawPath( member.empty() || member.front() != '/' ? "/" + std::string( member ) + : std::string( member ) ); + entry.normalize(); + path += '!'; + path += entry.getPath(); + uri.setRawPath( std::move( path ) ); + return uri; + } +}; + +}}} // namespace EE::VFS::detail + +#endif diff --git a/src/modules/vfs/src/eepp/vfs/providers/archiveprovider.cpp b/src/modules/vfs/src/eepp/vfs/providers/archiveprovider.cpp new file mode 100644 index 000000000..6403d8d7e --- /dev/null +++ b/src/modules/vfs/src/eepp/vfs/providers/archiveprovider.cpp @@ -0,0 +1,697 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace EE { namespace VFS { + +using detail::ArchiveLocation; +using detail::LibArchive; +using detail::LibArchiveReader; + +namespace { +constexpr ProviderCapabilities ArchiveCapabilities = + VFSCapability::Read | VFSCapability::List | VFSCapability::Seek | VFSCapability::CaseSensitive; +constexpr size_t MaxEntries = 262144; +constexpr size_t MaxMetadata = 64 * 1024 * 1024; +constexpr Uint64 NoEntry = std::numeric_limits::max(); + +VFSError readOnly( const URI& uri ) { + return VFSError( VFSErrorCode::ReadOnly, uri, "Archives are read-only" ); +} + +struct ArchiveNode { + VFSFileInfo info; + std::vector children; + std::string hardLink; + Uint64 entry = NoEntry; +}; + +/** Immutable index; readers are independent and retain the library and revision they opened. */ +struct ArchiveIndex { + std::string path; + std::vector nodes; + UnorderedMap paths; + std::shared_ptr library; + FileRevision revision; + LocalIOScheduler::Route route; + + struct IdleReader { + std::unique_ptr reader; + Uint64 entry = NoEntry; + }; + + IdleReader takeReader( Uint64 entry ) { + std::lock_guard lock( mReaderMutex ); + if ( mIdle.reader && mIdle.entry < entry ) + return std::move( mIdle ); + return {}; + } + + void recycleReader( std::unique_ptr reader, Uint64 entry ) { + if ( !reader ) + return; + IdleReader retired; + { + std::lock_guard lock( mReaderMutex ); + retired = std::move( mIdle ); + mIdle = { std::move( reader ), entry }; + } + // Native reader teardown never holds the index lock. At most one idle reader is kept, + // allowing sequential copies to advance without replaying each entry's archive prefix. + } + + VFSResult find( std::string member, const URI& uri, bool follow = true, + std::string* resolved = nullptr ) const { + // Resolve links only within the archive. URI normalization clamps parent components at + // the member root; no target is ever passed to a native filesystem API. + for ( size_t links = 0; links <= 40; ++links ) { + size_t start = 0; + bool redirected = false; + while ( start < member.size() ) { + size_t end = member.find( '/', start ); + if ( end == std::string::npos ) + end = member.size(); + auto found = paths.find( member.substr( 0, end ) ); + if ( found == paths.end() ) + return Err( VFSError( VFSErrorCode::NotFound, uri ) ); + const auto& node = nodes[found->second]; + if ( ( node.info.isSymlink() && ( follow || end < member.size() ) ) || + !node.hardLink.empty() ) { + const std::string& target = + node.hardLink.empty() ? node.info.symlinkTarget : node.hardLink; + if ( target.empty() || target.size() > 4096 || + target.find_first_of( "\\:" ) != std::string::npos ) + return Err( + VFSError( VFSErrorCode::Unsupported, uri, "Invalid archive link" ) ); + std::string path; + if ( !node.hardLink.empty() || target.front() == '/' ) + path = "/" + target; + else + path = "/" + member.substr( 0, start ) + target; + path += member.substr( end ); + URI normalized; + normalized.setRawPath( std::move( path ) ); + normalized.normalize(); + member = normalized.getPath(); + while ( !member.empty() && member.front() == '/' ) + member.erase( member.begin() ); + redirected = true; + break; + } + if ( end < member.size() && !node.info.isDirectory() ) + return Err( VFSError( VFSErrorCode::NotDirectory, uri ) ); + start = end + 1; + } + if ( redirected ) + continue; + auto found = paths.find( member ); + if ( found == paths.end() ) + return Err( VFSError( VFSErrorCode::NotFound, uri ) ); + if ( uri.getPath().ends_with( '/' ) && !nodes[found->second].info.isDirectory() ) + return Err( VFSError( VFSErrorCode::NotDirectory, uri ) ); + if ( resolved ) + *resolved = std::move( member ); + return Ok( found->second ); + } + return Err( VFSError( VFSErrorCode::TooManySymlinks, uri ) ); + } + + VFSResult index( LibArchiveReader& reader, const URI& uri, CancellationToken token ) { + VFSFileInfo root; + root.name = "/"; + root.type = FileType::Directory; + root.revision = revision; + nodes.push_back( { std::move( root ), {}, {}, NoEntry } ); + paths.emplace( "", 0 ); + size_t budget = MaxMetadata; + for ( Uint64 ordinal = 0;; ++ordinal ) { + auto next = reader.next( token ); + if ( !next ) + return Err( std::move( next ).error() ); + if ( !next.value() ) + break; + if ( ordinal >= MaxEntries ) + return Err( VFSError( VFSErrorCode::InvalidOperation, uri, + "Archive has too many entries" ) ); + const char* raw = reader.path(); + if ( !raw ) + return Err( VFSError( VFSErrorCode::Unsupported, uri, + "Archive filename cannot be decoded as UTF-8" ) ); + std::string_view name( raw ); + while ( name.starts_with( "./" ) ) + name.remove_prefix( 2 ); + if ( name == "." || name.empty() ) + continue; + if ( !ArchiveLocation::safeEntry( name ) ) + continue; + if ( name.ends_with( '/' ) ) + name.remove_suffix( 1 ); + auto info = reader.info(); + if ( !info ) + return Err( std::move( info ).error() ); + VFSFileInfo metadata = std::move( info ).value(); + const char* hardLink = reader.hardLink(); + if ( metadata.symlinkTarget.size() > 4096 || + ( hardLink && std::char_traits::length( hardLink ) > 4096 ) ) + return Err( + VFSError( VFSErrorCode::InvalidOperation, uri, "Archive link is too long" ) ); + size_t start = 0, parent = 0; + while ( start < name.size() ) { + size_t end = name.find( '/', start ); + if ( end == std::string_view::npos ) + end = name.size(); + bool last = end == name.size(); + bool folder = !last || metadata.isDirectory(); + std::string key( name.substr( 0, end ) ); + auto found = paths.find( key ); + size_t node; + if ( found == paths.end() ) { + const size_t cost = + 2 * ( sizeof( ArchiveNode ) + key.size() + end - start + 128 ); + if ( nodes.size() >= MaxEntries || cost > budget ) + return Err( VFSError( VFSErrorCode::InvalidOperation, uri, + "Archive metadata budget exceeded" ) ); + budget -= cost; + node = nodes.size(); + paths.emplace( std::move( key ), node ); + nodes[parent].children.push_back( node ); + VFSFileInfo info; + info.name = std::string( name.substr( start, end - start ) ); + info.type = folder ? FileType::Directory : FileType::Unknown; + info.hidden = info.name.starts_with( '.' ); + info.revision = revision; + nodes.push_back( { std::move( info ), {}, {}, NoEntry } ); + } else { + node = found->second; + } + auto& value = nodes[node]; + if ( folder && !value.info.isDirectory() ) { + VFSFileInfo directory; + directory.name = std::move( value.info.name ); + directory.hidden = value.info.hidden; + directory.type = FileType::Directory; + directory.revision = revision; + value.info = std::move( directory ); + value.entry = NoEntry; + value.hardLink.clear(); + } + // As with ZIP, directories win collisions and duplicate files keep the first. + if ( last && ( folder || !value.info.isDirectory() ) && value.entry == NoEntry ) { + size_t cost = + 2 * ( metadata.symlinkTarget.size() + + ( hardLink ? std::char_traits::length( hardLink ) : 0 ) ); + if ( cost > budget ) + return Err( VFSError( VFSErrorCode::InvalidOperation, uri, + "Archive metadata budget exceeded" ) ); + budget -= cost; + metadata.name = std::move( value.info.name ); + metadata.hidden = metadata.name.starts_with( '.' ); + metadata.revision = FileRevision::fromValues( + { std::hash{}( std::string_view( + reinterpret_cast( revision.data() ), revision.size() ) ), + ordinal } ); + value.info = std::move( metadata ); + value.entry = ordinal; + if ( hardLink ) + value.hardLink = hardLink; + } + parent = node; + start = end + 1; + } + } + return Ok(); + } + + private: + std::mutex mReaderMutex; + IdleReader mIdle; +}; + +class ArchiveFile : public FileHandleImpl, public std::enable_shared_from_this { + public: + ArchiveFile( std::shared_ptr scheduler, std::shared_ptr index, + std::unique_ptr reader, size_t node, URI uri ) : + mScheduler( std::move( scheduler ) ), + mIndex( std::move( index ) ), + mReader( std::move( reader ) ), + mUri( std::move( uri ) ), + mNode( static_cast( node ) ) {} + + static VFSResult> openReader( ArchiveIndex& index, + size_t node, const URI& uri, + CancellationToken token, + bool reuse = true ) { + auto current = LocalProvider::statSync( URI::fromFSPath( index.path ), index.path, true ); + if ( !current || current.value().revision != index.revision ) + return Err( VFSError( VFSErrorCode::ChangedExternally, uri, + "Archive changed while opening entry" ) ); + auto idle = + reuse ? index.takeReader( index.nodes[node].entry ) : ArchiveIndex::IdleReader{}; + auto reader = std::move( idle.reader ); + Uint64 first = reader ? idle.entry + 1 : 0; + if ( reader ) { + reader->setURI( uri ); + auto resumed = reader->resume( token ); + if ( !resumed ) + return Err( std::move( resumed ).error() ); + } else { + reader = std::make_unique( index.library, index.path, uri ); + auto opened = reader->open( token ); + if ( !opened ) + return Err( std::move( opened ).error() ); + first = 0; + } + current = LocalProvider::statSync( URI::fromFSPath( index.path ), index.path, true ); + if ( !current || current.value().revision != index.revision ) + return Err( VFSError( VFSErrorCode::ChangedExternally, uri, + "Archive changed while opening entry" ) ); + reader->setURI( uri ); + for ( Uint64 i = first; i <= index.nodes[node].entry; ++i ) { + auto next = reader->next( token ); + if ( !next ) + return Err( std::move( next ).error() ); + if ( !next.value() ) + return Err( VFSError( VFSErrorCode::IO, uri, "Archive entry is missing" ) ); + } + return Ok( std::move( reader ) ); + } + + VFSTask read( Uint64 offset, std::span data, CancellationToken token ) override { + auto work = [self = shared_from_this(), offset, + token]( std::span buffer ) -> VFSResult { + std::lock_guard lock( self->mMutex ); + auto result = self->readInto( offset, buffer, token ); + if ( !result ) + self->mReusable = false; + return result; + }; + if ( !mIndex->route.isNetworkMount() ) { + return mScheduler->submit( + mIndex->route, + [work = std::move( work ), data]() mutable { return work( data ); } ); + } + // A mount can return after its timeout, when data no longer exists. Commit only if + // this call still owns completion; its work touches an independent buffer. + return mScheduler->submitCommitted( + mIndex->route, + [work = std::move( work ), + size = data.size()]() mutable -> VFSResult> { + std::vector buffer( size ); + auto result = work( buffer ); + if ( !result ) + return Err( std::move( result ).error() ); + buffer.resize( result.value() ); + return Ok( std::move( buffer ) ); + }, + [data]( VFSResult>&& result ) -> VFSResult { + if ( !result ) + return Err( std::move( result ).error() ); + std::copy( result.value().begin(), result.value().end(), data.begin() ); + return Ok( result.value().size() ); + } ); + } + + VFSTask write( Uint64, std::span, CancellationToken ) override { + return readyError( readOnly( mUri ) ); + } + + VFSTask truncate( Uint64 ) override { return readyError( readOnly( mUri ) ); } + + VFSTask flush() override { return readyValue(); } + + VFSTask stat() override { + return readyValue( mIndex->nodes[mNode].info ); + } + + VFSTask close() override { + return mScheduler->submit( + mIndex->route, [self = shared_from_this()]() -> VFSResult { + std::lock_guard lock( self->mMutex ); + if ( self->mReusable && self->mReader && self->mReader->canReuse() && + self->mReader->suspend() ) { + self->mIndex->recycleReader( std::move( self->mReader ), + self->mIndex->nodes[self->mNode].entry ); + } else { + self->mReader.reset(); + } + return Ok(); + } ); + } + + ProviderCapabilities capabilities() const override { return ArchiveCapabilities; } + + private: + VFSResult readInto( Uint64 offset, std::span data, CancellationToken token ) { + if ( token.isCancellationRequested() ) + return Err( cancelledError( mUri ) ); + if ( !mReader ) + return Err( VFSError( VFSErrorCode::InvalidOperation, mUri, "File closed" ) ); + if ( data.empty() ) + return Ok( size_t( 0 ) ); + const auto size = mIndex->nodes[mNode].info.size(); + if ( size && offset >= *size ) + return Ok( size_t( 0 ) ); + if ( offset < mPosition ) { + auto reopened = openReader( *mIndex, mNode, mUri, token, false ); + if ( !reopened ) + return Err( std::move( reopened ).error() ); + mReader = std::move( reopened ).value(); + mPosition = 0; + } + std::array scratch; + while ( mPosition < offset ) { + auto got = mReader->read( std::span( scratch ).first( + std::min( scratch.size(), offset - mPosition ) ), + token ); + if ( !got ) + return got; + if ( got.value() == 0 ) + return Err( VFSError( VFSErrorCode::IO, mUri, "Truncated archive entry" ) ); + mPosition += got.value(); + } + size_t done = 0; + const size_t wanted = size ? std::min( data.size(), *size - offset ) : data.size(); + while ( done < wanted ) { + auto got = mReader->read( + data.subspan( done, std::min( wanted - done, 65536 ) ), token ); + if ( !got ) + return got; + if ( got.value() == 0 ) { + if ( size ) + return Err( VFSError( VFSErrorCode::IO, mUri, "Truncated archive entry" ) ); + break; + } + done += got.value(); + mPosition += got.value(); + } + // Check the entry's end marker too: some compression/checksum errors surface only + // when a read advances past the final output byte. + if ( size && mPosition == *size ) { + auto end = mReader->read( std::span( scratch ).first( 1 ), token ); + if ( !end ) + return end; + if ( end.value() != 0 ) + return Err( + VFSError( VFSErrorCode::IO, mUri, "Archive entry exceeds declared size" ) ); + } + return Ok( done ); + } + + std::shared_ptr mScheduler; + std::shared_ptr mIndex; + std::unique_ptr mReader; + std::mutex mMutex; + URI mUri; + Uint64 mPosition = 0; + Uint32 mNode; // The index is capped at MaxEntries, well below UINT32_MAX. + bool mReusable = true; +}; +} // namespace + +class ArchiveProviderImpl : public std::enable_shared_from_this { + public: + ArchiveProviderImpl( std::shared_ptr scheduler, std::string path ) : + scheduler( std::move( scheduler ) ), mLibraryPath( std::move( path ) ) {} + + VFSResult> library( const URI& uri ) { + std::call_once( mLoad, [&] { + auto loaded = LibArchive::load( mLibraryPath, uri ); + if ( loaded ) + mLibrary = std::move( loaded ).value(); + } ); + if ( !mLibrary ) + return Err( + VFSError( VFSErrorCode::Unsupported, uri, + "A compatible libarchive 3.x runtime library is not available" ) ); + return Ok( mLibrary ); + } + + template + VFSTask run( const URI& uri, CancellationToken token, F&& fn ) { + if ( token.isCancellationRequested() ) + return readyError( cancelledError( uri ) ); + auto address = ArchiveLocation::parse( uri, "archive" ); + if ( !address ) + return readyError( std::move( address ).error() ); + auto native = LocalProvider::toNativePath( address.value().archive ); + if ( !native ) + return readyError( std::move( native ).error() ); + std::string path = std::move( native ).value(); + std::string member = std::move( address ).value().member; + while ( !member.empty() && member.front() == '/' ) + member.erase( member.begin() ); + while ( !member.empty() && member.back() == '/' ) + member.pop_back(); + return scheduler->submitResolved( + path, true, + [self = shared_from_this(), uri, path, member = std::move( member ), + token = std::move( token ), fn = std::forward( fn )]() mutable -> VFSResult { + if ( token.isCancellationRequested() ) + return Err( cancelledError( uri ) ); + auto index = self->load( path, uri, token ); + if ( !index ) + return Err( std::move( index ).error() ); + return fn( std::move( index ).value(), member, uri, token ); + } ); + } + + std::shared_ptr scheduler; + + private: + struct Cached { + std::shared_ptr index; + Uint64 used = 0; + }; + + VFSResult> load( const std::string& path, const URI& uri, + CancellationToken token ) { + auto loaded = library( uri ); + if ( !loaded ) + return Err( std::move( loaded ).error() ); + const URI backing = URI::fromFSPath( path ); + auto before = LocalProvider::statSync( backing, path, true ); + if ( !before ) + return Err( std::move( before ).error() ); + if ( !before.value().isRegularFile() ) + return Err( VFSError( VFSErrorCode::InvalidOperation, uri, + "Archive backing file is not a regular file" ) ); + const auto revision = before.value().revision; + { + std::lock_guard lock( mMutex ); + for ( auto& cached : mCache ) { + if ( cached.index && cached.index->path == path && + cached.index->revision == revision ) { + cached.used = ++mUsed; + return Ok( cached.index ); + } + } + } + auto index = std::make_shared(); + index->path = path; + index->revision = revision; + index->library = std::move( loaded ).value(); + index->route = scheduler->resolvedRouteFor( path, true ); + LibArchiveReader reader( index->library, path, uri ); + auto opened = reader.open( token ); + if ( !opened ) + return Err( std::move( opened ).error() ); + auto indexed = index->index( reader, uri, token ); + if ( !indexed ) + return Err( std::move( indexed ).error() ); + auto after = LocalProvider::statSync( backing, path, true ); + if ( !after || after.value().revision != revision ) + return Err( VFSError( VFSErrorCode::ChangedExternally, uri, + "Archive changed while indexing" ) ); + std::shared_ptr retired; + { + std::lock_guard lock( mMutex ); + auto slot = mCache.begin(); + for ( auto it = mCache.begin(); it != mCache.end(); ++it ) { + if ( it->index && it->index->path == path ) { + slot = it; + break; + } + if ( it->used < slot->used ) + slot = it; + } + retired = std::move( slot->index ); + slot->index = index; + slot->used = ++mUsed; + } + return Ok( std::move( index ) ); + } + + std::string mLibraryPath; + std::shared_ptr mLibrary; + std::once_flag mLoad; + std::mutex mMutex; + std::array mCache; + Uint64 mUsed = 0; +}; + +ArchiveProvider::ArchiveProvider( std::shared_ptr scheduler, + std::string libraryPath ) : + mImpl( std::make_shared( std::move( scheduler ), + std::move( libraryPath ) ) ) {} + +ArchiveProvider::~ArchiveProvider() = default; + +URI ArchiveProvider::makeURI( const URI& archive, std::string_view member ) { + return ArchiveLocation::make( "archive", archive, member ); +} + +VFSResult ArchiveProvider::archiveURI( const URI& uri ) { + auto address = ArchiveLocation::parse( uri, "archive" ); + if ( !address ) + return Err( std::move( address ).error() ); + return Ok( std::move( address ).value().archive ); +} + +VFSResult ArchiveProvider::memberPath( const URI& uri ) { + auto address = ArchiveLocation::parse( uri, "archive" ); + if ( !address ) + return Err( std::move( address ).error() ); + return Ok( std::move( address ).value().member ); +} + +VFSTask ArchiveProvider::support( CancellationToken token ) { + return mImpl->scheduler->submit( + LocalIOScheduler::Route{}, + [impl = mImpl, token = std::move( token )]() -> VFSResult { + if ( token.isCancellationRequested() ) + return Err( cancelledError( {} ) ); + auto library = impl->library( {} ); + if ( !library ) + return Err( std::move( library ).error() ); + return Ok( library.value()->support() ); + } ); +} + +ProviderCapabilities ArchiveProvider::capabilities( const URI& ) const { + return ArchiveCapabilities; +} + +VFSTask ArchiveProvider::stat( const URI& uri, const StatOptions& options, + CancellationToken token ) { + return mImpl->run( + uri, std::move( token ), + [options]( const std::shared_ptr& index, const std::string& member, + const URI& uri, CancellationToken ) -> VFSResult { + auto found = index->find( member, uri, options.followSymlinks ); + if ( !found ) + return Err( std::move( found ).error() ); + auto info = index->nodes[found.value()].info; + if ( !member.empty() ) + info.name = member.substr( member.find_last_of( '/' ) + 1 ); + info.hidden = info.name.starts_with( '.' ); + return Ok( std::move( info ) ); + } ); +} + +VFSTask ArchiveProvider::openDirectory( const URI& uri, + const DirectoryOpenOptions&, + CancellationToken token ) { + return mImpl->run( + uri, std::move( token ), + []( const std::shared_ptr& index, const std::string& member, const URI& uri, + CancellationToken token ) -> VFSResult { + auto found = index->find( member, uri ); + if ( !found ) + return Err( std::move( found ).error() ); + const auto& folder = index->nodes[found.value()]; + if ( !folder.info.isDirectory() ) + return Err( VFSError( VFSErrorCode::NotDirectory, uri ) ); + std::vector entries; + entries.reserve( folder.children.size() ); + for ( auto child : folder.children ) { + if ( token.isCancellationRequested() ) + return Err( cancelledError( uri ) ); + auto info = index->nodes[child].info; + if ( info.isSymlink() || !index->nodes[child].hardLink.empty() ) { + auto target = + index->find( member.empty() ? info.name : member + "/" + info.name, uri ); + if ( target ) { + if ( !index->nodes[child].hardLink.empty() ) { + std::string name = std::move( info.name ); + info = index->nodes[target.value()].info; + info.name = std::move( name ); + info.hidden = info.name.starts_with( '.' ); + } else { + info.targetType = index->nodes[target.value()].info.type; + if ( auto size = index->nodes[target.value()].info.size() ) + info.setSize( *size ); + } + } + } + entries.push_back( std::move( info ) ); + } + return Ok( DirectoryHandle( + uri, std::make_shared( std::move( entries ) ) ) ); + } ); +} + +VFSTask ArchiveProvider::open( const URI& uri, const OpenOptions& options, + CancellationToken token ) { + if ( hasFlag( options.flags, OpenFlags::Write ) || + hasFlag( options.flags, OpenFlags::Create ) || + hasFlag( options.flags, OpenFlags::Truncate ) ) + return readyError( readOnly( uri ) ); + return mImpl->run( + uri, std::move( token ), + [scheduler = mImpl->scheduler]( std::shared_ptr index, + const std::string& member, const URI& uri, + CancellationToken token ) -> VFSResult { + auto found = index->find( member, uri ); + if ( !found ) + return Err( std::move( found ).error() ); + const auto& node = index->nodes[found.value()]; + if ( node.info.isDirectory() ) + return Err( VFSError( VFSErrorCode::IsDirectory, uri ) ); + if ( !node.info.isRegularFile() || node.entry == NoEntry ) + return Err( VFSError( VFSErrorCode::Unsupported, uri, + "Archive entry is not a regular file" ) ); + auto reader = ArchiveFile::openReader( *index, found.value(), uri, token ); + if ( !reader ) + return Err( std::move( reader ).error() ); + return Ok( FileHandle( uri, std::make_shared( + scheduler, std::move( index ), + std::move( reader ).value(), found.value(), uri ) ) ); + } ); +} + +VFSTask ArchiveProvider::mkdir( const URI& uri, const MkDirOptions&, CancellationToken ) { + return readyError( readOnly( uri ) ); +} + +VFSTask ArchiveProvider::remove( const URI& uri, const RemoveOptions&, CancellationToken ) { + return readyError( readOnly( uri ) ); +} + +VFSTask ArchiveProvider::rename( const URI& uri, const URI&, const RenameOptions&, + CancellationToken ) { + return readyError( readOnly( uri ) ); +} + +VFSTask ArchiveProvider::canonicalize( const URI& uri, CancellationToken token ) { + return mImpl->run( uri, std::move( token ), + []( const std::shared_ptr& index, + const std::string& member, const URI& uri, + CancellationToken ) -> VFSResult { + std::string resolved; + auto found = index->find( member, uri, true, &resolved ); + if ( !found ) + return Err( std::move( found ).error() ); + auto backing = ArchiveProvider::archiveURI( uri ); + return Ok( ArchiveProvider::makeURI( backing.value(), resolved ) ); + } ); +} + +}} // namespace EE::VFS diff --git a/src/modules/vfs/src/eepp/vfs/providers/zipprovider.cpp b/src/modules/vfs/src/eepp/vfs/providers/zipprovider.cpp index 820576c8b..56523dcbf 100644 --- a/src/modules/vfs/src/eepp/vfs/providers/zipprovider.cpp +++ b/src/modules/vfs/src/eepp/vfs/providers/zipprovider.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -48,73 +49,6 @@ VFSError readerError( const Zip& reader, const URI& uri ) { return VFSError( code, uri, reader.errorMessage() ); } -/** Archive paths are untrusted. Reject names that could leave a copy destination or turn into - * native Windows paths/streams; do not resolve these names into aliases of safe entries. */ -bool safeEntry( std::string_view name ) { - if ( name.empty() || name.size() > 4096 || name.front() == '/' || - name.find_first_of( std::string_view( "\\:\0", 3 ) ) != std::string_view::npos ) - return false; - size_t start = 0; - while ( start < name.size() ) { - size_t end = name.find( '/', start ); - if ( end == std::string_view::npos ) - end = name.size(); - auto part = name.substr( start, end - start ); - // Windows trims trailing dots/spaces; an all-dot/space segment can become a parent. - if ( part.empty() || part.find_first_not_of( ". " ) == std::string_view::npos ) - return false; - start = end + 1; - } - return true; -} - -/** Parsed archive identity and normalized path within it. */ -struct ZipAddress { - URI archive; - std::string member; - - static VFSResult parse( const URI& uri ) { - if ( uri.getScheme() != "zip" || uri.getAuthority() != uri.getHost() || - !uri.getFragment().empty() || uri.getPath().find( '\0' ) != std::string::npos ) - return Err( VFSError( VFSErrorCode::InvalidURI, uri, "Invalid ZIP location" ) ); - if ( !uri.getRawQuery().empty() ) - return Err( VFSError( VFSErrorCode::Unsupported, uri, "Unsupported ZIP backing URI" ) ); - const auto& path = uri.getPath(); - if ( path.empty() || path.front() != '/' ) - return Err( VFSError( VFSErrorCode::InvalidURI, uri, "ZIP path must be absolute" ) ); - std::string backing; - backing.reserve( path.size() ); - size_t boundary = std::string::npos; - for ( size_t i = 0; i < path.size(); ++i ) { - if ( path[i] == '!' ) { - if ( i + 1 < path.size() && path[i + 1] == '!' ) { - backing += '!'; - ++i; - continue; - } - if ( i + 1 == path.size() || path[i + 1] == '/' ) { - boundary = i + 1; - break; - } - } - backing += path[i]; - } - if ( boundary == std::string::npos || backing.empty() || backing.back() == '/' ) - return Err( - VFSError( VFSErrorCode::InvalidURI, uri, "ZIP path needs archive.zip!/member" ) ); - ZipAddress result; - result.archive.setScheme( "file" ); - result.archive.setHost( uri.getHost() ); - result.archive.setRawPath( std::move( backing ) ); - URI member; - member.setScheme( "zip" ); - member.setRawPath( boundary == path.size() ? "/" : path.substr( boundary ) ); - member.normalize(); - result.member = member.getPath(); - return Ok( std::move( result ) ); - } -}; - struct ZipNode { VFSFileInfo info; std::vector children; @@ -163,7 +97,7 @@ struct ZipArchive { if ( !reader.entry( entry, metadata ) ) return Err( readerError( reader, uri ) ); std::string_view name( metadata.name ); - if ( !safeEntry( name ) ) + if ( !detail::ArchiveLocation::safeEntry( name ) ) continue; const bool directory = name.back() == '/'; if ( directory ) @@ -369,7 +303,7 @@ class ZipProviderImpl : public std::enable_shared_from_this { VFSTask run( const URI& uri, CancellationToken token, F&& fn ) { if ( token.isCancellationRequested() ) return readyError( cancelledError( uri ) ); - auto address = ZipAddress::parse( uri ); + auto address = detail::ArchiveLocation::parse( uri, "zip" ); if ( !address ) return readyError( std::move( address ).error() ); auto native = LocalProvider::toNativePath( address.value().archive ); @@ -471,43 +405,18 @@ ZipProvider::ZipProvider( std::shared_ptr scheduler ) : ZipProvider::~ZipProvider() = default; URI ZipProvider::makeURI( const URI& archive, std::string_view member ) { - if ( archive.getScheme() != "file" || !archive.getRawQuery().empty() || - !archive.getFragment().empty() || archive.getAuthority() != archive.getHost() || - archive.getPath().empty() || archive.getPath().front() != '/' || - archive.getPath().find( '\0' ) != std::string::npos ) - return {}; - URI uri; - uri.setScheme( "zip" ); - uri.setHost( archive.getHost() ); - std::string path; - path.reserve( archive.getPath().size() + member.size() + 2 ); - // Escape literal bangs in the backing path, so even a directory named "packages!" is - // unambiguous. Only the first unescaped "!/" separates the archive from its member. - for ( char c : archive.getPath() ) { - path += c; - if ( c == '!' ) - path += '!'; - } - URI entry; - entry.setScheme( "zip" ); - entry.setRawPath( member.empty() || member.front() != '/' ? "/" + std::string( member ) - : std::string( member ) ); - entry.normalize(); - path += '!'; - path += entry.getPath(); - uri.setRawPath( std::move( path ) ); - return uri; + return detail::ArchiveLocation::make( "zip", archive, member ); } VFSResult ZipProvider::archiveURI( const URI& uri ) { - auto address = ZipAddress::parse( uri ); + auto address = detail::ArchiveLocation::parse( uri, "zip" ); if ( !address ) return Err( std::move( address ).error() ); return Ok( std::move( address ).value().archive ); } VFSResult ZipProvider::memberPath( const URI& uri ) { - auto address = ZipAddress::parse( uri ); + auto address = detail::ArchiveLocation::parse( uri, "zip" ); if ( !address ) return Err( std::move( address ).error() ); return Ok( std::move( address ).value().member ); diff --git a/src/modules/vfs/src/eepp/vfs/transferengine.cpp b/src/modules/vfs/src/eepp/vfs/transferengine.cpp index 3bb9fc5cd..0f09a97fb 100644 --- a/src/modules/vfs/src/eepp/vfs/transferengine.cpp +++ b/src/modules/vfs/src/eepp/vfs/transferengine.cpp @@ -198,7 +198,8 @@ VFSResult TransferEngine::copyNode( const Node& node, const URI& source, if ( existing ) { if ( existing.value().isDirectory() ) return Err( VFSError( VFSErrorCode::IsDirectory, destination ) ); - if ( mOptions.overwrite == OverwritePolicy::Fail ) + if ( mOptions.overwrite == OverwritePolicy::Fail || + mOptions.overwrite == OverwritePolicy::MergeDirectories ) return Err( VFSError( VFSErrorCode::AlreadyExists, destination ) ); if ( mOptions.overwrite == OverwritePolicy::Skip ) { ++mResult.filesSkipped; diff --git a/src/modules/vfs/src/eepp/vfs/virtualfilesystem.cpp b/src/modules/vfs/src/eepp/vfs/virtualfilesystem.cpp index 1566fa8b5..dc1bad246 100644 --- a/src/modules/vfs/src/eepp/vfs/virtualfilesystem.cpp +++ b/src/modules/vfs/src/eepp/vfs/virtualfilesystem.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -71,6 +72,7 @@ VirtualFileSystem::~VirtualFileSystem() { void VirtualFileSystem::registerDefaultProviders() { registerProvider( std::make_shared( mLocalIO ) ); registerProvider( std::make_shared( mLocalIO ) ); + registerProvider( std::make_shared( mLocalIO ) ); registerProvider( std::make_shared( mIOPool ) ); // Network mounts are polled through the VFS (whose calls time out), not by efsw. auto polling = std::make_shared( *this ); diff --git a/src/tests/unit_tests/archiveprovider_tests.cpp b/src/tests/unit_tests/archiveprovider_tests.cpp new file mode 100644 index 000000000..6ad84c6ea --- /dev/null +++ b/src/tests/unit_tests/archiveprovider_tests.cpp @@ -0,0 +1,553 @@ +#include +#if EE_PLATFORM == EE_PLATFORM_WIN +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#endif + +#include "../../tools/eefm/location.hpp" +#include "utest.hpp" +#include "vfs_test_utils.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace EE; +using namespace EE::System; +using namespace EE::VFS; +using namespace vfstest; + +namespace { + +// Independent USTAR fixture writer. GNU/PAX extensions are explicit entries so the tests do +// not depend on a system tar command or on the same reader being used to produce fixtures. +struct TarFixture { + std::string bytes; + + void entry( std::string_view name, std::string_view data, char type = '0', + std::string_view link = {}, std::optional headerSize = {} ) { + std::array header{}; + std::copy_n( name.data(), std::min( name.size(), 100 ), header.data() ); + std::snprintf( header.data() + 100, 8, "%07o", type == '5' ? 0755 : 0644 ); + std::snprintf( header.data() + 108, 8, "%07o", 1000 ); + std::snprintf( header.data() + 116, 8, "%07o", 1000 ); + std::snprintf( header.data() + 124, 12, "%011llo", + static_cast( headerSize.value_or( data.size() ) & + 077777777777ULL ) ); + std::snprintf( header.data() + 136, 12, "%011o", 1700000000 ); + std::fill_n( header.data() + 148, 8, ' ' ); + header[156] = type; + std::copy_n( link.data(), std::min( link.size(), 100 ), header.data() + 157 ); + std::copy_n( "ustar", 5, header.data() + 257 ); + header[263] = header[264] = '0'; + unsigned sum = 0; + for ( unsigned char byte : header ) + sum += byte; + std::snprintf( header.data() + 148, 7, "%06o", sum ); + header[155] = ' '; + bytes.append( header.data(), header.size() ); + bytes.append( data ); + bytes.append( ( 512 - data.size() % 512 ) % 512, '\0' ); + } + + bool write( const std::string& path, bool gzip = false ) const { + std::string data = bytes + std::string( 1024, '\0' ); + if ( !gzip ) + return FileSystem::fileWrite( path, data ); + IOStreamMemory input( static_cast( data.data() ), data.size() ); + IOStreamFile output( path, "wb" ); + return output.isOpen() && + Compression::compress( output, input, Compression::MODE_GZIP ) == Compression::OK; + } + + static std::string pax( std::string value ) { + size_t size = value.size() + 3; + while ( std::to_string( size ).size() + value.size() + 2 != size ) + size = std::to_string( size ).size() + value.size() + 2; + return std::to_string( size ) + " " + value + "\n"; + } +}; + +bool available( int* utest_result, VirtualFileSystem& vfs ) { + auto provider = std::dynamic_pointer_cast( vfs.provider( "archive" ) ); + if ( !provider ) { + *utest_result = UTEST_TEST_FAILURE; + return false; + } + auto support = waitFor( provider->support() ); + if ( !support ) { + if ( !support.error().is( VFSErrorCode::Unsupported ) ) + *utest_result = UTEST_TEST_FAILURE; + UTEST_PRINT_INFO( + "libarchive runtime unavailable; integration requires an installed library" ); + if ( *utest_result != UTEST_TEST_FAILURE ) + *utest_result = UTEST_TEST_SKIPPED; + return false; + } + if ( support.value().version.empty() || + std::find( support.value().formats.begin(), support.value().formats.end(), "tar" ) == + support.value().formats.end() ) { + *utest_result = UTEST_TEST_FAILURE; + return false; + } + return true; +} + +} // namespace + +UTEST( ArchiveProvider, locationsAndEefmExtensions ) { + TempDir temp; + const URI backing = URI::fromFSPath( temp.path + "Archive! #%.tar.gz" ); + const URI root = ArchiveProvider::makeURI( backing ); + const URI member = root.withAppendedSegment( "dir" ).withAppendedSegment( "entry! #%" ); + EXPECT_TRUE( ArchiveProvider::archiveURI( URI( member.toString() ) ).value() == backing ); + EXPECT_STDSTREQ( "/dir/entry! #%", ArchiveProvider::memberPath( member ).value() ); + EXPECT_TRUE( eefm::Location::parse( eefm::Location::text( member ).toUtf8() ) == member ); + EXPECT_TRUE( eefm::Location::parent( root ) == backing.getParent() ); + EXPECT_STDSTREQ( backing.getLastPathSegment(), eefm::Location::entryName( root ) ); + EXPECT_STDSTREQ( backing.getLastPathSegment(), eefm::Location::displayName( root ).toUtf8() ); + EXPECT_TRUE( eefm::Location::parent( member ) == root.withAppendedSegment( "dir" ) ); + for ( const char* name : { "archive.tar", "archive.TAR.GZ", "archive.tgz", "archive.tar.xz", + "archive.tar.bz2", "archive.7z", "archive.rar", "package.deb" } ) { + const URI file = temp.uri().withAppendedSegment( name ); + EXPECT_TRUE( eefm::Location::archiveRoot( file ) == ArchiveProvider::makeURI( file ) ); + } + EXPECT_TRUE( ArchiveProvider::makeURI( URI( "sftp://host/a.tar" ) ).empty() ); + EXPECT_EQ( VFSErrorCode::InvalidURI, + codeOf( ArchiveProvider::archiveURI( URI( "archive:///no-boundary" ) ) ) ); +} + +UTEST( ArchiveProvider, largeSparseTarUses64BitInputOffsets ) { + TempDir temp; + VirtualFileSystem vfs; + if ( !available( utest_result, vfs ) ) + return; + const Uint64 largeSize = Uint64( 3 ) * 1024 * 1024 * 1024; + TarFixture header; + header.entry( "large", "", '0', {}, largeSize ); + const std::string path = temp.path + "large.tar"; + ASSERT_TRUE( FileSystem::fileWrite( path, header.bytes ) ); +#if EE_PLATFORM == EE_PLATFORM_WIN + // Mark sparse before extending: the test must not allocate 3 GiB on disk. + HANDLE file = CreateFileW( String::fromUtf8( path ).toWideString().c_str(), GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr ); + ASSERT_TRUE( file != INVALID_HANDLE_VALUE ); + DWORD returned = 0; + bool sparse = + DeviceIoControl( file, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &returned, nullptr ) != 0; + CloseHandle( file ); + if ( !sparse ) + UTEST_SKIP( "Temporary filesystem does not support sparse files" ); +#endif + TarFixture tail; + tail.entry( "tail", "payload" ); + { + const std::filesystem::path nativePath( std::u8string( path.begin(), path.end() ) ); + std::fstream stream( nativePath, std::ios::binary | std::ios::in | std::ios::out ); + stream.seekp( static_cast( 512 + largeSize ) ); + const std::string end = tail.bytes + std::string( 1024, '\0' ); + stream.write( end.data(), end.size() ); + ASSERT_TRUE( stream.good() ); + } + const URI root = ArchiveProvider::makeURI( URI::fromFSPath( path ) ); + auto listed = waitFor( vfs.listAll( root ) ); + ASSERT_TRUE( listed.hasValue() ); + EXPECT_EQ( 2u, listed.value().size() ); + EXPECT_STDSTREQ( "payload", readString( vfs, root.withAppendedSegment( "tail" ) ) ); + EXPECT_EQ( + largeSize, + waitFor( vfs.stat( root.withAppendedSegment( "large" ) ) ).value().size().value_or( 0 ) ); +} + +UTEST( ArchiveProvider, missingLibraryIsOptionalAndReadOnly ) { + auto pool = ThreadPool::createShared( 1 ); + auto scheduler = std::make_shared( pool, Seconds( 1 ) ); + TempDir temp; + ArchiveProvider provider( scheduler, temp.path + "missing-libarchive" ); + EXPECT_EQ( VFSErrorCode::Unsupported, codeOf( waitFor( provider.support() ) ) ); + const URI file = + ArchiveProvider::makeURI( URI::fromFSPath( temp.path + "archive.tar" ), "/file" ); + EXPECT_EQ( VFSErrorCode::Unsupported, codeOf( waitFor( provider.stat( file, {}, {} ) ) ) ); + EXPECT_EQ( VFSErrorCode::ReadOnly, + codeOf( waitFor( provider.open( file, OpenFlags::Write, {} ) ) ) ); + EXPECT_EQ( VFSErrorCode::ReadOnly, codeOf( waitFor( provider.remove( file, {}, {} ) ) ) ); + CancellationSource cancelled; + cancelled.cancel(); + EXPECT_EQ( VFSErrorCode::Cancelled, + codeOf( waitFor( provider.support( cancelled.token() ) ) ) ); + EXPECT_EQ( VFSErrorCode::Cancelled, + codeOf( waitFor( provider.stat( file, {}, cancelled.token() ) ) ) ); +} + +UTEST( ArchiveProvider, tarAndGzipListStreamSeekAndCopy ) { + TempDir temp; + VirtualFileSystem vfs; + if ( !available( utest_result, vfs ) ) + return; + TarFixture fixture; + std::string content( 200000, 'x' ); + for ( size_t i = 0; i < content.size(); ++i ) + content[i] = static_cast( 'a' + i % 26 ); + fixture.entry( "./dir/file.txt", content ); + fixture.entry( "empty/", "", '5' ); + fixture.entry( "zero", "" ); + const auto generation = PackFileIndex::instance()->getGeneration(); + for ( bool gzip : { false, true } ) { + const std::string path = temp.path + ( gzip ? "archivo-ñ-雪.tar.gz" : "archive.tar" ); + ASSERT_TRUE( fixture.write( path, gzip ) ); + const URI root = ArchiveProvider::makeURI( URI::fromFSPath( path ) ); + auto list = waitFor( vfs.listAll( root ) ); + ASSERT_TRUE( list.hasValue() ); + EXPECT_TRUE( sortedNames( list.value() ) == + std::vector( { "dir", "empty", "zero" } ) ); + EXPECT_TRUE( + waitFor( vfs.listAll( root.withAppendedSegment( "empty" ) ) ).value().empty() ); + const URI file = root.withAppendedSegment( "dir" ).withAppendedSegment( "file.txt" ); + auto opened = waitFor( vfs.open( file ) ); + ASSERT_TRUE( opened.hasValue() ); + auto handle = std::move( opened ).value(); + std::array bytes{}; + for ( Uint64 offset : { 150000u, 0u, 199990u, 5u } ) { + auto got = waitFor( handle.read( offset, bytes ) ); + ASSERT_TRUE( got.hasValue() ); + EXPECT_EQ( std::min( bytes.size(), content.size() - offset ), got.value() ); + EXPECT_TRUE( std::string( bytes.begin(), bytes.begin() + got.value() ) == + content.substr( offset, got.value() ) ); + } + EXPECT_FALSE( handle.capabilities().has( VFSCapability::RandomAccess ) ); + EXPECT_TRUE( waitFor( handle.close() ).hasValue() ); + EXPECT_STDSTREQ( "", readString( vfs, root.withAppendedSegment( "zero" ) ) ); + const URI destination = URI::fromFSPath( temp.path + ( gzip ? "gzip-copy" : "tar-copy" ) ); + EXPECT_TRUE( + waitFor( vfs.copy( root.withAppendedSegment( "dir" ), destination ).completion() ) + .hasValue() ); + EXPECT_TRUE( content == readString( vfs, destination.withAppendedSegment( "file.txt" ) ) ); + const URI refused = destination.withAppendedSegment( "must-not-exist" ); + EXPECT_EQ( VFSErrorCode::Unsupported, + codeOf( waitFor( vfs.move( file, refused ).completion() ) ) ); + EXPECT_EQ( VFSErrorCode::NotFound, codeOf( waitFor( vfs.stat( refused ) ) ) ); + } + EXPECT_EQ( generation, PackFileIndex::instance()->getGeneration() ); +} + +UTEST( ArchiveProvider, extractRootIntoExistingFolder ) { + TempDir temp; + VirtualFileSystem vfs; + if ( !available( utest_result, vfs ) ) + return; + TarFixture tar; + tar.entry( ".hidden", "hidden" ); + tar.entry( "sub/file.txt", "contents" ); + tar.entry( "empty/", "", '5' ); + ASSERT_TRUE( tar.write( temp.path + "archive.tar.gz", true ) ); + const URI root = + eefm::Location::archiveRoot( temp.uri().withAppendedSegment( "archive.tar.gz" ) ); + CopyOptions options; + options.overwrite = OverwritePolicy::MergeDirectories; + auto copied = waitFor( vfs.copy( root, temp.uri(), options ).completion() ); + ASSERT_TRUE( copied ); + EXPECT_FALSE( copied.value().partial() ); + EXPECT_EQ( 2u, copied.value().filesTransferred ); + EXPECT_STDSTREQ( "hidden", readString( vfs, temp.uri().withAppendedSegment( ".hidden" ) ) ); + EXPECT_STDSTREQ( "contents", + readString( vfs, temp.uri().withAppendedSegment( "sub" ).withAppendedSegment( + "file.txt" ) ) ); + auto empty = waitFor( vfs.stat( temp.uri().withAppendedSegment( "empty" ) ) ); + ASSERT_TRUE( empty ); + EXPECT_TRUE( empty.value().isDirectory() ); + EXPECT_STDSTREQ( "hidden", readString( vfs, root.withAppendedSegment( ".hidden" ) ) ); +} + +UTEST( ArchiveProvider, concurrentReadersAndSequentialDirectoryCopy ) { + TempDir temp; + VirtualFileSystem vfs; + if ( !available( utest_result, vfs ) ) + return; + TarFixture fixture; + for ( int i = 0; i < 64; ++i ) + fixture.entry( "dir/file-" + std::to_string( i ), "contents " + std::to_string( i ) ); + const std::string path = temp.path + "sequential.tar.gz"; + ASSERT_TRUE( fixture.write( path, true ) ); + const URI root = ArchiveProvider::makeURI( URI::fromFSPath( path ) ); + const URI directory = root.withAppendedSegment( "dir" ); + auto first = waitFor( vfs.open( directory.withAppendedSegment( "file-0" ) ) ); + auto second = waitFor( vfs.open( directory.withAppendedSegment( "file-1" ) ) ); + ASSERT_TRUE( first.hasValue() ); + ASSERT_TRUE( second.hasValue() ); + auto a = std::move( first ).value(); + auto b = std::move( second ).value(); + std::array bytes{}; + EXPECT_EQ( 4u, waitFor( a.read( 2, bytes ) ).value() ); + EXPECT_TRUE( waitFor( b.close() ).hasValue() ); + // Closing one reader must not change another's position or its backward-seek behavior. + EXPECT_EQ( 4u, waitFor( a.read( 0, bytes ) ).value() ); + EXPECT_TRUE( std::string( bytes.begin(), bytes.end() ) == "cont" ); + EXPECT_TRUE( waitFor( a.close() ).hasValue() ); + const URI output = temp.uri().withAppendedSegment( "copied" ); + auto copied = waitFor( vfs.copy( directory, output ).completion() ); + ASSERT_TRUE( copied.hasValue() ); + EXPECT_EQ( 64u, copied.value().filesTransferred ); + for ( int i = 0; i < 64; ++i ) { + EXPECT_STDSTREQ( + "contents " + std::to_string( i ), + readString( vfs, output.withAppendedSegment( "file-" + std::to_string( i ) ) ) ); + } +} + +UTEST( ArchiveProvider, gnuAndPaxNamesAndSizeOverrides ) { + TempDir temp; + VirtualFileSystem vfs; + if ( !available( utest_result, vfs ) ) + return; + TarFixture fixture; + const std::string longName = "gnu/" + std::string( 180, 'n' ); + fixture.entry( "././@LongLink", longName + '\0', 'L' ); + fixture.entry( "placeholder", "gnu content" ); + const std::string paxName = "pax/" + std::string( 180, 'p' ); + fixture.entry( "PaxHeader", TarFixture::pax( "path=" + paxName ) + TarFixture::pax( "size=11" ), + 'x' ); + fixture.entry( "placeholder", "pax content", '0', {}, 0 ); + fixture.entry( "tail", "tail content" ); + const auto path = temp.path + "extensions.tar.gz"; + ASSERT_TRUE( fixture.write( path, true ) ); + const URI root = ArchiveProvider::makeURI( URI::fromFSPath( path ) ); + EXPECT_STDSTREQ( "gnu content", readString( vfs, ArchiveProvider::makeURI( + URI::fromFSPath( path ), longName ) ) ); + EXPECT_STDSTREQ( "pax content", readString( vfs, ArchiveProvider::makeURI( + URI::fromFSPath( path ), paxName ) ) ); + EXPECT_STDSTREQ( "tail content", readString( vfs, root.withAppendedSegment( "tail" ) ) ); +} + +UTEST( ArchiveProvider, runtimeFiltersAndSevenZip ) { + TempDir temp; + VirtualFileSystem vfs; + if ( !available( utest_result, vfs ) ) + return; + auto provider = std::dynamic_pointer_cast( vfs.provider( "archive" ) ); + auto support = waitFor( provider->support() ); + ASSERT_TRUE( support.hasValue() ); + struct Fixture { + const char* name; + const char* filter; + const char* format; + const char* data; + }; + // Generated independently using Python tarfile/bz2/lzma and libarchive's 7zip writer. + for ( const auto& fixture : + { Fixture{ "archive.tar.bz2", "bzip2", "tar", + "QlpoOTFBWSZTWRGkBHYAAG97gMmAAAhAAEWAAARnJN4gCAggAFQ0jUxDATR6maCSKaGmg0GTQfTiO" + "hBOCEIlVsTWUcECGBjRxOwjZ0UMctBnNo9/TCY+TXCvVVOZEQGxdyRThQkBGkBHYA==" }, + Fixture{ "archive.tar.xz", "xz", "tar", + "/Td6WFoAAATm1rRGAgAhARYAAAB0L+Wj4Cf/AF1dADMaSda2OCGvAe5uBXM7wPyAzzewSHXXbqSo/" + "hdgBa5T3GsxIACRvOjdMLsxnrOfIgBYUuJzInp0sdefWq+cGCAurmuD6EUsI9irRPY0Dg3L73/" + "zP4rTIqzcpMEqAAAAAAB5/rNswT5g4QABeYBQAAAA5DeTlbHEZ/sCAAAAAARZWg==" }, + Fixture{ "archive.7z", "none", "7zip", + "N3q8ryccAAP0BzRLBwAAAAAAAAA0AAAAAAAAAF5/" + "nFVwYXlsb2FkAQQGAAEJBwAHCwEAAQEADAcACAoBFWosQgAABQERCwBmAGkAbABlAAAAFQYBACGAA" + "IAAAA==" } } ) { + if ( std::find( support.value().filters.begin(), support.value().filters.end(), + fixture.filter ) == support.value().filters.end() || + std::find( support.value().formats.begin(), support.value().formats.end(), + fixture.format ) == support.value().formats.end() ) + continue; + const std::string path = temp.path + fixture.name; + std::string bytes; + Base64::decode( fixture.data, bytes ); + ASSERT_TRUE( FileSystem::fileWrite( path, bytes ) ); + EXPECT_STDSTREQ( "payload", readString( vfs, ArchiveProvider::makeURI( + URI::fromFSPath( path ), "/file" ) ) ); + } +} + +UTEST( ArchiveProvider, recognizedFormatCanHaveAnUnavailableCodec ) { + TempDir temp; + VirtualFileSystem vfs; + if ( !available( utest_result, vfs ) ) + return; + auto provider = std::dynamic_pointer_cast( vfs.provider( "archive" ) ); + auto support = waitFor( provider->support() ); + ASSERT_TRUE( support.hasValue() ); + if ( std::find( support.value().formats.begin(), support.value().formats.end(), "7zip" ) == + support.value().formats.end() ) + UTEST_SKIP( "7zip reader not available" ); + std::string bytes; + Base64::decode( + "N3q8ryccAANoCeHQEQAAAAAAAAA8AAAAAAAAAMgYqcgAOBhLmXUPRgxO4E///" + "d8QAAEEBgABCREABwsBAAEjAwEBBV0AAIAADAcACAoBFWosQgAABQERCwBmAGkAbABlAAAAFQYBACGAAIAAAA==", + bytes ); + const std::string path = temp.path + "lzma.7z"; + ASSERT_TRUE( FileSystem::fileWrite( path, bytes ) ); + auto result = + waitFor( vfs.readAll( ArchiveProvider::makeURI( URI::fromFSPath( path ), "/file" ) ) ); + if ( std::find( support.value().filters.begin(), support.value().filters.end(), "lzma" ) == + support.value().filters.end() ) { + EXPECT_TRUE( result.hasError() ); + } else { + ASSERT_TRUE( result.hasValue() ); + EXPECT_STDSTREQ( "payload", std::string( result.value().begin(), result.value().end() ) ); + } +} + +UTEST( ArchiveProvider, aggregateMetadataIsBounded ) { + TempDir temp; + VirtualFileSystem vfs; + if ( !available( utest_result, vfs ) ) + return; + TarFixture fixture; + std::string nested; + for ( int i = 0; i < 100; ++i ) + nested += "directory-name-long-enough-for-budget/"; + for ( int i = 0; i < 512; ++i ) { + fixture.entry( "././@LongLink", std::to_string( i ) + "/" + nested + "file" + '\0', 'L' ); + fixture.entry( "file", "" ); + } + const std::string path = temp.path + "metadata.tar"; + ASSERT_TRUE( fixture.write( path ) ); + EXPECT_EQ( + VFSErrorCode::InvalidOperation, + codeOf( waitFor( vfs.listAll( ArchiveProvider::makeURI( URI::fromFSPath( path ) ) ) ) ) ); +} + +UTEST( ArchiveProvider, unsafePathsLinksAndCollisionsStayInsideArchive ) { + TempDir temp; + VirtualFileSystem vfs; + if ( !available( utest_result, vfs ) ) + return; + TarFixture fixture; + fixture.entry( "dir/file", "inside" ); + fixture.entry( "dir/file", "duplicate" ); + fixture.entry( "collision", "hidden" ); + fixture.entry( "collision/child", "child" ); + fixture.entry( "../evil", "evil" ); + fixture.entry( "/absolute", "evil" ); + fixture.entry( "C:drive", "evil" ); + fixture.entry( "back\\slash", "evil" ); + fixture.entry( "file-link", "", '2', "dir/file" ); + fixture.entry( ".hidden-link", "", '2', "dir/file" ); + fixture.entry( "dir-link", "", '2', "dir" ); + fixture.entry( "hard-link", "", '1', "dir/file" ); + fixture.entry( "cycle", "", '2', "cycle" ); + fixture.entry( "outside", "", '2', "../../secret" ); + fixture.entry( "pipe", "", '6' ); + const auto path = temp.path + "links.tar"; + ASSERT_TRUE( fixture.write( path ) ); + const URI root = ArchiveProvider::makeURI( URI::fromFSPath( path ) ); + auto listed = waitFor( vfs.listAll( root ) ); + ASSERT_TRUE( listed.hasValue() ); + EXPECT_EQ( 9u, listed.value().size() ); + const URI hidden = root.withAppendedSegment( ".hidden-link" ); + auto hiddenInfo = waitFor( vfs.stat( hidden ) ); + ASSERT_TRUE( hiddenInfo.hasValue() ); + EXPECT_TRUE( hiddenInfo.value().hidden ); + EXPECT_STDSTREQ( ".hidden-link", hiddenInfo.value().name ); + EXPECT_TRUE( waitFor( vfs.canonicalize( hidden ) ).value() == + root.withAppendedSegment( "dir" ).withAppendedSegment( "file" ) ); + EXPECT_STDSTREQ( "inside", readString( vfs, root.withAppendedSegment( "file-link" ) ) ); + EXPECT_STDSTREQ( "inside", readString( vfs, root.withAppendedSegment( "hard-link" ) ) ); + EXPECT_STDSTREQ( + "inside", + readString( vfs, root.withAppendedSegment( "dir-link" ).withAppendedSegment( "file" ) ) ); + EXPECT_STDSTREQ( + "child", + readString( vfs, root.withAppendedSegment( "collision" ).withAppendedSegment( "child" ) ) ); + EXPECT_EQ( VFSErrorCode::TooManySymlinks, + codeOf( waitFor( vfs.open( root.withAppendedSegment( "cycle" ) ) ) ) ); + EXPECT_EQ( VFSErrorCode::NotFound, + codeOf( waitFor( vfs.open( root.withAppendedSegment( "outside" ) ) ) ) ); + EXPECT_EQ( VFSErrorCode::Unsupported, + codeOf( waitFor( vfs.open( root.withAppendedSegment( "pipe" ) ) ) ) ); + EXPECT_TRUE( + waitFor( vfs.stat( root.withAppendedSegment( "file-link" ), StatOptions{ false } ) ) + .value() + .isSymlink() ); +} + +UTEST( ArchiveProvider, missingInvalidCorruptAndTruncatedArchivesReportErrors ) { + TempDir temp; + VirtualFileSystem vfs; + if ( !available( utest_result, vfs ) ) + return; + EXPECT_EQ( VFSErrorCode::NotFound, codeOf( waitFor( vfs.stat( ArchiveProvider::makeURI( + URI::fromFSPath( temp.path + "missing.tar" ) ) ) ) ) ); + ASSERT_TRUE( FileSystem::fileWrite( temp.path + "invalid.tar", "not an archive" ) ); + EXPECT_TRUE( waitFor( vfs.listAll( ArchiveProvider::makeURI( + URI::fromFSPath( temp.path + "invalid.tar" ) ) ) ) + .hasError() ); + TarFixture fixture; + fixture.entry( "file", "contents" ); + fixture.bytes[10] ^= 1; + ASSERT_TRUE( fixture.write( temp.path + "checksum.tar" ) ); + EXPECT_TRUE( waitFor( vfs.listAll( ArchiveProvider::makeURI( + URI::fromFSPath( temp.path + "checksum.tar" ) ) ) ) + .hasError() ); + TarFixture truncated; + truncated.entry( "file", "short", '0', {}, 65536 ); + ASSERT_TRUE( truncated.write( temp.path + "truncated.tar" ) ); + EXPECT_TRUE( waitFor( vfs.listAll( ArchiveProvider::makeURI( + URI::fromFSPath( temp.path + "truncated.tar" ) ) ) ) + .hasError() ); +} + +UTEST( ArchiveProvider, truncatedGzipIsReported ) { + TempDir temp; + VirtualFileSystem vfs; + if ( !available( utest_result, vfs ) ) + return; + TarFixture fixture; + fixture.entry( "file", "contents" ); + const std::string path = temp.path + "corrupt.tar.gz"; + ASSERT_TRUE( fixture.write( path, true ) ); + std::string bytes; + ASSERT_TRUE( FileSystem::fileGet( path, bytes ) ); + ASSERT_GT( bytes.size(), 8u ); + const URI file = ArchiveProvider::makeURI( URI::fromFSPath( path ), "/file" ); + bytes.resize( bytes.size() - 6 ); + ASSERT_TRUE( FileSystem::fileWrite( path, bytes ) ); + EXPECT_TRUE( waitFor( vfs.readAll( file ) ).hasError() ); +} + +UTEST( ArchiveProvider, replacementCacheEvictionAndReaderLifetime ) { + TempDir temp; + VirtualFileSystem vfs; + if ( !available( utest_result, vfs ) ) + return; + TarFixture fixture; + fixture.entry( "file", "original" ); + const auto path = temp.path + "original.tar"; + ASSERT_TRUE( fixture.write( path ) ); + const URI root = ArchiveProvider::makeURI( URI::fromFSPath( path ) ); + auto opened = waitFor( vfs.open( root.withAppendedSegment( "file" ) ) ); + ASSERT_TRUE( opened.hasValue() ); + auto handle = std::move( opened ).value(); + for ( int i = 0; i < 10; ++i ) { + const auto other = temp.path + std::to_string( i ) + ".tar"; + ASSERT_TRUE( fixture.write( other ) ); + ASSERT_TRUE( waitFor( vfs.listAll( ArchiveProvider::makeURI( URI::fromFSPath( other ) ) ) ) + .hasValue() ); + } + vfs.unregisterProvider( "archive" ); + std::array bytes{}; + auto read = waitFor( handle.read( 0, bytes ) ); + ASSERT_TRUE( read.hasValue() ); + EXPECT_TRUE( std::string( bytes.begin(), bytes.end() ) == "original" ); + EXPECT_TRUE( waitFor( handle.close() ).hasValue() ); + + // Invalidate an index after replacement, including same-length changes. + VirtualFileSystem next; + EXPECT_STDSTREQ( "original", readString( next, root.withAppendedSegment( "file" ) ) ); + ASSERT_TRUE( FileSystem::fileRemove( path ) ); + TarFixture replacement; + replacement.entry( "file", "replaced" ); + ASSERT_TRUE( replacement.write( path ) ); + EXPECT_STDSTREQ( "replaced", readString( next, root.withAppendedSegment( "file" ) ) ); +} diff --git a/src/tests/unit_tests/vfs_tests.cpp b/src/tests/unit_tests/vfs_tests.cpp index ed11669c2..cb8ce2fb4 100644 --- a/src/tests/unit_tests/vfs_tests.cpp +++ b/src/tests/unit_tests/vfs_tests.cpp @@ -346,6 +346,49 @@ UTEST( VFSTransfer, crossProviderCopyAndProgress ) { EXPECT_STDSTREQ( "new", readString( vfs, destination.withAppendedSegment( "a.txt" ) ) ); } +UTEST( VFSTransfer, directoryMergeRequiresExplicitFileOverwrite ) { + TempDir tmp; + VirtualFileSystem vfs; + auto mem = std::make_shared(); + vfs.registerProvider( mem ); + const URI source( "mem://s/tree" ); + const URI nested = tmp.uri().withAppendedSegment( "nested" ); + const URI file = nested.withAppendedSegment( "file.txt" ); + mem->putDirectory( source.withAppendedSegment( "empty" ) ); + mem->putFile( source.withAppendedSegment( "nested" ).withAppendedSegment( "file.txt" ), "new" ); + ASSERT_TRUE( waitFor( vfs.makeDir( nested ) ) ); + ASSERT_TRUE( waitFor( vfs.writeAll( tmp.uri().withAppendedSegment( "unrelated" ), "keep" ) ) ); + // The ordinary Fail policy still rejects the existing root directory. + EXPECT_EQ( VFSErrorCode::AlreadyExists, + codeOf( waitFor( vfs.copy( source, tmp.uri() ).completion() ) ) ); + CopyOptions options; + options.overwrite = OverwritePolicy::MergeDirectories; + auto copied = waitFor( vfs.copy( source, tmp.uri(), options ).completion() ); + ASSERT_TRUE( copied ); + EXPECT_EQ( 1u, copied.value().filesTransferred ); + EXPECT_STDSTREQ( "new", readString( vfs, file ) ); + EXPECT_STDSTREQ( "keep", readString( vfs, tmp.uri().withAppendedSegment( "unrelated" ) ) ); + ASSERT_TRUE( + waitFor( vfs.stat( tmp.uri().withAppendedSegment( "empty" ) ) ).value().isDirectory() ); + + ASSERT_TRUE( waitFor( vfs.writeAll( file, "existing" ) ) ); + auto conflict = waitFor( vfs.copy( source, tmp.uri(), options ).completion() ); + ASSERT_FALSE( conflict ); + EXPECT_EQ( VFSErrorCode::AlreadyExists, conflict.error().code ); + EXPECT_TRUE( conflict.error().uri == file ); + EXPECT_STDSTREQ( "existing", readString( vfs, file ) ); + // The policy also protects a file copied directly, without a directory root. + EXPECT_EQ( + VFSErrorCode::AlreadyExists, + codeOf( waitFor( + vfs.copy( source.withAppendedSegment( "nested" ).withAppendedSegment( "file.txt" ), + file, options ) + .completion() ) ) ); + options.overwrite = OverwritePolicy::Overwrite; + ASSERT_TRUE( waitFor( vfs.copy( source, tmp.uri(), options ).completion() ) ); + EXPECT_STDSTREQ( "new", readString( vfs, file ) ); +} + UTEST( VFSTransfer, moveNeverRemovesUncopiedSources ) { VirtualFileSystem vfs; auto memA = std::make_shared( "mema" ); @@ -748,7 +791,8 @@ UTEST( VFSSafety, unsafeReplacementIsRefusedWithoutAtomicReplace ) { } UTEST( VFSSafety, concurrentDestinationRespectsFailAndSkip ) { - for ( auto policy : { OverwritePolicy::Fail, OverwritePolicy::Skip } ) { + for ( auto policy : + { OverwritePolicy::Fail, OverwritePolicy::Skip, OverwritePolicy::MergeDirectories } ) { VirtualFileSystem vfs; auto destination = std::make_shared(); auto source = std::make_shared( destination ); @@ -760,7 +804,7 @@ UTEST( VFSSafety, concurrentDestinationRespectsFailAndSkip ) { auto result = waitFor( vfs.move( URI( "race://t/source" ), URI( "mem://t/destination" ), options ) .completion() ); - if ( policy == OverwritePolicy::Fail ) { + if ( policy != OverwritePolicy::Skip ) { ASSERT_FALSE( result.hasValue() ); EXPECT_EQ( VFSErrorCode::AlreadyExists, result.error().code ); } else { diff --git a/src/tests/unit_tests/zipprovider_tests.cpp b/src/tests/unit_tests/zipprovider_tests.cpp index 7d15953ae..f30cbe3fa 100644 --- a/src/tests/unit_tests/zipprovider_tests.cpp +++ b/src/tests/unit_tests/zipprovider_tests.cpp @@ -302,6 +302,52 @@ UTEST( Zip, standaloneReaderIsReadOnlyAndHasBoundedSeek ) { EXPECT_TRUE( missing.error() == Zip::Error::InvalidArchive ); } +UTEST( ZipProvider, extractRootIntoExistingFolder ) { + TempDir temp; + VirtualFileSystem vfs; + const URI file = temp.uri().withAppendedSegment( "archive.zip" ); + ASSERT_TRUE( + makeZip( file.getFSPath(), + { { ".hidden", "hidden" }, { "sub/file.txt", "contents" }, { "empty", "" } } ) ); + CopyOptions options; + options.overwrite = OverwritePolicy::MergeDirectories; + auto copied = + waitFor( vfs.copy( ZipProvider::makeURI( file ), temp.uri(), options ).completion() ); + ASSERT_TRUE( copied ); + EXPECT_FALSE( copied.value().partial() ); + EXPECT_EQ( 3u, copied.value().filesTransferred ); + EXPECT_STDSTREQ( "hidden", readString( vfs, temp.uri().withAppendedSegment( ".hidden" ) ) ); + EXPECT_STDSTREQ( "contents", + readString( vfs, temp.uri().withAppendedSegment( "sub" ).withAppendedSegment( + "file.txt" ) ) ); + EXPECT_STDSTREQ( "", readString( vfs, temp.uri().withAppendedSegment( "empty" ) ) ); + // The backing archive remains usable after extraction into its containing folder. + EXPECT_STDSTREQ( "hidden", readString( vfs, ZipProvider::makeURI( file, "/.hidden" ) ) ); +} + +UTEST( ZipLocation, extractionFolderNames ) { + TempDir temp; + for ( const auto& [file, folder] : { std::pair{ "Mixed Case.zip", "Mixed Case" }, + { "release.TAR.GZ", "release" }, + { "package.tar.bz2", "package" }, + { "package.tar.xz", "package" }, + { "package.tar.lzma", "package" }, + { "package.tar.lz", "package" }, + { "package.tar.zst", "package" }, + { "package.tar.lz4", "package" }, + { "package.tar.Z", "package" }, + { "package.tgz", "package" }, + { "release.v1.7z", "release.v1" }, + { "a! #%雪.epub", "a! #%雪" }, + { ".hidden.zip", ".hidden" }, + { ".tar.gz", "archive" }, + { "..zip", "archive" }, + { "...zip", "archive" } } ) { + EXPECT_STDSTREQ( + folder, eefm::Location::archiveFolderName( temp.uri().withAppendedSegment( file ) ) ); + } +} + UTEST( ZipLocation, zipBasedFormatsUseTheSameProvider ) { TempDir temp; VirtualFileSystem vfs; diff --git a/src/tools/eefm/fileoperations.cpp b/src/tools/eefm/fileoperations.cpp index 14f88bf7d..0edfe637f 100644 --- a/src/tools/eefm/fileoperations.cpp +++ b/src/tools/eefm/fileoperations.cpp @@ -393,6 +393,41 @@ void FileOperations::showContextMenu( FileTab* tab, bool background ) { ->setId( "open-other" ); } } + if ( !inTrash && uris.size() == 1 && !anyFolder && + !Location::archiveRoot( uris[0] ).empty() ) { + const URI file = uris[0]; + const std::string folderName = Location::archiveFolderName( file ); + const URI folder = file.getParent().withAppendedSegment( folderName ); + const int paneIndex = tab->pane->index; + UIPopUpMenu* extract = UIPopUpMenu::New(); + extract->add( String::format( "Extract to \"%s/\"", folderName.c_str() ) ) + ->setId( "extract-folder" ); + extract->add( mUI->i18n( "eefm_extract_here", "Extract Here" ) ) + ->setId( "extract-here" ); + extract->add( mUI->i18n( "eefm_extract_to", "Extract To..." ) )->setId( "extract-to" ); + extract->on( + Event::OnItemClicked, [this, file, folder, paneIndex]( const Event* event ) { + const std::string& id = event->getNode()->getId(); + if ( id == "extract-here" ) { + extractArchive( file, file.getParent(), paneIndex ); + } else if ( id == "extract-folder" ) { + extractArchive( file, folder, paneIndex ); + } else if ( id == "extract-to" ) { + Dialogs::promptText( + mUI, mUI->i18n( "eefm_extract_to", "Extract To..." ), + mUI->i18n( "eefm_extract_destination", + "Destination folder (path or URI):" ), + Location::text( folder ).toUtf8(), false, + [this, file, paneIndex]( std::optional destination ) { + if ( !destination || String::trim( *destination ).empty() ) + return; + extractArchive( file, Location::parse( *destination ), paneIndex ); + } ); + } + } ); + menu->addSubMenu( mUI->i18n( "eefm_extract", "Extract" ), mApp.icon( "mime-archive" ), + extract ); + } // After the open actions, as Dolphin. if ( menu->getCount() > 0 ) menu->addSeparator(); @@ -640,22 +675,35 @@ void FileOperations::transferFiles( const std::vector& sources, const URI& auto [base, extension] = FileSystem::fileNameSplitExtension( name ); target = destination.withAppendedSegment( base + " (copy)" + extension ); } - startTransfer( source, target, move, paneIndex ); + startTransfer( source, target, move ? TransferKind::Move : TransferKind::Copy, paneIndex ); } } -void FileOperations::startTransfer( const URI& source, const URI& destination, bool move, +void FileOperations::extractArchive( const URI& file, const URI& destination, int paneIndex ) { + URI root = Location::archiveRoot( file ); + if ( root.empty() ) + return; + CopyOptions options; + options.overwrite = OverwritePolicy::MergeDirectories; + // Copy the root as one transfer: entries are streamed sequentially, and an empty archive + // still creates its destination folder. Existing files require an explicit overwrite. + startTransfer( root, destination, TransferKind::Extract, paneIndex, options ); +} + +void FileOperations::startTransfer( const URI& source, const URI& destination, TransferKind kind, int paneIndex, const CopyOptions& options ) { + const bool move = kind == TransferKind::Move; + String label = kind == TransferKind::Extract ? mUI->i18n( "eefm_extracting", "Extracting " ) + : move ? mUI->i18n( "eefm_moving", "Moving " ) + : mUI->i18n( "eefm_copying", "Copying " ); Uint64 id = ++mLastTransferId; - ActiveTransfer active{ - id, - paneIndex, - ( move ? mUI->i18n( "eefm_moving", "Moving " ) : mUI->i18n( "eefm_copying", "Copying " ) ) + - Location::displayName( source ), - move ? mVFS->move( source, destination, options ) - : mVFS->copy( source, destination, options ), - {}, - {} }; + ActiveTransfer active{ id, + paneIndex, + std::move( label ) + Location::displayName( source ), + move ? mVFS->move( source, destination, options ) + : mVFS->copy( source, destination, options ), + {}, + {} }; active.subscription = active.transfer.onProgress( [this, id]( const TransferProgress& progress ) { auto it = mTransfers.find( id ); @@ -669,7 +717,7 @@ void FileOperations::startTransfer( const URI& source, const URI& destination, b mTransfers.emplace( id, std::move( active ) ); updateTransfers(); std::move( completion ) - .then( mApp.executor(), [this, id, source, destination, move, paneIndex, + .then( mApp.executor(), [this, id, source, destination, kind, paneIndex, options]( VFSResult&& result ) { auto it = mTransfers.find( id ); String label = it != mTransfers.end() ? it->second.label : String(); @@ -690,25 +738,33 @@ void FileOperations::startTransfer( const URI& source, const URI& destination, b URI folder = destination.getParent(); for ( auto& pane : mApp.panes().all() ) { for ( auto& tab : pane->tabList ) { - if ( tab->model->uri() == folder.getParent() && - folder != tab->model->uri() ) + if ( ( tab->model->uri() == folder.getParent() && + folder != tab->model->uri() ) || + ( kind == TransferKind::Extract && tab->model->uri() == destination ) ) tab->model->refresh(); } } return; } if ( result.error().is( VFSErrorCode::AlreadyExists ) && - options.overwrite == OverwritePolicy::Fail ) { + ( options.overwrite == OverwritePolicy::Fail || + options.overwrite == OverwritePolicy::MergeDirectories ) ) { + String question = + kind == TransferKind::Extract + ? mUI->i18n( "eefm_extract_overwrite", + "The extraction destination contains existing files. " + "Overwrite them?\n" ) + + Location::text( destination ) + : String( String::format( "\"%s\" already exists. Overwrite it?", + destination.getLastPathSegment().c_str() ) ); Dialogs::confirm( - mUI, mUI->i18n( "eefm_file_exists", "File Already Exists" ), - String::format( "\"%s\" already exists. Overwrite it?", - destination.getLastPathSegment().c_str() ), - [this, source, destination, move, paneIndex, options]( bool overwrite ) { + mUI, mUI->i18n( "eefm_file_exists", "File Already Exists" ), question, + [this, source, destination, kind, paneIndex, options]( bool overwrite ) { if ( !overwrite ) return; CopyOptions retry = options; retry.overwrite = OverwritePolicy::Overwrite; - startTransfer( source, destination, move, paneIndex, retry ); + startTransfer( source, destination, kind, paneIndex, retry ); } ); return; } diff --git a/src/tools/eefm/fileoperations.hpp b/src/tools/eefm/fileoperations.hpp index f3ea2f7fa..1db155806 100644 --- a/src/tools/eefm/fileoperations.hpp +++ b/src/tools/eefm/fileoperations.hpp @@ -82,6 +82,8 @@ class FileOperations { void cancelTransfers(); private: + enum class TransferKind : Uint8 { Copy, Move, Extract }; + struct ActiveTransfer { Uint64 id; int pane; @@ -128,7 +130,9 @@ class FileOperations { void transferFiles( const std::vector& sources, const URI& destination, bool move, int paneIndex ); - void startTransfer( const URI& source, const URI& destination, bool move, int paneIndex, + void extractArchive( const URI& file, const URI& destination, int paneIndex ); + + void startTransfer( const URI& source, const URI& destination, TransferKind kind, int paneIndex, const EE::VFS::CopyOptions& options = {} ); void updateTransfers(); diff --git a/src/tools/eefm/location.cpp b/src/tools/eefm/location.cpp index 07d6245b7..4fff11485 100644 --- a/src/tools/eefm/location.cpp +++ b/src/tools/eefm/location.cpp @@ -1,6 +1,8 @@ #include "location.hpp" #include +#include #include +#include using namespace EE; using namespace EE::System; @@ -8,9 +10,17 @@ using namespace EE::VFS; namespace eefm { +namespace { + +constexpr std::string_view TarSuffixes[] = { ".tar.gz", ".tar.bz2", ".tar.xz", ".tar.lzma", + ".tar.lz", ".tar.zst", ".tar.lz4", ".tar.z" }; + +} // namespace + URI Location::parse( const std::string& text ) { std::string trimmed = String::trim( text ); - if ( trimmed.find( "://" ) != std::string::npos || trimmed.starts_with( "zip:" ) ) { + if ( trimmed.find( "://" ) != std::string::npos || trimmed.starts_with( "zip:" ) || + trimmed.starts_with( "archive:" ) ) { URI uri( trimmed ); if ( uri.getPath().empty() ) uri.setRawPath( "/" ); @@ -25,28 +35,59 @@ URI Location::archiveRoot( const URI& file ) { const std::string extension = String::toLower( FileSystem::fileExtension( file.getPath() ) ); if ( extension == "zip" || extension == "jar" || extension == "apk" || extension == "epub" ) return ZipProvider::makeURI( file ); + if ( extension == "tar" || extension == "tgz" || extension == "tbz" || extension == "tbz2" || + extension == "txz" || extension == "tlz" || extension == "tzst" || extension == "7z" || + extension == "rar" || extension == "iso" || extension == "cab" || extension == "cpio" || + extension == "ar" || extension == "deb" || extension == "xar" || extension == "lha" || + extension == "lzh" ) + return ArchiveProvider::makeURI( file ); + const std::string name = String::toLower( file.getLastPathSegment() ); + for ( const auto suffix : TarSuffixes ) { + if ( name.ends_with( suffix ) ) + return ArchiveProvider::makeURI( file ); + } return {}; } +std::string Location::archiveFolderName( const URI& file ) { + std::string name = file.getLastPathSegment(); + const std::string lower = String::toLower( name ); + size_t suffix = name.find_last_of( '.' ); + for ( const auto tarSuffix : TarSuffixes ) { + if ( lower.ends_with( tarSuffix ) ) { + suffix = name.size() - tarSuffix.size(); + break; + } + } + if ( suffix != std::string::npos ) + name.resize( suffix ); + if ( name.empty() || name == "." || name == ".." ) + return "archive"; + return name; +} + URI Location::parent( const URI& uri ) { - if ( uri.getScheme() == "zip" ) { - auto archive = ZipProvider::archiveURI( uri ); - auto member = ZipProvider::memberPath( uri ); + if ( uri.getScheme() == "zip" || uri.getScheme() == "archive" ) { + const bool zip = uri.getScheme() == "zip"; + auto archive = zip ? ZipProvider::archiveURI( uri ) : ArchiveProvider::archiveURI( uri ); + auto member = zip ? ZipProvider::memberPath( uri ) : ArchiveProvider::memberPath( uri ); if ( archive && member ) { if ( member.value() == "/" ) return archive.value().getParent(); URI entry; entry.setRawPath( std::move( member ).value() ); - return ZipProvider::makeURI( archive.value(), entry.getParent().getPath() ); + return zip ? ZipProvider::makeURI( archive.value(), entry.getParent().getPath() ) + : ArchiveProvider::makeURI( archive.value(), entry.getParent().getPath() ); } } return uri.getParent(); } std::string Location::entryName( const URI& uri ) { - if ( uri.getScheme() == "zip" ) { - auto archive = ZipProvider::archiveURI( uri ); - auto member = ZipProvider::memberPath( uri ); + if ( uri.getScheme() == "zip" || uri.getScheme() == "archive" ) { + const bool zip = uri.getScheme() == "zip"; + auto archive = zip ? ZipProvider::archiveURI( uri ) : ArchiveProvider::archiveURI( uri ); + auto member = zip ? ZipProvider::memberPath( uri ) : ArchiveProvider::memberPath( uri ); if ( archive && member && member.value() == "/" ) return archive.value().getLastPathSegment(); } @@ -55,7 +96,7 @@ std::string Location::entryName( const URI& uri ) { String Location::displayName( const URI& uri ) { std::string name = entryName( uri ); - if ( uri.getScheme() == "zip" && !name.empty() ) + if ( ( uri.getScheme() == "zip" || uri.getScheme() == "archive" ) && !name.empty() ) return String::fromUtf8( name ); if ( name.empty() || uri.getPath() == "/" ) name = uri.getScheme() == "file" ? std::string( "/" ) : uri.getHost(); @@ -63,7 +104,7 @@ String Location::displayName( const URI& uri ) { } String Location::text( const URI& uri ) { - if ( uri.getScheme() == "zip" ) { + if ( uri.getScheme() == "zip" || uri.getScheme() == "archive" ) { // URI::toString() omits an empty authority for non-file schemes. Spell it explicitly // in the location bar; both spellings parse into the same provider URI. return String::fromUtf8( uri.getSchemeAndAuthority() + uri.getPathEtc() ); diff --git a/src/tools/eefm/location.hpp b/src/tools/eefm/location.hpp index 1bdcd9344..0bf29bc15 100644 --- a/src/tools/eefm/location.hpp +++ b/src/tools/eefm/location.hpp @@ -15,9 +15,14 @@ class Location { /** A typed location: a URI, or a local path ("~" expands to the home folder). */ static URI parse( const std::string& text ); - /** Browsable ZIP archive root for a local .zip, .jar, .apk or .epub file; empty otherwise. */ + /** Browsable archive root for a recognized local archive filename; empty otherwise. + * Runtime archive support is checked asynchronously by the provider when navigating. */ static URI archiveRoot( const URI& file ); + /** Suggested extraction folder, without the archive suffix (including compound TAR suffixes). + */ + static std::string archiveFolderName( const URI& file ); + /** Parent location; leaving an archive root returns to the archive's containing folder. */ static URI parent( const URI& uri );