diff --git a/docs/articles/native_drag_and_drop.md b/docs/articles/native_drag_and_drop.md index 50ed12a68..b01a26c8d 100644 --- a/docs/articles/native_drag_and_drop.md +++ b/docs/articles/native_drag_and_drop.md @@ -216,7 +216,9 @@ including one the user switches to with Alt+Tab mid-drag. Internal drag feedback the source window still receives the pointer. Ignore the internal drag's release while the native session is active and resolve it from the completion instead. On X11, a release over the source window ends the session `Cancelled` with `DragResult::releasedOverSource` set, so the application -completes it as its internal drop. Elsewhere, such a drop arrives as `FileDropped` events. +completes it as its internal drop. Elsewhere, such a drop arrives as `FileDropped` events. Set +`DragOffer::sourceHandlesDrops` so the preview shows a pending drop, not a rejected one, while the +pointer is over the source window. Applications that install their own `SDL_SetX11EventHook` must coordinate its use: SDL3 has a single hook slot, which eepp uses during an active native drag and clears afterwards. diff --git a/docs/articles/vfs.md b/docs/articles/vfs.md index c0f42a2af..88d0a384f 100644 --- a/docs/articles/vfs.md +++ b/docs/articles/vfs.md @@ -110,6 +110,12 @@ Moves use a native rename when both sides are on the same provider/authority and plus removal of the entries that were completely copied; a source is never removed after a partial transfer. +File copies are written to a temporary sibling before publication. `Fail` and `Skip` also apply +when a destination appears while the copy is running. The `Overwrite` policy requires +`AtomicReplace`; otherwise the transfer returns `Unsupported` and preserves the old contents. +Creating a file with `Fail` or `Skip` requires `Rename`. Moving a symlink through the copy fallback +is unsupported; native same-filesystem renames still move the link itself. + ## Watching ```cpp diff --git a/include/eepp/core/string.hpp b/include/eepp/core/string.hpp index 480ce7ce3..2dd75c743 100644 --- a/include/eepp/core/string.hpp +++ b/include/eepp/core/string.hpp @@ -539,6 +539,10 @@ class EE_API String { static bool iequals( String::View str1, String::View str2 ); + /** Compares ignoring ASCII case, without allocating (as comparing String::toLower() copies + * would). @return Negative, zero or positive, like std::string::compare(). */ + static int icompare( std::string_view str1, std::string_view str2 ); + /** Creates a random string using the dictionary characters. */ static std::string randString( size_t len, diff --git a/include/eepp/scene/mainthreadexecutor.hpp b/include/eepp/scene/mainthreadexecutor.hpp index 0f7a36a3c..e2eeffc5b 100644 --- a/include/eepp/scene/mainthreadexecutor.hpp +++ b/include/eepp/scene/mainthreadexecutor.hpp @@ -15,8 +15,9 @@ class Node; * deferred to the next scene update, even when execute() is called from the main thread, which is * what Task::thenOn() requires. * - * The dispatcher node must outlive the executor. Destroying the executor drops queued functions - * that have not run yet (releasing what they own, for example abandoning task sources). + * The dispatcher node must outlive the executor, or shutdown() must be called before the node is + * destroyed. Destroying the executor drops queued functions that have not run yet (releasing what + * they own, for example abandoning task sources). */ class EE_API MainThreadExecutor : public Executor { public: @@ -30,6 +31,11 @@ class EE_API MainThreadExecutor : public Executor { void execute( Function function ) override; + /** Detaches the executor from its dispatcher node and drops the queued functions. Functions + * executed afterwards are dropped too. Call it before the dispatcher node is destroyed when + * work may still be executed on this executor (it must then outlive that work). */ + void shutdown(); + /** @return Number of functions waiting for the main thread. */ size_t pending() const; diff --git a/include/eepp/system/sys.hpp b/include/eepp/system/sys.hpp index 6e5aaa479..1f86fed77 100644 --- a/include/eepp/system/sys.hpp +++ b/include/eepp/system/sys.hpp @@ -42,6 +42,13 @@ class EE_API Sys { /** @return A storage path for config files for every platform */ static std::string getConfigPath( const std::string& appname ); + /** @return A storage path for cache files (data the application can recreate) for every + * platform, private to the user: $XDG_CACHE_HOME/appname (~/.cache/appname) on Linux and BSD, + * ~/Library/Caches/appname on macOS and iOS, %LOCALAPPDATA%\appname on Windows, + * ~/config/cache/appname on Haiku. Elsewhere, or without those variables, a directory in the + * temporary path. The directory is not created. */ + static std::string getCachePath( const std::string& appname ); + /** @return The path of the directory designated for temporary files. */ static std::string getTempPath(); diff --git a/include/eepp/window/draganddrop.hpp b/include/eepp/window/draganddrop.hpp index b9e53acd4..38d1ba327 100644 --- a/include/eepp/window/draganddrop.hpp +++ b/include/eepp/window/draganddrop.hpp @@ -47,6 +47,9 @@ struct DragOffer { SmallVector files; DragActions actions{ static_cast( DragAction::Copy ) }; DragPreview preview; + /** The application completes drops on its own window (DragResult::releasedOverSource on + * X11): the preview shows a pending drop there instead of a rejected one. */ + bool sourceHandlesDrops{ false }; }; struct DragResult { diff --git a/src/eepp/core/cancellationtoken.cpp b/src/eepp/core/cancellationtoken.cpp index c78fd68b0..aad9e69ae 100644 --- a/src/eepp/core/cancellationtoken.cpp +++ b/src/eepp/core/cancellationtoken.cpp @@ -107,7 +107,10 @@ CancellationSource::CancellationSource() : mState( std::make_shared() ) {} bool CancellationSource::cancel() { - return mState && mState->cancel(); + // A callback may destroy this source (and with it the last reference to the state) while the + // state is still running callbacks: keep the state alive until cancel() returns. + std::shared_ptr state( mState ); + return state && state->cancel(); } } // namespace EE diff --git a/src/eepp/core/string.cpp b/src/eepp/core/string.cpp index 29ce8017f..bb199c221 100644 --- a/src/eepp/core/string.cpp +++ b/src/eepp/core/string.cpp @@ -2828,6 +2828,17 @@ bool String::iequals( std::string_view str1, std::string_view str2 ) { } ); } +int String::icompare( std::string_view str1, std::string_view str2 ) { + const size_t length = std::min( str1.size(), str2.size() ); + for ( size_t i = 0; i < length; ++i ) { + const int c1 = std::tolower( static_cast( str1[i] ) ); + const int c2 = std::tolower( static_cast( str2[i] ) ); + if ( c1 != c2 ) + return c1 < c2 ? -1 : 1; + } + return str1.size() < str2.size() ? -1 : ( str1.size() > str2.size() ? 1 : 0 ); +} + bool String::iequals( String::View str1, String::View str2 ) { return str1.length() == str2.length() && std::equal( str1.begin(), str1.end(), str2.begin(), diff --git a/src/eepp/scene/mainthreadexecutor.cpp b/src/eepp/scene/mainthreadexecutor.cpp index 81f869564..bb8ef5610 100644 --- a/src/eepp/scene/mainthreadexecutor.cpp +++ b/src/eepp/scene/mainthreadexecutor.cpp @@ -28,6 +28,10 @@ MainThreadExecutor::MainThreadExecutor( Node* dispatcher ) : mState( std::make_s } MainThreadExecutor::~MainThreadExecutor() { + shutdown(); +} + +void MainThreadExecutor::shutdown() { std::vector dropped; { std::lock_guard lock( mState->mutex ); diff --git a/src/eepp/system/sys.cpp b/src/eepp/system/sys.cpp index 6a67ca144..507f4f791 100644 --- a/src/eepp/system/sys.cpp +++ b/src/eepp/system/sys.cpp @@ -807,6 +807,40 @@ std::string Sys::getConfigPath( const std::string& appname ) { return std::string( path ); } +std::string Sys::getCachePath( const std::string& appname ) { + auto under = []( std::string base, const char* subdirectory, const std::string& appname ) { + FileSystem::dirAddSlashAtEnd( base ); + return base + subdirectory + appname; + }; +#if EE_PLATFORM == EE_PLATFORM_WIN + std::string local = getEnv( "LOCALAPPDATA" ); + if ( !local.empty() ) + return under( std::move( local ), "", appname ); +#elif EE_PLATFORM == EE_PLATFORM_MACOS || EE_PLATFORM == EE_PLATFORM_IOS + // On iOS HOME is the application's container. + std::string home = getEnv( "HOME" ); + if ( !home.empty() ) + return under( std::move( home ), "Library/Caches/", appname ); +#elif EE_PLATFORM == EE_PLATFORM_HAIKU + std::string home = getEnv( "HOME" ); + if ( !home.empty() ) + return under( std::move( home ), "config/cache/", appname ); +#elif EE_PLATFORM == EE_PLATFORM_LINUX || EE_PLATFORM == EE_PLATFORM_BSD || \ + EE_PLATFORM == EE_PLATFORM_SOLARIS + // The XDG Base Directory specification ignores empty and relative values. + std::string cache = getEnv( "XDG_CACHE_HOME" ); + if ( !cache.empty() && cache[0] == '/' ) + return under( std::move( cache ), "", appname ); + std::string home = getEnv( "HOME" ); + if ( !home.empty() ) + return under( std::move( home ), ".cache/", appname ); +#elif EE_PLATFORM == EE_PLATFORM_ANDROID + return under( Window::Engine::instance()->getPlatformHelper()->getInternalStoragePath(), + "cache/", appname ); +#endif + return getTempPath() + appname; +} + std::string Sys::getTempPath() { char path[EE_MAX_CFG_PATH_LEN + 1]; diff --git a/src/eepp/window/draganddrop/x11draganddrop.cpp b/src/eepp/window/draganddrop/x11draganddrop.cpp index 9f5c9dbde..22a8f6b3e 100644 --- a/src/eepp/window/draganddrop/x11draganddrop.cpp +++ b/src/eepp/window/draganddrop/x11draganddrop.cpp @@ -140,6 +140,7 @@ class X11DragSource : public DragSource { !( mask & Button1Mask ) ) return { "A left-button pointer gesture is required", DragStartStatus::InvalidGesture }; preview = NativeDragAccess::createNativeDragPreview( *owner, offer, true ); + sourceHandlesDrops = offer.sourceHandlesDrops; if ( !preview ) return { "Could not create the drag preview", DragStartStatus::Failed }; XErrorScope setup( x11, display ); @@ -481,9 +482,10 @@ class X11DragSource : public DragSource { accepted = false; waitingStatus = false; uriRequested = false; - // Over the source window the application resolves the drop itself. - preview->setFeedback( target || overOrigin ? DragFeedback::Pending - : DragFeedback::Rejected ); + // Over the source window, an application that handles its own drops resolves it. + preview->setFeedback( target || ( overOrigin && sourceHandlesDrops ) + ? DragFeedback::Pending + : DragFeedback::Rejected ); if ( target ) send( atoms[Enter], static_cast( version ) << 24, atoms[UriList] ); } @@ -728,6 +730,8 @@ class X11DragSource : public DragSource { bool pointerGrabbed{ false }; /** The pointer is over the source window (never a target), as of the last position(). */ bool overOrigin{ false }; + /** DragOffer::sourceHandlesDrops of the session. */ + bool sourceHandlesDrops{ false }; bool dirty{ true }; bool released{ false }; bool waitingStatus{ false }; diff --git a/src/modules/vfs/include/eepp/vfs/ssh/sshconfig.hpp b/src/modules/vfs/include/eepp/vfs/ssh/sshconfig.hpp index 1fb27ac1b..30737f5cf 100644 --- a/src/modules/vfs/include/eepp/vfs/ssh/sshconfig.hpp +++ b/src/modules/vfs/include/eepp/vfs/ssh/sshconfig.hpp @@ -146,9 +146,11 @@ struct SSHResolvedConfig { /** Non-fatal problems (unknown directives, failed expansions, unreadable includes). */ std::vector warnings; - /** @return A key identifying meaningfully different SSH sessions (host, port, user, proxy - * chain, authentication profile, host key alias). Never contains a filesystem path of the - * remote side. */ + /** @return A key identifying meaningfully different SSH sessions: destination (host, port, + * user, proxy chain, host key alias) and the security policy that verified and + * authenticated it (known hosts, host key checking, algorithms, authentication settings). + * Sessions are only shared between configurations with equal keys. Never contains a + * filesystem path of the remote side. */ std::string connectionKey() const; }; diff --git a/src/modules/vfs/include/eepp/vfs/ui/vfsdirectorymodel.hpp b/src/modules/vfs/include/eepp/vfs/ui/vfsdirectorymodel.hpp index aeb1a574f..deadbafb6 100644 --- a/src/modules/vfs/include/eepp/vfs/ui/vfsdirectorymodel.hpp +++ b/src/modules/vfs/include/eepp/vfs/ui/vfsdirectorymodel.hpp @@ -229,11 +229,22 @@ class VFSDirectoryModel : public Model, public std::enable_shared_from_this& entries ) const; + void setLoadState( LoadState state, std::optional error = std::nullopt ); void notifyChanged(); - bool less( const FileInfo& a, const FileInfo& b ) const; + /** The sort order. aType and bType are the entries' types when already known. */ + bool less( const FileInfo& a, const FileInfo& b, const std::string* aType = nullptr, + const std::string* bType = nullptr ) const; bool isVisible( const FileInfo& info ) const; @@ -282,6 +293,8 @@ class VFSDirectoryModel : public Model, public std::enable_shared_from_this mFilterRegEx; NameFilterMode mFilterMode{ NameFilterMode::Glob }; bool mFilterCaseSensitive{ false }; diff --git a/src/modules/vfs/include/eepp/vfs/virtualfilesystem.hpp b/src/modules/vfs/include/eepp/vfs/virtualfilesystem.hpp index 1493cca11..da3834225 100644 --- a/src/modules/vfs/include/eepp/vfs/virtualfilesystem.hpp +++ b/src/modules/vfs/include/eepp/vfs/virtualfilesystem.hpp @@ -27,15 +27,19 @@ using EE::System::ThreadPool; * - Tasks resolve on VFS or provider threads; use thenOn() with a UI executor before touching UI * or model state. * - Multi-step operations run on a dedicated orchestration pool that waits on provider tasks. - * Provider work never runs on that pool, so waits there cannot starve provider I/O. + * Provider work never runs on that pool, so waits there cannot starve provider I/O. Copies and + * moves run on a pool of their own: long transfers never delay the other operations. */ class VirtualFileSystem { public: struct Config { /** Worker threads for blocking local I/O (LocalProvider and similar). */ Uint32 ioThreads{ 4 }; - /** Worker threads for multi-step orchestration (transfers, recursive operations). */ + /** Worker threads for multi-step orchestration (whole-file reads and writes, recursive + * operations). */ Uint32 orchestrationThreads{ 2 }; + /** Copies and moves running at the same time; more wait for one to finish. */ + Uint32 transferThreads{ 2 }; /** Register file:// and res:// (and sftp:// when built with SFTP support). */ bool registerDefaultProviders{ true }; }; @@ -154,6 +158,7 @@ class VirtualFileSystem { std::unordered_map> mProviders; std::shared_ptr mIOPool; std::shared_ptr mOrchestrationPool; + std::shared_ptr mTransferPool; std::unique_ptr mWatchManager; }; diff --git a/src/modules/vfs/include/eepp/vfs/watchers/localwatchbackend.hpp b/src/modules/vfs/include/eepp/vfs/watchers/localwatchbackend.hpp index b0cb51b70..42b017e31 100644 --- a/src/modules/vfs/include/eepp/vfs/watchers/localwatchbackend.hpp +++ b/src/modules/vfs/include/eepp/vfs/watchers/localwatchbackend.hpp @@ -1,6 +1,7 @@ #ifndef EE_VFS_WATCHERS_LOCALWATCHBACKEND_HPP #define EE_VFS_WATCHERS_LOCALWATCHBACKEND_HPP +#include #include #include @@ -18,11 +19,16 @@ class LocalWatchBackendImpl; * (efsw handleMissedFileActions) become RescanRequired. * * Several logical watches on the same directory share one efsw watch. + * + * Registration and removal touch the filesystem (the generic watcher takes a snapshot of the + * tree), so they run on the I/O pool, never on the caller's thread. */ class LocalWatchBackend : public WatchBackend { public: - /** @param genericPollingInterval Polling interval for the generic efsw watcher. */ - explicit LocalWatchBackend( Time genericPollingInterval = System::Seconds( 1 ) ); + /** @param pool Where watches are registered and removed. + * @param genericPollingInterval Polling interval for the generic efsw watcher. */ + explicit LocalWatchBackend( std::shared_ptr pool, + Time genericPollingInterval = System::Seconds( 1 ) ); ~LocalWatchBackend() override; @@ -30,6 +36,7 @@ class LocalWatchBackend : public WatchBackend { std::shared_ptr sink ) override; private: + std::shared_ptr mPool; std::shared_ptr mImpl; }; diff --git a/src/modules/vfs/src/eepp/vfs/sftp/remotewatchbackend.cpp b/src/modules/vfs/src/eepp/vfs/sftp/remotewatchbackend.cpp index a21bd8687..e07555199 100644 --- a/src/modules/vfs/src/eepp/vfs/sftp/remotewatchbackend.cpp +++ b/src/modules/vfs/src/eepp/vfs/sftp/remotewatchbackend.cpp @@ -58,10 +58,7 @@ class AgentClient : public SSHChannelPump, public std::enable_shared_from_this frame ) { auto self = shared_from_this(); - auto context = ioContext(); - if ( !context ) - return; - context->post( [self, frame = std::move( frame )]() mutable { + mConnection->ioContext()->post( [self, frame = std::move( frame )]() mutable { if ( self->mState.load() == State::Dead ) return; self->mOut.insert( self->mOut.end(), frame.begin(), frame.end() ); @@ -72,9 +69,14 @@ class AgentClient : public SSHChannelPump, public std::enable_shared_from_this ioContext() const; + /** Thread-safe: gives up on the agent (as if its channel closed). */ + void stop( std::string reason ) { + auto self = shared_from_this(); + mConnection->ioContext()->post( + [self, reason = std::move( reason )] { self->die( reason ); } ); + } + private: void die( const std::string& reason ); void handle( const agent::Frame& frame ); @@ -148,6 +150,10 @@ class RemoteWatchBackendImpl : public std::enable_shared_from_this provider, VirtualFileSystem& vfs ) : mProvider( provider ), mPolling( std::make_shared( vfs ) ), @@ -157,11 +163,17 @@ class RemoteWatchBackendImpl : public std::enable_shared_from_this> watches; + std::map agents; { std::lock_guard lock( mMutex ); watches.swap( mWatches ); - mAgents.clear(); + agents.swap( mAgents ); + for ( auto& watch : watches ) + watch.second->mode = RemoteWatch::Mode::Closed; } + for ( auto& entry : agents ) + if ( entry.second.agent ) + entry.second.agent->stop( "Watch backend closed" ); for ( auto& watch : watches ) watch.second->polling.reset(); } @@ -210,6 +222,7 @@ class RemoteWatchBackendImpl : public std::enable_shared_from_thisagent->send( w.frame( agent::MessageUnwatch ) ); } + stopUnusedAgent( watch->agent ); watch->polling.reset(); } @@ -350,11 +363,6 @@ class RemoteWatchBackendImpl : public std::enable_shared_from_thisrescanRequired(); } - std::shared_ptr ioContext() const { - auto provider = mProvider.lock(); - return provider ? provider->ioContext() : nullptr; - } - private: struct AgentEntry { std::shared_ptr agent; @@ -363,6 +371,22 @@ class RemoteWatchBackendImpl : public std::enable_shared_from_this& client ) { + if ( !client ) + return; + { + std::lock_guard lock( mMutex ); + for ( const auto& watch : mWatches ) + if ( watch.second->agent == client ) + return; + auto it = mAgents.find( client->key() ); + if ( it != mAgents.end() && it->second.agent == client ) + it->second.agent.reset(); + } + // An unused pump otherwise counts as an active user forever, preventing idle cleanup. + client->stop( "No watches remain" ); + } + void completeEstablishment( RemoteWatch& watch, VFSResult&& result ) { std::optional>> source; { @@ -401,13 +425,15 @@ class RemoteWatchBackendImpl : public std::enable_shared_from_this previousAgent; { std::lock_guard lock( mMutex ); if ( watch->mode == RemoteWatch::Mode::Closed ) return; watch->mode = RemoteWatch::Mode::Polling; - watch->agent.reset(); + previousAgent = std::move( watch->agent ); } + stopUnusedAgent( previousAgent ); std::weak_ptr weak = weak_from_this(); std::weak_ptr weakWatch = watch; mPolling->watch( watch->root, watch->options, sink ) @@ -539,6 +565,35 @@ class RemoteWatchBackendImpl : public std::enable_shared_from_this& watch, + const std::shared_ptr& client ) { + std::weak_ptr weak = weak_from_this(); + std::weak_ptr weakWatch = watch; + std::weak_ptr weakClient = client; + mRunner.runAfter( EstablishTimeout, [weak, weakWatch, weakClient] { + auto self = weak.lock(); + auto watch = weakWatch.lock(); + auto client = weakClient.lock(); + if ( !self || !watch || !client ) + return; + { + std::lock_guard lock( self->mMutex ); + if ( watch->mode == RemoteWatch::Mode::Closed || + watch->mode == RemoteWatch::Mode::Agent || watch->agent != client ) + return; + } + if ( client->state() != AgentClient::State::Ready ) { + // No handshake: the agent is marked unavailable and its watches poll. + client->stop( "efsw-agent did not answer" ); + return; + } + self->usePolling( watch ); + } ); } std::weak_ptr mProvider; @@ -555,11 +610,6 @@ class RemoteWatchBackendImpl : public std::enable_shared_from_this AgentClient::ioContext() const { - auto backend = mBackend.lock(); - return backend ? backend->ioContext() : nullptr; -} - void AgentClient::start( const std::string& command ) { auto self = shared_from_this(); mConnection diff --git a/src/modules/vfs/src/eepp/vfs/sftp/sftpprovider.cpp b/src/modules/vfs/src/eepp/vfs/sftp/sftpprovider.cpp index 32ad348a5..e8473d07e 100644 --- a/src/modules/vfs/src/eepp/vfs/sftp/sftpprovider.cpp +++ b/src/modules/vfs/src/eepp/vfs/sftp/sftpprovider.cpp @@ -506,76 +506,112 @@ Time SFTPProviderImpl::connectTimeout( const SSHResolvedConfig& config ) const { return config.connectTimeout ? *config.connectTimeout : mConfig.defaultConnectTimeout; } -/** Builds (blocking, on a helper thread) a ready connection for a resolved configuration, - * including its ProxyJump chain or ProxyCommand transport. */ -VFSResult> -SFTPProviderImpl::buildConnection( const SSHResolvedConfig& config ) { - std::unique_ptr transport; - if ( !config.proxyJump.empty() ) { - // Each jump host uses its own ssh_config (like `ssh -J`); the chain is the target's. - std::shared_ptr parent; - for ( size_t i = 0; i <= config.proxyJump.size(); ++i ) { - bool last = i == config.proxyJump.size(); - SSHResolvedConfig hopConfig; - if ( last ) { - hopConfig = config; - } else { - const auto& jump = config.proxyJump[i]; - auto resolved = resolveSync( - jump.host, jump.user.empty() ? std::nullopt : std::optional( jump.user ), - jump.port ? std::optional( jump.port ) : std::nullopt ); - if ( !resolved ) - return Err( std::move( resolved ).error() ); - hopConfig = std::move( resolved ).value(); - hopConfig.proxyJump.clear(); - hopConfig.proxyCommand.reset(); - } - std::unique_ptr hopTransport; - if ( !parent ) { - std::string error; - SocketHandle socket = connectTcp( - hopConfig.hostName, hopConfig.port, - static_cast( connectTimeout( hopConfig ).asMilliseconds() ), error ); - if ( socket == InvalidSocket ) - return Err( VFSError( VFSErrorCode::ConnectionLost, URI(), error ) ); - hopTransport = std::make_unique( socket ); - } else { - auto channelTransport = std::make_unique( - parent, hopConfig.hostName, hopConfig.port ); - ChannelTransport* raw = channelTransport.get(); - // Opening the channel is an operation on the parent: it is serialized with the - // parent's own SFTP traffic. - auto opened = await( parent->submit( - {}, URI(), [raw]( SSHConnection& ) -> std::optional> { - VFSError error; - if ( raw->open( error ) == Step::Again ) - return std::nullopt; - if ( !raw->channel() ) - return VFSResult( Err( std::move( error ) ) ); - return VFSResult( Ok() ); - } ) ); - if ( !opened ) - return Err( std::move( opened ).error() ); - hopTransport = std::move( channelTransport ); - } - auto connection = std::make_shared( mContext, hopConfig, hooks(), - std::move( hopTransport ), - connectTimeout( hopConfig ) ); - mContext->post( - [context = mContext, connection] { context->addConnection( connection ); } ); - auto ready = await( connection->whenReady() ); +/** A connection being built, one hop of its ProxyJump chain at a time. */ +struct SFTPProviderImpl::Build { + SSHResolvedConfig config; + /** The previous hop, which carries the next one's channel. */ + std::shared_ptr parent; + size_t hop{ 0 }; + TaskSource>> source; +}; + +VFSTask> +SFTPProviderImpl::buildConnection( SSHResolvedConfig config ) { + auto build = std::make_shared(); + build->config = std::move( config ); + auto task = build->source.task(); + buildHop( std::move( build ) ); + return task; +} + +void SFTPProviderImpl::buildHop( std::shared_ptr build ) { + // Transports are set up on a helper thread (TCP connect blocks, bounded by ConnectTimeout). + // Waiting for the session is not: the handshake and authentication may wait on the user + // (host key, password), so it continues from the connection's ready task instead. + auto pool = mHelperPool; + if ( !pool ) { + build->source.resolve( + Err( VFSError( VFSErrorCode::ConnectionLost, URI(), "SFTP provider shutting down" ) ) ); + return; + } + std::weak_ptr weak = weak_from_this(); + pool->run( [weak, build = std::move( build )]() mutable { + HelperThreadScope scope; + auto self = weak.lock(); + if ( !self ) + return; // The source is abandoned with the build. + auto connection = self->startHop( *build ); + if ( !connection ) { + build->source.resolve( Err( std::move( connection ).error() ) ); + return; + } + bool last = build->hop == build->config.proxyJump.size(); + std::string hopName = connection.value()->config().hostName; + connection.value()->whenReady().then( [weak, build, connection = connection.value(), last, + hopName]( VFSResult&& ready ) mutable { + auto self = weak.lock(); + if ( !self ) + return; if ( !ready ) { VFSError error = std::move( ready ).error(); if ( !last ) - error.message = "ProxyJump " + hopConfig.hostName + ": " + error.message; - return Err( std::move( error ) ); + error.message = "ProxyJump " + hopName + ": " + error.message; + build->source.resolve( Err( std::move( error ) ) ); + return; } - parent = std::move( connection ); - } - return Ok( std::move( parent ) ); + if ( !last ) { + build->parent = std::move( connection ); + ++build->hop; + self->buildHop( std::move( build ) ); + return; + } + self->probeExtensions( connection ) + .then( [build, connection]( VFSResult&& ) mutable { + build->source.resolve( Ok( std::move( connection ) ) ); + } ); + } ); + } ); +} + +VFSResult> SFTPProviderImpl::startHop( Build& build ) { + const SSHResolvedConfig& config = build.config; + bool last = build.hop == config.proxyJump.size(); + SSHResolvedConfig hopConfig; + if ( last ) { + hopConfig = config; + } else { + // Each jump host uses its own ssh_config (like `ssh -J`); the chain is the target's. + const auto& jump = config.proxyJump[build.hop]; + auto resolved = + resolveSync( jump.host, jump.user.empty() ? std::nullopt : std::optional( jump.user ), + jump.port ? std::optional( jump.port ) : std::nullopt ); + if ( !resolved ) + return Err( std::move( resolved ).error() ); + hopConfig = std::move( resolved ).value(); + hopConfig.proxyJump.clear(); + hopConfig.proxyCommand.reset(); } - if ( config.proxyCommand ) { + std::unique_ptr transport; + if ( build.parent ) { + auto channelTransport = + std::make_unique( build.parent, hopConfig.hostName, hopConfig.port ); + ChannelTransport* raw = channelTransport.get(); + // Opening the channel is an operation on the parent: it is serialized with the parent's + // own traffic. Bounded (no user interaction), so waiting here is fine. + auto opened = await( build.parent->submit( + {}, URI(), [raw]( SSHConnection& ) -> std::optional> { + VFSError error; + if ( raw->open( error ) == Step::Again ) + return std::nullopt; + if ( !raw->channel() ) + return VFSResult( Err( std::move( error ) ) ); + return VFSResult( Ok() ); + } ) ); + if ( !opened ) + return Err( std::move( opened ).error() ); + transport = std::move( channelTransport ); + } else if ( last && config.proxyCommand ) { auto process = std::make_unique( mContext, *config.proxyCommand ); if ( !process->started() ) return Err( VFSError( VFSErrorCode::ConnectionLost, URI(), @@ -584,26 +620,28 @@ SFTPProviderImpl::buildConnection( const SSHResolvedConfig& config ) { } else { std::string error; SocketHandle socket = - connectTcp( config.hostName, config.port, - static_cast( connectTimeout( config ).asMilliseconds() ), error ); + connectTcp( hopConfig.hostName, hopConfig.port, + static_cast( connectTimeout( hopConfig ).asMilliseconds() ), error ); if ( socket == InvalidSocket ) return Err( VFSError( VFSErrorCode::ConnectionLost, URI(), error ) ); transport = std::make_unique( socket ); } - auto connection = std::make_shared( - mContext, config, hooks(), std::move( transport ), connectTimeout( config ) ); + + Time timeout = connectTimeout( hopConfig ); + auto connection = std::make_shared( mContext, std::move( hopConfig ), hooks(), + std::move( transport ), timeout ); + connection->setIdleTimeout( mConfig.idleTimeout ); + connection->setSFTPRequired( last ); mContext->post( [context = mContext, connection] { context->addConnection( connection ); } ); - auto ready = await( connection->whenReady() ); - if ( !ready ) - return Err( std::move( ready ).error() ); return Ok( std::move( connection ) ); } /** Probes SFTP extensions once per connection: posix-rename (atomic replace), statvfs. fsync is * detected on first use. */ -void SFTPProviderImpl::probeExtensions( const std::shared_ptr& connection ) { +VFSTask +SFTPProviderImpl::probeExtensions( const std::shared_ptr& connection ) { static const std::string probe = "/.eepp-vfs-extension-probe-does-not-exist"; - await( connection->submit( + return connection->submit( {}, URI(), []( SSHConnection& c ) -> std::optional> { if ( c.posixRenameSupport < 0 ) { int rc = libssh2_sftp_posix_rename_ex( @@ -628,19 +666,19 @@ void SFTPProviderImpl::probeExtensions( const std::shared_ptr& co if ( c.posixRenameSupport == 1 && c.fsyncSupport < 0 ) c.fsyncSupport = 1; // Both arrived in OpenSSH 6.5; confirmed on first use. return VFSResult( Ok() ); - } ) ); + } ); } void SFTPProviderImpl::pruneIdle() { - auto now = std::chrono::steady_clock::now(); - auto idle = - std::chrono::milliseconds( static_cast( mConfig.idleTimeout.asMilliseconds() ) ); + // Idle connections close themselves on the I/O thread (SSHConnection::setIdleTimeout()); + // here the pool forgets the closed ones, and closes those whose timer has not fired yet. for ( auto it = mPool.begin(); it != mPool.end(); ) { auto& entry = it->second; - if ( entry.connection && - ( !entry.connection->isUsable() || ( entry.connection.use_count() == 1 && - now - entry.connection->lastUse() > idle ) ) ) { - entry.connection->close(); + auto connection = entry.connection.lock(); + if ( connection && ( !connection->isUsable() || connection->idleExpired() ) ) { + connection->close(); + it = mPool.erase( it ); + } else if ( !connection && entry.waiters.empty() ) { it = mPool.erase( it ); } else { ++it; @@ -651,18 +689,21 @@ void SFTPProviderImpl::pruneIdle() { VFSTask> SFTPProviderImpl::connection( const URI& uri ) { using Result = VFSResult>; std::string key = authorityKey( uri ); + Uint64 generation; TaskSource source; auto task = source.task(); { std::lock_guard lock( mPoolMutex ); + generation = mPoolGeneration; pruneIdle(); auto it = mPool.find( key ); if ( it != mPool.end() ) { - if ( it->second.connection && it->second.connection->isUsable() ) { - source.resolve( Ok( it->second.connection ) ); + auto connection = it->second.connection.lock(); + if ( connection && connection->isUsable() ) { + source.resolve( Ok( std::move( connection ) ) ); return task; } - if ( !it->second.connection ) { + if ( !connection && !it->second.waiters.empty() ) { it->second.waiters.push_back( std::move( source ) ); return task; } @@ -675,7 +716,7 @@ VFSTask> SFTPProviderImpl::connection( const URI& return readyError>( VFSErrorCode::ConnectionLost, uri, "SFTP provider shutting down" ); std::weak_ptr weak = weak_from_this(); - pool->run( [weak, uri, key]() { + pool->run( [weak, uri, key, generation]() { HelperThreadScope scope; auto self = weak.lock(); if ( !self ) @@ -691,10 +732,13 @@ VFSTask> SFTPProviderImpl::connection( const URI& std::shared_ptr existing; { std::lock_guard lock( self->mPoolMutex ); + if ( self->mPoolGeneration != generation ) + return; for ( auto& entry : self->mPool ) { - if ( entry.second.connection && entry.second.connection->isUsable() && - entry.second.connection->config().connectionKey() == connectionKey ) { - existing = entry.second.connection; + auto connection = entry.second.connection.lock(); + if ( connection && connection->isUsable() && + connection->config().connectionKey() == connectionKey ) { + existing = std::move( connection ); break; } } @@ -702,37 +746,51 @@ VFSTask> SFTPProviderImpl::connection( const URI& if ( existing ) { result = Ok( existing ); } else { - result = self->buildConnection( config.value() ); - if ( result ) - self->probeExtensions( result.value() ); - } - if ( !result && result.error().uri.empty() ) - result.error().uri = uri; - } - std::vector> waiters; - { - std::lock_guard lock( self->mPoolMutex ); - auto it = self->mPool.find( key ); - if ( it != self->mPool.end() ) { - waiters = std::move( it->second.waiters ); - if ( result ) - it->second.connection = result.value(); - else - self->mPool.erase( it ); + self->buildConnection( std::move( config ).value() ) + .then( [weak, uri, key, generation]( Result&& built ) { + if ( auto self = weak.lock() ) + self->finishConnection( key, uri, generation, std::move( built ) ); + } ); + return; } } - for ( auto& waiter : waiters ) - waiter.resolve( Result( result ) ); + self->finishConnection( key, uri, generation, std::move( result ) ); } ); return task; } +void SFTPProviderImpl::finishConnection( const std::string& key, const URI& uri, Uint64 generation, + VFSResult>&& result ) { + if ( !result && result.error().uri.empty() ) + result.error().uri = uri; + std::vector>>> waiters; + { + std::lock_guard lock( mPoolMutex ); + if ( generation != mPoolGeneration ) { + if ( result ) + result.value()->close(); + return; + } + auto it = mPool.find( key ); + if ( it != mPool.end() ) { + waiters = std::move( it->second.waiters ); + if ( result ) + it->second.connection = result.value(); + else + mPool.erase( it ); + } + } + for ( auto& waiter : waiters ) + waiter.resolve( VFSResult>( result ) ); +} + ProviderCapabilities SFTPProviderImpl::capabilities( const URI& uri ) const { ProviderCapabilities caps( BaseCapabilities ); std::lock_guard lock( mPoolMutex ); auto it = mPool.find( authorityKey( uri ) ); - if ( it != mPool.end() && it->second.connection ) { - const auto& c = *it->second.connection; + auto connection = it != mPool.end() ? it->second.connection.lock() : nullptr; + if ( connection ) { + const auto& c = *connection; caps.set( VFSCapability::AtomicRename, c.posixRenameSupport == 1 ); caps.set( VFSCapability::AtomicReplace, c.posixRenameSupport == 1 ); caps.set( VFSCapability::Fsync, c.fsyncSupport == 1 ); @@ -742,19 +800,32 @@ ProviderCapabilities SFTPProviderImpl::capabilities( const URI& uri ) const { } void SFTPProviderImpl::disconnectAll() { - std::lock_guard lock( mPoolMutex ); - for ( auto& entry : mPool ) - if ( entry.second.connection ) - entry.second.connection->close(); - mPool.clear(); + std::map removed; + { + std::lock_guard lock( mPoolMutex ); + ++mPoolGeneration; + removed.swap( mPool ); + } + // Completing waiters can run arbitrary continuations: never under the pool lock. Include + // pending connections, whose authentication may be waiting on UI that is shutting down. + mContext->closeConnections(); + for ( auto& entry : removed ) { + if ( auto connection = entry.second.connection.lock() ) + connection->close(); + for ( auto& waiter : entry.second.waiters ) + waiter.resolve( + Err( VFSError( VFSErrorCode::ConnectionLost, URI(), "Connection closed" ) ) ); + } } size_t SFTPProviderImpl::connectionCount() const { std::lock_guard lock( mPoolMutex ); size_t count = 0; - for ( const auto& entry : mPool ) - if ( entry.second.connection && entry.second.connection->isUsable() ) + for ( const auto& entry : mPool ) { + if ( auto connection = entry.second.connection.lock(); + connection && connection->isUsable() ) ++count; + } return count; } diff --git a/src/modules/vfs/src/eepp/vfs/sftp/sftpproviderimpl.hpp b/src/modules/vfs/src/eepp/vfs/sftp/sftpproviderimpl.hpp index e0738ebdb..a2287807d 100644 --- a/src/modules/vfs/src/eepp/vfs/sftp/sftpproviderimpl.hpp +++ b/src/modules/vfs/src/eepp/vfs/sftp/sftpproviderimpl.hpp @@ -44,7 +44,9 @@ class SFTPProviderImpl : public std::enable_shared_from_this { private: struct PoolEntry { - std::shared_ptr connection; + /** The I/O context owns live sessions; the pool must not retain their resources after + * the context closes them. An expired entry with waiters is still being built. */ + std::weak_ptr connection; std::vector>>> waiters; }; @@ -52,9 +54,23 @@ class SFTPProviderImpl : public std::enable_shared_from_this { std::optional user, std::optional port ) const; - VFSResult> buildConnection( const SSHResolvedConfig& config ); + struct Build; - void probeExtensions( const std::shared_ptr& connection ); + /** Connects to config through its ProxyJump chain or ProxyCommand. The task resolves on the + * I/O thread once the session is ready (authenticated, SFTP started). */ + VFSTask> buildConnection( SSHResolvedConfig config ); + + /** Sets up the next hop of build on a helper thread and continues when it is ready. */ + void buildHop( std::shared_ptr build ); + + /** Creates the transport and connection of build's current hop (helper thread). */ + VFSResult> startHop( Build& build ); + + VFSTask probeExtensions( const std::shared_ptr& connection ); + + /** Hands the connection (or the failure) to every request waiting on the pool key. */ + void finishConnection( const std::string& key, const URI& uri, Uint64 generation, + VFSResult>&& result ); SSHConnectionHooks hooks() const; @@ -69,6 +85,8 @@ class SFTPProviderImpl : public std::enable_shared_from_this { std::shared_ptr mHelperPool; mutable std::mutex mPoolMutex; std::map mPool; + /** Invalidates connection builds still in flight when disconnectAll() is called. */ + Uint64 mPoolGeneration{ 0 }; }; }}} // namespace EE::VFS::sftp diff --git a/src/modules/vfs/src/eepp/vfs/sftp/sshagent.cpp b/src/modules/vfs/src/eepp/vfs/sftp/sshagent.cpp new file mode 100644 index 000000000..3c23ec00e --- /dev/null +++ b/src/modules/vfs/src/eepp/vfs/sftp/sshagent.cpp @@ -0,0 +1,275 @@ +#include "sshagent.hpp" + +#if EE_PLATFORM != EE_PLATFORM_WIN +#include +#include +#include +#include +#include +#endif + +namespace EE { namespace VFS { namespace sftp { + +namespace { + +// draft-miller-ssh-agent message numbers and flags. +constexpr Uint8 AgentFailure = 5; +constexpr Uint8 RequestIdentities = 11; +constexpr Uint8 IdentitiesAnswer = 12; +constexpr Uint8 SignRequest = 13; +constexpr Uint8 SignResponse = 14; +constexpr Uint32 RSASHA256 = 2; +constexpr Uint32 RSASHA512 = 4; + +/** Agent replies larger than this are refused (OpenSSH uses 256 KiB). */ +constexpr Uint32 MaxMessage = 256 * 1024; + +void putU32( std::string& out, Uint32 value ) { + out += static_cast( ( value >> 24 ) & 0xFF ); + out += static_cast( ( value >> 16 ) & 0xFF ); + out += static_cast( ( value >> 8 ) & 0xFF ); + out += static_cast( value & 0xFF ); +} + +void putString( std::string& out, std::string_view value ) { + putU32( out, static_cast( value.size() ) ); + out.append( value ); +} + +/** Sequential reader of SSH wire data; any overrun makes it invalid. */ +class WireReader { + public: + explicit WireReader( std::string_view data ) : mData( data ) {} + + bool valid() const { return mValid; } + + Uint8 u8() { + if ( !need( 1 ) ) + return 0; + return static_cast( mData[mPosition++] ); + } + + Uint32 u32() { + if ( !need( 4 ) ) + return 0; + Uint32 value = 0; + for ( int i = 0; i < 4; ++i ) + value = ( value << 8 ) | static_cast( mData[mPosition++] ); + return value; + } + + std::string_view string() { + Uint32 length = u32(); + if ( !need( length ) ) + return {}; + std::string_view value = mData.substr( mPosition, length ); + mPosition += length; + return value; + } + + private: + bool need( size_t bytes ) { + if ( !mValid || mData.size() - mPosition < bytes ) + mValid = false; + return mValid; + } + + std::string_view mData; + size_t mPosition{ 0 }; + bool mValid{ true }; +}; + +VFSError agentError( std::string message ) { + return VFSError( VFSErrorCode::AuthenticationFailed, URI(), "ssh-agent: " + message ); +} + +} // namespace + +bool SSHAgent::supported() { +#if EE_PLATFORM != EE_PLATFORM_WIN + return true; +#else + return false; +#endif +} + +SSHAgent::SSHAgent( std::string socketPath ) : mPath( std::move( socketPath ) ) {} + +SSHAgent::~SSHAgent() { + disconnect(); +} + +void SSHAgent::disconnect() { + std::lock_guard lock( mSocketMutex ); +#if EE_PLATFORM != EE_PLATFORM_WIN + if ( mSocket >= 0 ) + ::close( mSocket ); +#endif + mSocket = -1; +} + +void SSHAgent::cancel() { + std::lock_guard lock( mSocketMutex ); + mCancelled.store( true, std::memory_order_release ); +#if EE_PLATFORM != EE_PLATFORM_WIN + // Do not close here: the worker may still use this descriptor, and closing could let a + // different thread reuse its number. shutdown wakes its blocking receive; it closes it. + if ( mSocket >= 0 ) + ::shutdown( mSocket, SHUT_RDWR ); +#endif +} + +VFSResult SSHAgent::transact( std::string_view request ) { +#if EE_PLATFORM != EE_PLATFORM_WIN + if ( mCancelled.load( std::memory_order_acquire ) ) + return Err( agentError( "request cancelled" ) ); + if ( mSocket < 0 ) { + sockaddr_un address{}; + if ( mPath.empty() || mPath.size() >= sizeof( address.sun_path ) ) + return Err( agentError( "invalid socket path \"" + mPath + "\"" ) ); + address.sun_family = AF_UNIX; + std::memcpy( address.sun_path, mPath.c_str(), mPath.size() + 1 ); + { + std::lock_guard lock( mSocketMutex ); + if ( mCancelled.load( std::memory_order_acquire ) ) + return Err( agentError( "request cancelled" ) ); + mSocket = ::socket( AF_UNIX, SOCK_STREAM, 0 ); + } +#ifdef SO_NOSIGPIPE + // No MSG_NOSIGNAL on Apple platforms: an agent that went away must not kill the process. + int noSigPipe = 1; + if ( mSocket >= 0 ) + ::setsockopt( mSocket, SOL_SOCKET, SO_NOSIGPIPE, &noSigPipe, sizeof( noSigPipe ) ); +#endif + if ( mSocket < 0 || ::connect( mSocket, reinterpret_cast( &address ), + sizeof( address ) ) != 0 ) { + std::string error( std::strerror( errno ) ); + disconnect(); + return Err( agentError( "cannot connect to " + mPath + ": " + error ) ); + } + } + if ( mCancelled.load( std::memory_order_acquire ) ) { + disconnect(); + return Err( agentError( "request cancelled" ) ); + } + std::string message; + message.reserve( 4 + request.size() ); + putString( message, request ); + size_t sent = 0; + while ( sent < message.size() ) { +#ifdef MSG_NOSIGNAL + constexpr int flags = MSG_NOSIGNAL; +#else + constexpr int flags = 0; +#endif + ssize_t rc = ::send( mSocket, message.data() + sent, message.size() - sent, flags ); + if ( rc < 0 && errno == EINTR ) + continue; + if ( rc <= 0 ) { + disconnect(); + return Err( agentError( "write failed" ) ); + } + sent += static_cast( rc ); + } + auto receive = [this]( char* buffer, size_t size ) { + size_t received = 0; + while ( received < size ) { + ssize_t rc = ::recv( mSocket, buffer + received, size - received, 0 ); + if ( rc < 0 && errno == EINTR ) + continue; + if ( rc <= 0 ) + return false; + received += static_cast( rc ); + } + return true; + }; + char header[4]; + if ( !receive( header, sizeof( header ) ) ) { + disconnect(); + return Err( agentError( "read failed" ) ); + } + Uint32 length = WireReader( std::string_view( header, sizeof( header ) ) ).u32(); + if ( length == 0 || length > MaxMessage ) { + disconnect(); + return Err( agentError( "invalid reply length" ) ); + } + std::string reply( length, '\0' ); + if ( !receive( reply.data(), reply.size() ) ) { + disconnect(); + return Err( agentError( "read failed" ) ); + } + return Ok( std::move( reply ) ); +#else + (void)request; + return Err( agentError( "not supported on this platform" ) ); +#endif +} + +VFSResult> SSHAgent::listIdentities() { + std::string request( 1, static_cast( RequestIdentities ) ); + auto reply = transact( request ); + if ( !reply ) + return Err( std::move( reply ).error() ); + WireReader reader( reply.value() ); + if ( reader.u8() != IdentitiesAnswer ) + return Err( agentError( "the agent refused to list its keys" ) ); + Uint32 count = reader.u32(); + std::vector identities; + for ( Uint32 i = 0; i < count && reader.valid(); ++i ) { + std::string_view blob = reader.string(); + std::string_view comment = reader.string(); + if ( reader.valid() ) + identities.push_back( { std::string( blob ), std::string( comment ) } ); + } + if ( !reader.valid() ) + return Err( agentError( "malformed key list" ) ); + return Ok( std::move( identities ) ); +} + +VFSResult SSHAgent::sign( std::string_view blob, std::string_view data, + std::string_view algorithm ) { + Uint32 flags = algorithm == "rsa-sha2-512" ? RSASHA512 + : algorithm == "rsa-sha2-256" ? RSASHA256 + : 0; + std::string request( 1, static_cast( SignRequest ) ); + putString( request, blob ); + putString( request, data ); + putU32( request, flags ); + auto reply = transact( request ); + if ( !reply ) + return Err( std::move( reply ).error() ); + WireReader reader( reply.value() ); + Uint8 type = reader.u8(); + if ( type == AgentFailure ) + return Err( agentError( "signing was refused" ) ); + if ( type != SignResponse ) + return Err( agentError( "unexpected reply to a signing request" ) ); + WireReader signature( reader.string() ); + std::string_view signedWith = signature.string(); + std::string_view raw = signature.string(); + if ( !reader.valid() || !signature.valid() ) + return Err( agentError( "malformed signature" ) ); + // The server verifies the algorithm announced in the request. + if ( !algorithm.empty() && signedWith != algorithm ) + return Err( agentError( "signed with " + std::string( signedWith ) + " instead of " + + std::string( algorithm ) ) ); + return Ok( std::string( raw ) ); +} + +std::string_view SSHAgent::algorithmOfSignedRequest( std::string_view data ) { + // string session identifier, byte SSH_MSG_USERAUTH_REQUEST, string user, string service, + // string "publickey", boolean TRUE, string algorithm, string public key. + WireReader reader( data ); + reader.string(); + reader.u8(); + reader.string(); + reader.string(); + std::string_view method = reader.string(); + reader.u8(); + std::string_view algorithm = reader.string(); + if ( !reader.valid() || method != "publickey" ) + return {}; + return algorithm; +} + +}}} // namespace EE::VFS::sftp diff --git a/src/modules/vfs/src/eepp/vfs/sftp/sshagent.hpp b/src/modules/vfs/src/eepp/vfs/sftp/sshagent.hpp new file mode 100644 index 000000000..93a775a9e --- /dev/null +++ b/src/modules/vfs/src/eepp/vfs/sftp/sshagent.hpp @@ -0,0 +1,71 @@ +#ifndef EE_VFS_SFTP_SSHAGENT_HPP +#define EE_VFS_SFTP_SSHAGENT_HPP + +#include +#include +#include +#include +#include +#include +#include + +namespace EE { namespace VFS { namespace sftp { + +/** @brief Minimal ssh-agent client: lists the agent's keys and signs with them. + * + * Every call blocks on the agent, and signing may wait for the user (a hardware key touch, a + * confirmation dialog): use it from a worker thread, never from the SSH I/O thread. One call at a + * time. Speaks the agent protocol over Unix-domain sockets; elsewhere (Windows named pipes, + * Pageant) supported() is false and libssh2's own agent support is used. + */ +class SSHAgent { + public: + struct Identity { + /** Public key blob (SSH wire format). */ + std::string blob; + std::string comment; + }; + + /** @return True where agents are reached through Unix-domain sockets, which this client + * speaks: every POSIX platform that builds SFTP (Linux, BSD, macOS, iOS). It does not mean an + * agent runs: without one, connecting fails and authentication goes on with identity files. + * Windows agents are named pipes or Pageant, left to libssh2. */ + static bool supported(); + + explicit SSHAgent( std::string socketPath ); + + ~SSHAgent(); + + /** Interrupts a pending request and prevents further requests. Safe from any thread. */ + void cancel(); + + SSHAgent( const SSHAgent& ) = delete; + + SSHAgent& operator=( const SSHAgent& ) = delete; + + VFSResult> listIdentities(); + + /** Signs data with the key of blob for the signature algorithm (for example "rsa-sha2-512", + * which selects the agent's RSA SHA-2 flags). @return The raw signature, without its + * algorithm name (as libssh2 sign callbacks return it). */ + VFSResult sign( std::string_view blob, std::string_view data, + std::string_view algorithm ); + + /** @return The signature algorithm of a SSH_MSG_USERAUTH_REQUEST "publickey" signing payload + * (session identifier followed by the request), empty when it cannot be parsed. */ + static std::string_view algorithmOfSignedRequest( std::string_view data ); + + private: + VFSResult transact( std::string_view request ); + + void disconnect(); + + std::string mPath; + std::mutex mSocketMutex; + int mSocket{ -1 }; + std::atomic mCancelled{ false }; +}; + +}}} // namespace EE::VFS::sftp + +#endif diff --git a/src/modules/vfs/src/eepp/vfs/sftp/sshsession.cpp b/src/modules/vfs/src/eepp/vfs/sftp/sshsession.cpp index 1f9ac796a..831322af6 100644 --- a/src/modules/vfs/src/eepp/vfs/sftp/sshsession.cpp +++ b/src/modules/vfs/src/eepp/vfs/sftp/sshsession.cpp @@ -1,8 +1,10 @@ #include "sshsession.hpp" #include #include +#include #include #include +#include #include #include #include @@ -80,10 +82,20 @@ int knownHostKeyBits( int type ) { } } +/** Decodes base64 text (as in known_hosts and .pub files), nullopt when it is not valid. */ +std::optional decodeBase64( std::string_view text ) { + std::string decoded; + if ( System::Base64::decode( text, decoded ) == static_cast( -1 ) ) + return std::nullopt; + return decoded; +} + /** Loads known_hosts line by line: libssh2_knownhost_readfile() stops at the first entry it * cannot parse (certificate authorities, security-key types...), which would hide every later - * host. */ -void loadKnownHosts( LIBSSH2_KNOWNHOSTS* hosts, const std::string& path ) { + * host. libssh2 does not know markers: the key blobs of "@revoked" lines are added to revoked + * (when given), "@cert-authority" lines are ignored. */ +void loadKnownHosts( LIBSSH2_KNOWNHOSTS* hosts, const std::string& path, + std::vector* revoked ) { std::string data; if ( !FileSystem::fileExists( path ) || !FileSystem::fileGet( path, data ) ) return; @@ -97,8 +109,18 @@ void loadKnownHosts( LIBSSH2_KNOWNHOSTS* hosts, const std::string& path ) { while ( !line.empty() && ( line.back() == '\r' || line.back() == ' ' ) ) line.remove_suffix( 1 ); size_t first = line.find_first_not_of( " \t" ); - if ( first == std::string_view::npos || line[first] == '#' || line[first] == '@' ) + if ( first == std::string_view::npos || line[first] == '#' ) continue; + if ( line[first] == '@' ) { + // "@revoked hosts keytype base64key [comment]": the key is never accepted, for any + // host (stricter than matching the host patterns, never weaker). + auto fields = String::split( std::string( line.substr( first ) ), " \t" ); + if ( revoked && fields.size() >= 4 && fields[0] == "@revoked" ) { + if ( auto blob = decodeBase64( fields[3] ) ) + revoked->push_back( std::move( *blob ) ); + } + continue; + } // libssh2 inspects the byte after the given length (expecting an optional newline), so // hand it a NUL-terminated copy rather than a view into the file buffer. std::string entry( line ); @@ -120,29 +142,13 @@ bool isKeyEncrypted( const std::string& path ) { if ( begin == std::string::npos ) return false; size_t end = data.find( "-----END", begin + Begin.size() ); - std::string base64; - for ( size_t i = begin + Begin.size(); i < end && i < data.size(); ++i ) - if ( !std::isspace( static_cast( data[i] ) ) ) - base64 += data[i]; - // Decode just the header: "openssh-key-v1\0" + uint32 length + cipher name. - static constexpr char Alphabet[] = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - std::string decoded; - Uint32 buffer = 0; - int bits = 0; - for ( char c : base64 ) { - if ( c == '=' || decoded.size() > 64 ) - break; - const char* pos = std::strchr( Alphabet, c ); - if ( !pos ) - break; - buffer = ( buffer << 6 ) | static_cast( pos - Alphabet ); - bits += 6; - if ( bits >= 8 ) { - bits -= 8; - decoded += static_cast( ( buffer >> bits ) & 0xFF ); - } - } + auto decodedText = decodeBase64( std::string_view( data ).substr( + begin + Begin.size(), + end == std::string::npos ? std::string::npos : end - begin - Begin.size() ) ); + if ( !decodedText ) + return false; + // Only the header matters: "openssh-key-v1\0" + uint32 length + cipher name. + const std::string& decoded = *decodedText; static constexpr size_t Magic = 15; // "openssh-key-v1" + NUL if ( decoded.size() < Magic + 4 ) return false; @@ -161,25 +167,9 @@ std::optional readPublicKeyBlob( const std::string& path ) { auto fields = String::split( data, ' ' ); if ( fields.size() < 2 ) return std::nullopt; - static constexpr char Alphabet[] = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - std::string decoded; - Uint32 buffer = 0; - int bits = 0; - for ( char c : fields[1] ) { - if ( c == '=' || c == '\n' || c == '\r' ) - break; - const char* pos = std::strchr( Alphabet, c ); - if ( !pos ) - return std::nullopt; - buffer = ( buffer << 6 ) | static_cast( pos - Alphabet ); - bits += 6; - if ( bits >= 8 ) { - bits -= 8; - decoded += static_cast( ( buffer >> bits ) & 0xFF ); - } - } - return decoded; + String::trimInPlace( fields[1], '\n' ); + String::trimInPlace( fields[1], '\r' ); + return decodeBase64( fields[1] ); } /** Host key algorithms of a known_hosts key type, in libssh2 method names. */ @@ -210,32 +200,17 @@ bool isSocketError( int rc ) { rc == LIBSSH2_ERROR_KEX_FAILURE; } -} // namespace - +/** Base64 without padding, as used by OpenSSH fingerprints. */ std::string base64Unpadded( const unsigned char* data, size_t length ) { - static constexpr char Alphabet[] = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; std::string out; - size_t i = 0; - for ( ; i + 2 < length; i += 3 ) { - Uint32 v = ( data[i] << 16 ) | ( data[i + 1] << 8 ) | data[i + 2]; - out += Alphabet[( v >> 18 ) & 63]; - out += Alphabet[( v >> 12 ) & 63]; - out += Alphabet[( v >> 6 ) & 63]; - out += Alphabet[v & 63]; - } - if ( i < length ) { - Uint32 v = data[i] << 16; - if ( i + 1 < length ) - v |= data[i + 1] << 8; - out += Alphabet[( v >> 18 ) & 63]; - out += Alphabet[( v >> 12 ) & 63]; - if ( i + 1 < length ) - out += Alphabet[( v >> 6 ) & 63]; - } + System::Base64::encode( std::string_view( reinterpret_cast( data ), length ), + out ); + while ( !out.empty() && out.back() == '=' ) + out.pop_back(); return out; } +/** Maps an SFTP status code to a VFS error code. */ VFSErrorCode mapSFTPStatus( unsigned long status ) { switch ( status ) { case LIBSSH2_FX_OK: @@ -274,6 +249,8 @@ VFSErrorCode mapSFTPStatus( unsigned long status ) { } } +} // namespace + // --------------------------------------------------------------------------------------------- // I/O context // --------------------------------------------------------------------------------------------- @@ -315,6 +292,8 @@ void SSHIOContext::stop() { void SSHIOContext::post( Command command ) { { std::lock_guard lock( mMutex ); + if ( mStopping.load( std::memory_order_acquire ) ) + return; mCommands.emplace_back( std::move( command ) ); } mWake.signal(); @@ -324,6 +303,15 @@ void SSHIOContext::addConnection( std::shared_ptr connection ) { mConnections.push_back( std::move( connection ) ); } +void SSHIOContext::closeConnections() { + post( [this] { + for ( auto& connection : mConnections ) { + connection->fail( + VFSError( VFSErrorCode::ConnectionLost, URI(), "Connection closed" ) ); + } + } ); +} + void SSHIOContext::loop() { std::vector commands; std::vector fds; @@ -341,7 +329,7 @@ void SSHIOContext::loop() { // Drop finished connections; their owners (pool, handles) may still reference them. mConnections.erase( std::remove_if( mConnections.begin(), mConnections.end(), []( const std::shared_ptr& c ) { - return !c->isUsable() && c->mQueue.empty(); + return !c->isUsable() && !c->hasQueuedOperations(); } ), mConnections.end() ); @@ -449,20 +437,43 @@ void SSHConnection::markUsed() { } void SSHConnection::enqueue( std::unique_ptr operation ) { + std::optional failure; { std::lock_guard lock( mQueueMutex ); - mLastUse = std::chrono::steady_clock::now(); - mQueue.push_back( std::move( operation ) ); + // A failed or closed connection is dropped by the I/O context: nobody would run the + // operation. + if ( mQueueFailure ) { + failure = mQueueFailure; + } else { + mLastUse = std::chrono::steady_clock::now(); + mQueue.push_back( std::move( operation ) ); + } + } + if ( failure ) { + operation->fail( *failure ); + return; } mContext->wake(); } +bool SSHConnection::hasQueuedOperations() const { + std::lock_guard lock( mQueueMutex ); + return !mQueue.empty(); +} + void SSHConnection::fail( VFSError error ) { State state = mState.load(); if ( state == State::Failed || state == State::Closed ) return; mFailure = std::move( error ); mState = State::Failed; + if ( mAgentClient ) + mAgentClient->cancel(); + { + // From now on enqueue() fails operations itself; drive() fails the queued ones. + std::lock_guard lock( mQueueMutex ); + mQueueFailure = mFailure; + } for ( auto& waiter : mReadyWaiters ) waiter.resolve( Err( mFailure ) ); mReadyWaiters.clear(); @@ -540,9 +551,20 @@ int SSHConnection::nextTimeoutMs() const { consider( mDeadline ); if ( state == State::Ready && mConfig.serverAliveInterval ) consider( mNextKeepalive ); + if ( state == State::Ready && mIdleTimeout != Time::Zero && isIdle() ) + consider( lastUse() + idleTimeout() ); return timeout; } +std::chrono::milliseconds SSHConnection::idleTimeout() const { + return std::chrono::milliseconds( static_cast( mIdleTimeout.asMilliseconds() ) ); +} + +bool SSHConnection::idleExpired() const { + return mState.load() == State::Ready && mIdleTimeout != Time::Zero && isIdle() && + std::chrono::steady_clock::now() - lastUse() > idleTimeout(); +} + void SSHConnection::drive() { if ( mTransport && !mTransport->alive() && isUsable() ) fail( VFSError( VFSErrorCode::ConnectionLost, URI(), "Transport closed" ) ); @@ -566,8 +588,10 @@ void SSHConnection::drive() { operation->fail( mFailure ); auto pumps = std::move( mPumps ); mPumps.clear(); - for ( auto& pump : pumps ) + for ( auto& pump : pumps ) { + removeUser(); pump->connectionLost( mFailure ); + } return; } @@ -600,6 +624,7 @@ void SSHConnection::drive() { ++i; } else { mPumps.erase( mPumps.begin() + i ); + removeUser(); } if ( !isUsable() ) { drive(); @@ -607,6 +632,12 @@ void SSHConnection::drive() { } } + if ( idleExpired() ) { + fail( VFSError( VFSErrorCode::ConnectionLost, URI(), "Idle connection closed" ) ); + mState = State::Closed; + return; + } + if ( mConfig.serverAliveInterval && std::chrono::steady_clock::now() >= mNextKeepalive ) { int seconds = 0; int rc = libssh2_keepalive_send( mSession, &seconds ); @@ -625,9 +656,9 @@ std::vector SSHConnection::knownHostKeyAlgorithms() { if ( !hosts ) return known; for ( const auto& file : mConfig.userKnownHostsFiles ) - loadKnownHosts( hosts, file ); + loadKnownHosts( hosts, file, nullptr ); for ( const auto& file : mConfig.globalKnownHostsFiles ) - loadKnownHosts( hosts, file ); + loadKnownHosts( hosts, file, nullptr ); std::string checkHost = mConfig.hostKeyAlias.value_or( mConfig.hostName ); // A key that can never be stored: a "mismatch" means an entry of that type exists for the // host. This lets libssh2 handle hashed names and [host]:port entries. @@ -647,7 +678,7 @@ std::vector SSHConnection::knownHostKeyAlgorithms() { return known; } -void SSHConnection::applyAlgorithmPreferences() { +VFSResult SSHConnection::applyAlgorithmPreferences() { // Like OpenSSH, unless HostKeyAlgorithms is configured, prefer the host key algorithms that // known_hosts already has keys for, so a known host is not reported as unknown just because // the server also offers another key type. @@ -677,22 +708,30 @@ void SSHConnection::applyAlgorithmPreferences() { struct Pref { const std::optional* spec; int method; + const char* option; }; const Pref prefs[] = { - { &mConfig.kexAlgorithms, LIBSSH2_METHOD_KEX }, - { &mConfig.hostKeyAlgorithms, LIBSSH2_METHOD_HOSTKEY }, - { &mConfig.ciphers, LIBSSH2_METHOD_CRYPT_CS }, - { &mConfig.ciphers, LIBSSH2_METHOD_CRYPT_SC }, - { &mConfig.macs, LIBSSH2_METHOD_MAC_CS }, - { &mConfig.macs, LIBSSH2_METHOD_MAC_SC }, + { &mConfig.kexAlgorithms, LIBSSH2_METHOD_KEX, "KexAlgorithms" }, + { &mConfig.hostKeyAlgorithms, LIBSSH2_METHOD_HOSTKEY, "HostKeyAlgorithms" }, + { &mConfig.ciphers, LIBSSH2_METHOD_CRYPT_CS, "Ciphers" }, + { &mConfig.ciphers, LIBSSH2_METHOD_CRYPT_SC, "Ciphers" }, + { &mConfig.macs, LIBSSH2_METHOD_MAC_CS, "MACs" }, + { &mConfig.macs, LIBSSH2_METHOD_MAC_SC, "MACs" }, }; for ( const auto& pref : prefs ) { if ( !*pref.spec ) continue; + // A configured restriction is a policy: when nothing it permits is available, the + // connection fails instead of falling back to libssh2's defaults. + auto unsupported = [&]() -> VFSResult { + return Err( VFSError( VFSErrorCode::Unsupported, URI(), + std::string( "None of the configured " ) + pref.option + " (" + + **pref.spec + ") is supported" ) ); + }; const char** algs = nullptr; int count = libssh2_session_supported_algs( mSession, pref.method, &algs ); if ( count <= 0 ) - continue; + return unsupported(); std::vector supported( algs, algs + count ); libssh2_free( mSession, algs ); auto list = SSHConfigEvaluator::applyAlgorithmSpec( supported, **pref.spec ); @@ -705,9 +744,11 @@ void SSHConnection::applyAlgorithmPreferences() { joined += ','; joined += alg; } - if ( !joined.empty() ) - libssh2_session_method_pref( mSession, pref.method, joined.c_str() ); + if ( joined.empty() || + libssh2_session_method_pref( mSession, pref.method, joined.c_str() ) != 0 ) + return unsupported(); } + return Ok(); } Step SSHConnection::establish() { @@ -721,7 +762,10 @@ Step SSHConnection::establish() { libssh2_session_set_blocking( mSession, 0 ); if ( mConfig.compression ) libssh2_session_flag( mSession, LIBSSH2_FLAG_COMPRESS, 1 ); - applyAlgorithmPreferences(); + if ( auto preferred = applyAlgorithmPreferences(); !preferred ) { + fail( std::move( preferred ).error() ); + return Step::Done; + } mTransport->install( mSession ); mState = State::Handshake; [[fallthrough]]; @@ -747,8 +791,9 @@ Step SSHConnection::establish() { mState = State::SFTPInit; [[fallthrough]]; case State::SFTPInit: - mSFTP = libssh2_sftp_init( mSession ); - if ( !mSFTP ) { + // Jump hosts only forward channels: they need not offer SFTP. + mSFTP = mSFTPRequired ? libssh2_sftp_init( mSession ) : nullptr; + if ( !mSFTP && mSFTPRequired ) { if ( libssh2_session_last_errno( mSession ) == LIBSSH2_ERROR_EAGAIN ) return Step::Again; fail( lastError( URI(), "Unable to start the SFTP subsystem" ) ); @@ -818,10 +863,21 @@ Step SSHConnection::stepHostKey() { std::string checkHost = mConfig.hostKeyAlias.value_or( mConfig.hostName ); int keyBits = knownHostKeyBits( keyType ); LIBSSH2_KNOWNHOSTS* hosts = libssh2_knownhost_init( mSession ); + std::vector revoked; for ( const auto& file : mConfig.userKnownHostsFiles ) - loadKnownHosts( hosts, file ); + loadKnownHosts( hosts, file, &revoked ); for ( const auto& file : mConfig.globalKnownHostsFiles ) - loadKnownHosts( hosts, file ); + loadKnownHosts( hosts, file, &revoked ); + const std::string_view presented( key, keyLength ); + if ( std::find( revoked.begin(), revoked.end(), presented ) != revoked.end() ) { + // Checked before anything else: a revoked key is never accepted, even when it is also + // listed as known or StrictHostKeyChecking would accept new keys. + libssh2_knownhost_free( hosts ); + fail( VFSError( VFSErrorCode::HostKeyRejected, URI(), + "The " + std::string( hostKeyTypeName( keyType ) ) + " host key of " + + checkHost + " is revoked" ) ); + return Step::Done; + } struct libssh2_knownhost* entry = nullptr; int check = libssh2_knownhost_checkp( hosts, checkHost.c_str(), mConfig.port, key, keyLength, @@ -959,6 +1015,101 @@ bool SSHConnection::saveHostKey( const char* key, size_t keyLength, int keyBits, return true; } +void SSHConnection::startAgentListing() { + mAuthPhase = AuthPhase::AgentListing; + // Listing identities needs no user interaction. Keep the connection deadline active so an + // unresponsive agent cannot leave the connection pending forever. + mWaitingExternal = false; + auto agent = std::make_shared( *mConfig.identityAgent ); + mAgentClient = agent; + std::vector allowed; + if ( mConfig.identitiesOnly ) { + for ( const auto& file : mConfig.identityFiles ) + if ( auto blob = readPublicKeyBlob( file + ".pub" ) ) + allowed.push_back( std::move( *blob ) ); + } + std::weak_ptr weak = weak_from_this(); + std::weak_ptr context = mContext; + // The connection cancels the agent socket on failure or shutdown, waking this worker even + // if the agent never answers. The worker owns its captures until it exits. + std::thread( [agent, weak, context, identitiesOnly = mConfig.identitiesOnly, + allowed = std::move( allowed )]() mutable { + auto keys = agent->listIdentities(); + auto io = context.lock(); + if ( !io ) + return; + io->post( [agent = std::move( agent ), weak, identitiesOnly, allowed = std::move( allowed ), + keys = std::make_shared( std::move( keys ) )]() mutable { + auto self = weak.lock(); + if ( !self || !self->isUsable() || self->mAuthPhase != AuthPhase::AgentListing ) + return; + self->mWaitingExternal = false; + self->mAgentKeys.clear(); + self->mAgentKeyIndex = 0; + if ( *keys ) { + for ( auto& key : keys->value() ) { + if ( !identitiesOnly || + std::find( allowed.begin(), allowed.end(), key.blob ) != allowed.end() ) + self->mAgentKeys.push_back( std::move( key ) ); + } + } + self->mAgentClient = self->mAgentKeys.empty() ? nullptr : std::move( agent ); + self->mAuthPhase = + self->mAgentKeys.empty() ? AuthPhase::KeyNext : AuthPhase::AgentKeyNext; + } ); + } ).detach(); +} + +void SSHConnection::startAgentSignature( std::string_view payload ) { + mAgentSignPayload.assign( payload ); + mAgentSignature.reset(); + mAgentSigning = true; + mWaitingExternal = true; + std::weak_ptr weak = weak_from_this(); + std::weak_ptr context = mContext; + std::thread( [agent = mAgentClient, weak, context, blob = mAgentKeys[mAgentKeyIndex].blob, + payload = mAgentSignPayload, + algorithm = std::string( SSHAgent::algorithmOfSignedRequest( payload ) )] { + auto signature = + std::make_shared>( agent->sign( blob, payload, algorithm ) ); + if ( auto io = context.lock() ) { + io->post( [weak, signature] { + auto self = weak.lock(); + if ( !self || !self->isUsable() || !self->mAgentSigning ) + return; + self->mAgentSigning = false; + self->mWaitingExternal = false; + self->mAgentSignature = std::move( *signature ); + } ); + } + } ).detach(); +} + +int SSHConnection::agentSign( LIBSSH2_SESSION*, unsigned char** signature, size_t* signatureLength, + const unsigned char* data, size_t dataLength, void** abstract ) { + auto* self = static_cast( *abstract ); + std::string_view payload( reinterpret_cast( data ), dataLength ); + // libssh2 calls again with the same payload after an EAGAIN. + if ( self->mAgentSignature && self->mAgentSignPayload == payload ) { + auto result = std::move( *self->mAgentSignature ); + self->mAgentSignature.reset(); + self->mAgentSignPayload.clear(); + if ( !result ) + return LIBSSH2_ERROR_AGENT_PROTOCOL; + const std::string& raw = result.value(); + // Released by libssh2 with the session's (default) allocator. + *signature = static_cast( std::malloc( raw.size() ) ); + if ( !*signature ) + return LIBSSH2_ERROR_ALLOC; + std::memcpy( *signature, raw.data(), raw.size() ); + *signatureLength = raw.size(); + return 0; + } + if ( !self->mAgentSigning ) + self->startAgentSignature( payload ); + return LIBSSH2_ERROR_EAGAIN; +} + void SSHConnection::requestExternal( const SSHAuthenticationRequest& request ) { mWaitingExternal = true; mExternalAnswer.reset(); @@ -1075,6 +1226,10 @@ Step SSHConnection::stepAuthentication() { mAuthPhase = AuthPhase::KeyNext; continue; } + if ( SSHAgent::supported() ) { + startAgentListing(); + return Step::Again; + } mAgent = libssh2_agent_init( mSession ); if ( mAgent ) { libssh2_agent_set_identity_path( mAgent, mConfig.identityAgent->c_str() ); @@ -1132,6 +1287,43 @@ Step SSHConnection::stepAuthentication() { mAuthPhase = AuthPhase::AgentNext; continue; } + case AuthPhase::AgentListing: + // startAgentListing() moves on once the agent answered. + return Step::Again; + case AuthPhase::AgentKeyNext: { + if ( mAgentKeyIndex >= mAgentKeys.size() ) { + mAgentClient.reset(); + mAgentKeys.clear(); + mAuthPhase = AuthPhase::KeyNext; + continue; + } + mAgentSignPayload.clear(); + mAgentSignature.reset(); + mAuthPhase = AuthPhase::AgentKeyAuth; + continue; + } + case AuthPhase::AgentKeyAuth: { + const std::string& blob = mAgentKeys[mAgentKeyIndex].blob; + void* abstract = this; + int rc = libssh2_userauth_publickey( + mSession, user.c_str(), reinterpret_cast( blob.data() ), + blob.size(), &SSHConnection::agentSign, &abstract ); + if ( rc == LIBSSH2_ERROR_EAGAIN ) + return Step::Again; + if ( rc == 0 ) { + mAgentClient.reset(); + mAgentKeys.clear(); + mAuthPhase = AuthPhase::Done; + continue; + } + if ( isSocketError( libssh2_session_last_errno( mSession ) ) ) { + fail( lastError( URI(), "Authentication failed" ) ); + return Step::Done; + } + ++mAgentKeyIndex; + mAuthPhase = AuthPhase::AgentKeyNext; + continue; + } case AuthPhase::KeyNext: { while ( mKeyIndex < mConfig.identityFiles.size() && !FileSystem::fileExists( mConfig.identityFiles[mKeyIndex] ) ) @@ -1327,9 +1519,14 @@ short SocketTransport::pollEvents( short directions ) const { ChannelTransport::ChannelTransport( std::shared_ptr parent, std::string host, Uint16 port ) : - mParent( std::move( parent ) ), mHost( std::move( host ) ), mPort( port ) {} + mParent( std::move( parent ) ), mHost( std::move( host ) ), mPort( port ) { + // The child's traffic is invisible to the parent's queue: keep the parent from closing as + // idle while the child lives. + mParent->addUser(); +} ChannelTransport::~ChannelTransport() { + mParent->removeUser(); // The channel belongs to the parent session; libssh2_session_free() of the parent releases // it. Freeing it here could race with the parent's I/O thread usage. } diff --git a/src/modules/vfs/src/eepp/vfs/sftp/sshsession.hpp b/src/modules/vfs/src/eepp/vfs/sftp/sshsession.hpp index 633b292e8..1ea7018ff 100644 --- a/src/modules/vfs/src/eepp/vfs/sftp/sshsession.hpp +++ b/src/modules/vfs/src/eepp/vfs/sftp/sshsession.hpp @@ -1,6 +1,7 @@ #ifndef EE_VFS_SFTP_SSHSESSION_HPP #define EE_VFS_SFTP_SSHSESSION_HPP +#include "sshagent.hpp" #include "sshsocket.hpp" #include #include @@ -128,6 +129,9 @@ class SSHIOContext : public std::enable_shared_from_this { /** Registers a connection (I/O thread only). */ void addConnection( std::shared_ptr connection ); + /** Fails registered sessions, including sessions still awaiting authentication. */ + void closeConnections(); + private: void loop(); @@ -221,6 +225,8 @@ class SSHConnection : public std::enable_shared_from_this { const SSHResolvedConfig& config() const { return mConfig; } + const std::shared_ptr& ioContext() const { return mContext; } + /** Last activity time (any thread), for idle pooling. */ std::chrono::steady_clock::time_point lastUse() const; @@ -234,6 +240,11 @@ class SSHConnection : public std::enable_shared_from_this { /** @return Milliseconds until a timer of this connection fires, -1 if none. */ int nextTimeoutMs() const; + std::chrono::milliseconds idleTimeout() const; + + /** @return True when the connection stayed idle longer than its idle timeout. */ + bool idleExpired() const; + LIBSSH2_SESSION* session() const { return mSession; } LIBSSH2_SFTP* sftp() const { return mSFTP; } @@ -253,14 +264,44 @@ class SSHConnection : public std::enable_shared_from_this { /** Tracks an open SFTP handle (I/O thread only). Handles still open when the connection is * destroyed (for example because their owners died after the connection failed) are closed * then, so libssh2 releases them. */ - void trackHandle( LIBSSH2_SFTP_HANDLE* handle ) { mHandles.push_back( handle ); } + void trackHandle( LIBSSH2_SFTP_HANDLE* handle ) { + mHandles.push_back( handle ); + addUser(); + } void untrackHandle( LIBSSH2_SFTP_HANDLE* handle ) { - mHandles.erase( std::remove( mHandles.begin(), mHandles.end(), handle ), mHandles.end() ); + auto it = std::find( mHandles.begin(), mHandles.end(), handle ); + if ( it != mHandles.end() ) { + mHandles.erase( it ); + removeUser(); + } } /** Registers a channel pump (I/O thread only, connection must be ready). */ - void addPump( std::shared_ptr pump ) { mPumps.push_back( std::move( pump ) ); } + void addPump( std::shared_ptr pump ) { + mPumps.push_back( std::move( pump ) ); + addUser(); + } + + /** Something keeps using the connection without queued operations: an open handle, a pump, + * a jump child's channel. A connection in use is never closed as idle. Any thread. */ + void addUser() { mUsers.fetch_add( 1, std::memory_order_relaxed ); } + + void removeUser() { mUsers.fetch_sub( 1, std::memory_order_relaxed ); } + + /** @return True when nothing uses the connection (no users, no queued operations). */ + bool isIdle() const { + return mUsers.load( std::memory_order_relaxed ) == 0 && !hasQueuedOperations(); + } + + /** Closes the connection once it stayed idle this long (zero: never). Set it before the + * connection is added to the I/O context. */ + void setIdleTimeout( Time timeout ) { mIdleTimeout = timeout; } + + /** Whether Ready includes a started SFTP subsystem (true by default). Jump hosts only open + * channels for the next hop and need no SFTP. Set it before the connection is added to the + * I/O context. */ + void setSFTPRequired( bool required ) { mSFTPRequired = required; } /** Opens a direct-tcpip channel for a child session (ProxyJump). I/O thread only. */ LIBSSH2_CHANNEL* openDirectTcpip( const std::string& host, Uint16 port, bool& again ); @@ -286,8 +327,11 @@ class SSHConnection : public std::enable_shared_from_this { friend class ChannelTransport; friend class ProcessTransport; + /** Queues operation, or fails it right away when the connection is already terminal. */ void enqueue( std::unique_ptr operation ); + bool hasQueuedOperations() const; + void fail( VFSError error ); Step establish(); @@ -302,7 +346,19 @@ class SSHConnection : public std::enable_shared_from_this { void requestExternal( const SSHAuthenticationRequest& request ); - void applyAlgorithmPreferences(); + /** Lists the agent's keys on a worker thread (AgentListing until it answers). */ + void startAgentListing(); + + /** Asks the agent for the signature of payload on a worker thread. */ + void startAgentSignature( std::string_view payload ); + + /** libssh2 sign callback of agent keys: EAGAIN until the agent thread answered. */ + static int agentSign( LIBSSH2_SESSION* session, unsigned char** signature, + size_t* signatureLength, const unsigned char* data, size_t dataLength, + void** abstract ); + + /** Applies the configured algorithm restrictions; fails when one permits nothing available. */ + VFSResult applyAlgorithmPreferences(); /** Host key algorithms for which known_hosts holds a key of this host. */ std::vector knownHostKeyAlgorithms(); @@ -319,6 +375,9 @@ class SSHConnection : public std::enable_shared_from_this { mutable std::mutex mQueueMutex; std::deque> mQueue; + /** Set (under mQueueMutex) once the connection failed or closed: new operations fail with it + * instead of being queued. */ + std::optional mQueueFailure; std::vector> mPumps; std::vector mHandles; std::vector>> mReadyWaiters; @@ -333,8 +392,13 @@ class SSHConnection : public std::enable_shared_from_this { List, NextMethod, AgentConnect, + /** libssh2's agent (where SSHAgent is not supported). */ AgentNext, AgentAuth, + /** SSHAgent on a worker thread: listing, then each key. */ + AgentListing, + AgentKeyNext, + AgentKeyAuth, KeyNext, KeyAuth, KbdAuth, @@ -349,6 +413,15 @@ class SSHConnection : public std::enable_shared_from_this { std::string mServerMethods; struct libssh2_agent_publickey* mAgentIdentity{ nullptr }; std::vector mAllowedAgentBlobs; + /** Agent calls block (signing may wait for a hardware key touch): they run on a worker + * thread while the I/O thread keeps serving the other sessions. */ + std::shared_ptr mAgentClient; + std::vector mAgentKeys; + size_t mAgentKeyIndex{ 0 }; + /** The signature libssh2 asked for (its payload), and the agent's answer once it arrived. */ + std::string mAgentSignPayload; + std::optional> mAgentSignature; + bool mAgentSigning{ false }; size_t mKeyIndex{ 0 }; std::string mPassphrase; Uint32 mAttempts{ 0 }; @@ -365,15 +438,12 @@ class SSHConnection : public std::enable_shared_from_this { int mPendingHostKeyBits{ 0 }; std::atomic mState{ State::Starting }; + std::atomic mUsers{ 0 }; + Time mIdleTimeout; + bool mSFTPRequired{ true }; bool mWaitingExternal{ false }; }; -/** Base64 without padding, as used by OpenSSH fingerprints. */ -std::string base64Unpadded( const unsigned char* data, size_t length ); - -/** Maps an SFTP status code to a VFS error code. */ -VFSErrorCode mapSFTPStatus( unsigned long status ); - /** TCP transport. */ class SocketTransport : public SSHTransport { public: diff --git a/src/modules/vfs/src/eepp/vfs/ssh/sshconfig.cpp b/src/modules/vfs/src/eepp/vfs/ssh/sshconfig.cpp index c5d4f553c..98538e513 100644 --- a/src/modules/vfs/src/eepp/vfs/ssh/sshconfig.cpp +++ b/src/modules/vfs/src/eepp/vfs/ssh/sshconfig.cpp @@ -1307,8 +1307,32 @@ std::string SSHResolvedConfig::connectionKey() const { add( identitiesOnly ? "1" : "0" ); for ( const auto& identity : identityFiles ) add( identity ); + for ( const auto& certificate : certificateFiles ) + add( certificate ); add( identityAgent.value_or( "" ) ); add( hostKeyAlias.value_or( "" ) ); + // Security policy: a connection verified or authenticated under a weaker policy must never + // be handed to a destination that demands a stricter one. + add( "known" ); + for ( const auto& file : userKnownHostsFiles ) + add( file ); + add( "global" ); + for ( const auto& file : globalKnownHostsFiles ) + add( file ); + add( String::toString( static_cast( strictHostKeyChecking ) ) ); + add( checkHostIP ? "1" : "0" ); + for ( const auto* algorithms : + { &ciphers, &macs, &kexAlgorithms, &hostKeyAlgorithms, &pubkeyAcceptedAlgorithms } ) + add( algorithms->has_value() ? "+" + **algorithms : std::string() ); + add( "auth" ); + for ( const auto& method : preferredAuthentications ) + add( method ); + add( pubkeyAuthentication ? "1" : "0" ); + add( passwordAuthentication ? "1" : "0" ); + add( kbdInteractiveAuthentication ? "1" : "0" ); + add( String::toString( numberOfPasswordPrompts ) ); + add( batchMode ? "1" : "0" ); + add( compression ? "1" : "0" ); return key; } diff --git a/src/modules/vfs/src/eepp/vfs/ui/vfsdirectorymodel.cpp b/src/modules/vfs/src/eepp/vfs/ui/vfsdirectorymodel.cpp index cec51fc2f..f4d640808 100644 --- a/src/modules/vfs/src/eepp/vfs/ui/vfsdirectorymodel.cpp +++ b/src/modules/vfs/src/eepp/vfs/ui/vfsdirectorymodel.cpp @@ -359,13 +359,15 @@ void VFSDirectoryModel::rescan() { // Entry storage // --------------------------------------------------------------------------------------------- -bool VFSDirectoryModel::less( const FileInfo& a, const FileInfo& b ) const { +bool VFSDirectoryModel::less( const FileInfo& a, const FileInfo& b, const std::string* aType, + const std::string* bType ) const { bool aDir = a.isDirectory(), bDir = b.isDirectory(); if ( aDir != bDir ) return aDir; // Directories first, whatever the order. int cmp = 0; + // Runs O(n log n) times per sort on the UI thread: no allocations. auto compareNames = [&] { - int c = String::toLower( a.name ).compare( String::toLower( b.name ) ); + int c = String::icompare( a.name, b.name ); return c != 0 ? c : a.name.compare( b.name ); }; auto compareNumbers = []( auto x, auto y ) { return x < y ? -1 : ( y < x ? 1 : 0 ); }; @@ -378,7 +380,7 @@ bool VFSDirectoryModel::less( const FileInfo& a, const FileInfo& b ) const { cmp = compareNumbers( a.size().value_or( 0 ), b.size().value_or( 0 ) ); break; case Type: - cmp = typeOf( a ).compare( typeOf( b ) ); + cmp = aType && bType ? aType->compare( *bType ) : typeOf( a ).compare( typeOf( b ) ); break; case Modified: cmp = compareNumbers( a.modifiedTime().value_or( 0 ), b.modifiedTime().value_or( 0 ) ); @@ -400,6 +402,47 @@ bool VFSDirectoryModel::less( const FileInfo& a, const FileInfo& b ) const { return mSortOrder == SortOrder::Descending ? cmp > 0 : cmp < 0; } +void VFSDirectoryModel::sortEntries( std::vector& entries ) const { + if ( mSortColumn != Type ) { + std::stable_sort( entries.begin(), entries.end(), + [this]( const FileInfo& a, const FileInfo& b ) { return less( a, b ); } ); + return; + } + // The type provider (a std::function returning a string) runs once per entry, not twice per + // comparison. + std::vector types; + types.reserve( entries.size() ); + for ( const auto& entry : entries ) + types.push_back( typeOf( entry ) ); + std::vector order( entries.size() ); + for ( size_t i = 0; i < order.size(); ++i ) + order[i] = i; + std::stable_sort( order.begin(), order.end(), [&]( size_t x, size_t y ) { + return less( entries[x], entries[y], &types[x], &types[y] ); + } ); + std::vector sorted; + sorted.reserve( entries.size() ); + for ( size_t index : order ) + sorted.push_back( std::move( entries[index] ) ); + entries.swap( sorted ); +} + +void VFSDirectoryModel::rowsInserted( size_t entryIndex, bool visible ) { + size_t row = visibleRowFor( entryIndex ); + for ( size_t i = row; i < mRows.size(); ++i ) + ++mRows[i]; + if ( visible ) + mRows.insert( mRows.begin() + row, entryIndex ); +} + +void VFSDirectoryModel::rowsErased( size_t entryIndex ) { + size_t row = visibleRowFor( entryIndex ); + if ( row < mRows.size() && mRows[row] == entryIndex ) + mRows.erase( mRows.begin() + row ); + for ( size_t i = row; i < mRows.size(); ++i ) + --mRows[i]; +} + void VFSDirectoryModel::rebuildRows() { mRows.clear(); mRows.reserve( mEntries.size() ); @@ -419,7 +462,7 @@ void VFSDirectoryModel::mergeBatch( std::vector&& batch ) { for ( const auto& info : batch ) queueItemCount( info ); auto cmp = [this]( const FileInfo& a, const FileInfo& b ) { return less( a, b ); }; - std::sort( batch.begin(), batch.end(), cmp ); + sortEntries( batch ); if ( mEntries.empty() ) { mEntries = std::move( batch ); } else { @@ -446,15 +489,17 @@ void VFSDirectoryModel::removeByName( const std::string& name ) { size_t index = static_cast( it - mEntries.begin() ); bool visible = isVisible( *it ); mItemCounts.erase( name ); + // Rows are updated in place: watch events arrive one by one, a full rebuild would cost a + // pass over every entry each time. if ( visible ) { int row = static_cast( visibleRowFor( index ) ); beginDeleteRows( {}, row, row ); mEntries.erase( it ); - rebuildRows(); + rowsErased( index ); endDeleteRows(); } else { mEntries.erase( it ); - rebuildRows(); + rowsErased( index ); } onModelUpdate( Model::UpdateFlag::DontInvalidateIndexes ); notifyChanged(); @@ -491,11 +536,11 @@ void VFSDirectoryModel::upsert( FileInfo&& info ) { int row = static_cast( visibleRowFor( index ) ); beginInsertRows( {}, row, row ); mEntries.insert( pos, std::move( info ) ); - rebuildRows(); + rowsInserted( index, true ); endInsertRows(); } else { mEntries.insert( pos, std::move( info ) ); - rebuildRows(); + rowsInserted( index, false ); } onModelUpdate( Model::UpdateFlag::DontInvalidateIndexes ); notifyChanged(); @@ -683,8 +728,9 @@ bool VFSDirectoryModel::matchesNameFilter( const std::string& name ) const { case NameFilterMode::PlainText: if ( mFilterCaseSensitive ) return name.find( mFilterPattern ) != std::string::npos; - return String::toLower( name ).find( String::toLower( mFilterPattern ) ) != - std::string::npos; + // Every entry is matched on each rebuild: no lowercase copies. + return String::BMH::find( std::string_view( name ), mFilterPattern, 0, + mFilterOccurrences, true ) != -1; case NameFilterMode::Glob: return String::globMatch( name, mFilterGlob, !mFilterCaseSensitive ); case NameFilterMode::RegEx: @@ -702,6 +748,12 @@ void VFSDirectoryModel::setNameFilter( std::string pattern, NameFilterMode mode, mFilterCaseSensitive = caseSensitive; mFilterRegEx.reset(); mFilterValid = true; + mFilterOccurrences.clear(); + if ( mode == NameFilterMode::PlainText && !caseSensitive && !mFilterPattern.empty() ) { + mFilterOccurrences = String::BMH::createOccTable( + reinterpret_cast( mFilterPattern.data() ), + mFilterPattern.size(), true ); + } // Unanchored like Dolphin ("md" matches "notes.md"). A doubled '*' would mean "**" (any // path) to globMatch, so only missing wildcards are added. mFilterGlob.clear(); @@ -893,8 +945,7 @@ Variant VFSDirectoryModel::data( const ModelIndex& index, ModelRole role ) const void VFSDirectoryModel::sort( const size_t& column, const SortOrder& order ) { mSortColumn = column < ColumnCount ? column : Name; mSortOrder = order == SortOrder::None ? SortOrder::Ascending : order; - std::stable_sort( mEntries.begin(), mEntries.end(), - [this]( const FileInfo& a, const FileInfo& b ) { return less( a, b ); } ); + sortEntries( mEntries ); rebuildRows(); invalidate( Model::UpdateFlag::InvalidateAllIndexes ); } diff --git a/src/modules/vfs/src/eepp/vfs/virtualfilesystem.cpp b/src/modules/vfs/src/eepp/vfs/virtualfilesystem.cpp index 5855957c7..d58221a2b 100644 --- a/src/modules/vfs/src/eepp/vfs/virtualfilesystem.cpp +++ b/src/modules/vfs/src/eepp/vfs/virtualfilesystem.cpp @@ -46,22 +46,142 @@ std::string temporarySiblingName( std::string_view name ) { return tmp; } -/** Writes every byte of data through handle, starting at offset 0. */ +/** Writes every byte of data through handle, starting at offset. */ VFSResult writeFully( FileHandle& handle, std::span data, - const CancellationToken& token ) { - Uint64 offset = 0; - while ( offset < data.size() ) { - size_t chunk = std::min( IOChunkSize, data.size() - offset ); - auto written = await( handle.write( offset, data.subspan( offset, chunk ), token ) ); + const CancellationToken& token, Uint64 offset = 0 ) { + size_t done = 0; + while ( done < data.size() ) { + size_t chunk = std::min( IOChunkSize, data.size() - done ); + auto written = await( handle.write( offset + done, data.subspan( done, chunk ), token ) ); if ( !written ) return Err( std::move( written ).error() ); if ( written.value() == 0 ) return Err( VFSError( VFSErrorCode::IO, handle.uri(), "Short write" ) ); - offset += written.value(); + done += written.value(); } return Ok(); } +/** makeDirs() body. Orchestration jobs call it directly: waiting on a makeDirs() task from the + * orchestration pool would wait on a job queued behind the waiting one. */ +VFSResult makeDirsBlocking( VFSProvider& provider, const URI& uri, + const MkDirOptions& options, const CancellationToken& token ) { + std::vector missing; + URI current( uri ); + while ( true ) { + auto info = await( provider.stat( current, StatOptions{}, token ) ); + if ( info ) { + if ( !info.value().isDirectory() ) + return Err( VFSError( VFSErrorCode::NotDirectory, current ) ); + break; + } + if ( !info.error().is( VFSErrorCode::NotFound ) ) + return Err( std::move( info ).error() ); + missing.push_back( current ); + URI parent( current.getParent() ); + if ( parent.getPath() == current.getPath() ) + break; + current = std::move( parent ); + } + for ( auto it = missing.rbegin(); it != missing.rend(); ++it ) { + auto made = await( provider.mkdir( *it, options, token ) ); + if ( !made ) { + if ( !made.error().is( VFSErrorCode::AlreadyExists ) ) + return made; + // Lost a race with another creator: fine if it is a directory. + auto info = await( provider.stat( *it, StatOptions{}, token ) ); + if ( !info || !info.value().isDirectory() ) + return made; + } + } + return Ok(); +} + +/** The file a write to uri replaces: the target of a symlink (so the link survives the + * replacement), otherwise uri itself. */ +VFSResult resolveWriteTarget( VFSProvider& provider, const URI& uri, + const CancellationToken& token ) { + auto linkInfo = await( provider.stat( uri, StatOptions{ false }, token ) ); + if ( !linkInfo || !linkInfo.value().isSymlink() ) + return Ok( URI( uri ) ); + auto resolved = await( provider.canonicalize( uri, token ) ); + // A dangling link is replaced by the new file. + if ( !resolved && resolved.error().is( VFSErrorCode::NotFound ) ) + return Ok( URI( uri ) ); + return resolved; +} + +struct ReplaceOptions { + /** Mode of a newly created file (the provider filters it through the umask). */ + std::optional createPermissions; + /** Exact mode to give the new version (for example the replaced file's), applied after it is + * written: creation modes are filtered through the umask. */ + std::optional exactPermissions; + bool fsync{ false }; + /** Replace through a temporary sibling. When false, target is rewritten in place. */ + bool atomic{ true }; + /** Without atomic replacement support, rewrite target in place instead of failing. */ + bool allowInPlace{ false }; + /** Fail if another entry appeared at target while the temporary was being written. */ + bool overwrite{ true }; +}; + +/** Writes a new version of target with write( FileHandle& ). The data goes to a temporary sibling + * that replaces target only once write() and beforeReplace() succeeded, so a failure never damages + * the current version, nor another name of the same file (a hard link to it). Non-atomic writes, + * and providers without atomic replacement (when allowed), rewrite target in place. */ +template +VFSResult replaceFile( VFSProvider& provider, const URI& target, + const ReplaceOptions& options, const CancellationToken& token, + Write&& write, BeforeReplace&& beforeReplace ) { + ProviderCapabilities caps = provider.capabilities( target ); + auto writeTo = [&]( const URI& file, OpenFlags flags ) -> VFSResult { + OpenOptions openOptions( flags ); + openOptions.createPermissions = + options.exactPermissions ? options.exactPermissions : options.createPermissions; + auto opened = await( provider.open( file, openOptions, token ) ); + if ( !opened ) + return Err( std::move( opened ).error() ); + FileHandle handle( std::move( opened ).value() ); + VFSResult written = write( handle ); + if ( written && options.fsync && caps.has( VFSCapability::Fsync ) ) + written = await( handle.flush() ); + auto closed = await( handle.close() ); + if ( !written ) + return written; + if ( !closed ) + return closed; + if ( options.exactPermissions && caps.has( VFSCapability::Permissions ) ) + return await( provider.setPermissions( file, *options.exactPermissions, token ) ); + return Ok(); + }; + + if ( !options.atomic || !caps.has( VFSCapability::Rename ) || + ( options.overwrite && !caps.has( VFSCapability::AtomicReplace ) ) ) { + if ( options.atomic && !options.allowInPlace ) { + return Err( VFSError( VFSErrorCode::Unsupported, target, + "Atomic replacement is not supported by this provider" ) ); + } + // beforeReplace() guards a replacement; in place, the old version is already gone. + return writeTo( target, OpenFlags::Write | OpenFlags::Create | OpenFlags::Truncate ); + } + + URI temporary( target.getParent().withAppendedSegment( + temporarySiblingName( target.getLastPathSegment() ) ) ); + auto written = + writeTo( temporary, OpenFlags::Write | OpenFlags::Create | OpenFlags::Exclusive ); + if ( written ) + written = beforeReplace(); + if ( written ) + written = await( + provider.rename( temporary, target, RenameOptions{ options.overwrite }, token ) ); + if ( !written ) { + // Best effort: never leave the temporary behind. + await( provider.remove( temporary, {}, CancellationToken() ) ); + } + return written; +} + } // namespace VirtualFileSystem& VirtualFileSystem::instance() { @@ -85,6 +205,7 @@ VirtualFileSystem::VirtualFileSystem( const Config& config ) : mIOPool( ThreadPool::createShared( std::max( 1, config.ioThreads ) ) ), mOrchestrationPool( ThreadPool::createShared( std::max( 1, config.orchestrationThreads ) ) ), + mTransferPool( ThreadPool::createShared( std::max( 1, config.transferThreads ) ) ), mWatchManager( std::make_unique() ) { if ( config.registerDefaultProviders ) registerDefaultProviders(); @@ -92,6 +213,7 @@ VirtualFileSystem::VirtualFileSystem( const Config& config ) : VirtualFileSystem::~VirtualFileSystem() { // Orchestration jobs wait on provider tasks: stop them first, while providers still run. + mTransferPool.reset(); mOrchestrationPool.reset(); mWatchManager.reset(); { @@ -104,7 +226,7 @@ VirtualFileSystem::~VirtualFileSystem() { void VirtualFileSystem::registerDefaultProviders() { registerProvider( std::make_shared( mIOPool ) ); registerProvider( std::make_shared( mIOPool ) ); - mWatchManager->registerBackend( "file", std::make_shared() ); + mWatchManager->registerBackend( "file", std::make_shared( mIOPool ) ); mWatchManager->setFallbackBackend( std::make_shared( *this ) ); #ifdef EE_VFS_SFTP auto sftp = std::make_shared(); @@ -372,103 +494,64 @@ VFSTask VirtualFileSystem::writeAll( const URI& uri, std::vector( uri ); // The buffer is moved into the job: the caller's data must outlive the asynchronous write. - return orchestrate( [this, provider, uri, data = std::move( data ), options, - token]() -> VFSResult { - auto checkRevision = [&]( const VFSResult& current ) -> VFSResult { - if ( !options.expectedRevision ) - return Ok(); - if ( !current ) { - if ( current.error().is( VFSErrorCode::NotFound ) ) + return orchestrate( + [provider, uri, data = std::move( data ), options, token]() -> VFSResult { + auto checkRevision = [&]( const VFSResult& current ) -> VFSResult { + if ( !options.expectedRevision ) + return Ok(); + if ( !current ) { + if ( current.error().is( VFSErrorCode::NotFound ) ) + return Err( VFSError( VFSErrorCode::ChangedExternally, uri, + "The file was removed externally" ) ); + return Err( current.error() ); + } + if ( !current.value().revision.matches( *options.expectedRevision ) ) return Err( VFSError( VFSErrorCode::ChangedExternally, uri, - "The file was removed externally" ) ); - return Err( current.error() ); + "The file was modified externally" ) ); + return Ok(); + }; + + auto target = resolveWriteTarget( *provider, uri, token ); + if ( !target ) + return Err( std::move( target ).error() ); + + auto existing = await( provider->stat( target.value(), StatOptions{}, token ) ); + if ( !existing && !existing.error().is( VFSErrorCode::NotFound ) ) + return Err( std::move( existing ).error() ); + if ( auto check = checkRevision( existing ); !check ) + return Err( std::move( check ).error() ); + if ( existing && existing.value().isDirectory() ) + return Err( VFSError( VFSErrorCode::IsDirectory, uri ) ); + + if ( options.createParents && !existing ) { + auto parents = makeDirsBlocking( *provider, target.value().getParent(), {}, token ); + if ( !parents ) + return Err( std::move( parents ).error() ); } - if ( !current.value().revision.matches( *options.expectedRevision ) ) - return Err( VFSError( VFSErrorCode::ChangedExternally, uri, - "The file was modified externally" ) ); - return Ok(); - }; - // Saving through a symlink writes the link target, so the link survives the atomic - // replace. - URI target( uri ); - auto linkInfo = await( provider->stat( uri, StatOptions{ false }, token ) ); - if ( linkInfo && linkInfo.value().isSymlink() ) { - auto resolved = await( provider->canonicalize( uri, token ) ); - if ( !resolved ) - return Err( std::move( resolved ).error() ); - target = std::move( resolved ).value(); - } - - auto existing = await( provider->stat( target, StatOptions{}, token ) ); - if ( !existing && !existing.error().is( VFSErrorCode::NotFound ) ) - return Err( std::move( existing ).error() ); - if ( auto check = checkRevision( existing ); !check ) - return Err( std::move( check ).error() ); - if ( existing && existing.value().isDirectory() ) - return Err( VFSError( VFSErrorCode::IsDirectory, uri ) ); - - if ( options.createParents && !existing ) { - auto parents = await( makeDirs( target.getParent(), {}, token ) ); - if ( !parents ) - return Err( std::move( parents ).error() ); - } - - ProviderCapabilities caps = provider->capabilities( target ); - bool atomic = options.atomic && caps.has( VFSCapability::Rename ) && - caps.has( VFSCapability::AtomicReplace ); - if ( options.atomic && !atomic && !options.allowNonAtomicFallback ) { - return Err( VFSError( VFSErrorCode::Unsupported, uri, - "Atomic replacement is not supported by this provider" ) ); - } - - OpenOptions openOptions; - openOptions.createPermissions = - existing ? existing.value().permissions() : options.permissions; - std::span bytes( data.data(), data.size() ); - - auto writeTo = [&]( const URI& file, OpenFlags flags ) -> VFSResult { - openOptions.flags = flags; - auto opened = await( provider->open( file, openOptions, token ) ); - if ( !opened ) - return Err( std::move( opened ).error() ); - FileHandle handle( std::move( opened ).value() ); - auto written = writeFully( handle, bytes, token ); - if ( written && options.fsync && caps.has( VFSCapability::Fsync ) ) - written = await( handle.flush() ); - auto closed = await( handle.close() ); - if ( !written ) - return written; - return closed; - }; - - if ( !atomic ) { - auto written = - writeTo( target, OpenFlags::Write | OpenFlags::Create | OpenFlags::Truncate ); + ReplaceOptions replace; + if ( existing ) + replace.exactPermissions = existing.value().permissions(); + else + replace.createPermissions = options.permissions; + replace.fsync = options.fsync; + replace.atomic = options.atomic; + replace.allowInPlace = options.allowNonAtomicFallback; + std::span bytes( data.data(), data.size() ); + auto written = replaceFile( + *provider, target.value(), replace, token, + [&]( FileHandle& handle ) { return writeFully( handle, bytes, token ); }, + [&]() -> VFSResult { + // Narrow the race window: re-check right before replacing. + if ( !options.expectedRevision ) + return Ok(); + return checkRevision( + await( provider->stat( target.value(), StatOptions{}, token ) ) ); + } ); if ( !written ) return Err( std::move( written ).error() ); - } else { - URI temporary( target.getParent().withAppendedSegment( - temporarySiblingName( target.getLastPathSegment() ) ) ); - auto written = - writeTo( temporary, OpenFlags::Write | OpenFlags::Create | OpenFlags::Exclusive ); - if ( written && options.expectedRevision ) { - // Narrow the race window: re-check right before replacing. - written = checkRevision( await( provider->stat( target, StatOptions{}, token ) ) ); - } - if ( written ) { - written = - await( provider->rename( temporary, target, RenameOptions{ true }, token ) ); - } - if ( !written ) { - // Best effort: never leave the temporary behind. - await( provider->remove( temporary, {}, CancellationToken() ) ); - return Err( std::move( written ).error() ); - } - } - - return await( provider->stat( target, StatOptions{}, token ) ); - } ); + return await( provider->stat( target.value(), StatOptions{}, token ) ); + } ); } VFSTask VirtualFileSystem::makeDirs( const URI& uri, const MkDirOptions& options, @@ -477,35 +560,7 @@ VFSTask VirtualFileSystem::makeDirs( const URI& uri, const MkDirOptions& o if ( !provider ) return unsupportedScheme( uri ); return orchestrate( [provider, uri, options, token]() -> VFSResult { - std::vector missing; - URI current( uri ); - while ( true ) { - auto info = await( provider->stat( current, StatOptions{}, token ) ); - if ( info ) { - if ( !info.value().isDirectory() ) - return Err( VFSError( VFSErrorCode::NotDirectory, current ) ); - break; - } - if ( !info.error().is( VFSErrorCode::NotFound ) ) - return Err( std::move( info ).error() ); - missing.push_back( current ); - URI parent( current.getParent() ); - if ( parent.getPath() == current.getPath() ) - break; - current = std::move( parent ); - } - for ( auto it = missing.rbegin(); it != missing.rend(); ++it ) { - auto made = await( provider->mkdir( *it, options, token ) ); - if ( !made ) { - if ( !made.error().is( VFSErrorCode::AlreadyExists ) ) - return made; - // Lost a race with another creator: fine if it is a directory. - auto info = await( provider->stat( *it, StatOptions{}, token ) ); - if ( !info || !info.value().isDirectory() ) - return made; - } - } - return Ok(); + return makeDirsBlocking( *provider, uri, options, token ); } ); } @@ -600,9 +655,9 @@ class TransferEngine { return Err( VFSError( VFSErrorCode::InvalidOperation, mSource, "Source and destination are the same" ) ); - bool sameProvider = mSrcProvider == mDstProvider && sameAuthority( mSource, mDestination ); + mSameProvider = mSrcProvider == mDstProvider && sameAuthority( mSource, mDestination ); - if ( mMove && sameProvider ) { + if ( mMove && mSameProvider ) { auto renamed = await( mSrcProvider->rename( mSource, mDestination, RenameOptions{ mOptions.overwrite == OverwritePolicy::Overwrite }, mToken ) ); @@ -628,11 +683,23 @@ class TransferEngine { return Err( std::move( renamed ).error() ); } - auto rootInfo = await( mSrcProvider->stat( mSource, StatOptions{ true }, mToken ) ); + auto rootInfo = await( mSrcProvider->stat( mSource, StatOptions{ false }, mToken ) ); if ( !rootInfo ) return Err( std::move( rootInfo ).error() ); + if ( rootInfo.value().isSymlink() ) { + // Providers cannot create links, so a copy takes the link's content. A move cannot: + // removing what was copied would empty the link's target, not move the link. + if ( mMove ) { + return Err( VFSError( VFSErrorCode::Unsupported, mSource, + "Symbolic links can only be moved within their file " + "system" ) ); + } + rootInfo = await( mSrcProvider->stat( mSource, StatOptions{ true }, mToken ) ); + if ( !rootInfo ) + return Err( std::move( rootInfo ).error() ); + } - if ( rootInfo.value().isDirectory() && sameProvider && + if ( rootInfo.value().isDirectory() && mSameProvider && mDestination.isSameOrChildOf( mSource ) ) { return Err( VFSError( VFSErrorCode::InvalidOperation, mDestination, "Cannot copy a directory into itself" ) ); @@ -736,12 +803,30 @@ class TransferEngine { advanceFiles( info ); return Ok( false ); } + if ( mSameProvider ) { + auto same = isSameFile( source, destination ); + if ( !same ) + return Err( std::move( same ).error() ); + if ( same.value() ) { + return Err( VFSError( VFSErrorCode::InvalidOperation, destination, + "Source and destination are the same file" ) ); + } + } } else if ( !existing.error().is( VFSErrorCode::NotFound ) ) { return Err( std::move( existing ).error() ); } - if ( auto copied = copyFile( info, source, destination ); !copied ) + if ( auto copied = copyFile( info, source, destination ); !copied ) { + // A destination can appear while the temporary is written. Skip still applies at + // publication time, and a move must keep that source. + if ( mOptions.overwrite == OverwritePolicy::Skip && + copied.error().is( VFSErrorCode::AlreadyExists ) ) { + ++mResult.filesSkipped; + advanceFiles( info ); + return Ok( false ); + } return Err( std::move( copied ).error() ); + } ++mResult.filesTransferred; advanceFiles( info, false ); @@ -805,6 +890,18 @@ class TransferEngine { true ); } + /** @return True when both URIs name the same file through symlinks or "..". Hard links are + * not detected; replaceFile() keeps their data safe. */ + VFSResult isSameFile( const URI& source, const URI& destination ) { + auto canonicalSource = await( mSrcProvider->canonicalize( source, mToken ) ); + if ( !canonicalSource ) + return Err( std::move( canonicalSource ).error() ); + auto canonicalDestination = await( mDstProvider->canonicalize( destination, mToken ) ); + if ( !canonicalDestination ) + return Err( std::move( canonicalDestination ).error() ); + return Ok( canonicalSource.value() == canonicalDestination.value() ); + } + VFSResult copyFile( const FileInfo& info, const URI& source, const URI& destination ) { mState->updateProgress( [&]( TransferProgress& p ) { @@ -813,80 +910,45 @@ class TransferEngine { }, true ); + auto target = resolveWriteTarget( *mDstProvider, destination, mToken ); + if ( !target ) + return Err( std::move( target ).error() ); + auto srcOpened = await( mSrcProvider->open( source, OpenFlags::Read, mToken ) ); if ( !srcOpened ) return Err( std::move( srcOpened ).error() ); FileHandle src( std::move( srcOpened ).value() ); - OpenOptions dstOptions( OpenFlags::Write | OpenFlags::Create | OpenFlags::Truncate ); - bool preserve = mOptions.preservePermissions && info.permissions() && - mDstProvider->capabilities( destination ).has( VFSCapability::Permissions ); - if ( preserve ) - dstOptions.createPermissions = info.permissions(); - auto dstOpened = await( mDstProvider->open( destination, dstOptions, mToken ) ); - if ( !dstOpened ) { - src.close(); - return Err( std::move( dstOpened ).error() ); - } - FileHandle dst( std::move( dstOpened ).value() ); - + ReplaceOptions replace; + if ( mOptions.preservePermissions && info.permissions() ) + replace.exactPermissions = info.permissions(); + replace.overwrite = mOptions.overwrite == OverwritePolicy::Overwrite; std::vector buffer( mOptions.chunkSize ); - Uint64 offset = 0; - VFSResult status = Ok(); - while ( true ) { - if ( auto c = checkCancelled( source ); !c ) { - status = std::move( c ); - break; - } - auto read = await( src.read( offset, buffer, mToken ) ); - if ( !read ) { - status = Err( std::move( read ).error() ); - break; - } - size_t got = read.value(); - if ( got == 0 ) - break; - if ( auto written = writeFully( dst, std::span( buffer.data(), got ), - mToken, offset ); - !written ) { - status = std::move( written ); - break; - } - offset += got; - mResult.bytesTransferred += got; - mState->updateProgress( [got]( TransferProgress& p ) { p.bytesDone += got; } ); - } - - auto srcClosed = await( src.close() ); - auto dstClosed = await( dst.close() ); - if ( status && !dstClosed ) - status = std::move( dstClosed ); - (void)srcClosed; - - if ( !status ) { - // Never leave a truncated file that looks complete. - await( mDstProvider->remove( destination, {}, CancellationToken() ) ); - return status; - } - if ( preserve ) { - // Creation permissions are filtered by the umask; apply the exact mode. - await( mDstProvider->setPermissions( destination, *info.permissions(), mToken ) ); - } - return Ok(); - } - - static VFSResult writeFully( FileHandle& handle, std::span data, - const CancellationToken& token, Uint64 baseOffset ) { - size_t done = 0; - while ( done < data.size() ) { - auto written = await( handle.write( baseOffset + done, data.subspan( done ), token ) ); - if ( !written ) - return Err( std::move( written ).error() ); - if ( written.value() == 0 ) - return Err( VFSError( VFSErrorCode::IO, handle.uri(), "Short write" ) ); - done += written.value(); - } - return Ok(); + auto copied = replaceFile( + *mDstProvider, target.value(), replace, mToken, + [&]( FileHandle& dst ) -> VFSResult { + Uint64 offset = 0; + while ( true ) { + if ( auto c = checkCancelled( source ); !c ) + return c; + auto read = await( src.read( offset, buffer, mToken ) ); + if ( !read ) + return Err( std::move( read ).error() ); + size_t got = read.value(); + if ( got == 0 ) + return Ok(); + if ( auto written = writeFully( + dst, std::span( buffer.data(), got ), mToken, offset ); + !written ) + return written; + offset += got; + mResult.bytesTransferred += got; + mState->updateProgress( [got]( TransferProgress& p ) { p.bytesDone += got; } ); + } + }, + []() -> VFSResult { return Ok(); } ); + await( src.close() ); + return copied; } VirtualFileSystem& mVfs; @@ -899,6 +961,8 @@ class TransferEngine { VFSProvider* mDstProvider{ nullptr }; TransferResult mResult; bool mMove; + /** Source and destination are served by the same provider and authority. */ + bool mSameProvider{ false }; }; namespace { @@ -922,12 +986,12 @@ Transfer startTransfer( VirtualFileSystem& vfs, std::shared_ptr pool Transfer VirtualFileSystem::copy( const URI& source, const URI& destination, const CopyOptions& options ) { - return startTransfer( *this, mOrchestrationPool, source, destination, options, false ); + return startTransfer( *this, mTransferPool, source, destination, options, false ); } Transfer VirtualFileSystem::move( const URI& source, const URI& destination, const CopyOptions& options ) { - return startTransfer( *this, mOrchestrationPool, source, destination, options, true ); + return startTransfer( *this, mTransferPool, source, destination, options, true ); } }} // namespace EE::VFS diff --git a/src/modules/vfs/src/eepp/vfs/watchers/localwatchbackend.cpp b/src/modules/vfs/src/eepp/vfs/watchers/localwatchbackend.cpp index e33d006d0..dd734afd1 100644 --- a/src/modules/vfs/src/eepp/vfs/watchers/localwatchbackend.cpp +++ b/src/modules/vfs/src/eepp/vfs/watchers/localwatchbackend.cpp @@ -331,19 +331,31 @@ class LocalWatchBackendImpl : public std::enable_shared_from_this impl, Uint64 token ) : - mImpl( std::move( impl ) ), mToken( token ) {} + LocalWatchRegistration( std::weak_ptr pool, + std::shared_ptr impl, Uint64 token ) : + mPool( std::move( pool ) ), mImpl( std::move( impl ) ), mToken( token ) {} - ~LocalWatchRegistration() override { mImpl->remove( mToken ); } + ~LocalWatchRegistration() override { + // Removing an efsw watch may wait on its thread: off the caller's (UI) thread. Events + // stop anyway, the handle closes its delivery right after this. + if ( auto pool = mPool.lock() ) { + pool->run( [impl = std::move( mImpl ), token = mToken] { impl->remove( token ); } ); + } else { + mImpl->remove( mToken ); + } + } private: + std::weak_ptr mPool; std::shared_ptr mImpl; Uint64 mToken; }; } // namespace detail -LocalWatchBackend::LocalWatchBackend( Time genericPollingInterval ) : +LocalWatchBackend::LocalWatchBackend( std::shared_ptr pool, + Time genericPollingInterval ) : + mPool( std::move( pool ) ), mImpl( std::make_shared( genericPollingInterval ) ) {} LocalWatchBackend::~LocalWatchBackend() = default; @@ -355,20 +367,25 @@ LocalWatchBackend::watch( const URI& root, const WatchOptions& options, auto path = LocalProvider::toNativePath( root ); if ( !path ) return readyError( std::move( path ).error() ); - if ( !System::FileSystem::isDirectory( path.value() ) ) { - return readyError( System::FileSystem::fileExists( path.value() ) - ? VFSErrorCode::NotDirectory - : VFSErrorCode::NotFound, - root ); - } - // efsw registration is synchronous and fast; once it returns the watch is active, so a - // listing started afterwards cannot miss changes. - auto added = mImpl->add( path.value(), options, sink ); - if ( !added ) - return readyError( std::move( added ).error() ); - sink->setStatus( added.value().second ? WatchStatus::Polling : WatchStatus::Active ); - return readyValue( - std::make_unique( mImpl, added.value().first ) ); + std::weak_ptr weakPool = mPool; + // Checking the directory and registering it touch the filesystem (a slow mount blocks): on + // the I/O pool. Once the task resolves the watch is active, so a listing started afterwards + // cannot miss changes. + return mPool->submit( [impl = mImpl, weakPool, root, options, sink = std::move( sink ), + nativeRoot = std::move( path ).value()]() -> VFSResult { + if ( !System::FileSystem::isDirectory( nativeRoot ) ) { + return Err( VFSError( System::FileSystem::fileExists( nativeRoot ) + ? VFSErrorCode::NotDirectory + : VFSErrorCode::NotFound, + root ) ); + } + auto added = impl->add( nativeRoot, options, sink ); + if ( !added ) + return Err( std::move( added ).error() ); + sink->setStatus( added.value().second ? WatchStatus::Polling : WatchStatus::Active ); + return Ok( Registration( + std::make_unique( weakPool, impl, added.value().first ) ) ); + } ); } }} // namespace EE::VFS diff --git a/src/tests/unit_tests/async_core_tests.cpp b/src/tests/unit_tests/async_core_tests.cpp index 220198833..a25dfe445 100644 --- a/src/tests/unit_tests/async_core_tests.cpp +++ b/src/tests/unit_tests/async_core_tests.cpp @@ -308,6 +308,22 @@ UTEST( Cancellation, callbackMayDestroyItsOwnRegistration ) { EXPECT_FALSE( static_cast( reg ) ); } +UTEST( Cancellation, callbackMayDestroyTheSource ) { + // The state must outlive the callbacks even when one of them destroys the source that is + // cancelling (use-after-free under AddressSanitizer otherwise). + auto source = std::make_unique(); + CancellationRegistration reg; + int calls = 0; + reg = source->token().onCancel( [&] { + ++calls; + reg.reset(); + source.reset(); + } ); + source->cancel(); + EXPECT_EQ( 1, calls ); + EXPECT_TRUE( source == nullptr ); +} + UTEST( Cancellation, sourceAndTokenLifetimeIndependence ) { CancellationToken token; { diff --git a/src/tests/unit_tests/sftp_tests.cpp b/src/tests/unit_tests/sftp_tests.cpp index b6d6dca42..bff44d66b 100644 --- a/src/tests/unit_tests/sftp_tests.cpp +++ b/src/tests/unit_tests/sftp_tests.cpp @@ -3,14 +3,21 @@ #if defined( EE_VFS_SFTP ) && EE_PLATFORM != EE_PLATFORM_WIN +#include "../../modules/vfs/src/eepp/vfs/sftp/sshagent.hpp" #include "vfs_test_utils.hpp" +#include #include +#include #include #include #include #include #include +#include #include +#if defined( __linux__ ) +#include +#endif using namespace vfstest; using EE::System::Process; @@ -211,7 +218,21 @@ class TestSSHServer { // preferred by default: connecting needs the known-type ordering. "Host edhost\n StrictHostKeyChecking yes\n UserKnownHostsFile " + mDir + "kh_ed25519\n" + common + - "Host * !secret !edkey !ednopub !edsecret\n IdentityFile " + mDir + "id_ecdsa\n" + + // Every host key of the server is @revoked: never accepted, not even as new. + "Host revoked\n StrictHostKeyChecking accept-new\n UserKnownHostsFile " + mDir + + "kh_revoked\n" + common + + // Only an algorithm libssh2 does not support: the connection must fail. + "Host badalgo\n HostKeyAlgorithms sk-ssh-ed25519@openssh.com\n" + " StrictHostKeyChecking accept-new\n UserKnownHostsFile " + + mDir + "kh_badalgo\n" + common + + // Only an ssh-agent can authenticate these: their identity file does not exist. + "Host agent\n IdentityAgent " + mDir + "agent.sock\n IdentitiesOnly no\n" + + " IdentityFile " + mDir + "no-such-key\n PreferredAuthentications publickey\n" + + common + "Host stalledagent\n IdentityAgent " + mDir + + "stalled-agent.sock\n IdentitiesOnly no\n IdentityFile " + mDir + + "no-such-key\n PreferredAuthentications publickey\n" + common + + "Host * !secret !edkey !ednopub !edsecret !agent !stalledagent\n IdentityFile " + + mDir + "id_ecdsa\n" + "Host *\n StrictHostKeyChecking accept-new\n UserKnownHostsFile " + mDir + "known_hosts\n" + common; FileSystem::fileWrite( mDir + "ssh_config", std::string_view( sshConfig ) ); @@ -222,6 +243,15 @@ class TestSSHServer { FileSystem::fileWrite( mDir + "kh_mismatch", std::string_view( "mismatch-alias " + fields[0] + " " + fields[1] + "\n" ) ); + std::string revoked; + for ( const char* key : { "host_ecdsa.pub", "host_ed25519.pub" } ) { + std::string pub; + FileSystem::fileGet( mDir + key, pub ); + auto keyFields = String::split( pub, ' ' ); + revoked += + "@revoked [127.0.0.1]:" + port + " " + keyFields[0] + " " + keyFields[1] + "\n"; + } + FileSystem::fileWrite( mDir + "kh_revoked", std::string_view( revoked ) ); std::string hostEd; FileSystem::fileGet( mDir + "host_ed25519.pub", hostEd ); fields = String::split( hostEd, ' ' ); @@ -474,6 +504,30 @@ UTEST( SFTP, hostKeyPolicies ) { } } +UTEST( SFTP, revokedHostKeysAreRejected ) { + REQUIRE_SSHD(); + SFTPFixture fx; + auto revoked = waitFor( fx.vfs.stat( server.uri( "revoked" ) ) ); + EXPECT_EQ( VFSErrorCode::HostKeyRejected, codeOf( revoked ) ); +} + +UTEST( SFTP, unsupportedAlgorithmRestrictionFails ) { + REQUIRE_SSHD(); + SFTPFixture fx; + auto connected = waitFor( fx.vfs.stat( server.uri( "badalgo" ) ) ); + EXPECT_FALSE( connected.hasValue() ); +} + +UTEST( SFTP, reuseKeepsTheStricterHostKeyPolicy ) { + REQUIRE_SSHD(); + // "testhost" accepts new keys, "strict" does not know the server: both reach the same server + // and user, but "strict" must not reuse the connection "testhost" made. + SFTPFixture fx; + REQUIRE_OK( waitFor( fx.vfs.stat( server.uri( "testhost" ) ) ) ); + auto strict = waitFor( fx.vfs.stat( server.uri( "strict" ) ) ); + EXPECT_EQ( VFSErrorCode::HostKeyRejected, codeOf( strict ) ); +} + UTEST( SFTP, authenticationPrompts ) { REQUIRE_SSHD(); { @@ -584,6 +638,365 @@ UTEST( SFTP, connectionReuseReconnectAndErrors ) { EXPECT_EQ( 1u, config.value().proxyJump.size() ); } +UTEST( SFTP, operationsOnAClosedConnectionFail ) { + REQUIRE_SSHD(); + SFTPFixture fx; + URI file = server.uri( "testhost", uniqueName( "closed" ) ); + REQUIRE_OK( waitFor( fx.vfs.writeAll( file, std::string_view( "data" ) ) ) ); + auto opened = waitFor( fx.vfs.open( file ) ); + REQUIRE_OK( opened ); + FileHandle handle( std::move( opened ).value() ); + fx.sftp->disconnectAll(); + // Give the I/O thread time to drop the closed connection. + std::this_thread::sleep_for( std::chrono::milliseconds( 200 ) ); + Uint8 buffer[8]; + auto read = handle.read( 0, buffer ); + ASSERT_EQ( TaskState::Ready, read.waitFor( std::chrono::seconds( 5 ) ) ); + EXPECT_EQ( VFSErrorCode::ConnectionLost, codeOf( std::move( read ).takeValue() ) ); + auto closed = handle.close(); + ASSERT_EQ( TaskState::Ready, closed.waitFor( std::chrono::seconds( 5 ) ) ); +} + +UTEST( SFTP, idleConnectionsCloseButNotWhileInUse ) { + REQUIRE_SSHD(); + SFTPProviderConfig config; + config.idleTimeout = System::Milliseconds( 300 ); + SFTPFixture fx( config ); + URI file = server.uri( "testhost", uniqueName( "idle" ) ); + REQUIRE_OK( waitFor( fx.vfs.writeAll( file, std::string_view( "data" ) ) ) ); + auto opened = waitFor( fx.vfs.open( file ) ); + REQUIRE_OK( opened ); + FileHandle handle( std::move( opened ).value() ); + // An open handle keeps the connection. + std::this_thread::sleep_for( std::chrono::milliseconds( 800 ) ); + EXPECT_EQ( 1u, fx.sftp->connectionCount() ); + Uint8 buffer[4]; + auto read = waitFor( handle.read( 0, buffer ) ); + REQUIRE_OK( read ); + EXPECT_EQ( 4u, read.value() ); + REQUIRE_OK( waitFor( handle.close() ) ); + // Unused, it closes after the idle timeout; the next operation reconnects. + bool closed = false; + for ( int i = 0; i < 50 && !closed; ++i ) { + std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); + closed = fx.sftp->connectionCount() == 0; + } + EXPECT_TRUE( closed ); + EXPECT_TRUE( waitFor( fx.vfs.stat( file ) ).hasValue() ); +} + +#if defined( __linux__ ) +UTEST( SFTP, idleConnectionReleasesItsSocket ) { + REQUIRE_SSHD(); + auto connectedSockets = [&server] { + int count = 0; + DIR* fds = ::opendir( "/proc/self/fd" ); + if ( !fds ) + return -1; + while ( auto* entry = ::readdir( fds ) ) { + if ( entry->d_name[0] == '.' ) + continue; + int fd = std::atoi( entry->d_name ); + sockaddr_in peer{}; + socklen_t length = sizeof( peer ); + if ( ::getpeername( fd, reinterpret_cast( &peer ), &length ) == 0 && + peer.sin_family == AF_INET && ntohs( peer.sin_port ) == server.port() ) + ++count; + } + ::closedir( fds ); + return count; + }; + int before = connectedSockets(); + ASSERT_GE( before, 0 ); + SFTPProviderConfig config; + config.idleTimeout = System::Milliseconds( 300 ); + SFTPFixture fx( config ); + REQUIRE_OK( waitFor( fx.vfs.stat( server.uri( "testhost" ) ) ) ); + EXPECT_EQ( before + 1, connectedSockets() ); + for ( int i = 0; i < 50 && connectedSockets() != before; ++i ) + std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); + EXPECT_EQ( before, connectedSockets() ); +} +#endif + +namespace { + +/** Never answers until released: a user who leaves a password dialog open. */ +class PendingAuthenticator : public SSHAuthenticationHandler { + public: + Task>> + request( const SSHAuthenticationRequest& ) override { + std::lock_guard lock( mMutex ); + if ( mReleased ) + return makeReadyTask( std::optional>() ); + mPending.emplace_back(); + return mPending.back().task(); + } + + size_t pending() { + std::lock_guard lock( mMutex ); + return mPending.size(); + } + + void release() { + std::vector>>> pending; + { + std::lock_guard lock( mMutex ); + mReleased = true; + pending.swap( mPending ); + } + // Declined: later prompts (other methods, attempts) are declined right away. + for ( auto& source : pending ) + source.resolve( std::nullopt ); + } + + private: + std::mutex mMutex; + std::vector>>> mPending; + bool mReleased{ false }; +}; + +} // namespace + +UTEST( SFTP, pendingPromptsDoNotBlockOtherConnections ) { + REQUIRE_SSHD(); + auto prompts = std::make_shared(); + SFTPProviderConfig config; + config.authenticationHandler = prompts; + SFTPFixture fx( config ); + // Two connections wait for the user (as many as connection helper threads). + auto pw = fx.vfs.stat( server.uri( "pw" ) ); + auto secret = fx.vfs.stat( server.uri( "secret" ) ); + for ( int i = 0; i < 100 && prompts->pending() < 2; ++i ) + std::this_thread::sleep_for( std::chrono::milliseconds( 50 ) ); + EXPECT_EQ( 2u, prompts->pending() ); + auto other = fx.vfs.stat( server.uri( "testhost" ) ); + ASSERT_EQ( TaskState::Ready, other.waitFor( std::chrono::seconds( 10 ) ) ); + EXPECT_TRUE( std::move( other ).takeValue().hasValue() ); + prompts->release(); + EXPECT_EQ( TaskState::Ready, pw.waitFor( std::chrono::seconds( 10 ) ) ); + EXPECT_EQ( TaskState::Ready, secret.waitFor( std::chrono::seconds( 10 ) ) ); +} + +UTEST( SFTP, disconnectCompletesOperationsWaitingForAPrompt ) { + REQUIRE_SSHD(); + auto auth = std::make_shared(); + SFTPProviderConfig config; + config.authenticationHandler = auth; + SFTPFixture fx( config ); + auto pending = fx.vfs.stat( server.uri( "pw" ) ); + for ( int i = 0; i < 200 && auth->pending() == 0; ++i ) + std::this_thread::sleep_for( std::chrono::milliseconds( 10 ) ); + EXPECT_GT( auth->pending(), 0u ); + fx.sftp->disconnectAll(); + auto state = pending.waitFor( std::chrono::seconds( 2 ) ); + auth->release(); + ASSERT_EQ( TaskState::Ready, state ); + EXPECT_EQ( VFSErrorCode::ConnectionLost, codeOf( std::move( pending ).takeValue() ) ); +} + +namespace { + +/** An ssh-agent that lists one key and answers signing requests only once released (a hardware + * key waiting for a touch). */ +class StalledAgent { + public: + StalledAgent( const std::string& path, std::string keyBlob ) : mKey( std::move( keyBlob ) ) { + ::unlink( path.c_str() ); + mListener = ::socket( AF_UNIX, SOCK_STREAM, 0 ); + sockaddr_un address{}; + address.sun_family = AF_UNIX; + std::strncpy( address.sun_path, path.c_str(), sizeof( address.sun_path ) - 1 ); + ::bind( mListener, reinterpret_cast( &address ), sizeof( address ) ); + ::listen( mListener, 4 ); + mThread = std::thread( [this] { serve(); } ); + } + + ~StalledAgent() { + release(); + ::shutdown( mListener, SHUT_RDWR ); + ::close( mListener ); + { + std::lock_guard lock( mMutex ); + if ( mClient >= 0 ) + ::shutdown( mClient, SHUT_RDWR ); + } + mThread.join(); + } + + bool signRequested() { + std::lock_guard lock( mMutex ); + return mSignRequested; + } + + void release() { + std::lock_guard lock( mMutex ); + mReleased = true; + mChanged.notify_all(); + } + + private: + static void putU32( std::string& out, Uint32 value ) { + for ( int shift = 24; shift >= 0; shift -= 8 ) + out += static_cast( ( value >> shift ) & 0xFF ); + } + + static bool readAll( int fd, char* data, size_t size ) { + while ( size > 0 ) { + ssize_t rc = ::recv( fd, data, size, 0 ); + if ( rc <= 0 ) + return false; + data += rc; + size -= static_cast( rc ); + } + return true; + } + + void serve() { + while ( true ) { + int client = ::accept( mListener, nullptr, nullptr ); + if ( client < 0 ) + return; +#ifdef SO_NOSIGPIPE + int noSigPipe = 1; + ::setsockopt( client, SOL_SOCKET, SO_NOSIGPIPE, &noSigPipe, sizeof( noSigPipe ) ); +#endif + { + std::lock_guard lock( mMutex ); + mClient = client; + } + while ( true ) { + unsigned char header[4]; + if ( !readAll( client, reinterpret_cast( header ), 4 ) ) + break; + Uint32 length = + ( header[0] << 24 ) | ( header[1] << 16 ) | ( header[2] << 8 ) | header[3]; + std::string request( length, '\0' ); + if ( !readAll( client, request.data(), length ) ) + break; + std::string body; + if ( request[0] == 11 ) { // Request identities. + body += static_cast( 12 ); + putU32( body, 1 ); + putU32( body, static_cast( mKey.size() ) ); + body += mKey; + putU32( body, 0 ); + } else { + std::unique_lock lock( mMutex ); + mSignRequested = true; + mChanged.wait( lock, [this] { return mReleased; } ); + body += static_cast( 5 ); // Failure. + } + std::string reply; + putU32( reply, static_cast( body.size() ) ); + reply += body; +#ifdef MSG_NOSIGNAL + ::send( client, reply.data(), reply.size(), MSG_NOSIGNAL ); +#else + ::send( client, reply.data(), reply.size(), 0 ); +#endif + } + { + std::lock_guard lock( mMutex ); + mClient = -1; + } + ::close( client ); + } + } + + std::string mKey; + int mListener{ -1 }; + int mClient{ -1 }; + std::mutex mMutex; + std::condition_variable mChanged; + bool mSignRequested{ false }; + bool mReleased{ false }; + std::thread mThread; +}; + +std::string publicKeyBlob( const std::string& pubFile ) { + std::string text, blob; + FileSystem::fileGet( pubFile, text ); + auto fields = String::split( text, ' ' ); + if ( fields.size() >= 2 ) + System::Base64::decode( fields[1], blob ); + return blob; +} + +} // namespace + +UTEST( SFTP, agentAuthentication ) { + REQUIRE_SSHD(); + std::string agentBinary = Sys::which( "ssh-agent" ); + std::string addBinary = Sys::which( "ssh-add" ); + if ( agentBinary.empty() || addBinary.empty() ) { + printf( " ssh-agent not found, skipped\n" ); + return; + } + std::string socket = server.dir() + "agent.sock"; + FileSystem::fileRemove( socket ); + Process agent; + ASSERT_TRUE( agent.create( agentBinary, { "-D", "-a", socket } ) ); + for ( int i = 0; i < 100 && !FileSystem::fileExists( socket ); ++i ) + std::this_thread::sleep_for( std::chrono::milliseconds( 20 ) ); + Process add; + ASSERT_TRUE( add.create( "/bin/sh", { "-c", "SSH_AUTH_SOCK='" + socket + "' '" + addBinary + + "' '" + server.dir() + "id_ecdsa'" } ) ); + int code = -1; + add.join( &code ); + ASSERT_EQ( 0, code ); + { + // The identity file does not exist: only the agent's key can authenticate. + SFTPFixture fx; + REQUIRE_OK( waitFor( fx.vfs.stat( server.uri( "agent" ) ) ) ); + } + agent.kill(); +} + +UTEST( SFTP, stalledAgentDoesNotBlockOtherConnections ) { + REQUIRE_SSHD(); + StalledAgent agent( server.dir() + "stalled-agent.sock", + publicKeyBlob( server.dir() + "id_ecdsa.pub" ) ); + SFTPFixture fx; + auto stalled = fx.vfs.stat( server.uri( "stalledagent" ) ); + for ( int i = 0; i < 200 && !agent.signRequested(); ++i ) + std::this_thread::sleep_for( std::chrono::milliseconds( 25 ) ); + EXPECT_TRUE( agent.signRequested() ); + // The agent waits for a touch: every other session keeps working meanwhile. + auto other = fx.vfs.stat( server.uri( "testhost" ) ); + ASSERT_EQ( TaskState::Ready, other.waitFor( std::chrono::seconds( 10 ) ) ); + EXPECT_TRUE( std::move( other ).takeValue().hasValue() ); + EXPECT_TRUE( stalled.isPending() ); + agent.release(); + ASSERT_EQ( TaskState::Ready, stalled.waitFor( std::chrono::seconds( 10 ) ) ); + EXPECT_EQ( VFSErrorCode::AuthenticationFailed, codeOf( std::move( stalled ).takeValue() ) ); +} + +UTEST( SFTP, cancelledAgentRequestDoesNotWaitForTheAgent ) { + REQUIRE_SSHD(); + std::string path = server.dir() + "cancel-agent.sock"; + StalledAgent agent( path, publicKeyBlob( server.dir() + "id_ecdsa.pub" ) ); + sftp::SSHAgent client( path ); + auto keys = client.listIdentities(); + REQUIRE_OK( keys ); + ASSERT_FALSE( keys.value().empty() ); + TaskSource> source; + auto task = source.task(); + std::thread worker( [&client, &keys, source = std::move( source )]() mutable { + source.resolve( client.sign( keys.value()[0].blob, "payload", "ecdsa-sha2-nistp256" ) ); + } ); + for ( int i = 0; i < 200 && !agent.signRequested(); ++i ) + std::this_thread::sleep_for( std::chrono::milliseconds( 10 ) ); + EXPECT_TRUE( agent.signRequested() ); + client.cancel(); + auto state = task.waitFor( std::chrono::seconds( 2 ) ); + // Also release on failure so a regression cannot hang the suite during thread destruction. + agent.release(); + worker.join(); + ASSERT_EQ( TaskState::Ready, state ); + EXPECT_FALSE( std::move( task ).takeValue().hasValue() ); +} + UTEST( SFTP, remoteWatchThroughAgent ) { REQUIRE_SSHD(); std::string agent = Sys::getProcessPath(); @@ -652,4 +1065,44 @@ UTEST( SFTP, remoteWatchFallsBackToPolling ) { loop.runUntil( [&] { return events.has( WatchEventType::Added, "polled.txt" ); } ) ); } +UTEST( SFTP, closingTheLastWatchAllowsIdleCleanup ) { + REQUIRE_SSHD(); + std::string agent = Sys::getProcessPath(); + FileSystem::dirAddSlashAtEnd( agent ); + agent += "../efsw-agent"; + if ( !FileSystem::fileExists( agent ) ) { + printf( " efsw-agent not built, remote watch test skipped\n" ); + return; + } + SFTPProviderConfig config; + config.watchAgentCommand = agent + " --stdio"; + config.idleTimeout = System::Milliseconds( 300 ); + SFTPFixture fx( config ); + auto opened = waitFor( fx.vfs.watch( server.uri( "testhost" ) ) ); + REQUIRE_OK( opened ); + WatchHandle watch( std::move( opened ).value() ); + std::this_thread::sleep_for( std::chrono::milliseconds( 700 ) ); + EXPECT_EQ( 1u, fx.sftp->connectionCount() ); + watch.reset(); + for ( int i = 0; i < 50 && fx.sftp->connectionCount() != 0; ++i ) + std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); + EXPECT_EQ( 0u, fx.sftp->connectionCount() ); +} + +UTEST( SFTP, silentWatchAgentFallsBackToPolling ) { + REQUIRE_SSHD(); + // The command runs but never answers the handshake: the watch (and the listing that waits + // for it) must not stay pending. + SFTPProviderConfig config; + config.watchAgentCommand = "sleep 60"; + SFTPFixture fx( config ); + std::string name = uniqueName( "silentwatch" ); + FileSystem::makeDir( server.remoteRoot() + "/" + name ); + auto task = fx.vfs.watch( server.uri( "testhost", name ) ); + ASSERT_EQ( TaskState::Ready, task.waitFor( std::chrono::seconds( 20 ) ) ); + auto opened = std::move( task ).takeValue(); + REQUIRE_OK( opened ); + EXPECT_EQ( WatchStatus::Polling, opened.value().status() ); +} + #endif diff --git a/src/tests/unit_tests/sshconfig_tests.cpp b/src/tests/unit_tests/sshconfig_tests.cpp index 810705384..10e3be9f5 100644 --- a/src/tests/unit_tests/sshconfig_tests.cpp +++ b/src/tests/unit_tests/sshconfig_tests.cpp @@ -485,6 +485,25 @@ UTEST( SSHConfig, connectionKeyIgnoresPathsAndDistinguishesProfiles ) { EXPECT_TRUE( a.connectionKey() != c.connectionKey() ); } +UTEST( SSHConfig, connectionKeyIncludesTheSecurityPolicy ) { + // A session verified under a permissive policy must not serve a stricter configuration. + FakeSystem fake; + fake.files[UserConfig] = "Host makyo\n User black\n"; + auto base = fake.resolve( "makyo" ); + auto stricter = base; + stricter.strictHostKeyChecking = SSHStrictHostKeyChecking::Yes; + EXPECT_TRUE( base.connectionKey() != stricter.connectionKey() ); + auto otherKnownHosts = base; + otherKnownHosts.userKnownHostsFiles = { "/different/known_hosts" }; + EXPECT_TRUE( base.connectionKey() != otherKnownHosts.connectionKey() ); + auto restricted = base; + restricted.hostKeyAlgorithms = std::string( "ssh-ed25519" ); + EXPECT_TRUE( base.connectionKey() != restricted.connectionKey() ); + auto noPassword = base; + noPassword.passwordAuthentication = false; + EXPECT_TRUE( base.connectionKey() != noPassword.connectionKey() ); +} + #if EE_PLATFORM != EE_PLATFORM_WIN namespace { diff --git a/src/tests/unit_tests/stringsoperations_tests.cpp b/src/tests/unit_tests/stringsoperations_tests.cpp index ba2bbc24f..6f284927e 100644 --- a/src/tests/unit_tests/stringsoperations_tests.cpp +++ b/src/tests/unit_tests/stringsoperations_tests.cpp @@ -66,6 +66,16 @@ class ScopedTestDirectory { } // namespace +UTEST( String, icompare ) { + EXPECT_EQ( 0, String::icompare( "Readme.MD", "readme.md" ) ); + EXPECT_TRUE( String::icompare( "alpha", "Beta" ) < 0 ); + EXPECT_TRUE( String::icompare( "Beta", "alpha" ) > 0 ); + EXPECT_TRUE( String::icompare( "abc", "ABCD" ) < 0 ); + EXPECT_TRUE( String::icompare( "", "a" ) < 0 ); + // Bytes above ASCII compare unsigned, as std::string::compare does. + EXPECT_TRUE( String::icompare( "z", "\xC3\xA1" ) < 0 ); +} + UTEST( String, countLines ) { EXPECT_EQ( static_cast( 0 ), String::countLines( "" ) ); EXPECT_EQ( static_cast( 1 ), String::countLines( "A" ) ); diff --git a/src/tests/unit_tests/systempath_tests.cpp b/src/tests/unit_tests/systempath_tests.cpp index ce12321fd..9c4b5645e 100644 --- a/src/tests/unit_tests/systempath_tests.cpp +++ b/src/tests/unit_tests/systempath_tests.cpp @@ -1,5 +1,6 @@ #include "utest.hpp" +#include #include #include @@ -46,3 +47,32 @@ UTEST( SystemPath, workingDirectoryRoundTripsUnicode ) { EXPECT_STDSTREQ( FileSystem::getRealPath( temp.path ), FileSystem::getCurrentWorkingDirectory() ); } + +#if EE_PLATFORM == EE_PLATFORM_LINUX || EE_PLATFORM == EE_PLATFORM_BSD + +UTEST( SystemPath, cachePathFollowsXDG ) { + auto restore = []( const char* name, const char* value ) { + if ( value ) + ::setenv( name, value, 1 ); + else + ::unsetenv( name ); + }; + const char* xdg = std::getenv( "XDG_CACHE_HOME" ); + std::string savedXdg( xdg ? xdg : "" ); + const char* home = std::getenv( "HOME" ); + std::string savedHome( home ? home : "" ); + + ::setenv( "XDG_CACHE_HOME", "/xdg/cache", 1 ); + ::setenv( "HOME", "/home/someone", 1 ); + EXPECT_STDSTREQ( "/xdg/cache/app", Sys::getCachePath( "app" ) ); + // The specification ignores relative and empty values. + ::setenv( "XDG_CACHE_HOME", "relative", 1 ); + EXPECT_STDSTREQ( "/home/someone/.cache/app", Sys::getCachePath( "app" ) ); + ::unsetenv( "XDG_CACHE_HOME" ); + EXPECT_STDSTREQ( "/home/someone/.cache/app", Sys::getCachePath( "app" ) ); + + restore( "XDG_CACHE_HOME", xdg ? savedXdg.c_str() : nullptr ); + restore( "HOME", home ? savedHome.c_str() : nullptr ); +} + +#endif diff --git a/src/tests/unit_tests/vfs_model_tests.cpp b/src/tests/unit_tests/vfs_model_tests.cpp index 2d66dd6d5..d6896d293 100644 --- a/src/tests/unit_tests/vfs_model_tests.cpp +++ b/src/tests/unit_tests/vfs_model_tests.cpp @@ -196,6 +196,44 @@ UTEST( VFSDirectoryModel, staysLiveThroughWatchEvents ) { EXPECT_EQ( 3u, model->rowCount() ); } +UTEST( VFSDirectoryModel, watchEventsKeepRowsInStepWithHiddenEntries ) { + // Rows are updated in place by watch events: hidden and filtered entries in between must + // keep the visible rows pointing at the right entries. + TempDir tmp; + VirtualFileSystem vfs; + for ( const char* name : { ".a-hidden", "b-keep.txt", "c-other.txt", ".d-hidden" } ) + FileSystem::fileWrite( tmp.path + name, std::string_view( "1" ) ); + TestLoop loop; + VFSDirectoryModel::Config config; + config.vfs = &vfs; + auto model = VFSDirectoryModel::New( loop, config ); + model->setURI( tmp.uri() ); + EXPECT_TRUE( loop.runUntil( + [&] { return model->loadState() == VFSDirectoryModel::LoadState::Ready; } ) ); + model->setNameFilter( "keep", VFSDirectoryModel::NameFilterMode::PlainText ); + EXPECT_EQ( 1u, model->rowCount() ); + + for ( const char* name : { ".0-hidden", "a-keep.txt", "bb-other.txt", "z-KEEP.txt" } ) + FileSystem::fileWrite( tmp.path + name, std::string_view( "1" ) ); + FileSystem::fileRemove( tmp.path + ".a-hidden" ); + EXPECT_TRUE( loop.runUntil( [&] { + return model->rowOf( "a-keep.txt" ) && model->rowOf( "z-KEEP.txt" ) && + model->rowCount() == 3; + } ) ); + FileSystem::fileRemove( tmp.path + "b-keep.txt" ); + EXPECT_TRUE( loop.runUntil( [&] { return !model->rowOf( "b-keep.txt" ); } ) ); + auto names = modelNames( *model ); + ASSERT_EQ( 2u, names.size() ); + EXPECT_STDSTREQ( "a-keep.txt", names[0] ); + EXPECT_STDSTREQ( "z-KEEP.txt", names[1] ); + // The same rows as a full rebuild. + model->setNameFilter( "" ); + model->setNameFilter( "keep", VFSDirectoryModel::NameFilterMode::PlainText ); + EXPECT_EQ( 2u, model->rowCount() ); + EXPECT_STDSTREQ( "a-keep.txt", model->entry( 0 ).name ); + EXPECT_STDSTREQ( "z-KEEP.txt", model->entry( 1 ).name ); +} + UTEST( MainThreadExecutor, runsDeferredOnSceneUpdate ) { UIApplication app( WindowSettings( 320, 240, "eepp - MainThreadExecutor Test", WindowStyle::Default, diff --git a/src/tests/unit_tests/vfs_tests.cpp b/src/tests/unit_tests/vfs_tests.cpp index f321fd729..5eab7b8da 100644 --- a/src/tests/unit_tests/vfs_tests.cpp +++ b/src/tests/unit_tests/vfs_tests.cpp @@ -2,6 +2,7 @@ #include "vfs_test_utils.hpp" #include #include +#include #include #include #include @@ -13,7 +14,8 @@ #include #if EE_PLATFORM != EE_PLATFORM_WIN -#include // symlink +#include // umask +#include // symlink, link #endif using namespace EE; @@ -645,3 +647,311 @@ UTEST( VFSIOStream, synchronousAdapterReadsSeeksAndWrites ) { EXPECT_FALSE( missing.isOpen() ); EXPECT_TRUE( missing.error().has_value() ); } + +// --------------------------------------------------------------------------------------------- +// Data safety: copies, moves and saves never lose existing data +// --------------------------------------------------------------------------------------------- + +namespace { + +/** Every read fails: a copy from it fails after the destination was opened. */ +class FailingReadHandle : public FileHandleImpl { + public: + VFSTask read( Uint64, std::span, CancellationToken ) override { + return readyError( VFSErrorCode::IO, URI(), "injected read failure" ); + } + + VFSTask write( Uint64, std::span, CancellationToken ) override { + return readyError( VFSErrorCode::IO, URI() ); + } + + VFSTask truncate( Uint64 ) override { return readyValue(); } + + VFSTask flush() override { return readyValue(); } + + VFSTask stat() override { return readyError( VFSErrorCode::IO, URI() ); } + + VFSTask close() override { return readyValue(); } + + ProviderCapabilities capabilities() const override { return {}; } +}; + +class FailingReadProvider : public MemoryProvider { + public: + FailingReadProvider() : MemoryProvider( "fault" ) {} + + VFSTask open( const URI& uri, const OpenOptions&, CancellationToken ) override { + return readyValue( uri, std::make_shared() ); + } +}; + +class NonAtomicMemoryProvider : public MemoryProvider { + public: + ProviderCapabilities capabilities( const URI& uri ) const override { + auto caps = MemoryProvider::capabilities( uri ); + return caps.set( VFSCapability::AtomicReplace, false ); + } +}; + +/** Publishes another application's destination after the transfer's existence check. */ +class RacingReadHandle : public FailingReadHandle { + public: + explicit RacingReadHandle( std::shared_ptr destination ) : + mDestination( std::move( destination ) ) {} + + VFSTask read( Uint64 offset, std::span data, CancellationToken ) override { + if ( offset != 0 ) + return readyValue( 0 ); + mDestination->putFile( URI( "mem://t/destination" ), "concurrently created data" ); + data[0] = 'x'; + return readyValue( 1 ); + } + + private: + std::shared_ptr mDestination; +}; + +class RacingReadProvider : public MemoryProvider { + public: + explicit RacingReadProvider( std::shared_ptr destination ) : + MemoryProvider( "race" ), mDestination( std::move( destination ) ) {} + + VFSTask open( const URI& uri, const OpenOptions&, CancellationToken ) override { + return readyValue( uri, std::make_shared( mDestination ) ); + } + + private: + std::shared_ptr mDestination; +}; + +} // namespace + +UTEST( VFSSafety, unsafeReplacementIsRefusedWithoutAtomicReplace ) { + VirtualFileSystem vfs; + auto destination = std::make_shared(); + auto source = std::make_shared(); + vfs.registerProvider( destination ); + vfs.registerProvider( source ); + source->putFile( URI( "fault://t/source" ), "new content" ); + destination->putFile( URI( "mem://t/destination" ), "valuable data" ); + CopyOptions options; + options.overwrite = OverwritePolicy::Overwrite; + auto result = waitFor( + vfs.copy( URI( "fault://t/source" ), URI( "mem://t/destination" ), options ).completion() ); + ASSERT_FALSE( result.hasValue() ); + EXPECT_EQ( VFSErrorCode::Unsupported, result.error().code ); + EXPECT_STDSTREQ( "valuable data", *destination->fileContent( URI( "mem://t/destination" ) ) ); + // A new file still needs only a no-replace rename, not atomic replacement support. + auto memory = std::make_shared( "source" ); + memory->putFile( URI( "source://t/file" ), "new content" ); + vfs.registerProvider( memory ); + EXPECT_TRUE( + waitFor( vfs.copy( URI( "source://t/file" ), URI( "mem://t/new" ) ).completion() ) ); + EXPECT_STDSTREQ( "new content", *destination->fileContent( URI( "mem://t/new" ) ) ); +} + +UTEST( VFSSafety, concurrentDestinationRespectsFailAndSkip ) { + for ( auto policy : { OverwritePolicy::Fail, OverwritePolicy::Skip } ) { + VirtualFileSystem vfs; + auto destination = std::make_shared(); + auto source = std::make_shared( destination ); + source->putFile( URI( "race://t/source" ), "x" ); + vfs.registerProvider( destination ); + vfs.registerProvider( source ); + CopyOptions options; + options.overwrite = policy; + auto result = + waitFor( vfs.move( URI( "race://t/source" ), URI( "mem://t/destination" ), options ) + .completion() ); + if ( policy == OverwritePolicy::Fail ) { + ASSERT_FALSE( result.hasValue() ); + EXPECT_EQ( VFSErrorCode::AlreadyExists, result.error().code ); + } else { + ASSERT_TRUE( result.hasValue() ); + EXPECT_EQ( 1u, result.value().filesSkipped ); + EXPECT_EQ( 0u, result.value().sourcesRemoved ); + } + EXPECT_STDSTREQ( "concurrently created data", + *destination->fileContent( URI( "mem://t/destination" ) ) ); + EXPECT_TRUE( source->fileContent( URI( "race://t/source" ) ).has_value() ); + } +} + +UTEST( VFSSafety, failedOverwriteKeepsTheDestination ) { + VirtualFileSystem vfs; + auto mem = std::make_shared(); + auto fault = std::make_shared(); + vfs.registerProvider( mem ); + vfs.registerProvider( fault ); + fault->putFile( URI( "fault://t/source" ), "new content" ); + mem->putFile( URI( "mem://t/file" ), "valuable data" ); + CopyOptions options; + options.overwrite = OverwritePolicy::Overwrite; + auto result = waitFor( + vfs.copy( URI( "fault://t/source" ), URI( "mem://t/file" ), options ).completion() ); + EXPECT_FALSE( result.hasValue() ); + EXPECT_STDSTREQ( "valuable data", readString( vfs, URI( "mem://t/file" ) ) ); + // No temporary is left behind. + auto listed = waitFor( vfs.listAll( URI( "mem://t/" ) ) ); + ASSERT_TRUE( listed.hasValue() ); + EXPECT_EQ( 1u, listed.value().size() ); +} + +UTEST( VFSSafety, writeAllCreateParentsWithOneOrchestrationThread ) { + // writeAll() runs on the orchestration pool: creating the parents must not wait on another + // job of that pool. + VirtualFileSystem::Config config; + config.registerDefaultProviders = false; + config.orchestrationThreads = 1; + VirtualFileSystem vfs( config ); + vfs.registerProvider( std::make_shared() ); + WriteOptions options; + options.createParents = true; + auto task = vfs.writeAll( URI( "mem://t/new/dir/file" ), std::string_view( "data" ), options ); + ASSERT_EQ( TaskState::Ready, task.waitFor( std::chrono::seconds( 5 ) ) ); + EXPECT_TRUE( std::move( task ).takeValue().hasValue() ); + EXPECT_STDSTREQ( "data", readString( vfs, URI( "mem://t/new/dir/file" ) ) ); +} + +#if EE_PLATFORM != EE_PLATFORM_WIN + +UTEST( VFSSafety, copyOntoAHardLinkKeepsTheSource ) { + TempDir tmp; + VirtualFileSystem vfs; + URI source = tmp.uri().withAppendedSegment( "file" ); + URI alias = tmp.uri().withAppendedSegment( "hardlink" ); + ASSERT_TRUE( waitFor( vfs.writeAll( source, std::string_view( "valuable data" ) ) ) ); + ASSERT_EQ( 0, ::link( source.getFSPath().c_str(), alias.getFSPath().c_str() ) ); + CopyOptions options; + options.overwrite = OverwritePolicy::Overwrite; + waitFor( vfs.copy( source, alias, options ).completion() ); + EXPECT_STDSTREQ( "valuable data", readString( vfs, source ) ); + EXPECT_STDSTREQ( "valuable data", readString( vfs, alias ) ); +} + +UTEST( VFSSafety, copyOntoASymlinkToTheSourceFails ) { + TempDir tmp; + VirtualFileSystem vfs; + URI source = tmp.uri().withAppendedSegment( "file" ); + URI alias = tmp.uri().withAppendedSegment( "link" ); + ASSERT_TRUE( waitFor( vfs.writeAll( source, std::string_view( "valuable data" ) ) ) ); + ASSERT_EQ( 0, ::symlink( source.getFSPath().c_str(), alias.getFSPath().c_str() ) ); + CopyOptions options; + options.overwrite = OverwritePolicy::Overwrite; + auto result = waitFor( vfs.copy( source, alias, options ).completion() ); + EXPECT_EQ( VFSErrorCode::InvalidOperation, codeOf( result ) ); + EXPECT_STDSTREQ( "valuable data", readString( vfs, source ) ); +} + +UTEST( VFSSafety, movingADirectorySymlinkAcrossProvidersKeepsItsTarget ) { + TempDir tmp; + VirtualFileSystem vfs; + vfs.registerProvider( std::make_shared() ); + URI real = tmp.uri().withAppendedSegment( "real" ); + URI link = tmp.uri().withAppendedSegment( "link" ); + ASSERT_TRUE( waitFor( vfs.makeDirs( real ) ) ); + ASSERT_TRUE( + waitFor( vfs.writeAll( real.withAppendedSegment( "file" ), std::string_view( "data" ) ) ) ); + ASSERT_EQ( 0, ::symlink( real.getFSPath().c_str(), link.getFSPath().c_str() ) ); + auto result = waitFor( vfs.move( link, URI( "mem://t/moved" ) ).completion() ); + EXPECT_EQ( VFSErrorCode::Unsupported, codeOf( result ) ); + EXPECT_STDSTREQ( "data", readString( vfs, real.withAppendedSegment( "file" ) ) ); + EXPECT_TRUE( waitFor( vfs.stat( link, StatOptions{ false } ) ).hasValue() ); +} + +UTEST( VFSSafety, atomicSaveKeepsTheExactMode ) { + TempDir tmp; + VirtualFileSystem vfs; + URI file = tmp.uri().withAppendedSegment( "file" ); + ASSERT_TRUE( waitFor( vfs.writeAll( file, std::string_view( "v1" ) ) ) ); + ASSERT_TRUE( waitFor( vfs.setPermissions( file, 0777 ) ) ); + mode_t mask = ::umask( 0077 ); + auto saved = waitFor( vfs.writeAll( file, std::string_view( "v2" ) ) ); + ::umask( mask ); + ASSERT_TRUE( saved.hasValue() ); + EXPECT_EQ( 0777u, *saved.value().permissions() ); + EXPECT_STDSTREQ( "v2", readString( vfs, file ) ); +} + +#endif + +UTEST( VFSWatch, localRegistrationRunsOnThePool ) { + // Registering touches the filesystem (slow mounts block): it must not run on the caller's + // (UI) thread. With the backend's pool busy, watch() returns a pending task. + TempDir tmp; + auto pool = ThreadPool::createShared( 1 ); + std::mutex gate; + std::unique_lock hold( gate ); + pool->run( [&gate] { std::lock_guard wait( gate ); } ); + auto backend = std::make_shared( pool ); + WatchManager manager; + manager.registerBackend( "file", backend ); + auto task = manager.watch( tmp.uri() ); + EXPECT_EQ( TaskState::Pending, task.state() ); + hold.unlock(); + ASSERT_EQ( TaskState::Ready, task.waitFor( std::chrono::seconds( 5 ) ) ); + auto handle = std::move( task ).takeValue(); + ASSERT_TRUE( handle.hasValue() ); + EXPECT_EQ( WatchStatus::Active, handle.value().status() ); + handle.value().reset(); +} + +namespace { + +/** open() blocks until released: a transfer reading from it keeps its thread busy. */ +class GatedProvider : public MemoryProvider { + public: + GatedProvider() : MemoryProvider( "gated" ) {} + + VFSTask open( const URI& uri, const OpenOptions& options, + CancellationToken token ) override { + { + std::unique_lock lock( mMutex ); + mOpened = true; + mChanged.notify_all(); + mChanged.wait( lock, [this] { return mReleased; } ); + } + return MemoryProvider::open( uri, options, std::move( token ) ); + } + + void waitOpened() { + std::unique_lock lock( mMutex ); + mChanged.wait_for( lock, std::chrono::seconds( 5 ), [this] { return mOpened; } ); + } + + void release() { + std::lock_guard lock( mMutex ); + mReleased = true; + mChanged.notify_all(); + } + + private: + std::mutex mMutex; + std::condition_variable mChanged; + bool mOpened{ false }; + bool mReleased{ false }; +}; + +} // namespace + +UTEST( VFSSafety, transfersDoNotDelayOtherOperations ) { + // Copies run on their own pool: with every transfer thread busy, saves still complete. + VirtualFileSystem::Config config; + config.registerDefaultProviders = false; + config.orchestrationThreads = 1; + config.transferThreads = 1; + VirtualFileSystem vfs( config ); + auto mem = std::make_shared(); + auto gated = std::make_shared(); + vfs.registerProvider( mem ); + vfs.registerProvider( gated ); + gated->putFile( URI( "gated://t/big" ), std::string( 1024, 'x' ) ); + Transfer transfer = vfs.copy( URI( "gated://t/big" ), URI( "mem://t/copy" ) ); + gated->waitOpened(); + auto saved = vfs.writeAll( URI( "mem://t/other" ), std::string_view( "data" ) ); + EXPECT_EQ( TaskState::Ready, saved.waitFor( std::chrono::seconds( 5 ) ) ); + gated->release(); + auto done = transfer.completion(); + ASSERT_EQ( TaskState::Ready, done.waitFor( std::chrono::seconds( 5 ) ) ); + EXPECT_TRUE( std::move( done ).takeValue().hasValue() ); +} diff --git a/src/tools/eefm/app.cpp b/src/tools/eefm/app.cpp index ddb5d5161..43d32d6ac 100644 --- a/src/tools/eefm/app.cpp +++ b/src/tools/eefm/app.cpp @@ -417,13 +417,32 @@ int App::run( int argc, char* argv[] ) { if ( !init( locations.Get(), split.Get() ) ) return EXIT_FAILURE; int result = mApp->run(); + // Producers stop before the executor they deliver to goes away: continuations hold it by + // reference. Running transfers are cancelled (destroying the VFS waits for them). + for ( auto& transfer : mTransfers ) + transfer.second.transfer.cancel(); mTransfers.clear(); for ( auto& pane : mPanes ) if ( pane ) pane->tabList.clear(); - mExecutor.reset(); - mPool.reset(); +#ifdef EE_VFS_SFTP + if ( mSSHPrompts ) + mSSHPrompts->shutdown(); +#endif + // The scene is still alive: quit() stops the application loop without destroying it. + // Detach before waiting for producers, so late completions cannot schedule more UI work. + mExecutor->shutdown(); +#ifdef EE_VFS_SFTP + if ( auto sftp = std::dynamic_pointer_cast( mVFS->provider( "sftp" ) ) ) + sftp->disconnectAll(); +#endif VirtualFileSystem::destroyInstance(); + // The UI thread pool belongs to the scene too: work finishing while it is joined finds the + // executor detached (its functions are dropped), not destroyed. + mSystemIcons.reset(); + mPool.reset(); + mApp.reset(); + mExecutor.reset(); return result; } @@ -625,8 +644,9 @@ bool App::init( const std::vector& locations, bool forceSplit ) { } ); mApp->getWindow()->setCloseRequestCallback( [this]( EE::Window::Window* ) { - saveState(); - return true; + quit(); + // Keep the scene and its dispatcher alive until producers and the executor stop. + return false; } ); // The keyboard belongs to the files: a click on a pane's bars or empty areas focuses the @@ -753,9 +773,8 @@ void App::saveState() { } void App::quit() { - // Closing the window programmatically skips the close request callback. saveState(); - mApp->getWindow()->close(); + mApp->requestQuit(); } void App::rememberViewState( FileTab* tab ) { diff --git a/src/tools/eefm/dialogs.cpp b/src/tools/eefm/dialogs.cpp index 9f0335b8e..81a80cd85 100644 --- a/src/tools/eefm/dialogs.cpp +++ b/src/tools/eefm/dialogs.cpp @@ -65,12 +65,22 @@ void showError( UISceneNode* ui, const String& message ) { #ifdef EE_VFS_SFTP +void SSHPrompts::shutdown() { + std::lock_guard lock( mMutex ); + mExecutor = nullptr; +} + Task SSHPrompts::verify( const SSHHostKeyInfo& info ) { auto source = std::make_shared>(); auto task = source->task(); + std::lock_guard lock( mMutex ); + if ( !mExecutor ) { + source->resolve( SSHHostKeyDecision::Reject ); + return task; + } UISceneNode* ui = mUI; // If the executor drops this (shutdown) the source is abandoned and the key rejected. - mExecutor.execute( [ui, info, source] { + mExecutor->execute( [ui, info, source] { String message = String::format( ui->i18n( "eefm_unknown_host_key", "The authenticity of host '%s' (port %d) can't be established.\n%s key " @@ -92,8 +102,13 @@ SSHPrompts::request( const SSHAuthenticationRequest& request ) { using Answers = std::optional>; auto source = std::make_shared>(); auto task = source->task(); + std::lock_guard lock( mMutex ); + if ( !mExecutor ) { + source->resolve( std::nullopt ); + return task; + } UISceneNode* ui = mUI; - mExecutor.execute( [ui, request, source] { + mExecutor->execute( [ui, request, source] { String title = ui->i18n( "eefm_authentication", "Authentication" ) + " - " + String::fromUtf8( request.user + "@" + request.host ); // Prompts are asked one after another; any cancel aborts the whole request. diff --git a/src/tools/eefm/dialogs.hpp b/src/tools/eefm/dialogs.hpp index f2442d3ef..43b6f37b4 100644 --- a/src/tools/eefm/dialogs.hpp +++ b/src/tools/eefm/dialogs.hpp @@ -9,6 +9,7 @@ #include #endif #include +#include #include #include @@ -34,7 +35,10 @@ void showError( UISceneNode* ui, const String& message ); * I/O thread; dialogs are always created on the UI thread through the executor. */ class SSHPrompts : public SSHHostKeyVerifier, public SSHAuthenticationHandler { public: - SSHPrompts( UISceneNode* ui, Executor& uiExecutor ) : mUI( ui ), mExecutor( uiExecutor ) {} + SSHPrompts( UISceneNode* ui, Executor& uiExecutor ) : mUI( ui ), mExecutor( &uiExecutor ) {} + + /** Refuses late requests before the executor and scene are destroyed. */ + void shutdown(); Task verify( const SSHHostKeyInfo& info ) override; @@ -43,7 +47,8 @@ class SSHPrompts : public SSHHostKeyVerifier, public SSHAuthenticationHandler { private: UISceneNode* mUI; - Executor& mExecutor; + std::mutex mMutex; + Executor* mExecutor; }; #endif diff --git a/src/tools/eefm/fileops.cpp b/src/tools/eefm/fileops.cpp index 94d7854f4..aa00054d5 100644 --- a/src/tools/eefm/fileops.cpp +++ b/src/tools/eefm/fileops.cpp @@ -2,6 +2,7 @@ #include "apps.hpp" #include "mime.hpp" #include "trash.hpp" +#include namespace eefm { @@ -86,6 +87,15 @@ void App::openRow( FileTab* tab, size_t row ) { namespace { +/** Where remote files are downloaded to be opened: private to the user (the shared /tmp is + * not, on Linux and BSD). */ +std::string remoteCacheDirectory() { + std::string cache = Sys::getCachePath( "eefm" ); + FileSystem::dirAddSlashAtEnd( cache ); + cache += "remote/"; + return cache; +} + /** Calls done( localPaths ) with local copies of uris (remote files are downloaded to a cache * folder first). */ void withLocalFiles( VirtualFileSystem* vfs, Executor& executor, UISceneNode* ui, @@ -100,10 +110,12 @@ void withLocalFiles( VirtualFileSystem* vfs, Executor& executor, UISceneNode* ui continue; } ++*remaining; - std::string cache = Sys::getTempPath(); - FileSystem::dirAddSlashAtEnd( cache ); - cache += "eefm/" + uris[i].getScheme() + "/" + uris[i].getHost() + "/"; - FileSystem::makeDir( cache, true ); + // One private folder per remote file (its complete URI: user, port and path included), + // so files with the same name never overwrite each other; the file keeps its name for + // the application that opens it. + std::string cache = + remoteCacheDirectory() + SHA1::fromString( uris[i].toString() ).toHexString() + "/"; + FileSystem::makeDir( cache, true, 0700 ); std::string local = cache + uris[i].getLastPathSegment(); CopyOptions options; options.overwrite = OverwritePolicy::Overwrite; diff --git a/src/tools/eefm/fileview.cpp b/src/tools/eefm/fileview.cpp index 8bafab5cd..163534a52 100644 --- a/src/tools/eefm/fileview.cpp +++ b/src/tools/eefm/fileview.cpp @@ -243,6 +243,8 @@ class FileCell : public UITableCell { Image preview = snapshot( dragLabel( getUISceneNode() ) ); if ( preview.getPixels() ) offer.preview.image = &preview; + // Drops on eefm's own window complete as eefm's drag (see endNativeDrag()). + offer.sourceHandlesDrops = true; auto result = drag.startDrag( offer, []( DragResult result ) { FileCell* cell = sNativeDragCell; sNativeDragCell = nullptr;