A few optimizations to not affect first-frame time by the models catalog load, now everything runs async.

src/tools/ecode/plugins/aiassistant/aiassistantplugin.cpp

@@ -210,6 +210,9 @@ AIAssistantPlugin::AIAssistantPlugin( PluginManager* pluginManager, bool sync )
 AIAssistantPlugin::~AIAssistantPlugin() {
 	{
 		std::lock_guard<std::mutex> lock( mModelCatalogMutex );
+		mModelCatalogCancelled->store( true );
+		if ( mModelCatalog )
+			mModelCatalog->cancel();
 		mModelCatalog.reset();
 	}
 	if ( SceneManager::existsSingleton() && !SceneManager::instance()->isShuttingDown() ) {
@@ -287,9 +290,6 @@ void AIAssistantPlugin::load( PluginManager* pluginManager ) {
 					userConfig["model_catalog_refresh_hours"].get<std::uint32_t>();
 		}
 	}
-	LLMModelCatalog catalog( catalogSettings );
-	catalog.loadCached( mProviders );
-
 	if ( !mConfigPath.empty() ) {
 		try {
 			loadAIAssistantConfig( mConfigPath, true );
@@ -375,7 +375,8 @@ void AIAssistantPlugin::load( PluginManager* pluginManager ) {
 void AIAssistantPlugin::refreshModelCatalogAsync() {
 	LLMModelCatalog::Settings settings;
 	Node* mainThreadNode;
-	LLMModelCatalog* catalog;
+	std::shared_ptr<LLMModelCatalog> catalog;
+	std::shared_ptr<std::atomic_bool> cancelled;
 	{
 		std::lock_guard<std::mutex> lock( mModelCatalogMutex );
 		if ( mModelCatalogRefreshStarted || !mModelCatalogSettings ||
@@ -386,23 +387,39 @@ void AIAssistantPlugin::refreshModelCatalogAsync() {
 		if ( !mainThreadNode || !mainThreadNode->getSceneNode() )
 			return;
 		settings = *mModelCatalogSettings;
-		mModelCatalog = std::make_unique<LLMModelCatalog>( settings );
+		mModelCatalog = std::make_shared<LLMModelCatalog>( settings );
-		catalog = mModelCatalog.get();
+		catalog = mModelCatalog;
+		cancelled = mModelCatalogCancelled;
 		mModelCatalogRefreshStarted = true;
 	}
 	auto* manager = mManager;
-	catalog->refreshAsync( mProviders, [manager, mainThreadNode]( LLMProviders providers ) mutable {
+	auto applyProviders = [manager, mainThreadNode, cancelled]( LLMProviders providers ) mutable {
-		mainThreadNode->runOnMainThread( [providers = std::move( providers ), manager]() mutable {
+		if ( cancelled->load() )
-			if ( manager->isClosing() )
+			return;
-				return;
+		mainThreadNode->runOnMainThread(
-			auto* plugin = manager->get( AIAssistantPlugin::Definition().id );
+			[providers = std::move( providers ), manager, cancelled]() mutable {
-			if ( plugin )
+				if ( cancelled->load() || manager->isClosing() )
-				static_cast<AIAssistantPlugin*>( plugin )->applyModelCatalog(
+					return;
-					std::move( providers ) );
+				auto* plugin = manager->get( AIAssistantPlugin::Definition().id );
-			else
+				if ( plugin ) {
-				Log::warning(
+					static_cast<AIAssistantPlugin*>( plugin )->applyModelCatalog(
-					"Could not apply refreshed LLM model catalog: plugin is unavailable" );
+						std::move( providers ) );
-		} );
+				} else {
+					Log::warning(
+						"Could not apply refreshed LLM model catalog: plugin is unavailable" );
+				}
+			} );
+	};
+	LLMProviders providers = mProviders;
+	mThreadPool->run( [catalog = std::move( catalog ), providers = std::move( providers ),
+					   applyProviders = std::move( applyProviders ), cancelled]() mutable {
+		if ( cancelled->load() )
+			return;
+		if ( catalog->loadCached( providers ) && !cancelled->load() )
+			applyProviders( LLMProviders( providers ) );
+		if ( cancelled->load() )
+			return;
+		catalog->refreshAsync( std::move( providers ), std::move( applyProviders ) );
 	} );
 }
 

src/tools/ecode/plugins/aiassistant/aiassistantplugin.hpp

@@ -69,7 +69,9 @@ class AIAssistantPlugin : public PluginBase {
 	std::string mConfigFileError;
 	std::mutex mModelCatalogMutex;
 	std::optional<LLMModelCatalog::Settings> mModelCatalogSettings;
-	std::unique_ptr<LLMModelCatalog> mModelCatalog;
+	std::shared_ptr<LLMModelCatalog> mModelCatalog;
+	std::shared_ptr<std::atomic_bool> mModelCatalogCancelled{
+		std::make_shared<std::atomic_bool>( false ) };
 	bool mModelCatalogRefreshStarted{ false };
 
 	AIAssistantPlugin( PluginManager* pluginManager, bool sync );

src/tools/ecode/plugins/aiassistant/llmmodelcatalog.cpp

@@ -2,6 +2,7 @@
 
 #include <eepp/network/http.hpp>
 #include <eepp/system/clock.hpp>
+#include <eepp/system/fileinfo.hpp>
 #include <eepp/system/filesystem.hpp>
 #include <eepp/system/log.hpp>
 #include <eepp/system/sys.hpp>
@@ -32,9 +33,7 @@ static bool supportsTextChat( const json& model ) {
 }
 
 static json normalizeCatalog( const json& source ) {
-	json result = { { "version", CATALOG_VERSION },
+	json result = { { "version", CATALOG_VERSION }, { "providers", json::object() } };
-					{ "fetched_at", Sys::getUnixTimestamp() },
-					{ "providers", json::object() } };
 	for ( const auto& [providerId, provider] : source.items() ) {
 		if ( !provider.is_object() || !provider.contains( "models" ) ||
 			 !provider["models"].is_object() )
@@ -124,6 +123,10 @@ static bool writeAtomically( const std::string& path, const std::string& data )
 LLMModelCatalog::LLMModelCatalog( Settings settings ) : mSettings( std::move( settings ) ) {}
 
 LLMModelCatalog::~LLMModelCatalog() {
+	cancel();
+}
+
+void LLMModelCatalog::cancel() {
 	mCancelled->store( true );
 	if ( mRequestId && Http::Pool::getGlobal().exists( mRequestURI, mProxyURI ) )
 		Http::Pool::getGlobal().get( mRequestURI, mProxyURI )->setCancelRequest( mRequestId );
@@ -161,7 +164,8 @@ LLMModelCatalog::parseReasoningConfiguration( const json& options ) {
 	return fallback;
 }
 
-bool LLMModelCatalog::applyCatalog( const std::string& data, LLMProviders& providers ) const {
+bool LLMModelCatalog::applyCatalog( const std::string& data, LLMProviders& providers,
+									std::string* etag ) const {
 	const auto catalog = json::parse( data, nullptr, false, true );
 	if ( !catalog.is_object() || !catalog.contains( "version" ) ||
 		 !catalog["version"].is_number_integer() ||
@@ -243,15 +247,24 @@ bool LLMModelCatalog::applyCatalog( const std::string& data, LLMProviders& provi
 			catalogProviders.insert_or_assign( providerId, std::move( provider ) );
 	if ( catalogProviders.empty() )
 		return false;
+	if ( etag )
+		*etag = catalog.contains( "etag" ) && catalog["etag"].is_string()
+					? catalog["etag"].get<std::string>()
+					: "";
 	providers = std::move( catalogProviders );
 	return true;
 }
 
-bool LLMModelCatalog::loadCached( LLMProviders& providers ) const {
+bool LLMModelCatalog::loadCached( LLMProviders& providers ) {
 	if ( !mSettings.enabled )
 		return false;
 	std::string data;
-	return FileSystem::fileGet( mSettings.cachePath, data ) && applyCatalog( data, providers );
+	if ( !FileSystem::fileGet( mSettings.cachePath, data ) ||
+		 !applyCatalog( data, providers, &mCachedETag ) )
+		return false;
+	mCachedData = std::move( data );
+	mCachedCatalogValid = true;
+	return true;
 }
 
 std::uint64_t
@@ -260,36 +273,23 @@ LLMModelCatalog::refreshAsync( LLMProviders providers,
 	if ( !mSettings.enabled || mSettings.url.empty() )
 		return 0;
 	Clock clock;
-	std::string cached;
-	json cache;
-	if ( FileSystem::fileGet( mSettings.cachePath, cached ) )
-		cache = json::parse( cached, nullptr, false, true );
-	const bool validCache = cache.is_object() && cache.contains( "version" ) &&
-							cache["version"].is_number_integer() &&
-							cache["version"].get<int>() == CATALOG_VERSION;
-	const std::int64_t fetchedAt =
-		validCache && cache.contains( "fetched_at" ) && cache["fetched_at"].is_number_integer()
-			? cache["fetched_at"].get<std::int64_t>()
-			: 0;
 	const std::int64_t maxAge =
 		static_cast<std::int64_t>( mSettings.refreshIntervalHours ) * 60 * 60;
-	if ( fetchedAt > 0 && Sys::getUnixTimestamp() - fetchedAt < maxAge )
+	const FileInfo cacheInfo( mSettings.cachePath );
+	if ( mCachedCatalogValid && cacheInfo.exists() && cacheInfo.getModificationTime() > 0 &&
+		 Sys::getUnixTimestamp() - cacheInfo.getModificationTime() < maxAge )
 		return 0;
 
 	Http::Request::FieldTable headers;
-	if ( validCache ) {
+	if ( !mCachedETag.empty() )
-		const std::string etag = cache.contains( "etag" ) && cache["etag"].is_string()
+		headers["If-None-Match"] = mCachedETag;
-									 ? cache["etag"].get<std::string>()
-									 : "";
-		if ( !etag.empty() )
-			headers["If-None-Match"] = etag;
-	}
 	const Settings settings = mSettings;
+	std::string cachedData = std::move( mCachedData );
 	mCancelled->store( false );
 	mRequestURI = URI( mSettings.url );
 	mProxyURI = Http::getEnvProxyURI();
 	mRequestId = Http::getAsync(
-		[settings, cache = std::move( cache ), providers = std::move( providers ),
+		[settings, cachedData = std::move( cachedData ), providers = std::move( providers ),
 		 refreshedCallback = std::move( refreshedCallback ), clock,
 		 cancelled = mCancelled]( const Http&, Http::Request&, Http::Response& response ) mutable {
 			if ( cancelled->load() )
@@ -297,8 +297,8 @@ LLMModelCatalog::refreshAsync( LLMProviders providers,
 			if ( response.getStatus() == Http::Response::Status::NotModified ) {
 				if ( cancelled->load() )
 					return;
-				cache["fetched_at"] = Sys::getUnixTimestamp();
+				if ( !cachedData.empty() )
-				writeAtomically( settings.cachePath, cache.dump() );
+					writeAtomically( settings.cachePath, cachedData );
 				return;
 			}
 			if ( response.getStatus() != Http::Response::Status::Ok ) {

src/tools/ecode/plugins/aiassistant/llmmodelcatalog.hpp

@@ -23,8 +23,9 @@ class LLMModelCatalog {
 
 	explicit LLMModelCatalog( Settings settings );
 	~LLMModelCatalog();
+	void cancel();
 
-	bool loadCached( LLMProviders& providers ) const;
+	bool loadCached( LLMProviders& providers );
 
 	std::uint64_t refreshAsync( LLMProviders providers,
 								std::function<void( LLMProviders )> refreshedCallback );
@@ -38,8 +39,12 @@ class LLMModelCatalog {
 	EE::Network::URI mProxyURI;
 	std::uint64_t mRequestId{ 0 };
 	std::shared_ptr<std::atomic_bool> mCancelled{ std::make_shared<std::atomic_bool>( false ) };
+	std::string mCachedData;
+	std::string mCachedETag;
+	bool mCachedCatalogValid{ false };
 
-	bool applyCatalog( const std::string& data, LLMProviders& providers ) const;
+	bool applyCatalog( const std::string& data, LLMProviders& providers,
+					   std::string* etag = nullptr ) const;
 };
 
 } // namespace ecode