diff --git a/include/eepp/ui/doc/syntaxhighlighter.hpp b/include/eepp/ui/doc/syntaxhighlighter.hpp index e465c98a7..4aee18498 100644 --- a/include/eepp/ui/doc/syntaxhighlighter.hpp +++ b/include/eepp/ui/doc/syntaxhighlighter.hpp @@ -8,10 +8,10 @@ namespace EE { namespace UI { namespace Doc { struct TokenizedLine { - Uint64 initState; + Uint64 initState{ SYNTAX_TOKENIZER_STATE_NONE }; String::HashType hash; - std::vector tokens; - Uint64 state; + std::vector tokens; + Uint64 state{ SYNTAX_TOKENIZER_STATE_NONE }; }; class EE_API SyntaxHighlighter { @@ -24,7 +24,7 @@ class EE_API SyntaxHighlighter { void invalidate( Int64 lineIndex ); - const std::vector& getLine( const size_t& index ); + const std::vector& getLine( const size_t& index ); Int64 getFirstInvalidLine() const; @@ -38,12 +38,22 @@ class EE_API SyntaxHighlighter { SyntaxTokenPosition getTokenPositionAt( const TextPosition& pos ); + void setLine( const size_t& line, const TokenizedLine& tokenization ); + + void mergeLine( const size_t& line, const TokenizedLine& tokenization ); + + TokenizedLine tokenizeLine( const size_t& line, + const Uint64& state = SYNTAX_TOKENIZER_STATE_NONE ); + + Mutex& getLinesMutex(); + protected: TextDocument* mDoc; std::unordered_map mLines; + std::unordered_map mTokenizerLines; + Mutex mLinesMutex; Int64 mFirstInvalidLine; Int64 mMaxWantedLine; - TokenizedLine tokenizeLine( const size_t& line, const Uint64& state ); }; }}} // namespace EE::UI::Doc diff --git a/include/eepp/ui/doc/syntaxtokenizer.hpp b/include/eepp/ui/doc/syntaxtokenizer.hpp index 75478fdc3..56a4709a2 100644 --- a/include/eepp/ui/doc/syntaxtokenizer.hpp +++ b/include/eepp/ui/doc/syntaxtokenizer.hpp @@ -45,6 +45,10 @@ class EE_API SyntaxTokenizer { tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32& state, const size_t& startIndex = 0, bool skipSubSyntaxSeparator = false ); + static std::pair, Uint32> + tokenizePosition( const SyntaxDefinition& syntax, const std::string& text, const Uint32& state, + const size_t& startIndex = 0, bool skipSubSyntaxSeparator = false ); + static std::pair, Uint32> tokenizeComplete( const SyntaxDefinition& syntax, const std::string& text, const Uint32& state, const size_t& startIndex = 0, bool skipSubSyntaxSeparator = false ); diff --git a/src/eepp/ui/doc/syntaxdefinitionmanager.cpp b/src/eepp/ui/doc/syntaxdefinitionmanager.cpp index a19bf3f0e..46bce7ac3 100644 --- a/src/eepp/ui/doc/syntaxdefinitionmanager.cpp +++ b/src/eepp/ui/doc/syntaxdefinitionmanager.cpp @@ -513,7 +513,7 @@ static void addTypeScript() { { { "-?%d+[%d%.eE]*" }, "number" }, { { "-?%.?%d+" }, "number" }, { { "[%+%-=/%*%^%%<>!~|&]" }, "operator" }, - { { "[%a_][%w_]*%f[(]" }, "function" }, + { { "[%a_][%w_$]*%f[(]" }, "function" }, { { "[%a_][%w_]*" }, "symbol" }, }, { { "any", "keyword2" }, { "arguments", "keyword2" }, { "as", "keyword2" }, @@ -714,7 +714,7 @@ static void addCPP() { { "switch", "keyword" }, { "case", "keyword" }, { "default", "keyword" }, - { "auto", "keyword" }, + { "auto", "keyword2" }, { "const", "keyword" }, { "void", "keyword" }, { "int", "keyword2" }, diff --git a/src/eepp/ui/doc/syntaxhighlighter.cpp b/src/eepp/ui/doc/syntaxhighlighter.cpp index 0fdd3f175..f1e949888 100644 --- a/src/eepp/ui/doc/syntaxhighlighter.cpp +++ b/src/eepp/ui/doc/syntaxhighlighter.cpp @@ -16,6 +16,7 @@ void SyntaxHighlighter::changeDoc( TextDocument* doc ) { } void SyntaxHighlighter::reset() { + Lock l( mLinesMutex ); mLines.clear(); mFirstInvalidLine = 0; mMaxWantedLine = 0; @@ -30,19 +31,24 @@ TokenizedLine SyntaxHighlighter::tokenizeLine( const size_t& line, const Uint64& TokenizedLine tokenizedLine; tokenizedLine.initState = state; tokenizedLine.hash = mDoc->line( line ).getHash(); - std::pair, Uint64> res = SyntaxTokenizer::tokenize( - mDoc->getSyntaxDefinition(), mDoc->line( line ).toUtf8(), state ); + auto res = SyntaxTokenizer::tokenizePosition( mDoc->getSyntaxDefinition(), + mDoc->line( line ).toUtf8(), state ); tokenizedLine.tokens = std::move( res.first ); tokenizedLine.state = std::move( res.second ); return tokenizedLine; } -const std::vector& SyntaxHighlighter::getLine( const size_t& index ) { +Mutex& SyntaxHighlighter::getLinesMutex() { + return mLinesMutex; +} + +const std::vector& SyntaxHighlighter::getLine( const size_t& index ) { if ( mDoc->getSyntaxDefinition().getPatterns().empty() ) { - static std::vector noHighlightVector = { { "normal", 0 } }; + static std::vector noHighlightVector = { { "normal", 0 } }; noHighlightVector[0].len = mDoc->line( index ).size(); return noHighlightVector; } + Lock l( mLinesMutex ); const auto& it = mLines.find( index ); if ( it == mLines.end() || ( index < mDoc->linesCount() && mDoc->line( index ).getHash() != it->second.hash ) ) { @@ -54,6 +60,8 @@ const std::vector& SyntaxHighlighter::getLine( const size_t& index } } mLines[index] = tokenizeLine( index, prevState ); + mTokenizerLines[index] = mLines[index]; + mMaxWantedLine = eemax( mMaxWantedLine, index ); return mLines[index].tokens; } mMaxWantedLine = eemax( mMaxWantedLine, index ); @@ -88,6 +96,7 @@ bool SyntaxHighlighter::updateDirty( int visibleLinesCount ) { const auto& it = mLines.find( index ); if ( it == mLines.end() || it->second.initState != state ) { mLines[index] = tokenizeLine( index, state ); + mTokenizerLines[index] = mLines[index]; changed = true; } } @@ -100,6 +109,7 @@ bool SyntaxHighlighter::updateDirty( int visibleLinesCount ) { const SyntaxDefinition& SyntaxHighlighter::getSyntaxDefinitionFromTextPosition( const TextPosition& position ) { + Lock l( mLinesMutex ); auto found = mLines.find( position.line() ); if ( found == mLines.end() ) return SyntaxDefinitionManager::instance()->getPlainStyle(); @@ -117,7 +127,7 @@ SyntaxHighlighter::getSyntaxDefinitionFromTextPosition( const TextPosition& posi std::string SyntaxHighlighter::getTokenTypeAt( const TextPosition& pos ) { if ( !pos.isValid() || pos.line() < 0 || pos.line() >= (Int64)mDoc->linesCount() ) return "normal"; - const std::vector& tokens = getLine( pos.line() ); + auto tokens = getLine( pos.line() ); if ( tokens.empty() ) return "normal"; Int64 col = 0; @@ -132,7 +142,7 @@ std::string SyntaxHighlighter::getTokenTypeAt( const TextPosition& pos ) { SyntaxTokenPosition SyntaxHighlighter::getTokenPositionAt( const TextPosition& pos ) { if ( !pos.isValid() || pos.line() < 0 || pos.line() >= (Int64)mDoc->linesCount() ) return {}; - const std::vector& tokens = getLine( pos.line() ); + auto tokens = getLine( pos.line() ); if ( tokens.empty() ) return {}; Int64 col = 0; @@ -144,4 +154,53 @@ SyntaxTokenPosition SyntaxHighlighter::getTokenPositionAt( const TextPosition& p return {}; } +void SyntaxHighlighter::setLine( const size_t& line, const TokenizedLine& tokenization ) { + Lock l( mLinesMutex ); + mLines[line] = tokenization; +} + +void SyntaxHighlighter::mergeLine( const size_t& line, const TokenizedLine& tokenization ) { + TokenizedLine tline; + { + Lock l( mLinesMutex ); + auto found = mTokenizerLines.find( line ); + if ( found != mTokenizerLines.end() && + mDoc->line( line ).getHash() == found->second.hash ) { + tline = found->second; + } else { + tline = tokenizeLine( line ); + mTokenizerLines[line] = tline; + } + } + + for ( const auto& token : tokenization.tokens ) { + for ( size_t i = 0; i < tline.tokens.size(); ++i ) { + const auto ltoken = tline.tokens[i]; + if ( token.pos >= ltoken.pos && token.pos + token.len <= ltoken.pos + ltoken.len ) { + tline.tokens.erase( tline.tokens.begin() + i ); + + if ( token.pos > ltoken.pos ) { + tline.tokens.insert( tline.tokens.begin() + i, + { ltoken.type, ltoken.pos, + static_cast( token.pos - ltoken.pos ) } ); + } + + tline.tokens.insert( tline.tokens.begin() + i, token ); + + if ( token.pos + token.len < ltoken.pos + ltoken.len ) { + tline.tokens.insert( tline.tokens.begin() + i, + { ltoken.type, static_cast( token.pos + token.len ), + static_cast( ( ltoken.pos + ltoken.len ) - + ( token.pos + token.len ) ) } ); + } + + break; + } + } + } + + Lock l( mLinesMutex ); + mLines[line] = std::move( tline ); +} + }}} // namespace EE::UI::Doc diff --git a/src/eepp/ui/doc/syntaxtokenizer.cpp b/src/eepp/ui/doc/syntaxtokenizer.cpp index 49176c9da..34e34385e 100644 --- a/src/eepp/ui/doc/syntaxtokenizer.cpp +++ b/src/eepp/ui/doc/syntaxtokenizer.cpp @@ -54,6 +54,11 @@ static void pushToken( std::vector& tokens, const std::string& type, const st size_t len = String::utf8Length( substr ); if constexpr ( std::is_same_v ) { tokens.push_back( { type, std::move( substr ), len } ); + } else if constexpr ( std::is_same_v ) { + Int64 tpos = tokens.empty() ? 0 + : tokens[tokens.size() - 1].pos + + tokens[tokens.size() - 1].len; + tokens.push_back( { type, tpos, len } ); } else { tokens.push_back( { type, len } ); } @@ -63,6 +68,11 @@ static void pushToken( std::vector& tokens, const std::string& type, const st } else { if constexpr ( std::is_same_v ) { tokens.push_back( { type, text, String::utf8Length( text ) } ); + } else if constexpr ( std::is_same_v ) { + Int64 tpos = tokens.empty() + ? 0 + : tokens[tokens.size() - 1].pos + tokens[tokens.size() - 1].len; + tokens.push_back( { type, tpos, String::utf8Length( text ) } ); } else { tokens.push_back( { type, String::utf8Length( text ) } ); } @@ -347,6 +357,14 @@ SyntaxTokenizer::tokenize( const SyntaxDefinition& syntax, const std::string& te return _tokenize( syntax, text, state, startIndex, skipSubSyntaxSeparator ); } +std::pair, Uint32> +SyntaxTokenizer::tokenizePosition( const SyntaxDefinition& syntax, const std::string& text, + const Uint32& state, const size_t& startIndex, + bool skipSubSyntaxSeparator ) { + return _tokenize( syntax, text, state, startIndex, + skipSubSyntaxSeparator ); +} + std::pair, Uint32> SyntaxTokenizer::tokenizeComplete( const SyntaxDefinition& syntax, const std::string& text, const Uint32& state, const size_t& startIndex, diff --git a/src/eepp/ui/uicodeeditor.cpp b/src/eepp/ui/uicodeeditor.cpp index 759f27a92..cd4f4f213 100644 --- a/src/eepp/ui/uicodeeditor.cpp +++ b/src/eepp/ui/uicodeeditor.cpp @@ -306,6 +306,8 @@ void UICodeEditor::draw() { } for ( unsigned long i = lineRange.first; i <= lineRange.second; i++ ) { + Lock l( mDoc->getHighlighter()->getLinesMutex() ); + Vector2f curScroll( { startScroll.x, static_cast( startScroll.y + lineHeight * (double)i ) } ); @@ -343,8 +345,10 @@ void UICodeEditor::draw() { drawColorPreview( startScroll, lineHeight ); } - if ( mMinimapEnabled ) + if ( mMinimapEnabled ) { + Lock l( mDoc->getHighlighter()->getLinesMutex() ); drawMinimap( screenStart, lineRange ); + } for ( auto& plugin : mPlugins ) plugin->postDraw( this, startScroll, lineHeight, cursor ); diff --git a/src/tools/ecode/plugins/lsp/lspclientplugin.cpp b/src/tools/ecode/plugins/lsp/lspclientplugin.cpp index 867c059b1..3ecd8b7df 100644 --- a/src/tools/ecode/plugins/lsp/lspclientplugin.cpp +++ b/src/tools/ecode/plugins/lsp/lspclientplugin.cpp @@ -415,6 +415,14 @@ bool LSPClientPlugin::onMouseClick( UICodeEditor* editor, const Vector2i& pos, return true; } +bool LSPClientPlugin::semanticHighlightingEnabled() const { + return mSemanticHighlighting; +} + +void LSPClientPlugin::setSemanticHighlighting( bool semanticHighlighting ) { + mSemanticHighlighting = semanticHighlighting; +} + bool LSPClientPlugin::editorExists( UICodeEditor* editor ) { return mManager->getSplitter()->editorExists( editor ); } @@ -716,6 +724,11 @@ void LSPClientPlugin::loadLSPConfig( std::vector& lsps, const std Time::fromString( config["server_close_after_idle_time"].get() ) ); else if ( updateConfigFile ) config["server_close_after_idle_time"] = mClientManager.getLSPDecayTime().toString(); + + if ( config.contains( "semantic_highlighting" ) ) + mSemanticHighlighting = config.value( "semantic_highlighting", false ); + else if ( updateConfigFile ) + config["semantic_highlighting"] = mSemanticHighlighting; } if ( mKeyBindings.empty() ) { diff --git a/src/tools/ecode/plugins/lsp/lspclientplugin.hpp b/src/tools/ecode/plugins/lsp/lspclientplugin.hpp index 42a3cf63f..617907fc5 100644 --- a/src/tools/ecode/plugins/lsp/lspclientplugin.hpp +++ b/src/tools/ecode/plugins/lsp/lspclientplugin.hpp @@ -83,6 +83,10 @@ class LSPClientPlugin : public UICodeEditorPlugin { virtual bool onMouseClick( UICodeEditor* editor, const Vector2i& pos, const Uint32& flags ); + bool semanticHighlightingEnabled() const; + + void setSemanticHighlighting( bool semanticHighlighting ); + protected: friend class LSPDocumentClient; @@ -104,6 +108,7 @@ class LSPClientPlugin : public UICodeEditorPlugin { bool mOldDontAutoHideOnMouseMove{ false }; bool mOldUsingCustomStyling{ false }; bool mSymbolInfoShowing{ false }; + bool mSemanticHighlighting{ false }; std::map mKeyBindings; /* cmd, shortcut */ std::map> mDelayedDocs; Uint32 mHoverWaitCb{ 0 }; diff --git a/src/tools/ecode/plugins/lsp/lspclientserver.cpp b/src/tools/ecode/plugins/lsp/lspclientserver.cpp index 9a04b752d..2b46acae1 100644 --- a/src/tools/ecode/plugins/lsp/lspclientserver.cpp +++ b/src/tools/ecode/plugins/lsp/lspclientserver.cpp @@ -952,26 +952,30 @@ static json renameParams( const URI& document, const TextPosition& pos, static LSPSemanticTokensDelta parseSemanticTokensDelta( const json& result ) { LSPSemanticTokensDelta ret; + if ( result.is_null() ) + return ret; ret.resultId = result.value( "resultId", "" ); - const auto& edits = result["edits"]; - for ( const auto& edit : edits ) { - if ( !edit.is_object() ) - continue; - LSPSemanticTokensEdit e; - e.start = edit.value( "start", 0 ); - e.deleteCount = edit.value( "deleteCount", 0 ); - const auto& data = edit["data"]; - e.data.reserve( data.size() ); - std::transform( data.cbegin(), data.cend(), std::back_inserter( e.data ), - []( const json& jv ) { return jv.get(); } ); - ret.edits.push_back( e ); + if ( result.contains( "edits" ) ) { + const auto& edits = result["edits"]; + for ( const auto& edit : edits ) { + if ( !edit.is_object() ) + continue; + LSPSemanticTokensEdit e; + e.start = edit.value( "start", 0 ); + e.deleteCount = edit.value( "deleteCount", 0 ); + const auto& data = edit["data"]; + e.data.reserve( data.size() ); + std::transform( data.cbegin(), data.cend(), std::back_inserter( e.data ), + []( const json& jv ) { return jv.get(); } ); + ret.edits.push_back( e ); + } + } + if ( result.contains( "data" ) ) { + auto& data = result["data"]; + ret.data.reserve( data.size() ); + std::transform( data.cbegin(), data.cend(), std::back_inserter( ret.data ), + []( const json& jv ) { return jv.get(); } ); } - - auto& data = result["data"]; - ret.data.reserve( data.size() ); - std::transform( data.cbegin(), data.cend(), std::back_inserter( ret.data ), - []( const json& jv ) { return jv.get(); } ); - return ret; } @@ -1258,10 +1262,11 @@ LSPClientServer::LSPRequestHandle LSPClientServer::write( const json& msg, // notification == no handler if ( h ) { - ob[MEMBER_ID] = ++mLastMsgId; - ret.mId = mLastMsgId; + int msgId = ++mLastMsgId; + ob[MEMBER_ID] = msgId; + ret.mId = msgId; Lock l( mHandlersMutex ); - mHandlers[mLastMsgId] = { h, eh }; + mHandlers[msgId] = { h, eh }; } else if ( id ) { ob[MEMBER_ID] = id; } @@ -1966,34 +1971,36 @@ void LSPClientServer::executeCommand( const std::string& cmd, const json& params []( const auto&, const auto& ) {} ); } -LSPClientServer::LSPRequestHandle -LSPClientServer::documentSemanticTokensFull( const URI& document, bool delta, - const std::string& requestId, const TextRange& range, - const JsonReplyHandler& h ) { +void LSPClientServer::documentSemanticTokensFull( const URI& document, bool delta, + const std::string& requestId, + const TextRange& range, + const JsonReplyHandler& h ) { auto params = textDocumentParams( document ); // Delta if ( delta && !requestId.empty() ) { params[MEMBER_PREVIOUS_RESULT_ID] = requestId; - return send( newRequest( "textDocument/semanticTokens/full/delta", params ), h ); + sendAsync( newRequest( "textDocument/semanticTokens/full/delta", params ), h ); + return; } // Range if ( range.isValid() ) { params[MEMBER_RANGE] = toJson( range ); - return send( newRequest( "textDocument/semanticTokens/range", params ), h ); + sendAsync( newRequest( "textDocument/semanticTokens/range", params ), h ); + return; } - return send( newRequest( "textDocument/semanticTokens/full", params ), h ); + sendAsync( newRequest( "textDocument/semanticTokens/full", params ), h ); } -LSPClientServer::LSPRequestHandle -LSPClientServer::documentSemanticTokensFull( const URI& document, bool delta, - const std::string& requestId, const TextRange& range, - const SemanticTokensDeltaHandler& h ) { - return documentSemanticTokensFull( document, delta, requestId, range, - [h]( const IdType& id, const json& json ) { - if ( h ) - h( id, parseSemanticTokensDelta( json ) ); - } ); +void LSPClientServer::documentSemanticTokensFull( const URI& document, bool delta, + const std::string& requestId, + const TextRange& range, + const SemanticTokensDeltaHandler& h ) { + documentSemanticTokensFull( document, delta, requestId, range, + [h]( const IdType& id, const json& json ) { + if ( h ) + h( id, parseSemanticTokensDelta( json ) ); + } ); } } // namespace ecode diff --git a/src/tools/ecode/plugins/lsp/lspclientserver.hpp b/src/tools/ecode/plugins/lsp/lspclientserver.hpp index 6053dc029..d1a1d8581 100644 --- a/src/tools/ecode/plugins/lsp/lspclientserver.hpp +++ b/src/tools/ecode/plugins/lsp/lspclientserver.hpp @@ -5,6 +5,7 @@ #include "lspdefinition.hpp" #include "lspdocumentclient.hpp" #include "lspprotocol.hpp" +#include #include #include #include @@ -182,15 +183,11 @@ class LSPClientServer { const std::vector& positions, const SelectionRangeHandler& h ); - LSPRequestHandle documentSemanticTokensFull( const URI& document, bool delta, - const std::string& requestId, - const TextRange& range, - const JsonReplyHandler& h ); + void documentSemanticTokensFull( const URI& document, bool delta, const std::string& requestId, + const TextRange& range, const JsonReplyHandler& h ); - LSPRequestHandle documentSemanticTokensFull( const URI& document, bool delta, - const std::string& requestId, - const TextRange& range, - const SemanticTokensDeltaHandler& h ); + void documentSemanticTokensFull( const URI& document, bool delta, const std::string& requestId, + const TextRange& range, const SemanticTokensDeltaHandler& h ); LSPRequestHandle signatureHelp( const URI& document, const TextPosition& pos, const JsonReplyHandler& h ); @@ -255,7 +252,7 @@ class LSPClientServer { std::queue mDidChangeQueue; Mutex mDidChangeMutex; - int mLastMsgId{ 0 }; + std::atomic mLastMsgId{ 0 }; void readStdOut( const char* bytes, size_t n ); diff --git a/src/tools/ecode/plugins/lsp/lspdocumentclient.cpp b/src/tools/ecode/plugins/lsp/lspdocumentclient.cpp index c1acb34fc..fff3a13dd 100644 --- a/src/tools/ecode/plugins/lsp/lspdocumentclient.cpp +++ b/src/tools/ecode/plugins/lsp/lspdocumentclient.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include using namespace EE::System; @@ -16,6 +17,7 @@ LSPDocumentClient::LSPDocumentClient( LSPClientServer* server, TextDocument* doc refreshTag(); notifyOpen(); requestSymbolsDelayed(); + requestSemanticHighlightingDelayed(); } LSPDocumentClient::~LSPDocumentClient() { @@ -24,6 +26,10 @@ LSPDocumentClient::~LSPDocumentClient() { sceneNode->removeActionsByTag( mTag ); } +void LSPDocumentClient::onDocumentLoaded( TextDocument* ) { + requestSemanticHighlightingDelayed(); +} + void LSPDocumentClient::onDocumentTextChanged( const DocumentContentChange& change ) { ++mVersion; // If several change event are being fired, the thread pool can't guaranteed that it will be @@ -31,6 +37,7 @@ void LSPDocumentClient::onDocumentTextChanged( const DocumentContentChange& chan mServer->queueDidChange( mDoc->getURI(), mVersion, "", { change } ); mServer->getThreadPool()->run( [&, change]() { mServer->processDidChangeQueue(); } ); requestSymbolsDelayed(); + requestSemanticHighlightingDelayed(); } void LSPDocumentClient::onDocumentUndoRedo( const TextDocument::UndoRedo& /*eventType*/ ) {} @@ -87,16 +94,57 @@ int LSPDocumentClient::getVersion() const { void LSPDocumentClient::onServerInitialized() { requestSymbols(); + requestSemanticHighlighting(); } void LSPDocumentClient::refreshTag() { String::HashType oldTag = mTag; mTag = String::hash( mDoc->getURI().toString() ); + mTagSemanticTokens = String::hash( mDoc->getURI().toString() + ":semantictokens" ); UISceneNode* sceneNode = getUISceneNode(); if ( nullptr != sceneNode && 0 != oldTag ) sceneNode->removeActionsByTag( oldTag ); } +void LSPDocumentClient::requestSemanticHighlighting() { + if ( !mServer || !mServer->getManager()->getPlugin()->semanticHighlightingEnabled() ) + return; + const auto& cap = mServer->getCapabilities(); + if ( !cap.semanticTokenProvider.full && !cap.semanticTokenProvider.fullDelta /*&& + !cap.semanticTokenProvider.range*/ ) + return; + + TextRange range; + std::string reqId; + bool delta = false; + /*if ( cap.semanticTokenProvider.range ) { + range = mDoc->getDocRange(); + } else */ + if ( cap.semanticTokenProvider.fullDelta ) { + delta = true; + reqId = mSemanticeResultId; + } + + mServer->documentSemanticTokensFull( + mDoc->getURI(), delta, reqId, range, + [this]( const auto&, const LSPSemanticTokensDelta& deltas ) { processTokens( deltas ); } ); +} + +void LSPDocumentClient::requestSemanticHighlightingDelayed() { + if ( !mServer || !mServer->getManager()->getPlugin()->semanticHighlightingEnabled() ) + return; + const auto& cap = mServer->getCapabilities(); + if ( !cap.semanticTokenProvider.full && !cap.semanticTokenProvider.fullDelta /*&& + !cap.semanticTokenProvider.range*/ ) + return; + UISceneNode* sceneNode = getUISceneNode(); + if ( sceneNode ) { + sceneNode->removeActionsByTag( mTagSemanticTokens ); + sceneNode->runOnMainThread( [this]() { requestSemanticHighlighting(); }, Seconds( 0.1f ), + mTagSemanticTokens ); + } +} + UISceneNode* LSPDocumentClient::getUISceneNode() { LSPClientServer* server = mServer; if ( !server || !server->getManager() || !server->getManager()->getPluginManager() || @@ -105,6 +153,112 @@ UISceneNode* LSPDocumentClient::getUISceneNode() { return server->getManager()->getPluginManager()->getUISceneNode(); } +static std::string semanticTokenTypeToSyntaxType( const std::string& type, + const SyntaxDefinition& syn ) { + switch ( String::hash( type ) ) { + case SemanticTokenTypes::Namespace: + case SemanticTokenTypes::Type: + case SemanticTokenTypes::Class: + case SemanticTokenTypes::Enum: + case SemanticTokenTypes::Interface: + case SemanticTokenTypes::Struct: + case SemanticTokenTypes::TypeParameter: + return "keyword2"; + case SemanticTokenTypes::Parameter: + case SemanticTokenTypes::Variable: + return "symbol"; + case SemanticTokenTypes::Property: + if ( syn.getLSPName() == "typescript" || syn.getLSPName() == "javascript" ) + return "function"; + return "symbol"; + case SemanticTokenTypes::EnumMember: + case SemanticTokenTypes::Event: + return "keyword2"; + case SemanticTokenTypes::Function: + case SemanticTokenTypes::Method: + case SemanticTokenTypes::Member: + return "function"; + case SemanticTokenTypes::Macro: + case SemanticTokenTypes::Keyword: + return "keyword2"; + case SemanticTokenTypes::Modifier: + return "keyword"; + case SemanticTokenTypes::Comment: + return "comment"; + case SemanticTokenTypes::Str: + return "string"; + case SemanticTokenTypes::Number: + case SemanticTokenTypes::Regexp: + return "number"; + case SemanticTokenTypes::Operator: + return "operator"; + case SemanticTokenTypes::Decorator: + return "literal"; + case SemanticTokenTypes::Unknown: + break; + }; + return "normal"; +} + +void LSPDocumentClient::processTokens( const LSPSemanticTokensDelta& tokens ) { + if ( !tokens.resultId.empty() ) + mSemanticeResultId = tokens.resultId; + + for ( const auto& edit : tokens.edits ) { + auto& curTokens = mSemanticTokens.data; + if ( edit.deleteCount > 0 ) { + curTokens.erase( curTokens.begin() + edit.start, + curTokens.begin() + edit.start + edit.deleteCount ); + } + curTokens.insert( curTokens.begin() + edit.start, edit.data.begin(), edit.data.end() ); + } + + if ( !tokens.data.empty() ) { + mSemanticTokens = tokens; + } + + highlight(); +} + +void LSPDocumentClient::highlight() { + const auto& data = mSemanticTokens.data; + + if ( data.size() % 5 != 0 ) { + Log::warning( "LSPDocumentClient::highlight bad data format for doc: %s", + mDoc->getURI().toString().c_str() ); + return; + } + + const auto& caps = mServer->getCapabilities().semanticTokenProvider; + Uint32 currentLine = 0; + Uint32 start = 0; + std::map tokenizedLines; + for ( size_t i = 0; i < data.size(); i += 5 ) { + const Uint32 deltaLine = data[i]; + const Uint32 deltaStart = data[i + 1]; + const Uint32 len = data[i + 2]; + const Uint32 type = data[i + 3]; + // const Uint32 mod = data[i + 4]; + currentLine += deltaLine; + + if ( deltaLine == 0 ) { + start += deltaStart; + } else { + start = deltaStart; + } + + auto& line = tokenizedLines[currentLine]; + const auto& ltype = caps.legend.tokenTypes[type]; + line.tokens.push_back( + { semanticTokenTypeToSyntaxType( ltype, mDoc->getSyntaxDefinition() ), start, len } ); + line.hash = mDoc->line( currentLine ).getHash(); + } + + for ( const auto& tline : tokenizedLines ) { + mDoc->getHighlighter()->mergeLine( tline.first, tline.second ); + } +} + void LSPDocumentClient::notifyOpen() { eeASSERT( mDoc ); if ( Engine::instance()->isMainThread() ) { diff --git a/src/tools/ecode/plugins/lsp/lspdocumentclient.hpp b/src/tools/ecode/plugins/lsp/lspdocumentclient.hpp index 752188ebd..122694fd1 100644 --- a/src/tools/ecode/plugins/lsp/lspdocumentclient.hpp +++ b/src/tools/ecode/plugins/lsp/lspdocumentclient.hpp @@ -1,6 +1,7 @@ #ifndef ECODE_LSPDOCUMENTCLIENT_HPP #define ECODE_LSPDOCUMENTCLIENT_HPP +#include "lspprotocol.hpp" #include #include @@ -23,6 +24,7 @@ class LSPDocumentClient : public TextDocument::Client { ~LSPDocumentClient(); + virtual void onDocumentLoaded( TextDocument* ); virtual void onDocumentTextChanged( const DocumentContentChange& change ); virtual void onDocumentUndoRedo( const TextDocument::UndoRedo& eventType ); virtual void onDocumentCursorChange( const TextPosition& ); @@ -37,10 +39,6 @@ class LSPDocumentClient : public TextDocument::Client { void notifyOpen(); - void requestSymbols(); - - void requestSymbolsDelayed(); - TextDocument* getDoc() const; LSPClientServer* getServer() const; @@ -53,11 +51,26 @@ class LSPDocumentClient : public TextDocument::Client { LSPClientServer* mServer{ nullptr }; TextDocument* mDoc{ nullptr }; String::HashType mTag{ 0 }; + String::HashType mTagSemanticTokens{ 0 }; int mVersion{ 0 }; + std::string mSemanticeResultId; + LSPSemanticTokensDelta mSemanticTokens; void refreshTag(); UISceneNode* getUISceneNode(); + + void requestSymbols(); + + void requestSymbolsDelayed(); + + void requestSemanticHighlighting(); + + void requestSemanticHighlightingDelayed(); + + void processTokens( const LSPSemanticTokensDelta& tokens ); + + void highlight(); }; } // namespace ecode diff --git a/src/tools/ecode/plugins/lsp/lspprotocol.hpp b/src/tools/ecode/plugins/lsp/lspprotocol.hpp index 5d3bc1809..70d95609b 100644 --- a/src/tools/ecode/plugins/lsp/lspprotocol.hpp +++ b/src/tools/ecode/plugins/lsp/lspprotocol.hpp @@ -72,6 +72,36 @@ struct LSPSignatureHelpOptions { struct LSPDocumentOnTypeFormattingOptions : public LSPSignatureHelpOptions {}; +struct SemanticTokenTypes { + enum : Uint32 { + Unknown = String::hash( "unknown" ), + Namespace = String::hash( "namespace" ), + Type = String::hash( "type" ), + Class = String::hash( "class" ), + Enum = String::hash( "enum" ), + Interface = String::hash( "interface" ), + Struct = String::hash( "struct" ), + TypeParameter = String::hash( "typeParameter" ), + Parameter = String::hash( "parameter" ), + Variable = String::hash( "variable" ), + Property = String::hash( "property" ), + EnumMember = String::hash( "enumMember" ), + Event = String::hash( "event" ), + Function = String::hash( "function" ), + Method = String::hash( "method" ), + Macro = String::hash( "macro" ), + Keyword = String::hash( "keyword" ), + Modifier = String::hash( "modifier" ), + Comment = String::hash( "comment" ), + Str = String::hash( "string" ), + Number = String::hash( "number" ), + Regexp = String::hash( "regexp" ), + Operator = String::hash( "operator" ), + Decorator = String::hash( "decorator" ), + Member = String::hash( "member" ), + }; +}; + // Ref: // https://microsoft.github.io/language-server-protocol/specification#textDocument_semanticTokens struct SemanticTokensLegend {