From e62b42267f4ad1615d9ac8fa6b3dbe4d05ec16fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Sun, 2 Apr 2023 14:34:58 -0300 Subject: [PATCH] SyntaxHighlighter and semantic highlighting optimizations and improvements. Now the highlighters detect new lines and remove lines operations and translate all the already processed highlighted lines to it's new position, this means that only the lines with new content are processed again for highlighting tokenization. --- .gitignore | 2 + include/eepp/ui/doc/syntaxhighlighter.hpp | 7 +++ include/eepp/ui/doc/textdocument.hpp | 4 ++ src/eepp/ui/doc/syntaxhighlighter.cpp | 47 ++++++++++++++++ src/eepp/ui/doc/textdocument.cpp | 19 ++++++- .../ecode/plugins/lsp/lspdocumentclient.cpp | 54 +++++++++++++++++-- .../ecode/plugins/lsp/lspdocumentclient.hpp | 3 ++ 7 files changed, 130 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 9399bcb12..ec84ed505 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,5 @@ ecode.dmg /projects/**/*.zip /.ecode/.fstreeviewignore /.ecode/.prjallowed +/.cache/ +/compile_commands.json diff --git a/include/eepp/ui/doc/syntaxhighlighter.hpp b/include/eepp/ui/doc/syntaxhighlighter.hpp index 4aee18498..226b74a44 100644 --- a/include/eepp/ui/doc/syntaxhighlighter.hpp +++ b/include/eepp/ui/doc/syntaxhighlighter.hpp @@ -12,6 +12,11 @@ struct TokenizedLine { String::HashType hash; std::vector tokens; Uint64 state{ SYNTAX_TOKENIZER_STATE_NONE }; + Uint64 signature { 0 }; + + void updateSignature(); + + static Uint64 calcSignature( const std::vector& tokens ); }; class EE_API SyntaxHighlighter { @@ -47,6 +52,8 @@ class EE_API SyntaxHighlighter { Mutex& getLinesMutex(); + void moveHighlight( const Int64& fromLine, const Int64& numLines ); + protected: TextDocument* mDoc; std::unordered_map mLines; diff --git a/include/eepp/ui/doc/textdocument.hpp b/include/eepp/ui/doc/textdocument.hpp index ba8fc71e4..ef738793e 100644 --- a/include/eepp/ui/doc/textdocument.hpp +++ b/include/eepp/ui/doc/textdocument.hpp @@ -87,6 +87,8 @@ class EE_API TextDocument { onDocumentLoaded( doc ); } virtual void onDocumentSyntaxDefinitionChange( const SyntaxDefinition& ) {} + virtual void onDocumentMoveHighlight( const Int64& /*fromLine*/, + const Int64& /*numLines*/ ){}; }; TextDocument( bool verbose = true ); @@ -632,6 +634,8 @@ class EE_API TextDocument { void notifySyntaxDefinitionChange(); + void notifiyDocumentMoveHighlight( const Int64& fromLine, const Int64& numLines ); + void insertAtStartOfSelectedLines( const String& text, bool skipEmpty ); void removeFromStartOfSelectedLines( const String& text, bool skipEmpty, diff --git a/src/eepp/ui/doc/syntaxhighlighter.cpp b/src/eepp/ui/doc/syntaxhighlighter.cpp index f3f58575f..c2d2e88c8 100644 --- a/src/eepp/ui/doc/syntaxhighlighter.cpp +++ b/src/eepp/ui/doc/syntaxhighlighter.cpp @@ -1,9 +1,27 @@ +#include #include #include #include namespace EE { namespace UI { namespace Doc { +static constexpr void _hash( Uint64& signature, const String::HashType& val ) { + Int64 len = sizeof( decltype( val ) ); + while ( --len >= 0 ) + signature = ( ( signature << 5 ) + signature ) + ( ( val >> ( len * 8 ) ) & 0xFF ); +} + +Uint64 TokenizedLine::calcSignature( const std::vector& tokens ) { + Uint64 signature = 5381; + for ( const auto& token : tokens ) + _hash( signature, String::hash( token.type ) ); + return signature; +} + +void TokenizedLine::updateSignature() { + this->signature = calcSignature( tokens ); +} + SyntaxHighlighter::SyntaxHighlighter( TextDocument* doc ) : mDoc( doc ), mFirstInvalidLine( 0 ), mMaxWantedLine( 0 ) { reset(); @@ -35,6 +53,7 @@ TokenizedLine SyntaxHighlighter::tokenizeLine( const size_t& line, const Uint64& mDoc->line( line ).toUtf8(), state ); tokenizedLine.tokens = std::move( res.first ); tokenizedLine.state = std::move( res.second ); + tokenizedLine.updateSignature(); return tokenizedLine; } @@ -42,6 +61,34 @@ Mutex& SyntaxHighlighter::getLinesMutex() { return mLinesMutex; } +void SyntaxHighlighter::moveHighlight( const Int64& fromLine, const Int64& numLines ) { + if ( mLines.find( fromLine ) == mLines.end() ) + return; + Int64 linesCount = mDoc->linesCount(); + if ( numLines > 0 ) { + for ( Int64 i = linesCount - 1; i >= fromLine; --i ) { + auto lineIt = mLines.find( i - numLines ); + if ( lineIt != mLines.end() ) { + auto& line = lineIt->second; + if ( line.hash == mDoc->line( i ).getHash() ) { + auto nl = mLines.extract( lineIt ); + nl.key() = i; + mLines.insert( std::move( nl ) ); + } + } + } + } else if ( numLines < 0 ) { + for ( Int64 i = fromLine; i < linesCount; i++ ) { + auto lineIt = mLines.find( i - numLines ); + if ( lineIt != mLines.end() && lineIt->second.hash == mDoc->line( i ).getHash() ) { + auto nl = mLines.extract( lineIt ); + nl.key() = i; + mLines[i] = std::move( nl.mapped() ); + } + } + } +} + const std::vector& SyntaxHighlighter::getLine( const size_t& index ) { if ( mDoc->getSyntaxDefinition().getPatterns().empty() ) { static std::vector noHighlightVector = { { "normal", 0 } }; diff --git a/src/eepp/ui/doc/textdocument.cpp b/src/eepp/ui/doc/textdocument.cpp index 142708bc1..ec2bbdff8 100644 --- a/src/eepp/ui/doc/textdocument.cpp +++ b/src/eepp/ui/doc/textdocument.cpp @@ -915,6 +915,11 @@ TextPosition TextDocument::insert( const size_t& cursorIdx, TextPosition positio mUndoStack.pushSelection( undoStack, cursorIdx, mSelection, time ); mUndoStack.pushRemove( undoStack, cursorIdx, { position, cursor }, time ); + if ( linesAdd > 0 ) { + mHighlighter->moveHighlight( position.line(), linesAdd ); + notifiyDocumentMoveHighlight( position.line(), linesAdd ); + } + notifyTextChanged( { { position, position }, text } ); if ( lineCount != mLines.size() ) { @@ -982,8 +987,8 @@ size_t TextDocument::remove( const size_t& cursorIdx, TextRange range, if ( range.start().line() + 1 < range.end().line() ) { mLines.erase( mLines.begin() + range.start().line() + 1, mLines.begin() + range.end().line() ); - range.end().setLine( range.start().line() + 1 ); linesRemoved = range.end().line() - ( range.start().line() + 1 ); + range.end().setLine( range.start().line() + 1 ); } if ( range.start().line() == range.end().line() ) { @@ -1055,6 +1060,11 @@ size_t TextDocument::remove( const size_t& cursorIdx, TextRange range, } } + if ( linesRemoved > 0 ) { + mHighlighter->moveHighlight( range.start().line(), -linesRemoved ); + notifiyDocumentMoveHighlight( range.start().line(), -linesRemoved ); + } + notifyTextChanged( { originalRange, "" } ); notifyLineChanged( range.start().line() ); @@ -2588,6 +2598,13 @@ void TextDocument::notifySyntaxDefinitionChange() { } } +void TextDocument::notifiyDocumentMoveHighlight( const Int64& fromLine, const Int64& numLines ) { + Lock l( mClientsMutex ); + for ( auto& client : mClients ) { + client->onDocumentMoveHighlight( fromLine, numLines ); + } +} + void TextDocument::initializeCommands() { mCommands["reset"] = [&] { reset(); }; mCommands["save"] = [&] { save(); }; diff --git a/src/tools/ecode/plugins/lsp/lspdocumentclient.cpp b/src/tools/ecode/plugins/lsp/lspdocumentclient.cpp index 03fc00443..95b0d838e 100644 --- a/src/tools/ecode/plugins/lsp/lspdocumentclient.cpp +++ b/src/tools/ecode/plugins/lsp/lspdocumentclient.cpp @@ -151,7 +151,7 @@ void LSPDocumentClient::requestSemanticHighlightingDelayed() { UISceneNode* sceneNode = getUISceneNode(); if ( sceneNode ) { sceneNode->removeActionsByTag( mTagSemanticTokens ); - sceneNode->runOnMainThread( [this]() { requestSemanticHighlighting(); }, Seconds( 0.1f ), + sceneNode->runOnMainThread( [this]() { requestSemanticHighlighting(); }, Seconds( 0.5f ), mTagSemanticTokens ); } } @@ -233,6 +233,31 @@ void LSPDocumentClient::processTokens( const LSPSemanticTokensDelta& tokens ) { mRunningSemanticTokens = false; } +void LSPDocumentClient::onDocumentMoveHighlight( const Int64& fromLine, const Int64& numLines ) { + Int64 linesCount = mDoc->linesCount(); + if ( numLines > 0 ) { + for ( Int64 i = linesCount - 1; i >= fromLine; --i ) { + auto lineIt = mTokenizerLines.find( i - numLines ); + if ( lineIt != mTokenizerLines.end() && + lineIt->second.hash == mDoc->line( i ).getHash() ) { + auto nl = mTokenizerLines.extract( lineIt ); + nl.key() = i; + mTokenizerLines.insert( std::move( nl ) ); + } + } + } else if ( numLines < 0 ) { + for ( Int64 i = fromLine; i < linesCount; i++ ) { + auto lineIt = mTokenizerLines.find( i - numLines ); + if ( lineIt != mTokenizerLines.end() && + lineIt->second.hash == mDoc->line( i ).getHash() ) { + auto nl = mTokenizerLines.extract( lineIt ); + nl.key() = i; + mTokenizerLines[i] = std::move( nl.mapped() ); + } + } + } +} + void LSPDocumentClient::highlight() { if ( mShutdown ) return; @@ -243,11 +268,13 @@ void LSPDocumentClient::highlight() { mDoc->getURI().toString().c_str() ); return; } - + Clock clock; const auto& caps = mServer->getCapabilities().semanticTokenProvider; Uint32 currentLine = 0; Uint32 start = 0; - std::map tokenizedLines; + std::unordered_map tokenizerLines; + Int64 lastLine = 0; + TokenizedLine* lastLinePtr = nullptr; for ( size_t i = 0; i < data.size(); i += 5 ) { if ( mShutdown ) return; @@ -264,18 +291,35 @@ void LSPDocumentClient::highlight() { start = deltaStart; } - auto& line = tokenizedLines[currentLine]; + auto& line = tokenizerLines[currentLine]; const auto& ltype = caps.legend.tokenTypes[type]; line.tokens.push_back( { semanticTokenTypeToSyntaxType( ltype, mDoc->getSyntaxDefinition() ), start, len } ); line.hash = mDoc->line( currentLine ).getHash(); + line.updateSignature(); + + if ( lastLine != currentLine && nullptr != lastLinePtr ) { + auto lastLineTokIt = mTokenizerLines.find( lastLine ); + if ( lastLineTokIt != mTokenizerLines.end() && + lastLinePtr->signature == lastLineTokIt->second.signature ) { + tokenizerLines.erase( lastLine ); + } + } + + lastLine = currentLine; + lastLinePtr = &line; } - for ( const auto& tline : tokenizedLines ) { + for ( auto& tline : tokenizerLines ) { if ( mShutdown ) return; + mDoc->getHighlighter()->mergeLine( tline.first, tline.second ); + + mTokenizerLines[tline.first] = std::move( tline.second ); } + Log::debug( "LSPDocumentClient::highlight took: %.2f ms", + clock.getElapsedTime().asMilliseconds() ); } void LSPDocumentClient::notifyOpen() { diff --git a/src/tools/ecode/plugins/lsp/lspdocumentclient.hpp b/src/tools/ecode/plugins/lsp/lspdocumentclient.hpp index 35952f12a..e7ab596b8 100644 --- a/src/tools/ecode/plugins/lsp/lspdocumentclient.hpp +++ b/src/tools/ecode/plugins/lsp/lspdocumentclient.hpp @@ -3,6 +3,7 @@ #include "lspprotocol.hpp" #include +#include #include using namespace EE; @@ -36,6 +37,7 @@ class LSPDocumentClient : public TextDocument::Client { virtual void onDocumentDirtyOnFileSystem( TextDocument* ); virtual void onDocumentMoved( TextDocument* ); virtual void onDocumentReloaded( TextDocument* ); + virtual void onDocumentMoveHighlight( const Int64& fromLine, const Int64& numLines ); void notifyOpen(); @@ -57,6 +59,7 @@ class LSPDocumentClient : public TextDocument::Client { LSPSemanticTokensDelta mSemanticTokens; bool mRunningSemanticTokens{ false }; bool mShutdown{ false }; + std::unordered_map mTokenizerLines; void refreshTag();