From 48d70d11cf94d8792f761061ea6cd97bd112c841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Sat, 7 Oct 2023 01:42:39 -0300 Subject: [PATCH] Clean up. --- include/eepp/ui/doc/syntaxhighlighter.hpp | 3 +- include/eepp/ui/doc/syntaxtokenizer.hpp | 13 ++-- src/eepp/ui/doc/languages/markdown.cpp | 34 --------- src/eepp/ui/doc/syntaxhighlighter.cpp | 9 ++- src/eepp/ui/doc/syntaxtokenizer.cpp | 85 ++++++++++++----------- src/eepp/ui/uicodeeditor.cpp | 11 ++- 6 files changed, 70 insertions(+), 85 deletions(-) diff --git a/include/eepp/ui/doc/syntaxhighlighter.hpp b/include/eepp/ui/doc/syntaxhighlighter.hpp index 726d1a8c7..f9a64ad8f 100644 --- a/include/eepp/ui/doc/syntaxhighlighter.hpp +++ b/include/eepp/ui/doc/syntaxhighlighter.hpp @@ -62,7 +62,8 @@ class EE_API SyntaxHighlighter { void setMaxTokenizationLength( const Int64& maxTokenizationLength ); - void tokenizeAsync( std::shared_ptr pool ); + void tokenizeAsync( std::shared_ptr pool, + const std::function& onDone = {} ); bool isTokenizingAsync() const { return mTokenizeAsync; } diff --git a/include/eepp/ui/doc/syntaxtokenizer.hpp b/include/eepp/ui/doc/syntaxtokenizer.hpp index 2acc8240f..04cffa680 100644 --- a/include/eepp/ui/doc/syntaxtokenizer.hpp +++ b/include/eepp/ui/doc/syntaxtokenizer.hpp @@ -48,12 +48,15 @@ struct SyntaxStateRestored { Uint32 currentLevel{ 0 }; }; +#define MAX_SUB_SYNTAXS 4 + struct SyntaxState { - // 16 bits per pattern - max 4 languages - Uint16 state[4]{ SYNTAX_TOKENIZER_STATE_NONE, SYNTAX_TOKENIZER_STATE_NONE, - SYNTAX_TOKENIZER_STATE_NONE, SYNTAX_TOKENIZER_STATE_NONE }; - // 16 bits per language (language index) - max 4 languages - Uint16 langStack[4]{ 0, 0, 0, 0 }; + // 8 bits per pattern - max 4 sub-languages - max 254 patterns per language + Uint8 state[MAX_SUB_SYNTAXS]{ SYNTAX_TOKENIZER_STATE_NONE, SYNTAX_TOKENIZER_STATE_NONE, + SYNTAX_TOKENIZER_STATE_NONE, SYNTAX_TOKENIZER_STATE_NONE }; + + // 8 bits per language (language index) - max 4 sub-languages - max 255 languages + Uint8 langStack[MAX_SUB_SYNTAXS]{ 0, 0, 0, 0 }; bool operator==( const SyntaxState& other ) { return memcmp( this, &other, sizeof( SyntaxState ) ) == 0; diff --git a/src/eepp/ui/doc/languages/markdown.cpp b/src/eepp/ui/doc/languages/markdown.cpp index cd56dd01f..854728fcc 100644 --- a/src/eepp/ui/doc/languages/markdown.cpp +++ b/src/eepp/ui/doc/languages/markdown.cpp @@ -18,40 +18,6 @@ void addMarkdown() { { "%.md$", "%.markdown$" }, { { { "\\." }, "normal" }, - /*{ { "```[Xx][Mm][Ll]", "```" }, "function", "XML" }, - { { "```[Hh][Tt][Mm][Ll]", "```" }, "function", "html" }, - { { "```[Cc]++", "```" }, "function", "C++" }, - { { "```[Cc][Pp][Pp]", "```" }, "function", "C++" }, - { { "```[Cc]sharp", "```" }, "function", "C#" }, - { { "```[Cc][Ss][Ss]", "```" }, "function", "CSS" }, - { { "```[Cc]", "```" }, "function", "C" }, - { { "```[Dd][Aa][Rr][Tt]", "```" }, "function", "Dart" }, - { { "```[Dd]", "```" }, "function", "D" }, - { { "```[Ll]ua", "```" }, "function", "Lua" }, - { { "```[Jj][Ss][Oo][Nn]", "```" }, "function", "JSON" }, - { { "```[Ja]va[Ss]cript", "```" }, "function", "JavaScript" }, - { { "```[Tt]ype[Ss]cript", "```" }, "function", "TypeScript" }, - { { "```[Pp]ython", "```" }, "function", "Python" }, - { { "```[Bb]ash", "```" }, "function", "Bash" }, - { { "```[Pp][Hh][Pp]", "```" }, "function", "PHPCore" }, - { { "```[Ss][Qq][Ll]", "```" }, "function", "SQL" }, - { { "```[Gg][Ll][Ss][Ll]", "```" }, "function", "GLSL" }, - { { "```[Ii][Nn][Ii]", "```" }, "function", "Config File" }, - { { "```[Mm]akefile", "```" }, "function", "Makefile" }, - { { "```[Gg][Oo]", "```" }, "function", "Go" }, - { { "```[Rr]ust", "```" }, "function", "Rust" }, - { { "```[Rr]uby", "```" }, "function", "Ruby" }, - { { "```[Gg][Dd][Ss]cript", "```" }, "function", "GSCript" }, - { { "```[Jj]ava", "```" }, "function", "java" }, - { { "```[Ss]wift", "```" }, "function", "Swift" }, - { { "```[Oo]bjective[Cc]", "```" }, "function", "Objective-C" }, - { { "```[Yy][Aa][Mm][Ll]", "```" }, "function", "YAML" }, - { { "```[Kk]otlin", "```" }, "function", "Kotlin" }, - { { "```[Ss]olidity", "```" }, "function", "Solidity" }, - { { "```[Hh]askell", "```" }, "function", "Haskell" }, - { { "```[Oo]din", "```" }, "function", "Odin" }, - { { "```[Nn]im", "```" }, "function", "Nim" }, - { { "```[Zz]ig", "```" }, "function", "Zig" },*/ { { "```[%w%s+-#]+", "```" }, "function", dynSyntax }, { { "" }, "comment" }, { { "```", "```" }, "string" }, diff --git a/src/eepp/ui/doc/syntaxhighlighter.cpp b/src/eepp/ui/doc/syntaxhighlighter.cpp index 7dcb1e493..1d31db395 100644 --- a/src/eepp/ui/doc/syntaxhighlighter.cpp +++ b/src/eepp/ui/doc/syntaxhighlighter.cpp @@ -123,15 +123,18 @@ void SyntaxHighlighter::setMaxTokenizationLength( const Int64& maxTokenizationLe mMaxTokenizationLength = maxTokenizationLength; } -void SyntaxHighlighter::tokenizeAsync( std::shared_ptr pool ) { +void SyntaxHighlighter::tokenizeAsync( std::shared_ptr pool, + const std::function& onDone ) { if ( mTokenizeAsync ) return; mTokenizeAsync = true; - pool->run( [this] { + pool->run( [this, onDone] { for ( size_t i = mFirstInvalidLine; i < mDoc->linesCount() && !mStopTokenizing; i++ ) getLine( i ); mStopTokenizing = false; mTokenizeAsync = false; + if ( onDone ) + onDone(); } ); } @@ -225,7 +228,7 @@ SyntaxHighlighter::getSyntaxDefinitionFromTextPosition( const TextPosition& posi if ( found == mLines.end() ) return SyntaxDefinitionManager::instance()->getPlainDefinition(); - TokenizedLine& line = found->second; + const TokenizedLine& line = found->second; SyntaxStateRestored state = SyntaxTokenizer::retrieveSyntaxState( mDoc->getSyntaxDefinition(), line.state ); diff --git a/src/eepp/ui/doc/syntaxtokenizer.cpp b/src/eepp/ui/doc/syntaxtokenizer.cpp index a822dc842..f7fd581d4 100644 --- a/src/eepp/ui/doc/syntaxtokenizer.cpp +++ b/src/eepp/ui/doc/syntaxtokenizer.cpp @@ -97,6 +97,9 @@ bool isScaped( const std::string& text, const size_t& startIndex, const std::str std::pair findNonEscaped( const std::string& text, const std::string& pattern, int offset, const std::string& escapeStr ) { + eeASSERT( !pattern.empty() ); + if ( pattern.empty() ) + return std::make_pair( -1, -1 ); LuaPattern words( pattern ); int start, end; while ( words.find( text, start, end, offset ) ) { @@ -142,53 +145,55 @@ SyntaxStateRestored SyntaxTokenizer::retrieveSyntaxState( const SyntaxDefinition return syntaxState; } +static inline void setSubsyntaxPatternIdx( SyntaxStateRestored& curState, SyntaxState& retState, + const Uint32& patternIndex ) { + curState.currentPatternIdx = patternIndex; + retState.state[curState.currentLevel] = patternIndex; +}; + +static inline void pushSubsyntax( SyntaxStateRestored& curState, SyntaxState& retState, + const SyntaxPattern& enteringSubsyntax, + const Uint32& patternIndex, const std::string& patternStr ) { + if ( curState.currentLevel == MAX_SUB_SYNTAXS - 1 ) + return; + setSubsyntaxPatternIdx( curState, retState, patternIndex ); + curState.subsyntaxInfo = &enteringSubsyntax; + curState.currentSyntax = &SyntaxDefinitionManager::instance()->getByLanguageName( + curState.subsyntaxInfo->dynSyntax + ? curState.subsyntaxInfo->dynSyntax( enteringSubsyntax, patternStr ) + : curState.subsyntaxInfo->syntax ); + retState.langStack[curState.currentLevel] = curState.currentSyntax->getLanguageIndex(); + curState.currentLevel++; + setSubsyntaxPatternIdx( curState, retState, SYNTAX_TOKENIZER_STATE_NONE ); +}; + +static inline void popSubsyntax( SyntaxStateRestored& curState, SyntaxState& retState, + const SyntaxDefinition& syntax ) { + setSubsyntaxPatternIdx( curState, retState, SYNTAX_TOKENIZER_STATE_NONE ); + retState.langStack[curState.currentLevel] = 0; + curState.currentLevel--; + setSubsyntaxPatternIdx( curState, retState, SYNTAX_TOKENIZER_STATE_NONE ); + curState = SyntaxTokenizer::retrieveSyntaxState( syntax, retState ); +}; + template static inline std::pair, SyntaxState> _tokenize( const SyntaxDefinition& syntax, const std::string& text, const SyntaxState& state, const size_t& startIndex, bool skipSubSyntaxSeparator ) { std::vector tokens; - LuaPattern::Range matches[12]; - int start, end; - size_t numMatches; if ( syntax.getPatterns().empty() ) { pushToken( tokens, SyntaxStyleTypes::Normal, text ); return std::make_pair( std::move( tokens ), SyntaxState{} ); } + LuaPattern::Range matches[12]; + int start, end; + size_t numMatches; size_t i = startIndex; SyntaxState retState = state; SyntaxStateRestored curState = SyntaxTokenizer::retrieveSyntaxState( syntax, state ); - auto setSubsyntaxPatternIdx = [&curState, &retState]( const Uint32& patternIndex ) { - curState.currentPatternIdx = patternIndex; - retState.state[curState.currentLevel] = patternIndex; - }; - - auto pushSubsyntax = [&setSubsyntaxPatternIdx, &curState, - &retState]( const SyntaxPattern& enteringSubsyntax, - const Uint32& patternIndex, const std::string& patternStr ) { - if ( curState.currentLevel == 3 ) - return; - setSubsyntaxPatternIdx( patternIndex ); - curState.subsyntaxInfo = &enteringSubsyntax; - curState.currentSyntax = &SyntaxDefinitionManager::instance()->getByLanguageName( - curState.subsyntaxInfo->dynSyntax - ? curState.subsyntaxInfo->dynSyntax( enteringSubsyntax, patternStr ) - : curState.subsyntaxInfo->syntax ); - retState.langStack[curState.currentLevel] = curState.currentSyntax->getLanguageIndex(); - curState.currentLevel++; - setSubsyntaxPatternIdx( SYNTAX_TOKENIZER_STATE_NONE ); - }; - - auto popSubsyntax = [&setSubsyntaxPatternIdx, &curState, &syntax, &retState]() { - setSubsyntaxPatternIdx( SYNTAX_TOKENIZER_STATE_NONE ); - retState.langStack[curState.currentLevel] = 0; - curState.currentLevel--; - setSubsyntaxPatternIdx( SYNTAX_TOKENIZER_STATE_NONE ); - curState = SyntaxTokenizer::retrieveSyntaxState( syntax, retState ); - }; - size_t size = !text.empty() ? ( text[text.size() - 1] == '\n' ? text.size() - 1 : text.size() ) : 0; // skip last char ( new line char ) std::string patternStr; @@ -217,7 +222,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax pushToken( tokens, curState.subsyntaxInfo->types[0], text.substr( i, rangeSubsyntax.second - i ) ); } - popSubsyntax(); + popSubsyntax( curState, retState, syntax ); i = rangeSubsyntax.second; skip = true; } @@ -226,7 +231,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax if ( !skip ) { if ( range.first != -1 ) { pushToken( tokens, pattern.types[0], text.substr( i, range.second - i ) ); - setSubsyntaxPatternIdx( SYNTAX_TOKENIZER_STATE_NONE ); + setSubsyntaxPatternIdx( curState, retState, SYNTAX_TOKENIZER_STATE_NONE ); i = range.second; } else { pushToken( tokens, pattern.types[0], text.substr( i ) ); @@ -246,7 +251,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax pushToken( tokens, curState.subsyntaxInfo->types[0], text.substr( i, rangeSubsyntax.second - i ) ); } - popSubsyntax(); + popSubsyntax( curState, retState, syntax ); i = rangeSubsyntax.second; } } @@ -306,9 +311,10 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax } if ( pattern.hasSyntax() ) { - pushSubsyntax( pattern, patternIndex + 1, patternStr ); + pushSubsyntax( curState, retState, pattern, patternIndex + 1, + patternStr ); } else if ( pattern.patterns.size() > 1 ) { - setSubsyntaxPatternIdx( patternIndex + 1 ); + setSubsyntaxPatternIdx( curState, retState, patternIndex + 1 ); } i = end; @@ -350,9 +356,10 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax patternText ); } if ( pattern.hasSyntax() ) { - pushSubsyntax( pattern, patternIndex + 1, patternText ); + pushSubsyntax( curState, retState, pattern, patternIndex + 1, + patternText ); } else if ( pattern.patterns.size() > 1 ) { - setSubsyntaxPatternIdx( patternIndex + 1 ); + setSubsyntaxPatternIdx( curState, retState, patternIndex + 1 ); } i = end; matched = true; diff --git a/src/eepp/ui/uicodeeditor.cpp b/src/eepp/ui/uicodeeditor.cpp index 9b189b27a..9e6a2b90f 100644 --- a/src/eepp/ui/uicodeeditor.cpp +++ b/src/eepp/ui/uicodeeditor.cpp @@ -453,8 +453,11 @@ bool UICodeEditor::loadAsyncFromFile( } ); return; } - if ( mMinimapEnabled && getUISceneNode()->hasThreadPool() ) - mDoc->getHighlighter()->tokenizeAsync( getUISceneNode()->getThreadPool() ); + if ( mMinimapEnabled && getUISceneNode()->hasThreadPool() ) { + mDoc->getHighlighter()->tokenizeAsync( getUISceneNode()->getThreadPool(), [this] { + runOnMainThread( [this] { invalidateDraw(); } ); + } ); + } runOnMainThread( [&, onLoaded, wasLocked, success] { invalidateEditor(); invalidateDraw(); @@ -492,7 +495,9 @@ bool UICodeEditor::loadAsyncFromURL( url, headers, [this, onLoaded, wasLocked]( TextDocument*, bool success ) { if ( mMinimapEnabled && getUISceneNode()->hasThreadPool() ) - mDoc->getHighlighter()->tokenizeAsync( getUISceneNode()->getThreadPool() ); + mDoc->getHighlighter()->tokenizeAsync( getUISceneNode()->getThreadPool(), [this] { + runOnMainThread( [this] { invalidateDraw(); } ); + } ); runOnMainThread( [&, onLoaded, wasLocked] { invalidateEditor(); updateLongestLineWidth();