From a44328f5266edd4ebc0c7151e4be2f65afd6a4c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Sat, 7 Oct 2023 00:55:09 -0300 Subject: [PATCH] Reenable dyn syntax. --- include/eepp/ui/doc/syntaxdefinition.hpp | 5 +++ .../eepp/ui/doc/syntaxdefinitionmanager.hpp | 2 ++ include/eepp/ui/doc/syntaxtokenizer.hpp | 9 +++-- src/eepp/ui/doc/languages/markdown.cpp | 10 +++--- src/eepp/ui/doc/syntaxdefinitionmanager.cpp | 8 +++++ src/eepp/ui/doc/syntaxtokenizer.cpp | 34 +++++++++---------- 6 files changed, 43 insertions(+), 25 deletions(-) diff --git a/include/eepp/ui/doc/syntaxdefinition.hpp b/include/eepp/ui/doc/syntaxdefinition.hpp index 3788243c4..cdcaa6442 100644 --- a/include/eepp/ui/doc/syntaxdefinition.hpp +++ b/include/eepp/ui/doc/syntaxdefinition.hpp @@ -147,7 +147,11 @@ class EE_API SyntaxDefinition { UnorderedMap getSymbolNames() const; + const Uint16& getLanguageIndex() const { return mLanguageIndex; } + protected: + friend class SyntaxDefinitionManager; + std::string mLanguageName; String::HashType mLanguageId; std::vector mFiles; @@ -157,6 +161,7 @@ class EE_API SyntaxDefinition { std::string mComment; std::vector mHeaders; std::string mLSPName; + Uint16 mLanguageIndex{ 0 }; bool mAutoCloseXMLTags{ false }; bool mVisible{ true }; bool mHasExtensionPriority{ false }; diff --git a/include/eepp/ui/doc/syntaxdefinitionmanager.hpp b/include/eepp/ui/doc/syntaxdefinitionmanager.hpp index dc32e9951..a50dc6515 100644 --- a/include/eepp/ui/doc/syntaxdefinitionmanager.hpp +++ b/include/eepp/ui/doc/syntaxdefinitionmanager.hpp @@ -41,6 +41,8 @@ class EE_API SyntaxDefinitionManager { const SyntaxDefinition& getByLanguageName( const std::string& name ) const; + const SyntaxDefinition& getByLanguageIndex( const Uint32& index ) const; + const SyntaxDefinition& getByLanguageNameInsensitive( std::string name ) const; const SyntaxDefinition& getByLSPName( const std::string& name ) const; diff --git a/include/eepp/ui/doc/syntaxtokenizer.hpp b/include/eepp/ui/doc/syntaxtokenizer.hpp index cb741cc04..2acc8240f 100644 --- a/include/eepp/ui/doc/syntaxtokenizer.hpp +++ b/include/eepp/ui/doc/syntaxtokenizer.hpp @@ -49,11 +49,14 @@ struct SyntaxStateRestored { }; struct SyntaxState { - Uint32 state{ SYNTAX_TOKENIZER_STATE_NONE }; - String::HashType hash{ 0 }; + // 16 bits per pattern - max 4 languages + Uint16 state[4]{ SYNTAX_TOKENIZER_STATE_NONE, SYNTAX_TOKENIZER_STATE_NONE, + SYNTAX_TOKENIZER_STATE_NONE, SYNTAX_TOKENIZER_STATE_NONE }; + // 16 bits per language (language index) - max 4 languages + Uint16 langStack[4]{ 0, 0, 0, 0 }; bool operator==( const SyntaxState& other ) { - return state == other.state && hash == other.hash; + return memcmp( this, &other, sizeof( SyntaxState ) ) == 0; } bool operator!=( const SyntaxState& other ) { return !( *this == other ); } diff --git a/src/eepp/ui/doc/languages/markdown.cpp b/src/eepp/ui/doc/languages/markdown.cpp index 6d50e84ee..cd56dd01f 100644 --- a/src/eepp/ui/doc/languages/markdown.cpp +++ b/src/eepp/ui/doc/languages/markdown.cpp @@ -4,13 +4,13 @@ namespace EE { namespace UI { namespace Doc { namespace Language { void addMarkdown() { - /* auto dynSyntax = []( const SyntaxPattern&, const std::string_view& match ) -> std::string { + auto dynSyntax = []( const SyntaxPattern&, const std::string_view& match ) -> std::string { std::string lang = String::toLower( std::string{ match.substr( 3 ) } ); String::trimInPlace( lang ); if ( !lang.empty() && lang[lang.size() - 1] == '\n' ) lang.pop_back(); return SyntaxDefinitionManager::instance()->findFromString( lang ).getLanguageName(); - }; */ + }; SyntaxDefinitionManager::instance()->add( @@ -18,7 +18,7 @@ void addMarkdown() { { "%.md$", "%.markdown$" }, { { { "\\." }, "normal" }, - { { "```[Xx][Mm][Ll]", "```" }, "function", "XML" }, + /*{ { "```[Xx][Mm][Ll]", "```" }, "function", "XML" }, { { "```[Hh][Tt][Mm][Ll]", "```" }, "function", "html" }, { { "```[Cc]++", "```" }, "function", "C++" }, { { "```[Cc][Pp][Pp]", "```" }, "function", "C++" }, @@ -51,8 +51,8 @@ void addMarkdown() { { { "```[Hh]askell", "```" }, "function", "Haskell" }, { { "```[Oo]din", "```" }, "function", "Odin" }, { { "```[Nn]im", "```" }, "function", "Nim" }, - { { "```[Zz]ig", "```" }, "function", "Zig" }, - // { { "```[%w%s+-#]+", "```" }, "function", dynSyntax }, + { { "```[Zz]ig", "```" }, "function", "Zig" },*/ + { { "```[%w%s+-#]+", "```" }, "function", dynSyntax }, { { "" }, "comment" }, { { "```", "```" }, "string" }, { { "``", "``" }, "string" }, diff --git a/src/eepp/ui/doc/syntaxdefinitionmanager.cpp b/src/eepp/ui/doc/syntaxdefinitionmanager.cpp index f026ce235..d8bc61aa7 100644 --- a/src/eepp/ui/doc/syntaxdefinitionmanager.cpp +++ b/src/eepp/ui/doc/syntaxdefinitionmanager.cpp @@ -1911,6 +1911,7 @@ namespace EE { namespace UI { namespace Doc { namespace Language { } SyntaxDefinition& SyntaxDefinitionManager::add( SyntaxDefinition&& syntaxStyle ) { + syntaxStyle.mLanguageIndex = mDefinitions.size(); mDefinitions.emplace_back( std::move( syntaxStyle ) ); return mDefinitions.back(); } @@ -1932,6 +1933,11 @@ SyntaxDefinitionManager::getByLanguageName( const std::string& name ) const { return mDefinitions[0]; } +const SyntaxDefinition& SyntaxDefinitionManager::getByLanguageIndex( const Uint32& index ) const { + eeASSERT( index < mDefinitions.size() ); + return mDefinitions[index]; +} + const SyntaxDefinition& SyntaxDefinitionManager::getByLanguageNameInsensitive( std::string name ) const { String::toLowerInPlace( name ); @@ -2116,6 +2122,7 @@ bool SyntaxDefinitionManager::loadFromStream( IOStream& stream, } else { if ( addedLangs ) addedLangs->push_back( res.getLanguageName() ); + res.mLanguageIndex = mDefinitions.size(); mDefinitions.emplace_back( std::move( res ) ); } } @@ -2131,6 +2138,7 @@ bool SyntaxDefinitionManager::loadFromStream( IOStream& stream, } else { if ( addedLangs ) addedLangs->push_back( res.getLanguageName() ); + res.mLanguageIndex = mDefinitions.size(); mDefinitions.emplace_back( std::move( res ) ); } } diff --git a/src/eepp/ui/doc/syntaxtokenizer.cpp b/src/eepp/ui/doc/syntaxtokenizer.cpp index 1caba6539..a822dc842 100644 --- a/src/eepp/ui/doc/syntaxtokenizer.cpp +++ b/src/eepp/ui/doc/syntaxtokenizer.cpp @@ -111,21 +111,22 @@ std::pair findNonEscaped( const std::string& text, const std::string& SyntaxStateRestored SyntaxTokenizer::retrieveSyntaxState( const SyntaxDefinition& syntax, const SyntaxState& state ) { - SyntaxStateRestored syntaxState{ &syntax, nullptr, state.state, 0 }; - if ( state.state > 0 && - ( state.state > 255 || - ( state.state < syntaxState.currentSyntax->getPatterns().size() && - syntaxState.currentSyntax->getPatterns()[state.state - 1].hasSyntax() ) ) ) { + SyntaxStateRestored syntaxState{ &syntax, nullptr, state.state[0], 0 }; + if ( state.state[0] > 0 && + ( state.state[1] > 0 || + ( state.state[0] < syntaxState.currentSyntax->getPatterns().size() && + syntaxState.currentSyntax->getPatterns()[state.state[0] - 1].hasSyntax() ) ) ) { for ( size_t i = 0; i <= 2; ++i ) { - Uint32 target = ( state.state >> ( i << 3 ) ) & 0xFF; + Uint32 target = state.state[i]; if ( target != SYNTAX_TOKENIZER_STATE_NONE ) { if ( target < syntaxState.currentSyntax->getPatterns().size() && syntaxState.currentSyntax->getPatterns()[target - 1].hasSyntax() ) { syntaxState.subsyntaxInfo = &syntaxState.currentSyntax->getPatterns()[target - 1]; + Uint32 langIndex = state.langStack[i]; syntaxState.currentSyntax = - state.hash != 0 - ? &SyntaxDefinitionManager::instance()->getByLanguageId( state.hash ) + langIndex != 0 + ? &SyntaxDefinitionManager::instance()->getByLanguageIndex( langIndex ) : &SyntaxDefinitionManager::instance()->getByLanguageName( syntaxState.subsyntaxInfo->syntax ); syntaxState.currentPatternIdx = SYNTAX_TOKENIZER_STATE_NONE; @@ -161,30 +162,30 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax auto setSubsyntaxPatternIdx = [&curState, &retState]( const Uint32& patternIndex ) { curState.currentPatternIdx = patternIndex; - retState.state &= ~( 0xFF << ( curState.currentLevel << 3 ) ); - retState.state |= ( patternIndex << ( curState.currentLevel << 3 ) ); + retState.state[curState.currentLevel] = patternIndex; }; auto pushSubsyntax = [&setSubsyntaxPatternIdx, &curState, &retState]( const SyntaxPattern& enteringSubsyntax, const Uint32& patternIndex, const std::string& patternStr ) { + if ( curState.currentLevel == 3 ) + return; setSubsyntaxPatternIdx( patternIndex ); - curState.currentLevel++; curState.subsyntaxInfo = &enteringSubsyntax; curState.currentSyntax = &SyntaxDefinitionManager::instance()->getByLanguageName( curState.subsyntaxInfo->dynSyntax ? curState.subsyntaxInfo->dynSyntax( enteringSubsyntax, patternStr ) : curState.subsyntaxInfo->syntax ); - if ( curState.subsyntaxInfo->dynSyntax ) - retState.hash = curState.currentSyntax->getLanguageId(); + retState.langStack[curState.currentLevel] = curState.currentSyntax->getLanguageIndex(); + curState.currentLevel++; setSubsyntaxPatternIdx( SYNTAX_TOKENIZER_STATE_NONE ); }; auto popSubsyntax = [&setSubsyntaxPatternIdx, &curState, &syntax, &retState]() { setSubsyntaxPatternIdx( SYNTAX_TOKENIZER_STATE_NONE ); + retState.langStack[curState.currentLevel] = 0; curState.currentLevel--; setSubsyntaxPatternIdx( SYNTAX_TOKENIZER_STATE_NONE ); - retState.hash = 0; curState = SyntaxTokenizer::retrieveSyntaxState( syntax, retState ); }; @@ -408,9 +409,8 @@ Text& SyntaxTokenizer::tokenizeText( const SyntaxDefinition& syntax, const size_t& startIndex, const size_t& endIndex, bool skipSubSyntaxSeparator, const std::string& trimChars ) { - auto tokens = SyntaxTokenizer::tokenizeComplete( syntax, text.getString(), - { SYNTAX_TOKENIZER_STATE_NONE, 0 }, startIndex, - skipSubSyntaxSeparator ) + auto tokens = SyntaxTokenizer::tokenizeComplete( syntax, text.getString(), SyntaxState{}, + startIndex, skipSubSyntaxSeparator ) .first; if ( skipSubSyntaxSeparator || !trimChars.empty() ) {