diff --git a/include/eepp/ui/doc/syntaxdefinition.hpp b/include/eepp/ui/doc/syntaxdefinition.hpp index 3bcd9a7cf..a72c46366 100644 --- a/include/eepp/ui/doc/syntaxdefinition.hpp +++ b/include/eepp/ui/doc/syntaxdefinition.hpp @@ -4,10 +4,8 @@ #include #include #include -#include #include #include -#include #include namespace EE { namespace UI { namespace Doc { @@ -28,11 +26,13 @@ template static auto toSyntaxStyleTypeV( const std::vector& s ) } struct EE_API SyntaxPattern { + using DynamicSyntax = std::function; + std::vector patterns; std::vector types; std::vector typesNames; - std::string syntax{ "" }; + DynamicSyntax dynSyntax; SyntaxPattern( std::vector&& _patterns, const std::string& _type, const std::string& _syntax = "" ) : @@ -47,6 +47,22 @@ struct EE_API SyntaxPattern { types( toSyntaxStyleTypeV( _types ) ), typesNames( std::move( _types ) ), syntax( _syntax ) {} + + SyntaxPattern( std::vector&& _patterns, const std::string& _type, + DynamicSyntax&& _syntax ) : + patterns( std::move( _patterns ) ), + types( toSyntaxStyleTypeV( std::vector{ _type } ) ), + typesNames( { _type } ), + dynSyntax( std::move( _syntax ) ) {} + + SyntaxPattern( std::vector&& _patterns, std::vector&& _types, + DynamicSyntax&& _syntax ) : + patterns( std::move( _patterns ) ), + types( toSyntaxStyleTypeV( _types ) ), + typesNames( std::move( _types ) ), + dynSyntax( std::move( _syntax ) ) {} + + bool hasSyntax() const { return !syntax.empty() || dynSyntax; } }; class EE_API SyntaxDefinition { diff --git a/include/eepp/ui/doc/syntaxdefinitionmanager.hpp b/include/eepp/ui/doc/syntaxdefinitionmanager.hpp index ea4b98b14..dc32e9951 100644 --- a/include/eepp/ui/doc/syntaxdefinitionmanager.hpp +++ b/include/eepp/ui/doc/syntaxdefinitionmanager.hpp @@ -20,7 +20,7 @@ class EE_API SyntaxDefinitionManager { SyntaxDefinition& add( SyntaxDefinition&& syntaxStyle ); - const SyntaxDefinition& getPlainStyle() const; + const SyntaxDefinition& getPlainDefinition() const; std::vector languagesThatSupportExtension( std::string extension ) const; @@ -35,10 +35,14 @@ class EE_API SyntaxDefinitionManager { const SyntaxDefinition& find( const std::string& filePath, const std::string& header, bool hFileAsCPP = false ); + const SyntaxDefinition& findFromString( const std::string& str ) const; + SyntaxDefinition& getByExtensionRef( const std::string& filePath ); const SyntaxDefinition& getByLanguageName( const std::string& name ) const; + const SyntaxDefinition& getByLanguageNameInsensitive( std::string name ) const; + const SyntaxDefinition& getByLSPName( const std::string& name ) const; const SyntaxDefinition& getByLanguageId( const String::HashType& id ) const; diff --git a/include/eepp/ui/doc/syntaxhighlighter.hpp b/include/eepp/ui/doc/syntaxhighlighter.hpp index b5166197a..726d1a8c7 100644 --- a/include/eepp/ui/doc/syntaxhighlighter.hpp +++ b/include/eepp/ui/doc/syntaxhighlighter.hpp @@ -8,10 +8,10 @@ namespace EE { namespace UI { namespace Doc { struct EE_API TokenizedLine { - Uint64 initState{ SYNTAX_TOKENIZER_STATE_NONE }; + SyntaxState initState; String::HashType hash; std::vector tokens; - Uint64 state{ SYNTAX_TOKENIZER_STATE_NONE }; + SyntaxState state; Uint64 signature{ 0 }; void updateSignature(); @@ -50,8 +50,7 @@ class EE_API SyntaxHighlighter { void mergeLine( const size_t& line, const TokenizedLine& tokenization ); - TokenizedLine tokenizeLine( const size_t& line, - const Uint64& state = SYNTAX_TOKENIZER_STATE_NONE ); + TokenizedLine tokenizeLine( const size_t& line, const SyntaxState& state = SyntaxState{} ); Mutex& getLinesMutex(); diff --git a/include/eepp/ui/doc/syntaxtokenizer.hpp b/include/eepp/ui/doc/syntaxtokenizer.hpp index 70941fa7e..cb741cc04 100644 --- a/include/eepp/ui/doc/syntaxtokenizer.hpp +++ b/include/eepp/ui/doc/syntaxtokenizer.hpp @@ -6,7 +6,6 @@ #include #include #include -#include #include using namespace EE::Graphics; @@ -42,26 +41,39 @@ struct EE_API SyntaxTokenComplete { #define SYNTAX_TOKENIZER_STATE_NONE ( 0 ) -struct SyntaxState { +struct SyntaxStateRestored { const SyntaxDefinition* currentSyntax{ nullptr }; const SyntaxPattern* subsyntaxInfo{ nullptr }; Uint32 currentPatternIdx{ 0 }; Uint32 currentLevel{ 0 }; }; +struct SyntaxState { + Uint32 state{ SYNTAX_TOKENIZER_STATE_NONE }; + String::HashType hash{ 0 }; + + bool operator==( const SyntaxState& other ) { + return state == other.state && hash == other.hash; + } + + bool operator!=( const SyntaxState& other ) { return !( *this == other ); } +}; + class EE_API SyntaxTokenizer { public: - static std::pair, Uint32> - tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32& state, + static std::pair, SyntaxState> + tokenize( const SyntaxDefinition& syntax, const std::string& text, const SyntaxState& state, const size_t& startIndex = 0, bool skipSubSyntaxSeparator = false ); - static std::pair, Uint32> - tokenizePosition( const SyntaxDefinition& syntax, const std::string& text, const Uint32& state, - const size_t& startIndex = 0, bool skipSubSyntaxSeparator = false ); + static std::pair, SyntaxState> + tokenizePosition( const SyntaxDefinition& syntax, const std::string& text, + const SyntaxState& state, const size_t& startIndex = 0, + bool skipSubSyntaxSeparator = false ); - static std::pair, Uint32> - tokenizeComplete( const SyntaxDefinition& syntax, const std::string& text, const Uint32& state, - const size_t& startIndex = 0, bool skipSubSyntaxSeparator = false ); + static std::pair, SyntaxState> + tokenizeComplete( const SyntaxDefinition& syntax, const std::string& text, + const SyntaxState& state, const size_t& startIndex = 0, + bool skipSubSyntaxSeparator = false ); static Text& tokenizeText( const SyntaxDefinition& syntax, const SyntaxColorScheme& colorScheme, Text& text, const size_t& startIndex = 0, @@ -69,7 +81,8 @@ class EE_API SyntaxTokenizer { bool skipSubSyntaxSeparator = false, const std::string& trimChars = "" ); - static SyntaxState retrieveSyntaxState( const SyntaxDefinition& syntax, const Uint32& state ); + static SyntaxStateRestored retrieveSyntaxState( const SyntaxDefinition& syntax, + const SyntaxState& state ); }; }}} // namespace EE::UI::Doc diff --git a/src/eepp/core/memorymanager.cpp b/src/eepp/core/memorymanager.cpp index d330e2907..91d1708e7 100644 --- a/src/eepp/core/memorymanager.cpp +++ b/src/eepp/core/memorymanager.cpp @@ -5,8 +5,6 @@ #include #include #include -#include -#include using namespace EE::System; using namespace EE::Window; diff --git a/src/eepp/system/inifile.cpp b/src/eepp/system/inifile.cpp index 21d8dac27..0cfc00468 100644 --- a/src/eepp/system/inifile.cpp +++ b/src/eepp/system/inifile.cpp @@ -10,12 +10,6 @@ namespace EE { namespace System { -#if EE_PLATFORM == EE_PLATFORM_WIN -#define iniEOL std::endl -#else -#define iniEOL '\r' << std::endl -#endif - IniFile::IniFile( const std::string& iniPath, bool autoLoad ) { if ( autoLoad ) loadFromFile( iniPath ); @@ -97,7 +91,6 @@ bool IniFile::readFile() { isBOM = true; } - // ini_parse_string( mBuffer.c_str(), ValueHandler, this ); std::string_view buffer( mBuffer ); if ( isBOM ) buffer = std::string_view( mBuffer.data() + 3, mBuffer.size() - 3 ); diff --git a/src/eepp/ui/doc/languages/markdown.cpp b/src/eepp/ui/doc/languages/markdown.cpp index 52a86da34..eb7955438 100644 --- a/src/eepp/ui/doc/languages/markdown.cpp +++ b/src/eepp/ui/doc/languages/markdown.cpp @@ -4,6 +4,13 @@ namespace EE { namespace UI { namespace Doc { namespace Language { void addMarkdown() { + auto dynSyntax = []( const SyntaxPattern&, const std::string& match ) -> std::string { + std::string lang = String::toLower( match.substr( 3 ) ); + String::trimInPlace( lang ); + if ( !lang.empty() && lang[lang.size() - 1] == '\n' ) + lang.pop_back(); + return SyntaxDefinitionManager::instance()->findFromString( lang ).getLanguageName(); + }; SyntaxDefinitionManager::instance()->add( @@ -11,40 +18,7 @@ void addMarkdown() { { "%.md$", "%.markdown$" }, { { { "\\." }, "normal" }, - { { "```[Xx][Mm][Ll]", "```" }, "function", "XML" }, - { { "```[Hh][Tt][Mm][Ll]", "```" }, "function", "html" }, - { { "```[Cc]++", "```" }, "function", "C++" }, - { { "```[Cc][Pp][Pp]", "```" }, "function", "C++" }, - { { "```[Cc]sharp", "```" }, "function", "C#" }, - { { "```[Cc][Ss][Ss]", "```" }, "function", "CSS" }, - { { "```[Cc]", "```" }, "function", "C" }, - { { "```[Dd][Aa][Rr][Tt]", "```" }, "function", "Dart" }, - { { "```[Dd]", "```" }, "function", "D" }, - { { "```[Ll]ua", "```" }, "function", "Lua" }, - { { "```[Jj][Ss][Oo][Nn]", "```" }, "function", "JSON" }, - { { "```[Ja]va[Ss]cript", "```" }, "function", "JavaScript" }, - { { "```[Tt]ype[Ss]cript", "```" }, "function", "TypeScript" }, - { { "```[Pp]ython", "```" }, "function", "Python" }, - { { "```[Bb]ash", "```" }, "function", "Bash" }, - { { "```[Pp][Hh][Pp]", "```" }, "function", "PHPCore" }, - { { "```[Ss][Qq][Ll]", "```" }, "function", "SQL" }, - { { "```[Gg][Ll][Ss][Ll]", "```" }, "function", "GLSL" }, - { { "```[Ii][Nn][Ii]", "```" }, "function", "Config File" }, - { { "```[Mm]akefile", "```" }, "function", "Makefile" }, - { { "```[Gg][Oo]", "```" }, "function", "Go" }, - { { "```[Rr]ust", "```" }, "function", "Rust" }, - { { "```[Rr]uby", "```" }, "function", "Ruby" }, - { { "```[Gg][Dd][Ss]cript", "```" }, "function", "GSCript" }, - { { "```[Jj]ava", "```" }, "function", "java" }, - { { "```[Ss]wift", "```" }, "function", "Swift" }, - { { "```[Oo]bjective[Cc]", "```" }, "function", "Objective-C" }, - { { "```[Yy][Aa][Mm][Ll]", "```" }, "function", "YAML" }, - { { "```[Kk]otlin", "```" }, "function", "Kotlin" }, - { { "```[Ss]olidity", "```" }, "function", "Solidity" }, - { { "```[Hh]askell", "```" }, "function", "Haskell" }, - { { "```[Oo]din", "```" }, "function", "Odin" }, - { { "```[Nn]im", "```" }, "function", "Nim" }, - { { "```[Zz]ig", "```" }, "function", "Zig" }, + { { "```[%w%s+-#]+", "```" }, "function", dynSyntax }, { { "" }, "comment" }, { { "```", "```" }, "string" }, { { "``", "``" }, "string" }, diff --git a/src/eepp/ui/doc/syntaxdefinitionmanager.cpp b/src/eepp/ui/doc/syntaxdefinitionmanager.cpp index 31645966c..605099a78 100644 --- a/src/eepp/ui/doc/syntaxdefinitionmanager.cpp +++ b/src/eepp/ui/doc/syntaxdefinitionmanager.cpp @@ -1915,7 +1915,7 @@ SyntaxDefinition& SyntaxDefinitionManager::add( SyntaxDefinition&& syntaxStyle ) return mDefinitions.back(); } -const SyntaxDefinition& SyntaxDefinitionManager::getPlainStyle() const { +const SyntaxDefinition& SyntaxDefinitionManager::getPlainDefinition() const { return mDefinitions[0]; } @@ -1932,6 +1932,16 @@ SyntaxDefinitionManager::getByLanguageName( const std::string& name ) const { return mDefinitions[0]; } +const SyntaxDefinition& +SyntaxDefinitionManager::getByLanguageNameInsensitive( std::string name ) const { + String::toLowerInPlace( name ); + for ( auto& style : mDefinitions ) { + if ( String::toLower( style.getLanguageName() ) == name ) + return style; + } + return mDefinitions[0]; +} + const SyntaxDefinition& SyntaxDefinitionManager::getByLSPName( const std::string& name ) const { for ( auto& style : mDefinitions ) { if ( style.getLSPName() == name ) @@ -2303,4 +2313,14 @@ const SyntaxDefinition& SyntaxDefinitionManager::find( const std::string& filePa return def; } +const SyntaxDefinition& SyntaxDefinitionManager::findFromString( const std::string& lang ) const { + const auto& syn = getByLSPName( lang ); + if ( syn.getLSPName() != getPlainDefinition().getLSPName() ) + return syn; + const auto& syn2 = getByLanguageNameInsensitive( lang ); + if ( syn2.getLSPName() != getPlainDefinition().getLSPName() ) + return syn2; + return getPlainDefinition(); +} + }}} // namespace EE::UI::Doc diff --git a/src/eepp/ui/doc/syntaxhighlighter.cpp b/src/eepp/ui/doc/syntaxhighlighter.cpp index 036a0a270..b111ea123 100644 --- a/src/eepp/ui/doc/syntaxhighlighter.cpp +++ b/src/eepp/ui/doc/syntaxhighlighter.cpp @@ -46,7 +46,7 @@ void SyntaxHighlighter::invalidate( Int64 lineIndex ) { mMaxWantedLine = eemin( mMaxWantedLine, (Int64)mDoc->linesCount() - 1 ); } -TokenizedLine SyntaxHighlighter::tokenizeLine( const size_t& line, const Uint64& state ) { +TokenizedLine SyntaxHighlighter::tokenizeLine( const size_t& line, const SyntaxState& state ) { auto& ln = mDoc->line( line ); TokenizedLine tokenizedLine; tokenizedLine.initState = state; @@ -161,7 +161,7 @@ const std::vector& SyntaxHighlighter::getLine( const size_t return noHighlightVector; } - int prevState = SYNTAX_TOKENIZER_STATE_NONE; + SyntaxState prevState; if ( index > 0 ) { Lock l( mLinesMutex ); auto prevIt = mLines.find( index - 1 ); @@ -195,7 +195,7 @@ bool SyntaxHighlighter::updateDirty( int visibleLinesCount ) { Int64 max = eemax( 0LL, eemin( mFirstInvalidLine + visibleLinesCount, mMaxWantedLine ) ); for ( Int64 index = mFirstInvalidLine; index <= max; index++ ) { - Uint64 state = SYNTAX_TOKENIZER_STATE_NONE; + SyntaxState state; Lock l( mLinesMutex ); if ( index > 0 ) { auto prevIt = mLines.find( index - 1 ); @@ -223,14 +223,14 @@ SyntaxHighlighter::getSyntaxDefinitionFromTextPosition( const TextPosition& posi Lock l( mLinesMutex ); auto found = mLines.find( position.line() ); if ( found == mLines.end() ) - return SyntaxDefinitionManager::instance()->getPlainStyle(); + return SyntaxDefinitionManager::instance()->getPlainDefinition(); TokenizedLine& line = found->second; - SyntaxState state = + SyntaxStateRestored state = SyntaxTokenizer::retrieveSyntaxState( mDoc->getSyntaxDefinition(), line.state ); if ( nullptr == state.currentSyntax ) - return SyntaxDefinitionManager::instance()->getPlainStyle(); + return SyntaxDefinitionManager::instance()->getPlainDefinition(); return *state.currentSyntax; } diff --git a/src/eepp/ui/doc/syntaxtokenizer.cpp b/src/eepp/ui/doc/syntaxtokenizer.cpp index 1174108ca..8f82e8b6f 100644 --- a/src/eepp/ui/doc/syntaxtokenizer.cpp +++ b/src/eepp/ui/doc/syntaxtokenizer.cpp @@ -109,23 +109,25 @@ std::pair findNonEscaped( const std::string& text, const std::string& return std::make_pair( -1, -1 ); } -SyntaxState SyntaxTokenizer::retrieveSyntaxState( const SyntaxDefinition& syntax, - const Uint32& state ) { - SyntaxState syntaxState{ &syntax, nullptr, state, 0 }; - if ( state > 0 && - ( state > 255 || - ( state < syntaxState.currentSyntax->getPatterns().size() && - !syntaxState.currentSyntax->getPatterns()[state - 1].syntax.empty() ) ) ) { +SyntaxStateRestored SyntaxTokenizer::retrieveSyntaxState( const SyntaxDefinition& syntax, + const SyntaxState& state ) { + SyntaxStateRestored syntaxState{ &syntax, nullptr, state.state, 0 }; + if ( state.state > 0 && + ( state.state > 255 || + ( state.state < syntaxState.currentSyntax->getPatterns().size() && + syntaxState.currentSyntax->getPatterns()[state.state - 1].hasSyntax() ) ) ) { for ( size_t i = 0; i <= 2; ++i ) { - Uint32 target = ( state >> ( 8 * i ) ) & 0xFF; + Uint32 target = ( state.state >> ( i << 3 ) ) & 0xFF; if ( target != SYNTAX_TOKENIZER_STATE_NONE ) { if ( target < syntaxState.currentSyntax->getPatterns().size() && - !syntaxState.currentSyntax->getPatterns()[target - 1].syntax.empty() ) { + syntaxState.currentSyntax->getPatterns()[target - 1].hasSyntax() ) { syntaxState.subsyntaxInfo = &syntaxState.currentSyntax->getPatterns()[target - 1]; syntaxState.currentSyntax = - &SyntaxDefinitionManager::instance()->getByLanguageName( - syntaxState.subsyntaxInfo->syntax ); + state.hash != 0 + ? &SyntaxDefinitionManager::instance()->getByLanguageId( state.hash ) + : &SyntaxDefinitionManager::instance()->getByLanguageName( + syntaxState.subsyntaxInfo->syntax ); syntaxState.currentPatternIdx = SYNTAX_TOKENIZER_STATE_NONE; syntaxState.currentLevel++; } else { @@ -140,8 +142,8 @@ SyntaxState SyntaxTokenizer::retrieveSyntaxState( const SyntaxDefinition& syntax } template -static inline std::pair, Uint32> -_tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32& state, +static inline std::pair, SyntaxState> +_tokenize( const SyntaxDefinition& syntax, const std::string& text, const SyntaxState& state, const size_t& startIndex, bool skipSubSyntaxSeparator ) { std::vector tokens; LuaPattern::Range matches[12]; @@ -150,26 +152,31 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32 if ( syntax.getPatterns().empty() ) { pushToken( tokens, SyntaxStyleTypes::Normal, text ); - return std::make_pair( std::move( tokens ), SYNTAX_TOKENIZER_STATE_NONE ); + return std::make_pair( std::move( tokens ), SyntaxState{} ); } size_t i = startIndex; - int retState = state; - SyntaxState curState = SyntaxTokenizer::retrieveSyntaxState( syntax, state ); + SyntaxState retState = state; + SyntaxStateRestored curState = SyntaxTokenizer::retrieveSyntaxState( syntax, state ); auto setSubsyntaxPatternIdx = [&curState, &retState]( const Uint32& patternIndex ) { curState.currentPatternIdx = patternIndex; - retState &= ~( 0xFF << ( curState.currentLevel * 8 ) ); - retState |= ( patternIndex << ( curState.currentLevel * 8 ) ); + retState.state &= ~( 0xFF << ( curState.currentLevel << 3 ) ); + retState.state |= ( patternIndex << ( curState.currentLevel << 3 ) ); + if ( curState.currentSyntax ) + retState.hash = curState.currentSyntax->getLanguageId(); }; - auto pushSubsyntax = [&setSubsyntaxPatternIdx, &curState]( - const SyntaxPattern& enteringSubsyntax, const Uint32& patternIndex ) { + auto pushSubsyntax = [&setSubsyntaxPatternIdx, + &curState]( const SyntaxPattern& enteringSubsyntax, + const Uint32& patternIndex, const std::string& patternStr ) { setSubsyntaxPatternIdx( patternIndex ); curState.currentLevel++; curState.subsyntaxInfo = &enteringSubsyntax; curState.currentSyntax = &SyntaxDefinitionManager::instance()->getByLanguageName( - curState.subsyntaxInfo->syntax ); + curState.subsyntaxInfo->dynSyntax + ? curState.subsyntaxInfo->dynSyntax( enteringSubsyntax, patternStr ) + : curState.subsyntaxInfo->syntax ); setSubsyntaxPatternIdx( SYNTAX_TOKENIZER_STATE_NONE ); }; @@ -180,7 +187,8 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32 curState = SyntaxTokenizer::retrieveSyntaxState( syntax, retState ); }; - size_t size = !text.empty() ? text.size() - 1 : 0; // skip last char ( new line char ) + size_t size = !text.empty() ? ( text[text.size() - 1] == '\n' ? text.size() - 1 : text.size() ) + : 0; // skip last char ( new line char ) while ( i < size ) { if ( curState.currentPatternIdx != SYNTAX_TOKENIZER_STATE_NONE ) { @@ -240,14 +248,15 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32 } bool matched = false; + std::string patternStr; for ( size_t patternIndex = 0; patternIndex < curState.currentSyntax->getPatterns().size(); patternIndex++ ) { const SyntaxPattern& pattern = curState.currentSyntax->getPatterns()[patternIndex]; if ( i != 0 && pattern.patterns[0][0] == '^' ) continue; - const std::string& patternStr( - pattern.patterns[0][0] == '^' ? pattern.patterns[0] : "^" + pattern.patterns[0] ); + patternStr = + pattern.patterns[0][0] == '^' ? pattern.patterns[0] : "^" + pattern.patterns[0]; LuaPattern words( patternStr ); if ( words.matches( text, matches, i ) && ( numMatches = words.getNumMatches() ) > 0 ) { if ( numMatches > 1 ) { @@ -283,7 +292,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32 std::string patternText( text.substr( start, end - start ) ); SyntaxStyleType type = curState.currentSyntax->getSymbol( patternText ); - if ( !skipSubSyntaxSeparator || pattern.syntax.empty() ) { + if ( !skipSubSyntaxSeparator || !pattern.hasSyntax() ) { pushToken( tokens, type == SyntaxStyleEmpty() ? ( curMatch < pattern.types.size() @@ -293,8 +302,8 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32 patternText ); } - if ( !pattern.syntax.empty() ) { - pushSubsyntax( pattern, patternIndex + 1 ); + if ( pattern.hasSyntax() ) { + pushSubsyntax( pattern, patternIndex + 1, patternStr ); } else if ( pattern.patterns.size() > 1 ) { setSubsyntaxPatternIdx( patternIndex + 1 ); } @@ -328,7 +337,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32 } std::string patternText( text.substr( start, end - start ) ); SyntaxStyleType type = curState.currentSyntax->getSymbol( patternText ); - if ( !skipSubSyntaxSeparator || pattern.syntax.empty() ) { + if ( !skipSubSyntaxSeparator || !pattern.hasSyntax() ) { pushToken( tokens, type == SyntaxStyleEmpty() ? ( curMatch < pattern.types.size() @@ -337,8 +346,8 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32 : type, patternText ); } - if ( !pattern.syntax.empty() ) { - pushSubsyntax( pattern, patternIndex + 1 ); + if ( pattern.hasSyntax() ) { + pushSubsyntax( pattern, patternIndex + 1, patternText ); } else if ( pattern.patterns.size() > 1 ) { setSubsyntaxPatternIdx( patternIndex + 1 ); } @@ -369,24 +378,24 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32 return std::make_pair( std::move( tokens ), retState ); } -std::pair, Uint32> +std::pair, SyntaxState> SyntaxTokenizer::tokenize( const SyntaxDefinition& syntax, const std::string& text, - const Uint32& state, const size_t& startIndex, + const SyntaxState& state, const size_t& startIndex, bool skipSubSyntaxSeparator ) { return _tokenize( syntax, text, state, startIndex, skipSubSyntaxSeparator ); } -std::pair, Uint32> +std::pair, SyntaxState> SyntaxTokenizer::tokenizePosition( const SyntaxDefinition& syntax, const std::string& text, - const Uint32& state, const size_t& startIndex, + const SyntaxState& state, const size_t& startIndex, bool skipSubSyntaxSeparator ) { return _tokenize( syntax, text, state, startIndex, skipSubSyntaxSeparator ); } -std::pair, Uint32> +std::pair, SyntaxState> SyntaxTokenizer::tokenizeComplete( const SyntaxDefinition& syntax, const std::string& text, - const Uint32& state, const size_t& startIndex, + const SyntaxState& state, const size_t& startIndex, bool skipSubSyntaxSeparator ) { return _tokenize( syntax, text, state, startIndex, skipSubSyntaxSeparator ); @@ -397,10 +406,10 @@ Text& SyntaxTokenizer::tokenizeText( const SyntaxDefinition& syntax, const size_t& startIndex, const size_t& endIndex, bool skipSubSyntaxSeparator, const std::string& trimChars ) { - auto tokens = - SyntaxTokenizer::tokenizeComplete( syntax, text.getString(), SYNTAX_TOKENIZER_STATE_NONE, - startIndex, skipSubSyntaxSeparator ) - .first; + auto tokens = SyntaxTokenizer::tokenizeComplete( syntax, text.getString(), + { SYNTAX_TOKENIZER_STATE_NONE, 0 }, startIndex, + skipSubSyntaxSeparator ) + .first; if ( skipSubSyntaxSeparator || !trimChars.empty() ) { String txt; diff --git a/src/eepp/ui/doc/textdocument.cpp b/src/eepp/ui/doc/textdocument.cpp index c9a8384cc..5fb6d5f8f 100644 --- a/src/eepp/ui/doc/textdocument.cpp +++ b/src/eepp/ui/doc/textdocument.cpp @@ -86,7 +86,7 @@ void TextDocument::reset() { mLastSelection = 0; mLines.clear(); mLines.emplace_back( String( "\n" ) ); - mSyntaxDefinition = SyntaxDefinitionManager::instance()->getPlainStyle(); + mSyntaxDefinition = SyntaxDefinitionManager::instance()->getPlainDefinition(); mUndoStack.clear(); cleanChangeId(); notifySyntaxDefinitionChange(); diff --git a/src/tools/ecode/uitreeviewglobalsearch.cpp b/src/tools/ecode/uitreeviewglobalsearch.cpp index 0179d3058..26ac86caf 100644 --- a/src/tools/ecode/uitreeviewglobalsearch.cpp +++ b/src/tools/ecode/uitreeviewglobalsearch.cpp @@ -176,8 +176,7 @@ UIPushButton* UITreeViewCellGlobalSearch::updateText( const std::string& text ) mResultStr = ""; } - auto tokens = - SyntaxTokenizer::tokenize( styleDef, text, SYNTAX_TOKENIZER_STATE_NONE, to ).first; + auto tokens = SyntaxTokenizer::tokenize( styleDef, text, SyntaxState{}, to ).first; size_t start = to; for ( const auto& token : tokens ) {