diff --git a/bin/assets/plugins/aiassistant.json b/bin/assets/plugins/aiassistant.json index e09368f35..ab5e569ce 100644 --- a/bin/assets/plugins/aiassistant.json +++ b/bin/assets/plugins/aiassistant.json @@ -128,7 +128,12 @@ }, { "name": "gemini-2.5-pro-exp-03-25", - "display_name": "Gemini 2.5 Pro Experimental", + "display_name": "Gemini 2.5 Pro Experimental (03-25)", + "max_tokens": 1000000 + }, + { + "name": "gemini-2.5-pro-preview-05-06", + "display_name": "Gemini 2.5 Pro Preview (05-06)", "max_tokens": 1000000 }, { diff --git a/include/eepp/core/string.hpp b/include/eepp/core/string.hpp index 770589ed8..a1ba56068 100644 --- a/include/eepp/core/string.hpp +++ b/include/eepp/core/string.hpp @@ -97,6 +97,9 @@ class EE_API String { /** @return string hash */ static String::HashType hash( const std::string& str ); + /** @return string hash */ + static String::HashType hash( const std::string_view& str ); + /** @return string hash. Note: String::hash( std::string( "text" ) ) is != to String::hash( * String( "text" ) ) */ static String::HashType hash( const String& str ); diff --git a/include/eepp/ui/doc/syntaxdefinition.hpp b/include/eepp/ui/doc/syntaxdefinition.hpp index 165f9f720..86ce0fc67 100644 --- a/include/eepp/ui/doc/syntaxdefinition.hpp +++ b/include/eepp/ui/doc/syntaxdefinition.hpp @@ -5,6 +5,8 @@ #include #include #include +#include + #include #include #include @@ -31,6 +33,15 @@ enum class SyntaxPatternMatchType { LuaPattern, RegEx, Parser }; class SyntaxDefinition; struct EE_API SyntaxPattern { + enum Flags { + IsPure = 1 << 0, + IsInclude = 1 << 1, + IsRepositoryInclude = 1 << 2, + IsRootSelfInclude = 1 << 3, + IsInherited = 1 << 4, + IsRangedMatch = 1 << 5, + }; + static UnorderedMap SyntaxStyleTypeCache; using DynamicSyntax = @@ -45,6 +56,8 @@ struct EE_API SyntaxPattern { DynamicSyntax dynSyntax; SyntaxPatternMatchType matchType{ SyntaxPatternMatchType::LuaPattern }; const SyntaxDefinition* def{ nullptr }; + Uint16 flags{ 0 }; + Uint16 repositoryIdx{ 0 }; SyntaxPattern( std::vector&& _patterns, const std::string& _type, const std::string& _syntax = "", @@ -70,18 +83,41 @@ struct EE_API SyntaxPattern { std::vector&& _endTypes, DynamicSyntax&& _syntax, SyntaxPatternMatchType matchType = SyntaxPatternMatchType::LuaPattern ); - bool isIncludePattern() const { + inline bool hasSyntax() const { return !syntax.empty() || dynSyntax; } + + inline bool isPure() const { return flags & Flags::IsPure; } + + inline bool isInclude() const { return flags & Flags::IsInclude; } + + inline bool isRepositoryInclude() const { return flags & Flags::IsRepositoryInclude; } + + inline bool isRootSelfInclude() const { return flags & Flags::IsRootSelfInclude; } + + inline bool isInherited() const { return flags & Flags::IsInherited; } + + inline bool isRangedMatch() const { return flags & Flags::IsRangedMatch; } + + std::string_view getRepositoryName() const { + eeASSERT( isRepositoryInclude() ); + return std::string_view{ patterns[1] }.substr( 1 ); + } + + inline bool checkIsIncludePattern() const { return patterns.size() == 2 && patterns[0] == "include" && !patterns[1].empty() && ( patterns[1][0] == '#' || patterns[1][0] == '$' ); } - bool isRangedMatch() const { return !isIncludePattern() && patterns.size() >= 2; } + inline bool checkIsRangedMatch() const { + return !checkIsIncludePattern() && patterns.size() >= 2; + } - bool isRootSelfInclude() const { return isIncludePattern() && patterns[1] == "$self"; } + inline bool checkIsRootSelfInclude() const { + return checkIsIncludePattern() && patterns[1] == "$self"; + } - bool isRepositoryInclude() const { return isIncludePattern() && patterns[1][0] == '#'; } - - bool hasSyntax() const { return !syntax.empty() || dynSyntax; } + inline bool checkIsRepositoryInclude() const { + return checkIsIncludePattern() && patterns[1][0] == '#'; + } }; class EE_API SyntaxDefinition { @@ -183,12 +219,22 @@ class EE_API SyntaxDefinition { SyntaxDefinition& setRepository( const std::string& name, std::vector&& patterns ); - const std::vector& getRepository( const std::string& name ) const; + const std::vector& getRepository( String::HashType hash ) const; + + const std::vector& getRepository( std::string_view name ) const; + + Uint32 getRepositoryIndex( String::HashType hash ) const; + + Uint32 getRepositoryIndex( std::string_view name ) const; + + String::HashType getRepositoryHash( Uint32 index ) const; SyntaxDefinition& addAlternativeName( const std::string& name ); const std::vector& getAlternativeNames() const; + const SyntaxPattern* getPatternFromState( const SyntaxStateType& state ) const; + protected: friend class SyntaxDefinitionManager; @@ -208,8 +254,11 @@ class EE_API SyntaxDefinition { bool mVisible{ true }; bool mHasExtensionPriority{ false }; bool mCaseInsensitive{ false }; - UnorderedMap> mRepository; + UnorderedMap> mRepository; + UnorderedMap mRepositoryIndex; + UnorderedMap mRepositoryIndexInvert; std::vector mLanguageAlternativeNames; + Uint32 mRepositoryIndexCounter{ 0 }; }; }}} // namespace EE::UI::Doc diff --git a/include/eepp/ui/doc/syntaxtokenizer.hpp b/include/eepp/ui/doc/syntaxtokenizer.hpp index e365c622c..c09196779 100644 --- a/include/eepp/ui/doc/syntaxtokenizer.hpp +++ b/include/eepp/ui/doc/syntaxtokenizer.hpp @@ -5,13 +5,15 @@ #include #include #include -#include #include using namespace EE::Graphics; namespace EE { namespace UI { namespace Doc { +class SyntaxDefinition; +struct SyntaxPattern; + using SyntaxTokenLen = Uint32; struct EE_API SyntaxToken { @@ -41,20 +43,25 @@ struct EE_API SyntaxTokenComplete { #define SYNTAX_TOKENIZER_STATE_NONE ( 0 ) +struct SyntaxStateType { + Uint8 state{ SYNTAX_TOKENIZER_STATE_NONE }; + Uint8 repositoryIdx{ SYNTAX_TOKENIZER_STATE_NONE }; +}; + struct SyntaxStateRestored { const SyntaxDefinition* currentSyntax{ nullptr }; const SyntaxPattern* subsyntaxInfo{ nullptr }; - Uint32 currentPatternIdx{ 0 }; + SyntaxStateType currentPatternIdx{}; Uint32 currentLevel{ 0 }; }; #define MAX_SUB_SYNTAXS 8 struct SyntaxState { - // 8 bits per pattern - max 4 sub-languages - max 65k patterns per language - Uint16 state[MAX_SUB_SYNTAXS]{}; + // 16 bits per pattern - max 8 sub-languages - max 254 patterns per language or repository + SyntaxStateType state[MAX_SUB_SYNTAXS]{}; - // 16 bits per language (language index) - max 4 sub-languages - max 65k languages + // 16 bits per language (language index) - max 8 sub-languages - max 65k languages Uint16 langStack[MAX_SUB_SYNTAXS]{}; bool operator==( const SyntaxState& other ) const { diff --git a/src/eepp/core/string.cpp b/src/eepp/core/string.cpp index 6749ce1e3..8aff4c47f 100644 --- a/src/eepp/core/string.cpp +++ b/src/eepp/core/string.cpp @@ -624,6 +624,10 @@ String::HashType String::hash( const std::string& str ) { return String::hash( str.c_str() ); } +String::HashType String::hash( const std::string_view& str ) { + return String::hash( str.data(), str.length() ); +} + String::HashType String::hash( const String& str ) { return String::hash( reinterpret_cast( str.mString.data() ), str.size() * sizeof( String::StringBaseType ) ); diff --git a/src/eepp/ui/doc/syntaxdefinition.cpp b/src/eepp/ui/doc/syntaxdefinition.cpp index 11d905124..ba8d875df 100644 --- a/src/eepp/ui/doc/syntaxdefinition.cpp +++ b/src/eepp/ui/doc/syntaxdefinition.cpp @@ -2,6 +2,7 @@ #include #include #include +#include using namespace std::literals; @@ -37,9 +38,33 @@ static void updatePatternRefs( const SyntaxDefinition& def, SyntaxPattern& ptrn ptrn.syntax = def.getLanguageName(); } -static void updatePatternRefs( const SyntaxDefinition& def, std::vector& ptrns ) { - for ( auto& ptrn : ptrns ) +static void updatePatternState( SyntaxDefinition& def, std::vector& ptrns ) { + for ( auto& ptrn : ptrns ) { + if ( ptrn.checkIsRangedMatch() ) + ptrn.flags |= SyntaxPattern::IsRangedMatch; + + if ( ptrn.checkIsIncludePattern() ) { + ptrn.flags |= SyntaxPattern::IsInclude; + + if ( ptrn.checkIsRepositoryInclude() ) { + ptrn.flags |= SyntaxPattern::IsRepositoryInclude; + ptrn.repositoryIdx = def.getRepositoryIndex( ptrn.getRepositoryName() ); + } + + if ( ptrn.checkIsRootSelfInclude() ) + ptrn.flags |= SyntaxPattern::IsRootSelfInclude; + } else { + ptrn.flags |= SyntaxPattern::IsPure; + } + updatePatternRefs( def, ptrn ); + } +} + +static void updateRepoIndexState( SyntaxDefinition& def, std::vector& ptrns ) { + for ( auto& ptrn : ptrns ) + if ( ptrn.isRepositoryInclude() ) + ptrn.repositoryIdx = def.getRepositoryIndex( ptrn.getRepositoryName() ); } SyntaxDefinition::SyntaxDefinition() {} @@ -65,7 +90,7 @@ SyntaxDefinition::SyntaxDefinition( const std::string& languageName, return pattern.matchType == SyntaxPatternMatchType::Parser; } ) ) ParserMatcherManager::instance()->registerBaseParsers(); - updatePatternRefs( *this, mPatterns ); + updatePatternState( *this, mPatterns ); mPatterns.emplace_back( SyntaxPattern{ { "%s+" }, "normal" } ); mPatterns.emplace_back( SyntaxPattern{ { "%w+%f[%s]" }, "normal" } ); } @@ -360,17 +385,39 @@ SyntaxPattern::SyntaxPattern( std::vector&& _patterns, SyntaxDefinition& SyntaxDefinition::setRepository( const std::string& name, std::vector&& patterns ) { - updatePatternRefs( *this, patterns ); - mRepository[name] = std::move( patterns ); + updatePatternState( *this, patterns ); + updateRepoIndexState( *this, mPatterns ); + auto hash = String::hash( name ); + mRepository[hash] = std::move( patterns ); + mRepositoryIndex[hash] = ++mRepositoryIndexCounter; + mRepositoryIndexInvert[mRepositoryIndexCounter] = hash; return *this; } -const std::vector& SyntaxDefinition::getRepository( const std::string& name ) const { +const std::vector& SyntaxDefinition::getRepository( String::HashType hash ) const { static std::vector EMPTY = {}; - auto found = mRepository.find( name ); + auto found = mRepository.find( hash ); return found != mRepository.end() ? found->second : EMPTY; } +const std::vector& SyntaxDefinition::getRepository( std::string_view name ) const { + return getRepository( String::hash( name ) ); +} + +Uint32 SyntaxDefinition::getRepositoryIndex( String::HashType hash ) const { + auto it = mRepositoryIndex.find( hash ); + return it != mRepositoryIndex.end() ? it->second : 0; +} + +Uint32 SyntaxDefinition::getRepositoryIndex( std::string_view name ) const { + return getRepositoryIndex( String::hash( name ) ); +} + +String::HashType SyntaxDefinition::getRepositoryHash( Uint32 index ) const { + auto it = mRepositoryIndexInvert.find( index ); + return it != mRepositoryIndexInvert.end() ? it->second : 0; +} + SyntaxDefinition& SyntaxDefinition::addAlternativeName( const std::string& name ) { mLanguageAlternativeNames.emplace_back( name ); return *this; @@ -380,4 +427,21 @@ const std::vector& SyntaxDefinition::getAlternativeNames() const { return mLanguageAlternativeNames; } +const SyntaxPattern* SyntaxDefinition::getPatternFromState( const SyntaxStateType& state ) const { + if ( state.repositoryIdx == 0 && state.state > 0 && + state.state - 1 < static_cast( mPatterns.size() ) ) { + return &mPatterns[state.state - 1]; + } else if ( state.repositoryIdx != 0 ) { + auto hash = getRepositoryHash( state.repositoryIdx ); + if ( hash ) { + const auto& repo = mRepository.find( hash ); + if ( repo != mRepository.end() && state.state > 0 && + state.state - 1 < static_cast( repo->second.size() ) ) { + return &repo->second[state.state - 1]; + } + } + } + return nullptr; +} + }}} // namespace EE::UI::Doc diff --git a/src/eepp/ui/doc/syntaxtokenizer.cpp b/src/eepp/ui/doc/syntaxtokenizer.cpp index f0b363680..76e0fed22 100644 --- a/src/eepp/ui/doc/syntaxtokenizer.cpp +++ b/src/eepp/ui/doc/syntaxtokenizer.cpp @@ -140,27 +140,27 @@ static NonEscapedMatch findNonEscaped( const std::string& text, const std::strin SyntaxStateRestored SyntaxTokenizer::retrieveSyntaxState( const SyntaxDefinition& syntax, const SyntaxState& state ) { SyntaxStateRestored syntaxState{ &syntax, nullptr, state.state[0], 0 }; - if ( state.state[0] > 0 && - ( state.state[1] > 0 || - ( state.state[0] < syntaxState.currentSyntax->getPatterns().size() && - syntaxState.currentSyntax->getPatterns()[state.state[0] - 1].hasSyntax() ) ) ) { + const SyntaxPattern* curPattern = nullptr; + if ( state.state[0].state > 0 && + ( state.state[1].state > 0 || + ( ( curPattern = syntax.getPatternFromState( state.state[0] ) ) && + curPattern->hasSyntax() ) ) ) { for ( size_t i = 0; i < MAX_SUB_SYNTAXS - 1; ++i ) { - Uint32 target = state.state[i]; - if ( target != SYNTAX_TOKENIZER_STATE_NONE ) { - if ( target < syntaxState.currentSyntax->getPatterns().size() && - syntaxState.currentSyntax->getPatterns()[target - 1].hasSyntax() ) { - syntaxState.subsyntaxInfo = - &syntaxState.currentSyntax->getPatterns()[target - 1]; - Uint32 langIndex = state.langStack[i]; + if ( i != 0 || curPattern == nullptr ) + curPattern = syntaxState.currentSyntax->getPatternFromState( state.state[i] ); + if ( curPattern && state.state[i].state != SYNTAX_TOKENIZER_STATE_NONE ) { + if ( curPattern->hasSyntax() ) { + syntaxState.subsyntaxInfo = curPattern; + auto langIndex = state.langStack[i]; syntaxState.currentSyntax = langIndex != 0 ? &SyntaxDefinitionManager::instance()->getByLanguageIndex( langIndex ) : &SyntaxDefinitionManager::instance()->getByLanguageName( syntaxState.subsyntaxInfo->syntax ); - syntaxState.currentPatternIdx = SYNTAX_TOKENIZER_STATE_NONE; + syntaxState.currentPatternIdx = {}; syntaxState.currentLevel++; } else { - syntaxState.currentPatternIdx = target; + syntaxState.currentPatternIdx = state.state[i]; } } else { break; @@ -171,14 +171,15 @@ SyntaxStateRestored SyntaxTokenizer::retrieveSyntaxState( const SyntaxDefinition } static inline void setSubsyntaxPatternIdx( SyntaxStateRestored& curState, SyntaxState& retState, - const Uint32& patternIndex ) { + const SyntaxStateType& patternIndex ) { curState.currentPatternIdx = patternIndex; retState.state[curState.currentLevel] = patternIndex; }; static inline void pushSubsyntax( SyntaxStateRestored& curState, SyntaxState& retState, const SyntaxPattern& enteringSubsyntax, - const Uint32& patternIndex, std::string_view patternTextStr ) { + const SyntaxStateType& patternIndex, + std::string_view patternTextStr ) { if ( curState.currentLevel == MAX_SUB_SYNTAXS - 1 ) return; setSubsyntaxPatternIdx( curState, retState, patternIndex ); @@ -189,15 +190,15 @@ static inline void pushSubsyntax( SyntaxStateRestored& curState, SyntaxState& re : curState.subsyntaxInfo->syntax ); retState.langStack[curState.currentLevel] = curState.currentSyntax->getLanguageIndex(); curState.currentLevel++; - setSubsyntaxPatternIdx( curState, retState, SYNTAX_TOKENIZER_STATE_NONE ); + setSubsyntaxPatternIdx( curState, retState, SyntaxStateType{} ); }; static inline void popSubsyntax( SyntaxStateRestored& curState, SyntaxState& retState, const SyntaxDefinition& syntax ) { - setSubsyntaxPatternIdx( curState, retState, SYNTAX_TOKENIZER_STATE_NONE ); + setSubsyntaxPatternIdx( curState, retState, SyntaxStateType{} ); retState.langStack[curState.currentLevel] = 0; curState.currentLevel--; - setSubsyntaxPatternIdx( curState, retState, SYNTAX_TOKENIZER_STATE_NONE ); + setSubsyntaxPatternIdx( curState, retState, SyntaxStateType{} ); curState = SyntaxTokenizer::retrieveSyntaxState( syntax, retState ); }; @@ -272,7 +273,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax std::optional shouldCloseSubSyntax; const auto matchPattern = [&]( const SyntaxPattern& pattern, size_t& startIdx, - size_t patternIndex ) -> bool { + SyntaxStateType patternIndex ) -> bool { int start = 0, end = 0; patternStr = pattern.matchType != SyntaxPatternMatchType::Parser @@ -362,10 +363,10 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax if ( pattern.hasSyntax() ) { pushSubsyntax( - curState, retState, pattern, patternIndex + 1, + curState, retState, pattern, patternIndex, textv.substr( fullMatchStart, fullMatchEnd - fullMatchStart ) ); } else if ( pattern.patterns.size() > 1 ) { - setSubsyntaxPatternIdx( curState, retState, patternIndex + 1 ); + setSubsyntaxPatternIdx( curState, retState, patternIndex ); } startIdx = fullMatchEnd; @@ -420,10 +421,10 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax if ( pattern.hasSyntax() && curMatch == numMatches - 1 && end == fullMatchEnd ) { pushSubsyntax( - curState, retState, pattern, patternIndex + 1, + curState, retState, pattern, patternIndex, textv.substr( fullMatchStart, fullMatchEnd - fullMatchStart ) ); } else if ( pattern.patterns.size() > 1 ) { - setSubsyntaxPatternIdx( curState, retState, patternIndex + 1 ); + setSubsyntaxPatternIdx( curState, retState, patternIndex ); } startIdx = end; @@ -435,7 +436,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax if ( pattern.hasSyntax() && curMatch == numMatches - 1 ) { pushSubsyntax( - curState, retState, pattern, patternIndex + 1, + curState, retState, pattern, patternIndex, textv.substr( fullMatchStart, fullMatchEnd - fullMatchStart ) ); } } @@ -470,9 +471,9 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax patternText ); } if ( pattern.hasSyntax() ) { - pushSubsyntax( curState, retState, pattern, patternIndex + 1, patternText ); + pushSubsyntax( curState, retState, pattern, patternIndex, patternText ); } else if ( pattern.patterns.size() > 1 ) { - setSubsyntaxPatternIdx( curState, retState, patternIndex + 1 ); + setSubsyntaxPatternIdx( curState, retState, patternIndex ); } startIdx = end; return true; @@ -486,9 +487,9 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax size_t startIdx = startIndex; while ( startIdx < size ) { - if ( curState.currentPatternIdx != SYNTAX_TOKENIZER_STATE_NONE ) { + if ( curState.currentPatternIdx.state != SYNTAX_TOKENIZER_STATE_NONE ) { const SyntaxPattern& pattern = - curState.currentSyntax->getPatterns()[curState.currentPatternIdx - 1]; + *curState.currentSyntax->getPatternFromState( curState.currentPatternIdx ); auto range = findNonEscaped( text, pattern.patterns[1], startIdx, pattern.patterns.size() >= 3 ? pattern.patterns[2] : "", pattern.matchType ) @@ -528,7 +529,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax pushToken( tokens, pattern.types[0], textv.substr( startIdx, range.second - startIdx ) ); } - setSubsyntaxPatternIdx( curState, retState, SYNTAX_TOKENIZER_STATE_NONE ); + setSubsyntaxPatternIdx( curState, retState, SyntaxStateType{} ); startIdx = range.second; } else { pushToken( tokens, pattern.types[0], textv.substr( startIdx ) ); @@ -557,7 +558,29 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax if ( startIdx != 0 && pattern.patterns[0][0] == '^' ) continue; - if ( ( matched = matchPattern( pattern, startIdx, patternIndex ) ) ) + /* + if ( pattern.isRepositoryInclude() ) { + const auto& repo = + curState.currentSyntax->getRepository( pattern.getRepositoryName() ); + size_t repoPatternCount = repo.size(); + for ( size_t repoPatternIndex = 0; repoPatternIndex < repoPatternCount; + repoPatternIndex++ ) { + + if ( startIdx != 0 && repo[repoPatternIndex].patterns[0][0] == '^' ) + continue; + + if ( repo[repoPatternIndex].isPure() && + ( matched = + matchPattern( repo[repoPatternIndex], startIdx, + { static_cast( repoPatternIndex + 1 ), + static_cast( pattern.repositoryIdx ) } ) ) ) + break; + } + } + */ + + if ( ( matched = matchPattern( pattern, startIdx, + { static_cast( patternIndex + 1 ), 0 } ) ) ) break; } diff --git a/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/yaml.cpp b/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/yaml.cpp index c05b3fea1..26b9ab624 100644 --- a/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/yaml.cpp +++ b/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/yaml.cpp @@ -44,7 +44,7 @@ void addYAML() { }, "#", - {} + { "^%%YAML %d+%.%d+" } } );