From fbeadf7d7f47a0a95c3900f6b390f92f6dd6ef34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Fri, 6 Oct 2023 00:14:07 -0300 Subject: [PATCH] Optimizations in LuaPattern (don't allocate) and SyntaxTokenizer. Now it's possible to declare a dynamic syntax detection from it's context, this feature improves drastically Markdown parsing performance. --- include/eepp/core/string.hpp | 8 +++++++ include/eepp/system/luapattern.hpp | 17 +++++++++---- include/eepp/ui/doc/syntaxdefinition.hpp | 3 ++- src/eepp/core/string.cpp | 24 +++++++++++++++++-- src/eepp/system/luapattern.cpp | 12 ++++++---- src/eepp/ui/doc/languages/markdown.cpp | 4 ++-- src/eepp/ui/doc/syntaxdefinitionmanager.cpp | 5 +++- src/eepp/ui/doc/syntaxhighlighter.cpp | 2 +- src/eepp/ui/doc/syntaxtokenizer.cpp | 17 ++++++------- src/eepp/ui/doc/textdocument.cpp | 4 ++-- src/eepp/window/backend/SDL2/inputsdl2.cpp | 2 +- .../src/eepp/maps/mapeditor/mapeditor.cpp | 2 +- src/tests/test_all/test.cpp | 3 ++- 13 files changed, 74 insertions(+), 29 deletions(-) diff --git a/include/eepp/core/string.hpp b/include/eepp/core/string.hpp index 24309737e..653911915 100644 --- a/include/eepp/core/string.hpp +++ b/include/eepp/core/string.hpp @@ -352,6 +352,14 @@ class EE_API String { /** @return The number of codepoints of the utf8 string. */ static size_t utf8Length( const std::string& utf8String ); + /** @brief Construct from an UTF-8 string to UTF-32 according + ** @param utf8String UTF-8 string to convert + **/ + static String fromUtf8( const std::string_view& utf8String ); + + /** @return The number of codepoints of the utf8 string. */ + static size_t utf8Length( const std::string_view& utf8String ); + /** @return The next character in a utf8 null terminated string */ static Uint32 utf8Next( char*& utf8String ); diff --git a/include/eepp/system/luapattern.hpp b/include/eepp/system/luapattern.hpp index e31a8870a..17793d6bc 100644 --- a/include/eepp/system/luapattern.hpp +++ b/include/eepp/system/luapattern.hpp @@ -95,7 +95,7 @@ class EE_API LuaPattern { }; static std::string matchesAny( const std::vector& stringvec, - const std::string& pattern ); + const std::string& pattern ); static std::string match( const std::string& string, const std::string& pattern ); @@ -103,7 +103,7 @@ class EE_API LuaPattern { static bool matches( const std::string& string, const std::string& pattern ); - LuaPattern( const std::string& pattern ); + LuaPattern( const std::string_view& pattern ); bool matches( const char* stringSearch, int stringStartOffset, LuaPattern::Range* matchList, size_t stringLength ) const; @@ -122,7 +122,7 @@ class EE_API LuaPattern { bool range( int indexGet, int& startMatch, int& endMatch, LuaPattern::Range* returnedMatched ) const; - const std::string& getPatern() const { return mPattern; } + const std::string_view& getPatern() const { return mPattern; } LuaPattern::Match gmatch( const char* s ) &; @@ -137,11 +137,18 @@ class EE_API LuaPattern { std::string gsub( const std::string& text, const std::string& replace ); protected: - mutable std::string mErr; - std::string mPattern; + std::string_view mPattern; mutable size_t mMatchNum; }; +class EE_API LuaPatternStorage : public LuaPattern { + public: + LuaPatternStorage( const std::string& pattern ); + + protected: + std::string mPatternStorage; +}; + }} // namespace EE::System #endif // EE_SYSTEM_LUAPATTERNMATCHER_HPP diff --git a/include/eepp/ui/doc/syntaxdefinition.hpp b/include/eepp/ui/doc/syntaxdefinition.hpp index a72c46366..3788243c4 100644 --- a/include/eepp/ui/doc/syntaxdefinition.hpp +++ b/include/eepp/ui/doc/syntaxdefinition.hpp @@ -26,7 +26,8 @@ template static auto toSyntaxStyleTypeV( const std::vector& s ) } struct EE_API SyntaxPattern { - using DynamicSyntax = std::function; + using DynamicSyntax = + std::function; std::vector patterns; std::vector types; diff --git a/src/eepp/core/string.cpp b/src/eepp/core/string.cpp index c1c5f0959..081a30066 100644 --- a/src/eepp/core/string.cpp +++ b/src/eepp/core/string.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -1040,7 +1039,24 @@ String String::fromUtf8( const std::string& utf8String ) { return String( utf32 ); } -#define iscont( p ) ( ( *(p)&0xC0 ) == 0x80 ) +String String::fromUtf8( const std::string_view& utf8String ) { + String::StringType utf32; + + // Skip BOM + int skip = 0; + if ( utf8String.size() >= 3 && (char)0xef == utf8String[0] && (char)0xbb == utf8String[1] && + (char)0xbf == utf8String[2] ) { + skip = 3; + } + + utf32.reserve( utf8String.length() + 1 ); + + Utf8::toUtf32( utf8String.begin() + skip, utf8String.end(), std::back_inserter( utf32 ) ); + + return String( utf32 ); +} + +#define iscont( p ) ( ( *( p ) & 0xC0 ) == 0x80 ) static inline const char* utf8_next( const char* s, const char* e ) { while ( s < e && iscont( s + 1 ) ) @@ -1059,6 +1075,10 @@ size_t String::utf8Length( const std::string& utf8String ) { return utf8_length( utf8String.c_str(), utf8String.c_str() + utf8String.length() ); } +size_t String::utf8Length( const std::string_view& utf8String ) { + return utf8_length( utf8String.data(), utf8String.data() + utf8String.length() ); +} + Uint32 String::utf8Next( char*& utf8String ) { return utf8::unchecked::next( utf8String ); } diff --git a/src/eepp/system/luapattern.cpp b/src/eepp/system/luapattern.cpp index 69cab29ae..92a3e1276 100644 --- a/src/eepp/system/luapattern.cpp +++ b/src/eepp/system/luapattern.cpp @@ -29,7 +29,7 @@ std::string LuaPattern::match( const std::string& string, const std::string& pat } std::string LuaPattern::matchesAny( const std::vector& stringvec, - const std::string& pattern ) { + const std::string& pattern ) { LuaPattern matcher( pattern ); int start = 0, end = 0; for ( const auto& str : stringvec ) { @@ -52,7 +52,7 @@ bool LuaPattern::matches( const std::string& string, const std::string& pattern return find( string, pattern ).isValid(); } -LuaPattern::LuaPattern( const std::string& pattern ) : mPattern( pattern ) { +LuaPattern::LuaPattern( const std::string_view& pattern ) : mPattern( pattern ) { if ( !sFailHandlerInitialized ) { sFailHandlerInitialized = true; lua_str_fail_func( failHandler ); @@ -67,10 +67,9 @@ bool LuaPattern::matches( const char* stringSearch, int stringStartOffset, if ( stringLength == 0 ) stringLength = strlen( stringSearch ); try { - mMatchNum = lua_str_match( stringSearch, stringStartOffset, stringLength, mPattern.c_str(), + mMatchNum = lua_str_match( stringSearch, stringStartOffset, stringLength, mPattern.data(), (LuaMatch*)matchList ); } catch ( const std::string& patternError ) { - mErr = std::move( patternError ); mMatchNum = 0; } return mMatchNum == 0 ? false : true; @@ -240,4 +239,9 @@ std::string LuaPattern::gsub( const std::string& text, const std::string& replac return gsub( text.c_str(), replace.c_str() ); } +LuaPatternStorage::LuaPatternStorage( const std::string& pattern ) : + LuaPattern( "" ), mPatternStorage( pattern ) { + mPattern = std::string_view{ mPatternStorage }; +} + }} // namespace EE::System diff --git a/src/eepp/ui/doc/languages/markdown.cpp b/src/eepp/ui/doc/languages/markdown.cpp index eb7955438..854728fcc 100644 --- a/src/eepp/ui/doc/languages/markdown.cpp +++ b/src/eepp/ui/doc/languages/markdown.cpp @@ -4,8 +4,8 @@ namespace EE { namespace UI { namespace Doc { namespace Language { void addMarkdown() { - auto dynSyntax = []( const SyntaxPattern&, const std::string& match ) -> std::string { - std::string lang = String::toLower( match.substr( 3 ) ); + auto dynSyntax = []( const SyntaxPattern&, const std::string_view& match ) -> std::string { + std::string lang = String::toLower( std::string{ match.substr( 3 ) } ); String::trimInPlace( lang ); if ( !lang.empty() && lang[lang.size() - 1] == '\n' ) lang.pop_back(); diff --git a/src/eepp/ui/doc/syntaxdefinitionmanager.cpp b/src/eepp/ui/doc/syntaxdefinitionmanager.cpp index 605099a78..f026ce235 100644 --- a/src/eepp/ui/doc/syntaxdefinitionmanager.cpp +++ b/src/eepp/ui/doc/syntaxdefinitionmanager.cpp @@ -2317,9 +2317,12 @@ const SyntaxDefinition& SyntaxDefinitionManager::findFromString( const std::stri const auto& syn = getByLSPName( lang ); if ( syn.getLSPName() != getPlainDefinition().getLSPName() ) return syn; - const auto& syn2 = getByLanguageNameInsensitive( lang ); + const auto& syn2 = getByLanguageName( lang ); if ( syn2.getLSPName() != getPlainDefinition().getLSPName() ) return syn2; + const auto& syn3 = getByLanguageNameInsensitive( lang ); + if ( syn3.getLSPName() != getPlainDefinition().getLSPName() ) + return syn3; return getPlainDefinition(); } diff --git a/src/eepp/ui/doc/syntaxhighlighter.cpp b/src/eepp/ui/doc/syntaxhighlighter.cpp index b111ea123..7dcb1e493 100644 --- a/src/eepp/ui/doc/syntaxhighlighter.cpp +++ b/src/eepp/ui/doc/syntaxhighlighter.cpp @@ -128,7 +128,7 @@ void SyntaxHighlighter::tokenizeAsync( std::shared_ptr pool ) { return; mTokenizeAsync = true; pool->run( [this] { - for ( size_t i = 0; i < mDoc->linesCount() && !mStopTokenizing; i++ ) + for ( size_t i = mFirstInvalidLine; i < mDoc->linesCount() && !mStopTokenizing; i++ ) getLine( i ); mStopTokenizing = false; mTokenizeAsync = false; diff --git a/src/eepp/ui/doc/syntaxtokenizer.cpp b/src/eepp/ui/doc/syntaxtokenizer.cpp index 8f82e8b6f..7c8d696ed 100644 --- a/src/eepp/ui/doc/syntaxtokenizer.cpp +++ b/src/eepp/ui/doc/syntaxtokenizer.cpp @@ -32,7 +32,7 @@ static int isInMultiByteCodePoint( const char* text, const size_t& textSize, con template static void pushToken( std::vector& tokens, const SyntaxStyleType& type, - const std::string& text ) { + const std::string_view& text ) { if ( !tokens.empty() && ( tokens[tokens.size() - 1].type == type ) ) { size_t tpos = tokens.size() - 1; tokens[tpos].type = type; @@ -47,14 +47,14 @@ static void pushToken( std::vector& tokens, const SyntaxStyleType& type, while ( textSize > 0 ) { size_t chunkSize = textSize > MAX_TOKEN_SIZE ? MAX_TOKEN_SIZE : textSize; int multiByteCodePointPos = 0; - if ( ( multiByteCodePointPos = isInMultiByteCodePoint( text.c_str(), text.size(), + if ( ( multiByteCodePointPos = isInMultiByteCodePoint( text.data(), text.size(), pos + chunkSize ) ) > 0 ) { chunkSize = eemin( textSize, chunkSize + multiByteCodePointPos ); } - std::string substr = text.substr( pos, chunkSize ); + std::string_view substr = text.substr( pos, chunkSize ); SyntaxStyleType len = String::utf8Length( substr ); if constexpr ( std::is_same_v ) { - tokens.push_back( { type, std::move( substr ), len } ); + tokens.push_back( { type, std::string{ substr }, len } ); } else if constexpr ( std::is_same_v ) { SyntaxStyleType tpos = tokens.empty() ? 0 : tokens[tokens.size() - 1].pos + @@ -68,8 +68,8 @@ static void pushToken( std::vector& tokens, const SyntaxStyleType& type, } } else { if constexpr ( std::is_same_v ) { - tokens.push_back( - { type, text, static_cast( String::utf8Length( text ) ) } ); + tokens.push_back( { type, std::string{ text }, + static_cast( String::utf8Length( text ) ) } ); } else if constexpr ( std::is_same_v ) { SyntaxStyleType tpos = tokens.empty() ? 0 @@ -249,6 +249,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax bool matched = false; std::string patternStr; + std::string patternText; for ( size_t patternIndex = 0; patternIndex < curState.currentSyntax->getPatterns().size(); patternIndex++ ) { @@ -290,7 +291,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax text.substr( lastEnd, start - lastEnd ) ); } - std::string patternText( text.substr( start, end - start ) ); + patternText = text.substr( start, end - start ); SyntaxStyleType type = curState.currentSyntax->getSymbol( patternText ); if ( !skipSubSyntaxSeparator || !pattern.hasSyntax() ) { pushToken( tokens, @@ -335,7 +336,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax String::utf8Next( strEnd ); end = start + ( strEnd - strStart ); } - std::string patternText( text.substr( start, end - start ) ); + patternText = text.substr( start, end - start ); SyntaxStyleType type = curState.currentSyntax->getSymbol( patternText ); if ( !skipSubSyntaxSeparator || !pattern.hasSyntax() ) { pushToken( tokens, diff --git a/src/eepp/ui/doc/textdocument.cpp b/src/eepp/ui/doc/textdocument.cpp index 5fb6d5f8f..bac1a2faf 100644 --- a/src/eepp/ui/doc/textdocument.cpp +++ b/src/eepp/ui/doc/textdocument.cpp @@ -2108,7 +2108,7 @@ static std::pair findType( const String& str, const String& find const TextDocument::FindReplaceType& type ) { switch ( type ) { case TextDocument::FindReplaceType::LuaPattern: { - LuaPattern words( findStr ); + LuaPattern words( findStr.toUtf8() ); int start, end = 0; words.find( str, start, end ); if ( start < 0 ) @@ -2129,7 +2129,7 @@ static std::pair findLastType( const String& str, const String& switch ( type ) { case TextDocument::FindReplaceType::LuaPattern: { // TODO: Implement findLastType for Lua patterns - LuaPattern words( findStr ); + LuaPattern words( findStr.toUtf8() ); int start, end = 0; words.find( str, start, end ); if ( start < 0 ) diff --git a/src/eepp/window/backend/SDL2/inputsdl2.cpp b/src/eepp/window/backend/SDL2/inputsdl2.cpp index 2ad5f7590..e0e1e245c 100644 --- a/src/eepp/window/backend/SDL2/inputsdl2.cpp +++ b/src/eepp/window/backend/SDL2/inputsdl2.cpp @@ -241,7 +241,7 @@ void InputSDL::sendEvent( const SDL_Event& SDLEvent ) { break; } case SDL_TEXTINPUT: { - String txt = String::fromUtf8( SDLEvent.text.text ); + String txt = String::fromUtf8( std::string_view{ SDLEvent.text.text } ); event.Type = InputEvent::TextInput; event.text.timestamp = SDLEvent.text.timestamp; event.WinID = SDLEvent.text.windowID; diff --git a/src/modules/maps/src/eepp/maps/mapeditor/mapeditor.cpp b/src/modules/maps/src/eepp/maps/mapeditor/mapeditor.cpp index 8cea2afd6..c05f573a3 100644 --- a/src/modules/maps/src/eepp/maps/mapeditor/mapeditor.cpp +++ b/src/modules/maps/src/eepp/maps/mapeditor/mapeditor.cpp @@ -400,7 +400,7 @@ void MapEditor::createTextureRegionContainer( Int32 Width ) { ->setParent( mTextureRegionCont ) ->setPosition( mChkBlocked->getPosition().x, mChkBlocked->getPosition().y + mChkBlocked->getSize().getHeight() + 4 ); - mChkRot90->setText( String::fromUtf8( "Rotate 90º" ) ); + mChkRot90->setText( String::fromUtf8( std::string_view{ "Rotate 90º" } ) ); mChkRot90->addEventListener( Event::OnValueChange, cb::Make1( this, &MapEditor::chkClickRot90 ) ); diff --git a/src/tests/test_all/test.cpp b/src/tests/test_all/test.cpp index 9fd5c9542..5d24cd56c 100644 --- a/src/tests/test_all/test.cpp +++ b/src/tests/test_all/test.cpp @@ -1,4 +1,5 @@ #include "test.hpp" +using namespace std::literals; Demo_Test::EETest* TestInstance = NULL; @@ -269,7 +270,7 @@ void EETest::onFontLoaded() { "mejor, el real. Sufre porque es bueno y tiene compasión, lo ve y piensa: \"Pobre se está " "ahogando no puede respirar\". Y lo saca, lo saca y se queda tranquilo, por fin lo salvé. " "Pero el pez se retuerce de dolor y muere. Por eso te mostré el sueño, es imposible meter " - "el mar en tu cabeza, que es un balde." ); + "el mar en tu cabeza, que es un balde."sv ); createUI(); Con = UIConsole::NewOpt( monospace, true, true, 8191 );