diff --git a/include/eepp/core/string.hpp b/include/eepp/core/string.hpp index 3c26b63df..ab5ff6d5b 100644 --- a/include/eepp/core/string.hpp +++ b/include/eepp/core/string.hpp @@ -1087,10 +1087,18 @@ class EE_API String { std::function onSepChunkRead, char sep = '\n' ); + static void readBySeparatorStoppable( std::string_view buf, + std::function onSepChunkRead, + char sep = '\n' ); + static void readBySeparator( String::View buf, std::function onSepChunkRead, String::StringBaseType sep = L'\n' ); + static void readBySeparatorStoppable( String::View buf, + std::function onSepChunkRead, + String::StringBaseType sep = L'\n' ); + static size_t countLines( std::string_view text ); static size_t countLines( String::View text ); diff --git a/include/eepp/ui/doc/hextlanguagetype.hpp b/include/eepp/ui/doc/hextlanguagetype.hpp index 8f11a5b91..97f90347b 100644 --- a/include/eepp/ui/doc/hextlanguagetype.hpp +++ b/include/eepp/ui/doc/hextlanguagetype.hpp @@ -7,7 +7,7 @@ namespace EE { namespace UI { namespace Doc { enum class HExtLanguageType { AutoDetect, C, CPP, ObjectiveC, ObjectiveCPP }; struct EE_API HExtLanguageTypeHelper { - static HExtLanguageType detectLanguage( const std::string& buffer ); + static HExtLanguageType detectLanguage( std::string_view buffer ); static std::string toString( HExtLanguageType langType ); diff --git a/include/eepp/ui/doc/syntaxdefinitionmanager.hpp b/include/eepp/ui/doc/syntaxdefinitionmanager.hpp index b906c2957..431259341 100644 --- a/include/eepp/ui/doc/syntaxdefinitionmanager.hpp +++ b/include/eepp/ui/doc/syntaxdefinitionmanager.hpp @@ -37,10 +37,10 @@ class EE_API SyntaxDefinitionManager { const SyntaxDefinition& getByExtension( const std::string& filePath ) const; const SyntaxDefinition& - getByHeader( const std::string& header, const std::string& filePath = "", + getByHeader( std::string_view header, const std::string& filePath = "", HExtLanguageType hLangType = HExtLanguageType::AutoDetect ) const; - const SyntaxDefinition& find( const std::string& filePath, const std::string& header, + const SyntaxDefinition& find( const std::string& filePath, std::string_view header, HExtLanguageType hLangType = HExtLanguageType::AutoDetect ); const SyntaxDefinition& findFromString( const std::string_view& str ) const; @@ -84,6 +84,8 @@ class EE_API SyntaxDefinitionManager { const std::map& getLanguageExtensionsPriority(); + bool isFileFormatSupported( const std::string& filePath, std::string_view header ); + protected: SyntaxDefinitionManager( std::size_t reserveSpaceForLanguages = 12 ); @@ -95,8 +97,7 @@ class EE_API SyntaxDefinitionManager { std::optional getLanguageIndex( const std::string& langName ); const SyntaxDefinition* needsHFallback( HExtLanguageType langType, const std::string& lspName, - const std::string& ext, - const std::string& buffer ) const; + const std::string& ext, std::string_view buffer ) const; }; }}} // namespace EE::UI::Doc diff --git a/src/eepp/core/string.cpp b/src/eepp/core/string.cpp index 794114636..9da4fb599 100644 --- a/src/eepp/core/string.cpp +++ b/src/eepp/core/string.cpp @@ -2271,6 +2271,29 @@ void String::readBySeparator( std::string_view buf, } } +void String::readBySeparatorStoppable( std::string_view buf, + std::function onSepChunkRead, + char sep ) { + auto lastNL = 0; + auto nextNL = buf.find_first_of( sep ); + if ( nextNL != std::string_view::npos ) { + while ( nextNL != std::string_view::npos ) { + if ( onSepChunkRead( buf.substr( lastNL, nextNL - lastNL ) ) ) + return; + lastNL = nextNL + 1; + nextNL = buf.find_first_of( sep, nextNL + 1 ); + } + + if ( lastNL < static_cast( buf.size() ) ) { + if ( onSepChunkRead( buf.substr( lastNL ) ) ) + return; + } + } else { + if ( onSepChunkRead( buf ) ) + return; + } +} + size_t String::countLines( std::string_view text ) { const char* startPtr = text.data(); const char* endPtr = text.data() + text.size(); @@ -2294,6 +2317,19 @@ void String::readBySeparator( String::View buf, std::function onSepChunkRead, + String::StringBaseType sep ) { + auto lastNL = 0; + auto nextNL = buf.find_first_of( sep ); + while ( nextNL != String::View::npos ) { + if ( onSepChunkRead( buf.substr( lastNL, nextNL - lastNL ) ) ) + return; + lastNL = nextNL + 1; + nextNL = buf.find_first_of( sep, nextNL + 1 ); + } +} + size_t String::countLines( String::View text ) { const String::StringBaseType* startPtr = text.data(); const String::StringBaseType* endPtr = text.data() + text.size(); diff --git a/src/eepp/graphics/font.cpp b/src/eepp/graphics/font.cpp index 7297630c5..1de086f5c 100644 --- a/src/eepp/graphics/font.cpp +++ b/src/eepp/graphics/font.cpp @@ -4,18 +4,61 @@ #include #include +#include + namespace EE { namespace Graphics { +// Maximum Unicode code point for emojis (covering up to U+1FAFF) +constexpr size_t MAX_EMOJI_CODE_POINT = 0x1FAFF + 1; // 129791 + +// Static bitset to store emoji code points +static const std::bitset emojiLookup = []() { + std::bitset bits; + + // Define emoji ranges (Unicode 15.1, tailored for Noto Sans Color Emoji) + static constexpr struct { + uint32_t min, max; + } ranges[] = { + { 0x1F300, 0x1F5FF }, // Miscellaneous Symbols and Pictographs + { 0x1F600, 0x1F64F }, // Emoticons + { 0x1F680, 0x1F6FF }, // Transport and Map Symbols + { 0x1F900, 0x1F9FF }, // Supplemental Symbols and Pictographs + { 0x1FA70, 0x1FAFF }, // Symbols and Pictographs Extended-A + { 0x2600, 0x26FF }, // Miscellaneous Symbols + { 0x2700, 0x27BF }, // Dingbats + { 0x1F1E6, 0x1F1FF }, // Regional Indicator Symbols + { 0x1F000, 0x1F02F }, // Mahjong Tiles, Domino Tiles + { 0x1F0A0, 0x1F0FF }, // Playing Cards + { 0x1F100, 0x1F1FF }, // Enclosed Alphanumeric Supplement (partial) + { 0x1F200, 0x1F2FF }, // Enclosed Ideographic Supplement (partial) + }; + + // Define single emoji code points + static constexpr uint32_t singles[] = { + 0x00A9, 0x00AE, 0x203C, 0x2049, 0x2122, 0x2139, 0x231A, 0x231B, 0x2328, + 0x23CF, 0x23E9, 0x23F0, 0x23F3, 0x25AA, 0x25AB, 0x25B6, 0x25C0, 0x25FB, + 0x25FC, 0x25FD, 0x25FE, 0x2B50, 0x2B55, 0x3030, 0x303D, 0x3297, 0x3299, + }; + + // Set bits for ranges + for ( const auto& range : ranges ) + for ( uint32_t i = range.min; i <= range.max; ++i ) + bits.set( i ); + + // Set bits for single code points + for ( const auto& code : singles ) + bits.set( code ); + + // Set bits for emoji variation selector and ZWJ + bits.set( 0xFE0F ); // Variation Selector-16 + bits.set( 0x200D ); // Zero Width Joiner + + return bits; +}(); + bool Font::isEmojiCodePoint( const Uint32& codePoint ) { - static constexpr Uint32 rangeMin = 127744; - static constexpr Uint32 rangeMax = 131069; - static constexpr Uint32 rangeMin2 = 126980; - static constexpr Uint32 rangeMax2 = 127569; - static constexpr Uint32 rangeMin3 = 8986; - static constexpr Uint32 rangeMax3 = 12953; - return codePoint >= 8986 && ( ( rangeMin <= codePoint && codePoint <= rangeMax ) || - ( rangeMin2 <= codePoint && codePoint <= rangeMax2 ) || - ( rangeMin3 <= codePoint && codePoint <= rangeMax3 ) ); + // Check if code point is within valid range and is an emoji + return codePoint < MAX_EMOJI_CODE_POINT && emojiLookup[codePoint]; } bool Font::containsEmojiCodePoint( const String& string ) { diff --git a/src/eepp/ui/doc/hextlanguagetype.cpp b/src/eepp/ui/doc/hextlanguagetype.cpp index 2e27139e0..a1d18ca16 100644 --- a/src/eepp/ui/doc/hextlanguagetype.cpp +++ b/src/eepp/ui/doc/hextlanguagetype.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include namespace EE { namespace UI { namespace Doc { @@ -11,7 +11,7 @@ inline bool isWordBoundary( char c ) { } // Finds a keyword as a whole word (e.g., finds "class" but not "subclass"). -bool findStandaloneWord( const std::string& line, std::string_view keyword ) { +bool findStandaloneWord( std::string_view line, std::string_view keyword ) { size_t pos = 0; while ( ( pos = line.find( keyword, pos ) ) != std::string::npos ) { // Check character before the keyword @@ -36,10 +36,10 @@ bool findStandaloneWord( const std::string& line, std::string_view keyword ) { * to distinguish between C, C++, Objective-C, and Objective-C++. It uses a * fast, heuristic-based approach optimized with std::array and std::string_view. * - * @param buffer A std::string containing the source code from the .h file. + * @param buffer A std::string_view containing the source code from the .h file. * @return HExtLanguageType The detected language. */ -HExtLanguageType HExtLanguageTypeHelper::detectLanguage( const std::string& buffer ) { +HExtLanguageType HExtLanguageTypeHelper::detectLanguage( std::string_view buffer ) { bool hasCppFeature = false; bool hasObjcFeature = false; @@ -54,12 +54,10 @@ HExtLanguageType HExtLanguageTypeHelper::detectLanguage( const std::string& buff "final", "public", "private", "protected" }; static constexpr std::array cppTokens = { "::", "template<" }; - std::stringstream ss( buffer ); - std::string line; bool inMultilineComment = false; - while ( std::getline( ss, line ) ) { - std::string processedLine = line; + String::readBySeparatorStoppable( buffer, [&]( std::string_view line ) { + std::string_view processedLine = line; if ( inMultilineComment ) { size_t endCommentPos = processedLine.find( "*/" ); @@ -67,23 +65,31 @@ HExtLanguageType HExtLanguageTypeHelper::detectLanguage( const std::string& buff processedLine = processedLine.substr( endCommentPos + 2 ); inMultilineComment = false; } else { - continue; + return false; // Skip the line, still in multiline comment } } - size_t startCommentPos = processedLine.find( "/*" ); - if ( startCommentPos != std::string::npos ) { + // Handle multiline comments (/* ... */) + while ( true ) { + size_t startCommentPos = processedLine.find( "/*" ); + if ( startCommentPos == std::string_view::npos ) { + break; // No more multiline comments + } size_t endCommentPos = processedLine.find( "*/", startCommentPos ); - if ( endCommentPos != std::string::npos ) { - processedLine.erase( startCommentPos, endCommentPos - startCommentPos + 2 ); + if ( endCommentPos != std::string_view::npos ) { + // Skip the comment by creating a view after the comment + processedLine = processedLine.substr( endCommentPos + 2 ); } else { + // Multiline comment extends to next line inMultilineComment = true; processedLine = processedLine.substr( 0, startCommentPos ); + break; } } + // Handle single-line comments (//) size_t commentPos = processedLine.find( "//" ); - if ( commentPos != std::string::npos ) { + if ( commentPos != std::string_view::npos ) { processedLine = processedLine.substr( 0, commentPos ); } @@ -91,11 +97,11 @@ HExtLanguageType HExtLanguageTypeHelper::detectLanguage( const std::string& buff std::string_view lineView = processedLine; lineView.remove_prefix( std::min( lineView.find_first_not_of( " \t" ), lineView.size() ) ); - if ( lineView.rfind( objcDirective, 0 ) == 0 ) { + if ( lineView.starts_with( objcDirective ) ) { hasObjcFeature = true; } else { for ( const auto& keyword : objcKeywords ) { - if ( processedLine.find( keyword ) != std::string::npos ) { + if ( processedLine.find( keyword ) != std::string_view::npos ) { hasObjcFeature = true; break; } @@ -112,7 +118,7 @@ HExtLanguageType HExtLanguageTypeHelper::detectLanguage( const std::string& buff } if ( !hasCppFeature ) { for ( const auto& token : cppTokens ) { - if ( processedLine.find( token ) != std::string::npos ) { + if ( processedLine.find( token ) != std::string_view::npos ) { hasCppFeature = true; break; } @@ -120,16 +126,15 @@ HExtLanguageType HExtLanguageTypeHelper::detectLanguage( const std::string& buff } if ( !hasCppFeature ) { size_t ampPos = processedLine.find( '&' ); - if ( ampPos != std::string::npos && ampPos + 1 < processedLine.length() && + if ( ampPos != std::string_view::npos && ampPos + 1 < processedLine.length() && processedLine[ampPos + 1] != '&' ) { hasCppFeature = true; } } } - if ( hasCppFeature && hasObjcFeature ) - break; - } + return hasCppFeature && hasObjcFeature; // Stop if both features are found + } ); if ( hasCppFeature && hasObjcFeature ) return HExtLanguageType::ObjectiveCPP; diff --git a/src/eepp/ui/doc/languages/javascript.cpp b/src/eepp/ui/doc/languages/javascript.cpp index 9d23c3476..7b87d9b1c 100644 --- a/src/eepp/ui/doc/languages/javascript.cpp +++ b/src/eepp/ui/doc/languages/javascript.cpp @@ -58,7 +58,7 @@ void addJavaScript() { }, "//", - {} + { "^#!.*[ /]env node", "^#!.*[ /]env bun", "^#!.*[ /]env deno" } } ); diff --git a/src/eepp/ui/doc/syntaxdefinitionmanager.cpp b/src/eepp/ui/doc/syntaxdefinitionmanager.cpp index 8c1aa34dd..1e5cfad17 100644 --- a/src/eepp/ui/doc/syntaxdefinitionmanager.cpp +++ b/src/eepp/ui/doc/syntaxdefinitionmanager.cpp @@ -1234,7 +1234,7 @@ bool SyntaxDefinitionManager::extensionCanRepresentManyLanguages( std::string ex const SyntaxDefinition* SyntaxDefinitionManager::needsHFallback( HExtLanguageType langType, const std::string& lspName, const std::string& ext, - const std::string& buffer ) const { + std::string_view buffer ) const { if ( lspName != "c" || !( ext == "h" || ext == ".h" || ext == "%.h$" || ext == "%.h%.in$" ) ) return nullptr; switch ( langType ) { @@ -1337,7 +1337,7 @@ SyntaxDefinitionManager::getByExtension( const std::string& filePath ) const { return def != nullptr ? *def : *mDefinitions[0].get(); } -const SyntaxDefinition& SyntaxDefinitionManager::getByHeader( const std::string& header, +const SyntaxDefinition& SyntaxDefinitionManager::getByHeader( std::string_view header, const std::string& filePath, HExtLanguageType langType ) const { if ( !header.empty() ) { @@ -1354,7 +1354,7 @@ const SyntaxDefinition& SyntaxDefinitionManager::getByHeader( const std::string& for ( const auto& hdr : definition->get()->getHeaders() ) { LuaPattern words( hdr ); int start, end; - if ( words.find( header, start, end ) ) { + if ( words.find( header.data(), start, end ) ) { return *definition->get(); } } @@ -1371,7 +1371,7 @@ const SyntaxDefinition& SyntaxDefinitionManager::getByHeader( const std::string& for ( const auto& hdr : preDefinition->getHeaders() ) { LuaPattern words( hdr ); int start, end; - if ( words.find( header, start, end ) ) { + if ( words.find( header.data(), start, end ) ) { return preDefinition->load(); } } @@ -1381,7 +1381,7 @@ const SyntaxDefinition& SyntaxDefinitionManager::getByHeader( const std::string& } const SyntaxDefinition& SyntaxDefinitionManager::find( const std::string& filePath, - const std::string& header, + std::string_view header, HExtLanguageType langType ) { const SyntaxDefinition& def = getByHeader( header, filePath, langType ); if ( def.getLanguageName() == mDefinitions[0]->getLanguageName() ) @@ -1411,4 +1411,9 @@ std::size_t SyntaxDefinitionManager::count() const { return mDefinitions.size(); } +bool SyntaxDefinitionManager::isFileFormatSupported( const std::string& filePath, + std::string_view header ) { + return &find( filePath, header ) != mDefinitions[0].get(); +} + }}} // namespace EE::UI::Doc diff --git a/src/eepp/ui/doc/textdocument.cpp b/src/eepp/ui/doc/textdocument.cpp index ca51c3ebf..d2ce58e47 100644 --- a/src/eepp/ui/doc/textdocument.cpp +++ b/src/eepp/ui/doc/textdocument.cpp @@ -96,9 +96,14 @@ bool TextDocument::fileMightBeBinary( const std::string& file ) { ++nonPrintableCount; } } - // Consider file binary if >10% of characters are non-printable - if ( nonPrintableCount > bytesRead * 0.1 ) { - return true; + + // Consider file binary if >20% of characters are non-printable + if ( nonPrintableCount > bytesRead * 0.2 ) { + // Also white-list known extensions + if ( !SyntaxDefinitionManager::instance()->isFileFormatSupported( + file, std::string_view{ buffer.data(), buffer.size() } ) ) { + return true; + } } return false; // Likely a text file @@ -397,6 +402,12 @@ TextDocument::LoadStatus TextDocument::loadFromStream( IOStream& file, std::stri lineBuffer.resize( lineBufferSize - 1 ); } else if ( mLineEnding == TextFormat::LineEnding::CR && lineBufferSize > 0 ) { lineBuffer[lineBuffer.size() - 1] = '\n'; + } else if ( mLineEnding == TextFormat::LineEnding::LF ) { + if ( lineBufferSize > 1 && lineBuffer[lineBufferSize - 2] == '\r' && + lastChar == '\n' ) { + lineBuffer.pop_back(); + lineBuffer[lineBuffer.size() - 1] = '\n'; + } } { @@ -523,7 +534,8 @@ const SyntaxDefinition& TextDocument::guessSyntax() const { { { 0, 0 }, positionOffset( { 0, 0 }, FileSystem::fileExtension( mFilePath ) == "h" ? 5 * 1024 : 128 ) } ) ); - return SyntaxDefinitionManager::instance()->find( mFilePath, header, mHExtLanguageType ); + return SyntaxDefinitionManager::instance()->find( mFilePath, header.toUtf8(), + mHExtLanguageType ); } void TextDocument::resetSyntax() { @@ -534,8 +546,8 @@ void TextDocument::resetSyntax() { std::string oldDef = mSyntaxDefinition.getLSPName(); { Lock l( mSyntaxDefinitionMutex ); - mSyntaxDefinition = - SyntaxDefinitionManager::instance()->find( mFilePath, header, mHExtLanguageType ); + mSyntaxDefinition = SyntaxDefinitionManager::instance()->find( mFilePath, header.toUtf8(), + mHExtLanguageType ); } if ( mSyntaxDefinition.getLSPName() != oldDef ) notifySyntaxDefinitionChange();