From f721ef92df2bbb9481642449a12aaf7ef8a6a5a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Fri, 28 Mar 2025 02:15:20 -0300 Subject: [PATCH] Added a new tokenization method for the SyntaxTokenizer based on customized parsers. Initially is being used to parse numbers in C, C++ and JavaScript/TypeScript. This method is much more flexible and can be much faster than regex or lua patterns. Improvets in AI Assistant. Fix regression in size calculation of a UIAbstractTableView. Fix in project search path filters. --- include/eepp/system.hpp | 1 + include/eepp/system/parsermatcher.hpp | 59 ++ include/eepp/system/patternmatcher.hpp | 2 +- include/eepp/ui/doc/syntaxdefinition.hpp | 2 +- src/eepp/system/parsermatcher.cpp | 880 ++++++++++++++++++ src/eepp/system/patternmatcher.cpp | 4 + src/eepp/ui/abstract/uiabstracttableview.cpp | 2 +- src/eepp/ui/doc/languages/c.cpp | 16 +- src/eepp/ui/doc/languages/cpp.cpp | 21 +- src/eepp/ui/doc/languages/javascript.cpp | 5 +- src/eepp/ui/doc/syntaxdefinitionmanager.cpp | 41 +- src/eepp/ui/doc/syntaxtokenizer.cpp | 28 +- src/eepp/window/engine.cpp | 3 + .../src/eepp/ui/doc/languages/typescript.cpp | 5 +- .../ecode/plugins/aiassistant/chatui.cpp | 168 +++- .../ecode/plugins/aiassistant/chatui.hpp | 14 +- src/tools/ecode/projectsearch.cpp | 13 +- 17 files changed, 1164 insertions(+), 100 deletions(-) create mode 100644 include/eepp/system/parsermatcher.hpp create mode 100644 src/eepp/system/parsermatcher.cpp diff --git a/include/eepp/system.hpp b/include/eepp/system.hpp index eef8b13bd..eceb278af 100644 --- a/include/eepp/system.hpp +++ b/include/eepp/system.hpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include diff --git a/include/eepp/system/parsermatcher.hpp b/include/eepp/system/parsermatcher.hpp new file mode 100644 index 000000000..2eea279cb --- /dev/null +++ b/include/eepp/system/parsermatcher.hpp @@ -0,0 +1,59 @@ +#ifndef EE_SYSTEM_PARSERMATCHER_HPP +#define EE_SYSTEM_PARSERMATCHER_HPP + +#include +#include +#include + +#include +#include + +namespace EE { namespace System { + +using ParserMatcherFn = + std::function; + +class EE_API ParserMatcherManager { + SINGLETON_DECLARE_HEADERS( ParserMatcherManager ); + + public: + void registerBaseParsers(); + + void registerParser( std::string_view parserName, ParserMatcherFn fn ); + + bool hasParser( std::string_view parserName ) const; + + size_t matches( std::string_view parserName, const char* stringSearch, int stringStartOffset, + PatternMatcher::Range* matchList, size_t stringLength ); + + protected: + UnorderedMap mFns; +}; + +class EE_API ParserMatcher : public PatternMatcher { + public: + ParserMatcher( const std::string_view& parserName ); + + virtual ~ParserMatcher() override; + + virtual bool isValid() const override; + + virtual bool matches( const char* stringSearch, int stringStartOffset, + PatternMatcher::Range* matchList, size_t stringLength ) const override; + + virtual bool matches( const std::string& str, PatternMatcher::Range* matchList = nullptr, + int stringStartOffset = 0 ) const override; + + virtual const size_t& getNumMatches() const override; + + const std::string_view& getPattern() const override { return mParserName; } + + protected: + std::string_view mParserName; + mutable size_t mMatchNum; +}; + +}} // namespace EE::System + +#endif // EE_SYSTEM_PARSERMATCHER_HPP diff --git a/include/eepp/system/patternmatcher.hpp b/include/eepp/system/patternmatcher.hpp index 4f19cae98..3bfe04fd2 100644 --- a/include/eepp/system/patternmatcher.hpp +++ b/include/eepp/system/patternmatcher.hpp @@ -11,7 +11,7 @@ namespace EE { namespace System { class EE_API PatternMatcher { public: - enum class PatternType { LuaPattern, PCRE }; + enum class PatternType { LuaPattern, PCRE, Parser }; struct EE_API Range { int start{ -1 }; diff --git a/include/eepp/ui/doc/syntaxdefinition.hpp b/include/eepp/ui/doc/syntaxdefinition.hpp index a0165039e..16148d42b 100644 --- a/include/eepp/ui/doc/syntaxdefinition.hpp +++ b/include/eepp/ui/doc/syntaxdefinition.hpp @@ -26,7 +26,7 @@ template static auto toSyntaxStyleTypeV( const std::vector& s ) return std::vector{}; } -enum class SyntaxPatternMatchType { LuaPattern, RegEx }; +enum class SyntaxPatternMatchType { LuaPattern, RegEx, Parser }; struct EE_API SyntaxPattern { static UnorderedMap SyntaxStyleTypeCache; diff --git a/src/eepp/system/parsermatcher.cpp b/src/eepp/system/parsermatcher.cpp new file mode 100644 index 000000000..6406aebc4 --- /dev/null +++ b/src/eepp/system/parsermatcher.cpp @@ -0,0 +1,880 @@ +#include + +namespace EE { namespace System { + +// Enum to specify the language standard context +enum LanguageStandard { + LANG_C, // Target C (roughly C99/C11 - no binary, no separators, no 'z' suffix) + LANG_CPP // Target C++ (roughly C++14/C++23 - includes binary, separators, 'z' suffix) +}; + +// Helper functions (needed by the logic, inferred from original code) +namespace { // Use an anonymous namespace to keep helpers local to this translation unit + +inline bool isBinaryDigit( char c ) { + return c == '0' || c == '1'; +} + +inline bool isOctalDigit( char c ) { + return c >= '0' && c <= '7'; +} + +// Checks if a character 'c' is a valid digit for the given base. +// Handles C++ digit separators (') within the main logic, not here. +inline bool isValidDigitChar( char c, int base ) { + switch ( base ) { + case 2: + return isBinaryDigit( c ); + case 8: + return isOctalDigit( c ); + case 10: + return std::isdigit( c ); + case 16: + return std::isxdigit( c ); + default: + return false; + } +} + +/* + consumeDigitsWithSep + + Consumes digits (or letters for hex) in the supplied input string starting at pos, + while allowing numeric separators ('_') under these restrictions: + - An underscore is allowed only between valid digits. + - Not allowed at the start or end of the sequence. + - Not allowed consecutively. + + The flag "requireDigits" indicates if at least one digit is required to be present. + + Parameters: + input - the C-string to parse. + pos - current position (will be updated). + length - total length of input. + base - numeric base (2, 8, 10, or 16). + requireDigits - if true, at least one digit is required. + + Returns: + true if a valid sequence according to the rules was consumed, false otherwise. + Updates pos to point after the consumed sequence ONLY on success. + On failure, the value of pos is undefined/unreliable. +*/ +inline bool consumeDigitsWithSep( const char* input, int& pos, int length, int base, + bool requireDigits ) { + int currentPos = pos; // Use temporary position to avoid altering pos on failure + int startPos = currentPos; + int digitCount = 0; + bool lastWasDigit = false; // Track if the immediately preceding char was a digit + + while ( currentPos < length ) { + char c = input[currentPos]; + if ( c == '_' ) { + // Invalid if: + // 1. It's the first character being consumed in this sequence. + // 2. The immediately preceding character was not a digit. + if ( currentPos == startPos || !lastWasDigit ) { + return false; // Invalid underscore placement (start or consecutive) + } + // Valid underscore position, consume it. + lastWasDigit = false; // Next char must be a digit + currentPos++; + } else if ( isValidDigitChar( c, base ) ) { + digitCount++; + lastWasDigit = true; // Mark that we saw a digit + currentPos++; + } else { + break; // Not a digit or underscore, end of sequence for this part + } + } + + // After the loop, check validity: + // 1. Cannot end with an underscore + // This is implicitly checked by !lastWasDigit. If the loop ended normally + // (not via break) or broke on a non-digit/non-underscore, and the last + // character *consumed* was '_', then lastWasDigit will be false. + if ( !lastWasDigit && currentPos > startPos ) { + // Check if the loop didn't just break on a valid char (e.g. '.' or 'e') + // If the previous char was indeed '_', it's an error. + if ( input[currentPos - 1] == '_' ) { + return false; // Invalid: ends with underscore + } + } + + // 2. Check if digits were required and if we got any + if ( requireDigits && digitCount == 0 ) { + return false; // Required digits but found none + } + + // If we reached here, the sequence consumed (if any) is valid. + // Update the original pos. + pos = currentPos; + return true; +} + +} // anonymous namespace + +/** + isNumberLiteralJS checks if a substring (starting at stringStartOffset) + in the given C-string is a valid JavaScript/TypeScript number literal. + + It supports: + - Decimal literal, including fractional parts and exponent. + - Hexadecimal (0x or 0X), binary (0b or 0B), octal (0o or 0O) literals. + - BigInt literal with trailing "n", but only on integer literals. + - Numeric separators "_" between digits (but not multiple in a row, not + at the start or end of a digit sequence, and not immediately after a prefix). + + If a valid literal is found, the function writes the start and end indices + (zero-based positions into stringSearch) to matchList, and returns 1. + If no valid number literal is found at the offset, the function returns 0. +*/ +inline size_t isNumberLiteralJS( const char* stringSearch, int stringStartOffset, + PatternMatcher::Range* matchList, size_t stringLength ) { + if ( stringStartOffset < 0 || (size_t)stringStartOffset >= stringLength ) + return 0; + + int pos = stringStartOffset; + const int start = pos; // Keep original start index + + bool isSigned = false; + bool hasDecimalPoint = false; + bool hasExponent = false; + bool consumedSomethingAfterSign = + false; // Track if any part of number (digit, dot) is consumed after sign + + // 1. Check for optional leading sign (+ or -) + if ( stringSearch[pos] == '+' || stringSearch[pos] == '-' ) { + isSigned = true; + pos++; + if ( pos >= (int)stringLength ) + return 0; // Sign alone is invalid + } + + // Store the position after the potential sign + int afterSignPos = pos; + + // Cannot have underscore immediately after sign + if ( pos < (int)stringLength && stringSearch[pos] == '_' ) { + return 0; + } + + // 2. Handle different literal types based on the character AFTER the sign (if any) + if ( stringSearch[afterSignPos] == '0' ) { + // Potential 0, 0x, 0b, 0o, 0.123, 0e5, 0123 (decimal) + if ( isSigned && ( pos + 1 < (int)stringLength && + ( stringSearch[pos + 1] == 'x' || stringSearch[pos + 1] == 'X' || + stringSearch[pos + 1] == 'b' || stringSearch[pos + 1] == 'B' || + stringSearch[pos + 1] == 'o' || stringSearch[pos + 1] == 'O' ) ) ) { + // Signed non-decimal (e.g., +0x1) is invalid in JS/TS + return 0; + } + + pos++; // Consume '0' + consumedSomethingAfterSign = true; + + if ( pos < (int)stringLength ) { + char next = stringSearch[pos]; + int base = 0; // 0 indicates potential decimal or just '0' + + if ( next == 'x' || next == 'X' ) + base = 16; + else if ( next == 'b' || next == 'B' ) + base = 2; + else if ( next == 'o' || next == 'O' ) + base = 8; + + if ( base != 0 ) { // Hex, Bin, Octal (0x, 0b, 0o) + pos++; // Consume 'x'/'b'/'o' + // Cannot have underscore right after prefix + if ( pos < (int)stringLength && stringSearch[pos] == '_' ) + return 0; + if ( !consumeDigitsWithSep( stringSearch, pos, stringLength, base, true ) ) + return 0; // Must have digits & valid separators + // No fractional or exponent allowed for these bases + } else { // Decimal starting with '0' (e.g., 0, 0123, 0.5, 0e1) + // Check for more digits (e.g. 0123) or invalid things like 0_... + if ( !consumeDigitsWithSep( stringSearch, pos, stringLength, 10, false ) ) { + // Failed immediately after '0', likely invalid separator like "0_1" + return 0; + } + // Now check for optional fractional part. + if ( pos < (int)stringLength && stringSearch[pos] == '.' ) { + // Check for underscore immediately before dot (invalid) e.g. "1_"." + if ( pos > afterSignPos && stringSearch[pos - 1] == '_' ) + return 0; + + hasDecimalPoint = true; + pos++; + consumedSomethingAfterSign = true; // Consumed dot + + // Check for underscore immediately after dot (invalid) e.g. "1"._" + if ( pos < (int)stringLength && stringSearch[pos] == '_' ) + return 0; + + // Digits after dot are optional if we started with '0'. e.g. "0." is valid + if ( !consumeDigitsWithSep( stringSearch, pos, stringLength, 10, false ) ) + return 0; // Check separators + } + + // Now check for optional exponent part. + if ( pos < (int)stringLength && + ( stringSearch[pos] == 'e' || stringSearch[pos] == 'E' ) ) { + // Check for underscore immediately before 'e' (invalid) e.g. "1_"e + if ( pos > afterSignPos && stringSearch[pos - 1] == '_' ) + return 0; + + hasExponent = true; + pos++; + consumedSomethingAfterSign = true; // Consumed 'e' + + // Optional sign for exponent + if ( pos < (int)stringLength && + ( stringSearch[pos] == '+' || stringSearch[pos] == '-' ) ) { + pos++; + } + + // Check for underscore immediately after 'e'/'E' or exponent sign (invalid) + // e.g. "1e_" or "1e+_" + if ( pos < (int)stringLength && stringSearch[pos] == '_' ) + return 0; + + // Exponent *must* have digits + if ( !consumeDigitsWithSep( stringSearch, pos, stringLength, 10, true ) ) + return 0; + } + // If we got here, it's a valid decimal form starting with 0. + } + } else { + // Input is just "0" (or "+0" / "-0") + // pos is already correct (after '0') + } + + } else if ( std::isdigit( stringSearch[afterSignPos] ) ) { + // Decimal starting with 1-9 (potentially signed) + pos = afterSignPos; // Start consuming from the first digit + if ( !consumeDigitsWithSep( stringSearch, pos, stringLength, 10, true ) ) + return 0; // Must have digits & valid separators + consumedSomethingAfterSign = true; + + // Optional fractional part. + if ( pos < (int)stringLength && stringSearch[pos] == '.' ) { + // Check for underscore immediately before dot (invalid) e.g. "1_"." + if ( pos > afterSignPos && stringSearch[pos - 1] == '_' ) + return 0; + + hasDecimalPoint = true; + pos++; + consumedSomethingAfterSign = true; // Consumed dot + + // Check for underscore immediately after dot (invalid) e.g. "1"._" + if ( pos < (int)stringLength && stringSearch[pos] == '_' ) + return 0; + + // Fractional digits are optional if integer part exists (e.g., "123.") + if ( !consumeDigitsWithSep( stringSearch, pos, stringLength, 10, false ) ) + return 0; // Check separators + } + + // Optional exponent part. + if ( pos < (int)stringLength && ( stringSearch[pos] == 'e' || stringSearch[pos] == 'E' ) ) { + // Check for underscore immediately before 'e' (invalid) e.g. "1_"e or "1.5_"e + if ( pos > afterSignPos && stringSearch[pos - 1] == '_' ) + return 0; + + hasExponent = true; + pos++; + consumedSomethingAfterSign = true; // Consumed 'e' + + // Optional sign for exponent + if ( pos < (int)stringLength && + ( stringSearch[pos] == '+' || stringSearch[pos] == '-' ) ) { + pos++; + } + + // Check for underscore immediately after 'e'/'E' or exponent sign (invalid) e.g. "1e_" + // or "1e+_" + if ( pos < (int)stringLength && stringSearch[pos] == '_' ) + return 0; + + // Exponent *must* have digits + if ( !consumeDigitsWithSep( stringSearch, pos, stringLength, 10, true ) ) + return 0; + } + + } else if ( stringSearch[afterSignPos] == '.' ) { + // Decimal starting with '.' (potentially signed) + // Cannot have underscore immediately before dot (checked earlier by afterSignPos check) + pos = afterSignPos + 1; // Consume '.' + hasDecimalPoint = true; + consumedSomethingAfterSign = true; // Consumed dot + + // Check for underscore immediately after dot (invalid) e.g. "._" + if ( pos < (int)stringLength && stringSearch[pos] == '_' ) + return 0; + + // Must have digits *after* the dot if it's the start (e.g., ".5", "+.5") + if ( !consumeDigitsWithSep( stringSearch, pos, stringLength, 10, true ) ) + return 0; // Require digits & valid separators + + // Optional exponent part. + if ( pos < (int)stringLength && ( stringSearch[pos] == 'e' || stringSearch[pos] == 'E' ) ) { + // Check for underscore immediately before 'e' (invalid) e.g. ".5_"e + if ( pos > afterSignPos + 1 && stringSearch[pos - 1] == '_' ) + return 0; // pos > afterSignPos+1 ensures we had digits after '.' + + hasExponent = true; + pos++; + consumedSomethingAfterSign = true; // Consumed 'e' + + // Optional sign for exponent + if ( pos < (int)stringLength && + ( stringSearch[pos] == '+' || stringSearch[pos] == '-' ) ) { + pos++; + } + + // Check for underscore immediately after 'e'/'E' or exponent sign (invalid) e.g. ".1e_" + // or ".1e+_" + if ( pos < (int)stringLength && stringSearch[pos] == '_' ) + return 0; + + // Exponent *must* have digits + if ( !consumeDigitsWithSep( stringSearch, pos, stringLength, 10, true ) ) + return 0; + } + } else { + // Invalid character after sign (or invalid starting character if not signed) + return 0; + } + + // If signed, we must have consumed something after the sign + if ( isSigned && !consumedSomethingAfterSign ) + return 0; + // If not signed, we must have consumed *something* (digit or dot followed by digit) + if ( !isSigned && pos == start ) + return 0; // Handles "." case correctly + + // Optional BigInt suffix 'n' + if ( pos < (int)stringLength && stringSearch[pos] == 'n' ) { + // BigInt 'n' is only allowed for integer literals (no decimal point, no exponent) + if ( hasDecimalPoint || hasExponent ) { + return 0; // Invalid BigInt syntax + } + // Check if the character *before* 'n' was an underscore - invalid + if ( pos > start && stringSearch[pos - 1] == '_' ) { + return 0; + } + pos++; // Consume 'n' + } + + // Final check: Ensure we actually consumed something valid beyond just a sign + // This is implicitly covered by the logic within the branches and the + // consumedSomethingAfterSign check. + + // Success + matchList->start = start; + matchList->end = pos; // end is exclusive index + + return 1; +} + +/** + * @brief Checks if the substring starting at stringStartOffset is a valid C or C++ number literal, + * optionally including a leading '+' or '-' sign. + * + * @param stringSearch The C-style string to search within. + * @param stringStartOffset The starting index within stringSearch. + * @param matchList Pointer to a Range struct to store the start and end indices of the match (if + * found). + * @param stringLength The total length of stringSearch. + * @param language The language standard to adhere to (LANG_C or LANG_CPP). + * @return The number of matches + */ +inline size_t isNumberLiteral( const char* stringSearch, int stringStartOffset, + PatternMatcher::Range* matchList, size_t stringLength, + LanguageStandard language ) { + // Use size_t for internal calculations to avoid overflow and easily compare with length + const size_t len = stringLength; // Use the provided length + + // --- Language Standard Flags --- + const bool isCPP = ( language == LANG_CPP ); + // Features specific to C++ (based on our target definition) + const bool allowBinaryLiteral = isCPP; // 0b prefix (C++14) + const bool allowDigitSeparators = isCPP; // ' separator (C++14) + const bool allowZuffix = isCPP; // z/Z suffix (C++23) + // Note: Hex floats (0x...p...) are C99 and C++17, so allowed in both modes here. + + // --- Basic Validation --- + if ( stringStartOffset < 0 || static_cast( stringStartOffset ) >= len ) { + return 0; + } + + // --- Use size_t for positions internally --- + size_t start_pos = static_cast( stringStartOffset ); + size_t pos = start_pos; // Current parsing position + + // --- 1. Handle Optional Leading Sign --- + // Check for '+' or '-' at the very beginning. + bool has_leading_sign = false; // Track if a sign was consumed + if ( stringSearch[pos] == '+' || stringSearch[pos] == '-' ) { + pos++; + has_leading_sign = true; + // If the sign is the *only* character, it's not a number literal. + if ( pos == len ) { + return 0; + } + } + + // --- State Flags --- + bool is_float = false; + bool is_hex = false; + bool is_binary = false; + bool has_prefix = false; // Had 0x, 0b + bool has_digits = false; // Consumed any valid digits (part of the value)? + bool has_decimal_point = false; + int base = 10; // Default base + + // --- 2. Handle Start after Sign: Prefix, Base detection, Leading decimal point --- + // Now check the character *after* the optional sign (or the first char if no sign) + if ( stringSearch[pos] == '.' ) { // Case: Starts with '.' (e.g., ".5f", "+.5", "-.5") + pos++; + if ( pos == len || !std::isdigit( static_cast( stringSearch[pos] ) ) ) { + // A standalone '.', or '.' after sign, or '.' followed by non-digit isn't a number + // literal. + return 0; + } + is_float = true; + has_decimal_point = true; + base = 10; + // Fall through to parsing digits (fractional part) + } else if ( stringSearch[pos] == + '0' ) { // Case: Starts with '0' (e.g., "0", "+0", "-0xff", "+0b10") + has_digits = true; // The '0' itself counts initially + pos++; + if ( pos < len ) { + char next_char_lower = std::tolower( static_cast( stringSearch[pos] ) ); + if ( next_char_lower == 'x' ) { // Hexadecimal "0x..." / "+0x..." / "-0x..." + pos++; + if ( pos == len || + !std::isxdigit( static_cast( stringSearch[pos] ) ) ) + return 0; // "0x" or "+0x" alone is invalid + is_hex = true; + base = 16; + has_prefix = true; + has_digits = false; // Reset: Need hex digits *after* 0x + } else if ( allowBinaryLiteral && next_char_lower == 'b' ) { // Binary "0b..." (C++14) + pos++; + if ( pos == len || !isBinaryDigit( stringSearch[pos] ) ) + return 0; // "0b" or "+0b" alone is invalid + is_binary = true; + base = 2; + has_prefix = true; + has_digits = false; // Reset: Need bin digits *after* 0b + } else if ( isOctalDigit( stringSearch[pos] ) || + ( allowDigitSeparators && stringSearch[pos] == '\'' && pos + 1 < len && + isOctalDigit( stringSearch[pos + 1] ) ) ) { + // Octal "0..." or C++ "0'..." (separator needs check in digit loop) + // If it's a separator, the digit loop needs to handle it correctly after '0'. + base = 8; + // Don't advance pos here if it's a digit. If it's a separator, let digit loop + // handle. + } else if ( stringSearch[pos] == '.' ) { // Decimal float "0." / "+0." / "-0." + base = 10; // Becomes float, handled below + } else if ( std::tolower( static_cast( stringSearch[pos] ) ) == + 'e' ) { // Decimal float "0e..." + base = 10; // Becomes float, handled below + } else { + // Could be just '0' or '+0' or '-0', or start of invalid octal like '08', '09' + if ( std::isdigit( static_cast( stringSearch[pos] ) ) ) { + // Looks like octal start but might have invalid digits 8/9 + base = 8; + } else { + // Just '0' / '+0' / '-0' followed by non-digit, non-special char. Base 10. + base = 10; + } + } + } else { + // Just "0" or "+0" or "-0" - valid. Base is 10 (or 8, doesn't matter for value 0). + base = 10; + } + } else if ( std::isdigit( static_cast( + stringSearch[pos] ) ) ) { // Case: Starts with '1'-'9' (e.g., "123", "+123", + // "-1") + base = 10; + // Fall through to parsing digits + } else { + // This path is reached if: + // 1. No leading sign, and first char is not '.', '0', or '1'-'9'. + // 2. Had a leading sign, but the char *after* it is not '.', '0', or '1'-'9'. + return 0; // Doesn't start correctly after the optional sign. + } + + // --- 3. Parse Integer or Mantissa Digits --- + while ( pos < len ) { + if ( isValidDigitChar( stringSearch[pos], base ) ) { + pos++; + has_digits = true; + } else if ( allowDigitSeparators && stringSearch[pos] == '\'' && + has_digits && // Separator requires preceding digit + pos + 1 < len && isValidDigitChar( stringSearch[pos + 1], base ) ) { + // C++14 digit separator: skip it if validly placed + pos++; // Consume separator placeholder; next loop iteration checks the required digit + // after it + } else if ( base == 8 && std::isdigit( static_cast( stringSearch[pos] ) ) ) { + // If parsing assumed octal (base 8) and encounter '8' or '9', it's invalid octal. + // Stop parsing digits here. C considers "078" ill-formed. For highlighting, we stop at + // '7'. + break; + } else { + break; // Not a valid digit or separator for the current base + } + } + + // After prefix (0x, 0b), digits are mandatory. Check has_digits flag. + if ( has_prefix && !has_digits ) + return 0; + + size_t end_integer_part = pos; // Position after integer/mantissa digits + + // --- 4. Handle Floating Point specific parts (Decimal Point, Exponent) --- + // Binary literals cannot be floating point. Octal cannot be float in standard C/C++. + bool possible_float_base = !is_binary && base != 8; + + // Check for Decimal Point '.' (only if not already seen at the start) + if ( possible_float_base && !has_decimal_point && pos < len && stringSearch[pos] == '.' ) { + // If hex, requires digits *before* '.' to be a valid hex float *prefix* (e.g., 0x1.). + // Standard C99/C++17 hex floats can be 0x.fP0 if digits follow '.', but "0x." needs + // exponent. + if ( is_hex && !has_digits ) { + // Allow "0x." only if fractional digits or 'p' exponent follows immediately. + // Check next char(s). + size_t next_pos = pos + 1; + bool next_is_hex_digit = + ( next_pos < len && + std::isxdigit( static_cast( stringSearch[next_pos] ) ) ); + bool next_is_p = + ( next_pos < len && + std::tolower( static_cast( stringSearch[next_pos] ) ) == 'p' ); + if ( !next_is_hex_digit && !next_is_p ) { + // Invalid hex float like "0x." or "+0x." followed by something else. Stop parsing. + goto after_float_parts; // Suffix check will likely fail for "0x" + } + // Otherwise, proceed to parse fraction/exponent after '.' + } + + pos++; // Consume '.' + is_float = true; + has_decimal_point = true; + if ( is_hex ) + base = 16; // Hex float fractional part uses hex digits + else + base = 10; // Confirm base for fraction + + bool consumed_frac_digit = false; + while ( pos < len ) { // Consume fractional digits + if ( isValidDigitChar( stringSearch[pos], base ) ) { + pos++; + consumed_frac_digit = true; + } else if ( allowDigitSeparators && stringSearch[pos] == '\'' && consumed_frac_digit && + pos + 1 < len && isValidDigitChar( stringSearch[pos + 1], base ) ) { + pos++; // Consume separator placeholder + } else { + break; + } + } + + // C/C++ requires digits somewhere for floats (e.g., "1." is ok, ".5" is ok) + // Hex floats require fractional digits OR an exponent if '.' is present (e.g. 0x1.p0 or + // 0x1.0p0 ok, 0x1. invalid unless p follows) + if ( !has_digits && !consumed_frac_digit ) { + // Case like "." handled earlier. This means like "0x." without fraction. + // Needs exponent 'p'. Check handled below. If no 'p', invalid. + if ( is_hex ) { + // Check if 'p' follows immediately. If not, backtrack. + if ( pos == len || + std::tolower( static_cast( stringSearch[pos] ) ) != 'p' ) { + pos = end_integer_part; // Backtrack before '.' + is_float = false; + has_decimal_point = false; + // goto after_float_parts; // Let execution continue to exponent check (which + // will fail) then suffix check + } + // If 'p' follows, it's okay, handled below. + } else { + // Decimal: "0." needs fractional digits or exponent. "N." is okay. + // If !has_digits, means started with '.' (or sign then '.'), which requires digits + // after. Handled earlier. This path for decimal likely means error state. + return 0; + } + } + // If we had digits before '.' (has_digits=true) OR consumed fractional digits, it's + // potentially valid. Mark overall number as having digits if it didn't already. + if ( consumed_frac_digit ) + has_digits = true; + + if ( is_hex && !consumed_frac_digit ) { + // Hex like "0x1." - must be followed by 'p' exponent. Check in exponent section. + if ( pos == len || + std::tolower( static_cast( stringSearch[pos] ) ) != 'p' ) { + // Invalid: backtrack to before the '.' + pos = end_integer_part; + is_float = false; // Revert status + has_decimal_point = false; + // Continue to suffix check for the integer part 0xN + goto after_float_parts; + } + // If 'p' follows, it's okay, handled below. + } + } + + // Check for Exponent 'e/E' (decimal) or 'p/P' (hex) + if ( possible_float_base && pos < len ) { // Allow exponent check even if base 8 was temporarily + // assigned but '.' made it float + // Re-evaluate base if '.' occurred + if ( has_decimal_point && !is_hex ) + base = 10; + + char exp_char_lower = std::tolower( static_cast( stringSearch[pos] ) ); + bool is_exponent_char = false; + bool is_hex_exponent = false; + + // Hex exponent 'p' (C99 / C++17) + if ( is_hex && exp_char_lower == 'p' ) { + // Allowed after hex digits or hex fraction. Requires digits somewhere before it + // (checked by has_digits). + if ( !has_digits ) + goto after_float_parts; // e.g. 0xp1 or +0xp1 invalid + is_exponent_char = true; + is_hex_exponent = true; + // Decimal exponent 'e' + } else if ( !is_hex && base == 10 && exp_char_lower == 'e' ) { + // Allowed after dec digits or dec fraction. Requires digits before it. + if ( !has_digits ) + goto after_float_parts; // e.g. .e1 or +.e1 or e1 invalid + is_exponent_char = true; + } + + if ( is_exponent_char ) { + pos++; // Consume exponent char 'e'/'E'/'p'/'P' + is_float = true; // Using an exponent makes it a float + // Exponent *value* is always decimal, even for hex floats + // int exponent_base = 10; // Implicit + + // Optional sign '+' or '-' for the exponent value + if ( pos < len && ( stringSearch[pos] == '+' || stringSearch[pos] == '-' ) ) { + pos++; + } + + // Required decimal digits for exponent value + size_t exp_digits_start = pos; + bool consumed_exp_digit = false; + while ( pos < len ) { + if ( std::isdigit( static_cast( stringSearch[pos] ) ) ) { + pos++; + consumed_exp_digit = true; + } else if ( allowDigitSeparators && stringSearch[pos] == '\'' && + consumed_exp_digit && pos + 1 < len && + std::isdigit( static_cast( stringSearch[pos + 1] ) ) ) { + pos++; // Consume separator placeholder + } else { + break; + } + } + + if ( !consumed_exp_digit ) { + // Invalid: Exponent char/sign not followed by digits. Backtrack. + // Backtrack past optional sign first + if ( pos > exp_digits_start && + ( stringSearch[pos - 1] == '+' || stringSearch[pos - 1] == '-' ) ) { + pos--; + } + // Backtrack past exponent character + if ( pos > start_pos + ( has_leading_sign + ? 1 + : 0 ) ) { // Ensure we don't backtrack past start/sign + char prev_char = + std::tolower( static_cast( stringSearch[pos - 1] ) ); + char expected_exp_char = is_hex_exponent ? 'p' : 'e'; + if ( prev_char == expected_exp_char ) { + pos--; // Backtrack exp char + } + } + + // The number ends before the invalid exponent attempt. + is_float = has_decimal_point; // Revert float status only if exponent was the sole + // float indicator + } else { + // Valid exponent consumed + has_digits = true; // Mark overall number as having digits + } + } + } + +after_float_parts:; // Label to jump to after float parsing attempts + + // --- 5. Handle Suffixes --- + if ( is_float ) { + // Floating-point suffixes: f, F, l, L (common to C99+ and C++) + // C++11 also adds f16, f32, f64, f128, etc. Not handling those here. + if ( pos < len ) { + char s1 = stringSearch[pos]; + if ( s1 == 'f' || s1 == 'F' || s1 == 'l' || s1 == 'L' ) { + pos++; // Consume one suffix character + } + } + } else { + // Integer suffixes: U, L, LL, Z (Z is C++23 only) combinations + // C/C++ allow U/L/LL in any order, but Z only mixes with U (C++23). + // Max one of U/Z. Max two L's. + + size_t current_pos = pos; + // Use a flexible parsing approach - scan potential suffix chars + std::vector suffix_chars; + while ( current_pos < len ) { + char c = stringSearch[current_pos]; + char c_lower = std::tolower( static_cast( c ) ); + if ( c_lower == 'u' || c_lower == 'l' || ( allowZuffix && c_lower == 'z' ) ) { + suffix_chars.push_back( c_lower ); + current_pos++; + } else { + break; + } + } + + // Validate the collected suffix chars + size_t valid_suffix_len = 0; + if ( !suffix_chars.empty() ) { + size_t check_len = suffix_chars.size(); + + // Try to parse known valid combinations first (simplifies logic) + if ( check_len == 1 ) { + if ( suffix_chars[0] == 'u' || suffix_chars[0] == 'l' || suffix_chars[0] == 'z' ) + valid_suffix_len = 1; + } else if ( check_len == 2 ) { + char c1 = suffix_chars[0], c2 = suffix_chars[1]; + if ( ( c1 == 'u' && c2 == 'l' ) || ( c1 == 'l' && c2 == 'u' ) ) + valid_suffix_len = 2; // UL, LU + else if ( c1 == 'l' && c2 == 'l' ) + valid_suffix_len = 2; // LL + else if ( ( c1 == 'u' && c2 == 'z' ) || ( c1 == 'z' && c2 == 'u' ) ) + valid_suffix_len = 2; // UZ, ZU (C++23) + } else if ( check_len == 3 ) { + char c1 = suffix_chars[0], c2 = suffix_chars[1], c3 = suffix_chars[2]; + // ULL, LUL, LLU + if ( c1 == 'l' && c2 == 'l' ) { // Starts LL + if ( c3 == 'u' ) + valid_suffix_len = 3; // LLU + } else if ( c2 == 'l' && c3 == 'l' ) { // Ends LL + if ( c1 == 'u' ) + valid_suffix_len = 3; // ULL + } else if ( c1 == 'l' && c3 == 'l' ) { // L_L + if ( c2 == 'u' ) + valid_suffix_len = 3; // LUL + } + } + + // If a valid combination was found, advance pos + if ( valid_suffix_len > 0 ) { + pos += valid_suffix_len; // Advance main position pointer + } + // else: pos remains at suffix_start, no valid suffix consumed. + } + // pos is now either after a valid suffix or back at suffix_start + } + + size_t final_end_pos = pos; // Position after number value and any valid suffix + + // --- Final Validation and Return --- + // A valid number literal must contain *at least one digit* somewhere, + // OR be a valid floating point literal structure like "1." or ".1" (which implies digits). + // The `has_digits` flag covers most cases. We need it to be true. + if ( !has_digits ) { + // If no digits were ever consumed, it's invalid (e.g. "0x", "+0b", ".", "+.", "-") + return 0; + } + + // The end position must be strictly after the start position. + // This also implicitly handles the case where only a sign was present ("+" or "-"), + // because has_digits would be false in that case, failing the check above. + if ( final_end_pos > start_pos ) { + // Success! Update the output struct and return true. + if ( matchList ) { + matchList->start = stringStartOffset; // Use original int offset + matchList->end = static_cast( final_end_pos ); // Cast end position + } + return 1; + } else { + // Should only happen if start_pos was the only character and failed validation early + // Or if for some reason parsing didn't advance pos (e.g., logic error) + return 0; + } +} + +SINGLETON_DECLARE_IMPLEMENTATION( ParserMatcherManager ); + +void ParserMatcherManager::registerBaseParsers() { + if ( !mFns.empty() ) + return; + + registerParser( "cpp_number_parser", + []( const char* stringSearch, int stringStartOffset, + PatternMatcher::Range* matchList, size_t stringLength ) { + return isNumberLiteral( stringSearch, stringStartOffset, matchList, + stringLength, LanguageStandard::LANG_CPP ); + } ); + + registerParser( "c_number_parser", []( const char* stringSearch, int stringStartOffset, + PatternMatcher::Range* matchList, size_t stringLength ) { + return isNumberLiteral( stringSearch, stringStartOffset, matchList, stringLength, + LanguageStandard::LANG_C ); + } ); + + registerParser( "js_number_parser", []( const char* stringSearch, int stringStartOffset, + PatternMatcher::Range* matchList, + size_t stringLength ) { + return isNumberLiteralJS( stringSearch, stringStartOffset, matchList, stringLength ); + } ); +} + +void ParserMatcherManager::registerParser( std::string_view parserName, ParserMatcherFn fn ) { + mFns.insert_or_assign( std::hash()( parserName ), std::move( fn ) ); +} + +bool ParserMatcherManager::hasParser( std::string_view parserName ) const { + return mFns.find( std::hash()( parserName ) ) != mFns.end(); +} + +size_t ParserMatcherManager::matches( std::string_view parserName, const char* stringSearch, + int stringStartOffset, PatternMatcher::Range* matchList, + size_t stringLength ) { + auto parserIt = mFns.find( std::hash()( parserName ) ); + if ( parserIt != mFns.end() ) + return parserIt->second( stringSearch, stringStartOffset, matchList, stringLength ); + return 0; +} + +ParserMatcher::ParserMatcher( const std::string_view& parserName ) : + PatternMatcher( PatternType::Parser ), mParserName( parserName ), mMatchNum( 0 ) {} + +ParserMatcher::~ParserMatcher() {} + +bool ParserMatcher::isValid() const { + return true; +} + +bool ParserMatcher::matches( const char* stringSearch, int stringStartOffset, + PatternMatcher::Range* matchList, size_t stringLength ) const { + mMatchNum = ParserMatcherManager::instance()->matches( + mParserName, stringSearch, stringStartOffset, matchList, stringLength ); + return mMatchNum > 0; +} + +bool ParserMatcher::matches( const std::string& str, PatternMatcher::Range* matchList, + int stringStartOffset ) const { + return matches( str.c_str(), stringStartOffset, matchList, str.length() ); +} + +const size_t& ParserMatcher::getNumMatches() const { + return mMatchNum; +} + +}} // namespace EE::System diff --git a/src/eepp/system/patternmatcher.cpp b/src/eepp/system/patternmatcher.cpp index e6dcc38a1..9301a7c41 100644 --- a/src/eepp/system/patternmatcher.cpp +++ b/src/eepp/system/patternmatcher.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -77,6 +78,9 @@ PatternMatcher::State::State( PatternMatcher* pattern, bool ownPattern ) : case PatternType::PCRE: mPattern = new RegEx( pattern->getPattern() ); break; + case PatternType::Parser: + mPattern = new ParserMatcher( pattern->getPattern() ); + break; } } else { mPattern = pattern; diff --git a/src/eepp/ui/abstract/uiabstracttableview.cpp b/src/eepp/ui/abstract/uiabstracttableview.cpp index c8e38daef..aaecd1329 100644 --- a/src/eepp/ui/abstract/uiabstracttableview.cpp +++ b/src/eepp/ui/abstract/uiabstracttableview.cpp @@ -146,7 +146,7 @@ void UIAbstractTableView::resetColumnData() { void UIAbstractTableView::createOrUpdateColumns( bool resetColumnData ) { Model* model = getModel(); - if ( !model || mSize.getWidth() <= 0 ) + if ( !model ) return; size_t count = model->columnCount(); diff --git a/src/eepp/ui/doc/languages/c.cpp b/src/eepp/ui/doc/languages/c.cpp index e56732eb6..a8cc8a383 100644 --- a/src/eepp/ui/doc/languages/c.cpp +++ b/src/eepp/ui/doc/languages/c.cpp @@ -1,9 +1,11 @@ +#include #include #include namespace EE { namespace UI { namespace Doc { namespace Language { void addC() { + ParserMatcherManager::instance()->registerBaseParsers(); auto& sd = SyntaxDefinitionManager::instance()->add( @@ -14,17 +16,12 @@ void addC() { { { "/%*", "%*/" }, "comment" }, { { "^%s*(#include)%s+([<%\"][%w%d%.%\\%/%_%-]+[>%\"])" }, { "keyword", "keyword", "literal" } }, - { { "^%s*(#e?l?n?d?ifn?d?e?f?)%s+" }, { "keyword", "keyword", "literal" } }, - { { "^%s*(#define)%s*" }, { "keyword", "keyword", "literal" } }, - { { "^%s*(#else)%s*" }, { "keyword", "keyword", "literal" } }, - { { "^%s*#", "[^\\]\n" }, "comment" }, { { "\"", "[\"\n]", "\\" }, "string" }, { { "'", "'", "\\" }, "string" }, - { { "-?0x%x+" }, "number" }, - { { "-?%d+[%d%.eE]*f?" }, "number" }, - { { "-?%.?%d+f?" }, "number" }, + { { "c_number_parser" }, "number", "", SyntaxPatternMatchType::Parser }, { { "[%+%-=/%*%^%%<>!~|&]" }, "operator" }, { { "[%a_][%w_]*%f[(]" }, "function" }, + { { "^%s*#[%a_][%w_]*" }, "symbol" }, { { "[%a_][%w_]*" }, "symbol" }, }, @@ -45,6 +42,11 @@ void addC() { { "uint8_t", "keyword2" }, { "uint64_t", "keyword2" }, { "case", "keyword" }, { "if", "keyword" }, { "do", "keyword" }, + { "#if", "keyword" }, { "#ifdef", "keyword" }, { "#ifndef", "keyword" }, + { "#elif", "keyword" }, { "#else", "keyword" }, { "#endif", "keyword" }, + { "#define", "keyword" }, { "#undef", "keyword" }, { "#include", "keyword" }, + { "#line", "keyword" }, { "#error", "keyword" }, { "#pragma", "keyword" }, + { "#warning ", "keyword" }, { "#elifdef ", "keyword" }, { "#elifndef ", "keyword" }, }, "//", {} diff --git a/src/eepp/ui/doc/languages/cpp.cpp b/src/eepp/ui/doc/languages/cpp.cpp index f3f86f932..a2c06c53d 100644 --- a/src/eepp/ui/doc/languages/cpp.cpp +++ b/src/eepp/ui/doc/languages/cpp.cpp @@ -1,9 +1,11 @@ +#include #include #include namespace EE { namespace UI { namespace Doc { namespace Language { void addCPP() { + ParserMatcherManager::instance()->registerBaseParsers(); auto& sd = SyntaxDefinitionManager::instance()->add( @@ -23,19 +25,13 @@ void addCPP() { { { "'", "'", "\\" }, "string" }, { { "^%s*(#include)%s+([<%\"][%w%d%.%\\%/%_%-]+[>%\"])" }, { "keyword", "keyword", "literal" } }, - { { "^%s*(#e?l?n?d?ifn?d?e?f?)%s+" }, { "keyword", "keyword", "literal" } }, - { { "^%s*(#define)%s*" }, { "keyword", "keyword", "literal" } }, - { { "^%s*(#else)%s*" }, { "keyword", "keyword", "literal" } }, - { { "^%s*#", "[^\\]\n" }, "comment" }, - { { "-?0x%x+" }, "number" }, - { { "-?0b[01]+" }, "number" }, - { { "-?%d+[%d%.eE]*f?" }, "number" }, - { { "-?%.?%d+f?" }, "number" }, + { { "cpp_number_parser" }, "number", "", SyntaxPatternMatchType::Parser }, { { "[%+%-=/%*%^%%<>!~|&]" }, "operator" }, { { "[%a_][%w_]*%f[(]" }, "function" }, { { "std%:%:[%w_]*" }, "keyword2" }, { { "(%[)(%[)(%a[%w_]+)(%])(%])" }, { "normal", "keyword", "keyword3", "keyword2", "keyword3", "keyword" } }, + { { "^%s*#[%a_][%w_]*" }, "symbol" }, { { "[%a_][%w_]*" }, "symbol" }, }, @@ -99,6 +95,15 @@ void addCPP() { { "uint64_t", "keyword2" }, { "char32_t", "keyword2" }, { "alignas", "keyword" }, { "export", "keyword" }, + { "#if", "keyword" }, { "#ifdef", "keyword" }, + { "#ifndef", "keyword" }, { "#else", "keyword" }, + { "#elif", "keyword" }, { "#elifdef", "keyword" }, // C++23 + { "#elifndef", "keyword" }, // C++23 + { "#endif", "keyword" }, { "#include", "keyword" }, + { "#define", "keyword" }, { "#undef", "keyword" }, + { "#line", "keyword" }, { "#error", "keyword" }, + { "#pragma", "keyword" }, + }, "//", {}, diff --git a/src/eepp/ui/doc/languages/javascript.cpp b/src/eepp/ui/doc/languages/javascript.cpp index 28ee90bde..86d95d473 100644 --- a/src/eepp/ui/doc/languages/javascript.cpp +++ b/src/eepp/ui/doc/languages/javascript.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -18,9 +19,7 @@ void addJavaScript() { { { "/[%+%-%*%^%!%=%&%|%?%:%;%,%(%[%{%<%>%\\%\"].*%f[/]", "/[igmsuyd\n]?[igmsuyd\n]?[igmsuyd\n]?", "\\" }, "string" }, - { { "0x[%da-fA-F]+" }, "number" }, - { { "-?%d+[%d%.eE]*" }, "number" }, - { { "-?%.?%d+" }, "number" }, + { { "js_number_parser" }, "number", "", SyntaxPatternMatchType::Parser }, { { "[%+%-=/%*%^%%<>!~|&]" }, "operator" }, { { "([%w_][%w_]+)%.([%w_][%w%d_]*)%s*(=)%s*(function)" }, { "normal", "keyword2", "function", "operator", "keyword" } }, diff --git a/src/eepp/ui/doc/syntaxdefinitionmanager.cpp b/src/eepp/ui/doc/syntaxdefinitionmanager.cpp index 3d5e5afe4..67f31071d 100644 --- a/src/eepp/ui/doc/syntaxdefinitionmanager.cpp +++ b/src/eepp/ui/doc/syntaxdefinitionmanager.cpp @@ -81,7 +81,10 @@ static json toJson( const SyntaxDefinition& def ) { j["patterns"] = json::array(); for ( const auto& ptrn : def.getPatterns() ) { json pattern; - auto ptrnType = ptrn.matchType == SyntaxPatternMatchType::RegEx ? "regex" : "pattern"; + auto ptrnType = + ptrn.matchType == SyntaxPatternMatchType::RegEx + ? "regex" + : ( ptrn.matchType == SyntaxPatternMatchType::Parser ? "parser" : "pattern" ); // Do not export injected patterns if ( ptrn.matchType == SyntaxPatternMatchType::LuaPattern && @@ -214,12 +217,20 @@ namespace EE { namespace UI { namespace Doc { namespace Language { for ( const auto& pattern : def.getPatterns() ) { buf += "{ " + join( pattern.patterns ) + ", " + join( pattern.typesNames, true, true, - pattern.matchType == SyntaxPatternMatchType::RegEx ) + + pattern.matchType != SyntaxPatternMatchType::LuaPattern ) + str( pattern.syntax, ", ", "", false ); - if ( pattern.matchType == SyntaxPatternMatchType::RegEx && pattern.syntax.empty() ) - buf += ", \"\", SyntaxPatternMatchType::RegEx"; - else if ( pattern.matchType == SyntaxPatternMatchType::RegEx ) - buf += ", SyntaxPatternMatchType::RegEx"; + if ( pattern.matchType != SyntaxPatternMatchType::LuaPattern && pattern.syntax.empty() ) { + if ( pattern.matchType == SyntaxPatternMatchType::RegEx ) + buf += ", \"\", SyntaxPatternMatchType::RegEx"; + else if ( pattern.matchType == SyntaxPatternMatchType::Parser ) + buf += ", \"\", SyntaxPatternMatchType::Parser"; + } + else if ( pattern.matchType != SyntaxPatternMatchType::LuaPattern ){ + if ( pattern.matchType == SyntaxPatternMatchType::RegEx ) + buf += ", SyntaxPatternMatchType::RegEx"; + else if ( pattern.matchType == SyntaxPatternMatchType::Parser ) + buf += ", SyntaxPatternMatchType::Parser"; + } buf += " },\n"; } buf += "\n},\n"; @@ -389,7 +400,7 @@ static SyntaxDefinition loadLanguage( const nlohmann::json& json ) { ? "" : pattern.value( "syntax", "" ); std::vector ptrns; - bool isRegEx = false; + auto ctype = SyntaxPatternMatchType::LuaPattern; if ( pattern.contains( "pattern" ) ) { if ( pattern["pattern"].is_array() ) { const auto& ptrnIt = pattern["pattern"]; @@ -399,7 +410,7 @@ static SyntaxDefinition loadLanguage( const nlohmann::json& json ) { ptrns.emplace_back( pattern["pattern"] ); } } else if ( pattern.contains( "regex" ) ) { - isRegEx = true; + ctype = SyntaxPatternMatchType::RegEx; if ( pattern["regex"].is_array() ) { const auto& ptrnIt = pattern["regex"]; for ( const auto& ptrn : ptrnIt ) @@ -407,10 +418,18 @@ static SyntaxDefinition loadLanguage( const nlohmann::json& json ) { } else if ( pattern["regex"].is_string() ) { ptrns.emplace_back( pattern["regex"] ); } + } else if ( pattern.contains( "parser" ) ) { + ctype = SyntaxPatternMatchType::Parser; + if ( pattern["parser"].is_array() ) { + const auto& ptrnIt = pattern["parser"]; + for ( const auto& ptrn : ptrnIt ) + ptrns.emplace_back( ptrn ); + } else if ( pattern["parser"].is_string() ) { + ptrns.emplace_back( pattern["parser"] ); + } } - def.addPattern( SyntaxPattern( std::move( ptrns ), std::move( type ), syntax, - isRegEx ? SyntaxPatternMatchType::RegEx - : SyntaxPatternMatchType::LuaPattern ) ); + def.addPattern( + SyntaxPattern( std::move( ptrns ), std::move( type ), syntax, ctype ) ); } } if ( json.contains( "symbols" ) ) { diff --git a/src/eepp/ui/doc/syntaxtokenizer.cpp b/src/eepp/ui/doc/syntaxtokenizer.cpp index 38c8a7fb7..a1f48a815 100644 --- a/src/eepp/ui/doc/syntaxtokenizer.cpp +++ b/src/eepp/ui/doc/syntaxtokenizer.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -110,10 +111,12 @@ static NonEscapedMatch findNonEscaped( const std::string& text, const std::strin eeASSERT( !pattern.empty() ); if ( pattern.empty() ) return {}; - std::variant wordsVar = - matchType == SyntaxPatternMatchType::RegEx - ? std::variant( RegEx( pattern ) ) - : std::variant( LuaPattern( pattern ) ); + std::variant wordsVar = + matchType == SyntaxPatternMatchType::LuaPattern + ? std::variant( LuaPattern( pattern ) ) + : ( matchType == SyntaxPatternMatchType::RegEx + ? std::variant( RegEx( pattern ) ) + : std::variant( ParserMatcher( pattern ) ) ); PatternMatcher& words = std::visit( []( auto& patternType ) -> PatternMatcher& { return patternType; }, wordsVar ); int start, end; @@ -341,12 +344,17 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax const SyntaxPattern& pattern = curState.currentSyntax->getPatterns()[patternIndex]; if ( i != 0 && pattern.patterns[0][0] == '^' ) continue; - patternStr = - pattern.patterns[0][0] == '^' ? pattern.patterns[0] : "^" + pattern.patterns[0]; - std::variant wordsVar = - pattern.matchType == SyntaxPatternMatchType::RegEx - ? std::variant( RegEx( patternStr ) ) - : std::variant( LuaPattern( patternStr ) ); + patternStr = pattern.matchType != SyntaxPatternMatchType::Parser + ? pattern.patterns[0][0] == '^' ? pattern.patterns[0] + : "^" + pattern.patterns[0] + : pattern.patterns[0]; + std::variant wordsVar = + pattern.matchType == SyntaxPatternMatchType::LuaPattern + ? std::variant( LuaPattern( patternStr ) ) + : ( pattern.matchType == SyntaxPatternMatchType::RegEx + ? std::variant( RegEx( patternStr ) ) + : std::variant( + ParserMatcher( patternStr ) ) ); PatternMatcher& words = std::visit( []( auto& patternType ) -> PatternMatcher& { return patternType; }, wordsVar ); if ( !words.isValid() ) // Skip invalid patterns diff --git a/src/eepp/window/engine.cpp b/src/eepp/window/engine.cpp index 857806c90..86f94843d 100644 --- a/src/eepp/window/engine.cpp +++ b/src/eepp/window/engine.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -110,6 +111,8 @@ Engine::~Engine() { RegExCache::destroySingleton(); + ParserMatcherManager::destroySingleton(); + Log::destroySingleton(); } diff --git a/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/typescript.cpp b/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/typescript.cpp index 3215f05e3..9f8d70d92 100644 --- a/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/typescript.cpp +++ b/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/typescript.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -18,9 +19,7 @@ void addTypeScript() { { { "/[%+%-%*%^%!%=%&%|%?%:%;%,%(%[%{%<%>%\\%\"].*%f[/]", "/[igmsuyd\n]?[igmsuyd\n]?[igmsuyd\n]?", "\\" }, "string" }, - { { "0x[%da-fA-F]+" }, "number" }, - { { "-?%d+[%d%.eE]*" }, "number" }, - { { "-?%.?%d+" }, "number" }, + { { "js_number_parser" }, "number", "", SyntaxPatternMatchType::Parser }, { { "[%+%-=/%*%^%%<>!~|&]" }, "operator" }, { { "(interface%s)([%a_][%w_]*)" }, { "normal", "keyword", "keyword2" } }, { { "(type%s)([%a_][%w_]*)" }, { "normal", "keyword", "keyword2" } }, diff --git a/src/tools/ecode/plugins/aiassistant/chatui.cpp b/src/tools/ecode/plugins/aiassistant/chatui.cpp index 2eb0fa452..ddca91811 100644 --- a/src/tools/ecode/plugins/aiassistant/chatui.cpp +++ b/src/tools/ecode/plugins/aiassistant/chatui.cpp @@ -146,6 +146,7 @@ DropDownList.role_ui { + @@ -177,7 +178,10 @@ static const char* DEFAULT_CHAT_GLOBE = R"xml( )xml"; -LLMChatUI::LLMChatUI( PluginManager* manager ) : UILinearLayout(), mManager( manager ) { +LLMChatUI::LLMChatUI( PluginManager* manager ) : + UILinearLayout(), + WidgetCommandExecuter( getUISceneNode()->getWindow()->getInput() ), + mManager( manager ) { setClass( "llm_chatui" ); setLayoutSizePolicy( SizePolicy::MatchParent, SizePolicy::MatchParent ); @@ -190,9 +194,11 @@ LLMChatUI::LLMChatUI( PluginManager* manager ) : UILinearLayout(), mManager( man mRefreshModels = find( "refresh_model_ui" ); mRefreshModels->onClick( [this]( auto ) { fillApiModels( mModelDDL ); } ); + mChatSave = find( "llm_save_but" ); + mChatSave->onClick( [this]( auto ) { execute( "ai-save-chat" ); } ); + mChatSettings = find( "llm_settings_but" ); - mChatSettings->onClick( - [this]( auto ) { mChatInput->getDocument().execute( "ai-settings" ); } ); + mChatSettings->onClick( [this]( auto ) { execute( "ai-settings" ); } ); mChatScrollView = findByClass( "llm_chat_scrollview" )->asType(); mChatScrollView->getVerticalScrollBar()->setValue( 1 ); @@ -206,12 +212,49 @@ LLMChatUI::LLMChatUI( PluginManager* manager ) : UILinearLayout(), mManager( man getPlugin()->getPluginContext()->getSplitter()->getCurrentColorScheme() ); } - mChatInput->getDocument().setCommand( "ai-settings", [this] { + mChatAdd = find( "llm_add_chat" ); + mChatAdd->onClick( [this]( auto ) { execute( "ai-add-chat" ); } ); + + const auto& markdown = SyntaxDefinitionManager::instance()->getByLSPName( "markdown" ); + mChatInput->setShowFoldingRegion( true ); + mChatInput->getDocument().getFoldRangeService().setEnabled( true ); + mChatInput->setFoldDrawable( findIcon( "chevron-down", PixelDensity::dpToPxI( 12 ) ) ); + mChatInput->setFoldedDrawable( findIcon( "chevron-right", PixelDensity::dpToPxI( 12 ) ) ); + + mChatInput->setSyntaxDefinition( markdown ); + + mChatRun = find( "llm_run" ); + mChatRun->onClick( [this]( auto ) { execute( "ai-prompt" ); } ); + + mChatStop = find( "llm_stop" ); + mChatStop->onClick( [this]( auto ) { execute( "ai-prompt-stop" ); } ); + + mChatUserRole = find( "llm_user" ); + mChatUserRole->onClick( [this]( auto ) { + if ( mChatUserRole->getText() == mChatUserRole->i18n( "user", "User" ) ) { + mChatUserRole->setText( mChatUserRole->i18n( "assistant", "Assistant" ) ); + } else if ( mChatUserRole->getText() == mChatUserRole->i18n( "assistant", "Assistant" ) ) { + mChatUserRole->setText( mChatUserRole->i18n( "system", "System" ) ); + } else if ( mChatUserRole->getText() == mChatUserRole->i18n( "system", "System" ) ) { + mChatUserRole->setText( mChatUserRole->i18n( "user", "User" ) ); + } + } ); + + mChatPrivate = find( "llm_private_chat" ); + mChatPrivate->on( Event::OnValueChange, + [this]( auto ) { mChatIsPrivate = mChatPrivate->isSelected(); } ); + + auto setCmd = [this]( const std::string& name, const CommandCallback& cb ) { + setCommand( name, cb ); + mChatInput->getDocument().setCommand( name, cb ); + }; + + setCmd( "ai-settings", [this] { if ( getPlugin() ) getPlugin()->getPluginContext()->focusOrLoadFile( getPlugin()->getFileConfigPath() ); } ); - mChatInput->getDocument().setCommand( "ai-add-chat", [this] { + setCmd( "ai-add-chat", [this] { if ( mChatInput->getDocument().isEmpty() ) return; @@ -222,7 +265,7 @@ LLMChatUI::LLMChatUI( PluginManager* manager ) : UILinearLayout(), mManager( man mChatInput->setFocus(); } ); - mChatInput->getDocument().setCommand( "ai-prompt", [this] { + setCmd( "ai-prompt", [this] { // "ai-prompt-stop" if ( mRequest ) { mRequest->cancel(); @@ -249,16 +292,16 @@ LLMChatUI::LLMChatUI( PluginManager* manager ) : UILinearLayout(), mManager( man } } - mChatInput->getDocument().execute( "ai-add-chat" ); + execute( "ai-add-chat" ); doRequest(); } ); - mChatInput->getDocument().setCommand( "ai-prompt-stop", [this] { + setCmd( "ai-prompt-stop", [this] { if ( mRequest ) mRequest->cancel(); } ); - mChatInput->getDocument().setCommand( "ai-clone-chat", [this] { + setCmd( "ai-clone-chat", [this] { if ( getPlugin() == nullptr ) return; auto chats = findAllByClass( "llm_conversation" ); @@ -275,45 +318,33 @@ LLMChatUI::LLMChatUI( PluginManager* manager ) : UILinearLayout(), mManager( man chatUI->setFocus(); } ); - mChatInput->getDocument().setCommand( "ai-chat-history", [this] { showChatHistory(); } ); + setCmd( "ai-save-chat", [this] { saveChat(); } ); - mChatAdd = find( "llm_add_chat" ); - mChatAdd->onClick( [this]( auto ) { mChatInput->getDocument().execute( "ai-add-chat" ); } ); + setCmd( "ai-chat-history", [this] { showChatHistory(); } ); - const auto& markdown = SyntaxDefinitionManager::instance()->getByLSPName( "markdown" ); - mChatInput->setShowFoldingRegion( true ); - mChatInput->getDocument().getFoldRangeService().setEnabled( true ); - mChatInput->setFoldDrawable( findIcon( "chevron-down", PixelDensity::dpToPxI( 12 ) ) ); - mChatInput->setFoldedDrawable( findIcon( "chevron-right", PixelDensity::dpToPxI( 12 ) ) ); + setCmd( "ai-toggle-private-chat", [this] { mChatPrivate->toggleSelection(); } ); - mChatInput->setSyntaxDefinition( markdown ); - - mChatRun = find( "llm_run" ); - mChatRun->onClick( [this]( auto ) { mChatInput->getDocument().execute( "ai-prompt" ); } ); - - mChatStop = find( "llm_stop" ); - mChatStop->onClick( [this]( auto ) { mChatInput->getDocument().execute( "ai-prompt-stop" ); } ); - - mChatUserRole = find( "llm_user" ); - mChatUserRole->onClick( [this]( auto ) { - if ( mChatUserRole->getText() == mChatUserRole->i18n( "user", "User" ) ) { - mChatUserRole->setText( mChatUserRole->i18n( "assistant", "Assistant" ) ); - } else if ( mChatUserRole->getText() == mChatUserRole->i18n( "assistant", "Assistant" ) ) { - mChatUserRole->setText( mChatUserRole->i18n( "system", "System" ) ); - } else if ( mChatUserRole->getText() == mChatUserRole->i18n( "system", "System" ) ) { - mChatUserRole->setText( mChatUserRole->i18n( "user", "User" ) ); - } + setCmd( "ai-rename-chat", [this] { + UIMessageBox* msgBox = UIMessageBox::New( UIMessageBox::INPUT, + i18n( "ai_rename_chat", "Rename Conversation" ) ); + msgBox->on( Event::OnConfirm, [this, msgBox]( const Event* ) { + std::string newName( msgBox->getTextInput()->getText().toUtf8() ); + if ( newName.empty() || mSummary == newName ) + return; + msgBox->closeWindow(); + renameChat( newName ); + } ); + msgBox->setCloseShortcut( { KEY_ESCAPE, KEYMOD_NONE } ); + msgBox->center(); + auto summary = mSummary; + String::replaceAll( summary, "\n", " " ); + String::trimInPlace( summary ); + msgBox->getTextInput()->setText( summary ); + msgBox->showWhenReady(); } ); - mChatPrivate = find( "llm_private_chat" ); - mChatPrivate->on( Event::OnValueChange, - [this]( auto ) { mChatIsPrivate = mChatPrivate->isSelected(); } ); - - mChatInput->getDocument().setCommand( "ai-toggle-private-chat", - [this] { mChatPrivate->toggleSelection(); } ); - mChatClone = find( "llm_clone_chat" ); - mChatClone->onClick( [this]( auto ) { mChatInput->getDocument().execute( "ai-clone-chat" ); } ); + mChatClone->onClick( [this]( auto ) { execute( "ai-clone-chat" ); } ); mChatHistory = find( "llm_chat_history" ); mChatHistory->onClick( [this]( auto ) { showChatHistory(); } ); @@ -344,18 +375,25 @@ LLMChatUI::LLMChatUI( PluginManager* manager ) : UILinearLayout(), mManager( man fillModelDropDownList( mModelDDL ); const auto appendShortcutToTooltip = [this]( UIPushButton* but, const std::string& cmd ) { - auto kb = mChatInput->getKeyBindings().getCommandKeybindString( cmd ); + auto kb = getKeyBindings().getCommandKeybindString( cmd ); if ( kb.empty() ) return; but->setTooltipText( but->getTooltipText() + " (" + kb + ")" ); }; - mChatInput->getKeyBindings().addKeybindString( "mod+return", "ai-prompt" ); - mChatInput->getKeyBindings().addKeybindString( "mod+shift+return", "ai-add-chat" ); - mChatInput->getKeyBindings().addKeybindString( "mod+h", "ai-chat-history" ); - mChatInput->getKeyBindings().addKeybindString( "mod+shift+c", "ai-clone-chat" ); - mChatInput->getKeyBindings().addKeybindString( "mod+shift+s", "ai-settings" ); - mChatInput->getKeyBindings().addKeybindString( "mod+shift+p", "ai-toggle-private-chat" ); + const auto addKb = [this]( const std::string& kb, const std::string& cmd ) { + getKeyBindings().addKeybindString( kb, cmd ); + mChatInput->addKeyBindingString( kb, cmd ); + }; + + addKb( "mod+return", "ai-prompt" ); + addKb( "mod+shift+return", "ai-add-chat" ); + addKb( "mod+h", "ai-chat-history" ); + addKb( "mod+shift+c", "ai-clone-chat" ); + addKb( "mod+shift+s", "ai-settings" ); + addKb( "mod+shift+p", "ai-toggle-private-chat" ); + addKb( "mod+s", "ai-save-chat" ); + addKb( "f2", "ai-rename-chat" ); appendShortcutToTooltip( mChatHistory, "ai-chat-history" ); appendShortcutToTooltip( mChatRun, "ai-prompt" ); @@ -364,9 +402,10 @@ LLMChatUI::LLMChatUI( PluginManager* manager ) : UILinearLayout(), mManager( man appendShortcutToTooltip( mChatClone, "ai-clone-chat" ); appendShortcutToTooltip( mChatSettings, "ai-settings" ); appendShortcutToTooltip( mChatPrivate, "ai-toggle-private-chat" ); + appendShortcutToTooltip( mChatSave, "ai-save-chat" ); - mChatInput->getKeyBindings().addKeybindString( "mod+keypad enter", "ai-prompt" ); - mChatInput->getKeyBindings().addKeybindString( "mod+shift+keypad enter", "ai-add-chat" ); + addKb( "mod+keypad enter", "ai-prompt" ); + addKb( "mod+shift+keypad enter", "ai-add-chat" ); } std::optional LLMChatUI::getModel( const std::string& provider, @@ -886,6 +925,8 @@ void LLMChatUI::doRequest() { auto status = response.getStatus(); if ( status == Http::Response::Ok ) { mSummary = req.getResponse(); + String::trimInPlace( mSummary, '\n' ); + String::trimInPlace( mSummary, ' ' ); runOnMainThread( [this] { updateTabTitle(); } ); saveChat(); } @@ -1057,6 +1098,31 @@ void LLMChatUI::updateTabTitle() { tab->setText( title ); } +static bool fsRenameFile( const std::string& fpath, const std::string& newFilePath ) { + try { +#if EE_PLATFORM == EE_PLATFORM_WIN + std::filesystem::rename( String( fpath ).toWideString(), + String( newFilePath ).toWideString() ); +#else + std::filesystem::rename( fpath, newFilePath ); +#endif + } catch ( const std::filesystem::filesystem_error& ) { + return false; + } + + return true; +} + +void LLMChatUI::renameChat( const std::string& newName ) { + auto oldPath = getNewFilePath( mUUID.toString(), mSummary ); + auto newPath = getNewFilePath( mUUID.toString(), newName ); + if ( fsRenameFile( oldPath, newPath ) ) { + mSummary = newName; + saveChat(); + updateTabTitle(); + } +} + UISplitter* LLMChatUI::getSplitter() const { return mChatSplitter; } diff --git a/src/tools/ecode/plugins/aiassistant/chatui.hpp b/src/tools/ecode/plugins/aiassistant/chatui.hpp index 0cc87ad34..9dc48d1c9 100644 --- a/src/tools/ecode/plugins/aiassistant/chatui.hpp +++ b/src/tools/ecode/plugins/aiassistant/chatui.hpp @@ -1,12 +1,13 @@ #pragma once +#include "../pluginmanager.hpp" #include "llmchatcompletionrequest.hpp" #include "protocol.hpp" -#include "../pluginmanager.hpp" #include +#include -#include "nlohmann/json_fwd.hpp" +#include namespace EE { namespace UI { class UIWidget; @@ -45,7 +46,7 @@ class LLMChat { static LLMChat::Role stringToRole( UIPushButton* userBut ); }; -class LLMChatUI : public UILinearLayout { +class LLMChatUI : public UILinearLayout, public WidgetCommandExecuter { public: static LLMChatUI* New( PluginManager* manager ) { return eeNew( LLMChatUI, ( manager ) ); } @@ -71,6 +72,12 @@ class LLMChatUI : public UILinearLayout { void updateTabTitle(); + void renameChat( const std::string& newName ); + + virtual Uint32 onKeyDown( const KeyEvent& event ) { + return WidgetCommandExecuter::onKeyDown( event ); + } + protected: UUID mUUID; std::string mSummary; @@ -86,6 +93,7 @@ class LLMChatUI : public UILinearLayout { UIPushButton* mChatStop{ nullptr }; UIPushButton* mChatHistory{ nullptr }; UIPushButton* mChatClone{ nullptr }; + UIPushButton* mChatSave{ nullptr }; UIPushButton* mRefreshModels{ nullptr }; UISelectButton* mChatPrivate{ nullptr }; UIScrollView* mChatScrollView{ nullptr }; diff --git a/src/tools/ecode/projectsearch.cpp b/src/tools/ecode/projectsearch.cpp index 9c6b37c5f..7a6bcffae 100644 --- a/src/tools/ecode/projectsearch.cpp +++ b/src/tools/ecode/projectsearch.cpp @@ -236,15 +236,26 @@ void ProjectSearch::find( const std::vector files, std::string stri for ( const auto& filter : pathFilters ) { bool matches = String::globMatch( fsv, filter.first ); - if ( ( matches && filter.second ) || ( !matches && !filter.second ) ) { + + // if it's inverted and the file matches with the glob it must be ignored/excluded + if ( filter.second && matches ) { skip = true; break; } + + // if the file is not inverted and the file matches with the glob, then it must be + // not skiped + if ( !filter.second && matches ) { + skip = false; + break; + } } + if ( skip ) { search[pos++] = false; continue; } + search[pos++] = true; count++; }