From d0f4a2eeea8e7f18ef4334a62f214dd464434cc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Sat, 12 Apr 2025 17:55:05 -0300 Subject: [PATCH] Fix some issues with RegEx captures. Added Janet syntax highlighting (SpartanJ/ecode#394). Fixes in syntax highlighter to C++ converter. --- include/eepp/system/regex.hpp | 6 +- projects/linux/ee.creator.user | 140 ++++++++++++++++-- src/eepp/system/regex.cpp | 28 ++-- src/eepp/ui/doc/syntaxdefinitionmanager.cpp | 4 +- src/eepp/ui/doc/syntaxtokenizer.cpp | 2 +- .../src/eepp/ui/doc/languages/janet.cpp | 104 +++++++++++++ .../src/eepp/ui/doc/languages/janet.hpp | 10 ++ .../ui/doc/languagessyntaxhighlighting.cpp | 2 + src/tools/ecode/ecode.cpp | 2 +- 9 files changed, 272 insertions(+), 26 deletions(-) create mode 100644 src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/janet.cpp create mode 100644 src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/janet.hpp diff --git a/include/eepp/system/regex.hpp b/include/eepp/system/regex.hpp index 50d9c8dba..d50df2a56 100644 --- a/include/eepp/system/regex.hpp +++ b/include/eepp/system/regex.hpp @@ -58,9 +58,12 @@ class EE_API RegEx : public PatternMatcher { ExtendedMore = 0x01000000u, // C Literal = 0x02000000u, // C MatchInvalidUtf = 0x04000000u, // J M D + FilterOutCaptures = + 0x08000000u, // It will filter out repeated captures and same range captures }; - RegEx( const std::string_view& pattern, Options options = Options::Utf, bool useCache = true ); + RegEx( const std::string_view& pattern, + Uint32 options = Options::Utf | Options::FilterOutCaptures, bool useCache = true ); virtual ~RegEx(); @@ -83,6 +86,7 @@ class EE_API RegEx : public PatternMatcher { int mCaptureCount; bool mValid{ false }; bool mCached{ false }; + bool mFilterOutCaptures{ false }; }; }} // namespace EE::System diff --git a/projects/linux/ee.creator.user b/projects/linux/ee.creator.user index afdebe4f2..68e1d9ac9 100644 --- a/projects/linux/ee.creator.user +++ b/projects/linux/ee.creator.user @@ -1,6 +1,6 @@ - + EnvironmentId @@ -13,8 +13,8 @@ ProjectExplorer.Project.EditorSettings + true false - false true Cpp @@ -39,6 +39,7 @@ false 4 false + 0 100 true true @@ -1035,17 +1036,18 @@ false -e cpu-cycles --call-graph dwarf,4096 -F 250 - %{buildDir}../../../bin/eeiv-debug + %{buildDir}/../../bin/eeiv-debug eeiv-debug ProjectExplorer.CustomExecutableRunConfiguration - false + true 0 false + 1 1 false false - %{buildDir}../../../bin/ + %{buildDir}/../../bin true @@ -1107,7 +1109,7 @@ ProjectExplorer.CustomExecutableRunConfiguration true - -u --xml /home/downloads/untitled_1.xml + -u true 1 false @@ -1197,7 +1199,7 @@ ProjectExplorer.CustomExecutableRunConfiguration true - --css=/root/.config/ecode/style.css + /home/downloads/files/svn/janet 0 false 1 @@ -1425,6 +1427,29 @@ false %{buildDir}/../../bin + + true + 0 + true + true + + 2 + + false + -e cpu-cycles --call-graph dwarf,4096 -F 250 + %{buildDir}/../../bin/eepp-ui-custom-widget-debug + eepp-ui-custom-widget-debug + ProjectExplorer.CustomExecutableRunConfiguration + + true + 0 + false + 1 + 1 + false + false + %{buildDir}../../../bin/ + true 0 @@ -1524,10 +1549,12 @@ eepp-sprites-debug ProjectExplorer.CustomExecutableRunConfiguration - false - 1 + true + 0 false - true + 1 + 1 + false false %{buildDir}../../../bin/ @@ -1574,7 +1601,7 @@ false %{buildDir}../../../bin/ - 29 + 30 @@ -1666,9 +1693,98 @@ 1 + + ProjectExplorer.Project.Target.2 + + Desktop + Python 2.7.18 + Python 2.7.18 + {0790f8fd-ca23-4e38-9ab1-4094ae2bb99e} + 0 + 0 + 0 + + /home/programming/eepp/projects/linux + + + + all + + true + GenericProjectManager.GenericMakeStep + + 1 + Build + Build + ProjectExplorer.BuildSteps.Build + + + + + clean + + true + GenericProjectManager.GenericMakeStep + + 1 + Clean + Clean + ProjectExplorer.BuildSteps.Clean + + 2 + false + + false + + Default + GenericProjectManager.GenericBuildConfiguration + + 1 + + + 0 + Deploy + Deploy + ProjectExplorer.BuildSteps.Deploy + + 1 + + false + ProjectExplorer.DefaultDeployConfiguration + + 1 + + true + true + 0 + false + false + false + /usr/bin/kcachegrind + 2 + 2 + true + true + /usr/bin/valgrind + + + 2 + + false + -e cpu-cycles --call-graph dwarf,4096 -F 250 + + ProjectExplorer.CustomExecutableRunConfiguration + + false + true + true + + 1 + + ProjectExplorer.Project.TargetCount - 2 + 3 ProjectExplorer.Project.Updater.FileVersion diff --git a/src/eepp/system/regex.cpp b/src/eepp/system/regex.cpp index 230e35082..c19d90d4f 100644 --- a/src/eepp/system/regex.cpp +++ b/src/eepp/system/regex.cpp @@ -25,17 +25,21 @@ void RegExCache::clear() { mCache.clear(); } -RegEx::RegEx( const std::string_view& pattern, Options options, bool useCache ) : +RegEx::RegEx( const std::string_view& pattern, Uint32 options, bool useCache ) : PatternMatcher( PatternType::PCRE ), mPattern( pattern ), mMatchNum( 0 ), mCompiledPattern( nullptr ), mCaptureCount( 0 ), - mValid( true ) { + mValid( true ), + mFilterOutCaptures( ( options & Options::FilterOutCaptures ) != 0 ) { int errornumber; PCRE2_SIZE erroroffset; PCRE2_SPTR pattern_sptr = reinterpret_cast( pattern.data() ); + if ( mFilterOutCaptures ) + options &= ~Options::FilterOutCaptures; + if ( useCache && RegExCache::instance()->isEnabled() && ( mCompiledPattern = RegExCache::instance()->find( pattern, options ) ) ) { mValid = true; @@ -108,17 +112,23 @@ bool RegEx::matches( const char* stringSearch, int stringStartOffset, mMatchNum = rc; - if ( matchList != nullptr ) { + if ( matchList != nullptr && mMatchNum > 0 ) { PCRE2_SIZE* ovector = pcre2_get_ovector_pointer( match_data ); + int curCap = 0; for ( size_t i = 0; i < static_cast( rc ); ++i ) { - matchList[i].start = stringStartOffset + static_cast( ovector[2 * i] ); - matchList[i].end = stringStartOffset + static_cast( ovector[2 * i + 1] ); - if ( matchList[i].start >= matchList[i].end ) { - matchList[i].start = matchList[i].end = -1; - mMatchNum--; - break; + int start = stringStartOffset + static_cast( ovector[2 * i] ); + int end = stringStartOffset + static_cast( ovector[2 * i + 1] ); + if ( !mFilterOutCaptures || + ( !( start == 0 && end == 0 ) && start != end && + ( curCap == 0 || !( matchList[curCap - 1].start == start && + matchList[curCap - 1].end == end ) ) ) ) { + matchList[curCap].start = start; + matchList[curCap].end = end; + curCap++; } } + + mMatchNum = curCap; } pcre2_match_data_free( match_data ); diff --git a/src/eepp/ui/doc/syntaxdefinitionmanager.cpp b/src/eepp/ui/doc/syntaxdefinitionmanager.cpp index 6d7d50d33..847b15485 100644 --- a/src/eepp/ui/doc/syntaxdefinitionmanager.cpp +++ b/src/eepp/ui/doc/syntaxdefinitionmanager.cpp @@ -283,14 +283,14 @@ namespace EE { namespace UI { namespace Doc { namespace Language { break; } if ( !fdtn.empty() ) - buf += String::format( ".setFoldRangeType( \"%s\" )\n", fdtn ); + buf += String::format( ".setFoldRangeType( %s )\n", fdtn ); } if ( !def.getFoldBraces().empty() ) { buf += ".setFoldBraces( { "; for ( const auto& brace : def.getFoldBraces() ) { buf += String::format( - "{ '%s', '%s' }", + "{ '%s', '%s' },", String( static_cast( brace.first ) ).toUtf8(), String( static_cast( brace.second ) ).toUtf8() ); } diff --git a/src/eepp/ui/doc/syntaxtokenizer.cpp b/src/eepp/ui/doc/syntaxtokenizer.cpp index e17822420..1d00def54 100644 --- a/src/eepp/ui/doc/syntaxtokenizer.cpp +++ b/src/eepp/ui/doc/syntaxtokenizer.cpp @@ -373,7 +373,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax for ( size_t curMatch = 1; curMatch < numMatches; curMatch++ ) { start = matches[curMatch].start; end = matches[curMatch].end; - if ( start == end && matches[curMatch - 1].end == start ) + if ( start == end || start < 0 || end < 0 ) continue; if ( pattern.patterns.size() >= 3 && i > 0 && text[i - 1] == pattern.patterns[2][0] ) diff --git a/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/janet.cpp b/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/janet.cpp new file mode 100644 index 000000000..e1f64b200 --- /dev/null +++ b/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/janet.cpp @@ -0,0 +1,104 @@ +#include +#include + +namespace EE { namespace UI { namespace Doc { namespace Language { + +void addJanet() { + + SyntaxDefinitionManager::instance() + ->add( + + { "Janet", + { "%.janet$" }, + { + { { "(@?)```", "```", "\\" }, "string" }, + { { "(@?)``", "``", "\\" }, "string" }, + { { "(@?)`", "`", "\\" }, "string" }, + { { "\"", "\"", "\\" }, "string" }, + { { "0x[%da-fA-F]+" }, "number" }, + { { "-?%d+[%d%.eE]*f?" }, "number" }, + { { "#.-\n" }, "comment" }, + { { "^\\((def|defglobal|defdyn)\\-?\\s+([a-zA-Z0-9!$%&*+-./:@^_\"]+)" }, + std::vector{ "operator", "keyword", "literal" }, + "", + SyntaxPatternMatchType::RegEx }, + { { "[';~,|]*\\((defn|defmacro)\\-?\\s+([a-zA-Z0-9!$%&*+-./:@^_\"]+)" }, + std::vector{ "operator", "keyword", "literal" }, + "", + SyntaxPatternMatchType::RegEx }, + { { "[';~,|]*\\(([';~,|]*)(break|def|do|fn|if|quasiquote|quote|set|splice|" + "unquote|upscope|var|while|call|maker|array|tuple|tablector|bufferctor|asm|" + "disasm|compile|dyn|setdyn|native|describe|string|symbol|keyword|buffer|" + "abstract\\?|scan-number|tuple|array|slice|table|getproto|struct|gensym|" + "gccollect|gcsetinterval|gcinterval|type|hash|getline|trace|untrace|int\\?|" + "nat\\?|signal|memcmp|sandbox|print|prin|eprint|eprin|xprint|xprin|printf|" + "prinf|eprintf|eprinf|xprintf|xprinf|flush|eflush|env-lookup|marshal|" + "unmarshal|not|debug|error|apply|yield|resume|in|put|length|add|sub|mul|div|" + "band|bor|bxor|lshift|rshift|rshiftu|bnot|gt|lt|gte|lte|eq|neq|propagate|get|" + "next|modulo|remainder|cmp|cancel|mod|sandbox|defmacro|defglobal|varglobal|" + "nan\\?|number\\?|fiber\\?|string\\?|symbol\\?|keyword\\?|buffer\\?|" + "function\\?|cfunction\\?|table\\?|struct\\?|array\\?|tuple\\?|boolean\\?|" + "bytes\\?|dictionary\\?|indexed\\?|truthy\\?|true\\?|false\\?|nil\\?|empty\\?" + "|odd\\?|inc|dec|errorf|return|sum|mean|product|comp|identity|complement|" + "extreme|max|min|max-of|min-of|first|last|compare|compare=|compare<|compare<=" + "|compare>|compare>=|zero\\?|pos\\?|neg\\?|one\\?|even\\?|odd\\?|sort|sort-" + "by|sorted|sorted-by|reduce|reduce2|accumulate|accumulate2|map|mapcat|filter|" + "count|keep|range|find-index|find|index-of|take|take-until|take-while|drop|" + "drop-until|drop-while|juxt\\*|walk|postwalk|prewalk|partial|every\\?|any\\?|" + "reverse!|reverse|invert|zipcoll|get-in|update-in|put-in|update|merge-into|" + "merge|keys|values|pairs|frequencies|group-by|partition-by|interleave|" + "distinct|flatten-into|flatten|kvs|from-pairs|interpose|partition|slurp|spit|" + "pp|maclintf|macex1|all|some|not=|deep-not=|deep=|freeze|macex|make-env|bad-" + "parse|warn-compile|bad-compile|curenv|run-context|quit|eval|parse|parse-all|" + "eval-string|make-image|load-image|debugger|debugger-on-status|dofile|" + "require|merge-module|import\\*|all-bindings|all-dynamics|doc-format|doc\\*|" + "doc-of|\\.fiber|\\.signal|\\.stack|\\.frame|\\.locals|\\.fn|\\.slots|\\." + "slot|\\.source|\\.break|\\.clear|\\.next|\\.nextc|\\.step|\\.locals|repl|" + "flycheck|cli-main|as-macro|defmacro-|defn-|def-|var-|toggle|assert|default|" + "comment|if-not|when|unless|cond|case|let|try|protect|and|or|with-syms|defer|" + "edefer|prompt|chr|label|with|when-with|if-with|forv|for|eachk|eachp|repeat|" + "forever|each|loop|seq|catseq|tabseq|generate|coro|fiber-fn|if-let|when-let|" + "juxt|defdyn|tracev|with-dyns|with-vars|match|varfn|short-fn|comptime|compif|" + "compwhen|import|use|doc|delay|keep-syntax|keep-syntax!|->|->>|-?>|-?>>|as->|" + "as?->|--|\\+=|\\+\\+|-=|\\*=|/=|%=|\\+|-|\\*|%|/" + "|>=|<=|=|<|>)((?=$|[\\s,()\\[\\]{}\\\";])+)" }, + std::vector{ "operator", "keyword", "literal" }, + "", + SyntaxPatternMatchType::RegEx }, + { { "%(?(%:[%a_][%w_]*)" }, { "operator", "keyword3", "keyword3" } }, + { { "[';~,|]*\\(([a-zA-Z!$%&*+-./:@^_\"]+/?[a-zA-Z!$%&*+-./:@^_\"]+)" }, + std::vector{ "operator", "function", "literal" }, + "", + SyntaxPatternMatchType::RegEx }, + { { "(nil|true|false)(?=$|[\\s,()\\[\\]{}\";])+" }, + "literal", + "", + SyntaxPatternMatchType::RegEx }, + { { "[a-zA-Z0-9!$%&*+-./:@^_\"]+" }, + "keyword2", + "", + SyntaxPatternMatchType::RegEx }, + { { "([';~,|]*)@?(\\()" }, + std::vector{ "operator", "operator", "operator" }, + "", + SyntaxPatternMatchType::RegEx }, + { { "[%)%[%]%{%}]" }, "operator" }, + + }, + { + + }, + "#", + {} + + } ) + .setFoldRangeType( FoldRangeType::Markdown ) + .setFoldBraces( { + { '(', ')' }, + { '{', '}' }, + { '[', ']' }, + } ); + ; +} + +}}}} // namespace EE::UI::Doc::Language diff --git a/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/janet.hpp b/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/janet.hpp new file mode 100644 index 000000000..449b9e199 --- /dev/null +++ b/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/janet.hpp @@ -0,0 +1,10 @@ +#ifndef EE_UI_DOC_Janet +#define EE_UI_DOC_Janet + +namespace EE { namespace UI { namespace Doc { namespace Language { + +extern void addJanet(); + +}}}} // namespace EE::UI::Doc::Language + +#endif diff --git a/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languagessyntaxhighlighting.cpp b/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languagessyntaxhighlighting.cpp index 78354786e..67f63002d 100644 --- a/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languagessyntaxhighlighting.cpp +++ b/src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languagessyntaxhighlighting.cpp @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -151,6 +152,7 @@ void LanguagesSyntaxHighlighting::load() { addHtaccessFile(); addIgnoreFile(); addJai(); + addJanet(); addJava(); addJule(); addJulia(); diff --git a/src/tools/ecode/ecode.cpp b/src/tools/ecode/ecode.cpp index 5fed0e04c..093fd9f67 100644 --- a/src/tools/ecode/ecode.cpp +++ b/src/tools/ecode/ecode.cpp @@ -3536,7 +3536,7 @@ void App::init( const LogLevel& logLevel, std::string file, const Float& pidelDe mThreadPool->run( [this] { // Load language definitions Clock defClock; - SyntaxDefinitionManager::createSingleton( 122 ); + SyntaxDefinitionManager::createSingleton( 123 ); Language::LanguagesSyntaxHighlighting::load(); SyntaxDefinitionManager::instance()->setLanguageExtensionsPriority( mConfig.languagesExtensions.priorities );