Mirrors/eepp
1
0
Fork 0
mirror of https://github.com/SpartanJ/eepp.git synced 2026-07-23 03:02:50 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added regex support to the tokenizer and syntax definition.

Browse Source
This commit is contained in:
Martín Lucas Golini
2024-09-22 22:53:24 -03:00
parent 4571cb2e4a
commit 69d2db6d12
7 changed files with 71 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
include/eepp/system/luapattern.hpp
View File

@@ -33,6 +33,8 @@ class EE_API LuaPattern : public PatternMatcher {
const std::string_view& getPattern() const { return mPattern; }
virtual bool isValid() const { return true; }
protected:
std::string_view mPattern;
mutable size_t mMatchNum;

2
include/eepp/system/patternmatcher.hpp
View File

@@ -128,6 +128,8 @@ class EE_API PatternMatcher {
virtual const size_t& getNumMatches() const = 0;
virtual bool isValid() const = 0;
protected:
PatternType mType;
};

2
include/eepp/system/regex.hpp
View File

@@ -64,7 +64,7 @@ class EE_API RegEx : public PatternMatcher {
virtual ~RegEx();
bool isValid() const { return mValid; }
virtual bool isValid() const override { return mValid; }
virtual bool matches( const char* stringSearch, int stringStartOffset,
PatternMatcher::Range* matchList, size_t stringLength ) const override;

9
include/eepp/ui/doc/syntaxdefinition.hpp
View File

@@ -37,18 +37,19 @@ struct EE_API SyntaxPattern {
std::vector<std::string> typesNames;
std::string syntax{ "" };
DynamicSyntax dynSyntax;
bool isRegEx{ false };
SyntaxPattern( std::vector<std::string>&& _patterns, const std::string& _type,
const std::string& _syntax = "" );
const std::string& _syntax = "", bool isRegEx = false );
SyntaxPattern( std::vector<std::string>&& _patterns, std::vector<std::string>&& _types,
const std::string& _syntax = "" );
const std::string& _syntax = "", bool isRegEx = false );
SyntaxPattern( std::vector<std::string>&& _patterns, const std::string& _type,
DynamicSyntax&& _syntax );
DynamicSyntax&& _syntax, bool isRegEx = false );
SyntaxPattern( std::vector<std::string>&& _patterns, std::vector<std::string>&& _types,
DynamicSyntax&& _syntax );
DynamicSyntax&& _syntax, bool isRegEx = false );
bool hasSyntax() const { return !syntax.empty() || dynSyntax; }
};

22
src/eepp/ui/doc/syntaxdefinition.cpp
View File

@@ -255,38 +255,44 @@ const String::HashType& SyntaxDefinition::getLanguageId() const {
}
SyntaxPattern::SyntaxPattern( std::vector<std::string>&& _patterns, const std::string& _type,
const std::string& _syntax ) :
const std::string& _syntax, bool isRegEx ) :
patterns( std::move( _patterns ) ),
types( toSyntaxStyleTypeV( std::vector<std::string>{ _type } ) ),
typesNames( { _type } ),
syntax( _syntax ) {
syntax( _syntax ),
isRegEx( isRegEx ) {
updateCache<SyntaxStyleType>( *this );
}
SyntaxPattern::SyntaxPattern( std::vector<std::string>&& _patterns,
std::vector<std::string>&& _types, const std::string& _syntax ) :
std::vector<std::string>&& _types, const std::string& _syntax,
bool isRegEx ) :
patterns( std::move( _patterns ) ),
types( toSyntaxStyleTypeV( _types ) ),
typesNames( std::move( _types ) ),
syntax( _syntax ) {
syntax( _syntax ),
isRegEx( isRegEx ) {
updateCache<SyntaxStyleType>( *this );
}
SyntaxPattern::SyntaxPattern( std::vector<std::string>&& _patterns, const std::string& _type,
DynamicSyntax&& _syntax ) :
DynamicSyntax&& _syntax, bool isRegEx ) :
patterns( std::move( _patterns ) ),
types( toSyntaxStyleTypeV( std::vector<std::string>{ _type } ) ),
typesNames( { _type } ),
dynSyntax( std::move( _syntax ) ) {
dynSyntax( std::move( _syntax ) ),
isRegEx( isRegEx ) {
updateCache<SyntaxStyleType>( *this );
}
SyntaxPattern::SyntaxPattern( std::vector<std::string>&& _patterns,
std::vector<std::string>&& _types, DynamicSyntax&& _syntax ) :
std::vector<std::string>&& _types, DynamicSyntax&& _syntax,
bool isRegEx ) :
patterns( std::move( _patterns ) ),
types( toSyntaxStyleTypeV( _types ) ),
typesNames( std::move( _types ) ),
dynSyntax( std::move( _syntax ) ) {
dynSyntax( std::move( _syntax ) ),
isRegEx( isRegEx ) {
updateCache<SyntaxStyleType>( *this );
}

26
src/eepp/ui/doc/syntaxdefinitionmanager.cpp
View File

@@ -233,10 +233,11 @@ static json toJson( const SyntaxDefinition& def ) {
j["patterns"] = json::array();
for ( const auto& ptrn : def.getPatterns() ) {
json pattern;
auto ptrnType = ptrn.isRegEx ? "regex" : "pattern";
if ( ptrn.patterns.size() == 1 ) {
pattern["pattern"] = ptrn.patterns[0];
pattern[ptrnType] = ptrn.patterns[0];
} else {
pattern["pattern"] = ptrn.patterns;
pattern[ptrnType] = ptrn.patterns;
}
if ( ptrn.typesNames.size() == 1 ) {
pattern["type"] = ptrn.typesNames[0];
@@ -352,9 +353,13 @@ namespace EE { namespace UI { namespace Doc { namespace Language {
buf += join( def.getFiles() ) + ",\n";
// patterns
buf += "{\n";
for ( const auto& pattern : def.getPatterns() )
for ( const auto& pattern : def.getPatterns() ) {
buf += "{ " + join( pattern.patterns ) + ", " + join( pattern.typesNames, true, true ) +
str( pattern.syntax, ", ", "", false ) + " },\n";
str( pattern.syntax, ", ", "", false );
if ( pattern.isRegEx )
buf += ", true";
buf += " },\n";
}
buf += "\n},\n";
// symbols
buf += "{\n";
@@ -521,6 +526,7 @@ static SyntaxDefinition loadLanguage( const nlohmann::json& json ) {
? ""
: pattern.value( "syntax", "" );
std::vector<std::string> ptrns;
bool isRegEx = false;
if ( pattern.contains( "pattern" ) ) {
if ( pattern["pattern"].is_array() ) {
const auto& ptrnIt = pattern["pattern"];
@@ -529,8 +535,18 @@ static SyntaxDefinition loadLanguage( const nlohmann::json& json ) {
} else if ( pattern["pattern"].is_string() ) {
ptrns.emplace_back( pattern["pattern"] );
}
} else if ( pattern.contains( "regex" ) ) {
isRegEx = true;
if ( pattern["regex"].is_array() ) {
const auto& ptrnIt = pattern["regex"];
for ( const auto& ptrn : ptrnIt )
ptrns.emplace_back( ptrn );
} else if ( pattern["regex"].is_string() ) {
ptrns.emplace_back( pattern["regex"] );
}
}
def.addPattern( SyntaxPattern( std::move( ptrns ), std::move( type ), syntax ) );
def.addPattern(
SyntaxPattern( std::move( ptrns ), std::move( type ), syntax, isRegEx ) );
}
}
if ( json.contains( "symbols" ) ) {

36
src/eepp/ui/doc/syntaxtokenizer.cpp
View File

@@ -1,7 +1,9 @@
#include <eepp/system/log.hpp>
#include <eepp/system/luapattern.hpp>
#include <eepp/system/regex.hpp>
#include <eepp/ui/doc/syntaxdefinitionmanager.hpp>
#include <eepp/ui/doc/syntaxtokenizer.hpp>
#include <variant>
using namespace EE::System;
@@ -84,7 +86,8 @@ static void pushToken( std::vector<T>& tokens, const SyntaxStyleType& type,
}
}
bool isScaped( const std::string& text, const size_t& startIndex, const std::string& escapeStr ) {
static bool isScaped( const std::string& text, const size_t& startIndex,
const std::string& escapeStr ) {
char escapeByte = escapeStr.empty() ? '\\' : escapeStr[0];
int count = 0;
for ( int i = startIndex - 1; i >= 0; i-- ) {
@@ -95,12 +98,17 @@ bool isScaped( const std::string& text, const size_t& startIndex, const std::str
return count % 2 == 1;
}
std::pair<int, int> findNonEscaped( const std::string& text, const std::string& pattern, int offset,
const std::string& escapeStr ) {
static std::pair<int, int> findNonEscaped( const std::string& text, const std::string& pattern,
int offset, const std::string& escapeStr,
bool isRegEx ) {
eeASSERT( !pattern.empty() );
if ( pattern.empty() )
return std::make_pair( -1, -1 );
LuaPattern words( pattern );
std::variant<RegEx, LuaPattern> wordsVar =
isRegEx ? std::variant<RegEx, LuaPattern>( RegEx( pattern ) )
: std::variant<RegEx, LuaPattern>( LuaPattern( pattern ) );
PatternMatcher& words =
std::visit( []( auto& patternType ) -> PatternMatcher& { return patternType; }, wordsVar );
int start, end;
while ( words.find( text, start, end, offset ) ) {
if ( !escapeStr.empty() && isScaped( text, start, escapeStr ) ) {
@@ -202,9 +210,9 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
if ( curState.currentPatternIdx != SYNTAX_TOKENIZER_STATE_NONE ) {
const SyntaxPattern& pattern =
curState.currentSyntax->getPatterns()[curState.currentPatternIdx - 1];
std::pair<int, int> range =
findNonEscaped( text, pattern.patterns[1], i,
pattern.patterns.size() >= 3 ? pattern.patterns[2] : "" );
std::pair<int, int> range = findNonEscaped(
text, pattern.patterns[1], i,
pattern.patterns.size() >= 3 ? pattern.patterns[2] : "", pattern.isRegEx );
bool skip = false;
@@ -213,7 +221,8 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
findNonEscaped( text, curState.subsyntaxInfo->patterns[1], i,
curState.subsyntaxInfo->patterns.size() >= 3
? curState.subsyntaxInfo->patterns[2]
: "" );
: "",
pattern.isRegEx );
if ( rangeSubsyntax.first != -1 &&
( range.first == -1 || rangeSubsyntax.first < range.first ) ) {
@@ -249,7 +258,8 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
std::pair<int, int> rangeSubsyntax = findNonEscaped(
text, "^" + curState.subsyntaxInfo->patterns[1], i,
curState.subsyntaxInfo->patterns.size() >= 3 ? curState.subsyntaxInfo->patterns[2]
: "" );
: "",
curState.subsyntaxInfo->isRegEx );
if ( rangeSubsyntax.first != -1 ) {
if ( !skipSubSyntaxSeparator ) {
@@ -270,7 +280,13 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
continue;
patternStr =
pattern.patterns[0][0] == '^' ? pattern.patterns[0] : "^" + pattern.patterns[0];
LuaPattern words( patternStr );
std::variant<RegEx, LuaPattern> wordsVar =
pattern.isRegEx ? std::variant<RegEx, LuaPattern>( RegEx( patternStr ) )
: std::variant<RegEx, LuaPattern>( LuaPattern( patternStr ) );
PatternMatcher& words = std::visit(
[]( auto& patternType ) -> PatternMatcher& { return patternType; }, wordsVar );
if ( !words.isValid() ) // Skip invalid patterns
continue;
if ( words.matches( text, matches, i ) && ( numMatches = words.getNumMatches() ) > 0 ) {
if ( numMatches > 1 ) {
int patternMatchStart = matches[0].start;