SyntaxTokenizer improvements for pushTokensToOpenCloseSubsyntax with non-sequential captures.

Added VimL support.

include/eepp/ui/doc/syntaxdefinition.hpp

@@ -55,6 +55,8 @@ struct EE_API SyntaxPattern {
 	std::vector<SyntaxStyleType> endTypes;
 	std::vector<std::string> typesNames;
 	std::vector<std::string> endTypesNames;
+	std::string contentTypeName;
+	SyntaxStyleType contentType{ SyntaxStyleEmpty() };
 	std::string syntax{ "" };
 	DynamicSyntax dynSyntax;
 	SyntaxPatternMatchType matchType{ SyntaxPatternMatchType::LuaPattern };

src/eepp/ui/doc/syntaxdefinitionmanager.cpp

@@ -52,6 +52,7 @@ class TextMateScopeMapper {
 		{ "entity.name.tag", "type" },			// HTML/XML tag name
 		{ "keyword.control", "keyword" },		// if, else, for, while, return etc.
 		{ "keyword.operator", "operator" },		// +, -, =, and, or, etc.
+		{ "punctuation.definition.comment", "comment" },
 		{ "punctuation.definition.string", "string" }, // <, >, </ in HTML/XML
 		{ "punctuation.definition.tag", "operator" },  // <, >, </ in HTML/XML
 		{ "support.function", "function" },			   // Built-in functions (print, len)
@@ -259,6 +260,9 @@ static std::optional<nlohmann::json> serializePattern( const SyntaxPattern& ptrn
 			pattern["patterns"] = std::move( ptrns );
 	}
 
+	if ( !ptrn.contentTypeName.empty() )
+		pattern["contentName"] = ptrn.contentTypeName;
+
 	return pattern;
 }
 
@@ -688,6 +692,8 @@ static SyntaxPattern parsePattern( const nlohmann::json& pattern ) {
 	std::vector<std::string> ptrns;
 	std::vector<SyntaxPattern> subPatterns;
 	auto ctype = SyntaxPatternMatchType::LuaPattern;
+	std::string contentTypeName;
+	SyntaxStyleType contentType{ SyntaxStyleEmpty() };
 	std::string syntax;
 
 	const auto fillTypes = []( const nlohmann::json& captures, std::vector<std::string>& type,
@@ -729,9 +735,9 @@ static SyntaxPattern parsePattern( const nlohmann::json& pattern ) {
 			type.emplace_back( TextMateScopeMapper::scopeToType( pattern.value( "name", "" ) ) );
 		}
 
-		if ( type.empty() && pattern.contains( "contentName" ) ) {
-			type.emplace_back(
-				TextMateScopeMapper::scopeToType( pattern.value( "contentName", "" ) ) );
+		if ( pattern.contains( "contentName" ) ) {
+			contentTypeName = pattern.value( "contentName", "" );
+			contentType = toSyntaxStyleType( TextMateScopeMapper::scopeToType( contentTypeName ) );
 		}
 
 		if ( pattern.contains( "beginCaptures" ) )
@@ -853,8 +859,17 @@ static SyntaxPattern parsePattern( const nlohmann::json& pattern ) {
 
 	eeASSERT( !ptrns.empty() );
 
-	return SyntaxPattern( std::move( ptrns ), std::move( type ), std::move( endType ), syntax,
-						  ctype, std::move( subPatterns ) );
+	SyntaxPattern ptrn( std::move( ptrns ), std::move( type ), std::move( endType ), syntax, ctype,
+						std::move( subPatterns ) );
+
+	if ( contentType != SyntaxStyleEmpty() ) {
+		ptrn.contentTypeName = std::move( contentTypeName );
+		ptrn.contentType = contentType;
+		if ( SyntaxStyleTypes::needsToBeCached( contentType ) ) {
+			SyntaxPattern::SyntaxStyleTypeCache[contentType] = ptrn.contentTypeName;
+		}
+	}
+	return ptrn;
 }
 
 static SyntaxDefinition loadTextMateLanguage( const nlohmann::json& json, SyntaxDefinition& def ) {

src/eepp/ui/doc/syntaxtokenizer.cpp

@@ -270,10 +270,10 @@ static inline void popStack( SyntaxStateRestored& curState, SyntaxState& retStat
 }
 
 template <typename T>
-static inline void pushTokensToOpenCloseSubsyntax( int i, std::string_view textv,
-												   const SyntaxPattern* subsyntaxInfo,
-												   const NonEscapedMatch& rangeSubsyntax,
-												   std::vector<T>& tokens, bool isClose = false ) {
+static inline void
+pushTokensToOpenCloseSubsyntax( int i, std::string_view textv, const SyntaxPattern* subsyntaxInfo,
+								const NonEscapedMatch& rangeSubsyntax, std::vector<T>& tokens,
+								std::vector<size_t>& priorityMap, bool isClose = false ) {
 	const auto& types = isClose && !subsyntaxInfo->endTypes.empty() ? subsyntaxInfo->endTypes
 																	: subsyntaxInfo->types;
 	if ( rangeSubsyntax.numMatches > 1 ) {
@@ -284,40 +284,100 @@ static inline void pushTokensToOpenCloseSubsyntax( int i, std::string_view textv
 			return;
 
 		auto patternType = types[0];
-		int lastStart = patternMatchStart;
-		int lastEnd = patternMatchEnd;
 
 		if ( i < patternMatchStart )
 			pushToken( tokens, patternType, textv.substr( i, patternMatchStart - i ) );
 
-		int start;
-		int end;
-
+		bool overlaps = false;
+		int lastEndCheck = patternMatchStart;
 		for ( int sidx = 1; sidx < rangeSubsyntax.numMatches; sidx++ ) {
-			start = rangeSubsyntax.matches[sidx].start;
-			end = rangeSubsyntax.matches[sidx].end;
-
-			if ( start == -1 || end == -1 )
+			int s = rangeSubsyntax.matches[sidx].start;
+			int e = rangeSubsyntax.matches[sidx].end;
+			if ( s == -1 || e == -1 )
 				continue;
-
-			if ( sidx == 1 && start > lastStart ) {
-				pushToken( tokens, patternType,
-						   textv.substr( patternMatchStart, start - patternMatchStart ) );
-			} else if ( start > lastEnd ) {
-				pushToken( tokens, patternType, textv.substr( lastEnd, start - lastEnd ) );
+			if ( s < lastEndCheck ) {
+				overlaps = true;
+				break;
 			}
+			lastEndCheck = e;
+		}
 
-			auto ss{ textv.substr( start, end - start ) };
+		if ( !overlaps ) {
+			int lastStart = patternMatchStart;
+			int lastEnd = patternMatchEnd;
+			int start;
+			int end;
 
-			pushToken( tokens, sidx < static_cast<int>( types.size() ) ? types[sidx] : types[0],
-					   ss );
+			for ( int sidx = 1; sidx < rangeSubsyntax.numMatches; sidx++ ) {
+				start = rangeSubsyntax.matches[sidx].start;
+				end = rangeSubsyntax.matches[sidx].end;
 
-			if ( sidx == rangeSubsyntax.numMatches - 1 && end < patternMatchEnd ) {
-				pushToken( tokens, patternType, textv.substr( end, patternMatchEnd - end ) );
+				if ( start == -1 || end == -1 )
+					continue;
+
+				if ( sidx == 1 && start > lastStart ) {
+					pushToken( tokens, patternType,
+							   textv.substr( patternMatchStart, start - patternMatchStart ) );
+				} else if ( start > lastEnd ) {
+					pushToken( tokens, patternType, textv.substr( lastEnd, start - lastEnd ) );
+				}
+
+				auto ss{ textv.substr( start, end - start ) };
+
+				pushToken( tokens, sidx < static_cast<int>( types.size() ) ? types[sidx] : types[0],
+						   ss );
+
+				if ( sidx == rangeSubsyntax.numMatches - 1 && end < patternMatchEnd ) {
+					pushToken( tokens, patternType, textv.substr( end, patternMatchEnd - end ) );
+				}
+
+				lastStart = start;
+				lastEnd = end;
 			}
+		} else {
+			int fullMatchLen = patternMatchEnd - patternMatchStart;
+			if ( fullMatchLen > 0 ) {
+				priorityMap.clear();
+				priorityMap.resize( fullMatchLen, 0 );
 
-			lastStart = start;
-			lastEnd = end;
+				for ( int sidx = 1; sidx < rangeSubsyntax.numMatches; sidx++ ) {
+					int start = rangeSubsyntax.matches[sidx].start;
+					int end = rangeSubsyntax.matches[sidx].end;
+
+					if ( start == -1 || end == -1 )
+						continue;
+
+					if ( start < patternMatchStart )
+						start = patternMatchStart;
+					if ( end > patternMatchEnd )
+						end = patternMatchEnd;
+					if ( start >= end )
+						continue;
+
+					for ( int k = start; k < end; ++k ) {
+						priorityMap[k - patternMatchStart] = sidx;
+					}
+				}
+
+				int currentBytePos = 0;
+				while ( currentBytePos < fullMatchLen ) {
+					size_t currentCaptureIndex = priorityMap[currentBytePos];
+					auto currentType =
+						currentCaptureIndex < types.size() ? types[currentCaptureIndex] : types[0];
+
+					int segmentEndBytePos = currentBytePos + 1;
+					while ( segmentEndBytePos < fullMatchLen &&
+							priorityMap[segmentEndBytePos] == currentCaptureIndex ) {
+						segmentEndBytePos++;
+					}
+
+					pushToken( tokens, currentType,
+							   textv.substr( patternMatchStart + currentBytePos,
+											 segmentEndBytePos - currentBytePos ) );
+
+					currentBytePos = segmentEndBytePos;
+				}
+			}
 		}
 	} else {
 		pushToken( tokens, types[0], textv.substr( i, rangeSubsyntax.range.second - i ) );
@@ -373,7 +433,8 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
 				if ( shouldCloseSubSyntax->range.second >= matches[0].end ) {
 					if ( !skipSubSyntaxSeparator ) {
 						pushTokensToOpenCloseSubsyntax( startIdx, textv, curState.subsyntaxInfo,
-														*shouldCloseSubSyntax, tokens );
+														*shouldCloseSubSyntax, tokens,
+														priorityMap );
 					}
 					popStack( curState, retState, syntax, *curState.subsyntaxInfo );
 					startIdx = shouldCloseSubSyntax->range.second;
@@ -645,7 +706,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
 			if ( activePattern->hasContentScope() ) {
 				if ( endRange.range.first == static_cast<Int64>( startIdx ) ) {
 					pushTokensToOpenCloseSubsyntax( startIdx, textv, activePattern, endRange,
-													tokens, true );
+													tokens, priorityMap, true );
 					popStack( curState, retState, syntax, *activePattern );
 					startIdx = endRange.range.second;
 					triedPatterns.clear(); // Position advanced
@@ -735,7 +796,11 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
 					String::utf8Next( strEnd );
 					int dist = strEnd - strStart;
 					if ( dist > 0 ) {
-						pushToken( tokens, activePattern->types[0], text.substr( startIdx, dist ) );
+						pushToken( tokens,
+								   activePattern->contentType != SyntaxStyleEmpty()
+									   ? activePattern->contentType
+									   : activePattern->types[0],
+								   text.substr( startIdx, dist ) );
 						startIdx += dist;
 						triedPatterns.clear(); // Position advanced
 					} else {
@@ -763,7 +828,8 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
 						   rangeSubsyntax.range.first < endRange.range.first ) ) {
 						if ( !skipSubSyntaxSeparator ) {
 							pushTokensToOpenCloseSubsyntax( startIdx, textv, curState.subsyntaxInfo,
-															rangeSubsyntax, tokens, true );
+															rangeSubsyntax, tokens, priorityMap,
+															true );
 						}
 						popStack( curState, retState, syntax, *curState.subsyntaxInfo );
 						startIdx = rangeSubsyntax.range.second;
@@ -775,7 +841,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
 				if ( !skip ) {
 					if ( endRange.range.first != -1 ) {
 						pushTokensToOpenCloseSubsyntax( startIdx, textv, activePattern, endRange,
-														tokens, true );
+														tokens, priorityMap, true );
 						popStack( curState, retState, syntax, *activePattern );
 						startIdx = endRange.range.second;
 						triedPatterns.clear(); // Position advanced
@@ -859,7 +925,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
 		if ( !matched && shouldCloseSubSyntax ) {
 			if ( !skipSubSyntaxSeparator ) {
 				pushTokensToOpenCloseSubsyntax( startIdx, textv, curState.subsyntaxInfo,
-												*shouldCloseSubSyntax, tokens );
+												*shouldCloseSubSyntax, tokens, priorityMap );
 			}
 			popStack( curState, retState, syntax, *curState.subsyntaxInfo );
 			startIdx = shouldCloseSubSyntax->range.second;

src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languages/viml.hpp

@@ -0,0 +1,11 @@
+#ifndef EE_UI_DOC_Viml
+#define EE_UI_DOC_Viml
+#include <eepp/ui/doc/syntaxdefinition.hpp>
+
+namespace EE { namespace UI { namespace Doc { namespace Language {
+
+extern SyntaxDefinition& addViml();
+
+}}}} // namespace EE::UI::Doc::Language
+
+#endif

src/modules/languages-syntax-highlighting/src/eepp/ui/doc/languagessyntaxhighlighting.cpp

@@ -123,6 +123,7 @@
 #include <eepp/ui/doc/languages/vala.hpp>
 #include <eepp/ui/doc/languages/vb.hpp>
 #include <eepp/ui/doc/languages/verilog.hpp>
+#include <eepp/ui/doc/languages/viml.hpp>
 #include <eepp/ui/doc/languages/vue.hpp>
 #include <eepp/ui/doc/languages/wren.hpp>
 #include <eepp/ui/doc/languages/x86assembly.hpp>
@@ -794,6 +795,9 @@ static void preDefinitionLangsChunk2( SyntaxDefinitionManager* sdm ) {
 		  "%.vba$", "%.vbs$" },
 	} );
 
+	sdm->addPreDefinition(
+		{ "VimL", []() -> SyntaxDefinition& { return addViml(); }, { "%.vim$", "%.vimrc$" } } );
+
 	sdm->addPreDefinition(
 		{ "Vue", []() -> SyntaxDefinition& { return addVue(); }, { "%.vue?$" } } );