Support dynamic syntax detection in tokenizer.

src/eepp/core/memorymanager.cpp

@@ -5,8 +5,6 @@
 #include <eepp/system/log.hpp>
 #include <eepp/system/mutex.hpp>
 #include <eepp/window/engine.hpp>
-#include <iostream>
-#include <sstream>
 
 using namespace EE::System;
 using namespace EE::Window;

src/eepp/system/inifile.cpp

@@ -10,12 +10,6 @@
 
 namespace EE { namespace System {
 
-#if EE_PLATFORM == EE_PLATFORM_WIN
-#define iniEOL std::endl
-#else
-#define iniEOL '\r' << std::endl
-#endif
-
 IniFile::IniFile( const std::string& iniPath, bool autoLoad ) {
 	if ( autoLoad )
 		loadFromFile( iniPath );
@@ -97,7 +91,6 @@ bool IniFile::readFile() {
 		isBOM = true;
 	}
 
-	// ini_parse_string( mBuffer.c_str(), ValueHandler, this );
 	std::string_view buffer( mBuffer );
 	if ( isBOM )
 		buffer = std::string_view( mBuffer.data() + 3, mBuffer.size() - 3 );

src/eepp/ui/doc/languages/markdown.cpp

@@ -4,6 +4,13 @@
 namespace EE { namespace UI { namespace Doc { namespace Language {
 
 void addMarkdown() {
+	auto dynSyntax = []( const SyntaxPattern&, const std::string& match ) -> std::string {
+		std::string lang = String::toLower( match.substr( 3 ) );
+		String::trimInPlace( lang );
+		if ( !lang.empty() && lang[lang.size() - 1] == '\n' )
+			lang.pop_back();
+		return SyntaxDefinitionManager::instance()->findFromString( lang ).getLanguageName();
+	};
 
 	SyntaxDefinitionManager::instance()->add(
 
@@ -11,40 +18,7 @@ void addMarkdown() {
 		  { "%.md$", "%.markdown$" },
 		  {
 			  { { "\\." }, "normal" },
-			  { { "```[Xx][Mm][Ll]", "```" }, "function", "XML" },
-			  { { "```[Hh][Tt][Mm][Ll]", "```" }, "function", "html" },
-			  { { "```[Cc]++", "```" }, "function", "C++" },
-			  { { "```[Cc][Pp][Pp]", "```" }, "function", "C++" },
-			  { { "```[Cc]sharp", "```" }, "function", "C#" },
-			  { { "```[Cc][Ss][Ss]", "```" }, "function", "CSS" },
-			  { { "```[Cc]", "```" }, "function", "C" },
-			  { { "```[Dd][Aa][Rr][Tt]", "```" }, "function", "Dart" },
-			  { { "```[Dd]", "```" }, "function", "D" },
-			  { { "```[Ll]ua", "```" }, "function", "Lua" },
-			  { { "```[Jj][Ss][Oo][Nn]", "```" }, "function", "JSON" },
-			  { { "```[Ja]va[Ss]cript", "```" }, "function", "JavaScript" },
-			  { { "```[Tt]ype[Ss]cript", "```" }, "function", "TypeScript" },
-			  { { "```[Pp]ython", "```" }, "function", "Python" },
-			  { { "```[Bb]ash", "```" }, "function", "Bash" },
-			  { { "```[Pp][Hh][Pp]", "```" }, "function", "PHPCore" },
-			  { { "```[Ss][Qq][Ll]", "```" }, "function", "SQL" },
-			  { { "```[Gg][Ll][Ss][Ll]", "```" }, "function", "GLSL" },
-			  { { "```[Ii][Nn][Ii]", "```" }, "function", "Config File" },
-			  { { "```[Mm]akefile", "```" }, "function", "Makefile" },
-			  { { "```[Gg][Oo]", "```" }, "function", "Go" },
-			  { { "```[Rr]ust", "```" }, "function", "Rust" },
-			  { { "```[Rr]uby", "```" }, "function", "Ruby" },
-			  { { "```[Gg][Dd][Ss]cript", "```" }, "function", "GSCript" },
-			  { { "```[Jj]ava", "```" }, "function", "java" },
-			  { { "```[Ss]wift", "```" }, "function", "Swift" },
-			  { { "```[Oo]bjective[Cc]", "```" }, "function", "Objective-C" },
-			  { { "```[Yy][Aa][Mm][Ll]", "```" }, "function", "YAML" },
-			  { { "```[Kk]otlin", "```" }, "function", "Kotlin" },
-			  { { "```[Ss]olidity", "```" }, "function", "Solidity" },
-			  { { "```[Hh]askell", "```" }, "function", "Haskell" },
-			  { { "```[Oo]din", "```" }, "function", "Odin" },
-			  { { "```[Nn]im", "```" }, "function", "Nim" },
-			  { { "```[Zz]ig", "```" }, "function", "Zig" },
+			  { { "```[%w%s+-#]+", "```" }, "function", dynSyntax },
 			  { { "<!%-%-", "%-%->" }, "comment" },
 			  { { "```", "```" }, "string" },
 			  { { "``", "``" }, "string" },

src/eepp/ui/doc/syntaxdefinitionmanager.cpp

@@ -1915,7 +1915,7 @@ SyntaxDefinition& SyntaxDefinitionManager::add( SyntaxDefinition&& syntaxStyle )
 	return mDefinitions.back();
 }
 
-const SyntaxDefinition& SyntaxDefinitionManager::getPlainStyle() const {
+const SyntaxDefinition& SyntaxDefinitionManager::getPlainDefinition() const {
 	return mDefinitions[0];
 }
 
@@ -1932,6 +1932,16 @@ SyntaxDefinitionManager::getByLanguageName( const std::string& name ) const {
 	return mDefinitions[0];
 }
 
+const SyntaxDefinition&
+SyntaxDefinitionManager::getByLanguageNameInsensitive( std::string name ) const {
+	String::toLowerInPlace( name );
+	for ( auto& style : mDefinitions ) {
+		if ( String::toLower( style.getLanguageName() ) == name )
+			return style;
+	}
+	return mDefinitions[0];
+}
+
 const SyntaxDefinition& SyntaxDefinitionManager::getByLSPName( const std::string& name ) const {
 	for ( auto& style : mDefinitions ) {
 		if ( style.getLSPName() == name )
@@ -2303,4 +2313,14 @@ const SyntaxDefinition& SyntaxDefinitionManager::find( const std::string& filePa
 	return def;
 }
 
+const SyntaxDefinition& SyntaxDefinitionManager::findFromString( const std::string& lang ) const {
+	const auto& syn = getByLSPName( lang );
+	if ( syn.getLSPName() != getPlainDefinition().getLSPName() )
+		return syn;
+	const auto& syn2 = getByLanguageNameInsensitive( lang );
+	if ( syn2.getLSPName() != getPlainDefinition().getLSPName() )
+		return syn2;
+	return getPlainDefinition();
+}
+
 }}} // namespace EE::UI::Doc

src/eepp/ui/doc/syntaxhighlighter.cpp

@@ -46,7 +46,7 @@ void SyntaxHighlighter::invalidate( Int64 lineIndex ) {
 	mMaxWantedLine = eemin<Int64>( mMaxWantedLine, (Int64)mDoc->linesCount() - 1 );
 }
 
-TokenizedLine SyntaxHighlighter::tokenizeLine( const size_t& line, const Uint64& state ) {
+TokenizedLine SyntaxHighlighter::tokenizeLine( const size_t& line, const SyntaxState& state ) {
 	auto& ln = mDoc->line( line );
 	TokenizedLine tokenizedLine;
 	tokenizedLine.initState = state;
@@ -161,7 +161,7 @@ const std::vector<SyntaxTokenPosition>& SyntaxHighlighter::getLine( const size_t
 		return noHighlightVector;
 	}
 
-	int prevState = SYNTAX_TOKENIZER_STATE_NONE;
+	SyntaxState prevState;
 	if ( index > 0 ) {
 		Lock l( mLinesMutex );
 		auto prevIt = mLines.find( index - 1 );
@@ -195,7 +195,7 @@ bool SyntaxHighlighter::updateDirty( int visibleLinesCount ) {
 		Int64 max = eemax( 0LL, eemin( mFirstInvalidLine + visibleLinesCount, mMaxWantedLine ) );
 
 		for ( Int64 index = mFirstInvalidLine; index <= max; index++ ) {
-			Uint64 state = SYNTAX_TOKENIZER_STATE_NONE;
+			SyntaxState state;
 			Lock l( mLinesMutex );
 			if ( index > 0 ) {
 				auto prevIt = mLines.find( index - 1 );
@@ -223,14 +223,14 @@ SyntaxHighlighter::getSyntaxDefinitionFromTextPosition( const TextPosition& posi
 	Lock l( mLinesMutex );
 	auto found = mLines.find( position.line() );
 	if ( found == mLines.end() )
-		return SyntaxDefinitionManager::instance()->getPlainStyle();
+		return SyntaxDefinitionManager::instance()->getPlainDefinition();
 
 	TokenizedLine& line = found->second;
-	SyntaxState state =
+	SyntaxStateRestored state =
 		SyntaxTokenizer::retrieveSyntaxState( mDoc->getSyntaxDefinition(), line.state );
 
 	if ( nullptr == state.currentSyntax )
-		return SyntaxDefinitionManager::instance()->getPlainStyle();
+		return SyntaxDefinitionManager::instance()->getPlainDefinition();
 
 	return *state.currentSyntax;
 }

src/eepp/ui/doc/syntaxtokenizer.cpp

@@ -109,23 +109,25 @@ std::pair<int, int> findNonEscaped( const std::string& text, const std::string&
 	return std::make_pair( -1, -1 );
 }
 
-SyntaxState SyntaxTokenizer::retrieveSyntaxState( const SyntaxDefinition& syntax,
-												  const Uint32& state ) {
-	SyntaxState syntaxState{ &syntax, nullptr, state, 0 };
-	if ( state > 0 &&
-		 ( state > 255 ||
-		   ( state < syntaxState.currentSyntax->getPatterns().size() &&
-			 !syntaxState.currentSyntax->getPatterns()[state - 1].syntax.empty() ) ) ) {
+SyntaxStateRestored SyntaxTokenizer::retrieveSyntaxState( const SyntaxDefinition& syntax,
+														  const SyntaxState& state ) {
+	SyntaxStateRestored syntaxState{ &syntax, nullptr, state.state, 0 };
+	if ( state.state > 0 &&
+		 ( state.state > 255 ||
+		   ( state.state < syntaxState.currentSyntax->getPatterns().size() &&
+			 syntaxState.currentSyntax->getPatterns()[state.state - 1].hasSyntax() ) ) ) {
 		for ( size_t i = 0; i <= 2; ++i ) {
-			Uint32 target = ( state >> ( 8 * i ) ) & 0xFF;
+			Uint32 target = ( state.state >> ( i << 3 ) ) & 0xFF;
 			if ( target != SYNTAX_TOKENIZER_STATE_NONE ) {
 				if ( target < syntaxState.currentSyntax->getPatterns().size() &&
-					 !syntaxState.currentSyntax->getPatterns()[target - 1].syntax.empty() ) {
+					 syntaxState.currentSyntax->getPatterns()[target - 1].hasSyntax() ) {
 					syntaxState.subsyntaxInfo =
 						&syntaxState.currentSyntax->getPatterns()[target - 1];
 					syntaxState.currentSyntax =
-						&SyntaxDefinitionManager::instance()->getByLanguageName(
-							syntaxState.subsyntaxInfo->syntax );
+						state.hash != 0
+							? &SyntaxDefinitionManager::instance()->getByLanguageId( state.hash )
+							: &SyntaxDefinitionManager::instance()->getByLanguageName(
+								  syntaxState.subsyntaxInfo->syntax );
 					syntaxState.currentPatternIdx = SYNTAX_TOKENIZER_STATE_NONE;
 					syntaxState.currentLevel++;
 				} else {
@@ -140,8 +142,8 @@ SyntaxState SyntaxTokenizer::retrieveSyntaxState( const SyntaxDefinition& syntax
 }
 
 template <typename T>
-static inline std::pair<std::vector<T>, Uint32>
-_tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32& state,
+static inline std::pair<std::vector<T>, SyntaxState>
+_tokenize( const SyntaxDefinition& syntax, const std::string& text, const SyntaxState& state,
 		   const size_t& startIndex, bool skipSubSyntaxSeparator ) {
 	std::vector<T> tokens;
 	LuaPattern::Range matches[12];
@@ -150,26 +152,31 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32
 
 	if ( syntax.getPatterns().empty() ) {
 		pushToken( tokens, SyntaxStyleTypes::Normal, text );
-		return std::make_pair( std::move( tokens ), SYNTAX_TOKENIZER_STATE_NONE );
+		return std::make_pair( std::move( tokens ), SyntaxState{} );
 	}
 
 	size_t i = startIndex;
-	int retState = state;
-	SyntaxState curState = SyntaxTokenizer::retrieveSyntaxState( syntax, state );
+	SyntaxState retState = state;
+	SyntaxStateRestored curState = SyntaxTokenizer::retrieveSyntaxState( syntax, state );
 
 	auto setSubsyntaxPatternIdx = [&curState, &retState]( const Uint32& patternIndex ) {
 		curState.currentPatternIdx = patternIndex;
-		retState &= ~( 0xFF << ( curState.currentLevel * 8 ) );
-		retState |= ( patternIndex << ( curState.currentLevel * 8 ) );
+		retState.state &= ~( 0xFF << ( curState.currentLevel << 3 ) );
+		retState.state |= ( patternIndex << ( curState.currentLevel << 3 ) );
+		if ( curState.currentSyntax )
+			retState.hash = curState.currentSyntax->getLanguageId();
 	};
 
-	auto pushSubsyntax = [&setSubsyntaxPatternIdx, &curState](
-							 const SyntaxPattern& enteringSubsyntax, const Uint32& patternIndex ) {
+	auto pushSubsyntax = [&setSubsyntaxPatternIdx,
+						  &curState]( const SyntaxPattern& enteringSubsyntax,
+									  const Uint32& patternIndex, const std::string& patternStr ) {
 		setSubsyntaxPatternIdx( patternIndex );
 		curState.currentLevel++;
 		curState.subsyntaxInfo = &enteringSubsyntax;
 		curState.currentSyntax = &SyntaxDefinitionManager::instance()->getByLanguageName(
-			curState.subsyntaxInfo->syntax );
+			curState.subsyntaxInfo->dynSyntax
+				? curState.subsyntaxInfo->dynSyntax( enteringSubsyntax, patternStr )
+				: curState.subsyntaxInfo->syntax );
 		setSubsyntaxPatternIdx( SYNTAX_TOKENIZER_STATE_NONE );
 	};
 
@@ -180,7 +187,8 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32
 		curState = SyntaxTokenizer::retrieveSyntaxState( syntax, retState );
 	};
 
-	size_t size = !text.empty() ? text.size() - 1 : 0; // skip last char ( new line char )
+	size_t size = !text.empty() ? ( text[text.size() - 1] == '\n' ? text.size() - 1 : text.size() )
+								: 0; // skip last char ( new line char )
 
 	while ( i < size ) {
 		if ( curState.currentPatternIdx != SYNTAX_TOKENIZER_STATE_NONE ) {
@@ -240,14 +248,15 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32
 		}
 
 		bool matched = false;
+		std::string patternStr;
 
 		for ( size_t patternIndex = 0; patternIndex < curState.currentSyntax->getPatterns().size();
 			  patternIndex++ ) {
 			const SyntaxPattern& pattern = curState.currentSyntax->getPatterns()[patternIndex];
 			if ( i != 0 && pattern.patterns[0][0] == '^' )
 				continue;
-			const std::string& patternStr(
-				pattern.patterns[0][0] == '^' ? pattern.patterns[0] : "^" + pattern.patterns[0] );
+			patternStr =
+				pattern.patterns[0][0] == '^' ? pattern.patterns[0] : "^" + pattern.patterns[0];
 			LuaPattern words( patternStr );
 			if ( words.matches( text, matches, i ) && ( numMatches = words.getNumMatches() ) > 0 ) {
 				if ( numMatches > 1 ) {
@@ -283,7 +292,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32
 
 						std::string patternText( text.substr( start, end - start ) );
 						SyntaxStyleType type = curState.currentSyntax->getSymbol( patternText );
-						if ( !skipSubSyntaxSeparator || pattern.syntax.empty() ) {
+						if ( !skipSubSyntaxSeparator || !pattern.hasSyntax() ) {
 							pushToken( tokens,
 									   type == SyntaxStyleEmpty()
 										   ? ( curMatch < pattern.types.size()
@@ -293,8 +302,8 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32
 									   patternText );
 						}
 
-						if ( !pattern.syntax.empty() ) {
-							pushSubsyntax( pattern, patternIndex + 1 );
+						if ( pattern.hasSyntax() ) {
+							pushSubsyntax( pattern, patternIndex + 1, patternStr );
 						} else if ( pattern.patterns.size() > 1 ) {
 							setSubsyntaxPatternIdx( patternIndex + 1 );
 						}
@@ -328,7 +337,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32
 						}
 						std::string patternText( text.substr( start, end - start ) );
 						SyntaxStyleType type = curState.currentSyntax->getSymbol( patternText );
-						if ( !skipSubSyntaxSeparator || pattern.syntax.empty() ) {
+						if ( !skipSubSyntaxSeparator || !pattern.hasSyntax() ) {
 							pushToken( tokens,
 									   type == SyntaxStyleEmpty()
 										   ? ( curMatch < pattern.types.size()
@@ -337,8 +346,8 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32
 										   : type,
 									   patternText );
 						}
-						if ( !pattern.syntax.empty() ) {
-							pushSubsyntax( pattern, patternIndex + 1 );
+						if ( pattern.hasSyntax() ) {
+							pushSubsyntax( pattern, patternIndex + 1, patternText );
 						} else if ( pattern.patterns.size() > 1 ) {
 							setSubsyntaxPatternIdx( patternIndex + 1 );
 						}
@@ -369,24 +378,24 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Uint32
 	return std::make_pair( std::move( tokens ), retState );
 }
 
-std::pair<std::vector<SyntaxToken>, Uint32>
+std::pair<std::vector<SyntaxToken>, SyntaxState>
 SyntaxTokenizer::tokenize( const SyntaxDefinition& syntax, const std::string& text,
-						   const Uint32& state, const size_t& startIndex,
+						   const SyntaxState& state, const size_t& startIndex,
 						   bool skipSubSyntaxSeparator ) {
 	return _tokenize<SyntaxToken>( syntax, text, state, startIndex, skipSubSyntaxSeparator );
 }
 
-std::pair<std::vector<SyntaxTokenPosition>, Uint32>
+std::pair<std::vector<SyntaxTokenPosition>, SyntaxState>
 SyntaxTokenizer::tokenizePosition( const SyntaxDefinition& syntax, const std::string& text,
-								   const Uint32& state, const size_t& startIndex,
+								   const SyntaxState& state, const size_t& startIndex,
 								   bool skipSubSyntaxSeparator ) {
 	return _tokenize<SyntaxTokenPosition>( syntax, text, state, startIndex,
 										   skipSubSyntaxSeparator );
 }
 
-std::pair<std::vector<SyntaxTokenComplete>, Uint32>
+std::pair<std::vector<SyntaxTokenComplete>, SyntaxState>
 SyntaxTokenizer::tokenizeComplete( const SyntaxDefinition& syntax, const std::string& text,
-								   const Uint32& state, const size_t& startIndex,
+								   const SyntaxState& state, const size_t& startIndex,
 								   bool skipSubSyntaxSeparator ) {
 	return _tokenize<SyntaxTokenComplete>( syntax, text, state, startIndex,
 										   skipSubSyntaxSeparator );
@@ -397,10 +406,10 @@ Text& SyntaxTokenizer::tokenizeText( const SyntaxDefinition& syntax,
 									 const size_t& startIndex, const size_t& endIndex,
 									 bool skipSubSyntaxSeparator, const std::string& trimChars ) {
 
-	auto tokens =
-		SyntaxTokenizer::tokenizeComplete( syntax, text.getString(), SYNTAX_TOKENIZER_STATE_NONE,
-										   startIndex, skipSubSyntaxSeparator )
-			.first;
+	auto tokens = SyntaxTokenizer::tokenizeComplete( syntax, text.getString(),
+													 { SYNTAX_TOKENIZER_STATE_NONE, 0 }, startIndex,
+													 skipSubSyntaxSeparator )
+					  .first;
 
 	if ( skipSubSyntaxSeparator || !trimChars.empty() ) {
 		String txt;

src/eepp/ui/doc/textdocument.cpp

@@ -86,7 +86,7 @@ void TextDocument::reset() {
 	mLastSelection = 0;
 	mLines.clear();
 	mLines.emplace_back( String( "\n" ) );
-	mSyntaxDefinition = SyntaxDefinitionManager::instance()->getPlainStyle();
+	mSyntaxDefinition = SyntaxDefinitionManager::instance()->getPlainDefinition();
 	mUndoStack.clear();
 	cleanChangeId();
 	notifySyntaxDefinitionChange();

src/tools/ecode/uitreeviewglobalsearch.cpp

@@ -176,8 +176,7 @@ UIPushButton* UITreeViewCellGlobalSearch::updateText( const std::string& text )
 			mResultStr = "";
 		}
 
-		auto tokens =
-			SyntaxTokenizer::tokenize( styleDef, text, SYNTAX_TOKENIZER_STATE_NONE, to ).first;
+		auto tokens = SyntaxTokenizer::tokenize( styleDef, text, SyntaxState{}, to ).first;
 
 		size_t start = to;
 		for ( const auto& token : tokens ) {