Improve Jai and Odin syntax highlighting.

Reduce memory usage of SyntaxTokens. Fix token signature calculation.

include/eepp/core/string.hpp

@@ -69,6 +69,20 @@ class EE_API String {
 		return 0;
 	}
 
+	static constexpr String::HashType hashCombine( String::HashType& hash, const char* str,
+												   Int64 len ) {
+		while ( --len >= 0 )
+			hash = ( ( hash << 5 ) + hash ) + *str++;
+		return hash;
+	}
+
+	static constexpr String::HashType hash( const char* str, Int64 len ) {
+		String::HashType hash = 5381;
+		while ( --len >= 0 )
+			hash = ( ( hash << 5 ) + hash ) + *str++;
+		return hash;
+	}
+
 	/** Escape string sequence */
 	static String escape( const String& str );
 

include/eepp/ui/doc/syntaxtokenizer.hpp

@@ -13,21 +13,31 @@ using namespace EE::Graphics;
 
 namespace EE { namespace UI { namespace Doc {
 
+using SyntaxTokenLen = Uint32;
+
 struct EE_API SyntaxToken {
 	SyntaxStyleType type;
-	size_t len{ 0 };
+	SyntaxTokenLen len{ 0 };
+
+	SyntaxToken( SyntaxStyleType type, SyntaxTokenLen len ) : type( type ), len( len ) {}
 };
 
 struct EE_API SyntaxTokenPosition {
 	SyntaxStyleType type;
-	Int64 pos{ 0 };
-	size_t len{ 0 };
+	SyntaxTokenLen pos{ 0 };
+	SyntaxTokenLen len{ 0 };
+
+	SyntaxTokenPosition( SyntaxStyleType type, SyntaxTokenLen pos, SyntaxTokenLen len ) :
+		type( type ), pos( pos ), len( len ) {}
 };
 
 struct EE_API SyntaxTokenComplete {
-	SyntaxStyleType type;
 	std::string text;
-	size_t len{ 0 };
+	SyntaxStyleType type;
+	SyntaxTokenLen len{ 0 };
+
+	SyntaxTokenComplete( SyntaxStyleType type, const std::string& text, SyntaxTokenLen len ) :
+		text( text ), type( type ), len( len ) {}
 };
 
 #define SYNTAX_TOKENIZER_STATE_NONE ( 0 )

include/eepp/ui/uiconsole.hpp

@@ -196,14 +196,6 @@ class EE_API UIConsole : public UIWidget,
 
 	virtual Uint32 onMouseUp( const Vector2i& position, const Uint32& flags );
 
-	virtual Uint32 onMouseClick( const Vector2i& position, const Uint32& flags );
-
-	virtual Uint32 onMouseDoubleClick( const Vector2i& position, const Uint32& flags );
-
-	virtual Uint32 onMouseOver( const Vector2i& position, const Uint32& flags );
-
-	virtual Uint32 onMouseLeave( const Vector2i& position, const Uint32& flags );
-
 	virtual Uint32 onFocus();
 
 	virtual Uint32 onFocusLoss();

src/eepp/ui/doc/languages/jai.cpp

@@ -19,12 +19,12 @@ void addJai() {
 			  { { "0[dz][%d_]+" }, "number" },
 			  { { "0x[%da-fA-F_]+" }, "number" },
 			  { { "-?%d+[%d%._e]*i?" }, "number" },
-			  { { "([%a_][%w_]+)(%s+::%s+)(%w+%s+)%f[(]" },
+			  { { "([%a_][%w_]+)(%s+::%s+)(%w+%s*)%f[(]" },
 				{ "normal", "function", "operator", "keyword" } },
 			  { { "([%a_][%w_]+)(%s+::%s+)%f[(]" }, { "normal", "function", "operator" } },
-			  { { "([%a_][%w_]+)(%s+::%s+)(struct%s+)%f[{]" },
+			  { { "([%a_][%w_]+)(%s+::%s+)(struct%s*)%f[{]" },
 				{ "normal", "keyword2", "operator", "keyword" } },
-			  { { "([%a_][%w_]+)(%s+::%s+)(enum%s+)(%w+%s)%f[{]" },
+			  { { "([%a_][%w_]+)(%s+::%s+)(enum%s*)(%w+%s)%f[{]" },
 				{ "normal", "keyword2", "operator", "keyword", "keyword2" } },
 			  { { "[<>~=+-*/]=" }, "operator" },
 			  { { "[%+%-=/%*%^%%<>!~|&:]" }, "operator" },

src/eepp/ui/doc/languages/odin.cpp

@@ -20,6 +20,13 @@ void addOdin() {
 			  { { "0[dz][%d_]+" }, "number" },
 			  { { "0x[%da-fA-F_]+" }, "number" },
 			  { { "-?%d+[%d%._e]*i?" }, "number" },
+			  { { "([%a_][%w_]+)(%s+::%s+)(proc%s*)%f[(]" },
+				{ "normal", "function", "operator", "keyword" } },
+			  { { "([%a_][%w_]+)(%s+::%s+)%f[(]" }, { "normal", "function", "operator" } },
+			  { { "([%a_][%w_]+)(%s+::%s+)(struct%s*)%f[{]" },
+				{ "normal", "keyword2", "operator", "keyword" } },
+			  { { "([%a_][%w_]+)(%s+::%s+)(enum%s*)(%w+%s*)%f[{]" },
+				{ "normal", "keyword2", "operator", "keyword", "keyword2" } },
 			  { { "[<>~=+-*/]=" }, "operator" },
 			  { { "[%+%-=/%*%^%%<>!~|&:]" }, "operator" },
 			  { { "%$[%a_][%w_]*" }, "operator" },
@@ -136,7 +143,8 @@ void addOdin() {
 			  { "in", "keyword" },
 			  { "break", "keyword" },
 			  { "else", "keyword" },
-
+			  { "or_break", "keyword" },
+			  { "or_continue", "keyword" },
 		  },
 		  "//",
 		  {}

src/eepp/ui/doc/syntaxhighlighter.cpp

@@ -5,17 +5,12 @@
 
 namespace EE { namespace UI { namespace Doc {
 
-static constexpr void _hash( Uint64& signature, const String::HashType& val ) {
-	Int64 len = sizeof( decltype( val ) );
-	while ( --len >= 0 )
-		signature = ( ( signature << 5 ) + signature ) + ( ( val >> ( len * 8 ) ) & 0xFF );
-}
-
 Uint64 TokenizedLine::calcSignature( const std::vector<SyntaxTokenPosition>& tokens ) {
-	Uint64 signature = 5381;
-	for ( const auto& token : tokens )
-		_hash( signature, SyntaxStyleTypeHash( token.type ) );
-	return signature;
+	if ( !tokens.empty() ) {
+		return String::hash( reinterpret_cast<const char*>( tokens.data() ),
+							 sizeof( SyntaxTokenPosition ) * tokens.size() );
+	}
+	return 0;
 }
 
 void TokenizedLine::updateSignature() {
@@ -52,9 +47,9 @@ TokenizedLine SyntaxHighlighter::tokenizeLine( const size_t& line, const Uint64&
 	tokenizedLine.hash = ln.getHash();
 	if ( mMaxTokenizationLength != 0 && (Int64)ln.size() > mMaxTokenizationLength ) {
 		Int64 textSize = ln.size();
-		Int64 pos = 0;
+		SyntaxTokenLen pos = 0;
 		while ( textSize > 0 ) {
-			size_t chunkSize =
+			SyntaxTokenLen chunkSize =
 				textSize > mMaxTokenizationLength ? mMaxTokenizationLength : textSize;
 			SyntaxTokenPosition token{ SyntaxStyleTypes::Normal, pos, chunkSize };
 			token.len = ln.size();
@@ -123,7 +118,8 @@ void SyntaxHighlighter::setMaxTokenizationLength( const Int64& maxTokenizationLe
 
 const std::vector<SyntaxTokenPosition>& SyntaxHighlighter::getLine( const size_t& index ) {
 	if ( mDoc->getSyntaxDefinition().getPatterns().empty() ) {
-		static std::vector<SyntaxTokenPosition> noHighlightVector = { { SyntaxStyleTypes::Normal, 0 } };
+		static std::vector<SyntaxTokenPosition> noHighlightVector = {
+			{ SyntaxStyleTypes::Normal, 0, 0 } };
 		noHighlightVector[0].len = mDoc->line( index ).size();
 		return noHighlightVector;
 	}
@@ -221,17 +217,17 @@ SyntaxStyleType SyntaxHighlighter::getTokenTypeAt( const TextPosition& pos ) {
 
 SyntaxTokenPosition SyntaxHighlighter::getTokenPositionAt( const TextPosition& pos ) {
 	if ( !pos.isValid() || pos.line() < 0 || pos.line() >= (Int64)mDoc->linesCount() )
-		return {};
+		return { SyntaxStyleTypes::Normal, 0, 0 };
 	auto tokens = getLine( pos.line() );
 	if ( tokens.empty() )
-		return {};
+		return { SyntaxStyleTypes::Normal, 0, 0 };
 	Int64 col = 0;
 	for ( const auto& token : tokens ) {
 		col += token.len;
 		if ( col > pos.column() )
-			return { token.type, static_cast<Int64>( col - token.len ), token.len };
+			return { token.type, static_cast<SyntaxStyleType>( col - token.len ), token.len };
 	}
-	return {};
+	return { SyntaxStyleTypes::Normal, 0, 0 };
 }
 
 void SyntaxHighlighter::setLine( const size_t& line, const TokenizedLine& tokenization ) {
@@ -263,18 +259,20 @@ void SyntaxHighlighter::mergeLine( const size_t& line, const TokenizedLine& toke
 
 				if ( token.pos > ltoken.pos ) {
 					++iDiff;
-					tline.tokens.insert( tline.tokens.begin() + i,
-										 { ltoken.type, ltoken.pos,
-										   static_cast<size_t>( token.pos - ltoken.pos ) } );
+					tline.tokens.insert(
+						tline.tokens.begin() + i,
+						{ ltoken.type, ltoken.pos,
+						  static_cast<SyntaxTokenLen>( token.pos - ltoken.pos ) } );
 				}
 
 				tline.tokens.insert( tline.tokens.begin() + iDiff, token );
 
 				if ( token.pos + token.len < ltoken.pos + ltoken.len ) {
-					tline.tokens.insert( tline.tokens.begin() + iDiff + 1,
-										 { ltoken.type, static_cast<Int64>( token.pos + token.len ),
-										   static_cast<size_t>( ( ltoken.pos + ltoken.len ) -
-																( token.pos + token.len ) ) } );
+					tline.tokens.insert(
+						tline.tokens.begin() + iDiff + 1,
+						{ ltoken.type, static_cast<SyntaxTokenLen>( token.pos + token.len ),
+						  static_cast<SyntaxTokenLen>( ( ltoken.pos + ltoken.len ) -
+													   ( token.pos + token.len ) ) } );
 				}
 
 				lastTokenPos = i;

src/eepp/ui/doc/syntaxtokenizer.cpp

@@ -52,13 +52,13 @@ static void pushToken( std::vector<T>& tokens, const SyntaxStyleType& type,
 					chunkSize = eemin( textSize, chunkSize + multiByteCodePointPos );
 				}
 				std::string substr = text.substr( pos, chunkSize );
-				size_t len = String::utf8Length( substr );
+				SyntaxStyleType len = String::utf8Length( substr );
 				if constexpr ( std::is_same_v<T, SyntaxTokenComplete> ) {
 					tokens.push_back( { type, std::move( substr ), len } );
 				} else if constexpr ( std::is_same_v<T, SyntaxTokenPosition> ) {
-					Int64 tpos = tokens.empty() ? 0
-												: tokens[tokens.size() - 1].pos +
-													  tokens[tokens.size() - 1].len;
+					SyntaxStyleType tpos = tokens.empty() ? 0
+														  : tokens[tokens.size() - 1].pos +
+																tokens[tokens.size() - 1].len;
 					tokens.push_back( { type, tpos, len } );
 				} else {
 					tokens.push_back( { type, len } );
@@ -68,14 +68,17 @@ static void pushToken( std::vector<T>& tokens, const SyntaxStyleType& type,
 			}
 		} else {
 			if constexpr ( std::is_same_v<T, SyntaxTokenComplete> ) {
-				tokens.push_back( { type, text, String::utf8Length( text ) } );
+				tokens.push_back(
+					{ type, text, static_cast<SyntaxTokenLen>( String::utf8Length( text ) ) } );
 			} else if constexpr ( std::is_same_v<T, SyntaxTokenPosition> ) {
-				Int64 tpos = tokens.empty()
-								 ? 0
-								 : tokens[tokens.size() - 1].pos + tokens[tokens.size() - 1].len;
-				tokens.push_back( { type, tpos, String::utf8Length( text ) } );
+				SyntaxStyleType tpos =
+					tokens.empty() ? 0
+								   : tokens[tokens.size() - 1].pos + tokens[tokens.size() - 1].len;
+				tokens.push_back(
+					{ type, tpos, static_cast<SyntaxTokenLen>( String::utf8Length( text ) ) } );
 			} else {
-				tokens.push_back( { type, String::utf8Length( text ) } );
+				tokens.push_back(
+					{ type, static_cast<SyntaxTokenLen>( String::utf8Length( text ) ) } );
 			}
 		}
 	}

src/eepp/ui/uiconsole.cpp

@@ -95,6 +95,16 @@ void UIConsole::setTheme( UITheme* Theme ) {
 }
 
 void UIConsole::scheduledUpdate( const Time& ) {
+	if ( mMouseDown ) {
+		if ( !( getUISceneNode()->getWindow()->getInput()->getPressTrigger() & EE_BUTTON_LMASK ) ) {
+			mMouseDown = false;
+			getUISceneNode()->getWindow()->getInput()->captureMouse( false );
+		} else {
+			onMouseDown( getUISceneNode()->getEventDispatcher()->getMousePos(),
+						 getUISceneNode()->getEventDispatcher()->getPressTrigger() );
+		}
+	}
+
 	if ( hasFocus() && getUISceneNode()->getWindow()->hasFocus() ) {
 		if ( mBlinkTime != Time::Zero && mBlinkTimer.getElapsedTime() > mBlinkTime ) {
 			mCursorVisible = !mCursorVisible;
@@ -910,7 +920,7 @@ Uint32 UIConsole::onMouseDown( const Vector2i& position, const Uint32& flags ) {
 	UIWidget::onMouseDown( position, flags );
 
 	if ( NULL != getEventDispatcher() && isTextSelectionEnabled() && ( flags & EE_BUTTON_LMASK ) &&
-		 getEventDispatcher()->getMouseDownNode() == this ) {
+		 getEventDispatcher()->getMouseDownNode() == this && !mMouseDown ) {
 		getUISceneNode()->getWindow()->getInput()->captureMouse( true );
 		mMouseDown = true;
 	}
@@ -942,22 +952,6 @@ Uint32 UIConsole::onMouseUp( const Vector2i& position, const Uint32& flags ) {
 	return UIWidget::onMouseUp( position, flags );
 }
 
-Uint32 UIConsole::onMouseClick( const Vector2i& position, const Uint32& flags ) {
-	return UIWidget::onMouseClick( position, flags );
-}
-
-Uint32 UIConsole::onMouseDoubleClick( const Vector2i& Pos, const Uint32& Flags ) {
-	return UIWidget::onMouseDoubleClick( Pos, Flags );
-}
-
-Uint32 UIConsole::onMouseOver( const Vector2i& position, const Uint32& flags ) {
-	return UIWidget::onMouseOver( position, flags );
-}
-
-Uint32 UIConsole::onMouseLeave( const Vector2i& Pos, const Uint32& Flags ) {
-	return UIWidget::onMouseLeave( Pos, Flags );
-}
-
 void UIConsole::onDocumentTextChanged( const DocumentContentChange& ) {
 	resetCursor();
 

src/eepp/ui/uitextinput.cpp

@@ -296,7 +296,7 @@ Uint32 UITextInput::onMouseDown( const Vector2i& position, const Uint32& flags )
 	UITextView::onMouseDown( position, flags );
 
 	if ( NULL != getEventDispatcher() && isTextSelectionEnabled() && ( flags & EE_BUTTON_LMASK ) &&
-		 getEventDispatcher()->getMouseDownNode() == this ) {
+		 getEventDispatcher()->getMouseDownNode() == this && !mMouseDown ) {
 		getUISceneNode()->getWindow()->getInput()->captureMouse( true );
 		mMouseDown = true;
 	}

src/tools/ecode/plugins/lsp/lspclientserver.hpp

@@ -47,7 +47,7 @@ class LSPClientServer {
 	using LocationHandler = ReplyHandler<std::vector<LSPLocation>>;
 	using TextEditArrayHandler = ReplyHandler<std::vector<LSPTextEdit>>;
 	using WorkspaceEditHandler = ReplyHandler<LSPWorkspaceEdit>;
-	using SemanticTokensDeltaHandler = ReplyHandler<LSPSemanticTokensDelta>;
+	using SemanticTokensDeltaHandler = WReplyHandler<LSPSemanticTokensDelta>;
 
 	class LSPRequestHandle : public PluginRequestHandle {
 	  public:

src/tools/ecode/plugins/lsp/lspdocumentclient.cpp

@@ -148,10 +148,10 @@ void LSPDocumentClient::requestSemanticHighlighting( bool reqFull ) {
 	Uint64 docModId = mDoc->getModificationId();
 	mServer->documentSemanticTokensFull(
 		mDoc->getURI(), delta, reqId, range,
-		[docClient, uri, server, docModId]( const auto&, const LSPSemanticTokensDelta& deltas ) {
+		[docClient, uri, server, docModId]( const auto&, LSPSemanticTokensDelta&& deltas ) {
 			if ( server->hasDocument( uri ) ) {
 				docClient->mWaitingSemanticTokensResponse = false;
-				docClient->processTokens( deltas, docModId );
+				docClient->processTokens( std::move( deltas ), docModId );
 			}
 		} );
 }
@@ -251,7 +251,7 @@ static SyntaxStyleType semanticTokenTypeToSyntaxType( const std::string& type,
 	return SyntaxStyleTypes::Normal;
 }
 
-void LSPDocumentClient::processTokens( const LSPSemanticTokensDelta& tokens,
+void LSPDocumentClient::processTokens( LSPSemanticTokensDelta&& tokens,
 									   const Uint64& docModificationId ) {
 	if ( mDoc == nullptr || mServer == nullptr )
 		return;
@@ -276,7 +276,7 @@ void LSPDocumentClient::processTokens( const LSPSemanticTokensDelta& tokens,
 	}
 
 	if ( !tokens.data.empty() ) {
-		mSemanticTokens = tokens;
+		mSemanticTokens = std::move( tokens );
 	}
 
 	highlight();

src/tools/ecode/plugins/lsp/lspdocumentclient.hpp

@@ -82,7 +82,7 @@ class LSPDocumentClient : public TextDocument::Client {
 
 	UISceneNode* getUISceneNode();
 
-	void processTokens( const LSPSemanticTokensDelta& tokens, const Uint64& docModificationId );
+	void processTokens( LSPSemanticTokensDelta&& tokens, const Uint64& docModificationId );
 
 	void highlight();
 };