Improve tokenizer performance by adding a couple of simple tokenizing hacks.

Added UUID class.
Fixed TextDocument folding ranges not updating on untitled docs.
Fixed UILinearLayout bug I introduced the other day.
Improved the Spreadsheet parser to allow direct use of arithmetic operators.

.ecode/project_build.json

@@ -204,7 +204,7 @@
     },
     "run": [
       {
-        "args": "-v",
+        "args": "",
         "command": "${project_root}/bin/ecode-debug",
         "name": "ecode-debug",
         "working_dir": "${project_root}/bin"
@@ -240,6 +240,18 @@
         "command": "${project_root}/bin/eepp-empty-window",
         "name": "eepp-empty_window",
         "working_dir": "${project_root}/bin"
+      },
+      {
+        "args": "-u",
+        "command": "${project_root}/bin/eepp-UIEditor-debug",
+        "name": "eepp-UIEditor-debug",
+        "working_dir": "${project_root}/bin"
+      },
+      {
+        "args": "",
+        "command": "${project_root}/bin/eepp-7guis-cells-debug",
+        "name": "eepp-7guis-cells-debug",
+        "working_dir": "${project_root}/bin"
       }
     ],
     "var": {

include/eepp/system.hpp

@@ -41,6 +41,7 @@
 #include <eepp/system/threadpool.hpp>
 #include <eepp/system/time.hpp>
 #include <eepp/system/translator.hpp>
+#include <eepp/system/uuid.hpp>
 #include <eepp/system/virtualfilesystem.hpp>
 #include <eepp/system/zip.hpp>
 

include/eepp/system/uuid.hpp

@@ -0,0 +1,55 @@
+#pragma once
+#include <eepp/config.hpp>
+
+#include <cstdint>
+#include <optional>
+#include <string>
+
+namespace EE { namespace System {
+
+class EE_API UUID {
+  public:
+	UUID();
+
+	UUID( uint64_t high, uint64_t low );
+
+	UUID( bool autocreate );
+
+	UUID( const UUID& other ) = default;
+
+	UUID( UUID&& other ) = default;
+
+	UUID& operator=( const UUID& other ) = default;
+
+	UUID& operator=( UUID&& other ) = default;
+
+	static std::optional<UUID> fromString( const std::string_view& str );
+
+	std::string toString() const;
+
+	void refresh();
+
+	bool isInitialized() const;
+
+	bool operator==( const UUID& other ) const {
+		return mHigh == other.mHigh && mLow == other.mLow;
+	}
+
+	bool operator!=( const UUID& other ) const { return !( *this == other ); }
+
+	bool operator<( const UUID& other ) const {
+		return mHigh < other.mHigh || ( mHigh == other.mHigh && mLow < other.mLow );
+	}
+
+	bool operator>( const UUID& other ) const { return other < *this; }
+
+	bool operator<=( const UUID& other ) const { return !( *this > other ); }
+
+	bool operator>=( const UUID& other ) const { return !( *this < other ); }
+
+  protected:
+	uint64_t mHigh{ 0 }; // Bits 0-63 of the UUID
+	uint64_t mLow{ 0 };	 // Bits 64-127 of the UUID
+};
+
+}} // namespace EE::System

include/eepp/ui/doc/textdocument.hpp

@@ -12,6 +12,7 @@
 #include <eepp/system/pack.hpp>
 #include <eepp/system/threadpool.hpp>
 #include <eepp/system/time.hpp>
+#include <eepp/system/uuid.hpp>
 #include <eepp/ui/doc/foldrangeservice.hpp>
 #include <eepp/ui/doc/syntaxdefinition.hpp>
 #include <eepp/ui/doc/textdocumentline.hpp>
@@ -157,6 +158,8 @@ class EE_API TextDocument {
 
 	LoadStatus reload();
 
+	const UUID& getUUID() const;
+
 	bool save();
 
 	bool save( const std::string& path );
@@ -668,6 +671,7 @@ class EE_API TextDocument {
 	friend class TextUndoStack;
 	friend class FoldRangeServive;
 
+	UUID mUUID;
 	Uint64 mModificationId{ 0 };
 	TextUndoStack mUndoStack;
 	std::string mFilePath;

src/eepp/system/uuid.cpp

@@ -0,0 +1,127 @@
+#include <eepp/system/uuid.hpp>
+
+#include <random>
+
+namespace EE { namespace System {
+
+namespace {
+
+static inline uint64_t hexToUint64( const std::string& hex ) {
+	uint64_t result = 0;
+	for ( char c : hex ) {
+		result <<= 4;
+		if ( c >= '0' && c <= '9' ) {
+			result |= ( c - '0' );
+		} else if ( c >= 'a' && c <= 'f' ) {
+			result |= ( c - 'a' + 10 );
+		} else {
+			return 0;
+		}
+	}
+	return result;
+}
+
+static inline void byteToHex( uint8_t byte, char* dest ) {
+	static const char* hexDigits = "0123456789abcdef";
+	dest[0] = hexDigits[( byte >> 4 ) & 0xF];
+	dest[1] = hexDigits[byte & 0xF];
+}
+
+} // namespace
+
+UUID::UUID() {
+	refresh();
+}
+
+UUID::UUID( uint64_t high, uint64_t low ) : mHigh( high ), mLow( low ) {}
+
+UUID::UUID( bool autocreate ) {
+	if ( autocreate )
+		refresh();
+}
+
+void UUID::refresh() {
+	// Thread-local random number generator for performance in multi-threaded environments
+	thread_local static std::random_device rd;
+	thread_local static std::mt19937_64 gen( rd() );
+
+	// Generate two 64-bit random numbers
+	mHigh = gen();
+	mLow = gen();
+
+	// Set version (4) in bits 48-51 of the UUID (bits 12-15 of high_)
+	// Clear bits 12-15 and set to 0b0100 (4 in hex)
+	mHigh = ( mHigh & ~( (uint64_t)0xF << 12 ) ) | ( (uint64_t)0x4 << 12 );
+
+	// Set variant (10) in bits 64-65 of the UUID (bits 63-62 of low_)
+	// Clear bits 63-62 and set to 0b10
+	mLow = ( mLow & ~( (uint64_t)0x3 << 62 ) ) | ( (uint64_t)0x2 << 62 );
+}
+
+std::optional<UUID> UUID::fromString( const std::string_view& uuidStr ) {
+	// Check length (36 characters including hyphens)
+	if ( uuidStr.length() != 36 )
+		return {};
+
+	// Verify hyphen positions
+	if ( uuidStr[8] != '-' || uuidStr[13] != '-' || uuidStr[18] != '-' || uuidStr[23] != '-' )
+		return {};
+
+	// Verify version (must be '4' for UUIDv4)
+	if ( uuidStr[14] != '4' )
+		return {};
+
+	// Verify variant (must be '8', '9', 'a', or 'b')
+	char variantChar = uuidStr[19];
+	if ( !( ( variantChar >= '8' && variantChar <= '9' ) ||
+			( variantChar >= 'a' && variantChar <= 'b' ) ) )
+		return {};
+
+	// Remove hyphens and convert to lowercase
+	std::string hexStr;
+	for ( char c : uuidStr ) {
+		if ( c != '-' ) {
+			if ( !std::isxdigit( c ) )
+				return {};
+			hexStr += std::tolower( c );
+		}
+	}
+
+	// Verify length after removing hyphens (32 hex chars)
+	if ( hexStr.length() != 32 )
+		return {};
+
+	// Convert to 128-bit binary (two 64-bit parts)
+	uint64_t high_ = hexToUint64( hexStr.substr( 0, 16 ) );
+	uint64_t low_ = hexToUint64( hexStr.substr( 16, 16 ) );
+
+	return UUID{ high_, low_ };
+}
+
+std::string UUID::toString() const {
+	char buf[36]; // 32 hex digits + 4 hyphens = 36 characters
+	int pos = 0;
+	uint64_t parts[2] = { mHigh, mLow };
+	int byteIndex = 0;
+
+	// Process all 16 bytes, inserting hyphens at the correct positions
+	for ( int part = 0; part < 2; ++part ) {
+		uint64_t val = parts[part];
+		for ( int i = 56; i >= 0; i -= 8 ) {
+			byteToHex( ( val >> i ) & 0xFF, buf + pos );
+			pos += 2;
+			byteIndex++;
+			// Insert hyphens after bytes 4, 6, 8, and 10 (positions 8, 13, 18, 23)
+			if ( byteIndex == 4 || byteIndex == 6 || byteIndex == 8 || byteIndex == 10 ) {
+				buf[pos++] = '-';
+			}
+		}
+	}
+	return std::string( buf, 36 );
+}
+
+bool UUID::isInitialized() const {
+	return mLow != 0 || mHigh != 0;
+}
+
+}} // namespace EE::System

src/eepp/ui/doc/syntaxdefinition.cpp

@@ -37,6 +37,10 @@ SyntaxDefinition::SyntaxDefinition( const std::string& languageName,
 	mHeaders( std::move( headers ) ),
 	mLSPName( lspName.empty() ? String::toLower( mLanguageName ) : lspName ) {
 	mSymbols.reserve( mSymbolNames.size() );
+	if ( !mPatterns.empty() ) {
+		mPatterns.emplace( mPatterns.begin(), SyntaxPattern{ { "%s+" }, "normal" } );
+		mPatterns.emplace_back( SyntaxPattern{ { "%w+%f[%s]" }, "normal" } );
+	}
 	for ( const auto& symbol : mSymbolNames )
 		mSymbols.insert( { symbol.first, toSyntaxStyleType( symbol.second ) } );
 }

src/eepp/ui/doc/textdocument.cpp

@@ -31,6 +31,7 @@ bool TextDocument::isNonWord( String::StringBaseType ch ) const {
 }
 
 TextDocument::TextDocument( bool verbose ) :
+	mUUID( true ),
 	mUndoStack( this ),
 	mVerbose( verbose ),
 	mAutoCloseBracketsPairs(
@@ -3833,4 +3834,8 @@ bool TextSearchParams::isEmpty() {
 	return text.empty();
 }
 
+const UUID& TextDocument::getUUID() const {
+	return mUUID;
+}
+
 }}} // namespace EE::UI::Doc

src/eepp/ui/uicodeeditor.cpp

@@ -185,8 +185,6 @@ UICodeEditor::~UICodeEditor() {
 		getUISceneNode()->getThreadPool()->removeWithTag( tag );
 	}
 
-	getUISceneNode()->removeActionsByTag( mTagFoldRange );
-
 	if ( mCurrentMenu ) {
 		mCurrentMenu->clearEventListener();
 		mCurrentMenu = nullptr;
@@ -205,6 +203,8 @@ UICodeEditor::~UICodeEditor() {
 
 	mDocView.setDocument( nullptr );
 	if ( mDoc.use_count() == 1 ) {
+		getUISceneNode()->removeActionsByTag( mTagFoldRange );
+
 		DocEvent event( this, mDoc.get(), Event::OnDocumentClosed );
 		sendEvent( &event );
 		mDoc->unregisterClient( this );
@@ -2200,6 +2200,7 @@ void UICodeEditor::onDocumentMoved( TextDocument* doc ) {
 }
 
 void UICodeEditor::onDocumentClosed( TextDocument* doc ) {
+	getUISceneNode()->removeActionsByTag( mTagFoldRange );
 	DocEvent event( this, doc, Event::OnDocumentClosed );
 	sendEvent( &event );
 }
@@ -5233,15 +5234,17 @@ void UICodeEditor::findRegionsDelayed() {
 		return;
 	UISceneNode* sceneNode = getUISceneNode();
 	if ( sceneNode ) {
-		sceneNode->debounce( [this]() { mDoc->getFoldRangeService().findRegions(); },
+		TextDocument* doc = mDoc.get();
+		sceneNode->debounce( [doc]() { doc->getFoldRangeService().findRegions(); },
 							 mFoldsIsFirst ? Milliseconds( 100 ) : mFoldsRefreshTime,
 							 mTagFoldRange );
+
 		mFoldsIsFirst = false;
 	}
 }
 
 void UICodeEditor::refreshTag() {
-	mTagFoldRange = String::hash( mDoc->getURI().toString() + ":foldrange" );
+	mTagFoldRange = String::hash( mDoc->getUUID().toString() + ":foldrange" );
 }
 
 bool UICodeEditor::isNotMonospace() const {

src/eepp/ui/uilinearlayout.cpp

@@ -210,7 +210,12 @@ void UILinearLayout::packVertical() {
 			Vector2f pos( mPaddingPx.Left, curY );
 
 			if ( widget->getLayoutWeight() != 0 ) {
-				Float totSize = getPixelsSize().getHeight() - mPaddingPx.Top - mPaddingPx.Bottom;
+				Float totSize =
+					( getLayoutHeightPolicy() == SizePolicy::MatchParent ||
+					  getLayoutHeightPolicy() == SizePolicy::Fixed )
+						? getPixelsSize().getHeight() - mPaddingPx.Top - mPaddingPx.Bottom
+						: getParent()->getPixelsSize().getHeight() - mLayoutMarginPx.Bottom -
+							  mLayoutMarginPx.Top - mPaddingPx.Top - mPaddingPx.Bottom;
 				Float newSize = eemax(
 					eeceil( totSize - freeSize.getHeight() ) * widget->getLayoutWeight(), 0.f );
 
@@ -336,7 +341,12 @@ void UILinearLayout::packHorizontal() {
 			Vector2f pos( curX, mPaddingPx.Top );
 
 			if ( widget->getLayoutWeight() != 0 ) {
-				Float totSize = getPixelsSize().getWidth() - mPaddingPx.Left - mPaddingPx.Right;
+				Float totSize =
+					( getLayoutWidthPolicy() == SizePolicy::MatchParent ||
+					  getLayoutWidthPolicy() == SizePolicy::Fixed )
+						? getPixelsSize().getWidth() - mPaddingPx.Left - mPaddingPx.Right
+						: getParent()->getPixelsSize().getWidth() - mLayoutMarginPx.Right -
+							  mLayoutMarginPx.Left - mPaddingPx.Left - mPaddingPx.Right;
 				Float newSize = eemax(
 					eeceil( totSize - freeSize.getWidth() ) * widget->getLayoutWeight(), 0.f );
 

src/examples/7guis/cells/parser.cpp

@@ -39,6 +39,11 @@ void FormulaParser::initTokenizer() {
 	tokenizer.add( ":", TokenType::COLON );
 	tokenizer.add( "%(", TokenType::OPEN_BRACKET );
 	tokenizer.add( "%)", TokenType::CLOSE_BRACKET );
+	tokenizer.add( "%+", TokenType::PLUS );
+	tokenizer.add( "%-", TokenType::MINUS );
+	tokenizer.add( "%*", TokenType::STAR );
+	tokenizer.add( "%/", TokenType::SLASH );
+	tokenizer.add( "%%", TokenType::PERCENT );
 }
 
 void FormulaParser::nextToken() {
@@ -62,12 +67,14 @@ std::shared_ptr<Formula> FormulaParser::application() {
 		args.emplace_back( expression() );
 		if ( lookahead.token == TokenType::COMMA )
 			nextToken();
-		if ( lookahead.token == TokenType::CLOSE_BRACKET )
+		if ( lookahead.token == TokenType::CLOSE_BRACKET ) {
+			nextToken();
 			return std::make_shared<SheetFunction>( opName, args );
+		}
 	}
 }
 
-std::shared_ptr<Formula> FormulaParser::expression() {
+std::shared_ptr<Formula> FormulaParser::factor() {
 	switch ( lookahead.token ) {
 		case TokenType::CELL: {
 			if ( lookahead.sequence.size() < 2 )
@@ -79,7 +86,7 @@ std::shared_ptr<Formula> FormulaParser::expression() {
 				return nullptr;
 			r = std::max( 0, r - 1 );
 			nextToken();
-			if ( lookahead.token == TokenType::COLON ) { // Range
+			if ( lookahead.token == TokenType::COLON ) {
 				nextToken();
 				if ( lookahead.token == TokenType::CELL ) {
 					String::toUpperInPlace( lookahead.sequence );
@@ -92,12 +99,10 @@ std::shared_ptr<Formula> FormulaParser::expression() {
 					return std::make_shared<RangeReference>(
 						std::make_shared<CellReference>( c, r ),
 						std::make_shared<CellReference>( c2, r2 ) );
-				} else {
-					return nullptr;
 				}
-			} else {
+				return nullptr;
-				return std::make_shared<CellReference>( c, r );
 			}
+			return std::make_shared<CellReference>( c, r );
 		}
 		case TokenType::DECIMAL: {
 			double val = 0;
@@ -105,33 +110,96 @@ std::shared_ptr<Formula> FormulaParser::expression() {
 			nextToken();
 			return std::make_shared<Number>( val );
 		}
-		case TokenType::IDENT:
+		case TokenType::IDENT: {
 			return application();
+		}
+		case TokenType::OPEN_BRACKET: {
+			nextToken();
+			auto expr = expression();
+			if ( lookahead.token != TokenType::CLOSE_BRACKET )
+				return nullptr;
+			nextToken();
+			return expr;
+		}
 		default:
 			return nullptr;
 	}
 }
 
-std::shared_ptr<Formula> FormulaParser::formula() {
+std::shared_ptr<Formula> FormulaParser::term() {
-	switch ( lookahead.token ) {
+	auto left = factor();
-		case TokenType::DECIMAL: {
+	if ( !left )
-			auto n = lookahead.sequence;
+		return nullptr;
+	while ( true ) {
+		if ( lookahead.token == TokenType::STAR ) {
 			nextToken();
-			double val = 0;
+			auto right = factor();
-			String::fromString( val, n );
+			if ( !right )
-			if ( lookahead.token == TokenType::EPSILON )
+				return nullptr;
-				return std::make_shared<Number>( val );
+			left = std::make_shared<SheetFunction>( "MUL", std::vector{ left, right } );
+		} else if ( lookahead.token == TokenType::SLASH ) {
+			nextToken();
+			auto right = factor();
+			if ( !right )
+				return nullptr;
+			left = std::make_shared<SheetFunction>( "DIV", std::vector{ left, right } );
+		} else if ( lookahead.token == TokenType::PERCENT ) {
+			nextToken();
+			auto right = factor();
+			if ( !right )
+				return nullptr;
+			left = std::make_shared<SheetFunction>( "MOD", std::vector{ left, right } );
+		} else {
 			break;
 		}
-		case TokenType::EQUALS:
-			nextToken();
-			return expression();
-		case TokenType::EPSILON:
-			return std::make_shared<Textual>();
-		default:
-			return std::make_shared<Textual>( formulaString );
 	}
-	return nullptr;
+	return left;
+}
+
+std::shared_ptr<Formula> FormulaParser::expression() {
+	auto left = term();
+	if ( !left )
+		return nullptr;
+	while ( true ) {
+		if ( lookahead.token == TokenType::PLUS ) {
+			nextToken();
+			auto right = term();
+			if ( !right )
+				return nullptr;
+			left = std::make_shared<SheetFunction>( "ADD", std::vector{ left, right } );
+		} else if ( lookahead.token == TokenType::MINUS ) {
+			nextToken();
+			auto right = term();
+			if ( !right )
+				return nullptr;
+			left = std::make_shared<SheetFunction>( "SUB", std::vector{ left, right } );
+		} else {
+			break;
+		}
+	}
+	return left;
+}
+
+std::shared_ptr<Formula> FormulaParser::formula() {
+	if ( lookahead.token == TokenType::EQUALS ) {
+		nextToken();
+		auto expr = expression();
+		if ( !expr || lookahead.token != TokenType::EPSILON )
+			return nullptr;
+		return expr;
+	} else if ( lookahead.token == TokenType::DECIMAL ) {
+		auto n = lookahead.sequence;
+		nextToken();
+		if ( lookahead.token != TokenType::EPSILON )
+			return nullptr;
+		double val = 0;
+		String::fromString( val, n );
+		return std::make_shared<Number>( val );
+	} else if ( lookahead.token == TokenType::EPSILON ) {
+		return std::make_shared<Textual>();
+	} else {
+		return std::make_shared<Textual>( formulaString );
+	}
 }
 
 std::shared_ptr<Formula> FormulaParser::parseFormula( std::string _formulaString ) {

src/examples/7guis/cells/parser.hpp

@@ -17,6 +17,11 @@ enum class TokenType {
 	COMMA = 6,
 	COLON = 7,
 	CELL = 8,
+	PLUS = 9,
+    MINUS = 10,
+    STAR = 11,
+    SLASH = 12,
+    PERCENT = 13,
 };
 
 struct Token {
@@ -60,6 +65,10 @@ class FormulaParser {
 	std::shared_ptr<Formula> expression();
 
 	std::shared_ptr<Formula> formula();
+
+	std::shared_ptr<Formula> factor();
+
+    std::shared_ptr<Formula> term();
 };
 
 #endif // PARSER_HPP