Added syntax highlighting for WebAssembly Text Format (SpartanJ/ecode#842) and WebAssembly Interface Types (SpartanJ/ecode#844).

Improvements in SyntaxTokenizer TextMate compatability.
Fixed debugging types definition scripts.
This commit is contained in:
Martín Lucas Golini
2026-03-16 01:11:23 -03:00
parent afb055405e
commit 9409a82092
13 changed files with 1855 additions and 145 deletions

View File

@@ -132,11 +132,6 @@
"display_name": "Gemini 3 Flash Preview",
"max_tokens": 1000000
},
{
"name": "gemini-3-pro-preview",
"display_name": "Gemini 3 Pro Preview",
"max_tokens": 1000000
},
{
"name": "gemini-3.1-pro-preview",
"display_name": "Gemini 3.1 Pro Preview",

View File

@@ -43,6 +43,7 @@ struct EE_API SyntaxPattern {
IsRangedMatch = 1 << 5,
IsSourceInclude = 1 << 6,
IsAutomaticallyAdded = 1 << 7,
IsApplyEndPatternLast = 1 << 8,
};
static SyntaxDefMap<SyntaxStyleType, std::string> SyntaxStyleTypeCache;
@@ -81,7 +82,8 @@ struct EE_API SyntaxPattern {
SyntaxPattern( std::vector<std::string>&& _patterns, std::vector<std::string>&& _types,
std::vector<std::string>&& _endTypes, const std::string& _syntax,
SyntaxPatternMatchType matchType, std::vector<SyntaxPattern>&& _subPatterns );
SyntaxPatternMatchType matchType, std::vector<SyntaxPattern>&& _subPatterns,
Uint16 flags = 0 );
SyntaxPattern( std::vector<std::string>&& _patterns, const std::string& _type,
DynamicSyntax&& _syntax,
@@ -117,6 +119,8 @@ struct EE_API SyntaxPattern {
return isRangedMatch() && !hasContentScope() && !hasSyntax();
}
inline bool isApplyEndPatternLast() const { return flags & Flags::IsApplyEndPatternLast; }
std::string_view getRepositoryName() const {
eeASSERT( isRepositoryInclude() || isSourceInclude() );
return isSourceInclude() ? std::string_view{ patterns[1] }

View File

@@ -1,5 +1,6 @@
import gdb
class EESmallVectorPrinter:
"""Pretty Printer for EE::SmallVector (ankerl::svector)"""
@@ -30,32 +31,36 @@ class EESmallVectorPrinter:
return f"EE::SmallVector<{self.type_t}> [Indirect] (size={size}, capacity={cap})"
def children(self):
# We need a char pointer type to do proper byte-level pointer math
char_ptr_type = gdb.lookup_type("char").pointer()
align_t = self.type_t.alignof
if self.is_direct():
size = int(self.m_data[0]) >> 1
# Data starts at the alignment of T
align_t = self.type_t.alignof
data_ptr = (
self.m_data.address.cast(self.type_t.pointer()) + 1
) # simplistic alignment check
# Real logic from svector.h: m_data.data() + std::alignment_of_v<T>
# We'll use the address offset directly
base_addr = self.m_data.address
base_addr = self.m_data.address.cast(char_ptr_type)
data_ptr = (base_addr + align_t).cast(self.type_t.pointer())
else:
void_ptr = self.m_data.address.cast(
gdb.lookup_type("void").pointer().pointer()
).dereference()
storage_ptr = void_ptr.cast(
gdb.lookup_type(f"ankerl::v1_0_3::detail::storage<{self.type_t}>").pointer()
)
size = int(storage_ptr["m_size"])
# In indirect mode, the storage object has a data() method,
# but we'll calculate the offset manually for GDB:
# offset_to_data = round_up(sizeof(header), alignment_of_t)
data_ptr = (
storage_ptr.cast(gdb.lookup_type("char").pointer()) + 16
) # Approx header size
data_ptr = data_ptr.cast(self.type_t.pointer())
# C++ logic: offset_to_data = detail::round_up(sizeof(header), alignment_of_t)
# header contains size_t m_size and size_t m_capacity, so sizeof(header) is usually 16.
# Get the exact size of size_t for the current architecture
size_t_type = gdb.lookup_type("size_t")
header_size = size_t_type.sizeof * 2 # header has two size_t fields
# Translate svector's detail::round_up(sizeof(header), alignment_of_t)
offset_to_data = ((header_size + align_t - 1) // align_t) * align_t
data_ptr = (storage_ptr.cast(char_ptr_type) + offset_to_data).cast(
self.type_t.pointer()
)
for i in range(size):
yield f"[{i}]", (data_ptr + i).dereference()

View File

@@ -6,50 +6,73 @@ class EESmallVectorSyntheticProvider:
self.valobj = valobj
self.update()
def _get_m_data(self):
# 1. Try direct access
m_data = self.valobj.GetChildMemberWithName("m_data")
if m_data and m_data.IsValid():
return m_data
# 2. Try looking inside the base class (ankerl::svector)
for i in range(self.valobj.GetNumChildren()):
child = self.valobj.GetChildAtIndex(i)
if child.GetName() and "svector" in child.GetName():
m_data = child.GetChildMemberWithName("m_data")
if m_data and m_data.IsValid():
return m_data
return None
def update(self):
self.size = 0
self.capacity = 0
self.is_direct = True
self.data_addr = lldb.LLDB_INVALID_ADDRESS
# 1. Get the type of T (e.g., Client*)
self.type_t = self.valobj.GetType().GetTemplateArgumentType(0)
self.m_data = self._get_m_data()
# 2. Find the m_data array
self.m_data = self.valobj.GetChildMemberWithName("m_data")
if not self.m_data.IsValid():
if not self.m_data or not self.m_data.IsValid():
return
process = self.valobj.GetProcess()
addr = self.m_data.GetLoadAddress()
# Use SBData instead of direct memory reading to support variables in registers
sb_data = self.m_data.GetData()
error = lldb.SBError()
if addr == lldb.LLDB_INVALID_ADDRESS or not process.IsValid():
if sb_data.GetByteSize() == 0:
return
first_byte = sb_data.GetUnsignedInt8(error, 0)
if error.Fail():
return
# 3. Read the first byte (the discriminator & size)
first_byte = process.ReadUnsignedIntegerFromMemory(addr, 1, error)
self.is_direct = (first_byte & 1) != 0
ptr_size = process.GetAddressByteSize()
target = self.valobj.GetTarget()
ptr_size = target.GetAddressByteSize()
# Safely get alignment
align_t = 0
if hasattr(self.type_t, "GetByteAlign"):
align_t = self.type_t.GetByteAlign()
if align_t == 0:
align_t = self.type_t.GetByteSize()
if align_t == 0:
align_t = ptr_size
if self.is_direct:
self.size = first_byte >> 1
self.capacity = 0 # Implied by N, but not explicitly stored
self.capacity = 0
# Data starts at offset equal to the alignment of T
align_t = self.type_t.GetByteAlign()
if align_t == 0:
align_t = self.type_t.GetByteSize() # Fallback
if align_t == 0:
align_t = ptr_size
self.data_addr = addr + align_t
addr = self.m_data.GetLoadAddress()
if addr != lldb.LLDB_INVALID_ADDRESS:
self.data_addr = addr + align_t
else:
# 4. Indirect Mode: read the pointer to the heap storage
void_ptr = process.ReadPointerFromMemory(addr, error)
# Indirect Mode: read the pointer to the heap storage
void_ptr = sb_data.GetAddress(error, 0)
process = self.valobj.GetProcess()
if error.Fail() or void_ptr == 0 or not process.IsValid():
return
# The storage header contains: size_t m_size, size_t m_capacity
self.size = process.ReadUnsignedIntegerFromMemory(void_ptr, ptr_size, error)
self.capacity = process.ReadUnsignedIntegerFromMemory(
void_ptr + ptr_size, ptr_size, error
@@ -57,12 +80,6 @@ class EESmallVectorSyntheticProvider:
# Calculate offset_to_data: round_up(sizeof(header), alignment_of_t)
header_size = 2 * ptr_size
align_t = self.type_t.GetByteAlign()
if align_t == 0:
align_t = self.type_t.GetByteSize()
if align_t == 0:
align_t = ptr_size
offset = ((header_size + (align_t - 1)) // align_t) * align_t
self.data_addr = void_ptr + offset
@@ -86,20 +103,53 @@ class EESmallVectorSyntheticProvider:
def EESmallVectorSummaryProvider(valobj, internal_dict):
provider = EESmallVectorSyntheticProvider(valobj, internal_dict)
if provider.is_direct:
return f"[Direct] size={provider.size}"
"""
Standalone summary provider. It's an LLDB anti-pattern to initialize the
SyntheticProvider class inside here, so we duplicate the tiny bit of logic
needed to read the state efficiently.
"""
# Find m_data, handling base classes
m_data = valobj.GetChildMemberWithName("m_data")
if not m_data or not m_data.IsValid():
for i in range(valobj.GetNumChildren()):
child = valobj.GetChildAtIndex(i)
if child.GetName() and "svector" in child.GetName():
m_data = child.GetChildMemberWithName("m_data")
break
if not m_data or not m_data.IsValid():
return f"size={valobj.GetNumChildren()}"
sb_data = m_data.GetData()
error = lldb.SBError()
first_byte = sb_data.GetUnsignedInt8(error, 0)
if error.Fail():
return f"size={valobj.GetNumChildren()}"
is_direct = (first_byte & 1) != 0
if is_direct:
size = first_byte >> 1
return f"[Direct] size={size}"
else:
return f"[Indirect] size={provider.size}, capacity={provider.capacity}"
target = valobj.GetTarget()
ptr_size = target.GetAddressByteSize()
void_ptr = sb_data.GetAddress(error, 0)
process = valobj.GetProcess()
if process.IsValid() and void_ptr != 0:
size = process.ReadUnsignedIntegerFromMemory(void_ptr, ptr_size, error)
capacity = process.ReadUnsignedIntegerFromMemory(void_ptr + ptr_size, ptr_size, error)
return f"[Indirect] size={size}, capacity={capacity}"
return "[Indirect]"
def __lldb_init_module(debugger, internal_dict):
# Register the Summary (the text next to the variable)
debugger.HandleCommand(
'type summary add -x "^EE::SmallVector<.+>$" -F eepp_lldb.EESmallVectorSummaryProvider'
)
# Register the Synthetic Children (the expandable array elements)
debugger.HandleCommand(
'type synthetic add -x "^EE::SmallVector<.+>$" -l eepp_lldb.EESmallVectorSyntheticProvider'
)

View File

@@ -422,7 +422,7 @@ SyntaxPattern::SyntaxPattern( std::vector<std::string>&& _patterns,
std::vector<std::string>&& _types,
std::vector<std::string>&& _endTypes, const std::string& _syntax,
SyntaxPatternMatchType matchType,
std::vector<SyntaxPattern>&& _subPatterns ) :
std::vector<SyntaxPattern>&& _subPatterns, Uint16 flags ) :
patterns( std::move( _patterns ) ),
types( toSyntaxStyleTypeV( _types ) ),
endTypes( toSyntaxStyleTypeV( _endTypes ) ),
@@ -430,6 +430,7 @@ SyntaxPattern::SyntaxPattern( std::vector<std::string>&& _patterns,
endTypesNames( std::move( _endTypes ) ),
syntax( _syntax ),
matchType( matchType ),
flags( flags ),
contentPatterns( std::move( _subPatterns ) ) {
eeASSERT( patterns.size() < std::numeric_limits<Uint8>::max() - 1 );
updateCache<SyntaxStyleType>( *this );
@@ -580,7 +581,8 @@ std::string SyntaxDefinition::getRepositoryName( String::HashType hash ) const {
return it != mRepositoryNames.end() ? it->second : "";
}
SyntaxDefinition& SyntaxDefinition::setBlockComment( SyntaxDefinition::BlockComment&& commentBlock ) {
SyntaxDefinition&
SyntaxDefinition::setBlockComment( SyntaxDefinition::BlockComment&& commentBlock ) {
mBlockComment = std::move( commentBlock );
return *this;
}

View File

@@ -255,6 +255,9 @@ static std::optional<nlohmann::json> serializePattern( const SyntaxPattern& ptrn
if ( !ptrn.contentTypeName.empty() )
pattern["contentName"] = ptrn.contentTypeName;
if ( ptrn.isApplyEndPatternLast() )
pattern["applyEndPatternLast"] = 1;
return pattern;
}
@@ -399,11 +402,12 @@ static std::string funcName( std::string name ) {
static void patternToCPP( std::string& buf, const SyntaxPattern& pattern,
const SyntaxDefinition& def ) {
bool allowReduce = ( pattern.patterns.size() == 1 && pattern.typesNames.size() <= 1 &&
pattern.endTypesNames.empty() ) ||
( pattern.patterns.size() <= 3 && pattern.typesNames.size() <= 1 &&
pattern.endTypesNames.empty() &&
pattern.matchType == SyntaxPatternMatchType::LuaPattern );
bool allowReduce = ( ( pattern.patterns.size() == 1 && pattern.typesNames.size() <= 1 &&
pattern.endTypesNames.empty() ) ||
( pattern.patterns.size() <= 3 && pattern.typesNames.size() <= 1 &&
pattern.endTypesNames.empty() &&
pattern.matchType == SyntaxPatternMatchType::LuaPattern ) ) &&
!pattern.isApplyEndPatternLast();
bool setType = allowReduce && pattern.matchType != SyntaxPatternMatchType::LuaPattern;
bool addPatternType = pattern.matchType != SyntaxPatternMatchType::LuaPattern ||
( pattern.patterns.size() == 2 && pattern.patterns[0] == "include" );
@@ -434,6 +438,13 @@ static void patternToCPP( std::string& buf, const SyntaxPattern& pattern,
patternToCPP( buf, ptrn, def );
buf += "\n}";
}
if ( pattern.isApplyEndPatternLast() ) {
if ( !pattern.hasContentScope() )
buf = ", {}";
buf += ", SyntaxPattern::IsApplyEndPatternLast";
}
buf += " },\n";
}
@@ -707,6 +718,7 @@ static SyntaxPattern parsePattern( const nlohmann::json& pattern ) {
std::string contentTypeName;
SyntaxStyleType contentType{ SyntaxStyleEmpty() };
std::string syntax;
bool applyEndPatternLast{ false };
const auto fillTypes = []( const nlohmann::json& captures, std::vector<std::string>& type,
const nlohmann::json& parent ) {
@@ -758,7 +770,7 @@ static SyntaxPattern parsePattern( const nlohmann::json& pattern ) {
if ( pattern.contains( "endCaptures" ) )
fillTypes( pattern["endCaptures"], endType, pattern );
if ( type.empty() && pattern.contains( "captures" ) )
if ( pattern.contains( "captures" ) )
fillTypes( pattern["captures"], type, pattern );
if ( pattern.contains( "match" ) && pattern["match"].is_string() ) {
@@ -771,9 +783,12 @@ static SyntaxPattern parsePattern( const nlohmann::json& pattern ) {
if ( pattern.contains( "begin" ) )
ptrns.emplace_back( pattern.value( "begin", "" ) );
if ( pattern.contains( "end" ) )
if ( pattern.contains( "end" ) ) {
ptrns.emplace_back( pattern.value( "end", "" ) );
if ( pattern.contains( "applyEndPatternLast" ) )
applyEndPatternLast = true;
}
// Sub-languages / Sub patterns?
if ( pattern.contains( "patterns" ) && !pattern["patterns"].empty() &&
pattern["patterns"].is_array() ) {
@@ -874,6 +889,9 @@ static SyntaxPattern parsePattern( const nlohmann::json& pattern ) {
SyntaxPattern ptrn( std::move( ptrns ), std::move( type ), std::move( endType ), syntax, ctype,
std::move( subPatterns ) );
if ( applyEndPatternLast )
ptrn.flags |= SyntaxPattern::Flags::IsApplyEndPatternLast;
if ( contentType != SyntaxStyleEmpty() ) {
ptrn.contentTypeName = std::move( contentTypeName );
ptrn.contentType = contentType;
@@ -917,7 +935,17 @@ static void parseRepositoryItem( SyntaxDefinition& def, const std::string& name,
for ( const auto& pattern : patterns ) {
if ( pattern.size() == 1 && pattern.contains( "comment" ) )
continue;
ptrns.emplace_back( parsePattern( pattern ) );
if ( !( pattern.contains( "match" ) || pattern.contains( "begin" ) ) &&
pattern.contains( "patterns" ) && pattern["patterns"].is_array() ) {
// Maybe do this recursive later, not a very common pattern
const auto& subPatterns = pattern["patterns"];
for ( const auto& subPattern : subPatterns ) {
if ( subPattern.size() == 1 && subPattern.contains( "comment" ) )
continue;
ptrns.emplace_back( parsePattern( subPattern ) );
}
} else
ptrns.emplace_back( parsePattern( pattern ) );
}
} else if ( item.is_array() ) {
for ( const auto& pattern : item )

View File

@@ -132,7 +132,7 @@ void SyntaxHighlighter::setMaxTokenizationLength( const Int64& maxTokenizationLe
void SyntaxHighlighter::tokenizeAsync( std::shared_ptr<ThreadPool> pool,
const std::function<void()>& onDone ) {
if ( mTokenizeAsync )
if ( mTokenizeAsync && false )
return;
mTokenizeAsync = true;
pool->run( [this, onDone] {

View File

@@ -7,7 +7,6 @@
#include <eepp/ui/doc/syntaxtokenizer.hpp>
#include <array>
#include <memory_resource>
#include <variant>
using namespace EE::System;
@@ -123,6 +122,10 @@ struct NonEscapedMatch {
std::pair<int, int> range{ -1, -1 };
PatternMatcher::Range matches[6];
int numMatches{ 0 };
inline bool isZeroWidthMatch() const {
return range.first != -1 && range.first == range.second;
}
};
static NonEscapedMatch findNonEscaped( const std::string& text, const std::string& pattern,
@@ -403,12 +406,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
std::string_view patternText;
size_t numMatches = 0;
std::optional<NonEscapedMatch> shouldCloseSubSyntax;
static constexpr auto TRIED_PATTERNS_BUFFER = MAX_TRIED_PATTERNS * sizeof( SyntaxStateType );
std::array<std::byte, TRIED_PATTERNS_BUFFER> triedPatternsBuffer;
std::pmr::monotonic_buffer_resource triedPatternsRes(
triedPatternsBuffer.data(), triedPatternsBuffer.size(), std::pmr::null_memory_resource() );
std::pmr::vector<SyntaxStateType> triedPatterns( &triedPatternsRes );
SmallVector<SyntaxStateType, MAX_TRIED_PATTERNS> triedPatterns;
const auto matchPattern = [&]( const SyntaxPattern& pattern, size_t& startIdx,
SyntaxStateType patternIndex,
@@ -678,16 +676,13 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
size_t startIdx = startIndex;
const SyntaxPattern* activePattern = nullptr;
static constexpr auto PATTERN_STACK_BUFFER =
MAX_PATTERN_STACK_SIZE * sizeof( PatternStackItem );
std::array<std::byte, PATTERN_STACK_BUFFER> patternStackBuffer;
std::pmr::monotonic_buffer_resource patternStackRes(
patternStackBuffer.data(), patternStackBuffer.size(), std::pmr::null_memory_resource() );
std::pmr::vector<PatternStackItem> patternStack( &patternStackRes );
SmallVector<PatternStackItem, MAX_PATTERN_STACK_SIZE> patternStack;
std::string emptyStr;
bool forceReevaluate = false;
while ( startIdx < size ) {
while ( startIdx < size || forceReevaluate ) {
bool matched = false;
forceReevaluate = false;
patternStack.clear();
activePattern = nullptr;
@@ -704,92 +699,122 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
activePattern->matchType, false );
if ( activePattern->hasContentScope() ) {
if ( endRange.range.first == static_cast<Int64>( startIdx ) ) {
bool applyEndLast = activePattern->isApplyEndPatternLast();
bool endMatchesAtStart = endRange.range.first == static_cast<Int64>( startIdx );
if ( endMatchesAtStart && !applyEndLast ) {
pushTokensToOpenCloseSubsyntax( startIdx, textv, activePattern, endRange,
tokens, priorityMap, true );
popStack( curState, retState, syntax, *activePattern );
startIdx = endRange.range.second;
triedPatterns.clear(); // Position advanced
// forceReevaluate = true;
continue;
}
const auto& contentScopeRepository =
curState.currentSyntax->getRepository( activePattern->contentScopeRepoHash );
if ( startIdx < size ) {
const auto& contentScopeRepository = curState.currentSyntax->getRepository(
activePattern->contentScopeRepoHash );
auto contentScopeRepoGlobalIndex = curState.currentSyntax->getRepositoryIndex(
activePattern->contentScopeRepoHash );
auto contentScopeRepoGlobalIndex = curState.currentSyntax->getRepositoryIndex(
activePattern->contentScopeRepoHash );
patternStack.push_back(
{ &contentScopeRepository.patterns, 0,
static_cast<SyntaxSyateHolderType>( contentScopeRepoGlobalIndex ) } );
patternStack.push_back(
{ &contentScopeRepository.patterns, 0,
static_cast<SyntaxSyateHolderType>( contentScopeRepoGlobalIndex ) } );
while ( !patternStack.empty() && !matched ) {
PatternStackItem& current = patternStack.back();
if ( current.index >= current.patterns->size() ) {
patternStack.pop_back();
continue;
}
const SyntaxPattern* innerPtrn = &current.patterns->data()[current.index];
SyntaxStateType patternState = {
static_cast<SyntaxSyateHolderType>( current.index + 1 ),
current.repositoryIdx };
current.index++;
while ( !patternStack.empty() && !matched ) {
PatternStackItem& current = patternStack.back();
if ( current.index >= current.patterns->size() ) {
patternStack.pop_back();
continue;
}
const SyntaxPattern* innerPtrn = &current.patterns->data()[current.index];
SyntaxStateType patternState = {
static_cast<SyntaxSyateHolderType>( current.index + 1 ),
current.repositoryIdx };
current.index++;
if ( innerPtrn->isRepositoryInclude() ) {
if ( patternStack.size() + 1 >= MAX_PATTERN_STACK_SIZE )
break;
const auto& targetRepo =
curState.currentSyntax->getRepository( innerPtrn->getRepositoryName() );
if ( innerPtrn->isRepositoryInclude() ) {
if ( patternStack.size() + 1 > MAX_PATTERN_STACK_SIZE )
break;
const auto& targetRepo = curState.currentSyntax->getRepository(
innerPtrn->getRepositoryName() );
#ifdef EE_DEBUG
const auto repoIndex = curState.currentSyntax->getRepositoryIndex(
innerPtrn->getRepositoryName() );
eeASSERT( repoIndex == innerPtrn->repositoryIdx );
const auto repoIndex = curState.currentSyntax->getRepositoryIndex(
innerPtrn->getRepositoryName() );
eeASSERT( repoIndex == innerPtrn->repositoryIdx );
#endif
patternStack.push_back(
{ &targetRepo.patterns, 0,
static_cast<SyntaxSyateHolderType>( innerPtrn->repositoryIdx ) } );
continue;
} else if ( innerPtrn->isRootSelfInclude() ) {
if ( patternStack.size() + 1 >= MAX_PATTERN_STACK_SIZE )
break;
patternStack.push_back( { &curState.currentSyntax->getPatterns(), 0, 0 } );
continue;
} else if ( innerPtrn->isSourceInclude() ) {
if ( patternStack.size() + 1 >= MAX_PATTERN_STACK_SIZE )
break;
const auto& targetRepo =
curState.currentSyntax->getRepository( innerPtrn->getRepositoryName() );
patternStack.push_back( { &targetRepo.patterns, 0,
static_cast<SyntaxSyateHolderType>(
innerPtrn->repositoryIdx ) } );
continue;
} else if ( innerPtrn->isRootSelfInclude() ) {
if ( patternStack.size() + 1 > MAX_PATTERN_STACK_SIZE )
break;
patternStack.push_back(
{ &curState.currentSyntax->getPatterns(), 0, 0 } );
continue;
} else if ( innerPtrn->isSourceInclude() ) {
if ( patternStack.size() + 1 > MAX_PATTERN_STACK_SIZE )
break;
const auto& targetRepo = curState.currentSyntax->getRepository(
innerPtrn->getRepositoryName() );
const auto repoIndex = curState.currentSyntax->getRepositoryIndex(
innerPtrn->getRepositoryName() );
const auto repoIndex = curState.currentSyntax->getRepositoryIndex(
innerPtrn->getRepositoryName() );
patternStack.push_back(
{ &targetRepo.patterns, 0,
static_cast<SyntaxSyateHolderType>( repoIndex ) } );
continue;
patternStack.push_back(
{ &targetRepo.patterns, 0,
static_cast<SyntaxSyateHolderType>( repoIndex ) } );
continue;
}
if ( startIdx != 0 &&
innerPtrn->matchType == SyntaxPatternMatchType::LuaPattern &&
innerPtrn->patterns[0][0] == '^' )
continue;
if ( patternStack.size() > 1 && innerPtrn->isAutomaticallyAdded() )
continue;
if ( std::find( triedPatterns.begin(), triedPatterns.end(),
patternState ) != triedPatterns.end() )
continue;
if ( matchPattern( *innerPtrn, startIdx, patternState,
endRange.numMatches && !applyEndLast ? &endRange
: nullptr ) ) {
matched = true;
}
}
if ( startIdx != 0 &&
innerPtrn->matchType == SyntaxPatternMatchType::LuaPattern &&
innerPtrn->patterns[0][0] == '^' )
if ( matched && endRange.isZeroWidthMatch() && startIdx == size &&
static_cast<Int64>( startIdx ) == endRange.range.second ) {
forceReevaluate = true;
continue;
if ( patternStack.size() > 1 && innerPtrn->isAutomaticallyAdded() )
continue;
if ( std::find( triedPatterns.begin(), triedPatterns.end(), patternState ) !=
triedPatterns.end() )
continue;
if ( matchPattern( *innerPtrn, startIdx, patternState,
endRange.numMatches ? &endRange : nullptr ) ) {
matched = true;
}
}
if ( matched )
continue;
// If applyEndLast is true, and NO inner pattern matched, the end pattern takes
// effect here.
bool isZeroWidthMatch = endRange.isZeroWidthMatch() &&
endRange.range.second == static_cast<Int64>( startIdx );
endMatchesAtStart |= isZeroWidthMatch;
if ( endMatchesAtStart && applyEndLast ) {
pushTokensToOpenCloseSubsyntax( startIdx, textv, activePattern, endRange,
tokens, priorityMap, true );
popStack( curState, retState, syntax, *activePattern );
startIdx = endRange.range.second;
triedPatterns.clear(); // Position advanced
forceReevaluate = true;
continue;
}
if ( !matched && startIdx < text.size() ) {
char* strStart = const_cast<char*>( text.c_str() + startIdx );
char* strEnd = strStart;
@@ -845,6 +870,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
popStack( curState, retState, syntax, *activePattern );
startIdx = endRange.range.second;
triedPatterns.clear(); // Position advanced
// forceReevaluate = true;
continue;
} else {
pushToken( tokens, activePattern->types[0], textv.substr( startIdx ) );
@@ -854,6 +880,12 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
}
}
// Prevent executing the rest of the loop (which tries to match new root patterns)
// if we've reached the end of the text and didn't trigger a pop cascade.
if ( startIdx >= size ) {
break;
}
if ( curState.subsyntaxInfo != nullptr && curState.subsyntaxInfo->patterns.size() > 1 ) {
auto rangeSubsyntax = findNonEscaped(
text, curState.subsyntaxInfo->patterns[1], startIdx,
@@ -878,7 +910,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
current.index++;
if ( pattern->isRepositoryInclude() ) {
if ( patternStack.size() + 1 >= MAX_PATTERN_STACK_SIZE )
if ( patternStack.size() + 1 > MAX_PATTERN_STACK_SIZE )
break;
const auto& repo =
@@ -887,12 +919,12 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
{ &repo.patterns, 0,
static_cast<SyntaxSyateHolderType>( pattern->repositoryIdx ) } );
} else if ( pattern->isRootSelfInclude() ) {
if ( patternStack.size() + 1 >= MAX_PATTERN_STACK_SIZE )
if ( patternStack.size() + 1 > MAX_PATTERN_STACK_SIZE )
break;
patternStack.push_back( { &curState.currentSyntax->getPatterns(), 0, 0 } );
} else if ( pattern->isSourceInclude() ) {
if ( patternStack.size() + 1 >= MAX_PATTERN_STACK_SIZE )
if ( patternStack.size() + 1 > MAX_PATTERN_STACK_SIZE )
break;
const auto& repo =

View File

@@ -0,0 +1,12 @@
#ifndef EE_UI_DOC_WEBASSEMBLYINTERFACETYPES
#define EE_UI_DOC_WEBASSEMBLYINTERFACETYPES
#include <eepp/ui/doc/syntaxdefinition.hpp>
namespace EE { namespace UI { namespace Doc { namespace Language {
extern SyntaxDefinition& addWebAssemblyInterfaceTypes();
}}}} // namespace EE::UI::Doc::Language
#endif

View File

@@ -0,0 +1,425 @@
#include <eepp/ui/doc/languages/webassemblytextformat.hpp>
#include <eepp/ui/doc/syntaxdefinitionmanager.hpp>
namespace EE { namespace UI { namespace Doc { namespace Language {
SyntaxDefinition& addWebAssemblyTextFormat() {
return SyntaxDefinitionManager::instance()
->add(
{ "WebAssembly Text Format",
{ "%.wat$" },
{
{ { "include", "#comments" }, "normal", "", SyntaxPatternMatchType::LuaPattern },
{ { "include", "#strings" }, "normal", "", SyntaxPatternMatchType::LuaPattern },
{ { "include", "#instructions" },
"normal",
"",
SyntaxPatternMatchType::LuaPattern },
{ { "include", "#types" }, "normal", "", SyntaxPatternMatchType::LuaPattern },
{ { "include", "#modules" }, "normal", "", SyntaxPatternMatchType::LuaPattern },
{ { "include", "#constants" }, "normal", "", SyntaxPatternMatchType::LuaPattern },
{ { "include", "#invalid" }, "normal", "", SyntaxPatternMatchType::LuaPattern },
},
{
},
"",
{}
} )
.addRepositories( {
{ "types",
{
{ { "\\bv128\\b(?!\\.)" }, "type", "", SyntaxPatternMatchType::RegEx },
{ { "\\b(?:externref|funcref|nullref)\\b(?!\\.)" },
"type",
"",
SyntaxPatternMatchType::RegEx },
{ { "\\bexnref\\b(?!\\.)" }, "type", "", SyntaxPatternMatchType::RegEx },
{ { "\\b(?:i32|i64|f32|f64)\\b(?!\\.)" },
"type",
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(?:i8|i16|ref|funcref|externref|anyref|eqref|i31ref|nullfuncref|"
"nullexternref|structref|arrayref|nullref)\\b(?!\\.)" },
"type",
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(?:type|func|extern|any|eq|nofunc|noextern|struct|array|none)\\b(?!\\."
")" },
"type",
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(?:struct|array|sub|final|rec|field|mut)\\b(?!\\.)" },
"type",
"",
SyntaxPatternMatchType::RegEx },
} },
{ "modules",
{
{ { "(?<=\\(data)\\s+(passive)\\b" },
{ "normal", "keyword" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "(?<=\\()(?:module|import|export|memory|data|table|elem|start|func|type|"
"param|result|global|local)\\b" },
"keyword",
"",
SyntaxPatternMatchType::RegEx },
{ { "(?<=\\()\\s*(mut)\\b" },
{ "keyword", "keyword" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "(?<=\\(func|\\(start|call|return_call|ref\\.func)\\s+(\\$[0-9A-Za-z!#$%&'*+"
"\\-./:<=>?@\\\\^_`|~]*)" },
{ "normal", "function" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\)\\s+(\\$[0-9A-Za-z!#$%&'*+\\-./:<=>?@\\\\^_`|~]*)", "\\)" },
{ "normal", "function" },
{},
"",
SyntaxPatternMatchType::RegEx,
{
{ { "(?<=\\s)\\$[0-9A-Za-z!#$%&'*+\\-./:<=>?@\\\\^_`|~]*" },
"function",
"",
SyntaxPatternMatchType::RegEx },
} },
{ { "(?<=\\(type)\\s+(\\$[0-9A-Za-z!#$%&'*+\\-./:<=>?@\\\\^_`|~]*)" },
{ "normal", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\$[0-9A-Za-z!#$%&'*+\\-./:<=>?@\\\\^_`|~]*\\b" },
"normal",
"",
SyntaxPatternMatchType::RegEx },
} },
{ "instructions",
{
{ { "\\b(i32|i64)\\.trunc_sat_f(?:32|64)_[su]\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i32)\\.(?:extend(?:8|16)_s)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i64)\\.(?:extend(?:8|16|32)_s)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(memory)\\.(?:copy|fill|init|drop)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(v128)\\.(?:const|and|or|xor|not|andnot|bitselect|load|store)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i8x16)\\.(?:shuffle|swizzle|splat|replace_lane|add|sub|mul|neg|shl|shr_["
"su]|eq|ne|lt_[su]|le_[su]|gt_[su]|ge_[su]|min_[su]|max_[su]|any_true|all_"
"true|extract_lane_[su]|add_saturate_[su]|sub_saturate_[su]|avgr_u|narrow_"
"i16x8_[su])\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i16x8)\\.(?:splat|replace_lane|add|sub|mul|neg|shl|shr_[su]|eq|ne|lt_["
"su]|le_[su]|gt_[su]|ge_[su]|min_[su]|max_[su]|any_true|all_true|extract_"
"lane_[su]|add_saturate_[su]|sub_saturate_[su]|avgr_u|load8x8_[su]|narrow_"
"i32x4_[su]|widen_(low|high)_i8x16_[su])\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i32x4)\\.(?:splat|replace_lane|add|sub|mul|neg|shl|shr_[su]|eq|ne|lt_["
"su]|le_[su]|gt_[su]|ge_[su]|min_[su]|max_[su]|any_true|all_true|extract_"
"lane|load16x4_[su]|trunc_sat_f32x4_[su]|widen_(low|high)_i16x8_[su])\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i64x2)\\.(?:splat|replace_lane|add|sub|mul|neg|shl|shr_[su]|extract_"
"lane|load32x2_[su])\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(f32x4)\\.(?:splat|replace_lane|add|sub|mul|neg|extract_lane|eq|ne|lt|le|"
"gt|ge|abs|min|max|div|sqrt|convert_i32x4_[su])\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(f64x2)\\.(?:splat|replace_lane|add|sub|mul|neg|extract_lane|eq|ne|lt|le|"
"gt|ge|abs|min|max|div|sqrt)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(v8x16)\\.(?:load_splat|shuffle|swizzle)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(v16x8)\\.load_splat\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(v32x4)\\.load_splat\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(v64x2)\\.load_splat\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i32)\\.(atomic)\\.(?:load(?:8_u|16_u)?|store(?:8|16)?|wait|(rmw)\\.(?:"
"add|sub|and|or|xor|xchg|cmpxchg)|(rmw8|rmw16)\\.(?:add_u|sub_u|and_u|or_u|"
"xor_u|xchg_u|cmpxchg_u))\\b" },
{ "operator", "type", "type", "type", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i64)\\.(atomic)\\.(?:load(?:8_u|16_u|32_u)?|store(?:8|16|32)?|wait|(rmw)"
"\\.(?:add|sub|and|or|xor|xchg|cmpxchg)|(rmw8|rmw16|rmw32)\\.(?:add_u|sub_u|"
"and_u|or_u|xor_u|xchg_u|cmpxchg_u))\\b" },
{ "operator", "type", "type", "type", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(atomic)\\.(?:notify|fence)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\bshared\\b" }, "keyword", "", SyntaxPatternMatchType::RegEx },
{ { "\\b(ref)\\.(?:null|is_null|func|extern)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(table)\\.(?:get|size|grow|fill|init|copy)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(?:externref|funcref|nullref)\\b" },
"type",
"",
SyntaxPatternMatchType::RegEx },
{ { "\\breturn_call(?:_indirect)?\\b" },
"keyword",
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(?:try|catch|throw|rethrow|br_on_exn)\\b" },
"keyword",
"",
SyntaxPatternMatchType::RegEx },
{ { "(?<=\\()event\\b" }, "keyword", "", SyntaxPatternMatchType::RegEx },
{ { "\\b(i32|i64|f32|f64|externref|funcref|nullref|exnref)\\.(?:push|pop)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i32)\\.(?:load|load(?:8|16)(?:_[su])?|store(?:8|16)?)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i64)\\.(?:load|load(?:8|16|32)(?:_[su])?|store(?:8|16|32)?)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(f32|f64)\\.(?:load|store)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(memory)\\.(?:size|grow)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(offset|align)=\\b" },
{ "normal", "normal" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(local)\\.(?:get|set|tee)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(global)\\.(?:get|set)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i32|i64)\\.(const|eqz|eq|ne|lt_[su]|gt_[su]|le_[su]|ge_[su]|clz|ctz|"
"popcnt|add|sub|mul|div_[su]|rem_[su]|and|or|xor|shl|shr_[su]|rotl|rotr)"
"\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(f32|f64)\\.(const|eq|ne|lt|gt|le|ge|abs|neg|ceil|floor|trunc|nearest|"
"sqrt|add|sub|mul|div|min|max|copysign)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i32)\\.(wrap_i64|trunc_(f32|f64)_[su]|reinterpret_f32)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i64)\\.(extend_i32_[su]|trunc_f(32|64)_[su]|reinterpret_f64)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(f32)\\.(convert_i(32|64)_[su]|demote_f64|reinterpret_i32)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(f64)\\.(convert_i(32|64)_[su]|promote_f32|reinterpret_i64)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(?:unreachable|nop|block|loop|if|then|else|end|br|br_if|br_table|return|"
"call|call_indirect)\\b" },
"keyword",
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(?:drop|select)\\b" }, "operator", "", SyntaxPatternMatchType::RegEx },
{ { "\\b(ref)\\.(?:eq|test|cast)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(struct)\\.(?:new_canon|new_canon_default|get|get_s|get_u|set)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(array)\\.(?:new_canon|new_canon_default|get|get_s|get_u|set|len|new_"
"canon_fixed|new_canon_data|new_canon_elem)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i31)\\.(?:new|get_s|get_u)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(?:br_on_non_null|br_on_cast|br_on_cast_fail)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(extern)\\.(?:internalize|externalize)\\b" },
{ "operator", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
} },
{ "strings",
{
{ { "\"", "\"", "\\\\(n|t|\\\\|'|\"|[0-9a-fA-F]{2})" },
{ "string" },
{ "string" },
"",
SyntaxPatternMatchType::RegEx },
} },
{ "constants",
{
{ { "\\b(i8x16)(?:\\s+0x[0-9a-fA-F]{1,2}){16}\\b" },
{ "number", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i16x8)(?:\\s+0x[0-9a-fA-F]{1,4}){8}\\b" },
{ "number", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i32x4)(?:\\s+0x[0-9a-fA-F]{1,8}){4}\\b" },
{ "number", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\b(i64x2)(?:\\s+0x[0-9a-fA-F]{1,16}){2}\\b" },
{ "number", "type" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "[+-]?\\b[0-9][0-9]*(?:\\.[0-9][0-9]*)?(?:[eE][+-]?[0-9]+)?\\b" },
"number",
"",
SyntaxPatternMatchType::RegEx },
{ { "[+-]?\\b0x([0-9a-fA-F]*\\.[0-9a-fA-F]+|[0-9a-fA-F]+\\.?)[Pp][+-]?[0-9]+"
"\\b" },
"number",
"",
SyntaxPatternMatchType::RegEx },
{ { "[+-]?\\binf\\b" }, "number", "", SyntaxPatternMatchType::RegEx },
{ { "[+-]?\\bnan:0x[0-9a-fA-F][0-9a-fA-F]*\\b" },
"number",
"",
SyntaxPatternMatchType::RegEx },
{ { "[+-]?\\b(?:0x[0-9a-fA-F][0-9a-fA-F]*|\\d[\\d]*)\\b" },
"number",
"",
SyntaxPatternMatchType::RegEx },
} },
{ "invalid",
{
{ { "[^\\s()]+" }, "normal", "", SyntaxPatternMatchType::RegEx },
} },
{ "comments",
{
{ { "(;;).*$" },
{ "comment", "comment" },
{},
"",
SyntaxPatternMatchType::RegEx },
{ { "\\(;", ";\\)" },
{ "comment" },
{ "comment" },
"",
SyntaxPatternMatchType::RegEx },
} },
} );
}
}}}} // namespace EE::UI::Doc::Language

View File

@@ -0,0 +1,12 @@
#ifndef EE_UI_DOC_WEBASSEMBLYTEXTFORMAT
#define EE_UI_DOC_WEBASSEMBLYTEXTFORMAT
#include <eepp/ui/doc/syntaxdefinition.hpp>
namespace EE { namespace UI { namespace Doc { namespace Language {
extern SyntaxDefinition& addWebAssemblyTextFormat();
}}}} // namespace EE::UI::Doc::Language
#endif

View File

@@ -127,6 +127,8 @@
#include <eepp/ui/doc/languages/verilog.hpp>
#include <eepp/ui/doc/languages/viml.hpp>
#include <eepp/ui/doc/languages/vue.hpp>
#include <eepp/ui/doc/languages/webassemblyinterfacetypes.hpp>
#include <eepp/ui/doc/languages/webassemblytextformat.hpp>
#include <eepp/ui/doc/languages/wren.hpp>
#include <eepp/ui/doc/languages/x86assembly.hpp>
#include <eepp/ui/doc/languages/xit.hpp>
@@ -816,6 +818,14 @@ static void preDefinitionLangsChunk2( SyntaxDefinitionManager* sdm ) {
sdm->addPreDefinition(
{ "Vue", []() -> SyntaxDefinition& { return addVue(); }, { "%.vue?$" } } );
sdm->addPreDefinition( { "WebAssembly Text Format",
[]() -> SyntaxDefinition& { return addWebAssemblyTextFormat(); },
{ "%.wat$" } } );
sdm->addPreDefinition( { "WebAssembly Interface Types",
[]() -> SyntaxDefinition& { return addWebAssemblyInterfaceTypes(); },
{ "%.wit$" } } );
sdm->addPreDefinition(
{ "Wren", []() -> SyntaxDefinition& { return addWren(); }, { "%.wren$" } } );