From 57754feb31c5e427f541f3b68435559e5b75629d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Thu, 3 Sep 2026 17:19:27 -0300 Subject: [PATCH 1/8] Added Kitty Graphics Protocol support for eterm module, still a WIP but working. --- docs/articles/eterm-kitty-graphics.md | 43 + include/eepp/system/base64.hpp | 22 +- src/eepp/system/base64.cpp | 182 +- .../eterm/terminal/ipseudoterminal.hpp | 4 +- .../eterm/terminal/iterminaldisplay.hpp | 4 + .../eterm/terminal/kittygraphicsprotocol.hpp | 243 +++ .../eterm/terminal/kittygraphicsrenderer.hpp | 42 + .../include/eterm/terminal/pseudoterminal.hpp | 2 +- .../eterm/terminal/terminaldisplay.hpp | 13 + .../eterm/terminal/terminalemulator.hpp | 31 +- .../eterm/terminal/terminalgraphics.hpp | 109 ++ .../eterm/terminal/terminalsession.hpp | 21 +- .../include/eterm/terminal/terminaltypes.hpp | 1 + .../src/eterm/terminal/iterminaldisplay.cpp | 3 + .../eterm/terminal/kittygraphicsprotocol.cpp | 1559 +++++++++++++++++ .../eterm/terminal/kittygraphicsrenderer.cpp | 167 ++ .../src/eterm/terminal/pseudoterminal.cpp | 8 +- .../src/eterm/terminal/terminaldisplay.cpp | 112 +- .../src/eterm/terminal/terminalemulator.cpp | 327 +++- .../src/eterm/terminal/terminalgraphics.cpp | 77 + .../src/eterm/terminal/terminalsession.cpp | 50 +- src/tests/unit_tests/eterm_tests.cpp | 642 ++++++- 22 files changed, 3543 insertions(+), 119 deletions(-) create mode 100644 docs/articles/eterm-kitty-graphics.md create mode 100644 src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp create mode 100644 src/modules/eterm/include/eterm/terminal/kittygraphicsrenderer.hpp create mode 100644 src/modules/eterm/include/eterm/terminal/terminalgraphics.hpp create mode 100644 src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp create mode 100644 src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp create mode 100644 src/modules/eterm/src/eterm/terminal/terminalgraphics.cpp diff --git a/docs/articles/eterm-kitty-graphics.md b/docs/articles/eterm-kitty-graphics.md new file mode 100644 index 000000000..a70aee53e --- /dev/null +++ b/docs/articles/eterm-kitty-graphics.md @@ -0,0 +1,43 @@ +# eTerm Kitty graphics protocol + +eTerm implements the terminal-receiver side of the Kitty graphics protocol. Protocol parsing, +decoding, image storage, placement state, animation, and terminal lifecycle integration run on the +terminal worker. The UI thread consumes immutable placement metadata and an ordered pixel-update +stream, and exclusively owns the OpenGL texture cache. + +## Supported features + +- direct RGB and RGBA transmission, including chunking and zlib compression; +- PNG transmission; +- image IDs, image numbers, placement IDs, source cropping, cell geometry, and pixel offsets; +- root-image rectangle updates, animation frames, frame composition, and animation control; +- deletion selectors, quiet modes, capability queries, storage quotas, usage hints, and eviction; +- primary/alternate-screen state, reset, clear, scrolling, margins, scrollback, and resize; +- Unicode placeholder placements (U+10EEEE and Kitty's combining-diacritic encoding); +- relative placements, including parent validation, cycle detection, and descendant movement; +- terminal pixel-size queries and SGR pixel mouse coordinates. + +Local file, temporary-file, and shared-memory transfer media are intentionally not enabled. Direct +transmission is the portable path and works through SSH without giving terminal applications access +to the terminal host's filesystem or shared-memory namespace. + +## Resource and threading behavior + +Decoded root images are stored as premultiplied-independent RGBA8 CPU buffers on the worker. GPU +textures exist only on the UI/GL thread. Incremental updates carry monotonically increasing sequence +numbers; a bounded queue gap triggers a complete graphics resynchronization. Image count, placement +count, APC size, decoded transfer size, and total image storage are bounded. Unreferenced images are +evicted oldest-first when storage is needed. + +Graphics follow the same presentation cadence and DEC synchronized-update boundary as text. A +placement anchored in normal-screen history moves with terminal scrolling and is projected into the +current scrollback viewport. Margin scrolling clips or removes only placements wholly participating +in the scrolled region. Resize preserves screen-coordinate placements and reflows Unicode placeholder +metadata with its text cells. + +## Compatibility testing + +The eTerm unit suite covers fragmented APC input, malformed/fuzzed commands, raw/zlib/PNG transfer, +independent image namespaces, replacement, partial updates, placement geometry, scrolling and +scrollback, screen lifecycle, animation, Unicode placeholders, relative placement graphs, ordered +worker/UI delivery, queue overflow, and resynchronization. diff --git a/include/eepp/system/base64.hpp b/include/eepp/system/base64.hpp index 640dcf9ef..1ebc7dd4c 100644 --- a/include/eepp/system/base64.hpp +++ b/include/eepp/system/base64.hpp @@ -11,22 +11,38 @@ namespace EE { namespace System { class EE_API Base64 { public: + enum class DecodeMode { + AllowWhitespace, + NoWhitespace, + }; + /** Encode binary data into base64 digits with MIME style === pads ** @return The final length of the output */ static size_t encode( size_t in_len, const unsigned char* in, size_t out_len, char* out ); - /** Decode base64 digits with MIME style === pads into binary data + /** Decode base64 digits with MIME style === pads into binary data. + ** Preserves the historical behavior of ignoring ASCII whitespace. ** @return The final length of the output */ static size_t decode( size_t in_len, const char* in, size_t out_len, unsigned char* out ); + /** Decode base64 digits with MIME style === pads into binary data. + ** DecodeMode::NoWhitespace avoids all whitespace handling and enables the fastest path. + ** @return The final length of the output */ + static size_t decode( size_t in_len, const char* in, size_t out_len, unsigned char* out, + DecodeMode mode ); + /** Encodes a string into a base64 string ** @return True if encoding was successful */ static bool encode( std::string_view in, std::string& out ); - /** Decodes a base64 string to a string - ** @return True if encoding was successful */ + /** Decodes a base64 string to a string, ignoring ASCII whitespace. + ** @return The final length of the output, or size_t(-1) on truncation */ static size_t decode( std::string_view in, std::string& out ); + /** Decodes a base64 string to a string. + ** @return The final length of the output, or size_t(-1) on truncation */ + static size_t decode( std::string_view in, std::string& out, DecodeMode mode ); + /** @return A safe encoding output length for an input of the length indicated */ static inline size_t encodeSafeOutLen( size_t in_len ) { return ( ( in_len + 2 ) / 3 ) * 4 + 1; diff --git a/src/eepp/system/base64.cpp b/src/eepp/system/base64.cpp index 9bee1e5d5..91ecf9d5b 100644 --- a/src/eepp/system/base64.cpp +++ b/src/eepp/system/base64.cpp @@ -1,63 +1,123 @@ #include +#include namespace EE { namespace System { -/* base64.c : base-64 / MIME encode/decode */ -/* PUBLIC DOMAIN - Jon Mayo - November 13, 2003 */ -/* $Id: base64.c 156 2007-07-12 23:29:10Z orange $ */ +namespace { -/* decode a base64 string in one shot */ -size_t Base64::decode( size_t in_len, const char* in, size_t out_len, unsigned char* out ) { - static const Uint8 base64dec_tab[256] = { - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63, 52, 53, 54, 55, 56, 57, - 58, 59, 60, 61, 255, 255, 255, 0, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, - 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, - 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, - }; +constexpr Uint8 BASE64_INVALID = 0xFF; +constexpr Uint8 BASE64_PADDING = 0xFE; +constexpr Uint8 BASE64_WHITESPACE = 0xFD; - size_t ii, io; - Uint32 v; - unsigned rem; +constexpr std::array makeBase64DecodeTable() { + std::array table{}; + + for ( size_t i = 0; i < table.size(); ++i ) + table[i] = BASE64_INVALID; + + for ( size_t i = 0; i < 26; ++i ) { + table[static_cast( 'A' + i )] = static_cast( i ); + table[static_cast( 'a' + i )] = static_cast( i + 26 ); + } + + for ( size_t i = 0; i < 10; ++i ) + table[static_cast( '0' + i )] = static_cast( i + 52 ); + + table[static_cast( '+' )] = 62; + table[static_cast( '/' )] = 63; + table[static_cast( '=' )] = BASE64_PADDING; + + // ASCII whitespace accepted by the historical decoder via isspace(): + table[static_cast( ' ' )] = BASE64_WHITESPACE; + table[static_cast( '\t' )] = BASE64_WHITESPACE; + table[static_cast( '\n' )] = BASE64_WHITESPACE; + table[static_cast( '\v' )] = BASE64_WHITESPACE; + table[static_cast( '\f' )] = BASE64_WHITESPACE; + table[static_cast( '\r' )] = BASE64_WHITESPACE; + + return table; +} + +constexpr auto base64dec_tab = makeBase64DecodeTable(); + +template +size_t decodeBase64( size_t in_len, const char* in, size_t out_len, unsigned char* out ) { + size_t ii = 0; + size_t io = 0; + Uint32 v = 0; + unsigned rem = 0; + + while ( ii < in_len ) { + /* + * Fast path: four ordinary base64 bytes become three output bytes. + * + * The OR validates all four table entries at once: valid base64 values + * are 0..63, while padding/whitespace/invalid entries have high bits set. + * + * This path is especially useful for protocol payloads that guarantee + * contiguous, whitespace-free base64 data. + */ + if ( rem == 0 && ii + 4 <= in_len && io + 3 <= out_len ) { + const Uint8 a = base64dec_tab[static_cast( in[ii] )]; + const Uint8 b = base64dec_tab[static_cast( in[ii + 1] )]; + const Uint8 c = base64dec_tab[static_cast( in[ii + 2] )]; + const Uint8 d = base64dec_tab[static_cast( in[ii + 3] )]; + + if ( ( a | b | c | d ) <= 63 ) { + out[io] = static_cast( ( a << 2 ) | ( b >> 4 ) ); + out[io + 1] = static_cast( ( b << 4 ) | ( c >> 2 ) ); + out[io + 2] = static_cast( ( c << 6 ) | d ); + + ii += 4; + io += 3; + continue; + } + } + + const Uint8 ch = base64dec_tab[static_cast( in[ii++] )]; + + if ( ch > 63 ) { + if constexpr ( AllowWhitespace ) { + if ( ch == BASE64_WHITESPACE ) + continue; + } + + // Preserve the old behavior: stop at '=' or the first parse error. + break; + } - for ( io = 0, ii = 0, v = 0, rem = 0; ii < in_len; ii++ ) { - unsigned char ch; - unsigned char c = (unsigned char)in[ii]; - if ( isspace( c ) ) - continue; - if ( c == '=' ) - break; /* stop at = */ - ch = base64dec_tab[c]; - if ( ch == 255 ) - break; /* stop at a parse error */ v = ( v << 6 ) | ch; rem += 6; + if ( rem >= 8 ) { rem -= 8; + if ( io >= out_len ) - return -1; /* truncation is failure */ - out[io++] = ( v >> rem ) & 255; + return static_cast( -1 ); + + out[io++] = static_cast( ( v >> rem ) & 255 ); + + // Every four valid base64 characters rem returns to zero. + if ( rem == 0 ) + v = 0; } } - if ( rem >= 8 ) { - rem -= 8; - if ( io >= out_len ) - return -1; /* truncation is failure */ - out[io++] = ( v >> rem ) & 255; - } + return io; } +} // namespace + +size_t Base64::decode( size_t in_len, const char* in, size_t out_len, unsigned char* out ) { + return decodeBase64( in_len, in, out_len, out ); +} + +size_t Base64::decode( size_t in_len, const char* in, size_t out_len, unsigned char* out, + DecodeMode mode ) { + return mode == DecodeMode::NoWhitespace ? decodeBase64( in_len, in, out_len, out ) + : decodeBase64( in_len, in, out_len, out ); +} + size_t Base64::encode( size_t in_len, const unsigned char* in, size_t out_len, char* out ) { static const Uint8 base64enc_tab[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -74,23 +134,27 @@ size_t Base64::encode( size_t in_len, const unsigned char* in, size_t out_len, c while ( rem >= 6 ) { rem -= 6; if ( io >= out_len ) - return -1; /* truncation is failure */ + return static_cast( -1 ); /* truncation is failure */ out[io++] = base64enc_tab[( v >> rem ) & 63]; } } + if ( rem ) { v <<= ( 6 - rem ); if ( io >= out_len ) - return -1; /* truncation is failure */ + return static_cast( -1 ); /* truncation is failure */ out[io++] = base64enc_tab[v & 63]; } + while ( io & 3 ) { if ( io >= out_len ) - return -1; /* truncation is failure */ + return static_cast( -1 ); /* truncation is failure */ out[io++] = '='; } + if ( io >= out_len ) - return -1; /* no room for null terminator */ + return static_cast( -1 ); /* no room for null terminator */ + out[io] = 0; return io; } @@ -98,31 +162,33 @@ size_t Base64::encode( size_t in_len, const unsigned char* in, size_t out_len, c bool Base64::encode( std::string_view in, std::string& out ) { size_t b64len = encodeSafeOutLen( in.size() ); - if ( out.size() < b64len ) { + if ( out.size() < b64len ) out.resize( b64len ); - } - int len = encode( in.size(), (const unsigned char*)in.data(), out.size(), (char*)&out[0] ); + const size_t len = + encode( in.size(), reinterpret_cast( in.data() ), out.size(), out.data() ); - if ( -1 != len && (size_t)len != out.size() ) { + if ( len != static_cast( -1 ) && len != out.size() ) out.resize( len ); - } - return -1 != len; + return len != static_cast( -1 ); } size_t Base64::decode( std::string_view in, std::string& out ) { + return decode( in, out, DecodeMode::AllowWhitespace ); +} + +size_t Base64::decode( std::string_view in, std::string& out, DecodeMode mode ) { size_t d64len = decodeSafeOutLen( in.size() ); - if ( out.size() < d64len ) { + if ( out.size() < d64len ) out.resize( d64len ); - } - int len = decode( in.size(), in.data(), out.size(), (unsigned char*)&out[0] ); + const size_t len = decode( in.size(), in.data(), out.size(), + reinterpret_cast( out.data() ), mode ); - if ( -1 != len && (size_t)len != out.size() ) { + if ( len != static_cast( -1 ) && len != out.size() ) out.resize( len ); - } return len; } diff --git a/src/modules/eterm/include/eterm/terminal/ipseudoterminal.hpp b/src/modules/eterm/include/eterm/terminal/ipseudoterminal.hpp index a6156a358..b6e981116 100644 --- a/src/modules/eterm/include/eterm/terminal/ipseudoterminal.hpp +++ b/src/modules/eterm/include/eterm/terminal/ipseudoterminal.hpp @@ -44,7 +44,9 @@ class IPseudoTerminal : public eterm::System::IPipe { virtual int getNumRows() const = 0; - virtual bool resize( int columns, int rows ) = 0; + virtual bool resize( int columns, int rows, int pixelWidth, int pixelHeight ) = 0; + + bool resize( int columns, int rows ) { return resize( columns, rows, 0, 0 ); } }; }} // namespace eterm::Terminal diff --git a/src/modules/eterm/include/eterm/terminal/iterminaldisplay.hpp b/src/modules/eterm/include/eterm/terminal/iterminaldisplay.hpp index 295be3a37..2d8dce80a 100644 --- a/src/modules/eterm/include/eterm/terminal/iterminaldisplay.hpp +++ b/src/modules/eterm/include/eterm/terminal/iterminaldisplay.hpp @@ -22,6 +22,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #include +#include #include using namespace EE; @@ -80,6 +81,9 @@ class ITerminalDisplay { virtual void drawCursor( int cx, int cy, TerminalGlyph g, int ox, int oy, TerminalGlyph og ) = 0; + virtual void drawGraphics( std::shared_ptr presentation, + std::vector updates ); + virtual void drawEnd() = 0; protected: diff --git a/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp b/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp new file mode 100644 index 000000000..bd18bf7f6 --- /dev/null +++ b/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp @@ -0,0 +1,243 @@ +#ifndef ETERM_KITTYGRAPHICSPROTOCOL_HPP +#define ETERM_KITTYGRAPHICSPROTOCOL_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace EE; + +namespace eterm { namespace Terminal { + +enum class KittyGraphicsError : Uint8 { + None, + InvalidArgument, + Unsupported, + InvalidData, + TooLarge, + DecodeFailed, + NotFound, + NoSpace, + NoParent, + Cycle, + TooDeep +}; + +struct KittyGraphicsCommandData { + std::string_view payload; + std::optional format; + std::optional dataSize; + std::optional more; + std::optional imageId; + std::optional imageNumber; + std::optional usageHint; + std::optional placementId; + std::optional quiet; + std::optional width; + std::optional height; + std::optional x; + std::optional y; + std::optional sourceWidth; + std::optional sourceHeight; + std::optional columns; + std::optional rows; + std::optional xOffset; + std::optional yOffset; + std::optional zIndex; + std::optional cursorMovement; + std::optional virtualPlacement; + std::optional parentImageId; + std::optional parentPlacementId; + std::optional parentOffsetX; + std::optional parentOffsetY; + char transmission{ 'd' }; + char compression{ 0 }; + char deletion{ 'a' }; +}; + +struct KittyTransmitCommand { + KittyGraphicsCommandData data; + bool display{ false }; +}; + +struct KittyPutCommand { + KittyGraphicsCommandData data; +}; + +struct KittyDeleteCommand { + KittyGraphicsCommandData data; +}; + +struct KittyFrameCommand { + KittyGraphicsCommandData data; +}; + +struct KittyAnimationCommand { + KittyGraphicsCommandData data; +}; + +struct KittyComposeCommand { + KittyGraphicsCommandData data; +}; + +struct KittyQueryCommand { + KittyGraphicsCommandData data; +}; + +using KittyGraphicsCommand = + std::variant; + +struct KittyGraphicsParseResult { + std::optional command; + KittyGraphicsError error{ KittyGraphicsError::None }; +}; + +struct KittyGraphicsHandleResult { + KittyGraphicsHandleResult() = default; + KittyGraphicsHandleResult( std::string response, KittyGraphicsError error, bool changed ) : + response( std::move( response ) ), error( error ), changed( changed ) {} + + std::string response; + Vector2i cursorMovement; + KittyGraphicsError error{ KittyGraphicsError::None }; + bool changed{ false }; +}; + +struct KittyGraphicsStats { + Uint64 decodedBytes{ 0 }; + Uint64 fullImageUpdates{ 0 }; + Uint64 rectangleUpdates{ 0 }; + Uint64 evictions{ 0 }; +}; + +class KittyGraphicsProtocol { + public: + explicit KittyGraphicsProtocol( size_t maxStorageBytes = 320 * 1024 * 1024, + size_t maxImages = 4096, size_t maxPlacements = 65536 ); + + static KittyGraphicsParseResult parse( std::string_view command ); + + KittyGraphicsHandleResult handle( std::string_view command, Vector2i cursor = {} ); + + std::vector takeUpdates(); + + std::shared_ptr takePresentation(); + + bool hasPendingPresentation() const { return mPresentationDirty; } + + const std::vector* imagePixels( KittyImageId imageId ) const; + + size_t imageCount() const { return mImages.size(); } + + const KittyGraphicsStats& stats() const { return mStats; } + + bool hasVirtualPlacements() const; + + void reset(); + + void resync(); + + void clearScreen(); + + void setAlternateScreen( bool alternate ); + + void scrollScreen( int top, int bottom, int rows, bool preserveHistory ); + + void setViewport( int scrollPosition, int historyLength, int screenRows ); + + bool updateAnimations(); + + void setCellPixelSize( Uint32 width, Uint32 height ); + + void setPlaceholderCells( std::vector cells ); + + private: + struct Image { + struct Frame { + std::vector rgba; + Int32 gapMs{ 40 }; + Uint32 usageHint{ 0 }; + }; + std::vector rgba; + std::unordered_map frames; + Sizei size; + Uint32 imageNumber{ 0 }; + Uint32 usageHint{ 0 }; + bool anonymous{ false }; + Uint64 creationSerial{ 0 }; + Uint32 currentFrame{ 1 }; + Uint32 loopCount{ 1 }; + Uint32 loopsCompleted{ 0 }; + Uint8 animationState{ 1 }; + Int32 rootGapMs{ 0 }; + EE::System::Clock frameClock; + }; + + struct PendingTransfer { + KittyGraphicsCommandData data; + std::vector decodedData; + bool display{ false }; + bool query{ false }; + bool frame{ false }; + bool active{ false }; + }; + + struct Placement { + TerminalVisiblePlacement visible; + Uint64 internalId{ 0 }; + bool virtualPlacement{ false }; + KittyImageId parentImageId{ 0 }; + KittyPlacementId parentPlacementId{ 0 }; + }; + + KittyGraphicsHandleResult handleTransmit( const KittyGraphicsCommandData& data, bool display, + bool query, bool frame, Vector2i cursor ); + KittyGraphicsHandleResult finishTransfer( PendingTransfer transfer, Vector2i cursor ); + KittyGraphicsHandleResult put( const KittyGraphicsCommandData& data, Vector2i cursor ); + KittyGraphicsHandleResult remove( const KittyGraphicsCommandData& data, Vector2i cursor ); + KittyGraphicsHandleResult controlAnimation( const KittyGraphicsCommandData& data ); + KittyGraphicsHandleResult composeFrames( const KittyGraphicsCommandData& data ); + KittyImageId allocateImageId(); + KittyImageId resolveImageId( const KittyGraphicsCommandData& data ) const; + bool ensureCapacity( size_t bytes, KittyImageId replacingId, bool addingImage ); + bool isImagePlaced( KittyImageId imageId ) const; + void eraseImage( KittyImageId imageId ); + std::string response( const KittyGraphicsCommandData& data, KittyGraphicsError error, + KittyImageId imageId = 0 ) const; + + std::unordered_map mImages; + std::vector mPlacements; + std::vector mPrimaryPlacements; + std::vector mPlaceholderCells; + std::vector mUpdates; + PendingTransfer mPending; + size_t mStorageBytes{ 0 }; + size_t mFrameStorageBytes{ 0 }; + Uint64 mCreationSerial{ 0 }; + Uint64 mPresentationGeneration{ 0 }; + Uint64 mPlacementSerial{ 0 }; + KittyImageId mNextImageId{ 1 }; + bool mPresentationDirty{ false }; + int mScrollPosition{ 0 }; + int mHistoryLength{ 0 }; + int mScreenRows{ 0 }; + Uint32 mCellPixelWidth{ 0 }; + Uint32 mCellPixelHeight{ 0 }; + size_t mMaxStorageBytes{ 0 }; + size_t mMaxImages{ 0 }; + size_t mMaxPlacements{ 0 }; + KittyGraphicsStats mStats; +}; + +}} // namespace eterm::Terminal + +#endif diff --git a/src/modules/eterm/include/eterm/terminal/kittygraphicsrenderer.hpp b/src/modules/eterm/include/eterm/terminal/kittygraphicsrenderer.hpp new file mode 100644 index 000000000..e0e9b8173 --- /dev/null +++ b/src/modules/eterm/include/eterm/terminal/kittygraphicsrenderer.hpp @@ -0,0 +1,42 @@ +#ifndef ETERM_KITTYGRAPHICSRENDERER_HPP +#define ETERM_KITTYGRAPHICSRENDERER_HPP + +#include +#include +#include + +using namespace EE::Graphics; + +namespace eterm { namespace Terminal { + +class KittyGraphicsRenderer { + public: + enum class Pass : Uint8 { VeryNegative, Negative, NonNegative }; + + bool applyUpdates( std::vector&& updates ); + + void setPresentation( std::shared_ptr presentation ); + + void draw( Pass pass, const Vector2f& origin, const Sizef& cellSize, const Sizef& gridSize ); + + bool hasPlacements( Pass pass ) const; + + void reset(); + + Uint64 lastAppliedSequence() const { return mLastAppliedSequence; } + + private: + struct GPUImage { + TexturePtr texture; + std::unordered_map frames; + Sizei size; + }; + + std::unordered_map mImages; + std::shared_ptr mPresentation; + Uint64 mLastAppliedSequence{ 0 }; +}; + +}} // namespace eterm::Terminal + +#endif diff --git a/src/modules/eterm/include/eterm/terminal/pseudoterminal.hpp b/src/modules/eterm/include/eterm/terminal/pseudoterminal.hpp index ad001c4c4..acafcbd76 100644 --- a/src/modules/eterm/include/eterm/terminal/pseudoterminal.hpp +++ b/src/modules/eterm/include/eterm/terminal/pseudoterminal.hpp @@ -55,7 +55,7 @@ class PseudoTerminal final : public IPseudoTerminal { virtual int getNumColumns() const override; virtual int getNumRows() const override; - virtual bool resize( int columns, int rows ) override; + virtual bool resize( int columns, int rows, int pixelWidth, int pixelHeight ) override; virtual int write( const char* s, size_t n ) override; virtual int read( char* buf, size_t n, bool block = false ) override; diff --git a/src/modules/eterm/include/eterm/terminal/terminaldisplay.hpp b/src/modules/eterm/include/eterm/terminal/terminaldisplay.hpp index bf8505f11..12116a1b7 100644 --- a/src/modules/eterm/include/eterm/terminal/terminaldisplay.hpp +++ b/src/modules/eterm/include/eterm/terminal/terminaldisplay.hpp @@ -30,6 +30,8 @@ class VertexBuffer; namespace eterm { namespace Terminal { +class KittyGraphicsRenderer; + enum class TerminalShortcutAction { PASTE, PASTE_SELECTION, @@ -169,6 +171,10 @@ class TerminalDisplay { bool update( bool isMouseOverMe = true ); + Sizei getCellPixelSize() const; + + Sizei getGridPixelSize() const; + void executeFile( const std::string& cmd ); void executeBinary( const std::string& binaryPath, const std::string& args = "" ); @@ -315,6 +321,7 @@ class TerminalDisplay { std::vector mColors; std::shared_ptr mSession; std::shared_ptr mSnapshot; + std::unique_ptr mGraphicsRenderer; mutable std::string mClipboardUtf8; Uint32 mNumCallBacks{ 0 }; std::map mCallbacks; @@ -349,6 +356,8 @@ class TerminalDisplay { Uint32 mRows{ 0 }; Uint32 mClickStep{ 5 }; Uint64 mSnapshotGeneration{ 0 }; + Uint64 mLastAppliedGraphicsSequence{ 0 }; + bool mGraphicsResyncPending{ false }; FontHinting mFontHinting{ FontHinting::Full }; FontAntialiasing mFontAntialiasing{ FontAntialiasing::Grayscale }; FrameBufferUniquePtr mFrameBuffer; @@ -377,12 +386,16 @@ class TerminalDisplay { Vector2i positionToGrid( const Vector2i& pos ); + Vector2i positionToPixel( const Vector2i& pos ) const; + void onSizeChange(); void onProcessExit( int exitCode ); void consumeSnapshot(); + void drainGraphicsUpdates(); + void drainSessionEvents(); TerminalColorPalette makeColorPalette() const; diff --git a/src/modules/eterm/include/eterm/terminal/terminalemulator.hpp b/src/modules/eterm/include/eterm/terminal/terminalemulator.hpp index 1ac34934d..24de7e7c6 100644 --- a/src/modules/eterm/include/eterm/terminal/terminalemulator.hpp +++ b/src/modules/eterm/include/eterm/terminal/terminalemulator.hpp @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -58,6 +59,10 @@ constexpr int ESC_BUF_SIZ = 512; constexpr int ESC_ARG_SIZ = 32; constexpr int STR_BUF_SIZ = ESC_BUF_SIZ; constexpr int STR_ARG_SIZ = ESC_ARG_SIZ; +// Kitty recommends small chunks, but unchunked direct transmissions are valid and common in +// simple clients. Keep this independently bounded while allowing useful image-sized APCs. +constexpr size_t MAX_KITTY_GRAPHICS_APC_SIZE = 16 * 1024 * 1024; +constexpr size_t MAX_GENERIC_STRING_SEQUENCE_SIZE = 1024 * 1024; /* Internal representation of the screen */ struct Term { @@ -119,7 +124,8 @@ struct STREscape { size_t siz; /* allocation size */ size_t len; /* raw string length */ char* args[STR_ARG_SIZ]; - int narg; /* nb of args */ + int narg; /* nb of args */ + bool discarded; /* oversized sequence: consume input through its terminator without storing */ }; enum class PromptState { @@ -156,6 +162,8 @@ class TerminalEmulator final { void resize( int columns, int rows ); + void resize( int columns, int rows, int pixelWidth, int pixelHeight ); + void redraw(); /** Worker-owned terminal state reset (RIS semantics without replacing the PTY/process). */ @@ -220,8 +228,8 @@ class TerminalEmulator final { std::string getSelection() const; - void mousereport( const TerminalMouseEventType& type, const Vector2i& pos, const Uint32& flags, - const Uint32& mod ); + void mousereport( const TerminalMouseEventType& type, const Vector2i& cellPosition, + const Vector2i& pixelPosition, const Uint32& flags, const Uint32& mod ); const bool& isDirty() const { return mDirty; } @@ -260,6 +268,8 @@ class TerminalEmulator final { /** Worker-only notification that the display palette changed. */ void notifyColorSchemeChanged(); + void requestGraphicsResync(); + Vector2i getSize() const; System::IProcess* getProcess() const; @@ -288,6 +298,10 @@ class TerminalEmulator final { bool mPendingPtyResize{ false }; int mPendingPtyColumns{ 0 }; int mPendingPtyRows{ 0 }; + int mPendingPtyPixelWidth{ 0 }; + int mPendingPtyPixelHeight{ 0 }; + int mPixelWidth{ 0 }; + int mPixelHeight{ 0 }; Clock mPendingPtyResizeClock; bool mDirty{ true }; @@ -307,6 +321,7 @@ class TerminalEmulator final { TerminalSelection mSel; CSIEscape mCsiescseq; STREscape mStrescseq; + KittyGraphicsProtocol mKittyGraphics; uint32_t mDefaultFg; uint32_t mDefaultBg; @@ -324,6 +339,16 @@ class TerminalEmulator final { PromptState mPromptState{ PromptState::Unknown }; PromptStateChangedCb mPromptStateChangedCb; DataCb mDataCb; + Vector2i mKittyPlaceholderCell{ -1, -1 }; + struct KittyPlaceholderMetadata { + Uint32 placementId{ 0 }; + Uint16 row{ UINT16_MAX }; + Uint16 column{ UINT16_MAX }; + Uint8 imageIdMsb{ 0 }; + Uint8 diacriticCount{ 0 }; + }; + std::unordered_map mKittyPlaceholderMetadata; + Uint32 mKittyUnderlineColor{ 0 }; void setClipboard( const char* str ); diff --git a/src/modules/eterm/include/eterm/terminal/terminalgraphics.hpp b/src/modules/eterm/include/eterm/terminal/terminalgraphics.hpp new file mode 100644 index 000000000..83b4ab6f0 --- /dev/null +++ b/src/modules/eterm/include/eterm/terminal/terminalgraphics.hpp @@ -0,0 +1,109 @@ +#ifndef ETERM_TERMINALGRAPHICS_HPP +#define ETERM_TERMINALGRAPHICS_HPP + +#include +#include +#include +#include +#include +#include +#include + +using namespace EE; +using namespace EE::Math; + +namespace eterm { namespace Terminal { + +using KittyImageId = Uint32; +using KittyPlacementId = Uint32; + +struct TerminalVisiblePlacement { + KittyImageId imageId{ 0 }; + KittyPlacementId placementId{ 0 }; + Uint32 frameNumber{ 1 }; + Vector2i visibleAnchorCell; + Rect sourcePixels; + Uint32 columns{ 0 }; + Uint32 rows{ 0 }; + Vector2i firstCellPixelOffset; + Int32 zIndex{ 0 }; +}; + +struct TerminalGraphicsPlaceholderCell { + KittyImageId imageId{ 0 }; + KittyPlacementId placementId{ 0 }; + Vector2i cell; + Uint32 imageRow{ 0 }; + Uint32 imageColumn{ 0 }; + + bool operator==( const TerminalGraphicsPlaceholderCell& other ) const { + return imageId == other.imageId && placementId == other.placementId && cell == other.cell && + imageRow == other.imageRow && imageColumn == other.imageColumn; + } +}; + +/** Small immutable graphics state associated with a terminal presentation. */ +struct TerminalGraphicsPresentation { + std::vector placements; + Uint64 generation{ 0 }; + Uint64 requiredUpdateSequence{ 0 }; +}; + +enum class TerminalGraphicsUpdateType : Uint8 { + CreateImage, + ReplaceImage, + UpdateRegion, + CreateFrame, + ReplaceFrame, + UpdateFrameRegion, + DeleteFrame, + DeleteImage, + ResetScreen, + ResetAll, + Resync +}; + +struct TerminalGraphicsUpdate { + std::shared_ptr> rgba; + Sizei imageSize; + Rect region; + Uint64 sequence{ 0 }; + KittyImageId imageId{ 0 }; + Uint32 frameNumber{ 1 }; + TerminalGraphicsUpdateType type{ TerminalGraphicsUpdateType::Resync }; + + size_t payloadBytes() const { return rgba ? rgba->size() : 0; } +}; + +/** Bounded ordered worker-to-UI mutation queue with explicit overflow recovery. */ +class TerminalGraphicsUpdateQueue { + public: + static constexpr size_t DefaultMaxUpdates = 1024; + static constexpr size_t DefaultMaxBytes = 32 * 1024 * 1024; + + explicit TerminalGraphicsUpdateQueue( size_t maxUpdates = DefaultMaxUpdates, + size_t maxBytes = DefaultMaxBytes ); + + Uint64 enqueue( TerminalGraphicsUpdate update ); + + std::vector drain(); + + size_t queuedBytes() const; + + bool needsResync() const; + + void resetResync(); + + private: + mutable std::mutex mMutex; + std::deque mUpdates; + size_t mMaxUpdates{ 0 }; + size_t mMaxBytes{ 0 }; + size_t mQueuedBytes{ 0 }; + Uint64 mNextSequence{ 0 }; + bool mNeedsResync{ false }; +}; + +}} // namespace eterm::Terminal + +#endif diff --git a/src/modules/eterm/include/eterm/terminal/terminalsession.hpp b/src/modules/eterm/include/eterm/terminal/terminalsession.hpp index 59be5bef2..37b27d7cd 100644 --- a/src/modules/eterm/include/eterm/terminal/terminalsession.hpp +++ b/src/modules/eterm/include/eterm/terminal/terminalsession.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,7 @@ namespace eterm { namespace Terminal { /** Immutable worker-to-UI presentation state. Cell selection is already applied as ATTR_REVERSE. */ struct TerminalSnapshot { + std::shared_ptr graphics; std::vector cells; std::vector dirtyRows; std::string title; @@ -118,6 +120,7 @@ class TerminalSession final : public std::enable_shared_from_this snapshot() const; std::vector drainEvents(); + std::vector drainGraphicsUpdates(); + void requestGraphicsResync(); /** Bounded exact-selection request. Returns no value on timeout or during shutdown. */ std::optional @@ -160,6 +165,8 @@ class TerminalSession final : public std::enable_shared_from_this response; }; struct SelectionClearCommand {}; + struct GraphicsResyncCommand {}; struct ResetCommand {}; struct TerminateCommand {}; struct AllowTrimCommand : BoolCommand {}; @@ -216,7 +225,7 @@ class TerminalSession final : public std::enable_shared_from_this; + SelectionRequestCommand, GraphicsResyncCommand>; TerminalSession( PtyPtr&& pty, ProcPtr&& process, size_t historySize, TerminalColorPalette palette ); @@ -228,6 +237,7 @@ class TerminalSession final : public std::enable_shared_from_this snapshot ); + Uint64 enqueueGraphicsUpdate( TerminalGraphicsUpdate update ); std::shared_ptr mWorkerDisplay; std::unique_ptr mEmulator; @@ -238,6 +248,7 @@ class TerminalSession final : public std::enable_shared_from_this mCommands; std::mutex mEventMutex; std::deque mEvents; + TerminalGraphicsUpdateQueue mGraphicsUpdates; mutable std::mutex mPublishedSnapshotMutex; std::shared_ptr mPublishedSnapshot; std::atomic mShutdownRequested{ false }; @@ -246,4 +257,4 @@ class TerminalSession final : public std::enable_shared_from_this, + std::vector ) {} + void ITerminalDisplay::onProcessExit( int /*exitCode*/ ) {} void ITerminalDisplay::onScrollPositionChange() {} diff --git a/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp b/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp new file mode 100644 index 000000000..1e561b0f6 --- /dev/null +++ b/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp @@ -0,0 +1,1559 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace EE::System; + +namespace eterm { namespace Terminal { + +KittyGraphicsProtocol::KittyGraphicsProtocol( size_t maxStorageBytes, size_t maxImages, + size_t maxPlacements ) : + mMaxStorageBytes( maxStorageBytes ), mMaxImages( maxImages ), mMaxPlacements( maxPlacements ) {} + +namespace { + +bool parseUnsigned( std::string_view value, Uint32& result ) { + if ( value.empty() ) + return false; + const char* end = value.data() + value.size(); + auto parsed = std::from_chars( value.data(), end, result ); + return parsed.ec == std::errc{} && parsed.ptr == end; +} + +bool parseSigned( std::string_view value, Int32& result ) { + if ( value.empty() ) + return false; + const char* end = value.data() + value.size(); + auto parsed = std::from_chars( value.data(), end, result ); + return parsed.ec == std::errc{} && parsed.ptr == end; +} + +bool parseCharacter( std::string_view value, char& result ) { + if ( value.size() != 1 ) + return false; + result = value.front(); + return true; +} + +bool checkedPixelBytes( Uint32 width, Uint32 height, size_t channels, size_t& result ) { + if ( width == 0 || height == 0 || width > std::numeric_limits::max() / height ) + return false; + const size_t pixels = static_cast( width ) * height; + if ( pixels > std::numeric_limits::max() / channels ) + return false; + result = pixels * channels; + return true; +} + +bool validBase64( std::string_view input, bool finalChunk ) { + if ( input.size() % 4 == 1 ) + return false; + size_t padding = 0; + for ( size_t i = 0; i < input.size(); ++i ) { + const unsigned char character = input[i]; + const bool alphabet = + ( character >= 'A' && character <= 'Z' ) || ( character >= 'a' && character <= 'z' ) || + ( character >= '0' && character <= '9' ) || character == '+' || character == '/'; + if ( character == '=' ) { + ++padding; + if ( !finalChunk || padding > 2 ) + return false; + } else if ( !alphabet || padding != 0 ) { + return false; + } + } + return finalChunk || input.size() % 4 == 0; +} + +bool decodeBase64( std::string_view input, bool finalChunk, std::vector& output ) { + if ( !validBase64( input, finalChunk ) ) + return false; + std::vector decoded( Base64::decodeSafeOutLen( input.size() ) ); + const size_t decodedSize = Base64::decode( input.size(), input.data(), decoded.size(), + decoded.data(), Base64::DecodeMode::NoWhitespace ); + if ( decodedSize == static_cast( -1 ) ) + return false; + decoded.resize( decodedSize ); + constexpr size_t MaxTransferBytes = 64 * 1024 * 1024; + if ( output.size() > MaxTransferBytes || decoded.size() > MaxTransferBytes - output.size() ) + return false; + output.insert( output.end(), decoded.begin(), decoded.end() ); + return true; +} + +bool placementContains( const TerminalVisiblePlacement& placement, Vector2i cell ) { + const Int64 right = static_cast( placement.visibleAnchorCell.x ) + placement.columns; + const Int64 bottom = static_cast( placement.visibleAnchorCell.y ) + placement.rows; + return cell.x >= placement.visibleAnchorCell.x && cell.y >= placement.visibleAnchorCell.y && + cell.x < right && cell.y < bottom; +} + +} // namespace + +KittyGraphicsParseResult KittyGraphicsProtocol::parse( std::string_view command ) { + KittyGraphicsParseResult result; + KittyGraphicsCommandData data; + char action = 't'; + + const size_t separator = command.find( ';' ); + std::string_view control = command.substr( 0, separator ); + if ( separator != std::string_view::npos ) + data.payload = command.substr( separator + 1 ); + + while ( !control.empty() ) { + const size_t comma = control.find( ',' ); + const std::string_view field = control.substr( 0, comma ); + control = + comma == std::string_view::npos ? std::string_view{} : control.substr( comma + 1 ); + const size_t equals = field.find( '=' ); + if ( equals == std::string_view::npos || equals == 0 || equals + 1 == field.size() ) { + result.error = KittyGraphicsError::InvalidArgument; + return result; + } + + const std::string_view key = field.substr( 0, equals ); + const std::string_view value = field.substr( equals + 1 ); + if ( key.size() != 1 ) + continue; // Unknown future keys are ignored for forward compatibility. + + Uint32 unsignedValue = 0; + bool valid = true; + switch ( key.front() ) { + case 'a': + valid = parseCharacter( value, action ); + break; + case 't': + valid = parseCharacter( value, data.transmission ); + break; + case 'o': + valid = parseCharacter( value, data.compression ); + break; +#define PARSE_UINT_FIELD( name ) \ + valid = parseUnsigned( value, unsignedValue ); \ + if ( valid ) \ + data.name = unsignedValue + case 'f': + PARSE_UINT_FIELD( format ); + break; + case 'S': + PARSE_UINT_FIELD( dataSize ); + break; + case 'm': + PARSE_UINT_FIELD( more ); + valid = valid && unsignedValue <= 1; + break; + case 'i': + PARSE_UINT_FIELD( imageId ); + break; + case 'I': + PARSE_UINT_FIELD( imageNumber ); + break; + case 'N': + PARSE_UINT_FIELD( usageHint ); + break; + case 'p': + PARSE_UINT_FIELD( placementId ); + break; + case 'q': + PARSE_UINT_FIELD( quiet ); + valid = valid && unsignedValue <= 2; + break; + case 's': + PARSE_UINT_FIELD( width ); + break; + case 'v': + PARSE_UINT_FIELD( height ); + break; + case 'x': + PARSE_UINT_FIELD( x ); + break; + case 'y': + PARSE_UINT_FIELD( y ); + break; + case 'w': + PARSE_UINT_FIELD( sourceWidth ); + break; + case 'h': + PARSE_UINT_FIELD( sourceHeight ); + break; + case 'c': + PARSE_UINT_FIELD( columns ); + break; + case 'r': + PARSE_UINT_FIELD( rows ); + break; + case 'X': + PARSE_UINT_FIELD( xOffset ); + break; + case 'Y': + PARSE_UINT_FIELD( yOffset ); + break; + case 'C': + PARSE_UINT_FIELD( cursorMovement ); + valid = valid && unsignedValue <= 1; + break; + case 'U': + PARSE_UINT_FIELD( virtualPlacement ); + valid = valid && unsignedValue <= 1; + break; + case 'P': + PARSE_UINT_FIELD( parentImageId ); + break; + case 'Q': + PARSE_UINT_FIELD( parentPlacementId ); + break; + case 'H': { + Int32 signedValue = 0; + valid = parseSigned( value, signedValue ); + if ( valid ) + data.parentOffsetX = signedValue; + break; + } + case 'V': { + Int32 signedValue = 0; + valid = parseSigned( value, signedValue ); + if ( valid ) + data.parentOffsetY = signedValue; + break; + } + case 'd': + valid = parseCharacter( value, data.deletion ); + break; + case 'z': { + Int32 signedValue = 0; + valid = parseSigned( value, signedValue ); + if ( valid ) + data.zIndex = signedValue; + break; + } + default: + break; + } +#undef PARSE_UINT_FIELD + if ( !valid ) { + result.error = KittyGraphicsError::InvalidArgument; + return result; + } + } + + if ( data.imageId && data.imageNumber ) { + result.error = KittyGraphicsError::InvalidArgument; + return result; + } + + switch ( action ) { + case 't': + result.command = KittyTransmitCommand{ data, false }; + break; + case 'T': + result.command = KittyTransmitCommand{ data, true }; + break; + case 'p': + result.command = KittyPutCommand{ data }; + break; + case 'd': + result.command = KittyDeleteCommand{ data }; + break; + case 'f': + result.command = KittyFrameCommand{ data }; + break; + case 'a': + result.command = KittyAnimationCommand{ data }; + break; + case 'c': + result.command = KittyComposeCommand{ data }; + break; + case 'q': + result.command = KittyQueryCommand{ data }; + break; + default: + result.error = KittyGraphicsError::InvalidArgument; + break; + } + return result; +} + +KittyGraphicsHandleResult KittyGraphicsProtocol::handle( std::string_view command, + Vector2i cursor ) { + auto parsed = parse( command ); + if ( !parsed.command ) + return { {}, parsed.error, false }; + + return std::visit( + [this, cursor]( const auto& value ) -> KittyGraphicsHandleResult { + using T = std::decay_t; + if constexpr ( !std::is_same_v && + !std::is_same_v && + !std::is_same_v && + !std::is_same_v ) { + if ( mPending.active ) { + mPending = {}; + return { response( value.data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + } + } + if constexpr ( std::is_same_v ) { + return handleTransmit( value.data, value.display, false, false, cursor ); + } else if constexpr ( std::is_same_v ) { + return handleTransmit( value.data, false, true, false, cursor ); + } else if constexpr ( std::is_same_v ) { + return handleTransmit( value.data, false, false, true, cursor ); + } else if constexpr ( std::is_same_v ) { + return put( value.data, cursor ); + } else if constexpr ( std::is_same_v ) { + return remove( value.data, cursor ); + } else if constexpr ( std::is_same_v ) { + return controlAnimation( value.data ); + } else if constexpr ( std::is_same_v ) { + return composeFrames( value.data ); + } else { + return { response( value.data, KittyGraphicsError::Unsupported ), + KittyGraphicsError::Unsupported, false }; + } + }, + *parsed.command ); +} + +KittyGraphicsHandleResult +KittyGraphicsProtocol::handleTransmit( const KittyGraphicsCommandData& data, bool display, + bool query, bool frame, Vector2i cursor ) { + const bool more = data.more.value_or( 0 ) != 0; + if ( mPending.active ) { + if ( frame != mPending.frame || data.format || data.dataSize || data.imageId || + data.imageNumber || data.usageHint || data.placementId || data.width || data.height || + data.x || data.y || data.sourceWidth || data.sourceHeight || data.columns || + data.rows || data.xOffset || data.yOffset || data.zIndex || data.cursorMovement || + data.virtualPlacement || data.parentImageId || data.parentPlacementId || + data.parentOffsetX || data.parentOffsetY || data.compression != 0 || + data.transmission != 'd' ) { + mPending = {}; + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + } + if ( !decodeBase64( data.payload, !more, mPending.decodedData ) ) { + mPending = {}; + return { response( data, KittyGraphicsError::InvalidData ), + KittyGraphicsError::InvalidData, false }; + } + if ( more ) + return {}; + auto transfer = std::move( mPending ); + mPending = {}; + return finishTransfer( std::move( transfer ), cursor ); + } + + if ( data.transmission != 'd' ) + return { response( data, KittyGraphicsError::Unsupported ), KittyGraphicsError::Unsupported, + false }; + PendingTransfer transfer; + transfer.data = data; + transfer.data.payload = {}; + transfer.display = display; + transfer.query = query; + transfer.frame = frame; + transfer.active = true; + if ( !decodeBase64( data.payload, !more, transfer.decodedData ) ) + return { response( data, KittyGraphicsError::InvalidData ), KittyGraphicsError::InvalidData, + false }; + if ( more ) { + mPending = std::move( transfer ); + return {}; + } + return finishTransfer( std::move( transfer ), cursor ); +} + +KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer transfer, + Vector2i cursor ) { + const auto& data = transfer.data; + mStats.decodedBytes += transfer.decodedData.size(); + const Uint32 format = data.format.value_or( 32 ); + if ( format != 24 && format != 32 && format != 100 ) + return { response( data, KittyGraphicsError::Unsupported ), KittyGraphicsError::Unsupported, + false }; + + Uint32 width = data.width.value_or( 0 ); + Uint32 height = data.height.value_or( 0 ); + std::vector pixels; + if ( format == 100 ) { + std::vector encoded; + if ( data.compression == 'z' ) { + if ( !data.dataSize || *data.dataSize == 0 || *data.dataSize > 64 * 1024 * 1024 ) + return { response( data, KittyGraphicsError::TooLarge ), + KittyGraphicsError::TooLarge, false }; + encoded.resize( *data.dataSize ); + IOStreamMemory source( reinterpret_cast( transfer.decodedData.data() ), + transfer.decodedData.size() ); + IOStreamMemory destination( reinterpret_cast( encoded.data() ), encoded.size() ); + if ( Compression::decompress( destination, source ) != Compression::OK || + static_cast( destination.tell() ) != encoded.size() ) { + return { response( data, KittyGraphicsError::DecodeFailed ), + KittyGraphicsError::DecodeFailed, false }; + } + } else if ( data.compression == 0 ) { + encoded = std::move( transfer.decodedData ); + } else { + return { response( data, KittyGraphicsError::Unsupported ), + KittyGraphicsError::Unsupported, false }; + } + if ( encoded.size() < 24 || std::memcmp( encoded.data(), "\x89PNG\r\n\x1a\n", 8 ) != 0 || + std::memcmp( encoded.data() + 12, "IHDR", 4 ) != 0 ) { + return { response( data, KittyGraphicsError::InvalidData ), + KittyGraphicsError::InvalidData, false }; + } + width = ( static_cast( encoded[16] ) << 24 ) | + ( static_cast( encoded[17] ) << 16 ) | + ( static_cast( encoded[18] ) << 8 ) | encoded[19]; + height = ( static_cast( encoded[20] ) << 24 ) | + ( static_cast( encoded[21] ) << 16 ) | + ( static_cast( encoded[22] ) << 8 ) | encoded[23]; + size_t rgbaBytes = 0; + if ( !checkedPixelBytes( width, height, 4, rgbaBytes ) || rgbaBytes > 128 * 1024 * 1024 ) { + return { response( data, KittyGraphicsError::TooLarge ), KittyGraphicsError::TooLarge, + false }; + } + EE::Graphics::Image decoded( encoded.data(), static_cast( encoded.size() ), + 4 ); + if ( !decoded.getPixelsPtr() || decoded.getWidth() != width || + decoded.getHeight() != height ) + return { response( data, KittyGraphicsError::DecodeFailed ), + KittyGraphicsError::DecodeFailed, false }; + pixels.assign( decoded.getPixelsPtr(), decoded.getPixelsPtr() + rgbaBytes ); + } else { + if ( width == 0 || height == 0 ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + size_t sourceBytes = 0; + if ( !checkedPixelBytes( width, height, format == 24 ? 3 : 4, sourceBytes ) || + sourceBytes > 128 * 1024 * 1024 ) { + return { response( data, KittyGraphicsError::TooLarge ), KittyGraphicsError::TooLarge, + false }; + } + if ( data.compression == 'z' ) { + pixels.resize( sourceBytes ); + IOStreamMemory source( reinterpret_cast( transfer.decodedData.data() ), + transfer.decodedData.size() ); + IOStreamMemory destination( reinterpret_cast( pixels.data() ), pixels.size() ); + if ( Compression::decompress( destination, source ) != Compression::OK || + static_cast( destination.tell() ) != sourceBytes ) { + return { response( data, KittyGraphicsError::DecodeFailed ), + KittyGraphicsError::DecodeFailed, false }; + } + } else if ( data.compression != 0 ) { + return { response( data, KittyGraphicsError::Unsupported ), + KittyGraphicsError::Unsupported, false }; + } else { + if ( transfer.decodedData.size() != sourceBytes ) + return { response( data, KittyGraphicsError::InvalidData ), + KittyGraphicsError::InvalidData, false }; + pixels = std::move( transfer.decodedData ); + } + } + + if ( format == 24 ) { + std::vector rgba; + rgba.resize( static_cast( width ) * height * 4 ); + for ( size_t sourceOffset = 0, destinationOffset = 0; sourceOffset < pixels.size(); + sourceOffset += 3, destinationOffset += 4 ) { + rgba[destinationOffset] = pixels[sourceOffset]; + rgba[destinationOffset + 1] = pixels[sourceOffset + 1]; + rgba[destinationOffset + 2] = pixels[sourceOffset + 2]; + rgba[destinationOffset + 3] = 255; + } + pixels = std::move( rgba ); + } + if ( transfer.frame ) { + const KittyImageId imageId = resolveImageId( data ); + auto image = mImages.find( imageId ); + if ( image == mImages.end() ) + return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, + false }; + Uint32 frameNumber = data.rows.value_or( 0 ); + if ( frameNumber == 0 ) { + frameNumber = 2; + for ( const auto& frame : image->second.frames ) + frameNumber = std::max( frameNumber, frame.first + 1 ); + } + const Uint32 destinationX = data.x.value_or( 0 ); + const Uint32 destinationY = data.y.value_or( 0 ); + if ( destinationX >= static_cast( image->second.size.getWidth() ) || + destinationY >= static_cast( image->second.size.getHeight() ) || + width > image->second.size.getWidth() - destinationX || + height > image->second.size.getHeight() - destinationY ) { + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + } + + std::vector* destinationPixels = nullptr; + bool createdFrame = false; + if ( frameNumber == 1 ) { + destinationPixels = &image->second.rgba; + } else { + auto frame = image->second.frames.find( frameNumber ); + if ( frame == image->second.frames.end() ) { + Image::Frame newFrame; + newFrame.rgba.resize( static_cast( image->second.size.getWidth() ) * + image->second.size.getHeight() * 4 ); + if ( data.columns ) { + const Uint32 baseFrame = *data.columns; + if ( baseFrame == 1 ) + newFrame.rgba = image->second.rgba; + else { + auto base = image->second.frames.find( baseFrame ); + if ( base == image->second.frames.end() ) + return { response( data, KittyGraphicsError::NotFound ), + KittyGraphicsError::NotFound, false }; + newFrame.rgba = base->second.rgba; + } + } else if ( data.yOffset ) { + const Uint32 color = *data.yOffset; + for ( size_t offset = 0; offset < newFrame.rgba.size(); offset += 4 ) { + newFrame.rgba[offset] = static_cast( color >> 24 ); + newFrame.rgba[offset + 1] = static_cast( color >> 16 ); + newFrame.rgba[offset + 2] = static_cast( color >> 8 ); + newFrame.rgba[offset + 3] = static_cast( color ); + } + } + const size_t maxFrameStorage = + mMaxStorageBytes > std::numeric_limits::max() / 5 + ? std::numeric_limits::max() + : mMaxStorageBytes * 5; + if ( mFrameStorageBytes > maxFrameStorage || + newFrame.rgba.size() > maxFrameStorage - mFrameStorageBytes ) + return { response( data, KittyGraphicsError::NoSpace ), + KittyGraphicsError::NoSpace, false }; + newFrame.gapMs = data.zIndex.value_or( 40 ); + newFrame.usageHint = data.usageHint.value_or( 0 ); + mFrameStorageBytes += newFrame.rgba.size(); + frame = image->second.frames.emplace( frameNumber, std::move( newFrame ) ).first; + createdFrame = true; + } else { + if ( data.zIndex && *data.zIndex != 0 ) + frame->second.gapMs = *data.zIndex; + } + destinationPixels = &frame->second.rgba; + } + + const bool replace = data.xOffset.value_or( 0 ) == 1; + std::vector finalPatch( pixels.size() ); + const size_t imageStride = static_cast( image->second.size.getWidth() ) * 4; + const size_t patchStride = static_cast( width ) * 4; + for ( Uint32 row = 0; row < height; ++row ) { + Uint8* destination = destinationPixels->data() + + static_cast( destinationY + row ) * imageStride + + static_cast( destinationX ) * 4; + const Uint8* source = pixels.data() + static_cast( row ) * patchStride; + Uint8* published = finalPatch.data() + static_cast( row ) * patchStride; + if ( replace ) { + std::memcpy( destination, source, patchStride ); + std::memcpy( published, source, patchStride ); + continue; + } + for ( size_t column = 0; column < patchStride; column += 4 ) { + const Uint32 alpha = source[column + 3]; + const Uint32 inverseAlpha = 255 - alpha; + for ( size_t channel = 0; channel < 3; ++channel ) + destination[column + channel] = + static_cast( ( source[column + channel] * alpha + + destination[column + channel] * inverseAlpha + 127 ) / + 255 ); + destination[column + 3] = static_cast( + alpha + ( destination[column + 3] * inverseAlpha + 127 ) / 255 ); + std::memcpy( published + column, destination + column, 4 ); + } + } + TerminalGraphicsUpdate update; + update.type = frameNumber == 1 ? TerminalGraphicsUpdateType::UpdateRegion + : createdFrame ? TerminalGraphicsUpdateType::CreateFrame + : TerminalGraphicsUpdateType::UpdateFrameRegion; + update.imageId = imageId; + update.frameNumber = frameNumber; + update.imageSize = image->second.size; + update.region = + createdFrame + ? Rect( 0, 0, image->second.size.getWidth(), image->second.size.getHeight() ) + : Rect( destinationX, destinationY, destinationX + width, destinationY + height ); + update.rgba = createdFrame + ? std::make_shared>( *destinationPixels ) + : std::make_shared>( std::move( finalPatch ) ); + if ( !createdFrame && !mUpdates.empty() && mUpdates.back().type == update.type && + mUpdates.back().imageId == update.imageId && + mUpdates.back().frameNumber == update.frameNumber && + mUpdates.back().region == update.region ) + mUpdates.back() = std::move( update ); + else + mUpdates.emplace_back( std::move( update ) ); + ++mStats.rectangleUpdates; + mPresentationDirty = true; + return { response( data, KittyGraphicsError::None, imageId ), KittyGraphicsError::None, + true }; + } + + const bool anonymous = !data.imageId && !data.imageNumber; + KittyImageId imageId = data.imageId.value_or( 0 ); + if ( anonymous && transfer.display ) { + auto placement = std::find_if( + mPlacements.begin(), mPlacements.end(), [&]( const Placement& candidate ) { + auto candidateImage = mImages.find( candidate.visible.imageId ); + return candidate.visible.visibleAnchorCell == cursor && + candidate.visible.placementId == 0 && candidateImage != mImages.end() && + candidateImage->second.anonymous; + } ); + if ( placement != mPlacements.end() ) + imageId = placement->visible.imageId; + } + if ( imageId == 0 ) + imageId = allocateImageId(); + if ( imageId == 0 ) + return { response( data, KittyGraphicsError::NoSpace ), KittyGraphicsError::NoSpace, + false }; + if ( transfer.query ) + return { response( data, KittyGraphicsError::None, imageId ), KittyGraphicsError::None, + false }; + + auto existing = mImages.find( imageId ); + const size_t oldBytes = existing == mImages.end() ? 0 : existing->second.rgba.size(); + if ( !ensureCapacity( pixels.size(), imageId, existing == mImages.end() ) ) + return { response( data, KittyGraphicsError::NoSpace ), KittyGraphicsError::NoSpace, + false }; + existing = mImages.find( imageId ); + if ( existing != mImages.end() ) { + for ( const auto& frame : existing->second.frames ) + mFrameStorageBytes -= frame.second.rgba.size(); + mPlacements.erase( std::remove_if( mPlacements.begin(), mPlacements.end(), + [imageId]( const Placement& p ) { + return p.visible.imageId == imageId; + } ), + mPlacements.end() ); + mPrimaryPlacements.erase( std::remove_if( mPrimaryPlacements.begin(), + mPrimaryPlacements.end(), + [imageId]( const Placement& p ) { + return p.visible.imageId == imageId; + } ), + mPrimaryPlacements.end() ); + } + Image image; + image.size = Sizei( width, height ); + image.imageNumber = data.imageNumber.value_or( 0 ); + image.usageHint = data.usageHint.value_or( 0 ); + image.anonymous = anonymous; + image.creationSerial = ++mCreationSerial; + image.rgba = std::move( pixels ); + mStorageBytes = mStorageBytes - oldBytes + image.rgba.size(); + const bool replaced = existing != mImages.end(); + auto inserted = mImages.insert_or_assign( imageId, std::move( image ) ).first; + + TerminalGraphicsUpdate update; + update.type = replaced ? TerminalGraphicsUpdateType::ReplaceImage + : TerminalGraphicsUpdateType::CreateImage; + update.imageId = imageId; + update.imageSize = inserted->second.size; + update.region = + Rect( 0, 0, inserted->second.size.getWidth(), inserted->second.size.getHeight() ); + update.rgba = std::make_shared>( inserted->second.rgba ); + mUpdates.emplace_back( std::move( update ) ); + ++mStats.fullImageUpdates; + ++mPresentationGeneration; + mPresentationDirty = true; + auto result = KittyGraphicsHandleResult{ + response( data, KittyGraphicsError::None, anonymous ? 0 : imageId ), + KittyGraphicsError::None, true }; + if ( transfer.display ) { + auto placementData = data; + placementData.imageId = imageId; + placementData.imageNumber.reset(); + auto placementResult = put( placementData, cursor ); + if ( placementResult.error != KittyGraphicsError::None ) + return placementResult; + result.cursorMovement = placementResult.cursorMovement; + } + return result; +} + +KittyGraphicsHandleResult KittyGraphicsProtocol::put( const KittyGraphicsCommandData& data, + Vector2i cursor ) { + const KittyImageId imageId = resolveImageId( data ); + auto image = mImages.find( imageId ); + if ( image == mImages.end() ) + return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, + false }; + const Uint32 sourceX = data.x.value_or( 0 ); + const Uint32 sourceY = data.y.value_or( 0 ); + if ( sourceX >= static_cast( image->second.size.getWidth() ) || + sourceY >= static_cast( image->second.size.getHeight() ) ) { + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + } + const Uint32 sourceWidth = + std::min( data.sourceWidth.value_or( image->second.size.getWidth() - sourceX ), + image->second.size.getWidth() - sourceX ); + const Uint32 sourceHeight = + std::min( data.sourceHeight.value_or( image->second.size.getHeight() - sourceY ), + image->second.size.getHeight() - sourceY ); + if ( sourceWidth == 0 || sourceHeight == 0 ) { + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + } + + TerminalVisiblePlacement visible; + visible.imageId = imageId; + visible.placementId = data.placementId.value_or( 0 ); + visible.visibleAnchorCell = cursor; + const bool relative = data.parentImageId.has_value() || data.parentPlacementId.has_value(); + if ( relative ) { + if ( data.virtualPlacement.value_or( 0 ) == 1 ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + if ( !data.parentImageId || !data.parentPlacementId ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + auto parent = + std::find_if( mPlacements.begin(), mPlacements.end(), [&]( const Placement& p ) { + return p.visible.imageId == *data.parentImageId && + p.visible.placementId == *data.parentPlacementId; + } ); + if ( parent == mPlacements.end() ) + return { response( data, KittyGraphicsError::NoParent ), KittyGraphicsError::NoParent, + false }; + KittyImageId ancestorImage = parent->visible.imageId; + KittyPlacementId ancestorPlacement = parent->visible.placementId; + for ( size_t depth = 0; ancestorImage != 0; ++depth ) { + if ( ancestorImage == imageId && ancestorPlacement == visible.placementId ) + return { response( data, KittyGraphicsError::Cycle ), KittyGraphicsError::Cycle, + false }; + if ( depth >= 64 ) + return { response( data, KittyGraphicsError::TooDeep ), KittyGraphicsError::TooDeep, + false }; + auto ancestor = std::find_if( + mPlacements.begin(), mPlacements.end(), [&]( const Placement& placement ) { + return placement.visible.imageId == ancestorImage && + placement.visible.placementId == ancestorPlacement; + } ); + if ( ancestor == mPlacements.end() || ancestor->parentImageId == 0 ) + break; + ancestorImage = ancestor->parentImageId; + ancestorPlacement = ancestor->parentPlacementId; + } + visible.visibleAnchorCell = + parent->visible.visibleAnchorCell + + Vector2i( data.parentOffsetX.value_or( 0 ), data.parentOffsetY.value_or( 0 ) ); + } + visible.sourcePixels = Rect( sourceX, sourceY, sourceX + sourceWidth, sourceY + sourceHeight ); + visible.columns = data.columns.value_or( 0 ); + visible.rows = data.rows.value_or( 0 ); + if ( visible.columns == 0 && visible.rows == 0 && mCellPixelWidth && mCellPixelHeight ) { + visible.columns = + static_cast( ( static_cast( sourceWidth ) + data.xOffset.value_or( 0 ) + + mCellPixelWidth - 1 ) / + mCellPixelWidth ); + visible.rows = static_cast( ( static_cast( sourceHeight ) + + data.yOffset.value_or( 0 ) + mCellPixelHeight - 1 ) / + mCellPixelHeight ); + } else if ( visible.columns != 0 && visible.rows == 0 && mCellPixelHeight ) { + const Uint64 scaledHeight = + static_cast( sourceHeight ) * visible.columns * mCellPixelWidth; + const Uint64 denominator = static_cast( sourceWidth ) * mCellPixelHeight; + visible.rows = static_cast( ( scaledHeight + denominator - 1 ) / denominator ); + } else if ( visible.rows != 0 && visible.columns == 0 && mCellPixelWidth ) { + const Uint64 scaledWidth = + static_cast( sourceWidth ) * visible.rows * mCellPixelHeight; + const Uint64 denominator = static_cast( sourceHeight ) * mCellPixelWidth; + visible.columns = static_cast( ( scaledWidth + denominator - 1 ) / denominator ); + } + if ( visible.columns == 0 ) + visible.columns = 1; + if ( visible.rows == 0 ) + visible.rows = 1; + visible.firstCellPixelOffset = + Vector2i( data.xOffset.value_or( 0 ), data.yOffset.value_or( 0 ) ); + if ( ( mCellPixelWidth && data.xOffset.value_or( 0 ) >= mCellPixelWidth ) || + ( mCellPixelHeight && data.yOffset.value_or( 0 ) >= mCellPixelHeight ) ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + visible.zIndex = data.zIndex.value_or( 0 ); + if ( visible.columns == 0 || visible.rows == 0 ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + bool replacingPlacement = false; + if ( visible.placementId != 0 ) { + for ( auto it = mPlacements.begin(); it != mPlacements.end(); ++it ) { + if ( it->visible.imageId == imageId && + it->visible.placementId == visible.placementId ) { + const Vector2i movement = visible.visibleAnchorCell - it->visible.visibleAnchorCell; + std::vector> movedParents{ + { imageId, visible.placementId } }; + for ( size_t parentIndex = 0; parentIndex < movedParents.size(); ++parentIndex ) { + for ( auto& child : mPlacements ) { + if ( child.parentImageId == movedParents[parentIndex].first && + child.parentPlacementId == movedParents[parentIndex].second ) { + child.visible.visibleAnchorCell += movement; + movedParents.emplace_back( child.visible.imageId, + child.visible.placementId ); + } + } + } + mPlacements.erase( it ); + replacingPlacement = true; + break; + } + } + } + if ( !replacingPlacement && mPlacements.size() >= mMaxPlacements ) + return { response( data, KittyGraphicsError::NoSpace ), KittyGraphicsError::NoSpace, + false }; + mPlacements.push_back( { visible, ++mPlacementSerial, data.virtualPlacement.value_or( 0 ) == 1, + data.parentImageId.value_or( 0 ), + data.parentPlacementId.value_or( 0 ) } ); + ++mPresentationGeneration; + mPresentationDirty = true; + KittyGraphicsHandleResult result{ response( data, KittyGraphicsError::None, imageId ), + KittyGraphicsError::None, true }; + if ( data.cursorMovement.value_or( 0 ) == 0 && !relative && + data.virtualPlacement.value_or( 0 ) == 0 ) + result.cursorMovement = Vector2i( visible.columns, visible.rows ); + return result; +} + +KittyGraphicsHandleResult KittyGraphicsProtocol::remove( const KittyGraphicsCommandData& data, + Vector2i cursor ) { + mPending = {}; + const char selector = data.deletion; + const bool deleteImageData = selector >= 'A' && selector <= 'Z'; + const char normalized = deleteImageData ? static_cast( selector - 'A' + 'a' ) : selector; + constexpr std::string_view ValidSelectors = "acfinpqrxyz"; + if ( ValidSelectors.find( normalized ) == std::string_view::npos ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + + const KittyImageId selectedImage = resolveImageId( data ); + if ( ( normalized == 'f' || normalized == 'i' || normalized == 'n' ) && selectedImage == 0 ) + return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, + false }; + if ( normalized == 'f' ) { + auto image = mImages.find( selectedImage ); + if ( image == mImages.end() ) + return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, + false }; + const bool changed = !image->second.frames.empty(); + for ( const auto& frame : image->second.frames ) { + mFrameStorageBytes -= frame.second.rgba.size(); + TerminalGraphicsUpdate update; + update.type = TerminalGraphicsUpdateType::DeleteFrame; + update.imageId = selectedImage; + update.frameNumber = frame.first; + mUpdates.emplace_back( std::move( update ) ); + } + image->second.frames.clear(); + image->second.currentFrame = 1; + image->second.animationState = 1; + if ( changed ) { + ++mPresentationGeneration; + mPresentationDirty = true; + } + return { response( data, KittyGraphicsError::None, selectedImage ), + KittyGraphicsError::None, changed }; + } + + Vector2i selectedCell = cursor; + if ( normalized == 'p' || normalized == 'q' ) { + if ( !data.x || !data.y || *data.x == 0 || *data.y == 0 ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + selectedCell = + Vector2i( static_cast( *data.x - 1 ), static_cast( *data.y - 1 ) ); + } + if ( normalized == 'x' && ( !data.x || *data.x == 0 ) ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + if ( normalized == 'y' && ( !data.y || *data.y == 0 ) ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + if ( normalized == 'r' && ( !data.x || !data.y || *data.x > *data.y ) ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + if ( normalized == 'z' && !data.zIndex ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + + std::vector affectedImages; + auto matches = [&]( const Placement& placement ) { + auto visible = placement.visible; + visible.visibleAnchorCell.y += mScrollPosition; + if ( placement.virtualPlacement && normalized != 'i' && normalized != 'n' && + normalized != 'r' ) + return false; + bool match = false; + switch ( normalized ) { + case 'a': + match = static_cast( visible.visibleAnchorCell.y ) + visible.rows > 0 && + ( mScreenRows == 0 || visible.visibleAnchorCell.y < mScreenRows ); + break; + case 'c': + match = placementContains( visible, cursor ); + break; + case 'i': + case 'n': + match = visible.imageId == selectedImage; + if ( match && data.placementId ) + match = visible.placementId == *data.placementId; + break; + case 'p': + match = placementContains( visible, selectedCell ); + break; + case 'q': + match = placementContains( visible, selectedCell ) && data.zIndex && + visible.zIndex == *data.zIndex; + break; + case 'r': + match = visible.imageId >= *data.x && visible.imageId <= *data.y; + break; + case 'x': + match = placementContains( visible, Vector2i( static_cast( *data.x - 1 ), + visible.visibleAnchorCell.y ) ); + break; + case 'y': + match = placementContains( visible, Vector2i( visible.visibleAnchorCell.x, + static_cast( *data.y - 1 ) ) ); + break; + case 'z': + match = visible.zIndex == *data.zIndex; + break; + } + if ( match ) + affectedImages.emplace_back( visible.imageId ); + return match; + }; + const size_t oldPlacementCount = mPlacements.size(); + mPlacements.erase( std::remove_if( mPlacements.begin(), mPlacements.end(), matches ), + mPlacements.end() ); + std::vector orphanedRelativeImages; + bool removedChild = true; + while ( removedChild ) { + removedChild = false; + mPlacements.erase( + std::remove_if( + mPlacements.begin(), mPlacements.end(), + [&]( const Placement& placement ) { + if ( placement.parentImageId == 0 ) + return false; + const bool parentExists = std::any_of( + mPlacements.begin(), mPlacements.end(), [&]( const Placement& parent ) { + return parent.visible.imageId == placement.parentImageId && + parent.visible.placementId == placement.parentPlacementId; + } ); + if ( !parentExists ) { + affectedImages.emplace_back( placement.visible.imageId ); + orphanedRelativeImages.emplace_back( placement.visible.imageId ); + removedChild = true; + } + return !parentExists; + } ), + mPlacements.end() ); + } + std::sort( orphanedRelativeImages.begin(), orphanedRelativeImages.end() ); + orphanedRelativeImages.erase( + std::unique( orphanedRelativeImages.begin(), orphanedRelativeImages.end() ), + orphanedRelativeImages.end() ); + for ( KittyImageId imageId : orphanedRelativeImages ) { + if ( !isImagePlaced( imageId ) ) + eraseImage( imageId ); + } + + bool deletedAnyImage = false; + if ( deleteImageData ) { + if ( normalized == 'a' ) { + for ( const auto& image : mImages ) + affectedImages.emplace_back( image.first ); + } else if ( normalized == 'i' || normalized == 'n' ) { + affectedImages.emplace_back( selectedImage ); + } else if ( normalized == 'r' ) { + for ( const auto& image : mImages ) { + if ( image.first >= *data.x && image.first <= *data.y ) + affectedImages.emplace_back( image.first ); + } + } + std::sort( affectedImages.begin(), affectedImages.end() ); + affectedImages.erase( std::unique( affectedImages.begin(), affectedImages.end() ), + affectedImages.end() ); + for ( KittyImageId imageId : affectedImages ) { + if ( isImagePlaced( imageId ) ) + continue; + auto image = mImages.find( imageId ); + if ( image == mImages.end() ) + continue; + eraseImage( imageId ); + deletedAnyImage = true; + } + } + + const bool changed = oldPlacementCount != mPlacements.size() || deletedAnyImage; + if ( changed ) { + ++mPresentationGeneration; + mPresentationDirty = true; + } + return { response( data, KittyGraphicsError::None, selectedImage ), KittyGraphicsError::None, + changed }; +} + +KittyGraphicsHandleResult +KittyGraphicsProtocol::controlAnimation( const KittyGraphicsCommandData& data ) { + const KittyImageId imageId = resolveImageId( data ); + auto image = mImages.find( imageId ); + if ( image == mImages.end() ) + return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, + false }; + bool changed = false; + if ( data.columns ) { + const Uint32 frameNumber = *data.columns; + if ( frameNumber == 0 || ( frameNumber != 1 && image->second.frames.find( frameNumber ) == + image->second.frames.end() ) ) + return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, + false }; + changed = image->second.currentFrame != frameNumber; + image->second.currentFrame = frameNumber; + } + if ( data.width ) { + if ( *data.width < 1 || *data.width > 3 ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + image->second.animationState = static_cast( *data.width ); + image->second.frameClock.restart(); + if ( *data.width == 1 ) + image->second.loopsCompleted = 0; + changed = true; + } + if ( data.height && *data.height != 0 ) { + image->second.loopCount = *data.height; + changed = true; + } + if ( data.zIndex && *data.zIndex != 0 ) { + const Uint32 frameNumber = data.rows.value_or( image->second.currentFrame ); + if ( frameNumber == 1 ) + image->second.rootGapMs = *data.zIndex; + else { + auto frame = image->second.frames.find( frameNumber ); + if ( frame == image->second.frames.end() ) + return { response( data, KittyGraphicsError::NotFound ), + KittyGraphicsError::NotFound, false }; + frame->second.gapMs = *data.zIndex; + } + changed = true; + } + if ( changed ) { + ++mPresentationGeneration; + mPresentationDirty = true; + } + return { response( data, KittyGraphicsError::None, imageId ), KittyGraphicsError::None, + changed }; +} + +bool KittyGraphicsProtocol::updateAnimations() { + bool changed = false; + for ( auto& entry : mImages ) { + auto& image = entry.second; + if ( image.animationState < 2 || image.frames.empty() ) + continue; + Int32 gap = image.rootGapMs; + if ( image.currentFrame != 1 ) { + auto current = image.frames.find( image.currentFrame ); + if ( current == image.frames.end() ) + continue; + gap = current->second.gapMs; + } + if ( gap > 0 && image.frameClock.getElapsedTime().asMilliseconds() < gap ) + continue; + Uint32 nextFrame = std::numeric_limits::max(); + for ( const auto& frame : image.frames ) { + if ( frame.first > image.currentFrame && frame.first < nextFrame ) + nextFrame = frame.first; + } + if ( nextFrame == std::numeric_limits::max() ) { + if ( image.animationState == 2 ) + continue; + ++image.loopsCompleted; + if ( image.loopCount > 1 && image.loopsCompleted >= image.loopCount - 1 ) { + image.animationState = 1; + continue; + } + nextFrame = 1; + } + image.currentFrame = nextFrame; + image.frameClock.restart(); + changed = true; + } + if ( changed ) { + ++mPresentationGeneration; + mPresentationDirty = true; + } + return changed; +} + +void KittyGraphicsProtocol::setCellPixelSize( Uint32 width, Uint32 height ) { + mCellPixelWidth = width; + mCellPixelHeight = height; +} + +void KittyGraphicsProtocol::setPlaceholderCells( + std::vector cells ) { + if ( mPlaceholderCells == cells ) + return; + mPlaceholderCells = std::move( cells ); + ++mPresentationGeneration; + mPresentationDirty = true; +} + +KittyGraphicsHandleResult +KittyGraphicsProtocol::composeFrames( const KittyGraphicsCommandData& data ) { + const KittyImageId imageId = resolveImageId( data ); + auto image = mImages.find( imageId ); + if ( image == mImages.end() || !data.columns || !data.rows ) + return { response( data, image == mImages.end() ? KittyGraphicsError::NotFound + : KittyGraphicsError::InvalidArgument ), + image == mImages.end() ? KittyGraphicsError::NotFound + : KittyGraphicsError::InvalidArgument, + false }; + auto pixelsFor = [&]( Uint32 frameNumber ) -> std::vector* { + if ( frameNumber == 1 ) + return &image->second.rgba; + auto frame = image->second.frames.find( frameNumber ); + return frame == image->second.frames.end() ? nullptr : &frame->second.rgba; + }; + const Uint32 sourceFrame = *data.rows; + const Uint32 destinationFrame = *data.columns; + auto* source = pixelsFor( sourceFrame ); + auto* destination = pixelsFor( destinationFrame ); + if ( !source || !destination ) + return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, + false }; + const Uint32 destinationX = data.x.value_or( 0 ); + const Uint32 destinationY = data.y.value_or( 0 ); + const Uint32 sourceX = data.xOffset.value_or( 0 ); + const Uint32 sourceY = data.yOffset.value_or( 0 ); + const Uint32 width = data.sourceWidth.value_or( image->second.size.getWidth() ); + const Uint32 height = data.sourceHeight.value_or( image->second.size.getHeight() ); + const Uint32 imageWidth = image->second.size.getWidth(); + const Uint32 imageHeight = image->second.size.getHeight(); + if ( width == 0 || height == 0 || sourceX > imageWidth || sourceY > imageHeight || + destinationX > imageWidth || destinationY > imageHeight || width > imageWidth - sourceX || + height > imageHeight - sourceY || width > imageWidth - destinationX || + height > imageHeight - destinationY ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + if ( sourceFrame == destinationFrame && sourceX < destinationX + width && + destinationX < sourceX + width && sourceY < destinationY + height && + destinationY < sourceY + height ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + + std::vector sourceCopy( static_cast( width ) * height * 4 ); + std::vector result( sourceCopy.size() ); + const size_t imageStride = static_cast( imageWidth ) * 4; + const size_t rowBytes = static_cast( width ) * 4; + for ( Uint32 row = 0; row < height; ++row ) + std::memcpy( sourceCopy.data() + static_cast( row ) * rowBytes, + source->data() + static_cast( sourceY + row ) * imageStride + + static_cast( sourceX ) * 4, + rowBytes ); + const bool replace = data.cursorMovement.value_or( 0 ) == 1; + for ( Uint32 row = 0; row < height; ++row ) { + Uint8* target = destination->data() + + static_cast( destinationY + row ) * imageStride + + static_cast( destinationX ) * 4; + const Uint8* overlay = sourceCopy.data() + static_cast( row ) * rowBytes; + Uint8* published = result.data() + static_cast( row ) * rowBytes; + for ( size_t offset = 0; offset < rowBytes; offset += 4 ) { + if ( replace ) { + std::memcpy( target + offset, overlay + offset, 4 ); + } else { + const Uint32 alpha = overlay[offset + 3]; + const Uint32 inverseAlpha = 255 - alpha; + for ( size_t channel = 0; channel < 3; ++channel ) + target[offset + channel] = + static_cast( ( overlay[offset + channel] * alpha + + target[offset + channel] * inverseAlpha + 127 ) / + 255 ); + target[offset + 3] = + static_cast( alpha + ( target[offset + 3] * inverseAlpha + 127 ) / 255 ); + } + std::memcpy( published + offset, target + offset, 4 ); + } + } + TerminalGraphicsUpdate update; + update.type = destinationFrame == 1 ? TerminalGraphicsUpdateType::UpdateRegion + : TerminalGraphicsUpdateType::UpdateFrameRegion; + update.imageId = imageId; + update.frameNumber = destinationFrame; + update.imageSize = image->second.size; + update.region = Rect( destinationX, destinationY, destinationX + width, destinationY + height ); + update.rgba = std::make_shared>( std::move( result ) ); + mUpdates.emplace_back( std::move( update ) ); + mPresentationDirty = true; + return { response( data, KittyGraphicsError::None, imageId ), KittyGraphicsError::None, true }; +} + +KittyImageId KittyGraphicsProtocol::resolveImageId( const KittyGraphicsCommandData& data ) const { + if ( data.imageId ) + return *data.imageId; + KittyImageId newestId = 0; + Uint64 newestSerial = 0; + if ( data.imageNumber ) { + for ( const auto& image : mImages ) { + if ( image.second.imageNumber == *data.imageNumber && + image.second.creationSerial > newestSerial ) { + newestId = image.first; + newestSerial = image.second.creationSerial; + } + } + } + return newestId; +} + +KittyImageId KittyGraphicsProtocol::allocateImageId() { + for ( Uint64 attempts = 0; attempts < std::numeric_limits::max(); ++attempts ) { + const KittyImageId candidate = mNextImageId++; + if ( mNextImageId == 0 ) + mNextImageId = 1; + if ( candidate != 0 && mImages.find( candidate ) == mImages.end() ) + return candidate; + } + return 0; +} + +bool KittyGraphicsProtocol::isImagePlaced( KittyImageId imageId ) const { + auto belongsToImage = [imageId]( const Placement& placement ) { + return placement.visible.imageId == imageId; + }; + return std::any_of( mPlacements.begin(), mPlacements.end(), belongsToImage ) || + std::any_of( mPrimaryPlacements.begin(), mPrimaryPlacements.end(), belongsToImage ); +} + +void KittyGraphicsProtocol::eraseImage( KittyImageId imageId ) { + auto image = mImages.find( imageId ); + if ( image == mImages.end() ) + return; + mStorageBytes -= image->second.rgba.size(); + for ( const auto& frame : image->second.frames ) + mFrameStorageBytes -= frame.second.rgba.size(); + mImages.erase( image ); + TerminalGraphicsUpdate update; + update.type = TerminalGraphicsUpdateType::DeleteImage; + update.imageId = imageId; + mUpdates.emplace_back( std::move( update ) ); +} + +bool KittyGraphicsProtocol::ensureCapacity( size_t bytes, KittyImageId replacingId, + bool addingImage ) { + auto replaced = mImages.find( replacingId ); + const size_t replacedBytes = replaced == mImages.end() ? 0 : replaced->second.rgba.size(); + auto hasCapacity = [&] { + return bytes <= mMaxStorageBytes && + mStorageBytes - replacedBytes <= mMaxStorageBytes - bytes && + ( !addingImage || mImages.size() < mMaxImages ); + }; + while ( !hasCapacity() ) { + auto candidate = mImages.end(); + for ( auto image = mImages.begin(); image != mImages.end(); ++image ) { + if ( image->first == replacingId || isImagePlaced( image->first ) ) + continue; + if ( candidate == mImages.end() || + ( image->second.usageHint != 0 && candidate->second.usageHint == 0 ) || + ( ( image->second.usageHint != 0 ) == ( candidate->second.usageHint != 0 ) && + image->second.creationSerial < candidate->second.creationSerial ) ) + candidate = image; + } + if ( candidate == mImages.end() ) + return false; + eraseImage( candidate->first ); + ++mStats.evictions; + } + return true; +} + +std::string KittyGraphicsProtocol::response( const KittyGraphicsCommandData& data, + KittyGraphicsError error, + KittyImageId imageId ) const { + const Uint32 quiet = data.quiet.value_or( 0 ); + if ( quiet == 2 || ( quiet == 1 && error == KittyGraphicsError::None ) ) + return {}; + const KittyImageId responseId = imageId != 0 ? imageId : data.imageId.value_or( 0 ); + if ( responseId == 0 && error == KittyGraphicsError::None ) + return {}; + std::string value = "\033_G"; + if ( responseId != 0 ) { + value += "i=" + std::to_string( responseId ); + if ( data.imageNumber ) + value += ",I=" + std::to_string( *data.imageNumber ); + if ( data.placementId ) + value += ",p=" + std::to_string( *data.placementId ); + } + value += ";"; + if ( error == KittyGraphicsError::None ) { + value += "OK"; + } else { + value += error == KittyGraphicsError::Unsupported ? "ENOTSUP" + : error == KittyGraphicsError::NoSpace ? "ENOSPC" + : error == KittyGraphicsError::NotFound ? "ENOENT" + : error == KittyGraphicsError::NoParent ? "ENOPARENT" + : error == KittyGraphicsError::Cycle ? "ECYCLE" + : error == KittyGraphicsError::TooDeep ? "ETOODEEP" + : "EINVAL"; + } + value += "\033\\"; + return value; +} + +std::vector KittyGraphicsProtocol::takeUpdates() { + std::vector updates; + updates.swap( mUpdates ); + return updates; +} + +std::shared_ptr KittyGraphicsProtocol::takePresentation() { + auto presentation = std::make_shared(); + presentation->generation = mPresentationGeneration; + presentation->placements.reserve( mPlacements.size() ); + for ( const auto& placement : mPlacements ) { + if ( placement.virtualPlacement ) + continue; + auto visible = placement.visible; + if ( placement.parentImageId != 0 ) { + auto parent = + std::find_if( mPlacements.begin(), mPlacements.end(), [&]( const Placement& p ) { + return p.visible.imageId == placement.parentImageId && + p.visible.placementId == placement.parentPlacementId; + } ); + if ( parent != mPlacements.end() && parent->virtualPlacement ) { + bool found = false; + Vector2i minimum( std::numeric_limits::max(), + std::numeric_limits::max() ); + for ( const auto& cell : mPlaceholderCells ) { + if ( cell.imageId == parent->visible.imageId && + ( parent->visible.placementId == 0 || + cell.placementId == parent->visible.placementId ) ) { + minimum.x = std::min( minimum.x, cell.cell.x ); + minimum.y = std::min( minimum.y, cell.cell.y ); + found = true; + } + } + if ( !found ) + continue; + visible.visibleAnchorCell += minimum - parent->visible.visibleAnchorCell; + } + } + auto image = mImages.find( visible.imageId ); + if ( image != mImages.end() ) + visible.frameNumber = image->second.currentFrame; + visible.visibleAnchorCell.y += mScrollPosition; + const Int64 bottom = static_cast( visible.visibleAnchorCell.y ) + visible.rows; + if ( bottom > 0 && ( mScreenRows == 0 || visible.visibleAnchorCell.y < mScreenRows ) ) + presentation->placements.emplace_back( std::move( visible ) ); + } + for ( const auto& cell : mPlaceholderCells ) { + auto prototype = + std::find_if( mPlacements.begin(), mPlacements.end(), [&]( const Placement& p ) { + return p.virtualPlacement && p.visible.imageId == cell.imageId && + ( cell.placementId == 0 || p.visible.placementId == cell.placementId ); + } ); + if ( prototype == mPlacements.end() || cell.imageRow >= prototype->visible.rows || + cell.imageColumn >= prototype->visible.columns ) + continue; + auto visible = prototype->visible; + visible.visibleAnchorCell = cell.cell; + visible.columns = 1; + visible.rows = 1; + const int sourceWidth = visible.sourcePixels.Right - visible.sourcePixels.Left; + const int sourceHeight = visible.sourcePixels.Bottom - visible.sourcePixels.Top; + const int left = visible.sourcePixels.Left + static_cast( sourceWidth ) * + cell.imageColumn / + prototype->visible.columns; + const int right = visible.sourcePixels.Left + static_cast( sourceWidth ) * + ( cell.imageColumn + 1 ) / + prototype->visible.columns; + const int top = visible.sourcePixels.Top + static_cast( sourceHeight ) * + cell.imageRow / prototype->visible.rows; + const int bottom = visible.sourcePixels.Top + static_cast( sourceHeight ) * + ( cell.imageRow + 1 ) / + prototype->visible.rows; + visible.sourcePixels = Rect( left, top, right, bottom ); + auto image = mImages.find( visible.imageId ); + if ( image != mImages.end() ) + visible.frameNumber = image->second.currentFrame; + presentation->placements.emplace_back( std::move( visible ) ); + } + std::stable_sort( + presentation->placements.begin(), presentation->placements.end(), + []( const TerminalVisiblePlacement& left, const TerminalVisiblePlacement& right ) { + if ( left.zIndex != right.zIndex ) + return left.zIndex < right.zIndex; + return left.imageId < right.imageId; + } ); + mPresentationDirty = false; + return presentation; +} + +const std::vector* KittyGraphicsProtocol::imagePixels( KittyImageId imageId ) const { + auto image = mImages.find( imageId ); + return image == mImages.end() ? nullptr : &image->second.rgba; +} + +bool KittyGraphicsProtocol::hasVirtualPlacements() const { + return std::any_of( mPlacements.begin(), mPlacements.end(), + []( const Placement& placement ) { return placement.virtualPlacement; } ); +} + +void KittyGraphicsProtocol::reset() { + mImages.clear(); + mPlacements.clear(); + mPrimaryPlacements.clear(); + mPlaceholderCells.clear(); + mUpdates.clear(); + mPending = {}; + mStorageBytes = 0; + mFrameStorageBytes = 0; + TerminalGraphicsUpdate reset; + reset.type = TerminalGraphicsUpdateType::ResetAll; + mUpdates.emplace_back( std::move( reset ) ); + ++mPresentationGeneration; + mPresentationDirty = true; +} + +void KittyGraphicsProtocol::clearScreen() { + const size_t oldSize = mPlacements.size(); + mPlacements.erase( + std::remove_if( mPlacements.begin(), mPlacements.end(), + [&]( const Placement& placement ) { + if ( placement.virtualPlacement ) + return false; + const int visibleTop = + placement.visible.visibleAnchorCell.y + mScrollPosition; + return static_cast( visibleTop ) + placement.visible.rows > 0 && + ( mScreenRows == 0 || visibleTop < mScreenRows ); + } ), + mPlacements.end() ); + if ( oldSize == mPlacements.size() ) + return; + ++mPresentationGeneration; + mPresentationDirty = true; +} + +void KittyGraphicsProtocol::setAlternateScreen( bool alternate ) { + if ( alternate ) { + mPrimaryPlacements = std::move( mPlacements ); + mPlacements.clear(); + } else { + mPlacements = std::move( mPrimaryPlacements ); + mPrimaryPlacements.clear(); + } + ++mPresentationGeneration; + mPresentationDirty = true; +} + +void KittyGraphicsProtocol::scrollScreen( int top, int bottom, int rows, bool preserveHistory ) { + if ( rows == 0 || top > bottom ) + return; + const size_t oldSize = mPlacements.size(); + std::vector scrolledOut; + for ( auto& placement : mPlacements ) { + auto& visible = placement.visible; + const Int64 placementBottom = + static_cast( visible.visibleAnchorCell.y ) + visible.rows; + if ( visible.visibleAnchorCell.y < top || + placementBottom > static_cast( bottom ) + 1 ) + continue; + visible.visibleAnchorCell.y += rows; + if ( preserveHistory ) + continue; + const Uint32 oldRows = visible.rows; + const int sourceTop = visible.sourcePixels.Top; + const int sourceHeight = visible.sourcePixels.Bottom - sourceTop; + const int clipTop = std::max( 0, top - visible.visibleAnchorCell.y ); + const int clipBottom = std::max( 0, visible.visibleAnchorCell.y + + static_cast( visible.rows ) - bottom - 1 ); + if ( clipTop + clipBottom >= static_cast( visible.rows ) ) { + scrolledOut.emplace_back( placement.internalId ); + continue; + } + visible.sourcePixels.Top = + sourceTop + static_cast( sourceHeight ) * clipTop / oldRows; + visible.sourcePixels.Bottom = + sourceTop + static_cast( sourceHeight ) * ( oldRows - clipBottom ) / oldRows; + visible.visibleAnchorCell.y += clipTop; + visible.rows -= clipTop + clipBottom; + if ( clipTop > 0 ) + visible.firstCellPixelOffset.y = 0; + } + if ( !scrolledOut.empty() ) { + mPlacements.erase( std::remove_if( mPlacements.begin(), mPlacements.end(), + [&]( const Placement& p ) { + return std::find( + scrolledOut.begin(), scrolledOut.end(), + p.internalId ) != scrolledOut.end(); + } ), + mPlacements.end() ); + } + if ( oldSize != mPlacements.size() || !mPlacements.empty() ) { + ++mPresentationGeneration; + mPresentationDirty = true; + } +} + +void KittyGraphicsProtocol::setViewport( int scrollPosition, int historyLength, int screenRows ) { + if ( mScrollPosition == scrollPosition && mHistoryLength == historyLength && + mScreenRows == screenRows ) + return; + mScrollPosition = scrollPosition; + mHistoryLength = historyLength; + mScreenRows = screenRows; + const size_t oldSize = mPlacements.size(); + mPlacements.erase( std::remove_if( mPlacements.begin(), mPlacements.end(), + [historyLength]( const Placement& p ) { + return p.visible.visibleAnchorCell.y < 0 && + static_cast( + p.visible.visibleAnchorCell.y ) + + p.visible.rows <= + -historyLength; + } ), + mPlacements.end() ); + if ( oldSize != 0 || oldSize != mPlacements.size() ) { + ++mPresentationGeneration; + mPresentationDirty = true; + } +} + +void KittyGraphicsProtocol::resync() { + mUpdates.clear(); + TerminalGraphicsUpdate reset; + reset.type = TerminalGraphicsUpdateType::ResetAll; + mUpdates.emplace_back( std::move( reset ) ); + for ( const auto& image : mImages ) { + TerminalGraphicsUpdate create; + create.type = TerminalGraphicsUpdateType::CreateImage; + create.imageId = image.first; + create.imageSize = image.second.size; + create.region = Rect( 0, 0, image.second.size.getWidth(), image.second.size.getHeight() ); + create.rgba = std::make_shared>( image.second.rgba ); + mUpdates.emplace_back( std::move( create ) ); + for ( const auto& frame : image.second.frames ) { + TerminalGraphicsUpdate createFrame; + createFrame.type = TerminalGraphicsUpdateType::CreateFrame; + createFrame.imageId = image.first; + createFrame.frameNumber = frame.first; + createFrame.imageSize = image.second.size; + createFrame.region = + Rect( 0, 0, image.second.size.getWidth(), image.second.size.getHeight() ); + createFrame.rgba = std::make_shared>( frame.second.rgba ); + mUpdates.emplace_back( std::move( createFrame ) ); + } + } + mPresentationDirty = true; +} + +}} // namespace eterm::Terminal diff --git a/src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp b/src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp new file mode 100644 index 000000000..d9d91735f --- /dev/null +++ b/src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp @@ -0,0 +1,167 @@ +#include + +#include +#include +#include +#include + +namespace eterm { namespace Terminal { + +static bool isInPass( Int32 zIndex, KittyGraphicsRenderer::Pass pass ) { + constexpr Int32 VeryNegativeThreshold = std::numeric_limits::min() / 2; + return pass == KittyGraphicsRenderer::Pass::VeryNegative ? zIndex < VeryNegativeThreshold + : pass == KittyGraphicsRenderer::Pass::Negative + ? zIndex < 0 && zIndex >= VeryNegativeThreshold + : zIndex >= 0; +} + +bool KittyGraphicsRenderer::applyUpdates( std::vector&& updates ) { + for ( auto& update : updates ) { + if ( update.type == TerminalGraphicsUpdateType::Resync ) { + reset(); + return false; + } + if ( update.type != TerminalGraphicsUpdateType::ResetAll && + update.sequence != mLastAppliedSequence + 1 ) + return false; + + switch ( update.type ) { + case TerminalGraphicsUpdateType::CreateImage: + case TerminalGraphicsUpdateType::ReplaceImage: { + if ( !update.rgba || update.imageSize.getWidth() <= 0 || + update.imageSize.getHeight() <= 0 ) + return false; + auto texture = TextureFactory::instance()->loadFromPixels( + update.rgba->data(), update.imageSize.getWidth(), update.imageSize.getHeight(), + 4, false, Texture::ClampMode::ClampToEdge, false, false ); + if ( !texture ) + return false; + texture->setFilter( Texture::Filter::Linear ); + mImages.insert_or_assign( update.imageId, + GPUImage{ std::move( texture ), {}, update.imageSize } ); + break; + } + case TerminalGraphicsUpdateType::UpdateRegion: { + auto image = mImages.find( update.imageId ); + if ( image == mImages.end() || !update.rgba ) + return false; + const int width = update.region.Right - update.region.Left; + const int height = update.region.Bottom - update.region.Top; + if ( width <= 0 || height <= 0 || update.region.Left < 0 || update.region.Top < 0 || + update.region.Right > image->second.size.getWidth() || + update.region.Bottom > image->second.size.getHeight() || + update.rgba->size() != static_cast( width ) * height * 4 ) + return false; + image->second.texture->update( update.rgba->data(), width, height, + update.region.Left, update.region.Top ); + break; + } + case TerminalGraphicsUpdateType::CreateFrame: + case TerminalGraphicsUpdateType::ReplaceFrame: { + auto image = mImages.find( update.imageId ); + if ( image == mImages.end() || !update.rgba || update.frameNumber <= 1 || + update.rgba->size() != static_cast( image->second.size.getWidth() ) * + image->second.size.getHeight() * 4 ) + return false; + auto texture = TextureFactory::instance()->loadFromPixels( + update.rgba->data(), image->second.size.getWidth(), + image->second.size.getHeight(), 4, false, Texture::ClampMode::ClampToEdge, + false, false ); + if ( !texture ) + return false; + texture->setFilter( Texture::Filter::Linear ); + image->second.frames.insert_or_assign( update.frameNumber, std::move( texture ) ); + break; + } + case TerminalGraphicsUpdateType::UpdateFrameRegion: { + auto image = mImages.find( update.imageId ); + if ( image == mImages.end() || !update.rgba ) + return false; + auto frame = image->second.frames.find( update.frameNumber ); + if ( frame == image->second.frames.end() ) + return false; + const int width = update.region.Right - update.region.Left; + const int height = update.region.Bottom - update.region.Top; + if ( width <= 0 || height <= 0 || update.region.Left < 0 || update.region.Top < 0 || + update.region.Right > image->second.size.getWidth() || + update.region.Bottom > image->second.size.getHeight() || + update.rgba->size() != static_cast( width ) * height * 4 ) + return false; + frame->second->update( update.rgba->data(), width, height, update.region.Left, + update.region.Top ); + break; + } + case TerminalGraphicsUpdateType::DeleteFrame: { + auto image = mImages.find( update.imageId ); + if ( image != mImages.end() ) + image->second.frames.erase( update.frameNumber ); + break; + } + case TerminalGraphicsUpdateType::DeleteImage: + mImages.erase( update.imageId ); + break; + case TerminalGraphicsUpdateType::ResetScreen: + break; + case TerminalGraphicsUpdateType::ResetAll: + mImages.clear(); + break; + default: + return false; + } + mLastAppliedSequence = update.sequence; + } + return true; +} + +void KittyGraphicsRenderer::setPresentation( + std::shared_ptr presentation ) { + mPresentation = std::move( presentation ); +} + +void KittyGraphicsRenderer::draw( Pass pass, const Vector2f& origin, const Sizef& cellSize, + const Sizef& gridSize ) { + if ( !mPresentation ) + return; + auto clippingMask = Renderer::instance()->getClippingMask(); + clippingMask->clipEnable( origin.x, origin.y, gridSize.getWidth(), gridSize.getHeight() ); + for ( const auto& placement : mPresentation->placements ) { + if ( !isInPass( placement.zIndex, pass ) ) + continue; + auto image = mImages.find( placement.imageId ); + if ( image == mImages.end() || !image->second.texture ) + continue; + const Vector2f position( origin.x + placement.visibleAnchorCell.x * cellSize.getWidth() + + placement.firstCellPixelOffset.x, + origin.y + placement.visibleAnchorCell.y * cellSize.getHeight() + + placement.firstCellPixelOffset.y ); + const Float width = placement.columns * cellSize.getWidth(); + const Float height = placement.rows * cellSize.getHeight(); + TexturePtr texture = image->second.texture; + if ( placement.frameNumber > 1 ) { + auto frame = image->second.frames.find( placement.frameNumber ); + if ( frame != image->second.frames.end() ) + texture = frame->second; + } + texture->drawEx( position.x, position.y, width, height, 0, Vector2f::One, Color::White, + Color::White, Color::White, Color::White, BlendMode::Alpha(), + RENDER_NORMAL, OriginPoint( OriginPoint::OriginTopLeft ), + placement.sourcePixels ); + } + clippingMask->clipDisable(); +} + +bool KittyGraphicsRenderer::hasPlacements( Pass pass ) const { + return mPresentation && + std::any_of( mPresentation->placements.begin(), mPresentation->placements.end(), + [pass]( const TerminalVisiblePlacement& placement ) { + return isInPass( placement.zIndex, pass ); + } ); +} + +void KittyGraphicsRenderer::reset() { + mImages.clear(); + mPresentation.reset(); + mLastAppliedSequence = 0; +} + +}} // namespace eterm::Terminal diff --git a/src/modules/eterm/src/eterm/terminal/pseudoterminal.cpp b/src/modules/eterm/src/eterm/terminal/pseudoterminal.cpp index 1c20ab538..d479c58f6 100644 --- a/src/modules/eterm/src/eterm/terminal/pseudoterminal.cpp +++ b/src/modules/eterm/src/eterm/terminal/pseudoterminal.cpp @@ -107,13 +107,13 @@ bool PseudoTerminal::isTTY() const { return true; } -bool PseudoTerminal::resize( int columns, int rows ) { +bool PseudoTerminal::resize( int columns, int rows, int pixelWidth, int pixelHeight ) { struct winsize w; w.ws_row = rows; w.ws_col = columns; - w.ws_xpixel = 0; - w.ws_ypixel = 0; + w.ws_xpixel = pixelWidth; + w.ws_ypixel = pixelHeight; bool masterResized = ioctl( (int)mMaster, TIOCSWINSZ, &w ) >= 0; bool slaveResized = @@ -297,7 +297,7 @@ bool PseudoTerminal::isTTY() const { return true; } -bool PseudoTerminal::resize( int columns, int rows ) { +bool PseudoTerminal::resize( int columns, int rows, int, int ) { if ( !pResizePseudoConsole ) return false; diff --git a/src/modules/eterm/src/eterm/terminal/terminaldisplay.cpp b/src/modules/eterm/src/eterm/terminal/terminaldisplay.cpp index 9ddda5c33..0fffc3ae1 100644 --- a/src/modules/eterm/src/eterm/terminal/terminaldisplay.cpp +++ b/src/modules/eterm/src/eterm/terminal/terminaldisplay.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -457,6 +458,9 @@ std::shared_ptr TerminalDisplay::create( eeSAFE_DELETE( processFactory ); return nullptr; } + terminal->mSession->resize( termSize.getWidth(), termSize.getHeight(), + terminal->getGridPixelSize().getWidth(), + terminal->getGridPixelSize().getHeight() ); terminal->mSession->setPresentationRate( presentationRateForWindow( window ) ); terminal->mProgram = program; terminal->mArgs = args; @@ -479,6 +483,7 @@ TerminalDisplay::~TerminalDisplay() { TerminalDisplay::TerminalDisplay( EE::Window::Window* window, Font* font, const Float& fontSize, const Sizef& pixelsSize, const bool& useFrameBuffer ) : mWindow( window ), + mGraphicsRenderer( std::make_unique() ), mFont( font ), mFontSize( fontSize ), mSize( pixelsSize ), @@ -493,6 +498,8 @@ TerminalDisplay::TerminalDisplay( EE::Window::Window* window, Font* font, const resetColors(); Sizei gridSize( gridSizeFromTermDimensions( mFont, mFontSize, mSize - mPadding * 2.f ) ); + mColumns = gridSize.getWidth(); + mRows = gridSize.getHeight(); mDirtyLines.resize( gridSize.getHeight(), 1 ); mQuadVertex = GLi->quadVertex(); @@ -715,7 +722,9 @@ void TerminalDisplay::setKeepAlive( bool keepAlive ) { } bool TerminalDisplay::update( bool isMouseOverMe ) { + drainGraphicsUpdates(); consumeSnapshot(); + drainGraphicsUpdates(); drainSessionEvents(); if ( mFocus && isBlinkingCursor() && mClock.getElapsedTime().asSeconds() > 0.7 ) { mMode ^= MODE_BLINK; @@ -734,6 +743,29 @@ bool TerminalDisplay::update( bool isMouseOverMe ) { return true; } +void TerminalDisplay::drainGraphicsUpdates() { + if ( !mSession ) + return; + auto updates = mSession->drainGraphicsUpdates(); + if ( !updates.empty() ) { + if ( !mGraphicsRenderer->applyUpdates( std::move( updates ) ) ) { + mGraphicsRenderer->reset(); + mLastAppliedGraphicsSequence = 0; + if ( !mGraphicsResyncPending ) { + mGraphicsResyncPending = true; + mSession->requestGraphicsResync(); + } + return; + } + mLastAppliedGraphicsSequence = mGraphicsRenderer->lastAppliedSequence(); + mGraphicsResyncPending = false; + if ( mSnapshot && mSnapshot->graphics && + mSnapshot->graphics->requiredUpdateSequence <= mLastAppliedGraphicsSequence ) + mGraphicsRenderer->setPresentation( mSnapshot->graphics ); + mDirty = true; + } +} + void TerminalDisplay::consumeSnapshot() { if ( !mSession ) return; @@ -768,6 +800,13 @@ void TerminalDisplay::consumeSnapshot() { mCursor = mSnapshot->cursor; mCursorGlyph = mSnapshot->cursorGlyph; mCursorMode = mSnapshot->cursorMode; + if ( mSnapshot->graphics && + mSnapshot->graphics->requiredUpdateSequence <= mGraphicsRenderer->lastAppliedSequence() ) { + mGraphicsRenderer->setPresentation( mSnapshot->graphics ); + } else if ( mSnapshot->graphics && !mGraphicsResyncPending ) { + mGraphicsResyncPending = true; + mSession->requestGraphicsResync(); + } const int presentationBits = mMode & MODE_BLINK; mMode = mSnapshot->windowMode | presentationBits; if ( mFocus ) @@ -1093,8 +1132,8 @@ void TerminalDisplay::onMouseMove( const Vector2i& pos, const Uint32& flags ) { mWindow->getInput()->getModState() & KEYMOD_SHIFT ? SEL_RECTANGULAR : SEL_REGULAR, false ); } - mSession->mouseReport( TerminalMouseEventType::MouseMotion, positionToGrid( pos ), flags, - mWindow->getInput()->getModState() ); + mSession->mouseReport( TerminalMouseEventType::MouseMotion, positionToGrid( pos ), + positionToPixel( pos ), flags, mWindow->getInput()->getModState() ); } void TerminalDisplay::onMouseDown( const Vector2i& pos, const Uint32& flags ) { @@ -1134,8 +1173,8 @@ void TerminalDisplay::onMouseDown( const Vector2i& pos, const Uint32& flags ) { } } - mSession->mouseReport( TerminalMouseEventType::MouseButtonDown, positionToGrid( pos ), flags, - mWindow->getInput()->getModState() ); + mSession->mouseReport( TerminalMouseEventType::MouseButtonDown, positionToGrid( pos ), + positionToPixel( pos ), flags, mWindow->getInput()->getModState() ); } void TerminalDisplay::onMouseUp( const Vector2i& pos, const Uint32& flags ) { @@ -1181,8 +1220,8 @@ void TerminalDisplay::onMouseUp( const Vector2i& pos, const Uint32& flags ) { } } - mSession->mouseReport( TerminalMouseEventType::MouseButtonRelease, positionToGrid( pos ), flags, - mWindow->getInput()->getModState() ); + mSession->mouseReport( TerminalMouseEventType::MouseButtonRelease, positionToGrid( pos ), + positionToPixel( pos ), flags, mWindow->getInput()->getModState() ); } static inline Color termColor( unsigned int terminalColor, const std::vector& colors ) { @@ -1368,6 +1407,7 @@ void TerminalDisplay::drawGrid( const Vector2f& pos ) { auto fontSize = mFont->getFontHeight( mFontSize ); auto spaceCharAdvanceX = mFont->getGlyph( 'A', mFontSize, false, false ).advance; + const Sizef cellSize( spaceCharAdvanceX, fontSize ); float x = 0.0f; float y = pos.y; @@ -1446,6 +1486,38 @@ void TerminalDisplay::drawGrid( const Vector2f& pos ) { mVBBackground->draw(); mVBBackground->unbind(); } + const Sizef graphicsGridSize( mColumns * cellSize.getWidth(), mRows * cellSize.getHeight() ); + mGraphicsRenderer->draw( KittyGraphicsRenderer::Pass::VeryNegative, pos, cellSize, + graphicsGridSize ); + if ( mGraphicsRenderer->hasPlacements( KittyGraphicsRenderer::Pass::VeryNegative ) ) { + y = std::floor( pos.y ); + for ( Uint32 row = 0; row < mRows; ++row ) { + x = std::floor( pos.x ); + for ( Uint32 column = 0; column < mColumns; ++column ) { + const auto& glyph = mSnapshot->cells[row * mColumns + column]; + if ( glyph.mode & ATTR_WDUMMY ) + continue; + auto foreground = termColor( glyph.fg, mColors ); + auto background = termColor( glyph.bg, mColors ); + if ( IS_SET( MODE_REVERSE ) ) { + foreground = foreground == defaultFg ? defaultBg : foreground.invert(); + background = background == defaultBg ? defaultFg : background.invert(); + } + if ( glyph.mode & ATTR_REVERSE ) + background = foreground; + const bool wide = glyph.mode & ATTR_WIDE; + const Float advance = spaceCharAdvanceX * ( wide ? 2.0f : 1.0f ); + if ( background != defaultBg ) { + mPrimitives.setColor( background ); + mPrimitives.drawRectangle( Rectf( { x, y }, { advance, lineHeight } ) ); + } + x += advance; + } + y += lineHeight; + } + } + mGraphicsRenderer->draw( KittyGraphicsRenderer::Pass::Negative, pos, cellSize, + graphicsGridSize ); y = std::floor( pos.y ); @@ -1683,6 +1755,8 @@ void TerminalDisplay::drawGrid( const Vector2f& pos ) { vbo->unbind(); } } + mGraphicsRenderer->draw( KittyGraphicsRenderer::Pass::NonNegative, pos, cellSize, + graphicsGridSize ); // Underline is rendered after foreground render because it usually clashes with the underlines // decorations and ends up being not visible, I prefer to do this even it it's not standard. @@ -1777,6 +1851,27 @@ Vector2i TerminalDisplay::positionToGrid( const Vector2i& pos ) { return { mouseX, mouseY }; } +Vector2i TerminalDisplay::positionToPixel( const Vector2i& pos ) const { + const Sizei gridPixels = getGridPixelSize(); + const int x = static_cast( std::floor( pos.x - mPosition.x - mPadding.Left ) ); + const int y = static_cast( std::floor( pos.y - mPosition.y - mPadding.Top ) ); + return { eeclamp( x, 0, eemax( 0, gridPixels.getWidth() - 1 ) ), + eeclamp( y, 0, eemax( 0, gridPixels.getHeight() - 1 ) ) }; +} + +Sizei TerminalDisplay::getCellPixelSize() const { + return { + static_cast( std::round( mFont->getGlyph( 'A', mFontSize, false, false ).advance ) ), + static_cast( std::round( mFont->getFontHeight( mFontSize ) ) ) }; +} + +Sizei TerminalDisplay::getGridPixelSize() const { + const Sizei cell = getCellPixelSize(); + const int columns = mSnapshot ? mSnapshot->columns : static_cast( mColumns ); + const int rows = mSnapshot ? mSnapshot->rows : static_cast( mRows ); + return { columns * cell.getWidth(), rows * cell.getHeight() }; +} + void TerminalDisplay::onSizeChange() { Sizei gridSize( gridSizeFromTermDimensions( mFont, mFontSize, @@ -1784,7 +1879,10 @@ void TerminalDisplay::onSizeChange() { if ( mSession && ( !mSnapshot || gridSize.getWidth() != mSnapshot->columns || gridSize.getHeight() != mSnapshot->rows ) ) { - mSession->resize( gridSize.getWidth(), gridSize.getHeight() ); + const Sizei cellSize = getCellPixelSize(); + mSession->resize( gridSize.getWidth(), gridSize.getHeight(), + gridSize.getWidth() * cellSize.getWidth(), + gridSize.getHeight() * cellSize.getHeight() ); mDirtyLines.resize( gridSize.getHeight(), 1 ); } diff --git a/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp b/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp index aaed40aa1..ad0295896 100644 --- a/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp +++ b/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp @@ -211,6 +211,46 @@ static const unsigned int tabspaces = 4; #define TRUECOLOR( r, g, b ) ( 1 << 24 | ( r ) << 16 | ( g ) << 8 | ( b ) ) #define IS_TRUECOL( x ) ( 1 << 24 & ( x ) ) +static int kittyDiacriticIndex( Rune value ) { + static constexpr Rune values[] = { + 0x0305, 0x030D, 0x030E, 0x0310, 0x0312, 0x033D, 0x033E, 0x033F, 0x0346, 0x034A, + 0x034B, 0x034C, 0x0350, 0x0351, 0x0352, 0x0357, 0x035B, 0x0363, 0x0364, 0x0365, + 0x0366, 0x0367, 0x0368, 0x0369, 0x036A, 0x036B, 0x036C, 0x036D, 0x036E, 0x036F, + 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, 0x0592, 0x0593, 0x0594, 0x0595, 0x0597, + 0x0598, 0x0599, 0x059C, 0x059D, 0x059E, 0x059F, 0x05A0, 0x05A1, 0x05A8, 0x05A9, + 0x05AB, 0x05AC, 0x05AF, 0x05C4, 0x0610, 0x0611, 0x0612, 0x0613, 0x0614, 0x0615, + 0x0616, 0x0617, 0x0657, 0x0658, 0x0659, 0x065A, 0x065B, 0x065D, 0x065E, 0x06D6, + 0x06D7, 0x06D8, 0x06D9, 0x06DA, 0x06DB, 0x06DC, 0x06DF, 0x06E0, 0x06E1, 0x06E2, + 0x06E4, 0x06E7, 0x06E8, 0x06EB, 0x06EC, 0x0730, 0x0732, 0x0733, 0x0735, 0x0736, + 0x073A, 0x073D, 0x073F, 0x0740, 0x0741, 0x0743, 0x0745, 0x0747, 0x0749, 0x074A, + 0x07EB, 0x07EC, 0x07ED, 0x07EE, 0x07EF, 0x07F0, 0x07F1, 0x07F3, 0x0816, 0x0817, + 0x0818, 0x0819, 0x081B, 0x081C, 0x081D, 0x081E, 0x081F, 0x0820, 0x0821, 0x0822, + 0x0823, 0x0825, 0x0826, 0x0827, 0x0829, 0x082A, 0x082B, 0x082C, 0x082D, 0x0951, + 0x0953, 0x0954, 0x0F82, 0x0F83, 0x0F86, 0x0F87, 0x135D, 0x135E, 0x135F, 0x17DD, + 0x193A, 0x1A17, 0x1A75, 0x1A76, 0x1A77, 0x1A78, 0x1A79, 0x1A7A, 0x1A7B, 0x1A7C, + 0x1B6B, 0x1B6D, 0x1B6E, 0x1B6F, 0x1B70, 0x1B71, 0x1B72, 0x1B73, 0x1CD0, 0x1CD1, + 0x1CD2, 0x1CDA, 0x1CDB, 0x1CE0, 0x1DC0, 0x1DC1, 0x1DC3, 0x1DC4, 0x1DC5, 0x1DC6, + 0x1DC7, 0x1DC8, 0x1DC9, 0x1DCB, 0x1DCC, 0x1DD1, 0x1DD2, 0x1DD3, 0x1DD4, 0x1DD5, + 0x1DD6, 0x1DD7, 0x1DD8, 0x1DD9, 0x1DDA, 0x1DDB, 0x1DDC, 0x1DDD, 0x1DDE, 0x1DDF, + 0x1DE0, 0x1DE1, 0x1DE2, 0x1DE3, 0x1DE4, 0x1DE5, 0x1DE6, 0x1DFE, 0x20D0, 0x20D1, + 0x20D4, 0x20D5, 0x20D6, 0x20D7, 0x20DB, 0x20DC, 0x20E1, 0x20E7, 0x20E9, 0x20F0, + 0x2CEF, 0x2CF0, 0x2CF1, 0x2DE0, 0x2DE1, 0x2DE2, 0x2DE3, 0x2DE4, 0x2DE5, 0x2DE6, + 0x2DE7, 0x2DE8, 0x2DE9, 0x2DEA, 0x2DEB, 0x2DEC, 0x2DED, 0x2DEE, 0x2DEF, 0x2DF0, + 0x2DF1, 0x2DF2, 0x2DF3, 0x2DF4, 0x2DF5, 0x2DF6, 0x2DF7, 0x2DF8, 0x2DF9, 0x2DFA, + 0x2DFB, 0x2DFC, 0x2DFD, 0x2DFE, 0x2DFF, 0xA66F, 0xA67C, 0xA67D, 0xA6F0, 0xA6F1, + 0xA8E0, 0xA8E1, 0xA8E2, 0xA8E3, 0xA8E4, 0xA8E5, 0xA8E6, 0xA8E7, 0xA8E8, 0xA8E9, + 0xA8EA, 0xA8EB, 0xA8EC, 0xA8ED, 0xA8EE, 0xA8EF, 0xA8F0, 0xA8F1, 0xAAB0, 0xAAB2, + 0xAAB3, 0xAAB7, 0xAAB8, 0xAABE, 0xAABF, 0xAAC1, 0xFE20, 0xFE21, 0xFE22, 0xFE23, + 0xFE24, 0xFE25, 0xFE26, 0x10A0F, 0x10A38, 0x1D185, 0x1D186, 0x1D187, 0x1D188, 0x1D189, + 0x1D1AA, 0x1D1AB, 0x1D1AC, 0x1D1AD, 0x1D242, 0x1D243, 0x1D244, + }; + for ( size_t i = 0; i < sizeof( values ) / sizeof( values[0] ); ++i ) { + if ( values[i] == value ) + return static_cast( i ); + } + return -1; +} + /* Arbitrary sizes */ #define UTF_INVALID 0xFFFD #define UTF_SIZ 4 @@ -867,8 +907,13 @@ void TerminalEmulator::trimMemory() { } void TerminalEmulator::clearHistory() { - for ( int i = 0; i < mTerm.histcursize; ++i ) + for ( int i = 0; i < mTerm.histcursize; ++i ) { + if ( mTerm.hist[i] ) { + for ( int column = 0; column < mTerm.col; ++column ) + mKittyPlaceholderMetadata.erase( &mTerm.hist[i][column] ); + } eeSAFE_FREE( mTerm.hist[i] ); + } eeSAFE_FREE( mTerm.hist ); mTerm.histcursize = 0; mTerm.histi = 0; @@ -906,6 +951,12 @@ void TerminalEmulator::notifyColorSchemeChanged() { mColorScheme = scheme; } +void TerminalEmulator::requestGraphicsResync() { + mKittyGraphics.resync(); + mDirty = true; + draw(); +} + Vector2i TerminalEmulator::getSize() const { return { mTerm.col, mTerm.row }; } @@ -1048,8 +1099,9 @@ void TerminalEmulator::treset( void ) { tswapscreen(); } - xsetmode( 0, MODE_MOUSE | MODE_MOUSESGR | MODE_APPKEYPAD | MODE_APPCURSOR | MODE_FOCUS | - MODE_BRCKTPASTE | MODE_MOUSEX10 | MODE_MOUSEMANY ); + xsetmode( 0, MODE_MOUSE | MODE_MOUSESGR | MODE_MOUSESGR_PIXELS | MODE_APPKEYPAD | + MODE_APPCURSOR | MODE_FOCUS | MODE_BRCKTPASTE | MODE_MOUSEX10 | + MODE_MOUSEMANY ); // Preserve eterm's established behavior and xterm's default alternateScroll resource. xsetmode( 1, MODE_ALTSCRROLL ); auto dpy = mDpy.lock(); @@ -1098,6 +1150,7 @@ void TerminalEmulator::tscrolldown( int top, int n ) { if ( mTerm.scr == 0 ) selscroll( top, n ); + mKittyGraphics.scrollScreen( top, mTerm.bot, n, false ); } void TerminalEmulator::tscrollup( int top, int n, int copyhist ) { @@ -1151,6 +1204,9 @@ void TerminalEmulator::tscrollup( int top, int n, int copyhist ) { if ( mTerm.scr == 0 ) selscroll( top, -n ); + mKittyGraphics.scrollScreen( top, mTerm.bot, -n, + copyhist && mTerm.histsize > 0 && !IS_SET( MODE_ALTSCREEN ) && + top == mTerm.top ); onScrollPositionChange(); } @@ -1559,6 +1615,7 @@ void TerminalEmulator::tsetchar( Rune u, TerminalGlyph* attr, int x, int y ) { TLINE( y )[x - 1].mode &= ~ATTR_WIDE; } + mKittyPlaceholderMetadata.erase( &TLINE( y )[x] ); mTerm.dirty[y] = 1; TLINE( y )[x] = *attr; TLINE( y )[x].u = u; @@ -1576,11 +1633,14 @@ void TerminalEmulator::tclearregion( int x1, int y1, int x2, int y2, bool skip_c temp = x1, x1 = x2, x2 = temp; if ( y1 > y2 ) temp = y1, y1 = y2, y2 = temp; - LIMIT( x1, 0, mTerm.col - 1 ); LIMIT( x2, 0, mTerm.col - 1 ); LIMIT( y1, 0, mTerm.row - 1 ); LIMIT( y2, 0, mTerm.row - 1 ); + for ( int clearY = y1; clearY <= y2; ++clearY ) { + for ( int clearX = x1; clearX <= x2; ++clearX ) + mKittyPlaceholderMetadata.erase( &TLINE( clearY )[clearX] ); + } /* * Fast path for the common full-row clear performed while scrolling: no @@ -1753,6 +1813,7 @@ void TerminalEmulator::tsetattr( int* attr, int l, const char* separators ) { ATTR_BLINK | ATTR_REVERSE | ATTR_INVISIBLE | ATTR_STRUCK ); mTerm.c.attr.fg = mDefaultFg; mTerm.c.attr.bg = mDefaultBg; + mKittyUnderlineColor = 0; break; case 1: mTerm.c.attr.mode |= ATTR_BOLD; @@ -1816,12 +1877,11 @@ void TerminalEmulator::tsetattr( int* attr, int l, const char* separators ) { mTerm.c.attr.bg = mDefaultBg; break; case 58: - /* This starts a sequence to change the color of - * "underline" pixels. We don't support that and - * instead eat up a following "5;n" or "2;r;g;b". */ - tdefcolor( attr, separators, &i, l ); + if ( ( idx = tdefcolor( attr, separators, &i, l ) ) >= 0 ) + mKittyUnderlineColor = idx; break; - case 59: /* reset underline color (unsupported, therefore a no-op) */ + case 59: + mKittyUnderlineColor = 0; break; default: if ( BETWEEN( attr[i], 30, 37 ) ) { @@ -1916,6 +1976,11 @@ void TerminalEmulator::tsetmode( int priv, int set, int* args, int narg ) { case 1006: /* 1006: extended reporting mode */ xsetmode( set, MODE_MOUSESGR ); break; + case 1016: /* 1016: extended reporting in terminal-grid pixels */ + xsetmode( set, MODE_MOUSESGR_PIXELS ); + if ( set ) + xsetmode( 1, MODE_MOUSESGR ); + break; case 1007: /* wheel sends cursor keys on the alternate screen */ xsetmode( set, MODE_ALTSCRROLL ); break; @@ -1936,7 +2001,10 @@ void TerminalEmulator::tsetmode( int priv, int set, int* args, int narg ) { tclearregion( 0, 0, mTerm.col - 1, mTerm.row - 1 ); } if ( set ^ alt ) /* set is always 1 or 0 */ + { tswapscreen(); + mKittyGraphics.setAlternateScreen( set != 0 ); + } if ( *args != 1049 ) break; /* FALLTHROUGH */ @@ -2153,6 +2221,7 @@ void TerminalEmulator::csihandle( void ) { // fallthrough case 2: /* all */ tclearregion( 0, 0, mTerm.col - 1, mTerm.row - 1 ); + mKittyGraphics.clearScreen(); break; default: goto unknown; @@ -2278,6 +2347,18 @@ void TerminalEmulator::csihandle( void ) { break; case 't': /* Window manipulation */ switch ( mCsiescseq.arg[0] ) { + case 14: /* Report terminal grid size in pixels. */ + len = + snprintf( buf, sizeof( buf ), "\033[4;%d;%dt", mPixelHeight, mPixelWidth ); + ttywrite( buf, len, 0 ); + break; + case 16: { /* Report terminal cell size in pixels. */ + const int cellWidth = mTerm.col > 0 ? mPixelWidth / mTerm.col : 0; + const int cellHeight = mTerm.row > 0 ? mPixelHeight / mTerm.row : 0; + len = snprintf( buf, sizeof( buf ), "\033[6;%d;%dt", cellHeight, cellWidth ); + ttywrite( buf, len, 0 ); + break; + } case 22: /* Save window title */ // Push current title to title stack mTerm.title_stack.push_back( mTerm.title ); @@ -2335,6 +2416,20 @@ void TerminalEmulator::strhandle( void ) { int j, narg, par; mTerm.esc &= ~( ESC_STR_END | ESC_STR ); + if ( mStrescseq.discarded ) + return; + if ( mStrescseq.type == '_' && mStrescseq.len > 0 && mStrescseq.buf[0] == 'G' ) { + auto result = + mKittyGraphics.handle( std::string_view{ mStrescseq.buf + 1, mStrescseq.len - 1 }, + Vector2i( mTerm.c.x, mTerm.c.y ) ); + if ( !result.response.empty() ) + write( result.response.data(), result.response.size() ); + if ( result.changed ) + mDirty = true; + if ( result.cursorMovement != Vector2i::Zero ) + tmoveto( mTerm.c.x + result.cursorMovement.x, mTerm.c.y + result.cursorMovement.y ); + return; + } strparse(); par = ( narg = mStrescseq.narg ) ? atoi( mStrescseq.args[0] ) : 0; @@ -2851,6 +2946,7 @@ int TerminalEmulator::eschandle( uchar ascii ) { ttywrite( vtiden, strlen( vtiden ), 0 ); break; case 'c': /* RIS -- Reset to initial state */ + mKittyGraphics.reset(); treset(); resettitle(); loadColors(); @@ -2979,23 +3075,28 @@ void TerminalEmulator::tputc( Rune u ) { goto check_control_code; } + if ( mStrescseq.discarded ) + return; + + const bool kittyGraphics = + mStrescseq.type == '_' && mStrescseq.len > 0 && mStrescseq.buf[0] == 'G'; + const size_t sequenceLimit = + kittyGraphics ? MAX_KITTY_GRAPHICS_APC_SIZE : MAX_GENERIC_STRING_SEQUENCE_SIZE; + if ( len > sequenceLimit - mStrescseq.len ) { + mStrescseq.discarded = true; + return; + } + if ( mStrescseq.len + len >= mStrescseq.siz ) { - /* - * Here is a bug in terminals. If the user never sends - * some code to stop the str or esc command, then st - * will stop responding. But this is better than - * silently failing with unknown characters. At least - * then users will report back. - * - * In the case users ever get fixed, here is the code: - */ - /* - * term.esc = 0; - * strhandle(); - */ - if ( mStrescseq.siz > ( SIZE_MAX - UTF_SIZ ) / 2 ) + const size_t required = mStrescseq.len + len + 1; + size_t newSize = mStrescseq.siz; + while ( newSize < required && newSize < sequenceLimit ) + newSize = eemin( newSize * 2, sequenceLimit ); + if ( newSize < required ) { + mStrescseq.discarded = true; return; - mStrescseq.siz *= 2; + } + mStrescseq.siz = newSize; mStrescseq.buf = (char*)xrealloc( mStrescseq.buf, mStrescseq.siz ); } @@ -3045,6 +3146,35 @@ check_control_code: */ return; } + if ( mKittyPlaceholderCell.x >= 0 ) { + const int diacritic = kittyDiacriticIndex( u ); + if ( diacritic >= 0 ) { + auto* placeholder = &mTerm.line[mKittyPlaceholderCell.y][mKittyPlaceholderCell.x]; + auto metadata = mKittyPlaceholderMetadata.find( placeholder ); + if ( metadata == mKittyPlaceholderMetadata.end() ) { + mKittyPlaceholderCell = Vector2i( -1, -1 ); + return; + } + switch ( metadata->second.diacriticCount++ ) { + case 0: + metadata->second.row = static_cast( diacritic ); + break; + case 1: + metadata->second.column = static_cast( diacritic ); + break; + case 2: + if ( diacritic <= 255 ) + metadata->second.imageIdMsb = static_cast( diacritic ); + break; + default: + break; + } + mTerm.dirty[mKittyPlaceholderCell.y] = 1; + mDirty = true; + return; + } + mKittyPlaceholderCell = Vector2i( -1, -1 ); + } if ( selected( mTerm.c.x, mTerm.c.y ) ) selclear(); @@ -3065,6 +3195,15 @@ check_control_code: } tsetchar( u, &mTerm.c.attr, mTerm.c.x, mTerm.c.y ); + if ( u == 0x10EEEE ) { + mKittyPlaceholderCell = Vector2i( mTerm.c.x, mTerm.c.y ); + const Uint32 placementId = IS_TRUECOL( mKittyUnderlineColor ) + ? mKittyUnderlineColor & 0xFFFFFF + : mKittyUnderlineColor <= 255 ? mKittyUnderlineColor + : 0; + mKittyPlaceholderMetadata[&mTerm.line[mTerm.c.y][mTerm.c.x]] = + KittyPlaceholderMetadata{ placementId }; + } mTerm.lastc = u; if ( width == 2 ) { @@ -3123,6 +3262,36 @@ void TerminalEmulator::tresize( int col, int row ) { int save_end = 0; int loaded = 0; bool is_alt = IS_SET( MODE_ALTSCREEN ); + std::vector primaryPlaceholderMetadata; + std::vector alternatePlaceholderMetadata; + auto collectPlaceholderMetadata = [&]( Line line, int columns, + std::vector& output ) { + if ( !line ) + return; + for ( int column = 0; column < columns; ++column ) { + if ( line[column].u != 0x10EEEE ) + continue; + auto metadata = mKittyPlaceholderMetadata.find( &line[column] ); + if ( metadata != mKittyPlaceholderMetadata.end() ) + output.emplace_back( metadata->second ); + } + }; + if ( mTerm.col > 0 ) { + for ( int history = 0; history < mTerm.histlen; ++history ) { + const int index = + ( mTerm.histi - mTerm.histlen + 1 + history + mTerm.histsize ) % mTerm.histsize; + collectPlaceholderMetadata( mTerm.hist[index], mTerm.col, primaryPlaceholderMetadata ); + } + Line* primaryLines = is_alt ? mTerm.alt : mTerm.line; + Line* alternateLines = is_alt ? mTerm.line : mTerm.alt; + for ( int line = 0; line < mTerm.row; ++line ) { + collectPlaceholderMetadata( primaryLines[line], mTerm.col, primaryPlaceholderMetadata ); + collectPlaceholderMetadata( alternateLines[line], mTerm.col, + alternatePlaceholderMetadata ); + } + } + mKittyPlaceholderMetadata.clear(); + mKittyPlaceholderCell = Vector2i( -1, -1 ); if ( col < 1 || row < 1 ) { terminalDiagnostic( "tresize: error resizing to %dx%d\n", col, row ); @@ -3304,6 +3473,32 @@ void TerminalEmulator::tresize( int col, int row ) { eemax( mTerm.scr - mTerm.histlen, eemin( mTerm.scr + mTerm.row - 1, mSel.oe.y ) ); selnormalize(); } + auto restorePlaceholderMetadata = [&]( Line line, int columns, + const std::vector& metadata, + size_t& index ) { + if ( !line ) + return; + for ( int column = 0; column < columns && index < metadata.size(); ++column ) { + if ( line[column].u == 0x10EEEE ) + mKittyPlaceholderMetadata[&line[column]] = metadata[index++]; + } + }; + size_t primaryMetadataIndex = 0; + for ( int history = 0; history < mTerm.histlen; ++history ) { + const int index = + ( mTerm.histi - mTerm.histlen + 1 + history + mTerm.histsize ) % mTerm.histsize; + restorePlaceholderMetadata( mTerm.hist[index], mTerm.col, primaryPlaceholderMetadata, + primaryMetadataIndex ); + } + Line* primaryLines = is_alt ? mTerm.alt : mTerm.line; + Line* alternateLines = is_alt ? mTerm.line : mTerm.alt; + size_t alternateMetadataIndex = 0; + for ( int line = 0; line < mTerm.row; ++line ) { + restorePlaceholderMetadata( primaryLines[line], mTerm.col, primaryPlaceholderMetadata, + primaryMetadataIndex ); + restorePlaceholderMetadata( alternateLines[line], mTerm.col, alternatePlaceholderMetadata, + alternateMetadataIndex ); + } mDirty = true; onScrollPositionChange(); @@ -3357,6 +3552,63 @@ void TerminalEmulator::draw() { cx--; drawregion( *dpy, 0, 0, mTerm.col, mTerm.row ); + std::vector placeholderCells; + const bool scanPlaceholders = mKittyGraphics.hasVirtualPlacements(); + for ( int y = 0; scanPlaceholders && y < mTerm.row; ++y ) { + const TerminalGlyph* previous = nullptr; + Uint32 previousPlacementId = 0; + Uint32 previousRow = 0; + Uint32 previousColumn = 0; + Uint8 previousMsb = 0; + for ( int x = 0; x < mTerm.col; ++x ) { + const auto& glyph = TLINE( y )[x]; + if ( glyph.u != 0x10EEEE ) { + previous = nullptr; + continue; + } + auto metadata = mKittyPlaceholderMetadata.find( &glyph ); + if ( metadata == mKittyPlaceholderMetadata.end() ) { + previous = nullptr; + continue; + } + Uint32 row = metadata->second.row; + Uint32 column = metadata->second.column; + Uint8 msb = metadata->second.imageIdMsb; + const Uint32 placementId = metadata->second.placementId; + const bool sameColors = + previous && previous->fg == glyph.fg && previousPlacementId == placementId; + if ( metadata->second.diacriticCount == 0 && sameColors ) { + row = previousRow; + column = previousColumn + 1; + msb = previousMsb; + } else if ( metadata->second.diacriticCount == 1 && sameColors && + row == previousRow ) { + column = previousColumn + 1; + msb = previousMsb; + } else if ( metadata->second.diacriticCount == 2 && sameColors && + row == previousRow && column == previousColumn + 1 ) { + msb = previousMsb; + } + if ( row == UINT16_MAX || column == UINT16_MAX ) { + previous = nullptr; + continue; + } + const Uint32 lowImageId = IS_TRUECOL( glyph.fg ) ? glyph.fg & 0xFFFFFF + : glyph.fg <= 255 ? glyph.fg + : 0; + placeholderCells.push_back( { lowImageId | ( static_cast( msb ) << 24 ), + placementId, Vector2i( x, y ), row, column } ); + previous = &glyph; + previousRow = row; + previousColumn = column; + previousMsb = msb; + previousPlacementId = placementId; + } + } + mKittyGraphics.setPlaceholderCells( std::move( placeholderCells ) ); + auto graphicsUpdates = mKittyGraphics.takeUpdates(); + if ( !graphicsUpdates.empty() || mKittyGraphics.hasPendingPresentation() ) + dpy->drawGraphics( mKittyGraphics.takePresentation(), std::move( graphicsUpdates ) ); if ( mTerm.scr == 0 ) dpy->drawCursor( cx, mTerm.c.y, mTerm.line[mTerm.c.y][cx], mTerm.ocx, mTerm.ocy, @@ -3379,6 +3631,7 @@ void TerminalEmulator::redraw() { } void TerminalEmulator::reset() { + mKittyGraphics.reset(); treset(); redraw(); } @@ -3448,8 +3701,10 @@ void TerminalEmulator::reportColorScheme() { ttywrite( buf, len, 0 ); } -void TerminalEmulator::mousereport( const TerminalMouseEventType& type, const Vector2i& pos, +void TerminalEmulator::mousereport( const TerminalMouseEventType& type, + const Vector2i& cellPosition, const Vector2i& pixelPosition, const Uint32& flags, const Uint32& mod ) { + const Vector2i& pos = xgetmode( MODE_MOUSESGR_PIXELS ) ? pixelPosition : cellPosition; if ( !xgetmode( (TerminalWinMode)MODE_MOUSE ) && !xgetmode( MODE_MOUSESGR ) && ( TerminalMouseEventType::MouseButtonDown == type || TerminalMouseEventType::MouseButtonRelease == type ) ) { @@ -3560,6 +3815,10 @@ void TerminalEmulator::mousereport( const TerminalMouseEventType& type, const Ve } void TerminalEmulator::setPtyAndProcess( PtyPtr&& pty, ProcPtr&& process ) { + mKittyGraphics.reset(); + mKittyPlaceholderMetadata.clear(); + mKittyPlaceholderCell = Vector2i( -1, -1 ); + mBuflen = 0; mStatus = STARTING; mExitCode = 1; mPty = std::move( pty ); @@ -3654,6 +3913,7 @@ void TerminalEmulator::onProcessExit( int exitCode ) { } void TerminalEmulator::onScrollPositionChange() { + mKittyGraphics.setViewport( mTerm.scr, mTerm.histlen, mTerm.row ); auto dpy = mDpy.lock(); if ( dpy ) dpy->onScrollPositionChange(); @@ -3704,11 +3964,19 @@ int TerminalEmulator::write( const char* buf, size_t buflen ) { } void TerminalEmulator::resize( int columns, int rows ) { + resize( columns, rows, mPixelWidth, mPixelHeight ); +} + +void TerminalEmulator::resize( int columns, int rows, int pixelWidth, int pixelHeight ) { + mPixelWidth = eemax( 0, pixelWidth ); + mPixelHeight = eemax( 0, pixelHeight ); + mKittyGraphics.setCellPixelSize( columns > 0 ? mPixelWidth / columns : 0, + rows > 0 ? mPixelHeight / rows : 0 ); bool is_alt = IS_SET( MODE_ALTSCREEN ); // Alt doesn't need reflow, we can resize and redraw instantly which looks and feels better if ( is_alt ) { - if ( !mPty->resize( columns, rows ) ) { + if ( !mPty->resize( columns, rows, mPixelWidth, mPixelHeight ) ) { _die( "Failed to resize pty!" ); return; } @@ -3722,6 +3990,8 @@ void TerminalEmulator::resize( int columns, int rows ) { redraw(); mPendingPtyColumns = columns; mPendingPtyRows = rows; + mPendingPtyPixelWidth = mPixelWidth; + mPendingPtyPixelHeight = mPixelHeight; mPendingPtyResize = true; mPendingPtyResizeClock.restart(); } @@ -3729,10 +3999,13 @@ void TerminalEmulator::resize( int columns, int rows ) { #define MAX_TTY_READS ( 1024 ) bool TerminalEmulator::update() { + if ( mKittyGraphics.updateAnimations() ) + mDirty = true; if ( mPendingPtyResize && mPendingPtyResizeClock.getElapsedTime() >= Milliseconds( 100 ) ) { mPendingPtyResize = false; - if ( !mPty->resize( mPendingPtyColumns, mPendingPtyRows ) ) { + if ( !mPty->resize( mPendingPtyColumns, mPendingPtyRows, mPendingPtyPixelWidth, + mPendingPtyPixelHeight ) ) { _die( "Failed to resize pty!" ); } diff --git a/src/modules/eterm/src/eterm/terminal/terminalgraphics.cpp b/src/modules/eterm/src/eterm/terminal/terminalgraphics.cpp new file mode 100644 index 000000000..39d5bc6ef --- /dev/null +++ b/src/modules/eterm/src/eterm/terminal/terminalgraphics.cpp @@ -0,0 +1,77 @@ +#include + +namespace eterm { namespace Terminal { + +TerminalGraphicsUpdateQueue::TerminalGraphicsUpdateQueue( size_t maxUpdates, size_t maxBytes ) : + mMaxUpdates( maxUpdates ), mMaxBytes( maxBytes ) {} + +Uint64 TerminalGraphicsUpdateQueue::enqueue( TerminalGraphicsUpdate update ) { + std::lock_guard lock( mMutex ); + const size_t payloadBytes = update.payloadBytes(); + if ( mNeedsResync ) + return ++mNextSequence; + + auto isFullImageUpdate = []( TerminalGraphicsUpdateType type ) { + return type == TerminalGraphicsUpdateType::CreateImage || + type == TerminalGraphicsUpdateType::ReplaceImage; + }; + if ( !mUpdates.empty() && isFullImageUpdate( update.type ) && + isFullImageUpdate( mUpdates.back().type ) && mUpdates.back().imageId == update.imageId ) { + const size_t previousBytes = mUpdates.back().payloadBytes(); + if ( payloadBytes <= mMaxBytes - ( mQueuedBytes - previousBytes ) ) { + update.sequence = mUpdates.back().sequence; + if ( mUpdates.back().type == TerminalGraphicsUpdateType::CreateImage ) + update.type = TerminalGraphicsUpdateType::CreateImage; + mQueuedBytes = mQueuedBytes - previousBytes + payloadBytes; + mUpdates.back() = std::move( update ); + return mUpdates.back().sequence; + } + } + + update.sequence = ++mNextSequence; + + if ( mUpdates.size() >= mMaxUpdates || mQueuedBytes > mMaxBytes || + payloadBytes > mMaxBytes - mQueuedBytes ) { + mUpdates.clear(); + mQueuedBytes = 0; + mNeedsResync = true; + TerminalGraphicsUpdate resync; + resync.sequence = update.sequence; + resync.type = TerminalGraphicsUpdateType::Resync; + mUpdates.emplace_back( std::move( resync ) ); + return update.sequence; + } + + mQueuedBytes += payloadBytes; + mUpdates.emplace_back( std::move( update ) ); + return mNextSequence; +} + +std::vector TerminalGraphicsUpdateQueue::drain() { + std::vector updates; + std::lock_guard lock( mMutex ); + updates.reserve( mUpdates.size() ); + while ( !mUpdates.empty() ) { + updates.emplace_back( std::move( mUpdates.front() ) ); + mUpdates.pop_front(); + } + mQueuedBytes = 0; + return updates; +} + +size_t TerminalGraphicsUpdateQueue::queuedBytes() const { + std::lock_guard lock( mMutex ); + return mQueuedBytes; +} + +bool TerminalGraphicsUpdateQueue::needsResync() const { + std::lock_guard lock( mMutex ); + return mNeedsResync; +} + +void TerminalGraphicsUpdateQueue::resetResync() { + std::lock_guard lock( mMutex ); + mNeedsResync = false; +} + +}} // namespace eterm::Terminal diff --git a/src/modules/eterm/src/eterm/terminal/terminalsession.cpp b/src/modules/eterm/src/eterm/terminal/terminalsession.cpp index eb270fc62..ae6132e05 100644 --- a/src/modules/eterm/src/eterm/terminal/terminalsession.cpp +++ b/src/modules/eterm/src/eterm/terminal/terminalsession.cpp @@ -20,7 +20,10 @@ struct TerminalSession::SelectionResponse { class TerminalSession::WorkerDisplay final : public ITerminalDisplay { public: WorkerDisplay( TerminalSession& session, TerminalColorPalette palette ) : - mSession( session ), mInitialPalette( std::move( palette ) ), mPalette( mInitialPalette ) { + mSession( session ), + mInitialPalette( std::move( palette ) ), + mPalette( mInitialPalette ), + mGraphics( std::make_shared() ) { mMode |= MODE_FOCUSED; } @@ -58,8 +61,20 @@ class TerminalSession::WorkerDisplay final : public ITerminalDisplay { mCursorVisible = true; } + void drawGraphics( std::shared_ptr presentation, + std::vector updates ) { + if ( !presentation ) + presentation = std::make_shared(); + presentation->requiredUpdateSequence = mGraphics ? mGraphics->requiredUpdateSequence : 0; + for ( auto& update : updates ) + presentation->requiredUpdateSequence = + mSession.enqueueGraphicsUpdate( std::move( update ) ); + mGraphics = std::move( presentation ); + } + void drawEnd() { auto snapshot = std::make_shared(); + snapshot->graphics = mGraphics; snapshot->cells = mCells; snapshot->dirtyRows = mDirtyRows; snapshot->title = mTitle; @@ -232,6 +247,7 @@ class TerminalSession::WorkerDisplay final : public ITerminalDisplay { std::vector mCells; std::vector mDirtyRows; std::string mTitle; + std::shared_ptr mGraphics; Uint64 mGeneration{ 0 }; Uint64 mLastAppliedScrollCommand{ 0 }; Vector2i mCursor; @@ -302,7 +318,11 @@ void TerminalSession::writeRaw( std::string data ) { } void TerminalSession::resize( int columns, int rows ) { - enqueue( ResizeCommand{ columns, rows } ); + resize( columns, rows, 0, 0 ); +} + +void TerminalSession::resize( int columns, int rows, int pixelWidth, int pixelHeight ) { + enqueue( ResizeCommand{ columns, rows, pixelWidth, pixelHeight } ); } void TerminalSession::scrollUp( int amount ) { @@ -334,9 +354,9 @@ void TerminalSession::selectionClear() { enqueue( SelectionClearCommand{} ); } -void TerminalSession::mouseReport( TerminalMouseEventType type, Vector2i position, Uint32 flags, - Uint32 modifiers ) { - enqueue( MouseCommand{ type, position, flags, modifiers } ); +void TerminalSession::mouseReport( TerminalMouseEventType type, Vector2i cellPosition, + Vector2i pixelPosition, Uint32 flags, Uint32 modifiers ) { + enqueue( MouseCommand{ type, cellPosition, pixelPosition, flags, modifiers } ); } void TerminalSession::setFocus( bool focus ) { @@ -383,6 +403,10 @@ void TerminalSession::restart( PtyPtr&& pty, ProcPtr&& process ) { enqueue( RestartCommand{ std::move( pty ), std::move( process ) } ); } +void TerminalSession::requestGraphicsResync() { + enqueue( GraphicsResyncCommand{} ); +} + std::shared_ptr TerminalSession::snapshot() const { std::lock_guard lock( mPublishedSnapshotMutex ); return mPublishedSnapshot; @@ -410,6 +434,14 @@ std::vector TerminalSession::drainEvents() { return events; } +std::vector TerminalSession::drainGraphicsUpdates() { + return mGraphicsUpdates.drain(); +} + +Uint64 TerminalSession::enqueueGraphicsUpdate( TerminalGraphicsUpdate update ) { + return mGraphicsUpdates.enqueue( std::move( update ) ); +} + void TerminalSession::enqueueEvent( Event event, bool coalescable ) { std::lock_guard lock( mEventMutex ); if ( coalescable ) { @@ -480,7 +512,7 @@ void TerminalSession::processCommand( Command&& command ) { } else if constexpr ( std::is_same_v ) { mEmulator->write( value.data.data(), value.data.size() ); } else if constexpr ( std::is_same_v ) { - mEmulator->resize( value.columns, value.rows ); + mEmulator->resize( value.columns, value.rows, value.pixelWidth, value.pixelHeight ); } else if constexpr ( std::is_same_v ) { TerminalArg argument( value.amount ); if ( value.direction < 0 ) @@ -502,7 +534,8 @@ void TerminalSession::processCommand( Command&& command ) { mEmulator->selclear(); mEmulator->redraw(); } else if constexpr ( std::is_same_v ) { - mEmulator->mousereport( value.type, value.position, value.flags, value.modifiers ); + mEmulator->mousereport( value.type, value.cellPosition, value.pixelPosition, + value.flags, value.modifiers ); } else if constexpr ( std::is_same_v ) { if ( mWorkerDisplay->getMode( MODE_FOCUS ) ) mEmulator->ttywrite( value.value ? "\033[I" : "\033[O", 3, false ); @@ -556,6 +589,9 @@ void TerminalSession::processCommand( Command&& command ) { value.response->selection = mEmulator->getSelection(); value.response->ready = true; value.response->condition.notify_one(); + } else if constexpr ( std::is_same_v ) { + mGraphicsUpdates.resetResync(); + mEmulator->requestGraphicsResync(); } }, std::move( command ) ); diff --git a/src/tests/unit_tests/eterm_tests.cpp b/src/tests/unit_tests/eterm_tests.cpp index 0570bec89..d5987e2a4 100644 --- a/src/tests/unit_tests/eterm_tests.cpp +++ b/src/tests/unit_tests/eterm_tests.cpp @@ -1,16 +1,21 @@ #include "utest.hpp" #include #include +#include +#include +#include #include #include #include #include +#include #include #include #include using namespace eterm::Terminal; using namespace eterm::System; +using namespace EE::System; class MockPty : public IPseudoTerminal { public: @@ -22,11 +27,15 @@ class MockPty : public IPseudoTerminal { std::atomic mBytesRead{ 0 }; int mCols = 80; int mRows = 24; + int mPixelWidth = 0; + int mPixelHeight = 0; int getNumColumns() const override { return mCols; } int getNumRows() const override { return mRows; } - bool resize( int columns, int rows ) override { + bool resize( int columns, int rows, int pixelWidth, int pixelHeight ) override { mCols = columns; mRows = rows; + mPixelWidth = pixelWidth; + mPixelHeight = pixelHeight; return true; } bool isTTY() const override { return true; } @@ -84,6 +93,8 @@ UTEST( eterm_session, command_wakeup_and_snapshot_immutability ) { snapshot.cells[1].u == 'B' && snapshot.cells[2].u == 'C'; } ); ASSERT_TRUE( first != nullptr ); + ASSERT_TRUE( first->graphics != nullptr ); + EXPECT_EQ( static_cast( 0 ), first->graphics->requiredUpdateSequence ); const Uint64 firstGeneration = first->generation; session->writeRaw( "\rXYZ" ); @@ -102,6 +113,68 @@ UTEST( eterm_session, skipped_snapshot_generation_requires_full_redraw ) { EXPECT_FALSE( snapshot.dirtyRowsFollow( 40 ) ); } +UTEST( eterm_session, graphics_update_queue_preserves_order_and_payloads ) { + TerminalGraphicsUpdateQueue queue; + auto pixels = std::make_shared>( 16, 0x7F ); + TerminalGraphicsUpdate create; + create.type = TerminalGraphicsUpdateType::CreateImage; + create.imageId = 7; + create.rgba = pixels; + TerminalGraphicsUpdate patch; + patch.type = TerminalGraphicsUpdateType::UpdateRegion; + patch.imageId = 7; + patch.rgba = pixels; + + EXPECT_EQ( static_cast( 1 ), queue.enqueue( std::move( create ) ) ); + EXPECT_EQ( static_cast( 2 ), queue.enqueue( std::move( patch ) ) ); + EXPECT_EQ( static_cast( 32 ), queue.queuedBytes() ); + auto updates = queue.drain(); + ASSERT_EQ( static_cast( 2 ), updates.size() ); + EXPECT_EQ( static_cast( 1 ), updates[0].sequence ); + EXPECT_EQ( static_cast( 2 ), updates[1].sequence ); + EXPECT_EQ( TerminalGraphicsUpdateType::CreateImage, updates[0].type ); + EXPECT_EQ( TerminalGraphicsUpdateType::UpdateRegion, updates[1].type ); + EXPECT_TRUE( pixels == updates[0].rgba ); +} + +UTEST( eterm_session, graphics_update_queue_overflow_requires_resync ) { + TerminalGraphicsUpdateQueue queue( 2, 8 ); + TerminalGraphicsUpdate update; + update.type = TerminalGraphicsUpdateType::UpdateRegion; + update.rgba = std::make_shared>( 8, 0xFF ); + queue.enqueue( update ); + queue.enqueue( std::move( update ) ); + + EXPECT_TRUE( queue.needsResync() ); + auto updates = queue.drain(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::Resync, updates[0].type ); + EXPECT_EQ( static_cast( 2 ), updates[0].sequence ); + EXPECT_EQ( static_cast( 0 ), queue.queuedBytes() ); +} + +UTEST( eterm_session, graphics_update_queue_coalesces_superseded_video_frames ) { + TerminalGraphicsUpdateQueue queue( 4, 8 ); + TerminalGraphicsUpdate create; + create.type = TerminalGraphicsUpdateType::CreateImage; + create.imageId = 7; + create.rgba = std::make_shared>( 8, 1 ); + EXPECT_EQ( static_cast( 1 ), queue.enqueue( std::move( create ) ) ); + for ( Uint8 frame = 2; frame < 20; ++frame ) { + TerminalGraphicsUpdate replacement; + replacement.type = TerminalGraphicsUpdateType::ReplaceImage; + replacement.imageId = 7; + replacement.rgba = std::make_shared>( 8, frame ); + EXPECT_EQ( static_cast( 1 ), queue.enqueue( std::move( replacement ) ) ); + } + EXPECT_FALSE( queue.needsResync() ); + EXPECT_EQ( static_cast( 8 ), queue.queuedBytes() ); + auto updates = queue.drain(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::CreateImage, updates[0].type ); + EXPECT_EQ( static_cast( 19 ), updates[0].rgba->front() ); +} + UTEST( eterm_session, ordered_selection_request ) { auto pty = std::make_unique(); pty->mBuffer = "ordered selection"; @@ -347,6 +420,7 @@ class MockDisplay : public ITerminalDisplay { std::vector mResetColorIndices; int mResetColorsCount{ 0 }; Uint32 mBackground{ 0x101010FF }; + std::shared_ptr mGraphics; bool drawBegin( Uint32, Uint32 ) override { return true; } void drawLine( Line line, int, int y, int ) override { ++mDrawLines; @@ -360,6 +434,10 @@ class MockDisplay : public ITerminalDisplay { void drawCursor( int, int, TerminalGlyph, int, int, TerminalGlyph ) override {} void drawEnd() override { ++mDrawEnds; } void resetColors() override { ++mResetColorsCount; } + void drawGraphics( std::shared_ptr presentation, + std::vector ) override { + mGraphics = std::move( presentation ); + } int resetColor( const Uint32& index, const char* ) override { mResetColorIndices.emplace_back( index ); return 0; @@ -381,6 +459,424 @@ class MockDisplay : public ITerminalDisplay { } }; +UTEST( eterm, kitty_graphics_parser_preserves_payload_and_types_action ) { + auto result = KittyGraphicsProtocol::parse( + "a=T,f=32,s=2,v=1,i=7,p=9,q=1,C=1,z=-3,future=value;AAAA;BBBB" ); + ASSERT_TRUE( result.command.has_value() ); + auto* transmit = std::get_if( &*result.command ); + ASSERT_TRUE( transmit != nullptr ); + EXPECT_TRUE( transmit->display ); + EXPECT_EQ( static_cast( 32 ), *transmit->data.format ); + EXPECT_EQ( static_cast( 2 ), *transmit->data.width ); + EXPECT_EQ( static_cast( 1 ), *transmit->data.height ); + EXPECT_EQ( static_cast( -3 ), *transmit->data.zIndex ); + EXPECT_STDSTREQ( "AAAA;BBBB", std::string( transmit->data.payload ) ); +} + +UTEST( eterm, kitty_graphics_parser_rejects_invalid_control_data ) { + EXPECT_EQ( KittyGraphicsError::InvalidArgument, + KittyGraphicsProtocol::parse( "a=T,m=2;AAAA" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidArgument, + KittyGraphicsProtocol::parse( "a=T,i=1,I=2;AAAA" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidArgument, + KittyGraphicsProtocol::parse( "a=T,s=4294967296;AAAA" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidArgument, + KittyGraphicsProtocol::parse( "a=unknown;AAAA" ).error ); +} + +UTEST( eterm, kitty_graphics_parser_fuzz_corpus_is_bounded_and_total ) { + Uint32 state = 0xC0FFEEu; + for ( size_t iteration = 0; iteration < 5000; ++iteration ) { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + const size_t length = state % 512; + std::string input( length, '\0' ); + for ( char& character : input ) { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + character = static_cast( state & 0x7F ); + } + const auto result = KittyGraphicsProtocol::parse( input ); + EXPECT_TRUE( result.command.has_value() || result.error != KittyGraphicsError::None ); + } +} + +UTEST( eterm, kitty_graphics_direct_rgba_chunks_create_worker_image ) { + KittyGraphicsProtocol protocol; + auto first = protocol.handle( "a=t,f=32,s=1,v=1,i=7,m=1;AQID" ); + EXPECT_EQ( KittyGraphicsError::None, first.error ); + EXPECT_FALSE( first.changed ); + auto final = protocol.handle( "m=0;BA==" ); + EXPECT_EQ( KittyGraphicsError::None, final.error ); + EXPECT_TRUE( final.changed ); + EXPECT_STDSTREQ( "\033_Gi=7;OK\033\\", final.response ); + + auto pixels = protocol.imagePixels( 7 ); + ASSERT_TRUE( pixels != nullptr ); + ASSERT_EQ( static_cast( 4 ), pixels->size() ); + EXPECT_EQ( static_cast( 1 ), ( *pixels )[0] ); + EXPECT_EQ( static_cast( 4 ), ( *pixels )[3] ); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::CreateImage, updates[0].type ); +} + +UTEST( eterm, kitty_graphics_chunk_continuations_reject_metadata_and_wrong_action ) { + KittyGraphicsProtocol protocol; + EXPECT_EQ( KittyGraphicsError::None, protocol.handle( "a=t,f=32,s=1,v=1,i=8,m=1;AQID" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidArgument, protocol.handle( "m=0,s=1;BA==" ).error ); + EXPECT_EQ( static_cast( 0 ), protocol.imageCount() ); + EXPECT_EQ( KittyGraphicsError::None, protocol.handle( "a=t,f=32,s=1,v=1,i=8,m=1;AQID" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidArgument, protocol.handle( "a=p,i=8" ).error ); + EXPECT_EQ( static_cast( 0 ), protocol.imageCount() ); + + protocol.handle( "a=t,f=32,s=1,v=1,i=8;AQIDBA==" ); + EXPECT_EQ( KittyGraphicsError::None, protocol.handle( "a=f,i=8,f=32,s=1,v=1,m=1;AQID" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidArgument, protocol.handle( "m=0;BA==" ).error ); +} + +UTEST( eterm, kitty_graphics_image_number_allocates_id_and_echoes_number ) { + KittyGraphicsProtocol protocol; + auto created = protocol.handle( "a=t,f=32,s=1,v=1,I=77;AQIDBA==" ); + EXPECT_TRUE( created.changed ); + EXPECT_TRUE( created.response.find( ",I=77;OK" ) != std::string::npos ); + auto placed = protocol.handle( "a=p,I=77,p=3" ); + EXPECT_EQ( KittyGraphicsError::None, placed.error ); + ASSERT_EQ( static_cast( 1 ), protocol.takePresentation()->placements.size() ); +} + +UTEST( eterm, kitty_graphics_rgb_and_zlib_normalize_to_rgba ) { + const std::vector rgb{ 10, 20, 30, 40, 50, 60 }; + std::vector compressed( Compression::getMaxCompressedBufferSize( rgb.size() ) ); + IOStreamMemory source( reinterpret_cast( rgb.data() ), rgb.size() ); + IOStreamMemory destination( reinterpret_cast( compressed.data() ), compressed.size() ); + ASSERT_EQ( Compression::OK, Compression::compress( destination, source ) ); + compressed.resize( destination.tell() ); + std::string encoded; + ASSERT_TRUE( Base64::encode( + std::string_view( reinterpret_cast( compressed.data() ), compressed.size() ), + encoded ) ); + + KittyGraphicsProtocol protocol; + auto result = protocol.handle( "a=t,f=24,s=2,v=1,i=9,o=z;" + encoded ); + EXPECT_TRUE( result.changed ); + auto pixels = protocol.imagePixels( 9 ); + ASSERT_TRUE( pixels != nullptr ); + const std::vector expected{ 10, 20, 30, 255, 40, 50, 60, 255 }; + EXPECT_TRUE( expected == *pixels ); +} + +UTEST( eterm, kitty_graphics_png_decodes_to_rgba ) { + KittyGraphicsProtocol protocol; + const auto result = protocol.handle( "a=t,f=100,i=10;" + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42" + "mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" ); + EXPECT_EQ( KittyGraphicsError::None, result.error ); + EXPECT_TRUE( result.changed ); + const auto* pixels = protocol.imagePixels( 10 ); + ASSERT_TRUE( pixels != nullptr ); + EXPECT_EQ( static_cast( 4 ), pixels->size() ); +} + +UTEST( eterm, kitty_graphics_placement_uses_final_cursor_and_geometry ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=T,f=32,s=1,v=1,i=21,p=4,c=2,r=3,C=1,x=0,y=0,w=1,h=1;AQIDBA==", + Vector2i( 5, 6 ) ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), presentation->placements.size() ); + const auto& placement = presentation->placements.front(); + EXPECT_EQ( static_cast( 21 ), placement.imageId ); + EXPECT_EQ( static_cast( 4 ), placement.placementId ); + EXPECT_EQ( 5, placement.visibleAnchorCell.x ); + EXPECT_EQ( 6, placement.visibleAnchorCell.y ); + EXPECT_EQ( static_cast( 2 ), placement.columns ); + EXPECT_EQ( static_cast( 3 ), placement.rows ); + + auto put = protocol.handle( "a=p,i=21,p=5,c=4,r=2", Vector2i( 1, 2 ) ); + EXPECT_EQ( 4, put.cursorMovement.x ); + EXPECT_EQ( 2, put.cursorMovement.y ); + presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 2 ), presentation->placements.size() ); +} + +UTEST( eterm, kitty_graphics_placement_derives_missing_cell_geometry ) { + KittyGraphicsProtocol protocol; + protocol.setCellPixelSize( 10, 20 ); + protocol.handle( "a=t,f=32,s=2,v=2,i=22;AAAAAAAAAAAAAAAAAAAAAA==" ); + auto result = protocol.handle( "a=p,i=22,X=9,Y=19,C=1" ); + EXPECT_TRUE( result.changed ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), presentation->placements.size() ); + EXPECT_EQ( static_cast( 2 ), presentation->placements[0].columns ); + EXPECT_EQ( static_cast( 2 ), presentation->placements[0].rows ); +} + +UTEST( eterm, kitty_graphics_retransmit_removes_old_placements_and_crop_intersects ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=T,f=32,s=2,v=2,i=23,p=4;AAAAAAAAAAAAAAAAAAAAAA==" ); + ASSERT_EQ( static_cast( 1 ), protocol.takePresentation()->placements.size() ); + protocol.handle( "a=t,f=32,s=1,v=1,i=23;AQIDBA==" ); + EXPECT_TRUE( protocol.takePresentation()->placements.empty() ); + protocol.handle( "a=t,f=32,s=2,v=2,i=24;AAAAAAAAAAAAAAAAAAAAAA==" ); + auto placed = protocol.handle( "a=p,i=24,p=5,x=1,y=1,w=99,h=99" ); + EXPECT_STDSTREQ( "\033_Gi=24,p=5;OK\033\\", placed.response ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), presentation->placements.size() ); + EXPECT_EQ( 2, presentation->placements[0].sourcePixels.Right ); + EXPECT_EQ( 2, presentation->placements[0].sourcePixels.Bottom ); +} + +UTEST( eterm, kitty_graphics_resync_republishes_authoritative_images ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=t,f=32,s=1,v=1,i=27;AQIDBA==" ); + protocol.takeUpdates(); + protocol.resync(); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 2 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::ResetAll, updates[0].type ); + EXPECT_EQ( TerminalGraphicsUpdateType::CreateImage, updates[1].type ); + EXPECT_EQ( static_cast( 27 ), updates[1].imageId ); + ASSERT_TRUE( updates[1].rgba != nullptr ); + EXPECT_EQ( static_cast( 4 ), updates[1].rgba->size() ); +} + +UTEST( eterm, kitty_graphics_root_frame_patch_publishes_only_changed_rectangle ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=t,f=32,s=2,v=1,i=29;AQIDBAUGBwg=" ); + protocol.takeUpdates(); + auto result = protocol.handle( "a=f,i=29,r=1,f=32,s=1,v=1,x=1,y=0,X=1;CQoLDA==" ); + EXPECT_TRUE( result.changed ); + const auto* pixels = protocol.imagePixels( 29 ); + ASSERT_TRUE( pixels != nullptr ); + const std::vector expected{ 1, 2, 3, 4, 9, 10, 11, 12 }; + EXPECT_TRUE( expected == *pixels ); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::UpdateRegion, updates[0].type ); + EXPECT_EQ( 1, updates[0].region.Left ); + EXPECT_EQ( 0, updates[0].region.Top ); + EXPECT_EQ( 2, updates[0].region.Right ); + EXPECT_EQ( 1, updates[0].region.Bottom ); + ASSERT_TRUE( updates[0].rgba != nullptr ); + EXPECT_EQ( static_cast( 4 ), updates[0].rgba->size() ); +} + +UTEST( eterm, kitty_graphics_animation_frame_create_control_and_compose ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=T,f=32,s=2,v=1,i=30;AQIDBAUGBwg=" ); + protocol.takeUpdates(); + protocol.takePresentation(); + auto frame = protocol.handle( "a=f,i=30,c=1,f=32,s=1,v=1,x=1,y=0,X=1,z=25;CQoLDA==" ); + EXPECT_EQ( KittyGraphicsError::None, frame.error ); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::CreateFrame, updates[0].type ); + EXPECT_EQ( static_cast( 2 ), updates[0].frameNumber ); + ASSERT_TRUE( updates[0].rgba != nullptr ); + const std::vector expectedFrame{ 1, 2, 3, 4, 9, 10, 11, 12 }; + EXPECT_TRUE( expectedFrame == *updates[0].rgba ); + + auto control = protocol.handle( "a=a,i=30,c=2" ); + EXPECT_TRUE( control.changed ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), presentation->placements.size() ); + EXPECT_EQ( static_cast( 2 ), presentation->placements[0].frameNumber ); + + auto compose = protocol.handle( "a=c,i=30,r=1,c=2,X=1,Y=0,x=0,y=0,w=1,h=1,C=1" ); + EXPECT_EQ( KittyGraphicsError::None, compose.error ); + updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::UpdateFrameRegion, updates[0].type ); + ASSERT_TRUE( updates[0].rgba != nullptr ); + const std::vector expectedPatch{ 5, 6, 7, 8 }; + EXPECT_TRUE( expectedPatch == *updates[0].rgba ); + EXPECT_TRUE( protocol.handle( "a=a,i=30,c=1,r=1,z=-1,s=3" ).changed ); + EXPECT_TRUE( protocol.updateAnimations() ); + presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), presentation->placements.size() ); + EXPECT_EQ( static_cast( 2 ), presentation->placements[0].frameNumber ); + EXPECT_TRUE( protocol.handle( "a=d,d=f,i=30" ).changed ); + updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::DeleteFrame, updates[0].type ); +} + +UTEST( eterm, kitty_graphics_delete_placements_and_uppercase_frees_data ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=T,f=32,s=1,v=1,i=31,p=1,c=2,r=2;AQIDBA==", Vector2i( 3, 4 ) ); + protocol.handle( "a=p,i=31,p=2,c=1,r=1", Vector2i( 8, 9 ) ); + protocol.takeUpdates(); + + auto result = protocol.handle( "a=d,d=c", Vector2i( 4, 5 ) ); + EXPECT_TRUE( result.changed ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), presentation->placements.size() ); + EXPECT_EQ( static_cast( 2 ), presentation->placements[0].placementId ); + EXPECT_TRUE( protocol.imagePixels( 31 ) != nullptr ); + + result = protocol.handle( "a=d,d=I,i=31,p=2" ); + EXPECT_TRUE( result.changed ); + EXPECT_TRUE( protocol.imagePixels( 31 ) == nullptr ); + EXPECT_TRUE( protocol.takePresentation()->placements.empty() ); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::DeleteImage, updates[0].type ); + EXPECT_EQ( static_cast( 31 ), updates[0].imageId ); +} + +UTEST( eterm, kitty_graphics_storage_quota_evicts_oldest_unplaced_image ) { + KittyGraphicsProtocol protocol( 8, 2, 2 ); + EXPECT_TRUE( protocol.handle( "a=t,f=32,s=1,v=1,i=41;AQIDBA==" ).changed ); + EXPECT_TRUE( protocol.handle( "a=t,f=32,s=1,v=1,i=42,N=1;BQYHCA==" ).changed ); + protocol.takeUpdates(); + EXPECT_TRUE( protocol.handle( "a=t,f=32,s=1,v=1,i=43;CQoLDA==" ).changed ); + EXPECT_TRUE( protocol.imagePixels( 41 ) != nullptr ); + EXPECT_TRUE( protocol.imagePixels( 42 ) == nullptr ); + EXPECT_TRUE( protocol.imagePixels( 43 ) != nullptr ); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 2 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::DeleteImage, updates[0].type ); + EXPECT_EQ( static_cast( 42 ), updates[0].imageId ); + EXPECT_EQ( TerminalGraphicsUpdateType::CreateImage, updates[1].type ); +} + +UTEST( eterm, kitty_graphics_screen_lifecycle_restores_primary_and_resets_gpu ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=T,f=32,s=1,v=1,i=51;AQIDBA==" ); + protocol.takeUpdates(); + protocol.takePresentation(); + protocol.setAlternateScreen( true ); + EXPECT_TRUE( protocol.takePresentation()->placements.empty() ); + protocol.handle( "a=p,i=51,p=2" ); + ASSERT_EQ( static_cast( 1 ), protocol.takePresentation()->placements.size() ); + protocol.setAlternateScreen( false ); + auto primary = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), primary->placements.size() ); + EXPECT_EQ( static_cast( 0 ), primary->placements[0].placementId ); + protocol.clearScreen(); + EXPECT_TRUE( protocol.takePresentation()->placements.empty() ); + EXPECT_TRUE( protocol.imagePixels( 51 ) != nullptr ); + protocol.reset(); + EXPECT_TRUE( protocol.imagePixels( 51 ) == nullptr ); + auto updates = protocol.takeUpdates(); + ASSERT_TRUE( !updates.empty() ); + EXPECT_EQ( TerminalGraphicsUpdateType::ResetAll, updates.back().type ); +} + +UTEST( eterm, kitty_graphics_scrolling_tracks_history_and_scrollback ) { + KittyGraphicsProtocol protocol; + protocol.setViewport( 0, 0, 4 ); + protocol.handle( "a=T,f=32,s=1,v=1,i=61,r=1;AQIDBA==", Vector2i( 0, 0 ) ); + protocol.takePresentation(); + protocol.scrollScreen( 0, 3, -1, true ); + protocol.setViewport( 0, 1, 4 ); + EXPECT_TRUE( protocol.takePresentation()->placements.empty() ); + protocol.setViewport( 1, 1, 4 ); + auto history = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), history->placements.size() ); + EXPECT_EQ( 0, history->placements[0].visibleAnchorCell.y ); + protocol.setViewport( 0, 0, 4 ); + EXPECT_TRUE( protocol.takePresentation()->placements.empty() ); +} + +UTEST( eterm, kitty_graphics_margin_scroll_discards_only_scrolled_out_placements ) { + KittyGraphicsProtocol protocol; + protocol.setViewport( 0, 0, 6 ); + protocol.handle( "a=T,f=32,s=1,v=1,i=62,p=1;AQIDBA==", Vector2i( 0, 0 ) ); + protocol.handle( "a=p,i=62,p=2", Vector2i( 0, 2 ) ); + protocol.handle( "a=p,i=62,p=3", Vector2i( 0, 5 ) ); + protocol.scrollScreen( 1, 4, -2, false ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 2 ), presentation->placements.size() ); + EXPECT_EQ( static_cast( 1 ), presentation->placements[0].placementId ); + EXPECT_EQ( static_cast( 3 ), presentation->placements[1].placementId ); +} + +UTEST( eterm, kitty_graphics_margin_scroll_clips_partially_visible_placements ) { + KittyGraphicsProtocol protocol; + protocol.setViewport( 0, 0, 6 ); + protocol.handle( "a=T,f=32,s=1,v=4,i=63,p=1,c=1,r=4;AAAAAAAAAAAAAAAAAAAAAA==", + Vector2i( 0, 1 ) ); + protocol.scrollScreen( 1, 4, -2, false ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), presentation->placements.size() ); + EXPECT_EQ( 1, presentation->placements[0].visibleAnchorCell.y ); + EXPECT_EQ( static_cast( 2 ), presentation->placements[0].rows ); + EXPECT_EQ( 2, presentation->placements[0].sourcePixels.Top ); + EXPECT_EQ( 4, presentation->placements[0].sourcePixels.Bottom ); +} + +UTEST( eterm, kitty_graphics_virtual_and_relative_placements_follow_protocol_rules ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=t,f=32,s=1,v=1,i=71;AQIDBA==" ); + auto virtualPlacement = protocol.handle( "a=p,i=71,p=10,U=1,c=2,r=2", Vector2i( 2, 3 ) ); + EXPECT_TRUE( virtualPlacement.changed ); + EXPECT_EQ( 0, virtualPlacement.cursorMovement.x ); + EXPECT_TRUE( protocol.takePresentation()->placements.empty() ); + auto relative = protocol.handle( "a=p,i=71,p=11,P=71,Q=10,H=4,V=-1,c=1,r=1" ); + EXPECT_TRUE( relative.changed ); + EXPECT_EQ( 0, relative.cursorMovement.x ); + protocol.setPlaceholderCells( { { 71, 10, Vector2i( 2, 3 ), 0, 0 } } ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 2 ), presentation->placements.size() ); + auto child = std::find_if( + presentation->placements.begin(), presentation->placements.end(), + []( const TerminalVisiblePlacement& placement ) { return placement.placementId == 11; } ); + ASSERT_TRUE( child != presentation->placements.end() ); + EXPECT_EQ( 6, child->visibleAnchorCell.x ); + EXPECT_EQ( 2, child->visibleAnchorCell.y ); + protocol.handle( "a=d,d=a" ); + ASSERT_EQ( static_cast( 1 ), protocol.takePresentation()->placements.size() ); + auto replacement = protocol.handle( "a=p,i=71,p=11,P=71,Q=10,H=1,V=1" ); + EXPECT_TRUE( replacement.changed ); +} + +UTEST( eterm, kitty_graphics_relative_placements_report_missing_parents_and_cycles ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=t,f=32,s=1,v=1,i=72;AQIDBA==" ); + EXPECT_EQ( KittyGraphicsError::NoParent, protocol.handle( "a=p,i=72,p=2,P=72,Q=99" ).error ); + EXPECT_TRUE( protocol.handle( "a=p,i=72,p=1", Vector2i( 1, 1 ) ).changed ); + EXPECT_TRUE( protocol.handle( "a=p,i=72,p=2,P=72,Q=1,H=1,V=0" ).changed ); + EXPECT_EQ( KittyGraphicsError::Cycle, + protocol.handle( "a=p,i=72,p=1,P=72,Q=2,H=1,V=0" ).error ); +} + +UTEST( eterm, kitty_graphics_independent_client_namespaces_coexist ) { + KittyGraphicsProtocol protocol; + auto first = protocol.handle( "a=T,f=32,s=1,v=1,I=101,p=1;AQIDBA==" ); + auto second = protocol.handle( "a=T,f=32,s=1,v=1,I=202,p=1;BQYHCA==", Vector2i( 2, 0 ) ); + EXPECT_EQ( KittyGraphicsError::None, first.error ); + EXPECT_EQ( KittyGraphicsError::None, second.error ); + ASSERT_EQ( static_cast( 2 ), protocol.imageCount() ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 2 ), presentation->placements.size() ); + EXPECT_NE( presentation->placements[0].imageId, presentation->placements[1].imageId ); + EXPECT_TRUE( protocol.handle( "a=d,d=n,I=101" ).changed ); + ASSERT_EQ( static_cast( 1 ), protocol.takePresentation()->placements.size() ); + EXPECT_EQ( static_cast( 2 ), protocol.imageCount() ); +} + +UTEST( eterm_session, kitty_graphics_update_and_metadata_cross_worker_boundary ) { + auto pty = std::make_unique(); + pty->mBuffer = "\033_Ga=t,f=32,s=1,v=1,i=13;AQIDBA==\033\\"; + pty->mLoopWrites = false; + MockPty* ptyPtr = pty.get(); + auto process = std::make_unique(); + auto session = TerminalSession::create( std::move( pty ), std::move( process ), 100 ); + auto snapshot = waitForSnapshot( session, []( const TerminalSnapshot& value ) { + return value.graphics && value.graphics->requiredUpdateSequence > 0; + } ); + ASSERT_TRUE( snapshot != nullptr ); + EXPECT_EQ( static_cast( 1 ), snapshot->graphics->requiredUpdateSequence ); + auto updates = session->drainGraphicsUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( static_cast( 1 ), updates[0].sequence ); + EXPECT_EQ( static_cast( 13 ), updates[0].imageId ); + EXPECT_STDSTREQ( "\033_Gi=13;OK\033\\", ptyPtr->mWrites ); +} + UTEST( eterm, modern_csi_prefixes_do_not_claim_unsupported_keyboard_protocol ) { auto pty = std::make_unique(); pty->mBuffer = "\033[?u\033[>7u\033[<1u\033[mWrites.empty() ); } +UTEST( eterm, kitty_graphics_unicode_placeholder_uses_color_and_diacritics ) { + auto pty = std::make_unique(); + pty->mBuffer = "\033_Ga=t,f=32,s=2,v=2,i=72,q=2;AAAAAAAAAAAAAAAAAAAAAA==\033\\" + "\033_Ga=p,i=72,p=3,U=1,c=2,r=2,q=2\033\\" + "\033[38;5;72m\033[58;5;3m\xF4\x8E\xBB\xAE\xCC\x85\xCC\x85"; + pty->mLoopWrites = false; + auto process = std::make_unique(); + auto display = std::make_shared(); + auto term = TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); + term->update(); + ASSERT_TRUE( display->mGraphics != nullptr ); + ASSERT_EQ( static_cast( 1 ), display->mGraphics->placements.size() ); + const auto& placement = display->mGraphics->placements[0]; + EXPECT_EQ( static_cast( 72 ), placement.imageId ); + EXPECT_EQ( static_cast( 3 ), placement.placementId ); + EXPECT_EQ( 0, placement.visibleAnchorCell.x ); + EXPECT_EQ( 0, placement.visibleAnchorCell.y ); + EXPECT_EQ( 0, placement.sourcePixels.Left ); + EXPECT_EQ( 0, placement.sourcePixels.Top ); + EXPECT_EQ( 1, placement.sourcePixels.Right ); + EXPECT_EQ( 1, placement.sourcePixels.Bottom ); + term->resize( 100, 30, 1000, 600 ); + ASSERT_TRUE( display->mGraphics != nullptr ); + ASSERT_EQ( static_cast( 1 ), display->mGraphics->placements.size() ); + EXPECT_EQ( static_cast( 72 ), display->mGraphics->placements[0].imageId ); +} + +UTEST( eterm, kitty_graphics_apc_is_fragmentation_safe_and_not_terminal_text ) { + auto pty = std::make_unique(); + pty->mBuffer = "\033_Ga=q,i=31,f=32,s=1,v=1;AAAAAA==\033\\OK"; + pty->mLoopWrites = false; + pty->mMaxRead = 1; + auto process = std::make_unique(); + auto display = std::make_shared(); + auto term = TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); + + term->update(); + while ( !term->update() ) { + } + + EXPECT_EQ( static_cast( 'O' ), display->mFirstGlyph.u ); + EXPECT_EQ( static_cast( 'K' ), display->mSecondGlyph.u ); +} + +UTEST( eterm, kitty_graphics_accepts_unchunked_direct_image_larger_than_eight_kibibytes ) { + std::vector rgb( 64 * 64 * 3 ); + for ( size_t offset = 0; offset < rgb.size(); offset += 3 ) + rgb[offset] = 255; + std::string encoded; + ASSERT_TRUE( Base64::encode( + std::string_view( reinterpret_cast( rgb.data() ), rgb.size() ), encoded ) ); + const std::string command = "a=T,f=24,s=64,v=64,c=10,r=5,C=1;" + encoded; + ASSERT_TRUE( KittyGraphicsProtocol::parse( command ).command.has_value() ); + KittyGraphicsProtocol directProtocol; + EXPECT_EQ( KittyGraphicsError::None, directProtocol.handle( command ).error ); + + auto pty = std::make_unique(); + pty->mBuffer = "\033_G" + command + "\033\\"; + pty->mLoopWrites = false; + MockPty* ptyPtr = pty.get(); + auto process = std::make_unique(); + auto display = std::make_shared(); + auto term = TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); + + term->update(); + while ( !term->update() ) { + } + + EXPECT_TRUE( ptyPtr->mWrites.empty() ); + ASSERT_TRUE( display->mGraphics != nullptr ); + ASSERT_EQ( static_cast( 1 ), display->mGraphics->placements.size() ); + EXPECT_EQ( static_cast( 10 ), display->mGraphics->placements[0].columns ); + EXPECT_EQ( static_cast( 5 ), display->mGraphics->placements[0].rows ); +} + +UTEST( eterm, kitty_graphics_anonymous_video_frames_replace_at_same_anchor ) { + KittyGraphicsProtocol protocol( 8, 8, 8 ); + for ( int frame = 0; frame < 20; ++frame ) { + const char* pixels = frame % 2 == 0 ? "AQIDBA==" : "BQYHCA=="; + auto result = + protocol.handle( std::string( "a=T,f=32,s=1,v=1,q=2;" ) + pixels, Vector2i( 3, 4 ) ); + EXPECT_EQ( KittyGraphicsError::None, result.error ); + EXPECT_TRUE( result.changed ); + EXPECT_EQ( static_cast( 1 ), protocol.imageCount() ); + ASSERT_EQ( static_cast( 1 ), protocol.takePresentation()->placements.size() ); + } + const auto* pixels = protocol.imagePixels( 1 ); + ASSERT_TRUE( pixels != nullptr ); + EXPECT_EQ( static_cast( 5 ), ( *pixels )[0] ); +} + +UTEST( eterm, oversized_kitty_graphics_apc_is_discarded_until_terminator ) { + auto pty = std::make_unique(); + pty->mBuffer = "\033_Ga=t;" + std::string( MAX_KITTY_GRAPHICS_APC_SIZE, 'A' ) + "\033\\OK"; + pty->mLoopWrites = false; + auto process = std::make_unique(); + auto display = std::make_shared(); + auto term = TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); + + term->update(); + while ( !term->update() ) { + } + + EXPECT_EQ( static_cast( 'O' ), display->mFirstGlyph.u ); + EXPECT_EQ( static_cast( 'K' ), display->mSecondGlyph.u ); +} + UTEST( eterm, cursor_style_and_xterm_version_queries ) { auto pty = std::make_unique(); pty->mBuffer = "\033[0 q\033[6 q\033[>0q"; @@ -412,6 +1015,37 @@ UTEST( eterm, cursor_style_and_xterm_version_queries ) { EXPECT_STDSTREQ( "\033\\", ptyPtr->mWrites.substr( ptyPtr->mWrites.size() - 2 ) ); } +UTEST( eterm, pixel_geometry_queries_use_latest_worker_resize ) { + auto pty = std::make_unique(); + pty->mBuffer = "\033[14t\033[16t"; + pty->mLoopWrites = false; + MockPty* ptyPtr = pty.get(); + auto process = std::make_unique(); + auto display = std::make_shared(); + auto term = TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); + term->resize( 40, 20, 400, 300 ); + + term->update(); + + EXPECT_TRUE( ptyPtr->mWrites.find( "\033[4;300;400t" ) != std::string::npos ); + EXPECT_TRUE( ptyPtr->mWrites.find( "\033[6;15;10t" ) != std::string::npos ); +} + +UTEST( eterm, sgr_pixel_mouse_mode_uses_grid_relative_pixels ) { + auto pty = std::make_unique(); + pty->mBuffer = "\033[?1000h\033[?1016h"; + pty->mLoopWrites = false; + MockPty* ptyPtr = pty.get(); + auto process = std::make_unique(); + auto display = std::make_shared(); + auto term = TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); + term->update(); + term->mousereport( TerminalMouseEventType::MouseButtonDown, { 2, 3 }, { 20, 30 }, + EE_BUTTON_LMASK, 0 ); + + EXPECT_STDSTREQ( "\033[<0;21;31M", ptyPtr->mWrites ); +} + UTEST( eterm, cursor_style_zero_uses_blinking_configured_shape ) { auto pty = std::make_unique(); pty->mBuffer = "\033[2 q\033[0 q"; @@ -451,12 +1085,14 @@ UTEST( eterm, alternate_scroll_mode_controls_wheel_key_translation ) { auto term = TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); term->update(); - term->mousereport( TerminalMouseEventType::MouseButtonDown, { 0, 0 }, EE_BUTTON_WUMASK, 0 ); + term->mousereport( TerminalMouseEventType::MouseButtonDown, { 0, 0 }, { 0, 0 }, + EE_BUTTON_WUMASK, 0 ); EXPECT_TRUE( ptyPtr->mWrites.empty() ); ptyPtr->mBuffer += "\033[?1007h"; term->update(); - term->mousereport( TerminalMouseEventType::MouseButtonDown, { 0, 0 }, EE_BUTTON_WUMASK, 0 ); + term->mousereport( TerminalMouseEventType::MouseButtonDown, { 0, 0 }, { 0, 0 }, + EE_BUTTON_WUMASK, 0 ); EXPECT_STDSTREQ( "\033[A", ptyPtr->mWrites ); } From 0fa1f65d04aa52b6cf51c96a7aa7a4edc971676d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Thu, 3 Sep 2026 19:39:47 -0300 Subject: [PATCH 2/8] Optimize eterm Kitty graphics rendering - bulk-ingest Kitty APC payloads without per-byte UTF-8 processing - preserve APC buffer capacity between graphics commands - update existing GPU textures when replacing video frames - validate Base64 while decoding directly into transfer storage - reserve known image sizes and share immutable pixel buffers - coalesce superseded anonymous video-frame updates - support POSIX shared-memory transmission with stable snapshots - fix large unchunked images and anonymous placement replacement - add fragmentation, malformed input, video, and shared-memory tests - update Kitty graphics protocol documentation --- docs/articles/eterm-kitty-graphics.md | 7 +- include/eepp/system/base64.hpp | 1 + src/eepp/system/base64.cpp | 48 ++++- .../eterm/terminal/kittygraphicsprotocol.hpp | 3 +- .../eterm/terminal/kittygraphicsprotocol.cpp | 173 ++++++++++++++---- .../eterm/terminal/kittygraphicsrenderer.cpp | 16 +- .../src/eterm/terminal/terminalemulator.cpp | 51 +++++- src/tests/unit_tests/eterm_tests.cpp | 71 ++++++- 8 files changed, 319 insertions(+), 51 deletions(-) diff --git a/docs/articles/eterm-kitty-graphics.md b/docs/articles/eterm-kitty-graphics.md index a70aee53e..ed1e77e9e 100644 --- a/docs/articles/eterm-kitty-graphics.md +++ b/docs/articles/eterm-kitty-graphics.md @@ -8,6 +8,7 @@ stream, and exclusively owns the OpenGL texture cache. ## Supported features - direct RGB and RGBA transmission, including chunking and zlib compression; +- POSIX shared-memory transmission, including byte offsets and explicit sizes; - PNG transmission; - image IDs, image numbers, placement IDs, source cropping, cell geometry, and pixel offsets; - root-image rectangle updates, animation frames, frame composition, and animation control; @@ -17,9 +18,9 @@ stream, and exclusively owns the OpenGL texture cache. - relative placements, including parent validation, cycle detection, and descendant movement; - terminal pixel-size queries and SGR pixel mouse coordinates. -Local file, temporary-file, and shared-memory transfer media are intentionally not enabled. Direct -transmission is the portable path and works through SSH without giving terminal applications access -to the terminal host's filesystem or shared-memory namespace. +Local file and temporary-file transfer media are intentionally not enabled. POSIX shared-memory +objects are opened read-only and immediately unlinked as required by the protocol. Direct +transmission remains the portable path and works through SSH. ## Resource and threading behavior diff --git a/include/eepp/system/base64.hpp b/include/eepp/system/base64.hpp index 1ebc7dd4c..b56c808d9 100644 --- a/include/eepp/system/base64.hpp +++ b/include/eepp/system/base64.hpp @@ -14,6 +14,7 @@ class EE_API Base64 { enum class DecodeMode { AllowWhitespace, NoWhitespace, + NoWhitespaceStrict, }; /** Encode binary data into base64 digits with MIME style === pads diff --git a/src/eepp/system/base64.cpp b/src/eepp/system/base64.cpp index 91ecf9d5b..e4cd6a3a6 100644 --- a/src/eepp/system/base64.cpp +++ b/src/eepp/system/base64.cpp @@ -1,5 +1,5 @@ -#include #include +#include namespace EE { namespace System { @@ -106,6 +106,46 @@ size_t decodeBase64( size_t in_len, const char* in, size_t out_len, unsigned cha return io; } +size_t decodeBase64Strict( size_t inLen, const char* input, size_t outLen, unsigned char* output ) { + size_t outputOffset = 0; + Uint32 value = 0; + unsigned bits = 0; + size_t padding = 0; + for ( size_t offset = 0; offset < inLen; ++offset ) { + const Uint8 decoded = base64dec_tab[static_cast( input[offset] )]; + if ( decoded == BASE64_PADDING ) { + padding = inLen - offset; + if ( padding > 2 || inLen % 4 != 0 ) + return static_cast( -1 ); + for ( size_t remainder = offset; remainder < inLen; ++remainder ) + if ( input[remainder] != '=' ) + return static_cast( -1 ); + break; + } + if ( decoded > 63 || padding != 0 ) + return static_cast( -1 ); + value = ( value << 6 ) | decoded; + bits += 6; + if ( bits >= 8 ) { + bits -= 8; + if ( outputOffset >= outLen ) + return static_cast( -1 ); + output[outputOffset++] = static_cast( ( value >> bits ) & 0xFF ); + if ( bits == 0 ) + value = 0; + } + } + const size_t dataLength = inLen - padding; + if ( dataLength % 4 == 1 || ( padding == 1 && dataLength % 4 != 3 ) || + ( padding == 2 && dataLength % 4 != 2 ) ) + return static_cast( -1 ); + // Reject non-canonical encodings whose unused bits are non-zero. Besides being strict, this + // avoids accepting multiple byte strings for the same payload at protocol boundaries. + if ( bits != 0 && ( value & ( ( 1u << bits ) - 1u ) ) != 0 ) + return static_cast( -1 ); + return outputOffset; +} + } // namespace size_t Base64::decode( size_t in_len, const char* in, size_t out_len, unsigned char* out ) { @@ -114,6 +154,8 @@ size_t Base64::decode( size_t in_len, const char* in, size_t out_len, unsigned c size_t Base64::decode( size_t in_len, const char* in, size_t out_len, unsigned char* out, DecodeMode mode ) { + if ( mode == DecodeMode::NoWhitespaceStrict ) + return decodeBase64Strict( in_len, in, out_len, out ); return mode == DecodeMode::NoWhitespace ? decodeBase64( in_len, in, out_len, out ) : decodeBase64( in_len, in, out_len, out ); } @@ -165,8 +207,8 @@ bool Base64::encode( std::string_view in, std::string& out ) { if ( out.size() < b64len ) out.resize( b64len ); - const size_t len = - encode( in.size(), reinterpret_cast( in.data() ), out.size(), out.data() ); + const size_t len = encode( in.size(), reinterpret_cast( in.data() ), + out.size(), out.data() ); if ( len != static_cast( -1 ) && len != out.size() ) out.resize( len ); diff --git a/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp b/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp index bd18bf7f6..16dc562f4 100644 --- a/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp +++ b/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp @@ -35,6 +35,7 @@ struct KittyGraphicsCommandData { std::string_view payload; std::optional format; std::optional dataSize; + std::optional dataOffset; std::optional more; std::optional imageId; std::optional imageNumber; @@ -167,7 +168,7 @@ class KittyGraphicsProtocol { Int32 gapMs{ 40 }; Uint32 usageHint{ 0 }; }; - std::vector rgba; + std::shared_ptr> rgba; std::unordered_map frames; Sizei size; Uint32 imageNumber{ 0 }; diff --git a/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp b/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp index 1e561b0f6..5f372a90c 100644 --- a/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp +++ b/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp @@ -9,8 +9,16 @@ #include #include #include +#include #include +#if EE_PLATFORM != EE_PLATFORM_WIN && EE_PLATFORM != EE_PLATFORM_EMSCRIPTEN +#include +#include +#include +#include +#endif + using namespace EE::System; namespace eterm { namespace Terminal { @@ -54,40 +62,96 @@ bool checkedPixelBytes( Uint32 width, Uint32 height, size_t channels, size_t& re return true; } -bool validBase64( std::string_view input, bool finalChunk ) { - if ( input.size() % 4 == 1 ) +bool decodeBase64( std::string_view input, bool finalChunk, std::vector& output ) { + if ( !finalChunk && input.size() % 4 != 0 ) + return false; + const size_t oldSize = output.size(); + const size_t capacity = Base64::decodeSafeOutLen( input.size() ); + constexpr size_t MaxTransferBytes = 64 * 1024 * 1024; + if ( oldSize > MaxTransferBytes || capacity > MaxTransferBytes - oldSize ) + return false; + output.resize( oldSize + capacity ); + const size_t decodedSize = + Base64::decode( input.size(), input.data(), capacity, output.data() + oldSize, + Base64::DecodeMode::NoWhitespaceStrict ); + if ( decodedSize == static_cast( -1 ) ) { + output.resize( oldSize ); return false; - size_t padding = 0; - for ( size_t i = 0; i < input.size(); ++i ) { - const unsigned char character = input[i]; - const bool alphabet = - ( character >= 'A' && character <= 'Z' ) || ( character >= 'a' && character <= 'z' ) || - ( character >= '0' && character <= '9' ) || character == '+' || character == '/'; - if ( character == '=' ) { - ++padding; - if ( !finalChunk || padding > 2 ) - return false; - } else if ( !alphabet || padding != 0 ) { - return false; - } } - return finalChunk || input.size() % 4 == 0; + output.resize( oldSize + decodedSize ); + return true; } -bool decodeBase64( std::string_view input, bool finalChunk, std::vector& output ) { - if ( !validBase64( input, finalChunk ) ) +bool readSharedMemory( const KittyGraphicsCommandData& data, std::vector& output ) { +#if EE_PLATFORM == EE_PLATFORM_WIN || EE_PLATFORM == EE_PLATFORM_EMSCRIPTEN + (void)data; + (void)output; + return false; +#else + std::vector decodedName; + if ( !decodeBase64( data.payload, true, decodedName ) || decodedName.empty() || + decodedName.size() > 255 || + std::find( decodedName.begin() + 1, decodedName.end(), '/' ) != decodedName.end() || + std::find( decodedName.begin(), decodedName.end(), 0 ) != decodedName.end() ) return false; - std::vector decoded( Base64::decodeSafeOutLen( input.size() ) ); - const size_t decodedSize = Base64::decode( input.size(), input.data(), decoded.size(), - decoded.data(), Base64::DecodeMode::NoWhitespace ); - if ( decodedSize == static_cast( -1 ) ) + std::string name( decodedName.begin(), decodedName.end() ); + const int descriptor = shm_open( name.c_str(), O_RDONLY, 0 ); + if ( descriptor == -1 ) return false; - decoded.resize( decodedSize ); - constexpr size_t MaxTransferBytes = 64 * 1024 * 1024; - if ( output.size() > MaxTransferBytes || decoded.size() > MaxTransferBytes - output.size() ) - return false; - output.insert( output.end(), decoded.begin(), decoded.end() ); - return true; + // POSIX Kitty transfers are single-use. Unlink immediately after opening so all error paths + // still retire the client-owned object while the descriptor keeps its contents alive. + shm_unlink( name.c_str() ); + struct stat status{}; + const size_t offset = data.dataOffset.value_or( 0 ); + bool valid = fstat( descriptor, &status ) == 0 && status.st_size >= 0 && + static_cast( status.st_size ) >= offset; + size_t bytes = 0; + if ( valid ) { + const size_t available = static_cast( status.st_size ) - offset; + bytes = data.dataSize.value_or( static_cast( + std::min( available, std::numeric_limits::max() ) ) ); + valid = bytes <= available && bytes <= 128 * 1024 * 1024; + } + if ( valid && bytes != 0 ) { + const size_t mappingBytes = offset + bytes; + valid = mappingBytes >= bytes && mappingBytes <= 128 * 1024 * 1024; + void* mapping = valid ? mmap( nullptr, mappingBytes, PROT_READ, MAP_SHARED, descriptor, 0 ) + : MAP_FAILED; + if ( mapping == MAP_FAILED ) { + valid = false; + } else { + const auto* source = static_cast( mapping ) + offset; + // mpv reuses the same shm name for every frame. If it reopened the object before + // we unlinked it above, its next memcpy can overlap this read. Require consecutive + // identical observations after yielding to the writer; otherwise retain the previous + // displayed frame instead of publishing visibly torn rows. + std::vector snapshot( source, source + bytes ); + bool stable = false; + unsigned stableObservations = 0; + constexpr unsigned RequiredStableObservations = 2; + constexpr unsigned MaxSnapshotAttempts = 6; + for ( unsigned attempt = 0; attempt < MaxSnapshotAttempts; ++attempt ) { + std::this_thread::yield(); + if ( std::memcmp( snapshot.data(), source, bytes ) == 0 ) { + if ( ++stableObservations == RequiredStableObservations ) { + stable = true; + break; + } + } else { + stableObservations = 0; + snapshot.assign( source, source + bytes ); + } + } + if ( stable ) + output = std::move( snapshot ); + else + valid = false; + munmap( mapping, mappingBytes ); + } + } + close( descriptor ); + return valid && bytes != 0; +#endif } bool placementContains( const TerminalVisiblePlacement& placement, Vector2i cell ) { @@ -147,6 +211,9 @@ KittyGraphicsParseResult KittyGraphicsProtocol::parse( std::string_view command case 'S': PARSE_UINT_FIELD( dataSize ); break; + case 'O': + PARSE_UINT_FIELD( dataOffset ); + break; case 'm': PARSE_UINT_FIELD( more ); valid = valid && unsignedValue <= 1; @@ -327,6 +394,20 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::handleTransmit( const KittyGraphicsCommandData& data, bool display, bool query, bool frame, Vector2i cursor ) { const bool more = data.more.value_or( 0 ) != 0; + if ( data.transmission == 's' ) { + if ( mPending.active ) + mPending = {}; + PendingTransfer transfer; + transfer.data = data; + transfer.data.payload = {}; + transfer.display = display; + transfer.query = query; + transfer.frame = frame; + if ( !readSharedMemory( data, transfer.decodedData ) ) + return { response( data, KittyGraphicsError::DecodeFailed ), + KittyGraphicsError::DecodeFailed, false }; + return finishTransfer( std::move( transfer ), cursor ); + } if ( mPending.active ) { if ( frame != mPending.frame || data.format || data.dataSize || data.imageId || data.imageNumber || data.usageHint || data.placementId || data.width || data.height || @@ -361,6 +442,14 @@ KittyGraphicsProtocol::handleTransmit( const KittyGraphicsCommandData& data, boo transfer.query = query; transfer.frame = frame; transfer.active = true; + const Uint32 format = data.format.value_or( 32 ); + if ( format == 24 || format == 32 ) { + size_t expectedBytes = 0; + if ( data.width && data.height && + checkedPixelBytes( *data.width, *data.height, format == 24 ? 3 : 4, expectedBytes ) && + expectedBytes <= 64 * 1024 * 1024 ) + transfer.decodedData.reserve( expectedBytes ); + } if ( !decodeBase64( data.payload, !more, transfer.decodedData ) ) return { response( data, KittyGraphicsError::InvalidData ), KittyGraphicsError::InvalidData, false }; @@ -495,7 +584,9 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer std::vector* destinationPixels = nullptr; bool createdFrame = false; if ( frameNumber == 1 ) { - destinationPixels = &image->second.rgba; + if ( !image->second.rgba.unique() ) + image->second.rgba = std::make_shared>( *image->second.rgba ); + destinationPixels = image->second.rgba.get(); } else { auto frame = image->second.frames.find( frameNumber ); if ( frame == image->second.frames.end() ) { @@ -505,7 +596,7 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer if ( data.columns ) { const Uint32 baseFrame = *data.columns; if ( baseFrame == 1 ) - newFrame.rgba = image->second.rgba; + newFrame.rgba = *image->second.rgba; else { auto base = image->second.frames.find( baseFrame ); if ( base == image->second.frames.end() ) @@ -620,7 +711,7 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer false }; auto existing = mImages.find( imageId ); - const size_t oldBytes = existing == mImages.end() ? 0 : existing->second.rgba.size(); + const size_t oldBytes = existing == mImages.end() ? 0 : existing->second.rgba->size(); if ( !ensureCapacity( pixels.size(), imageId, existing == mImages.end() ) ) return { response( data, KittyGraphicsError::NoSpace ), KittyGraphicsError::NoSpace, false }; @@ -646,8 +737,8 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer image.usageHint = data.usageHint.value_or( 0 ); image.anonymous = anonymous; image.creationSerial = ++mCreationSerial; - image.rgba = std::move( pixels ); - mStorageBytes = mStorageBytes - oldBytes + image.rgba.size(); + image.rgba = std::make_shared>( std::move( pixels ) ); + mStorageBytes = mStorageBytes - oldBytes + image.rgba->size(); const bool replaced = existing != mImages.end(); auto inserted = mImages.insert_or_assign( imageId, std::move( image ) ).first; @@ -658,7 +749,7 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer update.imageSize = inserted->second.size; update.region = Rect( 0, 0, inserted->second.size.getWidth(), inserted->second.size.getHeight() ); - update.rgba = std::make_shared>( inserted->second.rgba ); + update.rgba = inserted->second.rgba; mUpdates.emplace_back( std::move( update ) ); ++mStats.fullImageUpdates; ++mPresentationGeneration; @@ -1122,7 +1213,7 @@ KittyGraphicsProtocol::composeFrames( const KittyGraphicsCommandData& data ) { false }; auto pixelsFor = [&]( Uint32 frameNumber ) -> std::vector* { if ( frameNumber == 1 ) - return &image->second.rgba; + return image->second.rgba.get(); auto frame = image->second.frames.find( frameNumber ); return frame == image->second.frames.end() ? nullptr : &frame->second.rgba; }; @@ -1162,6 +1253,10 @@ KittyGraphicsProtocol::composeFrames( const KittyGraphicsCommandData& data ) { source->data() + static_cast( sourceY + row ) * imageStride + static_cast( sourceX ) * 4, rowBytes ); + if ( destinationFrame == 1 && !image->second.rgba.unique() ) { + image->second.rgba = std::make_shared>( *image->second.rgba ); + destination = image->second.rgba.get(); + } const bool replace = data.cursorMovement.value_or( 0 ) == 1; for ( Uint32 row = 0; row < height; ++row ) { Uint8* target = destination->data() + @@ -1239,7 +1334,7 @@ void KittyGraphicsProtocol::eraseImage( KittyImageId imageId ) { auto image = mImages.find( imageId ); if ( image == mImages.end() ) return; - mStorageBytes -= image->second.rgba.size(); + mStorageBytes -= image->second.rgba->size(); for ( const auto& frame : image->second.frames ) mFrameStorageBytes -= frame.second.rgba.size(); mImages.erase( image ); @@ -1252,7 +1347,7 @@ void KittyGraphicsProtocol::eraseImage( KittyImageId imageId ) { bool KittyGraphicsProtocol::ensureCapacity( size_t bytes, KittyImageId replacingId, bool addingImage ) { auto replaced = mImages.find( replacingId ); - const size_t replacedBytes = replaced == mImages.end() ? 0 : replaced->second.rgba.size(); + const size_t replacedBytes = replaced == mImages.end() ? 0 : replaced->second.rgba->size(); auto hasCapacity = [&] { return bytes <= mMaxStorageBytes && mStorageBytes - replacedBytes <= mMaxStorageBytes - bytes && @@ -1401,7 +1496,7 @@ std::shared_ptr KittyGraphicsProtocol::takePresent const std::vector* KittyGraphicsProtocol::imagePixels( KittyImageId imageId ) const { auto image = mImages.find( imageId ); - return image == mImages.end() ? nullptr : &image->second.rgba; + return image == mImages.end() ? nullptr : image->second.rgba.get(); } bool KittyGraphicsProtocol::hasVirtualPlacements() const { @@ -1539,7 +1634,7 @@ void KittyGraphicsProtocol::resync() { create.imageId = image.first; create.imageSize = image.second.size; create.region = Rect( 0, 0, image.second.size.getWidth(), image.second.size.getHeight() ); - create.rgba = std::make_shared>( image.second.rgba ); + create.rgba = image.second.rgba; mUpdates.emplace_back( std::move( create ) ); for ( const auto& frame : image.second.frames ) { TerminalGraphicsUpdate createFrame; diff --git a/src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp b/src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp index d9d91735f..7b8302d33 100644 --- a/src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp +++ b/src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp @@ -26,8 +26,22 @@ bool KittyGraphicsRenderer::applyUpdates( std::vector&& return false; switch ( update.type ) { - case TerminalGraphicsUpdateType::CreateImage: case TerminalGraphicsUpdateType::ReplaceImage: { + if ( !update.rgba || update.imageSize.getWidth() <= 0 || + update.imageSize.getHeight() <= 0 ) + return false; + auto existing = mImages.find( update.imageId ); + if ( existing != mImages.end() && existing->second.texture && + existing->second.size == update.imageSize ) { + existing->second.texture->update( update.rgba->data(), + update.imageSize.getWidth(), + update.imageSize.getHeight(), 0, 0 ); + existing->second.frames.clear(); + break; + } + [[fallthrough]]; + } + case TerminalGraphicsUpdateType::CreateImage: { if ( !update.rgba || update.imageSize.getWidth() <= 0 || update.imageSize.getHeight() <= 0 ) return false; diff --git a/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp b/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp index ad0295896..18c03c529 100644 --- a/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp +++ b/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp @@ -2664,10 +2664,16 @@ void TerminalEmulator::strdump( void ) { } void TerminalEmulator::strreset( void ) { - auto old = mStrescseq.buf; + char* buffer = mStrescseq.buf; + size_t capacity = mStrescseq.siz; + constexpr size_t MaxRetainedStringCapacity = 128 * 1024; + if ( !buffer || capacity > MaxRetainedStringCapacity ) { + buffer = (char*)xrealloc( buffer, STR_BUF_SIZ ); + capacity = STR_BUF_SIZ; + } mStrescseq = STREscape{}; - mStrescseq.buf = (char*)xrealloc( old, STR_BUF_SIZ ); - mStrescseq.siz = STR_BUF_SIZ; + mStrescseq.buf = buffer; + mStrescseq.siz = capacity; } void TerminalEmulator::sendbreak( const TerminalArg* ) { @@ -3230,6 +3236,45 @@ int TerminalEmulator::twrite( const char* buf, int buflen, int show_ctrl ) { int n; for ( n = 0; n < buflen; n += charsize ) { + /* Kitty control data and payload are ASCII transport bytes. Once ESC _ G has been + * recognized, append ordinary bytes in bulk instead of routing every Base64 byte through + * UTF-8 decoding and the terminal character state machine. Control bytes remain on the + * normal path so fragmented ESC \\ termination and malformed strings retain their exact + * behavior. */ + if ( !show_ctrl && ( mTerm.esc & ESC_STR ) && mStrescseq.type == '_' && + mStrescseq.len > 0 && mStrescseq.buf[0] == 'G' ) { + int end = n; + while ( end < buflen ) { + const unsigned char byte = static_cast( buf[end] ); + if ( byte == '\a' || byte == 030 || byte == 032 || byte == 033 || + ( byte >= 0x80 && byte <= 0x9F ) ) + break; + ++end; + } + const size_t bytes = static_cast( end - n ); + if ( bytes != 0 ) { + if ( !mStrescseq.discarded ) { + if ( mStrescseq.len > MAX_KITTY_GRAPHICS_APC_SIZE || + bytes > MAX_KITTY_GRAPHICS_APC_SIZE - mStrescseq.len ) { + mStrescseq.discarded = true; + } else { + const size_t required = mStrescseq.len + bytes + 1; + if ( required > mStrescseq.siz ) { + size_t capacity = mStrescseq.siz; + while ( capacity < required ) + capacity = eemin( capacity * 2, MAX_KITTY_GRAPHICS_APC_SIZE + 1 ); + mStrescseq.buf = (char*)xrealloc( mStrescseq.buf, capacity ); + mStrescseq.siz = capacity; + } + std::memcpy( mStrescseq.buf + mStrescseq.len, buf + n, bytes ); + mStrescseq.len += bytes; + } + } + n = end; + if ( n == buflen ) + return buflen; + } + } if ( IS_SET( MODE_UTF8 ) ) { /* process a complete utf8 char */ charsize = utf8decode( buf + n, &u, buflen - n ); diff --git a/src/tests/unit_tests/eterm_tests.cpp b/src/tests/unit_tests/eterm_tests.cpp index d5987e2a4..7936bd146 100644 --- a/src/tests/unit_tests/eterm_tests.cpp +++ b/src/tests/unit_tests/eterm_tests.cpp @@ -1,6 +1,7 @@ #include "utest.hpp" #include #include +#include #include #include #include @@ -13,6 +14,12 @@ #include #include +#if EE_PLATFORM != EE_PLATFORM_WIN && EE_PLATFORM != EE_PLATFORM_EMSCRIPTEN +#include +#include +#include +#endif + using namespace eterm::Terminal; using namespace eterm::System; using namespace EE::System; @@ -23,6 +30,7 @@ class MockPty : public IPseudoTerminal { std::string mWrites; bool mLoopWrites{ true }; size_t mMaxRead{ std::numeric_limits::max() }; + std::deque mReadSizes; size_t mReadOffset{ 0 }; std::atomic mBytesRead{ 0 }; int mCols = 80; @@ -48,7 +56,12 @@ class MockPty : public IPseudoTerminal { int read( char* buf, size_t n, bool ) override { if ( mReadOffset == mBuffer.size() ) return 0; - size_t toRead = std::min( { n, mBuffer.size() - mReadOffset, mMaxRead } ); + size_t readLimit = mMaxRead; + if ( !mReadSizes.empty() ) { + readLimit = mReadSizes.front(); + mReadSizes.pop_front(); + } + size_t toRead = std::min( { n, mBuffer.size() - mReadOffset, readLimit } ); memcpy( buf, mBuffer.data() + mReadOffset, toRead ); mReadOffset += toRead; mBytesRead.fetch_add( toRead, std::memory_order_relaxed ); @@ -935,6 +948,62 @@ UTEST( eterm, kitty_graphics_apc_is_fragmentation_safe_and_not_terminal_text ) { EXPECT_EQ( static_cast( 'K' ), display->mSecondGlyph.u ); } +UTEST( eterm, kitty_graphics_bulk_apc_accepts_every_input_split_boundary ) { + const std::string stream = "\033_Ga=T,f=32,s=1,v=1,q=2;AQIDBA==\033\\"; + for ( size_t split = 1; split < stream.size(); ++split ) { + auto pty = std::make_unique(); + pty->mBuffer = stream; + pty->mLoopWrites = false; + pty->mReadSizes = { split, stream.size() - split }; + auto process = std::make_unique(); + auto display = std::make_shared(); + auto term = + TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); + term->update(); + while ( !term->update() ) { + } + ASSERT_TRUE( display->mGraphics != nullptr ); + ASSERT_EQ( static_cast( 1 ), display->mGraphics->placements.size() ); + } +} + +UTEST( eterm, kitty_graphics_strict_base64_rejects_invalid_payload_bytes ) { + KittyGraphicsProtocol protocol; + EXPECT_EQ( KittyGraphicsError::InvalidData, + protocol.handle( "a=t,f=32,s=1,v=1;AQI BA==" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidData, + protocol.handle( "a=t,f=32,s=1,v=1;AQIDBA=$" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidData, + protocol.handle( "a=t,f=32,s=1,v=1;AQ=DBA==" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidData, protocol.handle( "a=t,f=32,s=1,v=1;AB==" ).error ); +} + +#if EE_PLATFORM != EE_PLATFORM_WIN && EE_PLATFORM != EE_PLATFORM_EMSCRIPTEN +UTEST( eterm, kitty_graphics_reads_and_unlinks_posix_shared_memory ) { + // mpv uses the Linux-compatible form without the optional leading slash. + const std::string name = "eterm-kitty-unit-" + std::to_string( getpid() ); + shm_unlink( name.c_str() ); + const int descriptor = shm_open( name.c_str(), O_CREAT | O_EXCL | O_RDWR, 0600 ); + ASSERT_TRUE( descriptor >= 0 ); + const Uint8 stored[] = { 99, 98, 1, 2, 3 }; + ASSERT_EQ( static_cast( sizeof( stored ) ), + write( descriptor, stored, sizeof( stored ) ) ); + close( descriptor ); + + std::string encodedName; + ASSERT_TRUE( Base64::encode( name, encodedName ) ); + KittyGraphicsProtocol protocol; + EXPECT_EQ( KittyGraphicsError::None, + protocol.handle( "a=T,t=s,f=24,s=1,v=1,O=2,S=3,q=2,m=1;" + encodedName ).error ); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + ASSERT_TRUE( updates[0].rgba != nullptr ); + const std::vector expected{ 1, 2, 3, 255 }; + EXPECT_TRUE( expected == *updates[0].rgba ); + EXPECT_EQ( -1, shm_open( name.c_str(), O_RDONLY, 0 ) ); +} +#endif + UTEST( eterm, kitty_graphics_accepts_unchunked_direct_image_larger_than_eight_kibibytes ) { std::vector rgb( 64 * 64 * 3 ); for ( size_t offset = 0; offset < rgb.size(); offset += 3 ) From e45ff6af007a8cb637ec23f46a83330d62b4fd6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Thu, 3 Sep 2026 20:18:30 -0300 Subject: [PATCH 3/8] eterm: reuse Kitty graphics transfer buffers Reduce allocation pressure in the Kitty graphics receive path by caching and reusing decoded, pixel-conversion, encoded-image, and frame-composition buffers. Reuse uniquely owned pixel storage when replacing an existing image while preserving immutable update data still referenced by the renderer. Add runtime SSSE3 capability detection to CPU for future optimized paths and extend Kitty graphics tests to cover buffer reuse and strict Base64 decoding across boundary and large payload sizes. --- include/eepp/system/cpu.hpp | 2 + src/eepp/system/cpu.cpp | 22 +++++++ .../eterm/terminal/kittygraphicsprotocol.hpp | 4 ++ .../eterm/terminal/kittygraphicsprotocol.cpp | 38 +++++++++++-- src/tests/unit_tests/eterm_tests.cpp | 57 +++++++++++++++++++ 5 files changed, 119 insertions(+), 4 deletions(-) diff --git a/include/eepp/system/cpu.hpp b/include/eepp/system/cpu.hpp index 5f5856832..194dacf82 100644 --- a/include/eepp/system/cpu.hpp +++ b/include/eepp/system/cpu.hpp @@ -7,6 +7,8 @@ namespace EE { namespace System { class EE_API CPU { public: + static bool hasSSSE3(); + static bool hasAVX2(); static bool hasNEON(); diff --git a/src/eepp/system/cpu.cpp b/src/eepp/system/cpu.cpp index 04b2a9a19..756aa0a1a 100644 --- a/src/eepp/system/cpu.cpp +++ b/src/eepp/system/cpu.cpp @@ -12,6 +12,28 @@ namespace EE { namespace System { +bool CPU::hasSSSE3() { +#ifdef EE_ARCH_X86_64 + static bool isSSSE3 = []() { +#if defined( COMPILER_MSVC ) + int cpuInfo[4]; + __cpuid( cpuInfo, 0 ); + if ( cpuInfo[0] < 1 ) + return false; + __cpuid( cpuInfo, 1 ); + return ( cpuInfo[2] & ( 1 << 9 ) ) != 0; +#elif defined( COMPILER_GCC_CLANG ) + return __builtin_cpu_supports( "ssse3" ); +#else + return false; +#endif + }(); + return isSSSE3; +#else + return false; +#endif +} + bool CPU::hasAVX2() { #ifdef EE_ARCH_X86_64 static bool isAVX2 = []() { diff --git a/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp b/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp index 16dc562f4..c137975fb 100644 --- a/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp +++ b/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp @@ -221,6 +221,10 @@ class KittyGraphicsProtocol { std::vector mPlaceholderCells; std::vector mUpdates; PendingTransfer mPending; + std::vector mDecodedScratch; + std::vector mPixelScratch; + std::vector mEncodedScratch; + std::vector mComposeSourceScratch; size_t mStorageBytes{ 0 }; size_t mFrameStorageBytes{ 0 }; Uint64 mCreationSerial{ 0 }; diff --git a/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp b/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp index 5f372a90c..b38b11901 100644 --- a/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp +++ b/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp @@ -62,6 +62,12 @@ bool checkedPixelBytes( Uint32 width, Uint32 height, size_t channels, size_t& re return true; } +void recycleBuffer( std::vector& buffer, std::vector& cache ) { + buffer.clear(); + if ( buffer.capacity() > cache.capacity() ) + buffer.swap( cache ); +} + bool decodeBase64( std::string_view input, bool finalChunk, std::vector& output ) { if ( !finalChunk && input.size() % 4 != 0 ) return false; @@ -442,6 +448,8 @@ KittyGraphicsProtocol::handleTransmit( const KittyGraphicsCommandData& data, boo transfer.query = query; transfer.frame = frame; transfer.active = true; + transfer.decodedData = std::move( mDecodedScratch ); + transfer.decodedData.clear(); const Uint32 format = data.format.value_or( 32 ); if ( format == 24 || format == 32 ) { size_t expectedBytes = 0; @@ -473,7 +481,8 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer Uint32 height = data.height.value_or( 0 ); std::vector pixels; if ( format == 100 ) { - std::vector encoded; + std::vector encoded = std::move( mEncodedScratch ); + encoded.clear(); if ( data.compression == 'z' ) { if ( !data.dataSize || *data.dataSize == 0 || *data.dataSize > 64 * 1024 * 1024 ) return { response( data, KittyGraphicsError::TooLarge ), @@ -488,6 +497,8 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer KittyGraphicsError::DecodeFailed, false }; } } else if ( data.compression == 0 ) { + mEncodedScratch = std::move( encoded ); + mEncodedScratch.clear(); encoded = std::move( transfer.decodedData ); } else { return { response( data, KittyGraphicsError::Unsupported ), @@ -516,6 +527,7 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer return { response( data, KittyGraphicsError::DecodeFailed ), KittyGraphicsError::DecodeFailed, false }; pixels.assign( decoded.getPixelsPtr(), decoded.getPixelsPtr() + rgbaBytes ); + recycleBuffer( encoded, mEncodedScratch ); } else { if ( width == 0 || height == 0 ) return { response( data, KittyGraphicsError::InvalidArgument ), @@ -548,7 +560,7 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer } if ( format == 24 ) { - std::vector rgba; + std::vector rgba = std::move( mPixelScratch ); rgba.resize( static_cast( width ) * height * 4 ); for ( size_t sourceOffset = 0, destinationOffset = 0; sourceOffset < pixels.size(); sourceOffset += 3, destinationOffset += 4 ) { @@ -557,6 +569,7 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer rgba[destinationOffset + 2] = pixels[sourceOffset + 2]; rgba[destinationOffset + 3] = 255; } + recycleBuffer( pixels, mDecodedScratch ); pixels = std::move( rgba ); } if ( transfer.frame ) { @@ -684,6 +697,10 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer mUpdates.emplace_back( std::move( update ) ); ++mStats.rectangleUpdates; mPresentationDirty = true; + if ( format == 24 ) + recycleBuffer( pixels, mPixelScratch ); + else if ( format == 32 ) + recycleBuffer( pixels, mDecodedScratch ); return { response( data, KittyGraphicsError::None, imageId ), KittyGraphicsError::None, true }; } @@ -731,13 +748,24 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer } ), mPrimaryPlacements.end() ); } + std::shared_ptr> pixelStorage; + if ( existing != mImages.end() && existing->second.rgba.unique() ) { + pixelStorage = existing->second.rgba; + pixelStorage->assign( pixels.begin(), pixels.end() ); + if ( format == 24 ) + recycleBuffer( pixels, mPixelScratch ); + else if ( format == 32 ) + recycleBuffer( pixels, mDecodedScratch ); + } else { + pixelStorage = std::make_shared>( std::move( pixels ) ); + } Image image; image.size = Sizei( width, height ); image.imageNumber = data.imageNumber.value_or( 0 ); image.usageHint = data.usageHint.value_or( 0 ); image.anonymous = anonymous; image.creationSerial = ++mCreationSerial; - image.rgba = std::make_shared>( std::move( pixels ) ); + image.rgba = std::move( pixelStorage ); mStorageBytes = mStorageBytes - oldBytes + image.rgba->size(); const bool replaced = existing != mImages.end(); auto inserted = mImages.insert_or_assign( imageId, std::move( image ) ).first; @@ -1244,7 +1272,8 @@ KittyGraphicsProtocol::composeFrames( const KittyGraphicsCommandData& data ) { return { response( data, KittyGraphicsError::InvalidArgument ), KittyGraphicsError::InvalidArgument, false }; - std::vector sourceCopy( static_cast( width ) * height * 4 ); + std::vector sourceCopy = std::move( mComposeSourceScratch ); + sourceCopy.resize( static_cast( width ) * height * 4 ); std::vector result( sourceCopy.size() ); const size_t imageStride = static_cast( imageWidth ) * 4; const size_t rowBytes = static_cast( width ) * 4; @@ -1281,6 +1310,7 @@ KittyGraphicsProtocol::composeFrames( const KittyGraphicsCommandData& data ) { std::memcpy( published + offset, target + offset, 4 ); } } + recycleBuffer( sourceCopy, mComposeSourceScratch ); TerminalGraphicsUpdate update; update.type = destinationFrame == 1 ? TerminalGraphicsUpdateType::UpdateRegion : TerminalGraphicsUpdateType::UpdateFrameRegion; diff --git a/src/tests/unit_tests/eterm_tests.cpp b/src/tests/unit_tests/eterm_tests.cpp index 7936bd146..d05946171 100644 --- a/src/tests/unit_tests/eterm_tests.cpp +++ b/src/tests/unit_tests/eterm_tests.cpp @@ -978,6 +978,63 @@ UTEST( eterm, kitty_graphics_strict_base64_rejects_invalid_payload_bytes ) { EXPECT_EQ( KittyGraphicsError::InvalidData, protocol.handle( "a=t,f=32,s=1,v=1;AB==" ).error ); } +UTEST( eterm, kitty_graphics_strict_base64_decodes_boundaries_and_large_payloads ) { + for ( size_t length = 1; length <= 257; ++length ) { + std::vector boundarySource( length ); + for ( size_t i = 0; i < boundarySource.size(); ++i ) + boundarySource[i] = static_cast( ( i * 197 + length ) & 0xFF ); + std::string boundaryEncoded; + ASSERT_TRUE( Base64::encode( + std::string_view( reinterpret_cast( boundarySource.data() ), + boundarySource.size() ), + boundaryEncoded ) ); + std::vector boundaryDecoded( Base64::decodeSafeOutLen( boundaryEncoded.size() ) ); + const size_t boundaryDecodedSize = + Base64::decode( boundaryEncoded.size(), boundaryEncoded.data(), boundaryDecoded.size(), + boundaryDecoded.data(), Base64::DecodeMode::NoWhitespaceStrict ); + ASSERT_EQ( boundarySource.size(), boundaryDecodedSize ); + boundaryDecoded.resize( boundaryDecodedSize ); + EXPECT_TRUE( boundarySource == boundaryDecoded ); + } + + std::vector source( 65537 ); + for ( size_t i = 0; i < source.size(); ++i ) + source[i] = static_cast( ( i * 131 + i / 7 ) & 0xFF ); + std::string encoded; + ASSERT_TRUE( Base64::encode( + std::string_view( reinterpret_cast( source.data() ), source.size() ), + encoded ) ); + std::vector decoded( Base64::decodeSafeOutLen( encoded.size() ) ); + const size_t decodedSize = + Base64::decode( encoded.size(), encoded.data(), decoded.size(), decoded.data(), + Base64::DecodeMode::NoWhitespaceStrict ); + ASSERT_EQ( source.size(), decodedSize ); + decoded.resize( decodedSize ); + EXPECT_TRUE( source == decoded ); + + encoded[encoded.size() / 2] = '!'; + EXPECT_EQ( static_cast( -1 ), + Base64::decode( encoded.size(), encoded.data(), decoded.size(), decoded.data(), + Base64::DecodeMode::NoWhitespaceStrict ) ); +} + +UTEST( eterm, kitty_graphics_reuses_unreferenced_replacement_pixel_storage ) { + KittyGraphicsProtocol protocol; + ASSERT_EQ( KittyGraphicsError::None, + protocol.handle( "a=t,f=24,s=2,v=1,i=91,q=2;AQIDBAUG" ).error ); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + updates.clear(); + const auto* storage = protocol.imagePixels( 91 ); + ASSERT_TRUE( storage != nullptr ); + + ASSERT_EQ( KittyGraphicsError::None, + protocol.handle( "a=t,f=24,s=2,v=1,i=91,q=2;BwgJCgsM" ).error ); + EXPECT_TRUE( storage == protocol.imagePixels( 91 ) ); + const std::vector expected{ 7, 8, 9, 255, 10, 11, 12, 255 }; + EXPECT_TRUE( expected == *protocol.imagePixels( 91 ) ); +} + #if EE_PLATFORM != EE_PLATFORM_WIN && EE_PLATFORM != EE_PLATFORM_EMSCRIPTEN UTEST( eterm, kitty_graphics_reads_and_unlinks_posix_shared_memory ) { // mpv uses the Linux-compatible form without the optional leading slash. From 090e7c1288a26f1792fffae0418231c48f3b4539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Thu, 3 Sep 2026 20:29:57 -0300 Subject: [PATCH 4/8] Compilation fixes. --- .../src/eterm/terminal/kittygraphicsprotocol.cpp | 12 +++++++----- src/tests/unit_tests/eterm_tests.cpp | 6 ++++-- .../ecode/plugins/aiassistant/aiassistantplugin.hpp | 4 ++-- .../plugins/autocomplete/autocompleteplugin.hpp | 4 ++-- src/tools/ecode/plugins/debugger/debuggerplugin.hpp | 4 ++-- .../ecode/plugins/discordRPC/discordRPCplugin.hpp | 4 ++-- .../ecode/plugins/formatter/formatterplugin.hpp | 4 ++-- src/tools/ecode/plugins/git/gitplugin.hpp | 4 ++-- src/tools/ecode/plugins/linter/linterplugin.hpp | 4 ++-- .../plugins/spellchecker/spellcheckerplugin.hpp | 4 ++-- src/tools/ecode/plugins/xmltools/xmltoolsplugin.hpp | 4 ++-- 11 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp b/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp index b38b11901..218b36a5e 100644 --- a/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp +++ b/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp @@ -12,7 +12,8 @@ #include #include -#if EE_PLATFORM != EE_PLATFORM_WIN && EE_PLATFORM != EE_PLATFORM_EMSCRIPTEN +#if EE_PLATFORM != EE_PLATFORM_WIN && EE_PLATFORM != EE_PLATFORM_EMSCRIPTEN && \ + EE_PLATFORM != EE_PLATFORM_ANDROID #include #include #include @@ -89,7 +90,8 @@ bool decodeBase64( std::string_view input, bool finalChunk, std::vector& } bool readSharedMemory( const KittyGraphicsCommandData& data, std::vector& output ) { -#if EE_PLATFORM == EE_PLATFORM_WIN || EE_PLATFORM == EE_PLATFORM_EMSCRIPTEN +#if EE_PLATFORM == EE_PLATFORM_WIN || EE_PLATFORM == EE_PLATFORM_EMSCRIPTEN || \ + EE_PLATFORM == EE_PLATFORM_ANDROID (void)data; (void)output; return false; @@ -597,7 +599,7 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer std::vector* destinationPixels = nullptr; bool createdFrame = false; if ( frameNumber == 1 ) { - if ( !image->second.rgba.unique() ) + if ( image->second.rgba.use_count() != 1 ) image->second.rgba = std::make_shared>( *image->second.rgba ); destinationPixels = image->second.rgba.get(); } else { @@ -749,7 +751,7 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer mPrimaryPlacements.end() ); } std::shared_ptr> pixelStorage; - if ( existing != mImages.end() && existing->second.rgba.unique() ) { + if ( existing != mImages.end() && existing->second.rgba.use_count() == 1 ) { pixelStorage = existing->second.rgba; pixelStorage->assign( pixels.begin(), pixels.end() ); if ( format == 24 ) @@ -1282,7 +1284,7 @@ KittyGraphicsProtocol::composeFrames( const KittyGraphicsCommandData& data ) { source->data() + static_cast( sourceY + row ) * imageStride + static_cast( sourceX ) * 4, rowBytes ); - if ( destinationFrame == 1 && !image->second.rgba.unique() ) { + if ( destinationFrame == 1 && image->second.rgba.use_count() != 1 ) { image->second.rgba = std::make_shared>( *image->second.rgba ); destination = image->second.rgba.get(); } diff --git a/src/tests/unit_tests/eterm_tests.cpp b/src/tests/unit_tests/eterm_tests.cpp index d05946171..c7493652e 100644 --- a/src/tests/unit_tests/eterm_tests.cpp +++ b/src/tests/unit_tests/eterm_tests.cpp @@ -14,7 +14,8 @@ #include #include -#if EE_PLATFORM != EE_PLATFORM_WIN && EE_PLATFORM != EE_PLATFORM_EMSCRIPTEN +#if EE_PLATFORM != EE_PLATFORM_WIN && EE_PLATFORM != EE_PLATFORM_EMSCRIPTEN && \ + EE_PLATFORM != EE_PLATFORM_ANDROID #include #include #include @@ -1035,7 +1036,8 @@ UTEST( eterm, kitty_graphics_reuses_unreferenced_replacement_pixel_storage ) { EXPECT_TRUE( expected == *protocol.imagePixels( 91 ) ); } -#if EE_PLATFORM != EE_PLATFORM_WIN && EE_PLATFORM != EE_PLATFORM_EMSCRIPTEN +#if EE_PLATFORM != EE_PLATFORM_WIN && EE_PLATFORM != EE_PLATFORM_EMSCRIPTEN && \ + EE_PLATFORM != EE_PLATFORM_ANDROID UTEST( eterm, kitty_graphics_reads_and_unlinks_posix_shared_memory ) { // mpv uses the Linux-compatible form without the optional leading slash. const std::string name = "eterm-kitty-unit-" + std::to_string( getpid() ); diff --git a/src/tools/ecode/plugins/aiassistant/aiassistantplugin.hpp b/src/tools/ecode/plugins/aiassistant/aiassistantplugin.hpp index a66d2df2f..daeac445e 100644 --- a/src/tools/ecode/plugins/aiassistant/aiassistantplugin.hpp +++ b/src/tools/ecode/plugins/aiassistant/aiassistantplugin.hpp @@ -40,9 +40,9 @@ class AIAssistantPlugin : public PluginBase { std::string getDescription() override { return Definition().description; } - bool hasSettingsPage() const override { return true; } + bool hasSettingsPage() const { return true; } - void registerSettings( SettingsPage& page ) override; + void registerSettings( SettingsPage& page ); const LLMProviders& getProviders() { return mProviders; } diff --git a/src/tools/ecode/plugins/autocomplete/autocompleteplugin.hpp b/src/tools/ecode/plugins/autocomplete/autocompleteplugin.hpp index b0abaf69e..12cb47cc3 100644 --- a/src/tools/ecode/plugins/autocomplete/autocompleteplugin.hpp +++ b/src/tools/ecode/plugins/autocomplete/autocompleteplugin.hpp @@ -95,9 +95,9 @@ class AutoCompletePlugin : public Plugin { bool isReady() const override { return true; } - bool hasSettingsPage() const override { return true; } + bool hasSettingsPage() const { return true; } - void registerSettings( SettingsPage& page ) override; + void registerSettings( SettingsPage& page ); void onRegister( UICodeEditor* ) override; diff --git a/src/tools/ecode/plugins/debugger/debuggerplugin.hpp b/src/tools/ecode/plugins/debugger/debuggerplugin.hpp index 1080b19c5..771b2f52b 100644 --- a/src/tools/ecode/plugins/debugger/debuggerplugin.hpp +++ b/src/tools/ecode/plugins/debugger/debuggerplugin.hpp @@ -68,9 +68,9 @@ class DebuggerPlugin : public PluginBase { std::string getDescription() override { return Definition().description; } - bool hasSettingsPage() const override { return true; } + bool hasSettingsPage() const { return true; } - void registerSettings( SettingsPage& page ) override; + void registerSettings( SettingsPage& page ); void onSaveState( IniFile* state ) override; diff --git a/src/tools/ecode/plugins/discordRPC/discordRPCplugin.hpp b/src/tools/ecode/plugins/discordRPC/discordRPCplugin.hpp index f1e8613d5..8b0c0005a 100644 --- a/src/tools/ecode/plugins/discordRPC/discordRPCplugin.hpp +++ b/src/tools/ecode/plugins/discordRPC/discordRPCplugin.hpp @@ -43,9 +43,9 @@ class DiscordRPCplugin : public PluginBase { std::string getDescription() override { return Definition().description; } - bool hasSettingsPage() const override { return true; } + bool hasSettingsPage() const { return true; } - void registerSettings( SettingsPage& page ) override; + void registerSettings( SettingsPage& page ); protected: DiscordIPC mIPC; diff --git a/src/tools/ecode/plugins/formatter/formatterplugin.hpp b/src/tools/ecode/plugins/formatter/formatterplugin.hpp index 7bb69d563..7dd5dbc93 100644 --- a/src/tools/ecode/plugins/formatter/formatterplugin.hpp +++ b/src/tools/ecode/plugins/formatter/formatterplugin.hpp @@ -50,9 +50,9 @@ class FormatterPlugin : public Plugin { std::string getDescription() { return Definition().description; } - bool hasSettingsPage() const override { return true; } + bool hasSettingsPage() const { return true; } - void registerSettings( SettingsPage& page ) override; + void registerSettings( SettingsPage& page ); virtual String::HashType getConfigFileHash() { return mConfigHash; } diff --git a/src/tools/ecode/plugins/git/gitplugin.hpp b/src/tools/ecode/plugins/git/gitplugin.hpp index dda3cf11e..98b327358 100644 --- a/src/tools/ecode/plugins/git/gitplugin.hpp +++ b/src/tools/ecode/plugins/git/gitplugin.hpp @@ -83,9 +83,9 @@ class GitPlugin : public PluginBase { std::string getDescription() override { return Definition().description; } - bool hasSettingsPage() const override { return true; } + bool hasSettingsPage() const { return true; } - void registerSettings( SettingsPage& page ) override; + void registerSettings( SettingsPage& page ); void onSaveState( IniFile* state ) override; diff --git a/src/tools/ecode/plugins/linter/linterplugin.hpp b/src/tools/ecode/plugins/linter/linterplugin.hpp index 3d0548273..ca45f69fb 100644 --- a/src/tools/ecode/plugins/linter/linterplugin.hpp +++ b/src/tools/ecode/plugins/linter/linterplugin.hpp @@ -76,9 +76,9 @@ class LinterPlugin : public Plugin { std::string getDescription() { return Definition().description; } - bool hasSettingsPage() const override { return true; } + bool hasSettingsPage() const { return true; } - void registerSettings( SettingsPage& page ) override; + void registerSettings( SettingsPage& page ); virtual String::HashType getConfigFileHash() { return mConfigHash; } diff --git a/src/tools/ecode/plugins/spellchecker/spellcheckerplugin.hpp b/src/tools/ecode/plugins/spellchecker/spellcheckerplugin.hpp index 75912dea5..0e414422a 100644 --- a/src/tools/ecode/plugins/spellchecker/spellcheckerplugin.hpp +++ b/src/tools/ecode/plugins/spellchecker/spellcheckerplugin.hpp @@ -39,9 +39,9 @@ class SpellCheckerPlugin : public PluginBase { std::string getDescription() override { return Definition().description; } - bool hasSettingsPage() const override { return true; } + bool hasSettingsPage() const { return true; } - void registerSettings( SettingsPage& page ) override; + void registerSettings( SettingsPage& page ); void drawAfterLineText( UICodeEditor* editor, const Int64& index, Vector2f position, const Float& fontSize, const Float& lineHeight ) override; diff --git a/src/tools/ecode/plugins/xmltools/xmltoolsplugin.hpp b/src/tools/ecode/plugins/xmltools/xmltoolsplugin.hpp index 0f951963b..3633abb92 100644 --- a/src/tools/ecode/plugins/xmltools/xmltoolsplugin.hpp +++ b/src/tools/ecode/plugins/xmltools/xmltoolsplugin.hpp @@ -36,9 +36,9 @@ class XMLToolsPlugin : public PluginBase { std::string getDescription() override { return Definition().description; } - bool hasSettingsPage() const override { return true; } + bool hasSettingsPage() const { return true; } - void registerSettings( SettingsPage& page ) override; + void registerSettings( SettingsPage& page ); bool getHighlightMatch() const; From 1d84bfda1b1107b158a294647397356a3f500b9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Fri, 4 Sep 2026 02:04:41 -0300 Subject: [PATCH 5/8] Nit. --- docs/articles/eterm-kitty-graphics.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/articles/eterm-kitty-graphics.md b/docs/articles/eterm-kitty-graphics.md index ed1e77e9e..522fe436b 100644 --- a/docs/articles/eterm-kitty-graphics.md +++ b/docs/articles/eterm-kitty-graphics.md @@ -1,6 +1,6 @@ -# eTerm Kitty graphics protocol +# eterm Kitty graphics protocol -eTerm implements the terminal-receiver side of the Kitty graphics protocol. Protocol parsing, +eterm implements the terminal-receiver side of the Kitty graphics protocol. Protocol parsing, decoding, image storage, placement state, animation, and terminal lifecycle integration run on the terminal worker. The UI thread consumes immutable placement metadata and an ordered pixel-update stream, and exclusively owns the OpenGL texture cache. @@ -38,7 +38,7 @@ metadata with its text cells. ## Compatibility testing -The eTerm unit suite covers fragmented APC input, malformed/fuzzed commands, raw/zlib/PNG transfer, +The eterm unit suite covers fragmented APC input, malformed/fuzzed commands, raw/zlib/PNG transfer, independent image namespaces, replacement, partial updates, placement geometry, scrolling and scrollback, screen lifecycle, animation, Unicode placeholders, relative placement graphs, ordered worker/UI delivery, queue overflow, and resynchronization. From 7ed9b7c517b48f5f19cd66bcee0b7f6a4145d54a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Fri, 4 Sep 2026 18:12:26 -0300 Subject: [PATCH 6/8] Optimize Kitty graphics decoding and RGB24 rendering - use fast Base64 decoding for non-final transfer chunks - rewrite strict Base64 decoding around complete quartets - vectorize Kitty APC control scanning with runtime AVX2 detection - preserve RGB24 frames through protocol storage and GPU upload - convert RGB24 roots to RGBA only when mutation requires it - handle tightly packed RGB texture uploads correctly - bridge transient PTY producer gaps to avoid per-chunk idle delays - remove the slower POSIX shared-memory transmission path - expand Kitty graphics and Base64 regression coverage - isolate the Git conflict test from user rerere configuration --- src/eepp/graphics/texture.cpp | 8 + src/eepp/system/base64.cpp | 71 ++++--- src/eepp/system/cpu.cpp | 6 + .../eterm/terminal/kittygraphicsprotocol.hpp | 12 +- .../eterm/terminal/kittygraphicsrenderer.hpp | 1 + .../eterm/terminal/terminalgraphics.hpp | 5 +- .../eterm/terminal/kittygraphicsprotocol.cpp | 195 +++++++----------- .../eterm/terminal/kittygraphicsrenderer.cpp | 54 +++-- .../src/eterm/terminal/terminalemulator.cpp | 68 +++++- src/tests/unit_tests/eterm_tests.cpp | 113 +++++----- src/tests/unit_tests/gitconflict_tests.cpp | 1 + 11 files changed, 294 insertions(+), 240 deletions(-) diff --git a/src/eepp/graphics/texture.cpp b/src/eepp/graphics/texture.cpp index 9c5dc43a0..0d0cfc922 100644 --- a/src/eepp/graphics/texture.cpp +++ b/src/eepp/graphics/texture.cpp @@ -530,10 +530,18 @@ void Texture::update( const Uint8* pixels, Uint32 width, Uint32 height, Uint32 x { ScopedTexture saver( mTexture ); + const bool tightlyPackedThreeChannelRows = + ( pf == Image::PixelFormat::PIXEL_FORMAT_RGB || + pf == Image::PixelFormat::PIXEL_FORMAT_BGR ) && + ( static_cast( width ) * 3 ) % 4 != 0; + if ( tightlyPackedThreeChannelRows ) + glPixelStorei( GL_UNPACK_ALIGNMENT, 1 ); glTexSubImage2D( GL_TEXTURE_2D, 0, x, y, width, height, (unsigned int)convertPixelFormatToGLFormat( pf ), GL_UNSIGNED_BYTE, pixels ); + if ( tightlyPackedThreeChannelRows ) + glPixelStorei( GL_UNPACK_ALIGNMENT, 4 ); if ( hasLocalCopy() ) { Image image( pixels, width, height, mChannels ); diff --git a/src/eepp/system/base64.cpp b/src/eepp/system/base64.cpp index e4cd6a3a6..d41235e0e 100644 --- a/src/eepp/system/base64.cpp +++ b/src/eepp/system/base64.cpp @@ -108,40 +108,51 @@ size_t decodeBase64( size_t in_len, const char* in, size_t out_len, unsigned cha size_t decodeBase64Strict( size_t inLen, const char* input, size_t outLen, unsigned char* output ) { size_t outputOffset = 0; - Uint32 value = 0; - unsigned bits = 0; - size_t padding = 0; - for ( size_t offset = 0; offset < inLen; ++offset ) { - const Uint8 decoded = base64dec_tab[static_cast( input[offset] )]; - if ( decoded == BASE64_PADDING ) { - padding = inLen - offset; - if ( padding > 2 || inLen % 4 != 0 ) - return static_cast( -1 ); - for ( size_t remainder = offset; remainder < inLen; ++remainder ) - if ( input[remainder] != '=' ) - return static_cast( -1 ); + size_t inputOffset = 0; + while ( inputOffset + 4 <= inLen ) { + const Uint8 a = base64dec_tab[static_cast( input[inputOffset] )]; + const Uint8 b = base64dec_tab[static_cast( input[inputOffset + 1] )]; + const Uint8 c = base64dec_tab[static_cast( input[inputOffset + 2] )]; + const Uint8 d = base64dec_tab[static_cast( input[inputOffset + 3] )]; + if ( ( a | b | c | d ) > 63 ) break; - } - if ( decoded > 63 || padding != 0 ) + if ( outputOffset + 3 > outLen ) return static_cast( -1 ); - value = ( value << 6 ) | decoded; - bits += 6; - if ( bits >= 8 ) { - bits -= 8; - if ( outputOffset >= outLen ) - return static_cast( -1 ); - output[outputOffset++] = static_cast( ( value >> bits ) & 0xFF ); - if ( bits == 0 ) - value = 0; - } + output[outputOffset] = static_cast( ( a << 2 ) | ( b >> 4 ) ); + output[outputOffset + 1] = static_cast( ( b << 4 ) | ( c >> 2 ) ); + output[outputOffset + 2] = static_cast( ( c << 6 ) | d ); + inputOffset += 4; + outputOffset += 3; } - const size_t dataLength = inLen - padding; - if ( dataLength % 4 == 1 || ( padding == 1 && dataLength % 4 != 3 ) || - ( padding == 2 && dataLength % 4 != 2 ) ) + + const size_t remaining = inLen - inputOffset; + if ( remaining == 0 ) + return outputOffset; + if ( remaining == 1 || remaining > 4 ) return static_cast( -1 ); - // Reject non-canonical encodings whose unused bits are non-zero. Besides being strict, this - // avoids accepting multiple byte strings for the same payload at protocol boundaries. - if ( bits != 0 && ( value & ( ( 1u << bits ) - 1u ) ) != 0 ) + + const Uint8 a = base64dec_tab[static_cast( input[inputOffset] )]; + const Uint8 b = base64dec_tab[static_cast( input[inputOffset + 1] )]; + if ( a > 63 || b > 63 || outputOffset >= outLen ) + return static_cast( -1 ); + output[outputOffset++] = static_cast( ( a << 2 ) | ( b >> 4 ) ); + + if ( remaining == 2 ) + return ( b & 0x0F ) == 0 ? outputOffset : static_cast( -1 ); + + const Uint8 c = base64dec_tab[static_cast( input[inputOffset + 2] )]; + if ( c == BASE64_PADDING ) { + return remaining == 4 && input[inputOffset + 3] == '=' && ( b & 0x0F ) == 0 + ? outputOffset + : static_cast( -1 ); + } + if ( c > 63 || outputOffset >= outLen ) + return static_cast( -1 ); + output[outputOffset++] = static_cast( ( b << 4 ) | ( c >> 2 ) ); + + if ( remaining == 3 ) + return ( c & 0x03 ) == 0 ? outputOffset : static_cast( -1 ); + if ( input[inputOffset + 3] != '=' || ( c & 0x03 ) != 0 ) return static_cast( -1 ); return outputOffset; } diff --git a/src/eepp/system/cpu.cpp b/src/eepp/system/cpu.cpp index 756aa0a1a..68ad612a6 100644 --- a/src/eepp/system/cpu.cpp +++ b/src/eepp/system/cpu.cpp @@ -42,6 +42,12 @@ bool CPU::hasAVX2() { __cpuid( cpuInfo, 0 ); if ( cpuInfo[0] < 7 ) return false; + __cpuid( cpuInfo, 1 ); + constexpr int OSXSAVE = 1 << 27; + constexpr int AVX = 1 << 28; + if ( ( cpuInfo[2] & ( OSXSAVE | AVX ) ) != ( OSXSAVE | AVX ) || + ( _xgetbv( 0 ) & 0x6 ) != 0x6 ) + return false; __cpuid( cpuInfo, 7 ); return ( cpuInfo[1] & ( 1 << 5 ) ) != 0; #elif defined( COMPILER_GCC_CLANG ) diff --git a/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp b/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp index c137975fb..7972c7547 100644 --- a/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp +++ b/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp @@ -168,19 +168,20 @@ class KittyGraphicsProtocol { Int32 gapMs{ 40 }; Uint32 usageHint{ 0 }; }; - std::shared_ptr> rgba; + std::shared_ptr> pixels; std::unordered_map frames; Sizei size; + Uint64 creationSerial{ 0 }; + EE::System::Clock frameClock; Uint32 imageNumber{ 0 }; Uint32 usageHint{ 0 }; - bool anonymous{ false }; - Uint64 creationSerial{ 0 }; Uint32 currentFrame{ 1 }; Uint32 loopCount{ 1 }; Uint32 loopsCompleted{ 0 }; - Uint8 animationState{ 1 }; Int32 rootGapMs{ 0 }; - EE::System::Clock frameClock; + Uint8 animationState{ 1 }; + Uint8 channels{ 4 }; + bool anonymous{ false }; }; struct PendingTransfer { @@ -210,6 +211,7 @@ class KittyGraphicsProtocol { KittyImageId allocateImageId(); KittyImageId resolveImageId( const KittyGraphicsCommandData& data ) const; bool ensureCapacity( size_t bytes, KittyImageId replacingId, bool addingImage ); + bool ensureRootRGBA( KittyImageId imageId, Image& image ); bool isImagePlaced( KittyImageId imageId ) const; void eraseImage( KittyImageId imageId ); std::string response( const KittyGraphicsCommandData& data, KittyGraphicsError error, diff --git a/src/modules/eterm/include/eterm/terminal/kittygraphicsrenderer.hpp b/src/modules/eterm/include/eterm/terminal/kittygraphicsrenderer.hpp index e0e9b8173..a165304ee 100644 --- a/src/modules/eterm/include/eterm/terminal/kittygraphicsrenderer.hpp +++ b/src/modules/eterm/include/eterm/terminal/kittygraphicsrenderer.hpp @@ -30,6 +30,7 @@ class KittyGraphicsRenderer { TexturePtr texture; std::unordered_map frames; Sizei size; + Uint8 channels{ 4 }; }; std::unordered_map mImages; diff --git a/src/modules/eterm/include/eterm/terminal/terminalgraphics.hpp b/src/modules/eterm/include/eterm/terminal/terminalgraphics.hpp index 83b4ab6f0..8f059a7b9 100644 --- a/src/modules/eterm/include/eterm/terminal/terminalgraphics.hpp +++ b/src/modules/eterm/include/eterm/terminal/terminalgraphics.hpp @@ -64,15 +64,16 @@ enum class TerminalGraphicsUpdateType : Uint8 { }; struct TerminalGraphicsUpdate { - std::shared_ptr> rgba; + std::shared_ptr> pixels; Sizei imageSize; Rect region; Uint64 sequence{ 0 }; KittyImageId imageId{ 0 }; Uint32 frameNumber{ 1 }; TerminalGraphicsUpdateType type{ TerminalGraphicsUpdateType::Resync }; + Uint8 channels{ 4 }; - size_t payloadBytes() const { return rgba ? rgba->size() : 0; } + size_t payloadBytes() const { return pixels ? pixels->size() : 0; } }; /** Bounded ordered worker-to-UI mutation queue with explicit overflow recovery. */ diff --git a/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp b/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp index 218b36a5e..1d32fdc82 100644 --- a/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp +++ b/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp @@ -9,17 +9,8 @@ #include #include #include -#include #include -#if EE_PLATFORM != EE_PLATFORM_WIN && EE_PLATFORM != EE_PLATFORM_EMSCRIPTEN && \ - EE_PLATFORM != EE_PLATFORM_ANDROID -#include -#include -#include -#include -#endif - using namespace EE::System; namespace eterm { namespace Terminal { @@ -78,10 +69,12 @@ bool decodeBase64( std::string_view input, bool finalChunk, std::vector& if ( oldSize > MaxTransferBytes || capacity > MaxTransferBytes - oldSize ) return false; output.resize( oldSize + capacity ); + const Base64::DecodeMode mode = + finalChunk ? Base64::DecodeMode::NoWhitespaceStrict : Base64::DecodeMode::NoWhitespace; const size_t decodedSize = - Base64::decode( input.size(), input.data(), capacity, output.data() + oldSize, - Base64::DecodeMode::NoWhitespaceStrict ); - if ( decodedSize == static_cast( -1 ) ) { + Base64::decode( input.size(), input.data(), capacity, output.data() + oldSize, mode ); + if ( decodedSize == static_cast( -1 ) || + ( !finalChunk && decodedSize != input.size() / 4 * 3 ) ) { output.resize( oldSize ); return false; } @@ -89,79 +82,6 @@ bool decodeBase64( std::string_view input, bool finalChunk, std::vector& return true; } -bool readSharedMemory( const KittyGraphicsCommandData& data, std::vector& output ) { -#if EE_PLATFORM == EE_PLATFORM_WIN || EE_PLATFORM == EE_PLATFORM_EMSCRIPTEN || \ - EE_PLATFORM == EE_PLATFORM_ANDROID - (void)data; - (void)output; - return false; -#else - std::vector decodedName; - if ( !decodeBase64( data.payload, true, decodedName ) || decodedName.empty() || - decodedName.size() > 255 || - std::find( decodedName.begin() + 1, decodedName.end(), '/' ) != decodedName.end() || - std::find( decodedName.begin(), decodedName.end(), 0 ) != decodedName.end() ) - return false; - std::string name( decodedName.begin(), decodedName.end() ); - const int descriptor = shm_open( name.c_str(), O_RDONLY, 0 ); - if ( descriptor == -1 ) - return false; - // POSIX Kitty transfers are single-use. Unlink immediately after opening so all error paths - // still retire the client-owned object while the descriptor keeps its contents alive. - shm_unlink( name.c_str() ); - struct stat status{}; - const size_t offset = data.dataOffset.value_or( 0 ); - bool valid = fstat( descriptor, &status ) == 0 && status.st_size >= 0 && - static_cast( status.st_size ) >= offset; - size_t bytes = 0; - if ( valid ) { - const size_t available = static_cast( status.st_size ) - offset; - bytes = data.dataSize.value_or( static_cast( - std::min( available, std::numeric_limits::max() ) ) ); - valid = bytes <= available && bytes <= 128 * 1024 * 1024; - } - if ( valid && bytes != 0 ) { - const size_t mappingBytes = offset + bytes; - valid = mappingBytes >= bytes && mappingBytes <= 128 * 1024 * 1024; - void* mapping = valid ? mmap( nullptr, mappingBytes, PROT_READ, MAP_SHARED, descriptor, 0 ) - : MAP_FAILED; - if ( mapping == MAP_FAILED ) { - valid = false; - } else { - const auto* source = static_cast( mapping ) + offset; - // mpv reuses the same shm name for every frame. If it reopened the object before - // we unlinked it above, its next memcpy can overlap this read. Require consecutive - // identical observations after yielding to the writer; otherwise retain the previous - // displayed frame instead of publishing visibly torn rows. - std::vector snapshot( source, source + bytes ); - bool stable = false; - unsigned stableObservations = 0; - constexpr unsigned RequiredStableObservations = 2; - constexpr unsigned MaxSnapshotAttempts = 6; - for ( unsigned attempt = 0; attempt < MaxSnapshotAttempts; ++attempt ) { - std::this_thread::yield(); - if ( std::memcmp( snapshot.data(), source, bytes ) == 0 ) { - if ( ++stableObservations == RequiredStableObservations ) { - stable = true; - break; - } - } else { - stableObservations = 0; - snapshot.assign( source, source + bytes ); - } - } - if ( stable ) - output = std::move( snapshot ); - else - valid = false; - munmap( mapping, mappingBytes ); - } - } - close( descriptor ); - return valid && bytes != 0; -#endif -} - bool placementContains( const TerminalVisiblePlacement& placement, Vector2i cell ) { const Int64 right = static_cast( placement.visibleAnchorCell.x ) + placement.columns; const Int64 bottom = static_cast( placement.visibleAnchorCell.y ) + placement.rows; @@ -402,20 +322,6 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::handleTransmit( const KittyGraphicsCommandData& data, bool display, bool query, bool frame, Vector2i cursor ) { const bool more = data.more.value_or( 0 ) != 0; - if ( data.transmission == 's' ) { - if ( mPending.active ) - mPending = {}; - PendingTransfer transfer; - transfer.data = data; - transfer.data.payload = {}; - transfer.display = display; - transfer.query = query; - transfer.frame = frame; - if ( !readSharedMemory( data, transfer.decodedData ) ) - return { response( data, KittyGraphicsError::DecodeFailed ), - KittyGraphicsError::DecodeFailed, false }; - return finishTransfer( std::move( transfer ), cursor ); - } if ( mPending.active ) { if ( frame != mPending.frame || data.format || data.dataSize || data.imageId || data.imageNumber || data.usageHint || data.placementId || data.width || data.height || @@ -561,7 +467,7 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer } } - if ( format == 24 ) { + if ( format == 24 && transfer.frame ) { std::vector rgba = std::move( mPixelScratch ); rgba.resize( static_cast( width ) * height * 4 ); for ( size_t sourceOffset = 0, destinationOffset = 0; sourceOffset < pixels.size(); @@ -580,6 +486,9 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer if ( image == mImages.end() ) return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, false }; + if ( !ensureRootRGBA( imageId, image->second ) ) + return { response( data, KittyGraphicsError::NoSpace ), KittyGraphicsError::NoSpace, + false }; Uint32 frameNumber = data.rows.value_or( 0 ); if ( frameNumber == 0 ) { frameNumber = 2; @@ -599,9 +508,10 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer std::vector* destinationPixels = nullptr; bool createdFrame = false; if ( frameNumber == 1 ) { - if ( image->second.rgba.use_count() != 1 ) - image->second.rgba = std::make_shared>( *image->second.rgba ); - destinationPixels = image->second.rgba.get(); + if ( image->second.pixels.use_count() != 1 ) + image->second.pixels = + std::make_shared>( *image->second.pixels ); + destinationPixels = image->second.pixels.get(); } else { auto frame = image->second.frames.find( frameNumber ); if ( frame == image->second.frames.end() ) { @@ -611,7 +521,7 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer if ( data.columns ) { const Uint32 baseFrame = *data.columns; if ( baseFrame == 1 ) - newFrame.rgba = *image->second.rgba; + newFrame.rgba = *image->second.pixels; else { auto base = image->second.frames.find( baseFrame ); if ( base == image->second.frames.end() ) @@ -687,9 +597,9 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer createdFrame ? Rect( 0, 0, image->second.size.getWidth(), image->second.size.getHeight() ) : Rect( destinationX, destinationY, destinationX + width, destinationY + height ); - update.rgba = createdFrame - ? std::make_shared>( *destinationPixels ) - : std::make_shared>( std::move( finalPatch ) ); + update.pixels = createdFrame + ? std::make_shared>( *destinationPixels ) + : std::make_shared>( std::move( finalPatch ) ); if ( !createdFrame && !mUpdates.empty() && mUpdates.back().type == update.type && mUpdates.back().imageId == update.imageId && mUpdates.back().frameNumber == update.frameNumber && @@ -730,7 +640,7 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer false }; auto existing = mImages.find( imageId ); - const size_t oldBytes = existing == mImages.end() ? 0 : existing->second.rgba->size(); + const size_t oldBytes = existing == mImages.end() ? 0 : existing->second.pixels->size(); if ( !ensureCapacity( pixels.size(), imageId, existing == mImages.end() ) ) return { response( data, KittyGraphicsError::NoSpace ), KittyGraphicsError::NoSpace, false }; @@ -751,11 +661,11 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer mPrimaryPlacements.end() ); } std::shared_ptr> pixelStorage; - if ( existing != mImages.end() && existing->second.rgba.use_count() == 1 ) { - pixelStorage = existing->second.rgba; + if ( existing != mImages.end() && existing->second.pixels.use_count() == 1 ) { + pixelStorage = existing->second.pixels; pixelStorage->assign( pixels.begin(), pixels.end() ); if ( format == 24 ) - recycleBuffer( pixels, mPixelScratch ); + recycleBuffer( pixels, mDecodedScratch ); else if ( format == 32 ) recycleBuffer( pixels, mDecodedScratch ); } else { @@ -767,8 +677,9 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer image.usageHint = data.usageHint.value_or( 0 ); image.anonymous = anonymous; image.creationSerial = ++mCreationSerial; - image.rgba = std::move( pixelStorage ); - mStorageBytes = mStorageBytes - oldBytes + image.rgba->size(); + image.channels = format == 24 ? 3 : 4; + image.pixels = std::move( pixelStorage ); + mStorageBytes = mStorageBytes - oldBytes + image.pixels->size(); const bool replaced = existing != mImages.end(); auto inserted = mImages.insert_or_assign( imageId, std::move( image ) ).first; @@ -777,9 +688,10 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer : TerminalGraphicsUpdateType::CreateImage; update.imageId = imageId; update.imageSize = inserted->second.size; + update.channels = inserted->second.channels; update.region = Rect( 0, 0, inserted->second.size.getWidth(), inserted->second.size.getHeight() ); - update.rgba = inserted->second.rgba; + update.pixels = inserted->second.pixels; mUpdates.emplace_back( std::move( update ) ); ++mStats.fullImageUpdates; ++mPresentationGeneration; @@ -1241,9 +1153,12 @@ KittyGraphicsProtocol::composeFrames( const KittyGraphicsCommandData& data ) { image == mImages.end() ? KittyGraphicsError::NotFound : KittyGraphicsError::InvalidArgument, false }; + if ( !ensureRootRGBA( imageId, image->second ) ) + return { response( data, KittyGraphicsError::NoSpace ), KittyGraphicsError::NoSpace, + false }; auto pixelsFor = [&]( Uint32 frameNumber ) -> std::vector* { if ( frameNumber == 1 ) - return image->second.rgba.get(); + return image->second.pixels.get(); auto frame = image->second.frames.find( frameNumber ); return frame == image->second.frames.end() ? nullptr : &frame->second.rgba; }; @@ -1284,9 +1199,9 @@ KittyGraphicsProtocol::composeFrames( const KittyGraphicsCommandData& data ) { source->data() + static_cast( sourceY + row ) * imageStride + static_cast( sourceX ) * 4, rowBytes ); - if ( destinationFrame == 1 && image->second.rgba.use_count() != 1 ) { - image->second.rgba = std::make_shared>( *image->second.rgba ); - destination = image->second.rgba.get(); + if ( destinationFrame == 1 && image->second.pixels.use_count() != 1 ) { + image->second.pixels = std::make_shared>( *image->second.pixels ); + destination = image->second.pixels.get(); } const bool replace = data.cursorMovement.value_or( 0 ) == 1; for ( Uint32 row = 0; row < height; ++row ) { @@ -1320,7 +1235,7 @@ KittyGraphicsProtocol::composeFrames( const KittyGraphicsCommandData& data ) { update.frameNumber = destinationFrame; update.imageSize = image->second.size; update.region = Rect( destinationX, destinationY, destinationX + width, destinationY + height ); - update.rgba = std::make_shared>( std::move( result ) ); + update.pixels = std::make_shared>( std::move( result ) ); mUpdates.emplace_back( std::move( update ) ); mPresentationDirty = true; return { response( data, KittyGraphicsError::None, imageId ), KittyGraphicsError::None, true }; @@ -1366,7 +1281,7 @@ void KittyGraphicsProtocol::eraseImage( KittyImageId imageId ) { auto image = mImages.find( imageId ); if ( image == mImages.end() ) return; - mStorageBytes -= image->second.rgba->size(); + mStorageBytes -= image->second.pixels->size(); for ( const auto& frame : image->second.frames ) mFrameStorageBytes -= frame.second.rgba.size(); mImages.erase( image ); @@ -1379,7 +1294,7 @@ void KittyGraphicsProtocol::eraseImage( KittyImageId imageId ) { bool KittyGraphicsProtocol::ensureCapacity( size_t bytes, KittyImageId replacingId, bool addingImage ) { auto replaced = mImages.find( replacingId ); - const size_t replacedBytes = replaced == mImages.end() ? 0 : replaced->second.rgba->size(); + const size_t replacedBytes = replaced == mImages.end() ? 0 : replaced->second.pixels->size(); auto hasCapacity = [&] { return bytes <= mMaxStorageBytes && mStorageBytes - replacedBytes <= mMaxStorageBytes - bytes && @@ -1404,6 +1319,37 @@ bool KittyGraphicsProtocol::ensureCapacity( size_t bytes, KittyImageId replacing return true; } +bool KittyGraphicsProtocol::ensureRootRGBA( KittyImageId imageId, Image& image ) { + if ( image.channels == 4 ) + return true; + const size_t pixelCount = static_cast( image.size.getWidth() ) * image.size.getHeight(); + const size_t rgbaBytes = pixelCount * 4; + if ( !ensureCapacity( rgbaBytes, imageId, false ) ) + return false; + std::vector rgba; + rgba.resize( rgbaBytes ); + for ( size_t pixel = 0; pixel < pixelCount; ++pixel ) { + const size_t source = pixel * 3; + const size_t destination = pixel * 4; + rgba[destination] = ( *image.pixels )[source]; + rgba[destination + 1] = ( *image.pixels )[source + 1]; + rgba[destination + 2] = ( *image.pixels )[source + 2]; + rgba[destination + 3] = 255; + } + const size_t oldBytes = image.pixels->size(); + image.pixels = std::make_shared>( std::move( rgba ) ); + image.channels = 4; + mStorageBytes = mStorageBytes - oldBytes + rgbaBytes; + TerminalGraphicsUpdate update; + update.type = TerminalGraphicsUpdateType::ReplaceImage; + update.imageId = imageId; + update.imageSize = image.size; + update.region = Rect( 0, 0, image.size.getWidth(), image.size.getHeight() ); + update.pixels = image.pixels; + mUpdates.emplace_back( std::move( update ) ); + return true; +} + std::string KittyGraphicsProtocol::response( const KittyGraphicsCommandData& data, KittyGraphicsError error, KittyImageId imageId ) const { @@ -1528,7 +1474,7 @@ std::shared_ptr KittyGraphicsProtocol::takePresent const std::vector* KittyGraphicsProtocol::imagePixels( KittyImageId imageId ) const { auto image = mImages.find( imageId ); - return image == mImages.end() ? nullptr : image->second.rgba.get(); + return image == mImages.end() ? nullptr : image->second.pixels.get(); } bool KittyGraphicsProtocol::hasVirtualPlacements() const { @@ -1665,8 +1611,9 @@ void KittyGraphicsProtocol::resync() { create.type = TerminalGraphicsUpdateType::CreateImage; create.imageId = image.first; create.imageSize = image.second.size; + create.channels = image.second.channels; create.region = Rect( 0, 0, image.second.size.getWidth(), image.second.size.getHeight() ); - create.rgba = image.second.rgba; + create.pixels = image.second.pixels; mUpdates.emplace_back( std::move( create ) ); for ( const auto& frame : image.second.frames ) { TerminalGraphicsUpdate createFrame; @@ -1676,7 +1623,7 @@ void KittyGraphicsProtocol::resync() { createFrame.imageSize = image.second.size; createFrame.region = Rect( 0, 0, image.second.size.getWidth(), image.second.size.getHeight() ); - createFrame.rgba = std::make_shared>( frame.second.rgba ); + createFrame.pixels = std::make_shared>( frame.second.rgba ); mUpdates.emplace_back( std::move( createFrame ) ); } } diff --git a/src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp b/src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp index 7b8302d33..75e4e44cd 100644 --- a/src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp +++ b/src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp @@ -27,58 +27,68 @@ bool KittyGraphicsRenderer::applyUpdates( std::vector&& switch ( update.type ) { case TerminalGraphicsUpdateType::ReplaceImage: { - if ( !update.rgba || update.imageSize.getWidth() <= 0 || - update.imageSize.getHeight() <= 0 ) + if ( !update.pixels || update.imageSize.getWidth() <= 0 || + update.imageSize.getHeight() <= 0 || + ( update.channels != 3 && update.channels != 4 ) || + update.pixels->size() != static_cast( update.imageSize.getWidth() ) * + update.imageSize.getHeight() * update.channels ) return false; auto existing = mImages.find( update.imageId ); if ( existing != mImages.end() && existing->second.texture && - existing->second.size == update.imageSize ) { - existing->second.texture->update( update.rgba->data(), - update.imageSize.getWidth(), - update.imageSize.getHeight(), 0, 0 ); + existing->second.size == update.imageSize && + existing->second.channels == update.channels ) { + existing->second.texture->update( + update.pixels->data(), update.imageSize.getWidth(), + update.imageSize.getHeight(), 0, 0, + Image::channelsToPixelFormat( update.channels ) ); existing->second.frames.clear(); break; } [[fallthrough]]; } case TerminalGraphicsUpdateType::CreateImage: { - if ( !update.rgba || update.imageSize.getWidth() <= 0 || - update.imageSize.getHeight() <= 0 ) + if ( !update.pixels || update.imageSize.getWidth() <= 0 || + update.imageSize.getHeight() <= 0 || + ( update.channels != 3 && update.channels != 4 ) || + update.pixels->size() != static_cast( update.imageSize.getWidth() ) * + update.imageSize.getHeight() * update.channels ) return false; auto texture = TextureFactory::instance()->loadFromPixels( - update.rgba->data(), update.imageSize.getWidth(), update.imageSize.getHeight(), - 4, false, Texture::ClampMode::ClampToEdge, false, false ); + update.pixels->data(), update.imageSize.getWidth(), + update.imageSize.getHeight(), update.channels, false, + Texture::ClampMode::ClampToEdge, false, false ); if ( !texture ) return false; texture->setFilter( Texture::Filter::Linear ); - mImages.insert_or_assign( update.imageId, - GPUImage{ std::move( texture ), {}, update.imageSize } ); + mImages.insert_or_assign( + update.imageId, + GPUImage{ std::move( texture ), {}, update.imageSize, update.channels } ); break; } case TerminalGraphicsUpdateType::UpdateRegion: { auto image = mImages.find( update.imageId ); - if ( image == mImages.end() || !update.rgba ) + if ( image == mImages.end() || !update.pixels ) return false; const int width = update.region.Right - update.region.Left; const int height = update.region.Bottom - update.region.Top; if ( width <= 0 || height <= 0 || update.region.Left < 0 || update.region.Top < 0 || update.region.Right > image->second.size.getWidth() || update.region.Bottom > image->second.size.getHeight() || - update.rgba->size() != static_cast( width ) * height * 4 ) + update.pixels->size() != static_cast( width ) * height * 4 ) return false; - image->second.texture->update( update.rgba->data(), width, height, + image->second.texture->update( update.pixels->data(), width, height, update.region.Left, update.region.Top ); break; } case TerminalGraphicsUpdateType::CreateFrame: case TerminalGraphicsUpdateType::ReplaceFrame: { auto image = mImages.find( update.imageId ); - if ( image == mImages.end() || !update.rgba || update.frameNumber <= 1 || - update.rgba->size() != static_cast( image->second.size.getWidth() ) * - image->second.size.getHeight() * 4 ) + if ( image == mImages.end() || !update.pixels || update.frameNumber <= 1 || + update.pixels->size() != static_cast( image->second.size.getWidth() ) * + image->second.size.getHeight() * 4 ) return false; auto texture = TextureFactory::instance()->loadFromPixels( - update.rgba->data(), image->second.size.getWidth(), + update.pixels->data(), image->second.size.getWidth(), image->second.size.getHeight(), 4, false, Texture::ClampMode::ClampToEdge, false, false ); if ( !texture ) @@ -89,7 +99,7 @@ bool KittyGraphicsRenderer::applyUpdates( std::vector&& } case TerminalGraphicsUpdateType::UpdateFrameRegion: { auto image = mImages.find( update.imageId ); - if ( image == mImages.end() || !update.rgba ) + if ( image == mImages.end() || !update.pixels ) return false; auto frame = image->second.frames.find( update.frameNumber ); if ( frame == image->second.frames.end() ) @@ -99,9 +109,9 @@ bool KittyGraphicsRenderer::applyUpdates( std::vector&& if ( width <= 0 || height <= 0 || update.region.Left < 0 || update.region.Top < 0 || update.region.Right > image->second.size.getWidth() || update.region.Bottom > image->second.size.getHeight() || - update.rgba->size() != static_cast( width ) * height * 4 ) + update.pixels->size() != static_cast( width ) * height * 4 ) return false; - frame->second->update( update.rgba->data(), width, height, update.region.Left, + frame->second->update( update.pixels->data(), width, height, update.region.Left, update.region.Top ); break; } diff --git a/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp b/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp index 18c03c529..0e80b02de 100644 --- a/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp +++ b/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp @@ -61,6 +61,7 @@ using namespace EE::System; #include #include #include +#include #if defined( EE_ARCH_X86_64 ) #include @@ -124,6 +125,39 @@ TerminalCursorMode blinkingCursorVariant( TerminalCursorMode mode ) { } // namespace #if defined( EE_ARCH_X86_64 ) +#if defined( __GNUC__ ) || defined( __clang__ ) +__attribute__( ( target( "avx2" ) ) ) +#endif +static int findKittyAPCControlAVX2( const char* data, int begin, int end ) { + const __m256i highThreeBitMask = _mm256_set1_epi8( static_cast( 0xE0 ) ); + const __m256i c1Prefix = _mm256_set1_epi8( static_cast( 0x80 ) ); + const __m256i bell = _mm256_set1_epi8( 0x07 ); + const __m256i cancel = _mm256_set1_epi8( 0x18 ); + const __m256i substitute = _mm256_set1_epi8( 0x1A ); + const __m256i escape = _mm256_set1_epi8( 0x1B ); + int offset = begin; + for ( ; offset + 32 <= end; offset += 32 ) { + const __m256i bytes = + _mm256_loadu_si256( reinterpret_cast( data + offset ) ); + const __m256i c1 = + _mm256_cmpeq_epi8( _mm256_and_si256( bytes, highThreeBitMask ), c1Prefix ); + const __m256i explicitControls = _mm256_or_si256( + _mm256_or_si256( _mm256_cmpeq_epi8( bytes, bell ), _mm256_cmpeq_epi8( bytes, cancel ) ), + _mm256_or_si256( _mm256_cmpeq_epi8( bytes, substitute ), + _mm256_cmpeq_epi8( bytes, escape ) ) ); + if ( _mm256_movemask_epi8( _mm256_or_si256( c1, explicitControls ) ) != 0 ) + break; + } + while ( offset < end ) { + const unsigned char byte = static_cast( data[offset] ); + if ( byte == '\a' || byte == 030 || byte == 032 || byte == 033 || + ( byte >= 0x80 && byte <= 0x9F ) ) + break; + ++offset; + } + return offset; +} + #if defined( __GNUC__ ) || defined( __clang__ ) __attribute__( ( target( "avx2" ) ) ) #endif @@ -165,6 +199,21 @@ static int trailingNonSpaceWidthNEON( Line line, int width, int minimumWidth ) { } #endif +static int findKittyAPCControl( const char* data, int begin, int end ) { +#if defined( EE_ARCH_X86_64 ) + if ( CPU::hasAVX2() ) + return findKittyAPCControlAVX2( data, begin, end ); +#endif + while ( begin < end ) { + const unsigned char byte = static_cast( data[begin] ); + if ( byte == '\a' || byte == 030 || byte == 032 || byte == 033 || + ( byte >= 0x80 && byte <= 0x9F ) ) + break; + ++begin; + } + return begin; +} + /* identification sequence returned in DA and DECID */ static const char* vtiden = "\033[?6c"; @@ -3243,14 +3292,7 @@ int TerminalEmulator::twrite( const char* buf, int buflen, int show_ctrl ) { * behavior. */ if ( !show_ctrl && ( mTerm.esc & ESC_STR ) && mStrescseq.type == '_' && mStrescseq.len > 0 && mStrescseq.buf[0] == 'G' ) { - int end = n; - while ( end < buflen ) { - const unsigned char byte = static_cast( buf[end] ); - if ( byte == '\a' || byte == 030 || byte == 032 || byte == 033 || - ( byte >= 0x80 && byte <= 0x9F ) ) - break; - ++end; - } + const int end = findKittyAPCControl( buf, n, buflen ); const size_t bytes = static_cast( end - n ); if ( bytes != 0 ) { if ( !mStrescseq.discarded ) { @@ -4108,6 +4150,16 @@ bool TerminalEmulator::update() { onProcessExit( mExitCode ); } + /* A non-blocking read can temporarily catch up with a producer that was blocked writing to the + * PTY. Do not enter the worker's 8 ms idle wait immediately after consuming data: give the + * producer a scheduling opportunity and probe the PTY once more. This matters for high-volume + * protocols such as Kitty graphics, where otherwise every transport chunk can pay one idle + * interval after the receiver becomes faster than the sender's wakeup latency. */ + if ( reads > 0 && !readBudgetSaturated ) { + std::this_thread::yield(); + return false; + } + return !readBudgetSaturated; } diff --git a/src/tests/unit_tests/eterm_tests.cpp b/src/tests/unit_tests/eterm_tests.cpp index c7493652e..d92f82456 100644 --- a/src/tests/unit_tests/eterm_tests.cpp +++ b/src/tests/unit_tests/eterm_tests.cpp @@ -14,13 +14,6 @@ #include #include -#if EE_PLATFORM != EE_PLATFORM_WIN && EE_PLATFORM != EE_PLATFORM_EMSCRIPTEN && \ - EE_PLATFORM != EE_PLATFORM_ANDROID -#include -#include -#include -#endif - using namespace eterm::Terminal; using namespace eterm::System; using namespace EE::System; @@ -133,11 +126,11 @@ UTEST( eterm_session, graphics_update_queue_preserves_order_and_payloads ) { TerminalGraphicsUpdate create; create.type = TerminalGraphicsUpdateType::CreateImage; create.imageId = 7; - create.rgba = pixels; + create.pixels = pixels; TerminalGraphicsUpdate patch; patch.type = TerminalGraphicsUpdateType::UpdateRegion; patch.imageId = 7; - patch.rgba = pixels; + patch.pixels = pixels; EXPECT_EQ( static_cast( 1 ), queue.enqueue( std::move( create ) ) ); EXPECT_EQ( static_cast( 2 ), queue.enqueue( std::move( patch ) ) ); @@ -148,14 +141,14 @@ UTEST( eterm_session, graphics_update_queue_preserves_order_and_payloads ) { EXPECT_EQ( static_cast( 2 ), updates[1].sequence ); EXPECT_EQ( TerminalGraphicsUpdateType::CreateImage, updates[0].type ); EXPECT_EQ( TerminalGraphicsUpdateType::UpdateRegion, updates[1].type ); - EXPECT_TRUE( pixels == updates[0].rgba ); + EXPECT_TRUE( pixels == updates[0].pixels ); } UTEST( eterm_session, graphics_update_queue_overflow_requires_resync ) { TerminalGraphicsUpdateQueue queue( 2, 8 ); TerminalGraphicsUpdate update; update.type = TerminalGraphicsUpdateType::UpdateRegion; - update.rgba = std::make_shared>( 8, 0xFF ); + update.pixels = std::make_shared>( 8, 0xFF ); queue.enqueue( update ); queue.enqueue( std::move( update ) ); @@ -172,13 +165,13 @@ UTEST( eterm_session, graphics_update_queue_coalesces_superseded_video_frames ) TerminalGraphicsUpdate create; create.type = TerminalGraphicsUpdateType::CreateImage; create.imageId = 7; - create.rgba = std::make_shared>( 8, 1 ); + create.pixels = std::make_shared>( 8, 1 ); EXPECT_EQ( static_cast( 1 ), queue.enqueue( std::move( create ) ) ); for ( Uint8 frame = 2; frame < 20; ++frame ) { TerminalGraphicsUpdate replacement; replacement.type = TerminalGraphicsUpdateType::ReplaceImage; replacement.imageId = 7; - replacement.rgba = std::make_shared>( 8, frame ); + replacement.pixels = std::make_shared>( 8, frame ); EXPECT_EQ( static_cast( 1 ), queue.enqueue( std::move( replacement ) ) ); } EXPECT_FALSE( queue.needsResync() ); @@ -186,7 +179,7 @@ UTEST( eterm_session, graphics_update_queue_coalesces_superseded_video_frames ) auto updates = queue.drain(); ASSERT_EQ( static_cast( 1 ), updates.size() ); EXPECT_EQ( TerminalGraphicsUpdateType::CreateImage, updates[0].type ); - EXPECT_EQ( static_cast( 19 ), updates[0].rgba->front() ); + EXPECT_EQ( static_cast( 19 ), updates[0].pixels->front() ); } UTEST( eterm_session, ordered_selection_request ) { @@ -561,7 +554,7 @@ UTEST( eterm, kitty_graphics_image_number_allocates_id_and_echoes_number ) { ASSERT_EQ( static_cast( 1 ), protocol.takePresentation()->placements.size() ); } -UTEST( eterm, kitty_graphics_rgb_and_zlib_normalize_to_rgba ) { +UTEST( eterm, kitty_graphics_rgb_and_zlib_preserve_rgb24 ) { const std::vector rgb{ 10, 20, 30, 40, 50, 60 }; std::vector compressed( Compression::getMaxCompressedBufferSize( rgb.size() ) ); IOStreamMemory source( reinterpret_cast( rgb.data() ), rgb.size() ); @@ -578,8 +571,12 @@ UTEST( eterm, kitty_graphics_rgb_and_zlib_normalize_to_rgba ) { EXPECT_TRUE( result.changed ); auto pixels = protocol.imagePixels( 9 ); ASSERT_TRUE( pixels != nullptr ); - const std::vector expected{ 10, 20, 30, 255, 40, 50, 60, 255 }; + const std::vector expected{ 10, 20, 30, 40, 50, 60 }; EXPECT_TRUE( expected == *pixels ); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( static_cast( 3 ), updates.front().channels ); + EXPECT_TRUE( updates.front().pixels && expected == *updates.front().pixels ); } UTEST( eterm, kitty_graphics_png_decodes_to_rgba ) { @@ -652,8 +649,8 @@ UTEST( eterm, kitty_graphics_resync_republishes_authoritative_images ) { EXPECT_EQ( TerminalGraphicsUpdateType::ResetAll, updates[0].type ); EXPECT_EQ( TerminalGraphicsUpdateType::CreateImage, updates[1].type ); EXPECT_EQ( static_cast( 27 ), updates[1].imageId ); - ASSERT_TRUE( updates[1].rgba != nullptr ); - EXPECT_EQ( static_cast( 4 ), updates[1].rgba->size() ); + ASSERT_TRUE( updates[1].pixels != nullptr ); + EXPECT_EQ( static_cast( 4 ), updates[1].pixels->size() ); } UTEST( eterm, kitty_graphics_root_frame_patch_publishes_only_changed_rectangle ) { @@ -673,8 +670,28 @@ UTEST( eterm, kitty_graphics_root_frame_patch_publishes_only_changed_rectangle ) EXPECT_EQ( 0, updates[0].region.Top ); EXPECT_EQ( 2, updates[0].region.Right ); EXPECT_EQ( 1, updates[0].region.Bottom ); - ASSERT_TRUE( updates[0].rgba != nullptr ); - EXPECT_EQ( static_cast( 4 ), updates[0].rgba->size() ); + ASSERT_TRUE( updates[0].pixels != nullptr ); + EXPECT_EQ( static_cast( 4 ), updates[0].pixels->size() ); +} + +UTEST( eterm, kitty_graphics_rgb24_root_converts_only_when_mutated ) { + KittyGraphicsProtocol protocol; + ASSERT_EQ( KittyGraphicsError::None, + protocol.handle( "a=t,f=24,s=2,v=1,i=92;AQIDBAUG" ).error ); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( static_cast( 3 ), updates[0].channels ); + + ASSERT_EQ( KittyGraphicsError::None, + protocol.handle( "a=f,i=92,r=1,f=32,s=1,v=1,x=1,y=0,X=1;BwgJCg==" ).error ); + const std::vector expected{ 1, 2, 3, 255, 7, 8, 9, 10 }; + ASSERT_TRUE( protocol.imagePixels( 92 ) != nullptr ); + EXPECT_TRUE( expected == *protocol.imagePixels( 92 ) ); + updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 2 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::ReplaceImage, updates[0].type ); + EXPECT_EQ( static_cast( 4 ), updates[0].channels ); + EXPECT_EQ( TerminalGraphicsUpdateType::UpdateRegion, updates[1].type ); } UTEST( eterm, kitty_graphics_animation_frame_create_control_and_compose ) { @@ -688,9 +705,9 @@ UTEST( eterm, kitty_graphics_animation_frame_create_control_and_compose ) { ASSERT_EQ( static_cast( 1 ), updates.size() ); EXPECT_EQ( TerminalGraphicsUpdateType::CreateFrame, updates[0].type ); EXPECT_EQ( static_cast( 2 ), updates[0].frameNumber ); - ASSERT_TRUE( updates[0].rgba != nullptr ); + ASSERT_TRUE( updates[0].pixels != nullptr ); const std::vector expectedFrame{ 1, 2, 3, 4, 9, 10, 11, 12 }; - EXPECT_TRUE( expectedFrame == *updates[0].rgba ); + EXPECT_TRUE( expectedFrame == *updates[0].pixels ); auto control = protocol.handle( "a=a,i=30,c=2" ); EXPECT_TRUE( control.changed ); @@ -703,9 +720,9 @@ UTEST( eterm, kitty_graphics_animation_frame_create_control_and_compose ) { updates = protocol.takeUpdates(); ASSERT_EQ( static_cast( 1 ), updates.size() ); EXPECT_EQ( TerminalGraphicsUpdateType::UpdateFrameRegion, updates[0].type ); - ASSERT_TRUE( updates[0].rgba != nullptr ); + ASSERT_TRUE( updates[0].pixels != nullptr ); const std::vector expectedPatch{ 5, 6, 7, 8 }; - EXPECT_TRUE( expectedPatch == *updates[0].rgba ); + EXPECT_TRUE( expectedPatch == *updates[0].pixels ); EXPECT_TRUE( protocol.handle( "a=a,i=30,c=1,r=1,z=-1,s=3" ).changed ); EXPECT_TRUE( protocol.updateAnimations() ); presentation = protocol.takePresentation(); @@ -977,6 +994,25 @@ UTEST( eterm, kitty_graphics_strict_base64_rejects_invalid_payload_bytes ) { EXPECT_EQ( KittyGraphicsError::InvalidData, protocol.handle( "a=t,f=32,s=1,v=1;AQ=DBA==" ).error ); EXPECT_EQ( KittyGraphicsError::InvalidData, protocol.handle( "a=t,f=32,s=1,v=1;AB==" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidData, + protocol.handle( "a=t,f=32,s=1,v=1,m=1;AQ==" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidData, + protocol.handle( "a=t,f=32,s=1,v=1,m=1;AQI!" ).error ); +} + +UTEST( eterm, kitty_graphics_strict_base64_validates_final_quantum ) { + auto decode = []( std::string_view encoded ) { + std::vector output( Base64::decodeSafeOutLen( encoded.size() ) ); + return Base64::decode( encoded.size(), encoded.data(), output.size(), output.data(), + Base64::DecodeMode::NoWhitespaceStrict ); + }; + EXPECT_EQ( static_cast( 1 ), decode( "AA" ) ); + EXPECT_EQ( static_cast( 2 ), decode( "AAA" ) ); + EXPECT_EQ( static_cast( 1 ), decode( "AA==" ) ); + EXPECT_EQ( static_cast( 2 ), decode( "AAA=" ) ); + for ( std::string_view invalid : + { "A", "AB", "AAB", "AB==", "AAB=", "====", "A===", "AA=A", "AAAA=", "AAAA====" } ) + EXPECT_EQ( static_cast( -1 ), decode( invalid ) ); } UTEST( eterm, kitty_graphics_strict_base64_decodes_boundaries_and_large_payloads ) { @@ -1032,36 +1068,15 @@ UTEST( eterm, kitty_graphics_reuses_unreferenced_replacement_pixel_storage ) { ASSERT_EQ( KittyGraphicsError::None, protocol.handle( "a=t,f=24,s=2,v=1,i=91,q=2;BwgJCgsM" ).error ); EXPECT_TRUE( storage == protocol.imagePixels( 91 ) ); - const std::vector expected{ 7, 8, 9, 255, 10, 11, 12, 255 }; + const std::vector expected{ 7, 8, 9, 10, 11, 12 }; EXPECT_TRUE( expected == *protocol.imagePixels( 91 ) ); } -#if EE_PLATFORM != EE_PLATFORM_WIN && EE_PLATFORM != EE_PLATFORM_EMSCRIPTEN && \ - EE_PLATFORM != EE_PLATFORM_ANDROID -UTEST( eterm, kitty_graphics_reads_and_unlinks_posix_shared_memory ) { - // mpv uses the Linux-compatible form without the optional leading slash. - const std::string name = "eterm-kitty-unit-" + std::to_string( getpid() ); - shm_unlink( name.c_str() ); - const int descriptor = shm_open( name.c_str(), O_CREAT | O_EXCL | O_RDWR, 0600 ); - ASSERT_TRUE( descriptor >= 0 ); - const Uint8 stored[] = { 99, 98, 1, 2, 3 }; - ASSERT_EQ( static_cast( sizeof( stored ) ), - write( descriptor, stored, sizeof( stored ) ) ); - close( descriptor ); - - std::string encodedName; - ASSERT_TRUE( Base64::encode( name, encodedName ) ); +UTEST( eterm, kitty_graphics_rejects_shared_memory_transmission ) { KittyGraphicsProtocol protocol; - EXPECT_EQ( KittyGraphicsError::None, - protocol.handle( "a=T,t=s,f=24,s=1,v=1,O=2,S=3,q=2,m=1;" + encodedName ).error ); - auto updates = protocol.takeUpdates(); - ASSERT_EQ( static_cast( 1 ), updates.size() ); - ASSERT_TRUE( updates[0].rgba != nullptr ); - const std::vector expected{ 1, 2, 3, 255 }; - EXPECT_TRUE( expected == *updates[0].rgba ); - EXPECT_EQ( -1, shm_open( name.c_str(), O_RDONLY, 0 ) ); + EXPECT_EQ( KittyGraphicsError::Unsupported, + protocol.handle( "a=T,t=s,f=24,s=1,v=1,q=2;L2VlcHAtc2ht" ).error ); } -#endif UTEST( eterm, kitty_graphics_accepts_unchunked_direct_image_larger_than_eight_kibibytes ) { std::vector rgb( 64 * 64 * 3 ); diff --git a/src/tests/unit_tests/gitconflict_tests.cpp b/src/tests/unit_tests/gitconflict_tests.cpp index 9f574cee8..4ec037a0f 100644 --- a/src/tests/unit_tests/gitconflict_tests.cpp +++ b/src/tests/unit_tests/gitconflict_tests.cpp @@ -83,6 +83,7 @@ UTEST( GitConflict, KeepsMergeOperationAfterAllConflictsAreStaged ) { ASSERT_EQ( EXIT_SUCCESS, run( { "init", "-b", "main" } ) ); ASSERT_EQ( EXIT_SUCCESS, run( { "config", "user.name", "eepp tests" } ) ); ASSERT_EQ( EXIT_SUCCESS, run( { "config", "user.email", "eepp-tests@example.invalid" } ) ); + ASSERT_EQ( EXIT_SUCCESS, run( { "config", "rerere.enabled", "false" } ) ); const std::string file = ( temp.path / "conflict.txt" ).string(); ASSERT_TRUE( FileSystem::fileWrite( file, "base\n" ) ); ASSERT_EQ( EXIT_SUCCESS, run( { "add", "conflict.txt" } ) ); From 0e9f2e832f0a8babdddb4d5a43d89af9f2922ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Fri, 4 Sep 2026 18:45:10 -0300 Subject: [PATCH 7/8] Optimize eterm Kitty graphics frame replacement - swap decoded frames into reusable image storage - recycle previous frame allocations for subsequent transfers - avoid full-frame copies during anonymous image replacement - extend tests to verify safe allocation reuse across frames - document eterm and eeiv in the project introduction --- .agent/rules/project-introduction.md | 13 ++++++++++++- .../src/eterm/terminal/kittygraphicsprotocol.cpp | 10 +++++----- src/tests/unit_tests/eterm_tests.cpp | 12 ++++++++++++ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/.agent/rules/project-introduction.md b/.agent/rules/project-introduction.md index 70a5681b9..d4a413ed5 100644 --- a/.agent/rules/project-introduction.md +++ b/.agent/rules/project-introduction.md @@ -1,6 +1,7 @@ # Project Architecture: eepp & ecode -This repository contains two primary components: a core framework (`eepp`) and an application built on top of it (`ecode`). +This repository contains the core framework (`eepp`) and several applications and tools built on +top of it, including `ecode`, `eterm`, and `eeiv`. ## 1. eepp (Core Framework) @@ -12,6 +13,16 @@ This repository contains two primary components: a core framework (`eepp`) and a * **Relationship:** `ecode` is built *using* the `eepp` GUI framework. It acts as the primary real-world consumer of `eepp`. * **Goal:** Development on `ecode` is often used to test, improve, and drive new features in the underlying `eepp` library. +## 3. eterm (Terminal Emulator) + +`eterm` is the repository's terminal emulator. Its name is always written in lowercase: `eterm`, +not `eTerm`. Its implementation lives under `src/modules/eterm/`. + +## 4. eeiv (Image Viewer) + +`eeiv` is the repository's image viewer. Its application entry point is +`src/tools/eeiv/eeiv.cpp`. + ## Documentation & Code References When working on this project, rely on the following resources to understand existing implementations: diff --git a/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp b/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp index 1d32fdc82..c3d5b8612 100644 --- a/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp +++ b/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp @@ -663,11 +663,11 @@ KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer std::shared_ptr> pixelStorage; if ( existing != mImages.end() && existing->second.pixels.use_count() == 1 ) { pixelStorage = existing->second.pixels; - pixelStorage->assign( pixels.begin(), pixels.end() ); - if ( format == 24 ) - recycleBuffer( pixels, mDecodedScratch ); - else if ( format == 32 ) - recycleBuffer( pixels, mDecodedScratch ); + // The existing image is no longer referenced by a queued UI update. Exchange its backing + // allocation with the freshly decoded frame instead of copying the complete image. The old + // image allocation becomes the next transfer's decode buffer. + pixelStorage->swap( pixels ); + recycleBuffer( pixels, mDecodedScratch ); } else { pixelStorage = std::make_shared>( std::move( pixels ) ); } diff --git a/src/tests/unit_tests/eterm_tests.cpp b/src/tests/unit_tests/eterm_tests.cpp index d92f82456..f9924522b 100644 --- a/src/tests/unit_tests/eterm_tests.cpp +++ b/src/tests/unit_tests/eterm_tests.cpp @@ -1064,12 +1064,24 @@ UTEST( eterm, kitty_graphics_reuses_unreferenced_replacement_pixel_storage ) { updates.clear(); const auto* storage = protocol.imagePixels( 91 ); ASSERT_TRUE( storage != nullptr ); + const auto* firstAllocation = storage->data(); ASSERT_EQ( KittyGraphicsError::None, protocol.handle( "a=t,f=24,s=2,v=1,i=91,q=2;BwgJCgsM" ).error ); EXPECT_TRUE( storage == protocol.imagePixels( 91 ) ); const std::vector expected{ 7, 8, 9, 10, 11, 12 }; EXPECT_TRUE( expected == *protocol.imagePixels( 91 ) ); + ASSERT_TRUE( firstAllocation != protocol.imagePixels( 91 )->data() ); + updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + updates.clear(); + + ASSERT_EQ( KittyGraphicsError::None, + protocol.handle( "a=t,f=24,s=2,v=1,i=91,q=2;DQ4PEBES" ).error ); + EXPECT_TRUE( storage == protocol.imagePixels( 91 ) ); + const std::vector recycledExpected{ 13, 14, 15, 16, 17, 18 }; + EXPECT_TRUE( recycledExpected == *protocol.imagePixels( 91 ) ); + EXPECT_TRUE( firstAllocation == protocol.imagePixels( 91 )->data() ); } UTEST( eterm, kitty_graphics_rejects_shared_memory_transmission ) { From adc562fb1dbef9a2dae143f1e35507767186a798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Fri, 4 Sep 2026 19:47:46 -0300 Subject: [PATCH 8/8] Fix regression due to kitty metadata being erased on every single pass. --- .../src/eterm/terminal/terminalemulator.cpp | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp b/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp index 0e80b02de..c9c249b3a 100644 --- a/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp +++ b/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp @@ -80,6 +80,8 @@ namespace eterm { namespace Terminal { namespace { +constexpr Rune KittyGraphicsPlaceholder = 0x10EEEE; + bool terminalDiagnosticsEnabled() { #ifdef EE_DEBUG return true; @@ -957,9 +959,11 @@ void TerminalEmulator::trimMemory() { void TerminalEmulator::clearHistory() { for ( int i = 0; i < mTerm.histcursize; ++i ) { - if ( mTerm.hist[i] ) { - for ( int column = 0; column < mTerm.col; ++column ) - mKittyPlaceholderMetadata.erase( &mTerm.hist[i][column] ); + if ( mTerm.hist[i] && !mKittyPlaceholderMetadata.empty() ) { + for ( int column = 0; column < mTerm.col; ++column ) { + if ( mTerm.hist[i][column].u == KittyGraphicsPlaceholder ) + mKittyPlaceholderMetadata.erase( &mTerm.hist[i][column] ); + } } eeSAFE_FREE( mTerm.hist[i] ); } @@ -1664,7 +1668,8 @@ void TerminalEmulator::tsetchar( Rune u, TerminalGlyph* attr, int x, int y ) { TLINE( y )[x - 1].mode &= ~ATTR_WIDE; } - mKittyPlaceholderMetadata.erase( &TLINE( y )[x] ); + if ( TLINE( y )[x].u == KittyGraphicsPlaceholder ) + mKittyPlaceholderMetadata.erase( &TLINE( y )[x] ); mTerm.dirty[y] = 1; TLINE( y )[x] = *attr; TLINE( y )[x].u = u; @@ -1686,9 +1691,13 @@ void TerminalEmulator::tclearregion( int x1, int y1, int x2, int y2, bool skip_c LIMIT( x2, 0, mTerm.col - 1 ); LIMIT( y1, 0, mTerm.row - 1 ); LIMIT( y2, 0, mTerm.row - 1 ); - for ( int clearY = y1; clearY <= y2; ++clearY ) { - for ( int clearX = x1; clearX <= x2; ++clearX ) - mKittyPlaceholderMetadata.erase( &TLINE( clearY )[clearX] ); + if ( !mKittyPlaceholderMetadata.empty() ) { + for ( int clearY = y1; clearY <= y2; ++clearY ) { + for ( int clearX = x1; clearX <= x2; ++clearX ) { + if ( TLINE( clearY )[clearX].u == KittyGraphicsPlaceholder ) + mKittyPlaceholderMetadata.erase( &TLINE( clearY )[clearX] ); + } + } } /* @@ -3250,7 +3259,7 @@ check_control_code: } tsetchar( u, &mTerm.c.attr, mTerm.c.x, mTerm.c.y ); - if ( u == 0x10EEEE ) { + if ( u == KittyGraphicsPlaceholder ) { mKittyPlaceholderCell = Vector2i( mTerm.c.x, mTerm.c.y ); const Uint32 placementId = IS_TRUECOL( mKittyUnderlineColor ) ? mKittyUnderlineColor & 0xFFFFFF @@ -3356,7 +3365,7 @@ void TerminalEmulator::tresize( int col, int row ) { if ( !line ) return; for ( int column = 0; column < columns; ++column ) { - if ( line[column].u != 0x10EEEE ) + if ( line[column].u != KittyGraphicsPlaceholder ) continue; auto metadata = mKittyPlaceholderMetadata.find( &line[column] ); if ( metadata != mKittyPlaceholderMetadata.end() ) @@ -3566,7 +3575,7 @@ void TerminalEmulator::tresize( int col, int row ) { if ( !line ) return; for ( int column = 0; column < columns && index < metadata.size(); ++column ) { - if ( line[column].u == 0x10EEEE ) + if ( line[column].u == KittyGraphicsPlaceholder ) mKittyPlaceholderMetadata[&line[column]] = metadata[index++]; } }; @@ -3649,7 +3658,7 @@ void TerminalEmulator::draw() { Uint8 previousMsb = 0; for ( int x = 0; x < mTerm.col; ++x ) { const auto& glyph = TLINE( y )[x]; - if ( glyph.u != 0x10EEEE ) { + if ( glyph.u != KittyGraphicsPlaceholder ) { previous = nullptr; continue; }