diff --git a/docs/articles/eterm-kitty-graphics.md b/docs/articles/eterm-kitty-graphics.md new file mode 100644 index 000000000..a70aee53e --- /dev/null +++ b/docs/articles/eterm-kitty-graphics.md @@ -0,0 +1,43 @@ +# eTerm Kitty graphics protocol + +eTerm implements the terminal-receiver side of the Kitty graphics protocol. Protocol parsing, +decoding, image storage, placement state, animation, and terminal lifecycle integration run on the +terminal worker. The UI thread consumes immutable placement metadata and an ordered pixel-update +stream, and exclusively owns the OpenGL texture cache. + +## Supported features + +- direct RGB and RGBA transmission, including chunking and zlib compression; +- PNG transmission; +- image IDs, image numbers, placement IDs, source cropping, cell geometry, and pixel offsets; +- root-image rectangle updates, animation frames, frame composition, and animation control; +- deletion selectors, quiet modes, capability queries, storage quotas, usage hints, and eviction; +- primary/alternate-screen state, reset, clear, scrolling, margins, scrollback, and resize; +- Unicode placeholder placements (U+10EEEE and Kitty's combining-diacritic encoding); +- relative placements, including parent validation, cycle detection, and descendant movement; +- terminal pixel-size queries and SGR pixel mouse coordinates. + +Local file, temporary-file, and shared-memory transfer media are intentionally not enabled. Direct +transmission is the portable path and works through SSH without giving terminal applications access +to the terminal host's filesystem or shared-memory namespace. + +## Resource and threading behavior + +Decoded root images are stored as premultiplied-independent RGBA8 CPU buffers on the worker. GPU +textures exist only on the UI/GL thread. Incremental updates carry monotonically increasing sequence +numbers; a bounded queue gap triggers a complete graphics resynchronization. Image count, placement +count, APC size, decoded transfer size, and total image storage are bounded. Unreferenced images are +evicted oldest-first when storage is needed. + +Graphics follow the same presentation cadence and DEC synchronized-update boundary as text. A +placement anchored in normal-screen history moves with terminal scrolling and is projected into the +current scrollback viewport. Margin scrolling clips or removes only placements wholly participating +in the scrolled region. Resize preserves screen-coordinate placements and reflows Unicode placeholder +metadata with its text cells. + +## Compatibility testing + +The eTerm unit suite covers fragmented APC input, malformed/fuzzed commands, raw/zlib/PNG transfer, +independent image namespaces, replacement, partial updates, placement geometry, scrolling and +scrollback, screen lifecycle, animation, Unicode placeholders, relative placement graphs, ordered +worker/UI delivery, queue overflow, and resynchronization. diff --git a/include/eepp/system/base64.hpp b/include/eepp/system/base64.hpp index 640dcf9ef..1ebc7dd4c 100644 --- a/include/eepp/system/base64.hpp +++ b/include/eepp/system/base64.hpp @@ -11,22 +11,38 @@ namespace EE { namespace System { class EE_API Base64 { public: + enum class DecodeMode { + AllowWhitespace, + NoWhitespace, + }; + /** Encode binary data into base64 digits with MIME style === pads ** @return The final length of the output */ static size_t encode( size_t in_len, const unsigned char* in, size_t out_len, char* out ); - /** Decode base64 digits with MIME style === pads into binary data + /** Decode base64 digits with MIME style === pads into binary data. + ** Preserves the historical behavior of ignoring ASCII whitespace. ** @return The final length of the output */ static size_t decode( size_t in_len, const char* in, size_t out_len, unsigned char* out ); + /** Decode base64 digits with MIME style === pads into binary data. + ** DecodeMode::NoWhitespace avoids all whitespace handling and enables the fastest path. + ** @return The final length of the output */ + static size_t decode( size_t in_len, const char* in, size_t out_len, unsigned char* out, + DecodeMode mode ); + /** Encodes a string into a base64 string ** @return True if encoding was successful */ static bool encode( std::string_view in, std::string& out ); - /** Decodes a base64 string to a string - ** @return True if encoding was successful */ + /** Decodes a base64 string to a string, ignoring ASCII whitespace. + ** @return The final length of the output, or size_t(-1) on truncation */ static size_t decode( std::string_view in, std::string& out ); + /** Decodes a base64 string to a string. + ** @return The final length of the output, or size_t(-1) on truncation */ + static size_t decode( std::string_view in, std::string& out, DecodeMode mode ); + /** @return A safe encoding output length for an input of the length indicated */ static inline size_t encodeSafeOutLen( size_t in_len ) { return ( ( in_len + 2 ) / 3 ) * 4 + 1; diff --git a/src/eepp/system/base64.cpp b/src/eepp/system/base64.cpp index 9bee1e5d5..91ecf9d5b 100644 --- a/src/eepp/system/base64.cpp +++ b/src/eepp/system/base64.cpp @@ -1,63 +1,123 @@ #include +#include namespace EE { namespace System { -/* base64.c : base-64 / MIME encode/decode */ -/* PUBLIC DOMAIN - Jon Mayo - November 13, 2003 */ -/* $Id: base64.c 156 2007-07-12 23:29:10Z orange $ */ +namespace { -/* decode a base64 string in one shot */ -size_t Base64::decode( size_t in_len, const char* in, size_t out_len, unsigned char* out ) { - static const Uint8 base64dec_tab[256] = { - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63, 52, 53, 54, 55, 56, 57, - 58, 59, 60, 61, 255, 255, 255, 0, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, - 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, - 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, - }; +constexpr Uint8 BASE64_INVALID = 0xFF; +constexpr Uint8 BASE64_PADDING = 0xFE; +constexpr Uint8 BASE64_WHITESPACE = 0xFD; - size_t ii, io; - Uint32 v; - unsigned rem; +constexpr std::array makeBase64DecodeTable() { + std::array table{}; + + for ( size_t i = 0; i < table.size(); ++i ) + table[i] = BASE64_INVALID; + + for ( size_t i = 0; i < 26; ++i ) { + table[static_cast( 'A' + i )] = static_cast( i ); + table[static_cast( 'a' + i )] = static_cast( i + 26 ); + } + + for ( size_t i = 0; i < 10; ++i ) + table[static_cast( '0' + i )] = static_cast( i + 52 ); + + table[static_cast( '+' )] = 62; + table[static_cast( '/' )] = 63; + table[static_cast( '=' )] = BASE64_PADDING; + + // ASCII whitespace accepted by the historical decoder via isspace(): + table[static_cast( ' ' )] = BASE64_WHITESPACE; + table[static_cast( '\t' )] = BASE64_WHITESPACE; + table[static_cast( '\n' )] = BASE64_WHITESPACE; + table[static_cast( '\v' )] = BASE64_WHITESPACE; + table[static_cast( '\f' )] = BASE64_WHITESPACE; + table[static_cast( '\r' )] = BASE64_WHITESPACE; + + return table; +} + +constexpr auto base64dec_tab = makeBase64DecodeTable(); + +template +size_t decodeBase64( size_t in_len, const char* in, size_t out_len, unsigned char* out ) { + size_t ii = 0; + size_t io = 0; + Uint32 v = 0; + unsigned rem = 0; + + while ( ii < in_len ) { + /* + * Fast path: four ordinary base64 bytes become three output bytes. + * + * The OR validates all four table entries at once: valid base64 values + * are 0..63, while padding/whitespace/invalid entries have high bits set. + * + * This path is especially useful for protocol payloads that guarantee + * contiguous, whitespace-free base64 data. + */ + if ( rem == 0 && ii + 4 <= in_len && io + 3 <= out_len ) { + const Uint8 a = base64dec_tab[static_cast( in[ii] )]; + const Uint8 b = base64dec_tab[static_cast( in[ii + 1] )]; + const Uint8 c = base64dec_tab[static_cast( in[ii + 2] )]; + const Uint8 d = base64dec_tab[static_cast( in[ii + 3] )]; + + if ( ( a | b | c | d ) <= 63 ) { + out[io] = static_cast( ( a << 2 ) | ( b >> 4 ) ); + out[io + 1] = static_cast( ( b << 4 ) | ( c >> 2 ) ); + out[io + 2] = static_cast( ( c << 6 ) | d ); + + ii += 4; + io += 3; + continue; + } + } + + const Uint8 ch = base64dec_tab[static_cast( in[ii++] )]; + + if ( ch > 63 ) { + if constexpr ( AllowWhitespace ) { + if ( ch == BASE64_WHITESPACE ) + continue; + } + + // Preserve the old behavior: stop at '=' or the first parse error. + break; + } - for ( io = 0, ii = 0, v = 0, rem = 0; ii < in_len; ii++ ) { - unsigned char ch; - unsigned char c = (unsigned char)in[ii]; - if ( isspace( c ) ) - continue; - if ( c == '=' ) - break; /* stop at = */ - ch = base64dec_tab[c]; - if ( ch == 255 ) - break; /* stop at a parse error */ v = ( v << 6 ) | ch; rem += 6; + if ( rem >= 8 ) { rem -= 8; + if ( io >= out_len ) - return -1; /* truncation is failure */ - out[io++] = ( v >> rem ) & 255; + return static_cast( -1 ); + + out[io++] = static_cast( ( v >> rem ) & 255 ); + + // Every four valid base64 characters rem returns to zero. + if ( rem == 0 ) + v = 0; } } - if ( rem >= 8 ) { - rem -= 8; - if ( io >= out_len ) - return -1; /* truncation is failure */ - out[io++] = ( v >> rem ) & 255; - } + return io; } +} // namespace + +size_t Base64::decode( size_t in_len, const char* in, size_t out_len, unsigned char* out ) { + return decodeBase64( in_len, in, out_len, out ); +} + +size_t Base64::decode( size_t in_len, const char* in, size_t out_len, unsigned char* out, + DecodeMode mode ) { + return mode == DecodeMode::NoWhitespace ? decodeBase64( in_len, in, out_len, out ) + : decodeBase64( in_len, in, out_len, out ); +} + size_t Base64::encode( size_t in_len, const unsigned char* in, size_t out_len, char* out ) { static const Uint8 base64enc_tab[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -74,23 +134,27 @@ size_t Base64::encode( size_t in_len, const unsigned char* in, size_t out_len, c while ( rem >= 6 ) { rem -= 6; if ( io >= out_len ) - return -1; /* truncation is failure */ + return static_cast( -1 ); /* truncation is failure */ out[io++] = base64enc_tab[( v >> rem ) & 63]; } } + if ( rem ) { v <<= ( 6 - rem ); if ( io >= out_len ) - return -1; /* truncation is failure */ + return static_cast( -1 ); /* truncation is failure */ out[io++] = base64enc_tab[v & 63]; } + while ( io & 3 ) { if ( io >= out_len ) - return -1; /* truncation is failure */ + return static_cast( -1 ); /* truncation is failure */ out[io++] = '='; } + if ( io >= out_len ) - return -1; /* no room for null terminator */ + return static_cast( -1 ); /* no room for null terminator */ + out[io] = 0; return io; } @@ -98,31 +162,33 @@ size_t Base64::encode( size_t in_len, const unsigned char* in, size_t out_len, c bool Base64::encode( std::string_view in, std::string& out ) { size_t b64len = encodeSafeOutLen( in.size() ); - if ( out.size() < b64len ) { + if ( out.size() < b64len ) out.resize( b64len ); - } - int len = encode( in.size(), (const unsigned char*)in.data(), out.size(), (char*)&out[0] ); + const size_t len = + encode( in.size(), reinterpret_cast( in.data() ), out.size(), out.data() ); - if ( -1 != len && (size_t)len != out.size() ) { + if ( len != static_cast( -1 ) && len != out.size() ) out.resize( len ); - } - return -1 != len; + return len != static_cast( -1 ); } size_t Base64::decode( std::string_view in, std::string& out ) { + return decode( in, out, DecodeMode::AllowWhitespace ); +} + +size_t Base64::decode( std::string_view in, std::string& out, DecodeMode mode ) { size_t d64len = decodeSafeOutLen( in.size() ); - if ( out.size() < d64len ) { + if ( out.size() < d64len ) out.resize( d64len ); - } - int len = decode( in.size(), in.data(), out.size(), (unsigned char*)&out[0] ); + const size_t len = decode( in.size(), in.data(), out.size(), + reinterpret_cast( out.data() ), mode ); - if ( -1 != len && (size_t)len != out.size() ) { + if ( len != static_cast( -1 ) && len != out.size() ) out.resize( len ); - } return len; } diff --git a/src/modules/eterm/include/eterm/terminal/ipseudoterminal.hpp b/src/modules/eterm/include/eterm/terminal/ipseudoterminal.hpp index a6156a358..b6e981116 100644 --- a/src/modules/eterm/include/eterm/terminal/ipseudoterminal.hpp +++ b/src/modules/eterm/include/eterm/terminal/ipseudoterminal.hpp @@ -44,7 +44,9 @@ class IPseudoTerminal : public eterm::System::IPipe { virtual int getNumRows() const = 0; - virtual bool resize( int columns, int rows ) = 0; + virtual bool resize( int columns, int rows, int pixelWidth, int pixelHeight ) = 0; + + bool resize( int columns, int rows ) { return resize( columns, rows, 0, 0 ); } }; }} // namespace eterm::Terminal diff --git a/src/modules/eterm/include/eterm/terminal/iterminaldisplay.hpp b/src/modules/eterm/include/eterm/terminal/iterminaldisplay.hpp index 295be3a37..2d8dce80a 100644 --- a/src/modules/eterm/include/eterm/terminal/iterminaldisplay.hpp +++ b/src/modules/eterm/include/eterm/terminal/iterminaldisplay.hpp @@ -22,6 +22,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #include +#include #include using namespace EE; @@ -80,6 +81,9 @@ class ITerminalDisplay { virtual void drawCursor( int cx, int cy, TerminalGlyph g, int ox, int oy, TerminalGlyph og ) = 0; + virtual void drawGraphics( std::shared_ptr presentation, + std::vector updates ); + virtual void drawEnd() = 0; protected: diff --git a/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp b/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp new file mode 100644 index 000000000..bd18bf7f6 --- /dev/null +++ b/src/modules/eterm/include/eterm/terminal/kittygraphicsprotocol.hpp @@ -0,0 +1,243 @@ +#ifndef ETERM_KITTYGRAPHICSPROTOCOL_HPP +#define ETERM_KITTYGRAPHICSPROTOCOL_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace EE; + +namespace eterm { namespace Terminal { + +enum class KittyGraphicsError : Uint8 { + None, + InvalidArgument, + Unsupported, + InvalidData, + TooLarge, + DecodeFailed, + NotFound, + NoSpace, + NoParent, + Cycle, + TooDeep +}; + +struct KittyGraphicsCommandData { + std::string_view payload; + std::optional format; + std::optional dataSize; + std::optional more; + std::optional imageId; + std::optional imageNumber; + std::optional usageHint; + std::optional placementId; + std::optional quiet; + std::optional width; + std::optional height; + std::optional x; + std::optional y; + std::optional sourceWidth; + std::optional sourceHeight; + std::optional columns; + std::optional rows; + std::optional xOffset; + std::optional yOffset; + std::optional zIndex; + std::optional cursorMovement; + std::optional virtualPlacement; + std::optional parentImageId; + std::optional parentPlacementId; + std::optional parentOffsetX; + std::optional parentOffsetY; + char transmission{ 'd' }; + char compression{ 0 }; + char deletion{ 'a' }; +}; + +struct KittyTransmitCommand { + KittyGraphicsCommandData data; + bool display{ false }; +}; + +struct KittyPutCommand { + KittyGraphicsCommandData data; +}; + +struct KittyDeleteCommand { + KittyGraphicsCommandData data; +}; + +struct KittyFrameCommand { + KittyGraphicsCommandData data; +}; + +struct KittyAnimationCommand { + KittyGraphicsCommandData data; +}; + +struct KittyComposeCommand { + KittyGraphicsCommandData data; +}; + +struct KittyQueryCommand { + KittyGraphicsCommandData data; +}; + +using KittyGraphicsCommand = + std::variant; + +struct KittyGraphicsParseResult { + std::optional command; + KittyGraphicsError error{ KittyGraphicsError::None }; +}; + +struct KittyGraphicsHandleResult { + KittyGraphicsHandleResult() = default; + KittyGraphicsHandleResult( std::string response, KittyGraphicsError error, bool changed ) : + response( std::move( response ) ), error( error ), changed( changed ) {} + + std::string response; + Vector2i cursorMovement; + KittyGraphicsError error{ KittyGraphicsError::None }; + bool changed{ false }; +}; + +struct KittyGraphicsStats { + Uint64 decodedBytes{ 0 }; + Uint64 fullImageUpdates{ 0 }; + Uint64 rectangleUpdates{ 0 }; + Uint64 evictions{ 0 }; +}; + +class KittyGraphicsProtocol { + public: + explicit KittyGraphicsProtocol( size_t maxStorageBytes = 320 * 1024 * 1024, + size_t maxImages = 4096, size_t maxPlacements = 65536 ); + + static KittyGraphicsParseResult parse( std::string_view command ); + + KittyGraphicsHandleResult handle( std::string_view command, Vector2i cursor = {} ); + + std::vector takeUpdates(); + + std::shared_ptr takePresentation(); + + bool hasPendingPresentation() const { return mPresentationDirty; } + + const std::vector* imagePixels( KittyImageId imageId ) const; + + size_t imageCount() const { return mImages.size(); } + + const KittyGraphicsStats& stats() const { return mStats; } + + bool hasVirtualPlacements() const; + + void reset(); + + void resync(); + + void clearScreen(); + + void setAlternateScreen( bool alternate ); + + void scrollScreen( int top, int bottom, int rows, bool preserveHistory ); + + void setViewport( int scrollPosition, int historyLength, int screenRows ); + + bool updateAnimations(); + + void setCellPixelSize( Uint32 width, Uint32 height ); + + void setPlaceholderCells( std::vector cells ); + + private: + struct Image { + struct Frame { + std::vector rgba; + Int32 gapMs{ 40 }; + Uint32 usageHint{ 0 }; + }; + std::vector rgba; + std::unordered_map frames; + Sizei size; + Uint32 imageNumber{ 0 }; + Uint32 usageHint{ 0 }; + bool anonymous{ false }; + Uint64 creationSerial{ 0 }; + Uint32 currentFrame{ 1 }; + Uint32 loopCount{ 1 }; + Uint32 loopsCompleted{ 0 }; + Uint8 animationState{ 1 }; + Int32 rootGapMs{ 0 }; + EE::System::Clock frameClock; + }; + + struct PendingTransfer { + KittyGraphicsCommandData data; + std::vector decodedData; + bool display{ false }; + bool query{ false }; + bool frame{ false }; + bool active{ false }; + }; + + struct Placement { + TerminalVisiblePlacement visible; + Uint64 internalId{ 0 }; + bool virtualPlacement{ false }; + KittyImageId parentImageId{ 0 }; + KittyPlacementId parentPlacementId{ 0 }; + }; + + KittyGraphicsHandleResult handleTransmit( const KittyGraphicsCommandData& data, bool display, + bool query, bool frame, Vector2i cursor ); + KittyGraphicsHandleResult finishTransfer( PendingTransfer transfer, Vector2i cursor ); + KittyGraphicsHandleResult put( const KittyGraphicsCommandData& data, Vector2i cursor ); + KittyGraphicsHandleResult remove( const KittyGraphicsCommandData& data, Vector2i cursor ); + KittyGraphicsHandleResult controlAnimation( const KittyGraphicsCommandData& data ); + KittyGraphicsHandleResult composeFrames( const KittyGraphicsCommandData& data ); + KittyImageId allocateImageId(); + KittyImageId resolveImageId( const KittyGraphicsCommandData& data ) const; + bool ensureCapacity( size_t bytes, KittyImageId replacingId, bool addingImage ); + bool isImagePlaced( KittyImageId imageId ) const; + void eraseImage( KittyImageId imageId ); + std::string response( const KittyGraphicsCommandData& data, KittyGraphicsError error, + KittyImageId imageId = 0 ) const; + + std::unordered_map mImages; + std::vector mPlacements; + std::vector mPrimaryPlacements; + std::vector mPlaceholderCells; + std::vector mUpdates; + PendingTransfer mPending; + size_t mStorageBytes{ 0 }; + size_t mFrameStorageBytes{ 0 }; + Uint64 mCreationSerial{ 0 }; + Uint64 mPresentationGeneration{ 0 }; + Uint64 mPlacementSerial{ 0 }; + KittyImageId mNextImageId{ 1 }; + bool mPresentationDirty{ false }; + int mScrollPosition{ 0 }; + int mHistoryLength{ 0 }; + int mScreenRows{ 0 }; + Uint32 mCellPixelWidth{ 0 }; + Uint32 mCellPixelHeight{ 0 }; + size_t mMaxStorageBytes{ 0 }; + size_t mMaxImages{ 0 }; + size_t mMaxPlacements{ 0 }; + KittyGraphicsStats mStats; +}; + +}} // namespace eterm::Terminal + +#endif diff --git a/src/modules/eterm/include/eterm/terminal/kittygraphicsrenderer.hpp b/src/modules/eterm/include/eterm/terminal/kittygraphicsrenderer.hpp new file mode 100644 index 000000000..e0e9b8173 --- /dev/null +++ b/src/modules/eterm/include/eterm/terminal/kittygraphicsrenderer.hpp @@ -0,0 +1,42 @@ +#ifndef ETERM_KITTYGRAPHICSRENDERER_HPP +#define ETERM_KITTYGRAPHICSRENDERER_HPP + +#include +#include +#include + +using namespace EE::Graphics; + +namespace eterm { namespace Terminal { + +class KittyGraphicsRenderer { + public: + enum class Pass : Uint8 { VeryNegative, Negative, NonNegative }; + + bool applyUpdates( std::vector&& updates ); + + void setPresentation( std::shared_ptr presentation ); + + void draw( Pass pass, const Vector2f& origin, const Sizef& cellSize, const Sizef& gridSize ); + + bool hasPlacements( Pass pass ) const; + + void reset(); + + Uint64 lastAppliedSequence() const { return mLastAppliedSequence; } + + private: + struct GPUImage { + TexturePtr texture; + std::unordered_map frames; + Sizei size; + }; + + std::unordered_map mImages; + std::shared_ptr mPresentation; + Uint64 mLastAppliedSequence{ 0 }; +}; + +}} // namespace eterm::Terminal + +#endif diff --git a/src/modules/eterm/include/eterm/terminal/pseudoterminal.hpp b/src/modules/eterm/include/eterm/terminal/pseudoterminal.hpp index ad001c4c4..acafcbd76 100644 --- a/src/modules/eterm/include/eterm/terminal/pseudoterminal.hpp +++ b/src/modules/eterm/include/eterm/terminal/pseudoterminal.hpp @@ -55,7 +55,7 @@ class PseudoTerminal final : public IPseudoTerminal { virtual int getNumColumns() const override; virtual int getNumRows() const override; - virtual bool resize( int columns, int rows ) override; + virtual bool resize( int columns, int rows, int pixelWidth, int pixelHeight ) override; virtual int write( const char* s, size_t n ) override; virtual int read( char* buf, size_t n, bool block = false ) override; diff --git a/src/modules/eterm/include/eterm/terminal/terminaldisplay.hpp b/src/modules/eterm/include/eterm/terminal/terminaldisplay.hpp index bf8505f11..12116a1b7 100644 --- a/src/modules/eterm/include/eterm/terminal/terminaldisplay.hpp +++ b/src/modules/eterm/include/eterm/terminal/terminaldisplay.hpp @@ -30,6 +30,8 @@ class VertexBuffer; namespace eterm { namespace Terminal { +class KittyGraphicsRenderer; + enum class TerminalShortcutAction { PASTE, PASTE_SELECTION, @@ -169,6 +171,10 @@ class TerminalDisplay { bool update( bool isMouseOverMe = true ); + Sizei getCellPixelSize() const; + + Sizei getGridPixelSize() const; + void executeFile( const std::string& cmd ); void executeBinary( const std::string& binaryPath, const std::string& args = "" ); @@ -315,6 +321,7 @@ class TerminalDisplay { std::vector mColors; std::shared_ptr mSession; std::shared_ptr mSnapshot; + std::unique_ptr mGraphicsRenderer; mutable std::string mClipboardUtf8; Uint32 mNumCallBacks{ 0 }; std::map mCallbacks; @@ -349,6 +356,8 @@ class TerminalDisplay { Uint32 mRows{ 0 }; Uint32 mClickStep{ 5 }; Uint64 mSnapshotGeneration{ 0 }; + Uint64 mLastAppliedGraphicsSequence{ 0 }; + bool mGraphicsResyncPending{ false }; FontHinting mFontHinting{ FontHinting::Full }; FontAntialiasing mFontAntialiasing{ FontAntialiasing::Grayscale }; FrameBufferUniquePtr mFrameBuffer; @@ -377,12 +386,16 @@ class TerminalDisplay { Vector2i positionToGrid( const Vector2i& pos ); + Vector2i positionToPixel( const Vector2i& pos ) const; + void onSizeChange(); void onProcessExit( int exitCode ); void consumeSnapshot(); + void drainGraphicsUpdates(); + void drainSessionEvents(); TerminalColorPalette makeColorPalette() const; diff --git a/src/modules/eterm/include/eterm/terminal/terminalemulator.hpp b/src/modules/eterm/include/eterm/terminal/terminalemulator.hpp index 1ac34934d..24de7e7c6 100644 --- a/src/modules/eterm/include/eterm/terminal/terminalemulator.hpp +++ b/src/modules/eterm/include/eterm/terminal/terminalemulator.hpp @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -58,6 +59,10 @@ constexpr int ESC_BUF_SIZ = 512; constexpr int ESC_ARG_SIZ = 32; constexpr int STR_BUF_SIZ = ESC_BUF_SIZ; constexpr int STR_ARG_SIZ = ESC_ARG_SIZ; +// Kitty recommends small chunks, but unchunked direct transmissions are valid and common in +// simple clients. Keep this independently bounded while allowing useful image-sized APCs. +constexpr size_t MAX_KITTY_GRAPHICS_APC_SIZE = 16 * 1024 * 1024; +constexpr size_t MAX_GENERIC_STRING_SEQUENCE_SIZE = 1024 * 1024; /* Internal representation of the screen */ struct Term { @@ -119,7 +124,8 @@ struct STREscape { size_t siz; /* allocation size */ size_t len; /* raw string length */ char* args[STR_ARG_SIZ]; - int narg; /* nb of args */ + int narg; /* nb of args */ + bool discarded; /* oversized sequence: consume input through its terminator without storing */ }; enum class PromptState { @@ -156,6 +162,8 @@ class TerminalEmulator final { void resize( int columns, int rows ); + void resize( int columns, int rows, int pixelWidth, int pixelHeight ); + void redraw(); /** Worker-owned terminal state reset (RIS semantics without replacing the PTY/process). */ @@ -220,8 +228,8 @@ class TerminalEmulator final { std::string getSelection() const; - void mousereport( const TerminalMouseEventType& type, const Vector2i& pos, const Uint32& flags, - const Uint32& mod ); + void mousereport( const TerminalMouseEventType& type, const Vector2i& cellPosition, + const Vector2i& pixelPosition, const Uint32& flags, const Uint32& mod ); const bool& isDirty() const { return mDirty; } @@ -260,6 +268,8 @@ class TerminalEmulator final { /** Worker-only notification that the display palette changed. */ void notifyColorSchemeChanged(); + void requestGraphicsResync(); + Vector2i getSize() const; System::IProcess* getProcess() const; @@ -288,6 +298,10 @@ class TerminalEmulator final { bool mPendingPtyResize{ false }; int mPendingPtyColumns{ 0 }; int mPendingPtyRows{ 0 }; + int mPendingPtyPixelWidth{ 0 }; + int mPendingPtyPixelHeight{ 0 }; + int mPixelWidth{ 0 }; + int mPixelHeight{ 0 }; Clock mPendingPtyResizeClock; bool mDirty{ true }; @@ -307,6 +321,7 @@ class TerminalEmulator final { TerminalSelection mSel; CSIEscape mCsiescseq; STREscape mStrescseq; + KittyGraphicsProtocol mKittyGraphics; uint32_t mDefaultFg; uint32_t mDefaultBg; @@ -324,6 +339,16 @@ class TerminalEmulator final { PromptState mPromptState{ PromptState::Unknown }; PromptStateChangedCb mPromptStateChangedCb; DataCb mDataCb; + Vector2i mKittyPlaceholderCell{ -1, -1 }; + struct KittyPlaceholderMetadata { + Uint32 placementId{ 0 }; + Uint16 row{ UINT16_MAX }; + Uint16 column{ UINT16_MAX }; + Uint8 imageIdMsb{ 0 }; + Uint8 diacriticCount{ 0 }; + }; + std::unordered_map mKittyPlaceholderMetadata; + Uint32 mKittyUnderlineColor{ 0 }; void setClipboard( const char* str ); diff --git a/src/modules/eterm/include/eterm/terminal/terminalgraphics.hpp b/src/modules/eterm/include/eterm/terminal/terminalgraphics.hpp new file mode 100644 index 000000000..83b4ab6f0 --- /dev/null +++ b/src/modules/eterm/include/eterm/terminal/terminalgraphics.hpp @@ -0,0 +1,109 @@ +#ifndef ETERM_TERMINALGRAPHICS_HPP +#define ETERM_TERMINALGRAPHICS_HPP + +#include +#include +#include +#include +#include +#include +#include + +using namespace EE; +using namespace EE::Math; + +namespace eterm { namespace Terminal { + +using KittyImageId = Uint32; +using KittyPlacementId = Uint32; + +struct TerminalVisiblePlacement { + KittyImageId imageId{ 0 }; + KittyPlacementId placementId{ 0 }; + Uint32 frameNumber{ 1 }; + Vector2i visibleAnchorCell; + Rect sourcePixels; + Uint32 columns{ 0 }; + Uint32 rows{ 0 }; + Vector2i firstCellPixelOffset; + Int32 zIndex{ 0 }; +}; + +struct TerminalGraphicsPlaceholderCell { + KittyImageId imageId{ 0 }; + KittyPlacementId placementId{ 0 }; + Vector2i cell; + Uint32 imageRow{ 0 }; + Uint32 imageColumn{ 0 }; + + bool operator==( const TerminalGraphicsPlaceholderCell& other ) const { + return imageId == other.imageId && placementId == other.placementId && cell == other.cell && + imageRow == other.imageRow && imageColumn == other.imageColumn; + } +}; + +/** Small immutable graphics state associated with a terminal presentation. */ +struct TerminalGraphicsPresentation { + std::vector placements; + Uint64 generation{ 0 }; + Uint64 requiredUpdateSequence{ 0 }; +}; + +enum class TerminalGraphicsUpdateType : Uint8 { + CreateImage, + ReplaceImage, + UpdateRegion, + CreateFrame, + ReplaceFrame, + UpdateFrameRegion, + DeleteFrame, + DeleteImage, + ResetScreen, + ResetAll, + Resync +}; + +struct TerminalGraphicsUpdate { + std::shared_ptr> rgba; + Sizei imageSize; + Rect region; + Uint64 sequence{ 0 }; + KittyImageId imageId{ 0 }; + Uint32 frameNumber{ 1 }; + TerminalGraphicsUpdateType type{ TerminalGraphicsUpdateType::Resync }; + + size_t payloadBytes() const { return rgba ? rgba->size() : 0; } +}; + +/** Bounded ordered worker-to-UI mutation queue with explicit overflow recovery. */ +class TerminalGraphicsUpdateQueue { + public: + static constexpr size_t DefaultMaxUpdates = 1024; + static constexpr size_t DefaultMaxBytes = 32 * 1024 * 1024; + + explicit TerminalGraphicsUpdateQueue( size_t maxUpdates = DefaultMaxUpdates, + size_t maxBytes = DefaultMaxBytes ); + + Uint64 enqueue( TerminalGraphicsUpdate update ); + + std::vector drain(); + + size_t queuedBytes() const; + + bool needsResync() const; + + void resetResync(); + + private: + mutable std::mutex mMutex; + std::deque mUpdates; + size_t mMaxUpdates{ 0 }; + size_t mMaxBytes{ 0 }; + size_t mQueuedBytes{ 0 }; + Uint64 mNextSequence{ 0 }; + bool mNeedsResync{ false }; +}; + +}} // namespace eterm::Terminal + +#endif diff --git a/src/modules/eterm/include/eterm/terminal/terminalsession.hpp b/src/modules/eterm/include/eterm/terminal/terminalsession.hpp index 59be5bef2..37b27d7cd 100644 --- a/src/modules/eterm/include/eterm/terminal/terminalsession.hpp +++ b/src/modules/eterm/include/eterm/terminal/terminalsession.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,7 @@ namespace eterm { namespace Terminal { /** Immutable worker-to-UI presentation state. Cell selection is already applied as ATTR_REVERSE. */ struct TerminalSnapshot { + std::shared_ptr graphics; std::vector cells; std::vector dirtyRows; std::string title; @@ -118,6 +120,7 @@ class TerminalSession final : public std::enable_shared_from_this snapshot() const; std::vector drainEvents(); + std::vector drainGraphicsUpdates(); + void requestGraphicsResync(); /** Bounded exact-selection request. Returns no value on timeout or during shutdown. */ std::optional @@ -160,6 +165,8 @@ class TerminalSession final : public std::enable_shared_from_this response; }; struct SelectionClearCommand {}; + struct GraphicsResyncCommand {}; struct ResetCommand {}; struct TerminateCommand {}; struct AllowTrimCommand : BoolCommand {}; @@ -216,7 +225,7 @@ class TerminalSession final : public std::enable_shared_from_this; + SelectionRequestCommand, GraphicsResyncCommand>; TerminalSession( PtyPtr&& pty, ProcPtr&& process, size_t historySize, TerminalColorPalette palette ); @@ -228,6 +237,7 @@ class TerminalSession final : public std::enable_shared_from_this snapshot ); + Uint64 enqueueGraphicsUpdate( TerminalGraphicsUpdate update ); std::shared_ptr mWorkerDisplay; std::unique_ptr mEmulator; @@ -238,6 +248,7 @@ class TerminalSession final : public std::enable_shared_from_this mCommands; std::mutex mEventMutex; std::deque mEvents; + TerminalGraphicsUpdateQueue mGraphicsUpdates; mutable std::mutex mPublishedSnapshotMutex; std::shared_ptr mPublishedSnapshot; std::atomic mShutdownRequested{ false }; @@ -246,4 +257,4 @@ class TerminalSession final : public std::enable_shared_from_this, + std::vector ) {} + void ITerminalDisplay::onProcessExit( int /*exitCode*/ ) {} void ITerminalDisplay::onScrollPositionChange() {} diff --git a/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp b/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp new file mode 100644 index 000000000..1e561b0f6 --- /dev/null +++ b/src/modules/eterm/src/eterm/terminal/kittygraphicsprotocol.cpp @@ -0,0 +1,1559 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace EE::System; + +namespace eterm { namespace Terminal { + +KittyGraphicsProtocol::KittyGraphicsProtocol( size_t maxStorageBytes, size_t maxImages, + size_t maxPlacements ) : + mMaxStorageBytes( maxStorageBytes ), mMaxImages( maxImages ), mMaxPlacements( maxPlacements ) {} + +namespace { + +bool parseUnsigned( std::string_view value, Uint32& result ) { + if ( value.empty() ) + return false; + const char* end = value.data() + value.size(); + auto parsed = std::from_chars( value.data(), end, result ); + return parsed.ec == std::errc{} && parsed.ptr == end; +} + +bool parseSigned( std::string_view value, Int32& result ) { + if ( value.empty() ) + return false; + const char* end = value.data() + value.size(); + auto parsed = std::from_chars( value.data(), end, result ); + return parsed.ec == std::errc{} && parsed.ptr == end; +} + +bool parseCharacter( std::string_view value, char& result ) { + if ( value.size() != 1 ) + return false; + result = value.front(); + return true; +} + +bool checkedPixelBytes( Uint32 width, Uint32 height, size_t channels, size_t& result ) { + if ( width == 0 || height == 0 || width > std::numeric_limits::max() / height ) + return false; + const size_t pixels = static_cast( width ) * height; + if ( pixels > std::numeric_limits::max() / channels ) + return false; + result = pixels * channels; + return true; +} + +bool validBase64( std::string_view input, bool finalChunk ) { + if ( input.size() % 4 == 1 ) + return false; + size_t padding = 0; + for ( size_t i = 0; i < input.size(); ++i ) { + const unsigned char character = input[i]; + const bool alphabet = + ( character >= 'A' && character <= 'Z' ) || ( character >= 'a' && character <= 'z' ) || + ( character >= '0' && character <= '9' ) || character == '+' || character == '/'; + if ( character == '=' ) { + ++padding; + if ( !finalChunk || padding > 2 ) + return false; + } else if ( !alphabet || padding != 0 ) { + return false; + } + } + return finalChunk || input.size() % 4 == 0; +} + +bool decodeBase64( std::string_view input, bool finalChunk, std::vector& output ) { + if ( !validBase64( input, finalChunk ) ) + return false; + std::vector decoded( Base64::decodeSafeOutLen( input.size() ) ); + const size_t decodedSize = Base64::decode( input.size(), input.data(), decoded.size(), + decoded.data(), Base64::DecodeMode::NoWhitespace ); + if ( decodedSize == static_cast( -1 ) ) + return false; + decoded.resize( decodedSize ); + constexpr size_t MaxTransferBytes = 64 * 1024 * 1024; + if ( output.size() > MaxTransferBytes || decoded.size() > MaxTransferBytes - output.size() ) + return false; + output.insert( output.end(), decoded.begin(), decoded.end() ); + return true; +} + +bool placementContains( const TerminalVisiblePlacement& placement, Vector2i cell ) { + const Int64 right = static_cast( placement.visibleAnchorCell.x ) + placement.columns; + const Int64 bottom = static_cast( placement.visibleAnchorCell.y ) + placement.rows; + return cell.x >= placement.visibleAnchorCell.x && cell.y >= placement.visibleAnchorCell.y && + cell.x < right && cell.y < bottom; +} + +} // namespace + +KittyGraphicsParseResult KittyGraphicsProtocol::parse( std::string_view command ) { + KittyGraphicsParseResult result; + KittyGraphicsCommandData data; + char action = 't'; + + const size_t separator = command.find( ';' ); + std::string_view control = command.substr( 0, separator ); + if ( separator != std::string_view::npos ) + data.payload = command.substr( separator + 1 ); + + while ( !control.empty() ) { + const size_t comma = control.find( ',' ); + const std::string_view field = control.substr( 0, comma ); + control = + comma == std::string_view::npos ? std::string_view{} : control.substr( comma + 1 ); + const size_t equals = field.find( '=' ); + if ( equals == std::string_view::npos || equals == 0 || equals + 1 == field.size() ) { + result.error = KittyGraphicsError::InvalidArgument; + return result; + } + + const std::string_view key = field.substr( 0, equals ); + const std::string_view value = field.substr( equals + 1 ); + if ( key.size() != 1 ) + continue; // Unknown future keys are ignored for forward compatibility. + + Uint32 unsignedValue = 0; + bool valid = true; + switch ( key.front() ) { + case 'a': + valid = parseCharacter( value, action ); + break; + case 't': + valid = parseCharacter( value, data.transmission ); + break; + case 'o': + valid = parseCharacter( value, data.compression ); + break; +#define PARSE_UINT_FIELD( name ) \ + valid = parseUnsigned( value, unsignedValue ); \ + if ( valid ) \ + data.name = unsignedValue + case 'f': + PARSE_UINT_FIELD( format ); + break; + case 'S': + PARSE_UINT_FIELD( dataSize ); + break; + case 'm': + PARSE_UINT_FIELD( more ); + valid = valid && unsignedValue <= 1; + break; + case 'i': + PARSE_UINT_FIELD( imageId ); + break; + case 'I': + PARSE_UINT_FIELD( imageNumber ); + break; + case 'N': + PARSE_UINT_FIELD( usageHint ); + break; + case 'p': + PARSE_UINT_FIELD( placementId ); + break; + case 'q': + PARSE_UINT_FIELD( quiet ); + valid = valid && unsignedValue <= 2; + break; + case 's': + PARSE_UINT_FIELD( width ); + break; + case 'v': + PARSE_UINT_FIELD( height ); + break; + case 'x': + PARSE_UINT_FIELD( x ); + break; + case 'y': + PARSE_UINT_FIELD( y ); + break; + case 'w': + PARSE_UINT_FIELD( sourceWidth ); + break; + case 'h': + PARSE_UINT_FIELD( sourceHeight ); + break; + case 'c': + PARSE_UINT_FIELD( columns ); + break; + case 'r': + PARSE_UINT_FIELD( rows ); + break; + case 'X': + PARSE_UINT_FIELD( xOffset ); + break; + case 'Y': + PARSE_UINT_FIELD( yOffset ); + break; + case 'C': + PARSE_UINT_FIELD( cursorMovement ); + valid = valid && unsignedValue <= 1; + break; + case 'U': + PARSE_UINT_FIELD( virtualPlacement ); + valid = valid && unsignedValue <= 1; + break; + case 'P': + PARSE_UINT_FIELD( parentImageId ); + break; + case 'Q': + PARSE_UINT_FIELD( parentPlacementId ); + break; + case 'H': { + Int32 signedValue = 0; + valid = parseSigned( value, signedValue ); + if ( valid ) + data.parentOffsetX = signedValue; + break; + } + case 'V': { + Int32 signedValue = 0; + valid = parseSigned( value, signedValue ); + if ( valid ) + data.parentOffsetY = signedValue; + break; + } + case 'd': + valid = parseCharacter( value, data.deletion ); + break; + case 'z': { + Int32 signedValue = 0; + valid = parseSigned( value, signedValue ); + if ( valid ) + data.zIndex = signedValue; + break; + } + default: + break; + } +#undef PARSE_UINT_FIELD + if ( !valid ) { + result.error = KittyGraphicsError::InvalidArgument; + return result; + } + } + + if ( data.imageId && data.imageNumber ) { + result.error = KittyGraphicsError::InvalidArgument; + return result; + } + + switch ( action ) { + case 't': + result.command = KittyTransmitCommand{ data, false }; + break; + case 'T': + result.command = KittyTransmitCommand{ data, true }; + break; + case 'p': + result.command = KittyPutCommand{ data }; + break; + case 'd': + result.command = KittyDeleteCommand{ data }; + break; + case 'f': + result.command = KittyFrameCommand{ data }; + break; + case 'a': + result.command = KittyAnimationCommand{ data }; + break; + case 'c': + result.command = KittyComposeCommand{ data }; + break; + case 'q': + result.command = KittyQueryCommand{ data }; + break; + default: + result.error = KittyGraphicsError::InvalidArgument; + break; + } + return result; +} + +KittyGraphicsHandleResult KittyGraphicsProtocol::handle( std::string_view command, + Vector2i cursor ) { + auto parsed = parse( command ); + if ( !parsed.command ) + return { {}, parsed.error, false }; + + return std::visit( + [this, cursor]( const auto& value ) -> KittyGraphicsHandleResult { + using T = std::decay_t; + if constexpr ( !std::is_same_v && + !std::is_same_v && + !std::is_same_v && + !std::is_same_v ) { + if ( mPending.active ) { + mPending = {}; + return { response( value.data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + } + } + if constexpr ( std::is_same_v ) { + return handleTransmit( value.data, value.display, false, false, cursor ); + } else if constexpr ( std::is_same_v ) { + return handleTransmit( value.data, false, true, false, cursor ); + } else if constexpr ( std::is_same_v ) { + return handleTransmit( value.data, false, false, true, cursor ); + } else if constexpr ( std::is_same_v ) { + return put( value.data, cursor ); + } else if constexpr ( std::is_same_v ) { + return remove( value.data, cursor ); + } else if constexpr ( std::is_same_v ) { + return controlAnimation( value.data ); + } else if constexpr ( std::is_same_v ) { + return composeFrames( value.data ); + } else { + return { response( value.data, KittyGraphicsError::Unsupported ), + KittyGraphicsError::Unsupported, false }; + } + }, + *parsed.command ); +} + +KittyGraphicsHandleResult +KittyGraphicsProtocol::handleTransmit( const KittyGraphicsCommandData& data, bool display, + bool query, bool frame, Vector2i cursor ) { + const bool more = data.more.value_or( 0 ) != 0; + if ( mPending.active ) { + if ( frame != mPending.frame || data.format || data.dataSize || data.imageId || + data.imageNumber || data.usageHint || data.placementId || data.width || data.height || + data.x || data.y || data.sourceWidth || data.sourceHeight || data.columns || + data.rows || data.xOffset || data.yOffset || data.zIndex || data.cursorMovement || + data.virtualPlacement || data.parentImageId || data.parentPlacementId || + data.parentOffsetX || data.parentOffsetY || data.compression != 0 || + data.transmission != 'd' ) { + mPending = {}; + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + } + if ( !decodeBase64( data.payload, !more, mPending.decodedData ) ) { + mPending = {}; + return { response( data, KittyGraphicsError::InvalidData ), + KittyGraphicsError::InvalidData, false }; + } + if ( more ) + return {}; + auto transfer = std::move( mPending ); + mPending = {}; + return finishTransfer( std::move( transfer ), cursor ); + } + + if ( data.transmission != 'd' ) + return { response( data, KittyGraphicsError::Unsupported ), KittyGraphicsError::Unsupported, + false }; + PendingTransfer transfer; + transfer.data = data; + transfer.data.payload = {}; + transfer.display = display; + transfer.query = query; + transfer.frame = frame; + transfer.active = true; + if ( !decodeBase64( data.payload, !more, transfer.decodedData ) ) + return { response( data, KittyGraphicsError::InvalidData ), KittyGraphicsError::InvalidData, + false }; + if ( more ) { + mPending = std::move( transfer ); + return {}; + } + return finishTransfer( std::move( transfer ), cursor ); +} + +KittyGraphicsHandleResult KittyGraphicsProtocol::finishTransfer( PendingTransfer transfer, + Vector2i cursor ) { + const auto& data = transfer.data; + mStats.decodedBytes += transfer.decodedData.size(); + const Uint32 format = data.format.value_or( 32 ); + if ( format != 24 && format != 32 && format != 100 ) + return { response( data, KittyGraphicsError::Unsupported ), KittyGraphicsError::Unsupported, + false }; + + Uint32 width = data.width.value_or( 0 ); + Uint32 height = data.height.value_or( 0 ); + std::vector pixels; + if ( format == 100 ) { + std::vector encoded; + if ( data.compression == 'z' ) { + if ( !data.dataSize || *data.dataSize == 0 || *data.dataSize > 64 * 1024 * 1024 ) + return { response( data, KittyGraphicsError::TooLarge ), + KittyGraphicsError::TooLarge, false }; + encoded.resize( *data.dataSize ); + IOStreamMemory source( reinterpret_cast( transfer.decodedData.data() ), + transfer.decodedData.size() ); + IOStreamMemory destination( reinterpret_cast( encoded.data() ), encoded.size() ); + if ( Compression::decompress( destination, source ) != Compression::OK || + static_cast( destination.tell() ) != encoded.size() ) { + return { response( data, KittyGraphicsError::DecodeFailed ), + KittyGraphicsError::DecodeFailed, false }; + } + } else if ( data.compression == 0 ) { + encoded = std::move( transfer.decodedData ); + } else { + return { response( data, KittyGraphicsError::Unsupported ), + KittyGraphicsError::Unsupported, false }; + } + if ( encoded.size() < 24 || std::memcmp( encoded.data(), "\x89PNG\r\n\x1a\n", 8 ) != 0 || + std::memcmp( encoded.data() + 12, "IHDR", 4 ) != 0 ) { + return { response( data, KittyGraphicsError::InvalidData ), + KittyGraphicsError::InvalidData, false }; + } + width = ( static_cast( encoded[16] ) << 24 ) | + ( static_cast( encoded[17] ) << 16 ) | + ( static_cast( encoded[18] ) << 8 ) | encoded[19]; + height = ( static_cast( encoded[20] ) << 24 ) | + ( static_cast( encoded[21] ) << 16 ) | + ( static_cast( encoded[22] ) << 8 ) | encoded[23]; + size_t rgbaBytes = 0; + if ( !checkedPixelBytes( width, height, 4, rgbaBytes ) || rgbaBytes > 128 * 1024 * 1024 ) { + return { response( data, KittyGraphicsError::TooLarge ), KittyGraphicsError::TooLarge, + false }; + } + EE::Graphics::Image decoded( encoded.data(), static_cast( encoded.size() ), + 4 ); + if ( !decoded.getPixelsPtr() || decoded.getWidth() != width || + decoded.getHeight() != height ) + return { response( data, KittyGraphicsError::DecodeFailed ), + KittyGraphicsError::DecodeFailed, false }; + pixels.assign( decoded.getPixelsPtr(), decoded.getPixelsPtr() + rgbaBytes ); + } else { + if ( width == 0 || height == 0 ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + size_t sourceBytes = 0; + if ( !checkedPixelBytes( width, height, format == 24 ? 3 : 4, sourceBytes ) || + sourceBytes > 128 * 1024 * 1024 ) { + return { response( data, KittyGraphicsError::TooLarge ), KittyGraphicsError::TooLarge, + false }; + } + if ( data.compression == 'z' ) { + pixels.resize( sourceBytes ); + IOStreamMemory source( reinterpret_cast( transfer.decodedData.data() ), + transfer.decodedData.size() ); + IOStreamMemory destination( reinterpret_cast( pixels.data() ), pixels.size() ); + if ( Compression::decompress( destination, source ) != Compression::OK || + static_cast( destination.tell() ) != sourceBytes ) { + return { response( data, KittyGraphicsError::DecodeFailed ), + KittyGraphicsError::DecodeFailed, false }; + } + } else if ( data.compression != 0 ) { + return { response( data, KittyGraphicsError::Unsupported ), + KittyGraphicsError::Unsupported, false }; + } else { + if ( transfer.decodedData.size() != sourceBytes ) + return { response( data, KittyGraphicsError::InvalidData ), + KittyGraphicsError::InvalidData, false }; + pixels = std::move( transfer.decodedData ); + } + } + + if ( format == 24 ) { + std::vector rgba; + rgba.resize( static_cast( width ) * height * 4 ); + for ( size_t sourceOffset = 0, destinationOffset = 0; sourceOffset < pixels.size(); + sourceOffset += 3, destinationOffset += 4 ) { + rgba[destinationOffset] = pixels[sourceOffset]; + rgba[destinationOffset + 1] = pixels[sourceOffset + 1]; + rgba[destinationOffset + 2] = pixels[sourceOffset + 2]; + rgba[destinationOffset + 3] = 255; + } + pixels = std::move( rgba ); + } + if ( transfer.frame ) { + const KittyImageId imageId = resolveImageId( data ); + auto image = mImages.find( imageId ); + if ( image == mImages.end() ) + return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, + false }; + Uint32 frameNumber = data.rows.value_or( 0 ); + if ( frameNumber == 0 ) { + frameNumber = 2; + for ( const auto& frame : image->second.frames ) + frameNumber = std::max( frameNumber, frame.first + 1 ); + } + const Uint32 destinationX = data.x.value_or( 0 ); + const Uint32 destinationY = data.y.value_or( 0 ); + if ( destinationX >= static_cast( image->second.size.getWidth() ) || + destinationY >= static_cast( image->second.size.getHeight() ) || + width > image->second.size.getWidth() - destinationX || + height > image->second.size.getHeight() - destinationY ) { + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + } + + std::vector* destinationPixels = nullptr; + bool createdFrame = false; + if ( frameNumber == 1 ) { + destinationPixels = &image->second.rgba; + } else { + auto frame = image->second.frames.find( frameNumber ); + if ( frame == image->second.frames.end() ) { + Image::Frame newFrame; + newFrame.rgba.resize( static_cast( image->second.size.getWidth() ) * + image->second.size.getHeight() * 4 ); + if ( data.columns ) { + const Uint32 baseFrame = *data.columns; + if ( baseFrame == 1 ) + newFrame.rgba = image->second.rgba; + else { + auto base = image->second.frames.find( baseFrame ); + if ( base == image->second.frames.end() ) + return { response( data, KittyGraphicsError::NotFound ), + KittyGraphicsError::NotFound, false }; + newFrame.rgba = base->second.rgba; + } + } else if ( data.yOffset ) { + const Uint32 color = *data.yOffset; + for ( size_t offset = 0; offset < newFrame.rgba.size(); offset += 4 ) { + newFrame.rgba[offset] = static_cast( color >> 24 ); + newFrame.rgba[offset + 1] = static_cast( color >> 16 ); + newFrame.rgba[offset + 2] = static_cast( color >> 8 ); + newFrame.rgba[offset + 3] = static_cast( color ); + } + } + const size_t maxFrameStorage = + mMaxStorageBytes > std::numeric_limits::max() / 5 + ? std::numeric_limits::max() + : mMaxStorageBytes * 5; + if ( mFrameStorageBytes > maxFrameStorage || + newFrame.rgba.size() > maxFrameStorage - mFrameStorageBytes ) + return { response( data, KittyGraphicsError::NoSpace ), + KittyGraphicsError::NoSpace, false }; + newFrame.gapMs = data.zIndex.value_or( 40 ); + newFrame.usageHint = data.usageHint.value_or( 0 ); + mFrameStorageBytes += newFrame.rgba.size(); + frame = image->second.frames.emplace( frameNumber, std::move( newFrame ) ).first; + createdFrame = true; + } else { + if ( data.zIndex && *data.zIndex != 0 ) + frame->second.gapMs = *data.zIndex; + } + destinationPixels = &frame->second.rgba; + } + + const bool replace = data.xOffset.value_or( 0 ) == 1; + std::vector finalPatch( pixels.size() ); + const size_t imageStride = static_cast( image->second.size.getWidth() ) * 4; + const size_t patchStride = static_cast( width ) * 4; + for ( Uint32 row = 0; row < height; ++row ) { + Uint8* destination = destinationPixels->data() + + static_cast( destinationY + row ) * imageStride + + static_cast( destinationX ) * 4; + const Uint8* source = pixels.data() + static_cast( row ) * patchStride; + Uint8* published = finalPatch.data() + static_cast( row ) * patchStride; + if ( replace ) { + std::memcpy( destination, source, patchStride ); + std::memcpy( published, source, patchStride ); + continue; + } + for ( size_t column = 0; column < patchStride; column += 4 ) { + const Uint32 alpha = source[column + 3]; + const Uint32 inverseAlpha = 255 - alpha; + for ( size_t channel = 0; channel < 3; ++channel ) + destination[column + channel] = + static_cast( ( source[column + channel] * alpha + + destination[column + channel] * inverseAlpha + 127 ) / + 255 ); + destination[column + 3] = static_cast( + alpha + ( destination[column + 3] * inverseAlpha + 127 ) / 255 ); + std::memcpy( published + column, destination + column, 4 ); + } + } + TerminalGraphicsUpdate update; + update.type = frameNumber == 1 ? TerminalGraphicsUpdateType::UpdateRegion + : createdFrame ? TerminalGraphicsUpdateType::CreateFrame + : TerminalGraphicsUpdateType::UpdateFrameRegion; + update.imageId = imageId; + update.frameNumber = frameNumber; + update.imageSize = image->second.size; + update.region = + createdFrame + ? Rect( 0, 0, image->second.size.getWidth(), image->second.size.getHeight() ) + : Rect( destinationX, destinationY, destinationX + width, destinationY + height ); + update.rgba = createdFrame + ? std::make_shared>( *destinationPixels ) + : std::make_shared>( std::move( finalPatch ) ); + if ( !createdFrame && !mUpdates.empty() && mUpdates.back().type == update.type && + mUpdates.back().imageId == update.imageId && + mUpdates.back().frameNumber == update.frameNumber && + mUpdates.back().region == update.region ) + mUpdates.back() = std::move( update ); + else + mUpdates.emplace_back( std::move( update ) ); + ++mStats.rectangleUpdates; + mPresentationDirty = true; + return { response( data, KittyGraphicsError::None, imageId ), KittyGraphicsError::None, + true }; + } + + const bool anonymous = !data.imageId && !data.imageNumber; + KittyImageId imageId = data.imageId.value_or( 0 ); + if ( anonymous && transfer.display ) { + auto placement = std::find_if( + mPlacements.begin(), mPlacements.end(), [&]( const Placement& candidate ) { + auto candidateImage = mImages.find( candidate.visible.imageId ); + return candidate.visible.visibleAnchorCell == cursor && + candidate.visible.placementId == 0 && candidateImage != mImages.end() && + candidateImage->second.anonymous; + } ); + if ( placement != mPlacements.end() ) + imageId = placement->visible.imageId; + } + if ( imageId == 0 ) + imageId = allocateImageId(); + if ( imageId == 0 ) + return { response( data, KittyGraphicsError::NoSpace ), KittyGraphicsError::NoSpace, + false }; + if ( transfer.query ) + return { response( data, KittyGraphicsError::None, imageId ), KittyGraphicsError::None, + false }; + + auto existing = mImages.find( imageId ); + const size_t oldBytes = existing == mImages.end() ? 0 : existing->second.rgba.size(); + if ( !ensureCapacity( pixels.size(), imageId, existing == mImages.end() ) ) + return { response( data, KittyGraphicsError::NoSpace ), KittyGraphicsError::NoSpace, + false }; + existing = mImages.find( imageId ); + if ( existing != mImages.end() ) { + for ( const auto& frame : existing->second.frames ) + mFrameStorageBytes -= frame.second.rgba.size(); + mPlacements.erase( std::remove_if( mPlacements.begin(), mPlacements.end(), + [imageId]( const Placement& p ) { + return p.visible.imageId == imageId; + } ), + mPlacements.end() ); + mPrimaryPlacements.erase( std::remove_if( mPrimaryPlacements.begin(), + mPrimaryPlacements.end(), + [imageId]( const Placement& p ) { + return p.visible.imageId == imageId; + } ), + mPrimaryPlacements.end() ); + } + Image image; + image.size = Sizei( width, height ); + image.imageNumber = data.imageNumber.value_or( 0 ); + image.usageHint = data.usageHint.value_or( 0 ); + image.anonymous = anonymous; + image.creationSerial = ++mCreationSerial; + image.rgba = std::move( pixels ); + mStorageBytes = mStorageBytes - oldBytes + image.rgba.size(); + const bool replaced = existing != mImages.end(); + auto inserted = mImages.insert_or_assign( imageId, std::move( image ) ).first; + + TerminalGraphicsUpdate update; + update.type = replaced ? TerminalGraphicsUpdateType::ReplaceImage + : TerminalGraphicsUpdateType::CreateImage; + update.imageId = imageId; + update.imageSize = inserted->second.size; + update.region = + Rect( 0, 0, inserted->second.size.getWidth(), inserted->second.size.getHeight() ); + update.rgba = std::make_shared>( inserted->second.rgba ); + mUpdates.emplace_back( std::move( update ) ); + ++mStats.fullImageUpdates; + ++mPresentationGeneration; + mPresentationDirty = true; + auto result = KittyGraphicsHandleResult{ + response( data, KittyGraphicsError::None, anonymous ? 0 : imageId ), + KittyGraphicsError::None, true }; + if ( transfer.display ) { + auto placementData = data; + placementData.imageId = imageId; + placementData.imageNumber.reset(); + auto placementResult = put( placementData, cursor ); + if ( placementResult.error != KittyGraphicsError::None ) + return placementResult; + result.cursorMovement = placementResult.cursorMovement; + } + return result; +} + +KittyGraphicsHandleResult KittyGraphicsProtocol::put( const KittyGraphicsCommandData& data, + Vector2i cursor ) { + const KittyImageId imageId = resolveImageId( data ); + auto image = mImages.find( imageId ); + if ( image == mImages.end() ) + return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, + false }; + const Uint32 sourceX = data.x.value_or( 0 ); + const Uint32 sourceY = data.y.value_or( 0 ); + if ( sourceX >= static_cast( image->second.size.getWidth() ) || + sourceY >= static_cast( image->second.size.getHeight() ) ) { + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + } + const Uint32 sourceWidth = + std::min( data.sourceWidth.value_or( image->second.size.getWidth() - sourceX ), + image->second.size.getWidth() - sourceX ); + const Uint32 sourceHeight = + std::min( data.sourceHeight.value_or( image->second.size.getHeight() - sourceY ), + image->second.size.getHeight() - sourceY ); + if ( sourceWidth == 0 || sourceHeight == 0 ) { + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + } + + TerminalVisiblePlacement visible; + visible.imageId = imageId; + visible.placementId = data.placementId.value_or( 0 ); + visible.visibleAnchorCell = cursor; + const bool relative = data.parentImageId.has_value() || data.parentPlacementId.has_value(); + if ( relative ) { + if ( data.virtualPlacement.value_or( 0 ) == 1 ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + if ( !data.parentImageId || !data.parentPlacementId ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + auto parent = + std::find_if( mPlacements.begin(), mPlacements.end(), [&]( const Placement& p ) { + return p.visible.imageId == *data.parentImageId && + p.visible.placementId == *data.parentPlacementId; + } ); + if ( parent == mPlacements.end() ) + return { response( data, KittyGraphicsError::NoParent ), KittyGraphicsError::NoParent, + false }; + KittyImageId ancestorImage = parent->visible.imageId; + KittyPlacementId ancestorPlacement = parent->visible.placementId; + for ( size_t depth = 0; ancestorImage != 0; ++depth ) { + if ( ancestorImage == imageId && ancestorPlacement == visible.placementId ) + return { response( data, KittyGraphicsError::Cycle ), KittyGraphicsError::Cycle, + false }; + if ( depth >= 64 ) + return { response( data, KittyGraphicsError::TooDeep ), KittyGraphicsError::TooDeep, + false }; + auto ancestor = std::find_if( + mPlacements.begin(), mPlacements.end(), [&]( const Placement& placement ) { + return placement.visible.imageId == ancestorImage && + placement.visible.placementId == ancestorPlacement; + } ); + if ( ancestor == mPlacements.end() || ancestor->parentImageId == 0 ) + break; + ancestorImage = ancestor->parentImageId; + ancestorPlacement = ancestor->parentPlacementId; + } + visible.visibleAnchorCell = + parent->visible.visibleAnchorCell + + Vector2i( data.parentOffsetX.value_or( 0 ), data.parentOffsetY.value_or( 0 ) ); + } + visible.sourcePixels = Rect( sourceX, sourceY, sourceX + sourceWidth, sourceY + sourceHeight ); + visible.columns = data.columns.value_or( 0 ); + visible.rows = data.rows.value_or( 0 ); + if ( visible.columns == 0 && visible.rows == 0 && mCellPixelWidth && mCellPixelHeight ) { + visible.columns = + static_cast( ( static_cast( sourceWidth ) + data.xOffset.value_or( 0 ) + + mCellPixelWidth - 1 ) / + mCellPixelWidth ); + visible.rows = static_cast( ( static_cast( sourceHeight ) + + data.yOffset.value_or( 0 ) + mCellPixelHeight - 1 ) / + mCellPixelHeight ); + } else if ( visible.columns != 0 && visible.rows == 0 && mCellPixelHeight ) { + const Uint64 scaledHeight = + static_cast( sourceHeight ) * visible.columns * mCellPixelWidth; + const Uint64 denominator = static_cast( sourceWidth ) * mCellPixelHeight; + visible.rows = static_cast( ( scaledHeight + denominator - 1 ) / denominator ); + } else if ( visible.rows != 0 && visible.columns == 0 && mCellPixelWidth ) { + const Uint64 scaledWidth = + static_cast( sourceWidth ) * visible.rows * mCellPixelHeight; + const Uint64 denominator = static_cast( sourceHeight ) * mCellPixelWidth; + visible.columns = static_cast( ( scaledWidth + denominator - 1 ) / denominator ); + } + if ( visible.columns == 0 ) + visible.columns = 1; + if ( visible.rows == 0 ) + visible.rows = 1; + visible.firstCellPixelOffset = + Vector2i( data.xOffset.value_or( 0 ), data.yOffset.value_or( 0 ) ); + if ( ( mCellPixelWidth && data.xOffset.value_or( 0 ) >= mCellPixelWidth ) || + ( mCellPixelHeight && data.yOffset.value_or( 0 ) >= mCellPixelHeight ) ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + visible.zIndex = data.zIndex.value_or( 0 ); + if ( visible.columns == 0 || visible.rows == 0 ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + bool replacingPlacement = false; + if ( visible.placementId != 0 ) { + for ( auto it = mPlacements.begin(); it != mPlacements.end(); ++it ) { + if ( it->visible.imageId == imageId && + it->visible.placementId == visible.placementId ) { + const Vector2i movement = visible.visibleAnchorCell - it->visible.visibleAnchorCell; + std::vector> movedParents{ + { imageId, visible.placementId } }; + for ( size_t parentIndex = 0; parentIndex < movedParents.size(); ++parentIndex ) { + for ( auto& child : mPlacements ) { + if ( child.parentImageId == movedParents[parentIndex].first && + child.parentPlacementId == movedParents[parentIndex].second ) { + child.visible.visibleAnchorCell += movement; + movedParents.emplace_back( child.visible.imageId, + child.visible.placementId ); + } + } + } + mPlacements.erase( it ); + replacingPlacement = true; + break; + } + } + } + if ( !replacingPlacement && mPlacements.size() >= mMaxPlacements ) + return { response( data, KittyGraphicsError::NoSpace ), KittyGraphicsError::NoSpace, + false }; + mPlacements.push_back( { visible, ++mPlacementSerial, data.virtualPlacement.value_or( 0 ) == 1, + data.parentImageId.value_or( 0 ), + data.parentPlacementId.value_or( 0 ) } ); + ++mPresentationGeneration; + mPresentationDirty = true; + KittyGraphicsHandleResult result{ response( data, KittyGraphicsError::None, imageId ), + KittyGraphicsError::None, true }; + if ( data.cursorMovement.value_or( 0 ) == 0 && !relative && + data.virtualPlacement.value_or( 0 ) == 0 ) + result.cursorMovement = Vector2i( visible.columns, visible.rows ); + return result; +} + +KittyGraphicsHandleResult KittyGraphicsProtocol::remove( const KittyGraphicsCommandData& data, + Vector2i cursor ) { + mPending = {}; + const char selector = data.deletion; + const bool deleteImageData = selector >= 'A' && selector <= 'Z'; + const char normalized = deleteImageData ? static_cast( selector - 'A' + 'a' ) : selector; + constexpr std::string_view ValidSelectors = "acfinpqrxyz"; + if ( ValidSelectors.find( normalized ) == std::string_view::npos ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + + const KittyImageId selectedImage = resolveImageId( data ); + if ( ( normalized == 'f' || normalized == 'i' || normalized == 'n' ) && selectedImage == 0 ) + return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, + false }; + if ( normalized == 'f' ) { + auto image = mImages.find( selectedImage ); + if ( image == mImages.end() ) + return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, + false }; + const bool changed = !image->second.frames.empty(); + for ( const auto& frame : image->second.frames ) { + mFrameStorageBytes -= frame.second.rgba.size(); + TerminalGraphicsUpdate update; + update.type = TerminalGraphicsUpdateType::DeleteFrame; + update.imageId = selectedImage; + update.frameNumber = frame.first; + mUpdates.emplace_back( std::move( update ) ); + } + image->second.frames.clear(); + image->second.currentFrame = 1; + image->second.animationState = 1; + if ( changed ) { + ++mPresentationGeneration; + mPresentationDirty = true; + } + return { response( data, KittyGraphicsError::None, selectedImage ), + KittyGraphicsError::None, changed }; + } + + Vector2i selectedCell = cursor; + if ( normalized == 'p' || normalized == 'q' ) { + if ( !data.x || !data.y || *data.x == 0 || *data.y == 0 ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + selectedCell = + Vector2i( static_cast( *data.x - 1 ), static_cast( *data.y - 1 ) ); + } + if ( normalized == 'x' && ( !data.x || *data.x == 0 ) ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + if ( normalized == 'y' && ( !data.y || *data.y == 0 ) ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + if ( normalized == 'r' && ( !data.x || !data.y || *data.x > *data.y ) ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + if ( normalized == 'z' && !data.zIndex ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + + std::vector affectedImages; + auto matches = [&]( const Placement& placement ) { + auto visible = placement.visible; + visible.visibleAnchorCell.y += mScrollPosition; + if ( placement.virtualPlacement && normalized != 'i' && normalized != 'n' && + normalized != 'r' ) + return false; + bool match = false; + switch ( normalized ) { + case 'a': + match = static_cast( visible.visibleAnchorCell.y ) + visible.rows > 0 && + ( mScreenRows == 0 || visible.visibleAnchorCell.y < mScreenRows ); + break; + case 'c': + match = placementContains( visible, cursor ); + break; + case 'i': + case 'n': + match = visible.imageId == selectedImage; + if ( match && data.placementId ) + match = visible.placementId == *data.placementId; + break; + case 'p': + match = placementContains( visible, selectedCell ); + break; + case 'q': + match = placementContains( visible, selectedCell ) && data.zIndex && + visible.zIndex == *data.zIndex; + break; + case 'r': + match = visible.imageId >= *data.x && visible.imageId <= *data.y; + break; + case 'x': + match = placementContains( visible, Vector2i( static_cast( *data.x - 1 ), + visible.visibleAnchorCell.y ) ); + break; + case 'y': + match = placementContains( visible, Vector2i( visible.visibleAnchorCell.x, + static_cast( *data.y - 1 ) ) ); + break; + case 'z': + match = visible.zIndex == *data.zIndex; + break; + } + if ( match ) + affectedImages.emplace_back( visible.imageId ); + return match; + }; + const size_t oldPlacementCount = mPlacements.size(); + mPlacements.erase( std::remove_if( mPlacements.begin(), mPlacements.end(), matches ), + mPlacements.end() ); + std::vector orphanedRelativeImages; + bool removedChild = true; + while ( removedChild ) { + removedChild = false; + mPlacements.erase( + std::remove_if( + mPlacements.begin(), mPlacements.end(), + [&]( const Placement& placement ) { + if ( placement.parentImageId == 0 ) + return false; + const bool parentExists = std::any_of( + mPlacements.begin(), mPlacements.end(), [&]( const Placement& parent ) { + return parent.visible.imageId == placement.parentImageId && + parent.visible.placementId == placement.parentPlacementId; + } ); + if ( !parentExists ) { + affectedImages.emplace_back( placement.visible.imageId ); + orphanedRelativeImages.emplace_back( placement.visible.imageId ); + removedChild = true; + } + return !parentExists; + } ), + mPlacements.end() ); + } + std::sort( orphanedRelativeImages.begin(), orphanedRelativeImages.end() ); + orphanedRelativeImages.erase( + std::unique( orphanedRelativeImages.begin(), orphanedRelativeImages.end() ), + orphanedRelativeImages.end() ); + for ( KittyImageId imageId : orphanedRelativeImages ) { + if ( !isImagePlaced( imageId ) ) + eraseImage( imageId ); + } + + bool deletedAnyImage = false; + if ( deleteImageData ) { + if ( normalized == 'a' ) { + for ( const auto& image : mImages ) + affectedImages.emplace_back( image.first ); + } else if ( normalized == 'i' || normalized == 'n' ) { + affectedImages.emplace_back( selectedImage ); + } else if ( normalized == 'r' ) { + for ( const auto& image : mImages ) { + if ( image.first >= *data.x && image.first <= *data.y ) + affectedImages.emplace_back( image.first ); + } + } + std::sort( affectedImages.begin(), affectedImages.end() ); + affectedImages.erase( std::unique( affectedImages.begin(), affectedImages.end() ), + affectedImages.end() ); + for ( KittyImageId imageId : affectedImages ) { + if ( isImagePlaced( imageId ) ) + continue; + auto image = mImages.find( imageId ); + if ( image == mImages.end() ) + continue; + eraseImage( imageId ); + deletedAnyImage = true; + } + } + + const bool changed = oldPlacementCount != mPlacements.size() || deletedAnyImage; + if ( changed ) { + ++mPresentationGeneration; + mPresentationDirty = true; + } + return { response( data, KittyGraphicsError::None, selectedImage ), KittyGraphicsError::None, + changed }; +} + +KittyGraphicsHandleResult +KittyGraphicsProtocol::controlAnimation( const KittyGraphicsCommandData& data ) { + const KittyImageId imageId = resolveImageId( data ); + auto image = mImages.find( imageId ); + if ( image == mImages.end() ) + return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, + false }; + bool changed = false; + if ( data.columns ) { + const Uint32 frameNumber = *data.columns; + if ( frameNumber == 0 || ( frameNumber != 1 && image->second.frames.find( frameNumber ) == + image->second.frames.end() ) ) + return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, + false }; + changed = image->second.currentFrame != frameNumber; + image->second.currentFrame = frameNumber; + } + if ( data.width ) { + if ( *data.width < 1 || *data.width > 3 ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + image->second.animationState = static_cast( *data.width ); + image->second.frameClock.restart(); + if ( *data.width == 1 ) + image->second.loopsCompleted = 0; + changed = true; + } + if ( data.height && *data.height != 0 ) { + image->second.loopCount = *data.height; + changed = true; + } + if ( data.zIndex && *data.zIndex != 0 ) { + const Uint32 frameNumber = data.rows.value_or( image->second.currentFrame ); + if ( frameNumber == 1 ) + image->second.rootGapMs = *data.zIndex; + else { + auto frame = image->second.frames.find( frameNumber ); + if ( frame == image->second.frames.end() ) + return { response( data, KittyGraphicsError::NotFound ), + KittyGraphicsError::NotFound, false }; + frame->second.gapMs = *data.zIndex; + } + changed = true; + } + if ( changed ) { + ++mPresentationGeneration; + mPresentationDirty = true; + } + return { response( data, KittyGraphicsError::None, imageId ), KittyGraphicsError::None, + changed }; +} + +bool KittyGraphicsProtocol::updateAnimations() { + bool changed = false; + for ( auto& entry : mImages ) { + auto& image = entry.second; + if ( image.animationState < 2 || image.frames.empty() ) + continue; + Int32 gap = image.rootGapMs; + if ( image.currentFrame != 1 ) { + auto current = image.frames.find( image.currentFrame ); + if ( current == image.frames.end() ) + continue; + gap = current->second.gapMs; + } + if ( gap > 0 && image.frameClock.getElapsedTime().asMilliseconds() < gap ) + continue; + Uint32 nextFrame = std::numeric_limits::max(); + for ( const auto& frame : image.frames ) { + if ( frame.first > image.currentFrame && frame.first < nextFrame ) + nextFrame = frame.first; + } + if ( nextFrame == std::numeric_limits::max() ) { + if ( image.animationState == 2 ) + continue; + ++image.loopsCompleted; + if ( image.loopCount > 1 && image.loopsCompleted >= image.loopCount - 1 ) { + image.animationState = 1; + continue; + } + nextFrame = 1; + } + image.currentFrame = nextFrame; + image.frameClock.restart(); + changed = true; + } + if ( changed ) { + ++mPresentationGeneration; + mPresentationDirty = true; + } + return changed; +} + +void KittyGraphicsProtocol::setCellPixelSize( Uint32 width, Uint32 height ) { + mCellPixelWidth = width; + mCellPixelHeight = height; +} + +void KittyGraphicsProtocol::setPlaceholderCells( + std::vector cells ) { + if ( mPlaceholderCells == cells ) + return; + mPlaceholderCells = std::move( cells ); + ++mPresentationGeneration; + mPresentationDirty = true; +} + +KittyGraphicsHandleResult +KittyGraphicsProtocol::composeFrames( const KittyGraphicsCommandData& data ) { + const KittyImageId imageId = resolveImageId( data ); + auto image = mImages.find( imageId ); + if ( image == mImages.end() || !data.columns || !data.rows ) + return { response( data, image == mImages.end() ? KittyGraphicsError::NotFound + : KittyGraphicsError::InvalidArgument ), + image == mImages.end() ? KittyGraphicsError::NotFound + : KittyGraphicsError::InvalidArgument, + false }; + auto pixelsFor = [&]( Uint32 frameNumber ) -> std::vector* { + if ( frameNumber == 1 ) + return &image->second.rgba; + auto frame = image->second.frames.find( frameNumber ); + return frame == image->second.frames.end() ? nullptr : &frame->second.rgba; + }; + const Uint32 sourceFrame = *data.rows; + const Uint32 destinationFrame = *data.columns; + auto* source = pixelsFor( sourceFrame ); + auto* destination = pixelsFor( destinationFrame ); + if ( !source || !destination ) + return { response( data, KittyGraphicsError::NotFound ), KittyGraphicsError::NotFound, + false }; + const Uint32 destinationX = data.x.value_or( 0 ); + const Uint32 destinationY = data.y.value_or( 0 ); + const Uint32 sourceX = data.xOffset.value_or( 0 ); + const Uint32 sourceY = data.yOffset.value_or( 0 ); + const Uint32 width = data.sourceWidth.value_or( image->second.size.getWidth() ); + const Uint32 height = data.sourceHeight.value_or( image->second.size.getHeight() ); + const Uint32 imageWidth = image->second.size.getWidth(); + const Uint32 imageHeight = image->second.size.getHeight(); + if ( width == 0 || height == 0 || sourceX > imageWidth || sourceY > imageHeight || + destinationX > imageWidth || destinationY > imageHeight || width > imageWidth - sourceX || + height > imageHeight - sourceY || width > imageWidth - destinationX || + height > imageHeight - destinationY ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + if ( sourceFrame == destinationFrame && sourceX < destinationX + width && + destinationX < sourceX + width && sourceY < destinationY + height && + destinationY < sourceY + height ) + return { response( data, KittyGraphicsError::InvalidArgument ), + KittyGraphicsError::InvalidArgument, false }; + + std::vector sourceCopy( static_cast( width ) * height * 4 ); + std::vector result( sourceCopy.size() ); + const size_t imageStride = static_cast( imageWidth ) * 4; + const size_t rowBytes = static_cast( width ) * 4; + for ( Uint32 row = 0; row < height; ++row ) + std::memcpy( sourceCopy.data() + static_cast( row ) * rowBytes, + source->data() + static_cast( sourceY + row ) * imageStride + + static_cast( sourceX ) * 4, + rowBytes ); + const bool replace = data.cursorMovement.value_or( 0 ) == 1; + for ( Uint32 row = 0; row < height; ++row ) { + Uint8* target = destination->data() + + static_cast( destinationY + row ) * imageStride + + static_cast( destinationX ) * 4; + const Uint8* overlay = sourceCopy.data() + static_cast( row ) * rowBytes; + Uint8* published = result.data() + static_cast( row ) * rowBytes; + for ( size_t offset = 0; offset < rowBytes; offset += 4 ) { + if ( replace ) { + std::memcpy( target + offset, overlay + offset, 4 ); + } else { + const Uint32 alpha = overlay[offset + 3]; + const Uint32 inverseAlpha = 255 - alpha; + for ( size_t channel = 0; channel < 3; ++channel ) + target[offset + channel] = + static_cast( ( overlay[offset + channel] * alpha + + target[offset + channel] * inverseAlpha + 127 ) / + 255 ); + target[offset + 3] = + static_cast( alpha + ( target[offset + 3] * inverseAlpha + 127 ) / 255 ); + } + std::memcpy( published + offset, target + offset, 4 ); + } + } + TerminalGraphicsUpdate update; + update.type = destinationFrame == 1 ? TerminalGraphicsUpdateType::UpdateRegion + : TerminalGraphicsUpdateType::UpdateFrameRegion; + update.imageId = imageId; + update.frameNumber = destinationFrame; + update.imageSize = image->second.size; + update.region = Rect( destinationX, destinationY, destinationX + width, destinationY + height ); + update.rgba = std::make_shared>( std::move( result ) ); + mUpdates.emplace_back( std::move( update ) ); + mPresentationDirty = true; + return { response( data, KittyGraphicsError::None, imageId ), KittyGraphicsError::None, true }; +} + +KittyImageId KittyGraphicsProtocol::resolveImageId( const KittyGraphicsCommandData& data ) const { + if ( data.imageId ) + return *data.imageId; + KittyImageId newestId = 0; + Uint64 newestSerial = 0; + if ( data.imageNumber ) { + for ( const auto& image : mImages ) { + if ( image.second.imageNumber == *data.imageNumber && + image.second.creationSerial > newestSerial ) { + newestId = image.first; + newestSerial = image.second.creationSerial; + } + } + } + return newestId; +} + +KittyImageId KittyGraphicsProtocol::allocateImageId() { + for ( Uint64 attempts = 0; attempts < std::numeric_limits::max(); ++attempts ) { + const KittyImageId candidate = mNextImageId++; + if ( mNextImageId == 0 ) + mNextImageId = 1; + if ( candidate != 0 && mImages.find( candidate ) == mImages.end() ) + return candidate; + } + return 0; +} + +bool KittyGraphicsProtocol::isImagePlaced( KittyImageId imageId ) const { + auto belongsToImage = [imageId]( const Placement& placement ) { + return placement.visible.imageId == imageId; + }; + return std::any_of( mPlacements.begin(), mPlacements.end(), belongsToImage ) || + std::any_of( mPrimaryPlacements.begin(), mPrimaryPlacements.end(), belongsToImage ); +} + +void KittyGraphicsProtocol::eraseImage( KittyImageId imageId ) { + auto image = mImages.find( imageId ); + if ( image == mImages.end() ) + return; + mStorageBytes -= image->second.rgba.size(); + for ( const auto& frame : image->second.frames ) + mFrameStorageBytes -= frame.second.rgba.size(); + mImages.erase( image ); + TerminalGraphicsUpdate update; + update.type = TerminalGraphicsUpdateType::DeleteImage; + update.imageId = imageId; + mUpdates.emplace_back( std::move( update ) ); +} + +bool KittyGraphicsProtocol::ensureCapacity( size_t bytes, KittyImageId replacingId, + bool addingImage ) { + auto replaced = mImages.find( replacingId ); + const size_t replacedBytes = replaced == mImages.end() ? 0 : replaced->second.rgba.size(); + auto hasCapacity = [&] { + return bytes <= mMaxStorageBytes && + mStorageBytes - replacedBytes <= mMaxStorageBytes - bytes && + ( !addingImage || mImages.size() < mMaxImages ); + }; + while ( !hasCapacity() ) { + auto candidate = mImages.end(); + for ( auto image = mImages.begin(); image != mImages.end(); ++image ) { + if ( image->first == replacingId || isImagePlaced( image->first ) ) + continue; + if ( candidate == mImages.end() || + ( image->second.usageHint != 0 && candidate->second.usageHint == 0 ) || + ( ( image->second.usageHint != 0 ) == ( candidate->second.usageHint != 0 ) && + image->second.creationSerial < candidate->second.creationSerial ) ) + candidate = image; + } + if ( candidate == mImages.end() ) + return false; + eraseImage( candidate->first ); + ++mStats.evictions; + } + return true; +} + +std::string KittyGraphicsProtocol::response( const KittyGraphicsCommandData& data, + KittyGraphicsError error, + KittyImageId imageId ) const { + const Uint32 quiet = data.quiet.value_or( 0 ); + if ( quiet == 2 || ( quiet == 1 && error == KittyGraphicsError::None ) ) + return {}; + const KittyImageId responseId = imageId != 0 ? imageId : data.imageId.value_or( 0 ); + if ( responseId == 0 && error == KittyGraphicsError::None ) + return {}; + std::string value = "\033_G"; + if ( responseId != 0 ) { + value += "i=" + std::to_string( responseId ); + if ( data.imageNumber ) + value += ",I=" + std::to_string( *data.imageNumber ); + if ( data.placementId ) + value += ",p=" + std::to_string( *data.placementId ); + } + value += ";"; + if ( error == KittyGraphicsError::None ) { + value += "OK"; + } else { + value += error == KittyGraphicsError::Unsupported ? "ENOTSUP" + : error == KittyGraphicsError::NoSpace ? "ENOSPC" + : error == KittyGraphicsError::NotFound ? "ENOENT" + : error == KittyGraphicsError::NoParent ? "ENOPARENT" + : error == KittyGraphicsError::Cycle ? "ECYCLE" + : error == KittyGraphicsError::TooDeep ? "ETOODEEP" + : "EINVAL"; + } + value += "\033\\"; + return value; +} + +std::vector KittyGraphicsProtocol::takeUpdates() { + std::vector updates; + updates.swap( mUpdates ); + return updates; +} + +std::shared_ptr KittyGraphicsProtocol::takePresentation() { + auto presentation = std::make_shared(); + presentation->generation = mPresentationGeneration; + presentation->placements.reserve( mPlacements.size() ); + for ( const auto& placement : mPlacements ) { + if ( placement.virtualPlacement ) + continue; + auto visible = placement.visible; + if ( placement.parentImageId != 0 ) { + auto parent = + std::find_if( mPlacements.begin(), mPlacements.end(), [&]( const Placement& p ) { + return p.visible.imageId == placement.parentImageId && + p.visible.placementId == placement.parentPlacementId; + } ); + if ( parent != mPlacements.end() && parent->virtualPlacement ) { + bool found = false; + Vector2i minimum( std::numeric_limits::max(), + std::numeric_limits::max() ); + for ( const auto& cell : mPlaceholderCells ) { + if ( cell.imageId == parent->visible.imageId && + ( parent->visible.placementId == 0 || + cell.placementId == parent->visible.placementId ) ) { + minimum.x = std::min( minimum.x, cell.cell.x ); + minimum.y = std::min( minimum.y, cell.cell.y ); + found = true; + } + } + if ( !found ) + continue; + visible.visibleAnchorCell += minimum - parent->visible.visibleAnchorCell; + } + } + auto image = mImages.find( visible.imageId ); + if ( image != mImages.end() ) + visible.frameNumber = image->second.currentFrame; + visible.visibleAnchorCell.y += mScrollPosition; + const Int64 bottom = static_cast( visible.visibleAnchorCell.y ) + visible.rows; + if ( bottom > 0 && ( mScreenRows == 0 || visible.visibleAnchorCell.y < mScreenRows ) ) + presentation->placements.emplace_back( std::move( visible ) ); + } + for ( const auto& cell : mPlaceholderCells ) { + auto prototype = + std::find_if( mPlacements.begin(), mPlacements.end(), [&]( const Placement& p ) { + return p.virtualPlacement && p.visible.imageId == cell.imageId && + ( cell.placementId == 0 || p.visible.placementId == cell.placementId ); + } ); + if ( prototype == mPlacements.end() || cell.imageRow >= prototype->visible.rows || + cell.imageColumn >= prototype->visible.columns ) + continue; + auto visible = prototype->visible; + visible.visibleAnchorCell = cell.cell; + visible.columns = 1; + visible.rows = 1; + const int sourceWidth = visible.sourcePixels.Right - visible.sourcePixels.Left; + const int sourceHeight = visible.sourcePixels.Bottom - visible.sourcePixels.Top; + const int left = visible.sourcePixels.Left + static_cast( sourceWidth ) * + cell.imageColumn / + prototype->visible.columns; + const int right = visible.sourcePixels.Left + static_cast( sourceWidth ) * + ( cell.imageColumn + 1 ) / + prototype->visible.columns; + const int top = visible.sourcePixels.Top + static_cast( sourceHeight ) * + cell.imageRow / prototype->visible.rows; + const int bottom = visible.sourcePixels.Top + static_cast( sourceHeight ) * + ( cell.imageRow + 1 ) / + prototype->visible.rows; + visible.sourcePixels = Rect( left, top, right, bottom ); + auto image = mImages.find( visible.imageId ); + if ( image != mImages.end() ) + visible.frameNumber = image->second.currentFrame; + presentation->placements.emplace_back( std::move( visible ) ); + } + std::stable_sort( + presentation->placements.begin(), presentation->placements.end(), + []( const TerminalVisiblePlacement& left, const TerminalVisiblePlacement& right ) { + if ( left.zIndex != right.zIndex ) + return left.zIndex < right.zIndex; + return left.imageId < right.imageId; + } ); + mPresentationDirty = false; + return presentation; +} + +const std::vector* KittyGraphicsProtocol::imagePixels( KittyImageId imageId ) const { + auto image = mImages.find( imageId ); + return image == mImages.end() ? nullptr : &image->second.rgba; +} + +bool KittyGraphicsProtocol::hasVirtualPlacements() const { + return std::any_of( mPlacements.begin(), mPlacements.end(), + []( const Placement& placement ) { return placement.virtualPlacement; } ); +} + +void KittyGraphicsProtocol::reset() { + mImages.clear(); + mPlacements.clear(); + mPrimaryPlacements.clear(); + mPlaceholderCells.clear(); + mUpdates.clear(); + mPending = {}; + mStorageBytes = 0; + mFrameStorageBytes = 0; + TerminalGraphicsUpdate reset; + reset.type = TerminalGraphicsUpdateType::ResetAll; + mUpdates.emplace_back( std::move( reset ) ); + ++mPresentationGeneration; + mPresentationDirty = true; +} + +void KittyGraphicsProtocol::clearScreen() { + const size_t oldSize = mPlacements.size(); + mPlacements.erase( + std::remove_if( mPlacements.begin(), mPlacements.end(), + [&]( const Placement& placement ) { + if ( placement.virtualPlacement ) + return false; + const int visibleTop = + placement.visible.visibleAnchorCell.y + mScrollPosition; + return static_cast( visibleTop ) + placement.visible.rows > 0 && + ( mScreenRows == 0 || visibleTop < mScreenRows ); + } ), + mPlacements.end() ); + if ( oldSize == mPlacements.size() ) + return; + ++mPresentationGeneration; + mPresentationDirty = true; +} + +void KittyGraphicsProtocol::setAlternateScreen( bool alternate ) { + if ( alternate ) { + mPrimaryPlacements = std::move( mPlacements ); + mPlacements.clear(); + } else { + mPlacements = std::move( mPrimaryPlacements ); + mPrimaryPlacements.clear(); + } + ++mPresentationGeneration; + mPresentationDirty = true; +} + +void KittyGraphicsProtocol::scrollScreen( int top, int bottom, int rows, bool preserveHistory ) { + if ( rows == 0 || top > bottom ) + return; + const size_t oldSize = mPlacements.size(); + std::vector scrolledOut; + for ( auto& placement : mPlacements ) { + auto& visible = placement.visible; + const Int64 placementBottom = + static_cast( visible.visibleAnchorCell.y ) + visible.rows; + if ( visible.visibleAnchorCell.y < top || + placementBottom > static_cast( bottom ) + 1 ) + continue; + visible.visibleAnchorCell.y += rows; + if ( preserveHistory ) + continue; + const Uint32 oldRows = visible.rows; + const int sourceTop = visible.sourcePixels.Top; + const int sourceHeight = visible.sourcePixels.Bottom - sourceTop; + const int clipTop = std::max( 0, top - visible.visibleAnchorCell.y ); + const int clipBottom = std::max( 0, visible.visibleAnchorCell.y + + static_cast( visible.rows ) - bottom - 1 ); + if ( clipTop + clipBottom >= static_cast( visible.rows ) ) { + scrolledOut.emplace_back( placement.internalId ); + continue; + } + visible.sourcePixels.Top = + sourceTop + static_cast( sourceHeight ) * clipTop / oldRows; + visible.sourcePixels.Bottom = + sourceTop + static_cast( sourceHeight ) * ( oldRows - clipBottom ) / oldRows; + visible.visibleAnchorCell.y += clipTop; + visible.rows -= clipTop + clipBottom; + if ( clipTop > 0 ) + visible.firstCellPixelOffset.y = 0; + } + if ( !scrolledOut.empty() ) { + mPlacements.erase( std::remove_if( mPlacements.begin(), mPlacements.end(), + [&]( const Placement& p ) { + return std::find( + scrolledOut.begin(), scrolledOut.end(), + p.internalId ) != scrolledOut.end(); + } ), + mPlacements.end() ); + } + if ( oldSize != mPlacements.size() || !mPlacements.empty() ) { + ++mPresentationGeneration; + mPresentationDirty = true; + } +} + +void KittyGraphicsProtocol::setViewport( int scrollPosition, int historyLength, int screenRows ) { + if ( mScrollPosition == scrollPosition && mHistoryLength == historyLength && + mScreenRows == screenRows ) + return; + mScrollPosition = scrollPosition; + mHistoryLength = historyLength; + mScreenRows = screenRows; + const size_t oldSize = mPlacements.size(); + mPlacements.erase( std::remove_if( mPlacements.begin(), mPlacements.end(), + [historyLength]( const Placement& p ) { + return p.visible.visibleAnchorCell.y < 0 && + static_cast( + p.visible.visibleAnchorCell.y ) + + p.visible.rows <= + -historyLength; + } ), + mPlacements.end() ); + if ( oldSize != 0 || oldSize != mPlacements.size() ) { + ++mPresentationGeneration; + mPresentationDirty = true; + } +} + +void KittyGraphicsProtocol::resync() { + mUpdates.clear(); + TerminalGraphicsUpdate reset; + reset.type = TerminalGraphicsUpdateType::ResetAll; + mUpdates.emplace_back( std::move( reset ) ); + for ( const auto& image : mImages ) { + TerminalGraphicsUpdate create; + create.type = TerminalGraphicsUpdateType::CreateImage; + create.imageId = image.first; + create.imageSize = image.second.size; + create.region = Rect( 0, 0, image.second.size.getWidth(), image.second.size.getHeight() ); + create.rgba = std::make_shared>( image.second.rgba ); + mUpdates.emplace_back( std::move( create ) ); + for ( const auto& frame : image.second.frames ) { + TerminalGraphicsUpdate createFrame; + createFrame.type = TerminalGraphicsUpdateType::CreateFrame; + createFrame.imageId = image.first; + createFrame.frameNumber = frame.first; + createFrame.imageSize = image.second.size; + createFrame.region = + Rect( 0, 0, image.second.size.getWidth(), image.second.size.getHeight() ); + createFrame.rgba = std::make_shared>( frame.second.rgba ); + mUpdates.emplace_back( std::move( createFrame ) ); + } + } + mPresentationDirty = true; +} + +}} // namespace eterm::Terminal diff --git a/src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp b/src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp new file mode 100644 index 000000000..d9d91735f --- /dev/null +++ b/src/modules/eterm/src/eterm/terminal/kittygraphicsrenderer.cpp @@ -0,0 +1,167 @@ +#include + +#include +#include +#include +#include + +namespace eterm { namespace Terminal { + +static bool isInPass( Int32 zIndex, KittyGraphicsRenderer::Pass pass ) { + constexpr Int32 VeryNegativeThreshold = std::numeric_limits::min() / 2; + return pass == KittyGraphicsRenderer::Pass::VeryNegative ? zIndex < VeryNegativeThreshold + : pass == KittyGraphicsRenderer::Pass::Negative + ? zIndex < 0 && zIndex >= VeryNegativeThreshold + : zIndex >= 0; +} + +bool KittyGraphicsRenderer::applyUpdates( std::vector&& updates ) { + for ( auto& update : updates ) { + if ( update.type == TerminalGraphicsUpdateType::Resync ) { + reset(); + return false; + } + if ( update.type != TerminalGraphicsUpdateType::ResetAll && + update.sequence != mLastAppliedSequence + 1 ) + return false; + + switch ( update.type ) { + case TerminalGraphicsUpdateType::CreateImage: + case TerminalGraphicsUpdateType::ReplaceImage: { + if ( !update.rgba || update.imageSize.getWidth() <= 0 || + update.imageSize.getHeight() <= 0 ) + return false; + auto texture = TextureFactory::instance()->loadFromPixels( + update.rgba->data(), update.imageSize.getWidth(), update.imageSize.getHeight(), + 4, false, Texture::ClampMode::ClampToEdge, false, false ); + if ( !texture ) + return false; + texture->setFilter( Texture::Filter::Linear ); + mImages.insert_or_assign( update.imageId, + GPUImage{ std::move( texture ), {}, update.imageSize } ); + break; + } + case TerminalGraphicsUpdateType::UpdateRegion: { + auto image = mImages.find( update.imageId ); + if ( image == mImages.end() || !update.rgba ) + return false; + const int width = update.region.Right - update.region.Left; + const int height = update.region.Bottom - update.region.Top; + if ( width <= 0 || height <= 0 || update.region.Left < 0 || update.region.Top < 0 || + update.region.Right > image->second.size.getWidth() || + update.region.Bottom > image->second.size.getHeight() || + update.rgba->size() != static_cast( width ) * height * 4 ) + return false; + image->second.texture->update( update.rgba->data(), width, height, + update.region.Left, update.region.Top ); + break; + } + case TerminalGraphicsUpdateType::CreateFrame: + case TerminalGraphicsUpdateType::ReplaceFrame: { + auto image = mImages.find( update.imageId ); + if ( image == mImages.end() || !update.rgba || update.frameNumber <= 1 || + update.rgba->size() != static_cast( image->second.size.getWidth() ) * + image->second.size.getHeight() * 4 ) + return false; + auto texture = TextureFactory::instance()->loadFromPixels( + update.rgba->data(), image->second.size.getWidth(), + image->second.size.getHeight(), 4, false, Texture::ClampMode::ClampToEdge, + false, false ); + if ( !texture ) + return false; + texture->setFilter( Texture::Filter::Linear ); + image->second.frames.insert_or_assign( update.frameNumber, std::move( texture ) ); + break; + } + case TerminalGraphicsUpdateType::UpdateFrameRegion: { + auto image = mImages.find( update.imageId ); + if ( image == mImages.end() || !update.rgba ) + return false; + auto frame = image->second.frames.find( update.frameNumber ); + if ( frame == image->second.frames.end() ) + return false; + const int width = update.region.Right - update.region.Left; + const int height = update.region.Bottom - update.region.Top; + if ( width <= 0 || height <= 0 || update.region.Left < 0 || update.region.Top < 0 || + update.region.Right > image->second.size.getWidth() || + update.region.Bottom > image->second.size.getHeight() || + update.rgba->size() != static_cast( width ) * height * 4 ) + return false; + frame->second->update( update.rgba->data(), width, height, update.region.Left, + update.region.Top ); + break; + } + case TerminalGraphicsUpdateType::DeleteFrame: { + auto image = mImages.find( update.imageId ); + if ( image != mImages.end() ) + image->second.frames.erase( update.frameNumber ); + break; + } + case TerminalGraphicsUpdateType::DeleteImage: + mImages.erase( update.imageId ); + break; + case TerminalGraphicsUpdateType::ResetScreen: + break; + case TerminalGraphicsUpdateType::ResetAll: + mImages.clear(); + break; + default: + return false; + } + mLastAppliedSequence = update.sequence; + } + return true; +} + +void KittyGraphicsRenderer::setPresentation( + std::shared_ptr presentation ) { + mPresentation = std::move( presentation ); +} + +void KittyGraphicsRenderer::draw( Pass pass, const Vector2f& origin, const Sizef& cellSize, + const Sizef& gridSize ) { + if ( !mPresentation ) + return; + auto clippingMask = Renderer::instance()->getClippingMask(); + clippingMask->clipEnable( origin.x, origin.y, gridSize.getWidth(), gridSize.getHeight() ); + for ( const auto& placement : mPresentation->placements ) { + if ( !isInPass( placement.zIndex, pass ) ) + continue; + auto image = mImages.find( placement.imageId ); + if ( image == mImages.end() || !image->second.texture ) + continue; + const Vector2f position( origin.x + placement.visibleAnchorCell.x * cellSize.getWidth() + + placement.firstCellPixelOffset.x, + origin.y + placement.visibleAnchorCell.y * cellSize.getHeight() + + placement.firstCellPixelOffset.y ); + const Float width = placement.columns * cellSize.getWidth(); + const Float height = placement.rows * cellSize.getHeight(); + TexturePtr texture = image->second.texture; + if ( placement.frameNumber > 1 ) { + auto frame = image->second.frames.find( placement.frameNumber ); + if ( frame != image->second.frames.end() ) + texture = frame->second; + } + texture->drawEx( position.x, position.y, width, height, 0, Vector2f::One, Color::White, + Color::White, Color::White, Color::White, BlendMode::Alpha(), + RENDER_NORMAL, OriginPoint( OriginPoint::OriginTopLeft ), + placement.sourcePixels ); + } + clippingMask->clipDisable(); +} + +bool KittyGraphicsRenderer::hasPlacements( Pass pass ) const { + return mPresentation && + std::any_of( mPresentation->placements.begin(), mPresentation->placements.end(), + [pass]( const TerminalVisiblePlacement& placement ) { + return isInPass( placement.zIndex, pass ); + } ); +} + +void KittyGraphicsRenderer::reset() { + mImages.clear(); + mPresentation.reset(); + mLastAppliedSequence = 0; +} + +}} // namespace eterm::Terminal diff --git a/src/modules/eterm/src/eterm/terminal/pseudoterminal.cpp b/src/modules/eterm/src/eterm/terminal/pseudoterminal.cpp index 1c20ab538..d479c58f6 100644 --- a/src/modules/eterm/src/eterm/terminal/pseudoterminal.cpp +++ b/src/modules/eterm/src/eterm/terminal/pseudoterminal.cpp @@ -107,13 +107,13 @@ bool PseudoTerminal::isTTY() const { return true; } -bool PseudoTerminal::resize( int columns, int rows ) { +bool PseudoTerminal::resize( int columns, int rows, int pixelWidth, int pixelHeight ) { struct winsize w; w.ws_row = rows; w.ws_col = columns; - w.ws_xpixel = 0; - w.ws_ypixel = 0; + w.ws_xpixel = pixelWidth; + w.ws_ypixel = pixelHeight; bool masterResized = ioctl( (int)mMaster, TIOCSWINSZ, &w ) >= 0; bool slaveResized = @@ -297,7 +297,7 @@ bool PseudoTerminal::isTTY() const { return true; } -bool PseudoTerminal::resize( int columns, int rows ) { +bool PseudoTerminal::resize( int columns, int rows, int, int ) { if ( !pResizePseudoConsole ) return false; diff --git a/src/modules/eterm/src/eterm/terminal/terminaldisplay.cpp b/src/modules/eterm/src/eterm/terminal/terminaldisplay.cpp index 9ddda5c33..0fffc3ae1 100644 --- a/src/modules/eterm/src/eterm/terminal/terminaldisplay.cpp +++ b/src/modules/eterm/src/eterm/terminal/terminaldisplay.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -457,6 +458,9 @@ std::shared_ptr TerminalDisplay::create( eeSAFE_DELETE( processFactory ); return nullptr; } + terminal->mSession->resize( termSize.getWidth(), termSize.getHeight(), + terminal->getGridPixelSize().getWidth(), + terminal->getGridPixelSize().getHeight() ); terminal->mSession->setPresentationRate( presentationRateForWindow( window ) ); terminal->mProgram = program; terminal->mArgs = args; @@ -479,6 +483,7 @@ TerminalDisplay::~TerminalDisplay() { TerminalDisplay::TerminalDisplay( EE::Window::Window* window, Font* font, const Float& fontSize, const Sizef& pixelsSize, const bool& useFrameBuffer ) : mWindow( window ), + mGraphicsRenderer( std::make_unique() ), mFont( font ), mFontSize( fontSize ), mSize( pixelsSize ), @@ -493,6 +498,8 @@ TerminalDisplay::TerminalDisplay( EE::Window::Window* window, Font* font, const resetColors(); Sizei gridSize( gridSizeFromTermDimensions( mFont, mFontSize, mSize - mPadding * 2.f ) ); + mColumns = gridSize.getWidth(); + mRows = gridSize.getHeight(); mDirtyLines.resize( gridSize.getHeight(), 1 ); mQuadVertex = GLi->quadVertex(); @@ -715,7 +722,9 @@ void TerminalDisplay::setKeepAlive( bool keepAlive ) { } bool TerminalDisplay::update( bool isMouseOverMe ) { + drainGraphicsUpdates(); consumeSnapshot(); + drainGraphicsUpdates(); drainSessionEvents(); if ( mFocus && isBlinkingCursor() && mClock.getElapsedTime().asSeconds() > 0.7 ) { mMode ^= MODE_BLINK; @@ -734,6 +743,29 @@ bool TerminalDisplay::update( bool isMouseOverMe ) { return true; } +void TerminalDisplay::drainGraphicsUpdates() { + if ( !mSession ) + return; + auto updates = mSession->drainGraphicsUpdates(); + if ( !updates.empty() ) { + if ( !mGraphicsRenderer->applyUpdates( std::move( updates ) ) ) { + mGraphicsRenderer->reset(); + mLastAppliedGraphicsSequence = 0; + if ( !mGraphicsResyncPending ) { + mGraphicsResyncPending = true; + mSession->requestGraphicsResync(); + } + return; + } + mLastAppliedGraphicsSequence = mGraphicsRenderer->lastAppliedSequence(); + mGraphicsResyncPending = false; + if ( mSnapshot && mSnapshot->graphics && + mSnapshot->graphics->requiredUpdateSequence <= mLastAppliedGraphicsSequence ) + mGraphicsRenderer->setPresentation( mSnapshot->graphics ); + mDirty = true; + } +} + void TerminalDisplay::consumeSnapshot() { if ( !mSession ) return; @@ -768,6 +800,13 @@ void TerminalDisplay::consumeSnapshot() { mCursor = mSnapshot->cursor; mCursorGlyph = mSnapshot->cursorGlyph; mCursorMode = mSnapshot->cursorMode; + if ( mSnapshot->graphics && + mSnapshot->graphics->requiredUpdateSequence <= mGraphicsRenderer->lastAppliedSequence() ) { + mGraphicsRenderer->setPresentation( mSnapshot->graphics ); + } else if ( mSnapshot->graphics && !mGraphicsResyncPending ) { + mGraphicsResyncPending = true; + mSession->requestGraphicsResync(); + } const int presentationBits = mMode & MODE_BLINK; mMode = mSnapshot->windowMode | presentationBits; if ( mFocus ) @@ -1093,8 +1132,8 @@ void TerminalDisplay::onMouseMove( const Vector2i& pos, const Uint32& flags ) { mWindow->getInput()->getModState() & KEYMOD_SHIFT ? SEL_RECTANGULAR : SEL_REGULAR, false ); } - mSession->mouseReport( TerminalMouseEventType::MouseMotion, positionToGrid( pos ), flags, - mWindow->getInput()->getModState() ); + mSession->mouseReport( TerminalMouseEventType::MouseMotion, positionToGrid( pos ), + positionToPixel( pos ), flags, mWindow->getInput()->getModState() ); } void TerminalDisplay::onMouseDown( const Vector2i& pos, const Uint32& flags ) { @@ -1134,8 +1173,8 @@ void TerminalDisplay::onMouseDown( const Vector2i& pos, const Uint32& flags ) { } } - mSession->mouseReport( TerminalMouseEventType::MouseButtonDown, positionToGrid( pos ), flags, - mWindow->getInput()->getModState() ); + mSession->mouseReport( TerminalMouseEventType::MouseButtonDown, positionToGrid( pos ), + positionToPixel( pos ), flags, mWindow->getInput()->getModState() ); } void TerminalDisplay::onMouseUp( const Vector2i& pos, const Uint32& flags ) { @@ -1181,8 +1220,8 @@ void TerminalDisplay::onMouseUp( const Vector2i& pos, const Uint32& flags ) { } } - mSession->mouseReport( TerminalMouseEventType::MouseButtonRelease, positionToGrid( pos ), flags, - mWindow->getInput()->getModState() ); + mSession->mouseReport( TerminalMouseEventType::MouseButtonRelease, positionToGrid( pos ), + positionToPixel( pos ), flags, mWindow->getInput()->getModState() ); } static inline Color termColor( unsigned int terminalColor, const std::vector& colors ) { @@ -1368,6 +1407,7 @@ void TerminalDisplay::drawGrid( const Vector2f& pos ) { auto fontSize = mFont->getFontHeight( mFontSize ); auto spaceCharAdvanceX = mFont->getGlyph( 'A', mFontSize, false, false ).advance; + const Sizef cellSize( spaceCharAdvanceX, fontSize ); float x = 0.0f; float y = pos.y; @@ -1446,6 +1486,38 @@ void TerminalDisplay::drawGrid( const Vector2f& pos ) { mVBBackground->draw(); mVBBackground->unbind(); } + const Sizef graphicsGridSize( mColumns * cellSize.getWidth(), mRows * cellSize.getHeight() ); + mGraphicsRenderer->draw( KittyGraphicsRenderer::Pass::VeryNegative, pos, cellSize, + graphicsGridSize ); + if ( mGraphicsRenderer->hasPlacements( KittyGraphicsRenderer::Pass::VeryNegative ) ) { + y = std::floor( pos.y ); + for ( Uint32 row = 0; row < mRows; ++row ) { + x = std::floor( pos.x ); + for ( Uint32 column = 0; column < mColumns; ++column ) { + const auto& glyph = mSnapshot->cells[row * mColumns + column]; + if ( glyph.mode & ATTR_WDUMMY ) + continue; + auto foreground = termColor( glyph.fg, mColors ); + auto background = termColor( glyph.bg, mColors ); + if ( IS_SET( MODE_REVERSE ) ) { + foreground = foreground == defaultFg ? defaultBg : foreground.invert(); + background = background == defaultBg ? defaultFg : background.invert(); + } + if ( glyph.mode & ATTR_REVERSE ) + background = foreground; + const bool wide = glyph.mode & ATTR_WIDE; + const Float advance = spaceCharAdvanceX * ( wide ? 2.0f : 1.0f ); + if ( background != defaultBg ) { + mPrimitives.setColor( background ); + mPrimitives.drawRectangle( Rectf( { x, y }, { advance, lineHeight } ) ); + } + x += advance; + } + y += lineHeight; + } + } + mGraphicsRenderer->draw( KittyGraphicsRenderer::Pass::Negative, pos, cellSize, + graphicsGridSize ); y = std::floor( pos.y ); @@ -1683,6 +1755,8 @@ void TerminalDisplay::drawGrid( const Vector2f& pos ) { vbo->unbind(); } } + mGraphicsRenderer->draw( KittyGraphicsRenderer::Pass::NonNegative, pos, cellSize, + graphicsGridSize ); // Underline is rendered after foreground render because it usually clashes with the underlines // decorations and ends up being not visible, I prefer to do this even it it's not standard. @@ -1777,6 +1851,27 @@ Vector2i TerminalDisplay::positionToGrid( const Vector2i& pos ) { return { mouseX, mouseY }; } +Vector2i TerminalDisplay::positionToPixel( const Vector2i& pos ) const { + const Sizei gridPixels = getGridPixelSize(); + const int x = static_cast( std::floor( pos.x - mPosition.x - mPadding.Left ) ); + const int y = static_cast( std::floor( pos.y - mPosition.y - mPadding.Top ) ); + return { eeclamp( x, 0, eemax( 0, gridPixels.getWidth() - 1 ) ), + eeclamp( y, 0, eemax( 0, gridPixels.getHeight() - 1 ) ) }; +} + +Sizei TerminalDisplay::getCellPixelSize() const { + return { + static_cast( std::round( mFont->getGlyph( 'A', mFontSize, false, false ).advance ) ), + static_cast( std::round( mFont->getFontHeight( mFontSize ) ) ) }; +} + +Sizei TerminalDisplay::getGridPixelSize() const { + const Sizei cell = getCellPixelSize(); + const int columns = mSnapshot ? mSnapshot->columns : static_cast( mColumns ); + const int rows = mSnapshot ? mSnapshot->rows : static_cast( mRows ); + return { columns * cell.getWidth(), rows * cell.getHeight() }; +} + void TerminalDisplay::onSizeChange() { Sizei gridSize( gridSizeFromTermDimensions( mFont, mFontSize, @@ -1784,7 +1879,10 @@ void TerminalDisplay::onSizeChange() { if ( mSession && ( !mSnapshot || gridSize.getWidth() != mSnapshot->columns || gridSize.getHeight() != mSnapshot->rows ) ) { - mSession->resize( gridSize.getWidth(), gridSize.getHeight() ); + const Sizei cellSize = getCellPixelSize(); + mSession->resize( gridSize.getWidth(), gridSize.getHeight(), + gridSize.getWidth() * cellSize.getWidth(), + gridSize.getHeight() * cellSize.getHeight() ); mDirtyLines.resize( gridSize.getHeight(), 1 ); } diff --git a/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp b/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp index aaed40aa1..ad0295896 100644 --- a/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp +++ b/src/modules/eterm/src/eterm/terminal/terminalemulator.cpp @@ -211,6 +211,46 @@ static const unsigned int tabspaces = 4; #define TRUECOLOR( r, g, b ) ( 1 << 24 | ( r ) << 16 | ( g ) << 8 | ( b ) ) #define IS_TRUECOL( x ) ( 1 << 24 & ( x ) ) +static int kittyDiacriticIndex( Rune value ) { + static constexpr Rune values[] = { + 0x0305, 0x030D, 0x030E, 0x0310, 0x0312, 0x033D, 0x033E, 0x033F, 0x0346, 0x034A, + 0x034B, 0x034C, 0x0350, 0x0351, 0x0352, 0x0357, 0x035B, 0x0363, 0x0364, 0x0365, + 0x0366, 0x0367, 0x0368, 0x0369, 0x036A, 0x036B, 0x036C, 0x036D, 0x036E, 0x036F, + 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, 0x0592, 0x0593, 0x0594, 0x0595, 0x0597, + 0x0598, 0x0599, 0x059C, 0x059D, 0x059E, 0x059F, 0x05A0, 0x05A1, 0x05A8, 0x05A9, + 0x05AB, 0x05AC, 0x05AF, 0x05C4, 0x0610, 0x0611, 0x0612, 0x0613, 0x0614, 0x0615, + 0x0616, 0x0617, 0x0657, 0x0658, 0x0659, 0x065A, 0x065B, 0x065D, 0x065E, 0x06D6, + 0x06D7, 0x06D8, 0x06D9, 0x06DA, 0x06DB, 0x06DC, 0x06DF, 0x06E0, 0x06E1, 0x06E2, + 0x06E4, 0x06E7, 0x06E8, 0x06EB, 0x06EC, 0x0730, 0x0732, 0x0733, 0x0735, 0x0736, + 0x073A, 0x073D, 0x073F, 0x0740, 0x0741, 0x0743, 0x0745, 0x0747, 0x0749, 0x074A, + 0x07EB, 0x07EC, 0x07ED, 0x07EE, 0x07EF, 0x07F0, 0x07F1, 0x07F3, 0x0816, 0x0817, + 0x0818, 0x0819, 0x081B, 0x081C, 0x081D, 0x081E, 0x081F, 0x0820, 0x0821, 0x0822, + 0x0823, 0x0825, 0x0826, 0x0827, 0x0829, 0x082A, 0x082B, 0x082C, 0x082D, 0x0951, + 0x0953, 0x0954, 0x0F82, 0x0F83, 0x0F86, 0x0F87, 0x135D, 0x135E, 0x135F, 0x17DD, + 0x193A, 0x1A17, 0x1A75, 0x1A76, 0x1A77, 0x1A78, 0x1A79, 0x1A7A, 0x1A7B, 0x1A7C, + 0x1B6B, 0x1B6D, 0x1B6E, 0x1B6F, 0x1B70, 0x1B71, 0x1B72, 0x1B73, 0x1CD0, 0x1CD1, + 0x1CD2, 0x1CDA, 0x1CDB, 0x1CE0, 0x1DC0, 0x1DC1, 0x1DC3, 0x1DC4, 0x1DC5, 0x1DC6, + 0x1DC7, 0x1DC8, 0x1DC9, 0x1DCB, 0x1DCC, 0x1DD1, 0x1DD2, 0x1DD3, 0x1DD4, 0x1DD5, + 0x1DD6, 0x1DD7, 0x1DD8, 0x1DD9, 0x1DDA, 0x1DDB, 0x1DDC, 0x1DDD, 0x1DDE, 0x1DDF, + 0x1DE0, 0x1DE1, 0x1DE2, 0x1DE3, 0x1DE4, 0x1DE5, 0x1DE6, 0x1DFE, 0x20D0, 0x20D1, + 0x20D4, 0x20D5, 0x20D6, 0x20D7, 0x20DB, 0x20DC, 0x20E1, 0x20E7, 0x20E9, 0x20F0, + 0x2CEF, 0x2CF0, 0x2CF1, 0x2DE0, 0x2DE1, 0x2DE2, 0x2DE3, 0x2DE4, 0x2DE5, 0x2DE6, + 0x2DE7, 0x2DE8, 0x2DE9, 0x2DEA, 0x2DEB, 0x2DEC, 0x2DED, 0x2DEE, 0x2DEF, 0x2DF0, + 0x2DF1, 0x2DF2, 0x2DF3, 0x2DF4, 0x2DF5, 0x2DF6, 0x2DF7, 0x2DF8, 0x2DF9, 0x2DFA, + 0x2DFB, 0x2DFC, 0x2DFD, 0x2DFE, 0x2DFF, 0xA66F, 0xA67C, 0xA67D, 0xA6F0, 0xA6F1, + 0xA8E0, 0xA8E1, 0xA8E2, 0xA8E3, 0xA8E4, 0xA8E5, 0xA8E6, 0xA8E7, 0xA8E8, 0xA8E9, + 0xA8EA, 0xA8EB, 0xA8EC, 0xA8ED, 0xA8EE, 0xA8EF, 0xA8F0, 0xA8F1, 0xAAB0, 0xAAB2, + 0xAAB3, 0xAAB7, 0xAAB8, 0xAABE, 0xAABF, 0xAAC1, 0xFE20, 0xFE21, 0xFE22, 0xFE23, + 0xFE24, 0xFE25, 0xFE26, 0x10A0F, 0x10A38, 0x1D185, 0x1D186, 0x1D187, 0x1D188, 0x1D189, + 0x1D1AA, 0x1D1AB, 0x1D1AC, 0x1D1AD, 0x1D242, 0x1D243, 0x1D244, + }; + for ( size_t i = 0; i < sizeof( values ) / sizeof( values[0] ); ++i ) { + if ( values[i] == value ) + return static_cast( i ); + } + return -1; +} + /* Arbitrary sizes */ #define UTF_INVALID 0xFFFD #define UTF_SIZ 4 @@ -867,8 +907,13 @@ void TerminalEmulator::trimMemory() { } void TerminalEmulator::clearHistory() { - for ( int i = 0; i < mTerm.histcursize; ++i ) + for ( int i = 0; i < mTerm.histcursize; ++i ) { + if ( mTerm.hist[i] ) { + for ( int column = 0; column < mTerm.col; ++column ) + mKittyPlaceholderMetadata.erase( &mTerm.hist[i][column] ); + } eeSAFE_FREE( mTerm.hist[i] ); + } eeSAFE_FREE( mTerm.hist ); mTerm.histcursize = 0; mTerm.histi = 0; @@ -906,6 +951,12 @@ void TerminalEmulator::notifyColorSchemeChanged() { mColorScheme = scheme; } +void TerminalEmulator::requestGraphicsResync() { + mKittyGraphics.resync(); + mDirty = true; + draw(); +} + Vector2i TerminalEmulator::getSize() const { return { mTerm.col, mTerm.row }; } @@ -1048,8 +1099,9 @@ void TerminalEmulator::treset( void ) { tswapscreen(); } - xsetmode( 0, MODE_MOUSE | MODE_MOUSESGR | MODE_APPKEYPAD | MODE_APPCURSOR | MODE_FOCUS | - MODE_BRCKTPASTE | MODE_MOUSEX10 | MODE_MOUSEMANY ); + xsetmode( 0, MODE_MOUSE | MODE_MOUSESGR | MODE_MOUSESGR_PIXELS | MODE_APPKEYPAD | + MODE_APPCURSOR | MODE_FOCUS | MODE_BRCKTPASTE | MODE_MOUSEX10 | + MODE_MOUSEMANY ); // Preserve eterm's established behavior and xterm's default alternateScroll resource. xsetmode( 1, MODE_ALTSCRROLL ); auto dpy = mDpy.lock(); @@ -1098,6 +1150,7 @@ void TerminalEmulator::tscrolldown( int top, int n ) { if ( mTerm.scr == 0 ) selscroll( top, n ); + mKittyGraphics.scrollScreen( top, mTerm.bot, n, false ); } void TerminalEmulator::tscrollup( int top, int n, int copyhist ) { @@ -1151,6 +1204,9 @@ void TerminalEmulator::tscrollup( int top, int n, int copyhist ) { if ( mTerm.scr == 0 ) selscroll( top, -n ); + mKittyGraphics.scrollScreen( top, mTerm.bot, -n, + copyhist && mTerm.histsize > 0 && !IS_SET( MODE_ALTSCREEN ) && + top == mTerm.top ); onScrollPositionChange(); } @@ -1559,6 +1615,7 @@ void TerminalEmulator::tsetchar( Rune u, TerminalGlyph* attr, int x, int y ) { TLINE( y )[x - 1].mode &= ~ATTR_WIDE; } + mKittyPlaceholderMetadata.erase( &TLINE( y )[x] ); mTerm.dirty[y] = 1; TLINE( y )[x] = *attr; TLINE( y )[x].u = u; @@ -1576,11 +1633,14 @@ void TerminalEmulator::tclearregion( int x1, int y1, int x2, int y2, bool skip_c temp = x1, x1 = x2, x2 = temp; if ( y1 > y2 ) temp = y1, y1 = y2, y2 = temp; - LIMIT( x1, 0, mTerm.col - 1 ); LIMIT( x2, 0, mTerm.col - 1 ); LIMIT( y1, 0, mTerm.row - 1 ); LIMIT( y2, 0, mTerm.row - 1 ); + for ( int clearY = y1; clearY <= y2; ++clearY ) { + for ( int clearX = x1; clearX <= x2; ++clearX ) + mKittyPlaceholderMetadata.erase( &TLINE( clearY )[clearX] ); + } /* * Fast path for the common full-row clear performed while scrolling: no @@ -1753,6 +1813,7 @@ void TerminalEmulator::tsetattr( int* attr, int l, const char* separators ) { ATTR_BLINK | ATTR_REVERSE | ATTR_INVISIBLE | ATTR_STRUCK ); mTerm.c.attr.fg = mDefaultFg; mTerm.c.attr.bg = mDefaultBg; + mKittyUnderlineColor = 0; break; case 1: mTerm.c.attr.mode |= ATTR_BOLD; @@ -1816,12 +1877,11 @@ void TerminalEmulator::tsetattr( int* attr, int l, const char* separators ) { mTerm.c.attr.bg = mDefaultBg; break; case 58: - /* This starts a sequence to change the color of - * "underline" pixels. We don't support that and - * instead eat up a following "5;n" or "2;r;g;b". */ - tdefcolor( attr, separators, &i, l ); + if ( ( idx = tdefcolor( attr, separators, &i, l ) ) >= 0 ) + mKittyUnderlineColor = idx; break; - case 59: /* reset underline color (unsupported, therefore a no-op) */ + case 59: + mKittyUnderlineColor = 0; break; default: if ( BETWEEN( attr[i], 30, 37 ) ) { @@ -1916,6 +1976,11 @@ void TerminalEmulator::tsetmode( int priv, int set, int* args, int narg ) { case 1006: /* 1006: extended reporting mode */ xsetmode( set, MODE_MOUSESGR ); break; + case 1016: /* 1016: extended reporting in terminal-grid pixels */ + xsetmode( set, MODE_MOUSESGR_PIXELS ); + if ( set ) + xsetmode( 1, MODE_MOUSESGR ); + break; case 1007: /* wheel sends cursor keys on the alternate screen */ xsetmode( set, MODE_ALTSCRROLL ); break; @@ -1936,7 +2001,10 @@ void TerminalEmulator::tsetmode( int priv, int set, int* args, int narg ) { tclearregion( 0, 0, mTerm.col - 1, mTerm.row - 1 ); } if ( set ^ alt ) /* set is always 1 or 0 */ + { tswapscreen(); + mKittyGraphics.setAlternateScreen( set != 0 ); + } if ( *args != 1049 ) break; /* FALLTHROUGH */ @@ -2153,6 +2221,7 @@ void TerminalEmulator::csihandle( void ) { // fallthrough case 2: /* all */ tclearregion( 0, 0, mTerm.col - 1, mTerm.row - 1 ); + mKittyGraphics.clearScreen(); break; default: goto unknown; @@ -2278,6 +2347,18 @@ void TerminalEmulator::csihandle( void ) { break; case 't': /* Window manipulation */ switch ( mCsiescseq.arg[0] ) { + case 14: /* Report terminal grid size in pixels. */ + len = + snprintf( buf, sizeof( buf ), "\033[4;%d;%dt", mPixelHeight, mPixelWidth ); + ttywrite( buf, len, 0 ); + break; + case 16: { /* Report terminal cell size in pixels. */ + const int cellWidth = mTerm.col > 0 ? mPixelWidth / mTerm.col : 0; + const int cellHeight = mTerm.row > 0 ? mPixelHeight / mTerm.row : 0; + len = snprintf( buf, sizeof( buf ), "\033[6;%d;%dt", cellHeight, cellWidth ); + ttywrite( buf, len, 0 ); + break; + } case 22: /* Save window title */ // Push current title to title stack mTerm.title_stack.push_back( mTerm.title ); @@ -2335,6 +2416,20 @@ void TerminalEmulator::strhandle( void ) { int j, narg, par; mTerm.esc &= ~( ESC_STR_END | ESC_STR ); + if ( mStrescseq.discarded ) + return; + if ( mStrescseq.type == '_' && mStrescseq.len > 0 && mStrescseq.buf[0] == 'G' ) { + auto result = + mKittyGraphics.handle( std::string_view{ mStrescseq.buf + 1, mStrescseq.len - 1 }, + Vector2i( mTerm.c.x, mTerm.c.y ) ); + if ( !result.response.empty() ) + write( result.response.data(), result.response.size() ); + if ( result.changed ) + mDirty = true; + if ( result.cursorMovement != Vector2i::Zero ) + tmoveto( mTerm.c.x + result.cursorMovement.x, mTerm.c.y + result.cursorMovement.y ); + return; + } strparse(); par = ( narg = mStrescseq.narg ) ? atoi( mStrescseq.args[0] ) : 0; @@ -2851,6 +2946,7 @@ int TerminalEmulator::eschandle( uchar ascii ) { ttywrite( vtiden, strlen( vtiden ), 0 ); break; case 'c': /* RIS -- Reset to initial state */ + mKittyGraphics.reset(); treset(); resettitle(); loadColors(); @@ -2979,23 +3075,28 @@ void TerminalEmulator::tputc( Rune u ) { goto check_control_code; } + if ( mStrescseq.discarded ) + return; + + const bool kittyGraphics = + mStrescseq.type == '_' && mStrescseq.len > 0 && mStrescseq.buf[0] == 'G'; + const size_t sequenceLimit = + kittyGraphics ? MAX_KITTY_GRAPHICS_APC_SIZE : MAX_GENERIC_STRING_SEQUENCE_SIZE; + if ( len > sequenceLimit - mStrescseq.len ) { + mStrescseq.discarded = true; + return; + } + if ( mStrescseq.len + len >= mStrescseq.siz ) { - /* - * Here is a bug in terminals. If the user never sends - * some code to stop the str or esc command, then st - * will stop responding. But this is better than - * silently failing with unknown characters. At least - * then users will report back. - * - * In the case users ever get fixed, here is the code: - */ - /* - * term.esc = 0; - * strhandle(); - */ - if ( mStrescseq.siz > ( SIZE_MAX - UTF_SIZ ) / 2 ) + const size_t required = mStrescseq.len + len + 1; + size_t newSize = mStrescseq.siz; + while ( newSize < required && newSize < sequenceLimit ) + newSize = eemin( newSize * 2, sequenceLimit ); + if ( newSize < required ) { + mStrescseq.discarded = true; return; - mStrescseq.siz *= 2; + } + mStrescseq.siz = newSize; mStrescseq.buf = (char*)xrealloc( mStrescseq.buf, mStrescseq.siz ); } @@ -3045,6 +3146,35 @@ check_control_code: */ return; } + if ( mKittyPlaceholderCell.x >= 0 ) { + const int diacritic = kittyDiacriticIndex( u ); + if ( diacritic >= 0 ) { + auto* placeholder = &mTerm.line[mKittyPlaceholderCell.y][mKittyPlaceholderCell.x]; + auto metadata = mKittyPlaceholderMetadata.find( placeholder ); + if ( metadata == mKittyPlaceholderMetadata.end() ) { + mKittyPlaceholderCell = Vector2i( -1, -1 ); + return; + } + switch ( metadata->second.diacriticCount++ ) { + case 0: + metadata->second.row = static_cast( diacritic ); + break; + case 1: + metadata->second.column = static_cast( diacritic ); + break; + case 2: + if ( diacritic <= 255 ) + metadata->second.imageIdMsb = static_cast( diacritic ); + break; + default: + break; + } + mTerm.dirty[mKittyPlaceholderCell.y] = 1; + mDirty = true; + return; + } + mKittyPlaceholderCell = Vector2i( -1, -1 ); + } if ( selected( mTerm.c.x, mTerm.c.y ) ) selclear(); @@ -3065,6 +3195,15 @@ check_control_code: } tsetchar( u, &mTerm.c.attr, mTerm.c.x, mTerm.c.y ); + if ( u == 0x10EEEE ) { + mKittyPlaceholderCell = Vector2i( mTerm.c.x, mTerm.c.y ); + const Uint32 placementId = IS_TRUECOL( mKittyUnderlineColor ) + ? mKittyUnderlineColor & 0xFFFFFF + : mKittyUnderlineColor <= 255 ? mKittyUnderlineColor + : 0; + mKittyPlaceholderMetadata[&mTerm.line[mTerm.c.y][mTerm.c.x]] = + KittyPlaceholderMetadata{ placementId }; + } mTerm.lastc = u; if ( width == 2 ) { @@ -3123,6 +3262,36 @@ void TerminalEmulator::tresize( int col, int row ) { int save_end = 0; int loaded = 0; bool is_alt = IS_SET( MODE_ALTSCREEN ); + std::vector primaryPlaceholderMetadata; + std::vector alternatePlaceholderMetadata; + auto collectPlaceholderMetadata = [&]( Line line, int columns, + std::vector& output ) { + if ( !line ) + return; + for ( int column = 0; column < columns; ++column ) { + if ( line[column].u != 0x10EEEE ) + continue; + auto metadata = mKittyPlaceholderMetadata.find( &line[column] ); + if ( metadata != mKittyPlaceholderMetadata.end() ) + output.emplace_back( metadata->second ); + } + }; + if ( mTerm.col > 0 ) { + for ( int history = 0; history < mTerm.histlen; ++history ) { + const int index = + ( mTerm.histi - mTerm.histlen + 1 + history + mTerm.histsize ) % mTerm.histsize; + collectPlaceholderMetadata( mTerm.hist[index], mTerm.col, primaryPlaceholderMetadata ); + } + Line* primaryLines = is_alt ? mTerm.alt : mTerm.line; + Line* alternateLines = is_alt ? mTerm.line : mTerm.alt; + for ( int line = 0; line < mTerm.row; ++line ) { + collectPlaceholderMetadata( primaryLines[line], mTerm.col, primaryPlaceholderMetadata ); + collectPlaceholderMetadata( alternateLines[line], mTerm.col, + alternatePlaceholderMetadata ); + } + } + mKittyPlaceholderMetadata.clear(); + mKittyPlaceholderCell = Vector2i( -1, -1 ); if ( col < 1 || row < 1 ) { terminalDiagnostic( "tresize: error resizing to %dx%d\n", col, row ); @@ -3304,6 +3473,32 @@ void TerminalEmulator::tresize( int col, int row ) { eemax( mTerm.scr - mTerm.histlen, eemin( mTerm.scr + mTerm.row - 1, mSel.oe.y ) ); selnormalize(); } + auto restorePlaceholderMetadata = [&]( Line line, int columns, + const std::vector& metadata, + size_t& index ) { + if ( !line ) + return; + for ( int column = 0; column < columns && index < metadata.size(); ++column ) { + if ( line[column].u == 0x10EEEE ) + mKittyPlaceholderMetadata[&line[column]] = metadata[index++]; + } + }; + size_t primaryMetadataIndex = 0; + for ( int history = 0; history < mTerm.histlen; ++history ) { + const int index = + ( mTerm.histi - mTerm.histlen + 1 + history + mTerm.histsize ) % mTerm.histsize; + restorePlaceholderMetadata( mTerm.hist[index], mTerm.col, primaryPlaceholderMetadata, + primaryMetadataIndex ); + } + Line* primaryLines = is_alt ? mTerm.alt : mTerm.line; + Line* alternateLines = is_alt ? mTerm.line : mTerm.alt; + size_t alternateMetadataIndex = 0; + for ( int line = 0; line < mTerm.row; ++line ) { + restorePlaceholderMetadata( primaryLines[line], mTerm.col, primaryPlaceholderMetadata, + primaryMetadataIndex ); + restorePlaceholderMetadata( alternateLines[line], mTerm.col, alternatePlaceholderMetadata, + alternateMetadataIndex ); + } mDirty = true; onScrollPositionChange(); @@ -3357,6 +3552,63 @@ void TerminalEmulator::draw() { cx--; drawregion( *dpy, 0, 0, mTerm.col, mTerm.row ); + std::vector placeholderCells; + const bool scanPlaceholders = mKittyGraphics.hasVirtualPlacements(); + for ( int y = 0; scanPlaceholders && y < mTerm.row; ++y ) { + const TerminalGlyph* previous = nullptr; + Uint32 previousPlacementId = 0; + Uint32 previousRow = 0; + Uint32 previousColumn = 0; + Uint8 previousMsb = 0; + for ( int x = 0; x < mTerm.col; ++x ) { + const auto& glyph = TLINE( y )[x]; + if ( glyph.u != 0x10EEEE ) { + previous = nullptr; + continue; + } + auto metadata = mKittyPlaceholderMetadata.find( &glyph ); + if ( metadata == mKittyPlaceholderMetadata.end() ) { + previous = nullptr; + continue; + } + Uint32 row = metadata->second.row; + Uint32 column = metadata->second.column; + Uint8 msb = metadata->second.imageIdMsb; + const Uint32 placementId = metadata->second.placementId; + const bool sameColors = + previous && previous->fg == glyph.fg && previousPlacementId == placementId; + if ( metadata->second.diacriticCount == 0 && sameColors ) { + row = previousRow; + column = previousColumn + 1; + msb = previousMsb; + } else if ( metadata->second.diacriticCount == 1 && sameColors && + row == previousRow ) { + column = previousColumn + 1; + msb = previousMsb; + } else if ( metadata->second.diacriticCount == 2 && sameColors && + row == previousRow && column == previousColumn + 1 ) { + msb = previousMsb; + } + if ( row == UINT16_MAX || column == UINT16_MAX ) { + previous = nullptr; + continue; + } + const Uint32 lowImageId = IS_TRUECOL( glyph.fg ) ? glyph.fg & 0xFFFFFF + : glyph.fg <= 255 ? glyph.fg + : 0; + placeholderCells.push_back( { lowImageId | ( static_cast( msb ) << 24 ), + placementId, Vector2i( x, y ), row, column } ); + previous = &glyph; + previousRow = row; + previousColumn = column; + previousMsb = msb; + previousPlacementId = placementId; + } + } + mKittyGraphics.setPlaceholderCells( std::move( placeholderCells ) ); + auto graphicsUpdates = mKittyGraphics.takeUpdates(); + if ( !graphicsUpdates.empty() || mKittyGraphics.hasPendingPresentation() ) + dpy->drawGraphics( mKittyGraphics.takePresentation(), std::move( graphicsUpdates ) ); if ( mTerm.scr == 0 ) dpy->drawCursor( cx, mTerm.c.y, mTerm.line[mTerm.c.y][cx], mTerm.ocx, mTerm.ocy, @@ -3379,6 +3631,7 @@ void TerminalEmulator::redraw() { } void TerminalEmulator::reset() { + mKittyGraphics.reset(); treset(); redraw(); } @@ -3448,8 +3701,10 @@ void TerminalEmulator::reportColorScheme() { ttywrite( buf, len, 0 ); } -void TerminalEmulator::mousereport( const TerminalMouseEventType& type, const Vector2i& pos, +void TerminalEmulator::mousereport( const TerminalMouseEventType& type, + const Vector2i& cellPosition, const Vector2i& pixelPosition, const Uint32& flags, const Uint32& mod ) { + const Vector2i& pos = xgetmode( MODE_MOUSESGR_PIXELS ) ? pixelPosition : cellPosition; if ( !xgetmode( (TerminalWinMode)MODE_MOUSE ) && !xgetmode( MODE_MOUSESGR ) && ( TerminalMouseEventType::MouseButtonDown == type || TerminalMouseEventType::MouseButtonRelease == type ) ) { @@ -3560,6 +3815,10 @@ void TerminalEmulator::mousereport( const TerminalMouseEventType& type, const Ve } void TerminalEmulator::setPtyAndProcess( PtyPtr&& pty, ProcPtr&& process ) { + mKittyGraphics.reset(); + mKittyPlaceholderMetadata.clear(); + mKittyPlaceholderCell = Vector2i( -1, -1 ); + mBuflen = 0; mStatus = STARTING; mExitCode = 1; mPty = std::move( pty ); @@ -3654,6 +3913,7 @@ void TerminalEmulator::onProcessExit( int exitCode ) { } void TerminalEmulator::onScrollPositionChange() { + mKittyGraphics.setViewport( mTerm.scr, mTerm.histlen, mTerm.row ); auto dpy = mDpy.lock(); if ( dpy ) dpy->onScrollPositionChange(); @@ -3704,11 +3964,19 @@ int TerminalEmulator::write( const char* buf, size_t buflen ) { } void TerminalEmulator::resize( int columns, int rows ) { + resize( columns, rows, mPixelWidth, mPixelHeight ); +} + +void TerminalEmulator::resize( int columns, int rows, int pixelWidth, int pixelHeight ) { + mPixelWidth = eemax( 0, pixelWidth ); + mPixelHeight = eemax( 0, pixelHeight ); + mKittyGraphics.setCellPixelSize( columns > 0 ? mPixelWidth / columns : 0, + rows > 0 ? mPixelHeight / rows : 0 ); bool is_alt = IS_SET( MODE_ALTSCREEN ); // Alt doesn't need reflow, we can resize and redraw instantly which looks and feels better if ( is_alt ) { - if ( !mPty->resize( columns, rows ) ) { + if ( !mPty->resize( columns, rows, mPixelWidth, mPixelHeight ) ) { _die( "Failed to resize pty!" ); return; } @@ -3722,6 +3990,8 @@ void TerminalEmulator::resize( int columns, int rows ) { redraw(); mPendingPtyColumns = columns; mPendingPtyRows = rows; + mPendingPtyPixelWidth = mPixelWidth; + mPendingPtyPixelHeight = mPixelHeight; mPendingPtyResize = true; mPendingPtyResizeClock.restart(); } @@ -3729,10 +3999,13 @@ void TerminalEmulator::resize( int columns, int rows ) { #define MAX_TTY_READS ( 1024 ) bool TerminalEmulator::update() { + if ( mKittyGraphics.updateAnimations() ) + mDirty = true; if ( mPendingPtyResize && mPendingPtyResizeClock.getElapsedTime() >= Milliseconds( 100 ) ) { mPendingPtyResize = false; - if ( !mPty->resize( mPendingPtyColumns, mPendingPtyRows ) ) { + if ( !mPty->resize( mPendingPtyColumns, mPendingPtyRows, mPendingPtyPixelWidth, + mPendingPtyPixelHeight ) ) { _die( "Failed to resize pty!" ); } diff --git a/src/modules/eterm/src/eterm/terminal/terminalgraphics.cpp b/src/modules/eterm/src/eterm/terminal/terminalgraphics.cpp new file mode 100644 index 000000000..39d5bc6ef --- /dev/null +++ b/src/modules/eterm/src/eterm/terminal/terminalgraphics.cpp @@ -0,0 +1,77 @@ +#include + +namespace eterm { namespace Terminal { + +TerminalGraphicsUpdateQueue::TerminalGraphicsUpdateQueue( size_t maxUpdates, size_t maxBytes ) : + mMaxUpdates( maxUpdates ), mMaxBytes( maxBytes ) {} + +Uint64 TerminalGraphicsUpdateQueue::enqueue( TerminalGraphicsUpdate update ) { + std::lock_guard lock( mMutex ); + const size_t payloadBytes = update.payloadBytes(); + if ( mNeedsResync ) + return ++mNextSequence; + + auto isFullImageUpdate = []( TerminalGraphicsUpdateType type ) { + return type == TerminalGraphicsUpdateType::CreateImage || + type == TerminalGraphicsUpdateType::ReplaceImage; + }; + if ( !mUpdates.empty() && isFullImageUpdate( update.type ) && + isFullImageUpdate( mUpdates.back().type ) && mUpdates.back().imageId == update.imageId ) { + const size_t previousBytes = mUpdates.back().payloadBytes(); + if ( payloadBytes <= mMaxBytes - ( mQueuedBytes - previousBytes ) ) { + update.sequence = mUpdates.back().sequence; + if ( mUpdates.back().type == TerminalGraphicsUpdateType::CreateImage ) + update.type = TerminalGraphicsUpdateType::CreateImage; + mQueuedBytes = mQueuedBytes - previousBytes + payloadBytes; + mUpdates.back() = std::move( update ); + return mUpdates.back().sequence; + } + } + + update.sequence = ++mNextSequence; + + if ( mUpdates.size() >= mMaxUpdates || mQueuedBytes > mMaxBytes || + payloadBytes > mMaxBytes - mQueuedBytes ) { + mUpdates.clear(); + mQueuedBytes = 0; + mNeedsResync = true; + TerminalGraphicsUpdate resync; + resync.sequence = update.sequence; + resync.type = TerminalGraphicsUpdateType::Resync; + mUpdates.emplace_back( std::move( resync ) ); + return update.sequence; + } + + mQueuedBytes += payloadBytes; + mUpdates.emplace_back( std::move( update ) ); + return mNextSequence; +} + +std::vector TerminalGraphicsUpdateQueue::drain() { + std::vector updates; + std::lock_guard lock( mMutex ); + updates.reserve( mUpdates.size() ); + while ( !mUpdates.empty() ) { + updates.emplace_back( std::move( mUpdates.front() ) ); + mUpdates.pop_front(); + } + mQueuedBytes = 0; + return updates; +} + +size_t TerminalGraphicsUpdateQueue::queuedBytes() const { + std::lock_guard lock( mMutex ); + return mQueuedBytes; +} + +bool TerminalGraphicsUpdateQueue::needsResync() const { + std::lock_guard lock( mMutex ); + return mNeedsResync; +} + +void TerminalGraphicsUpdateQueue::resetResync() { + std::lock_guard lock( mMutex ); + mNeedsResync = false; +} + +}} // namespace eterm::Terminal diff --git a/src/modules/eterm/src/eterm/terminal/terminalsession.cpp b/src/modules/eterm/src/eterm/terminal/terminalsession.cpp index eb270fc62..ae6132e05 100644 --- a/src/modules/eterm/src/eterm/terminal/terminalsession.cpp +++ b/src/modules/eterm/src/eterm/terminal/terminalsession.cpp @@ -20,7 +20,10 @@ struct TerminalSession::SelectionResponse { class TerminalSession::WorkerDisplay final : public ITerminalDisplay { public: WorkerDisplay( TerminalSession& session, TerminalColorPalette palette ) : - mSession( session ), mInitialPalette( std::move( palette ) ), mPalette( mInitialPalette ) { + mSession( session ), + mInitialPalette( std::move( palette ) ), + mPalette( mInitialPalette ), + mGraphics( std::make_shared() ) { mMode |= MODE_FOCUSED; } @@ -58,8 +61,20 @@ class TerminalSession::WorkerDisplay final : public ITerminalDisplay { mCursorVisible = true; } + void drawGraphics( std::shared_ptr presentation, + std::vector updates ) { + if ( !presentation ) + presentation = std::make_shared(); + presentation->requiredUpdateSequence = mGraphics ? mGraphics->requiredUpdateSequence : 0; + for ( auto& update : updates ) + presentation->requiredUpdateSequence = + mSession.enqueueGraphicsUpdate( std::move( update ) ); + mGraphics = std::move( presentation ); + } + void drawEnd() { auto snapshot = std::make_shared(); + snapshot->graphics = mGraphics; snapshot->cells = mCells; snapshot->dirtyRows = mDirtyRows; snapshot->title = mTitle; @@ -232,6 +247,7 @@ class TerminalSession::WorkerDisplay final : public ITerminalDisplay { std::vector mCells; std::vector mDirtyRows; std::string mTitle; + std::shared_ptr mGraphics; Uint64 mGeneration{ 0 }; Uint64 mLastAppliedScrollCommand{ 0 }; Vector2i mCursor; @@ -302,7 +318,11 @@ void TerminalSession::writeRaw( std::string data ) { } void TerminalSession::resize( int columns, int rows ) { - enqueue( ResizeCommand{ columns, rows } ); + resize( columns, rows, 0, 0 ); +} + +void TerminalSession::resize( int columns, int rows, int pixelWidth, int pixelHeight ) { + enqueue( ResizeCommand{ columns, rows, pixelWidth, pixelHeight } ); } void TerminalSession::scrollUp( int amount ) { @@ -334,9 +354,9 @@ void TerminalSession::selectionClear() { enqueue( SelectionClearCommand{} ); } -void TerminalSession::mouseReport( TerminalMouseEventType type, Vector2i position, Uint32 flags, - Uint32 modifiers ) { - enqueue( MouseCommand{ type, position, flags, modifiers } ); +void TerminalSession::mouseReport( TerminalMouseEventType type, Vector2i cellPosition, + Vector2i pixelPosition, Uint32 flags, Uint32 modifiers ) { + enqueue( MouseCommand{ type, cellPosition, pixelPosition, flags, modifiers } ); } void TerminalSession::setFocus( bool focus ) { @@ -383,6 +403,10 @@ void TerminalSession::restart( PtyPtr&& pty, ProcPtr&& process ) { enqueue( RestartCommand{ std::move( pty ), std::move( process ) } ); } +void TerminalSession::requestGraphicsResync() { + enqueue( GraphicsResyncCommand{} ); +} + std::shared_ptr TerminalSession::snapshot() const { std::lock_guard lock( mPublishedSnapshotMutex ); return mPublishedSnapshot; @@ -410,6 +434,14 @@ std::vector TerminalSession::drainEvents() { return events; } +std::vector TerminalSession::drainGraphicsUpdates() { + return mGraphicsUpdates.drain(); +} + +Uint64 TerminalSession::enqueueGraphicsUpdate( TerminalGraphicsUpdate update ) { + return mGraphicsUpdates.enqueue( std::move( update ) ); +} + void TerminalSession::enqueueEvent( Event event, bool coalescable ) { std::lock_guard lock( mEventMutex ); if ( coalescable ) { @@ -480,7 +512,7 @@ void TerminalSession::processCommand( Command&& command ) { } else if constexpr ( std::is_same_v ) { mEmulator->write( value.data.data(), value.data.size() ); } else if constexpr ( std::is_same_v ) { - mEmulator->resize( value.columns, value.rows ); + mEmulator->resize( value.columns, value.rows, value.pixelWidth, value.pixelHeight ); } else if constexpr ( std::is_same_v ) { TerminalArg argument( value.amount ); if ( value.direction < 0 ) @@ -502,7 +534,8 @@ void TerminalSession::processCommand( Command&& command ) { mEmulator->selclear(); mEmulator->redraw(); } else if constexpr ( std::is_same_v ) { - mEmulator->mousereport( value.type, value.position, value.flags, value.modifiers ); + mEmulator->mousereport( value.type, value.cellPosition, value.pixelPosition, + value.flags, value.modifiers ); } else if constexpr ( std::is_same_v ) { if ( mWorkerDisplay->getMode( MODE_FOCUS ) ) mEmulator->ttywrite( value.value ? "\033[I" : "\033[O", 3, false ); @@ -556,6 +589,9 @@ void TerminalSession::processCommand( Command&& command ) { value.response->selection = mEmulator->getSelection(); value.response->ready = true; value.response->condition.notify_one(); + } else if constexpr ( std::is_same_v ) { + mGraphicsUpdates.resetResync(); + mEmulator->requestGraphicsResync(); } }, std::move( command ) ); diff --git a/src/tests/unit_tests/eterm_tests.cpp b/src/tests/unit_tests/eterm_tests.cpp index 0570bec89..d5987e2a4 100644 --- a/src/tests/unit_tests/eterm_tests.cpp +++ b/src/tests/unit_tests/eterm_tests.cpp @@ -1,16 +1,21 @@ #include "utest.hpp" #include #include +#include +#include +#include #include #include #include #include +#include #include #include #include using namespace eterm::Terminal; using namespace eterm::System; +using namespace EE::System; class MockPty : public IPseudoTerminal { public: @@ -22,11 +27,15 @@ class MockPty : public IPseudoTerminal { std::atomic mBytesRead{ 0 }; int mCols = 80; int mRows = 24; + int mPixelWidth = 0; + int mPixelHeight = 0; int getNumColumns() const override { return mCols; } int getNumRows() const override { return mRows; } - bool resize( int columns, int rows ) override { + bool resize( int columns, int rows, int pixelWidth, int pixelHeight ) override { mCols = columns; mRows = rows; + mPixelWidth = pixelWidth; + mPixelHeight = pixelHeight; return true; } bool isTTY() const override { return true; } @@ -84,6 +93,8 @@ UTEST( eterm_session, command_wakeup_and_snapshot_immutability ) { snapshot.cells[1].u == 'B' && snapshot.cells[2].u == 'C'; } ); ASSERT_TRUE( first != nullptr ); + ASSERT_TRUE( first->graphics != nullptr ); + EXPECT_EQ( static_cast( 0 ), first->graphics->requiredUpdateSequence ); const Uint64 firstGeneration = first->generation; session->writeRaw( "\rXYZ" ); @@ -102,6 +113,68 @@ UTEST( eterm_session, skipped_snapshot_generation_requires_full_redraw ) { EXPECT_FALSE( snapshot.dirtyRowsFollow( 40 ) ); } +UTEST( eterm_session, graphics_update_queue_preserves_order_and_payloads ) { + TerminalGraphicsUpdateQueue queue; + auto pixels = std::make_shared>( 16, 0x7F ); + TerminalGraphicsUpdate create; + create.type = TerminalGraphicsUpdateType::CreateImage; + create.imageId = 7; + create.rgba = pixels; + TerminalGraphicsUpdate patch; + patch.type = TerminalGraphicsUpdateType::UpdateRegion; + patch.imageId = 7; + patch.rgba = pixels; + + EXPECT_EQ( static_cast( 1 ), queue.enqueue( std::move( create ) ) ); + EXPECT_EQ( static_cast( 2 ), queue.enqueue( std::move( patch ) ) ); + EXPECT_EQ( static_cast( 32 ), queue.queuedBytes() ); + auto updates = queue.drain(); + ASSERT_EQ( static_cast( 2 ), updates.size() ); + EXPECT_EQ( static_cast( 1 ), updates[0].sequence ); + EXPECT_EQ( static_cast( 2 ), updates[1].sequence ); + EXPECT_EQ( TerminalGraphicsUpdateType::CreateImage, updates[0].type ); + EXPECT_EQ( TerminalGraphicsUpdateType::UpdateRegion, updates[1].type ); + EXPECT_TRUE( pixels == updates[0].rgba ); +} + +UTEST( eterm_session, graphics_update_queue_overflow_requires_resync ) { + TerminalGraphicsUpdateQueue queue( 2, 8 ); + TerminalGraphicsUpdate update; + update.type = TerminalGraphicsUpdateType::UpdateRegion; + update.rgba = std::make_shared>( 8, 0xFF ); + queue.enqueue( update ); + queue.enqueue( std::move( update ) ); + + EXPECT_TRUE( queue.needsResync() ); + auto updates = queue.drain(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::Resync, updates[0].type ); + EXPECT_EQ( static_cast( 2 ), updates[0].sequence ); + EXPECT_EQ( static_cast( 0 ), queue.queuedBytes() ); +} + +UTEST( eterm_session, graphics_update_queue_coalesces_superseded_video_frames ) { + TerminalGraphicsUpdateQueue queue( 4, 8 ); + TerminalGraphicsUpdate create; + create.type = TerminalGraphicsUpdateType::CreateImage; + create.imageId = 7; + create.rgba = std::make_shared>( 8, 1 ); + EXPECT_EQ( static_cast( 1 ), queue.enqueue( std::move( create ) ) ); + for ( Uint8 frame = 2; frame < 20; ++frame ) { + TerminalGraphicsUpdate replacement; + replacement.type = TerminalGraphicsUpdateType::ReplaceImage; + replacement.imageId = 7; + replacement.rgba = std::make_shared>( 8, frame ); + EXPECT_EQ( static_cast( 1 ), queue.enqueue( std::move( replacement ) ) ); + } + EXPECT_FALSE( queue.needsResync() ); + EXPECT_EQ( static_cast( 8 ), queue.queuedBytes() ); + auto updates = queue.drain(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::CreateImage, updates[0].type ); + EXPECT_EQ( static_cast( 19 ), updates[0].rgba->front() ); +} + UTEST( eterm_session, ordered_selection_request ) { auto pty = std::make_unique(); pty->mBuffer = "ordered selection"; @@ -347,6 +420,7 @@ class MockDisplay : public ITerminalDisplay { std::vector mResetColorIndices; int mResetColorsCount{ 0 }; Uint32 mBackground{ 0x101010FF }; + std::shared_ptr mGraphics; bool drawBegin( Uint32, Uint32 ) override { return true; } void drawLine( Line line, int, int y, int ) override { ++mDrawLines; @@ -360,6 +434,10 @@ class MockDisplay : public ITerminalDisplay { void drawCursor( int, int, TerminalGlyph, int, int, TerminalGlyph ) override {} void drawEnd() override { ++mDrawEnds; } void resetColors() override { ++mResetColorsCount; } + void drawGraphics( std::shared_ptr presentation, + std::vector ) override { + mGraphics = std::move( presentation ); + } int resetColor( const Uint32& index, const char* ) override { mResetColorIndices.emplace_back( index ); return 0; @@ -381,6 +459,424 @@ class MockDisplay : public ITerminalDisplay { } }; +UTEST( eterm, kitty_graphics_parser_preserves_payload_and_types_action ) { + auto result = KittyGraphicsProtocol::parse( + "a=T,f=32,s=2,v=1,i=7,p=9,q=1,C=1,z=-3,future=value;AAAA;BBBB" ); + ASSERT_TRUE( result.command.has_value() ); + auto* transmit = std::get_if( &*result.command ); + ASSERT_TRUE( transmit != nullptr ); + EXPECT_TRUE( transmit->display ); + EXPECT_EQ( static_cast( 32 ), *transmit->data.format ); + EXPECT_EQ( static_cast( 2 ), *transmit->data.width ); + EXPECT_EQ( static_cast( 1 ), *transmit->data.height ); + EXPECT_EQ( static_cast( -3 ), *transmit->data.zIndex ); + EXPECT_STDSTREQ( "AAAA;BBBB", std::string( transmit->data.payload ) ); +} + +UTEST( eterm, kitty_graphics_parser_rejects_invalid_control_data ) { + EXPECT_EQ( KittyGraphicsError::InvalidArgument, + KittyGraphicsProtocol::parse( "a=T,m=2;AAAA" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidArgument, + KittyGraphicsProtocol::parse( "a=T,i=1,I=2;AAAA" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidArgument, + KittyGraphicsProtocol::parse( "a=T,s=4294967296;AAAA" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidArgument, + KittyGraphicsProtocol::parse( "a=unknown;AAAA" ).error ); +} + +UTEST( eterm, kitty_graphics_parser_fuzz_corpus_is_bounded_and_total ) { + Uint32 state = 0xC0FFEEu; + for ( size_t iteration = 0; iteration < 5000; ++iteration ) { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + const size_t length = state % 512; + std::string input( length, '\0' ); + for ( char& character : input ) { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + character = static_cast( state & 0x7F ); + } + const auto result = KittyGraphicsProtocol::parse( input ); + EXPECT_TRUE( result.command.has_value() || result.error != KittyGraphicsError::None ); + } +} + +UTEST( eterm, kitty_graphics_direct_rgba_chunks_create_worker_image ) { + KittyGraphicsProtocol protocol; + auto first = protocol.handle( "a=t,f=32,s=1,v=1,i=7,m=1;AQID" ); + EXPECT_EQ( KittyGraphicsError::None, first.error ); + EXPECT_FALSE( first.changed ); + auto final = protocol.handle( "m=0;BA==" ); + EXPECT_EQ( KittyGraphicsError::None, final.error ); + EXPECT_TRUE( final.changed ); + EXPECT_STDSTREQ( "\033_Gi=7;OK\033\\", final.response ); + + auto pixels = protocol.imagePixels( 7 ); + ASSERT_TRUE( pixels != nullptr ); + ASSERT_EQ( static_cast( 4 ), pixels->size() ); + EXPECT_EQ( static_cast( 1 ), ( *pixels )[0] ); + EXPECT_EQ( static_cast( 4 ), ( *pixels )[3] ); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::CreateImage, updates[0].type ); +} + +UTEST( eterm, kitty_graphics_chunk_continuations_reject_metadata_and_wrong_action ) { + KittyGraphicsProtocol protocol; + EXPECT_EQ( KittyGraphicsError::None, protocol.handle( "a=t,f=32,s=1,v=1,i=8,m=1;AQID" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidArgument, protocol.handle( "m=0,s=1;BA==" ).error ); + EXPECT_EQ( static_cast( 0 ), protocol.imageCount() ); + EXPECT_EQ( KittyGraphicsError::None, protocol.handle( "a=t,f=32,s=1,v=1,i=8,m=1;AQID" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidArgument, protocol.handle( "a=p,i=8" ).error ); + EXPECT_EQ( static_cast( 0 ), protocol.imageCount() ); + + protocol.handle( "a=t,f=32,s=1,v=1,i=8;AQIDBA==" ); + EXPECT_EQ( KittyGraphicsError::None, protocol.handle( "a=f,i=8,f=32,s=1,v=1,m=1;AQID" ).error ); + EXPECT_EQ( KittyGraphicsError::InvalidArgument, protocol.handle( "m=0;BA==" ).error ); +} + +UTEST( eterm, kitty_graphics_image_number_allocates_id_and_echoes_number ) { + KittyGraphicsProtocol protocol; + auto created = protocol.handle( "a=t,f=32,s=1,v=1,I=77;AQIDBA==" ); + EXPECT_TRUE( created.changed ); + EXPECT_TRUE( created.response.find( ",I=77;OK" ) != std::string::npos ); + auto placed = protocol.handle( "a=p,I=77,p=3" ); + EXPECT_EQ( KittyGraphicsError::None, placed.error ); + ASSERT_EQ( static_cast( 1 ), protocol.takePresentation()->placements.size() ); +} + +UTEST( eterm, kitty_graphics_rgb_and_zlib_normalize_to_rgba ) { + const std::vector rgb{ 10, 20, 30, 40, 50, 60 }; + std::vector compressed( Compression::getMaxCompressedBufferSize( rgb.size() ) ); + IOStreamMemory source( reinterpret_cast( rgb.data() ), rgb.size() ); + IOStreamMemory destination( reinterpret_cast( compressed.data() ), compressed.size() ); + ASSERT_EQ( Compression::OK, Compression::compress( destination, source ) ); + compressed.resize( destination.tell() ); + std::string encoded; + ASSERT_TRUE( Base64::encode( + std::string_view( reinterpret_cast( compressed.data() ), compressed.size() ), + encoded ) ); + + KittyGraphicsProtocol protocol; + auto result = protocol.handle( "a=t,f=24,s=2,v=1,i=9,o=z;" + encoded ); + EXPECT_TRUE( result.changed ); + auto pixels = protocol.imagePixels( 9 ); + ASSERT_TRUE( pixels != nullptr ); + const std::vector expected{ 10, 20, 30, 255, 40, 50, 60, 255 }; + EXPECT_TRUE( expected == *pixels ); +} + +UTEST( eterm, kitty_graphics_png_decodes_to_rgba ) { + KittyGraphicsProtocol protocol; + const auto result = protocol.handle( "a=t,f=100,i=10;" + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42" + "mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" ); + EXPECT_EQ( KittyGraphicsError::None, result.error ); + EXPECT_TRUE( result.changed ); + const auto* pixels = protocol.imagePixels( 10 ); + ASSERT_TRUE( pixels != nullptr ); + EXPECT_EQ( static_cast( 4 ), pixels->size() ); +} + +UTEST( eterm, kitty_graphics_placement_uses_final_cursor_and_geometry ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=T,f=32,s=1,v=1,i=21,p=4,c=2,r=3,C=1,x=0,y=0,w=1,h=1;AQIDBA==", + Vector2i( 5, 6 ) ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), presentation->placements.size() ); + const auto& placement = presentation->placements.front(); + EXPECT_EQ( static_cast( 21 ), placement.imageId ); + EXPECT_EQ( static_cast( 4 ), placement.placementId ); + EXPECT_EQ( 5, placement.visibleAnchorCell.x ); + EXPECT_EQ( 6, placement.visibleAnchorCell.y ); + EXPECT_EQ( static_cast( 2 ), placement.columns ); + EXPECT_EQ( static_cast( 3 ), placement.rows ); + + auto put = protocol.handle( "a=p,i=21,p=5,c=4,r=2", Vector2i( 1, 2 ) ); + EXPECT_EQ( 4, put.cursorMovement.x ); + EXPECT_EQ( 2, put.cursorMovement.y ); + presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 2 ), presentation->placements.size() ); +} + +UTEST( eterm, kitty_graphics_placement_derives_missing_cell_geometry ) { + KittyGraphicsProtocol protocol; + protocol.setCellPixelSize( 10, 20 ); + protocol.handle( "a=t,f=32,s=2,v=2,i=22;AAAAAAAAAAAAAAAAAAAAAA==" ); + auto result = protocol.handle( "a=p,i=22,X=9,Y=19,C=1" ); + EXPECT_TRUE( result.changed ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), presentation->placements.size() ); + EXPECT_EQ( static_cast( 2 ), presentation->placements[0].columns ); + EXPECT_EQ( static_cast( 2 ), presentation->placements[0].rows ); +} + +UTEST( eterm, kitty_graphics_retransmit_removes_old_placements_and_crop_intersects ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=T,f=32,s=2,v=2,i=23,p=4;AAAAAAAAAAAAAAAAAAAAAA==" ); + ASSERT_EQ( static_cast( 1 ), protocol.takePresentation()->placements.size() ); + protocol.handle( "a=t,f=32,s=1,v=1,i=23;AQIDBA==" ); + EXPECT_TRUE( protocol.takePresentation()->placements.empty() ); + protocol.handle( "a=t,f=32,s=2,v=2,i=24;AAAAAAAAAAAAAAAAAAAAAA==" ); + auto placed = protocol.handle( "a=p,i=24,p=5,x=1,y=1,w=99,h=99" ); + EXPECT_STDSTREQ( "\033_Gi=24,p=5;OK\033\\", placed.response ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), presentation->placements.size() ); + EXPECT_EQ( 2, presentation->placements[0].sourcePixels.Right ); + EXPECT_EQ( 2, presentation->placements[0].sourcePixels.Bottom ); +} + +UTEST( eterm, kitty_graphics_resync_republishes_authoritative_images ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=t,f=32,s=1,v=1,i=27;AQIDBA==" ); + protocol.takeUpdates(); + protocol.resync(); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 2 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::ResetAll, updates[0].type ); + EXPECT_EQ( TerminalGraphicsUpdateType::CreateImage, updates[1].type ); + EXPECT_EQ( static_cast( 27 ), updates[1].imageId ); + ASSERT_TRUE( updates[1].rgba != nullptr ); + EXPECT_EQ( static_cast( 4 ), updates[1].rgba->size() ); +} + +UTEST( eterm, kitty_graphics_root_frame_patch_publishes_only_changed_rectangle ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=t,f=32,s=2,v=1,i=29;AQIDBAUGBwg=" ); + protocol.takeUpdates(); + auto result = protocol.handle( "a=f,i=29,r=1,f=32,s=1,v=1,x=1,y=0,X=1;CQoLDA==" ); + EXPECT_TRUE( result.changed ); + const auto* pixels = protocol.imagePixels( 29 ); + ASSERT_TRUE( pixels != nullptr ); + const std::vector expected{ 1, 2, 3, 4, 9, 10, 11, 12 }; + EXPECT_TRUE( expected == *pixels ); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::UpdateRegion, updates[0].type ); + EXPECT_EQ( 1, updates[0].region.Left ); + EXPECT_EQ( 0, updates[0].region.Top ); + EXPECT_EQ( 2, updates[0].region.Right ); + EXPECT_EQ( 1, updates[0].region.Bottom ); + ASSERT_TRUE( updates[0].rgba != nullptr ); + EXPECT_EQ( static_cast( 4 ), updates[0].rgba->size() ); +} + +UTEST( eterm, kitty_graphics_animation_frame_create_control_and_compose ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=T,f=32,s=2,v=1,i=30;AQIDBAUGBwg=" ); + protocol.takeUpdates(); + protocol.takePresentation(); + auto frame = protocol.handle( "a=f,i=30,c=1,f=32,s=1,v=1,x=1,y=0,X=1,z=25;CQoLDA==" ); + EXPECT_EQ( KittyGraphicsError::None, frame.error ); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::CreateFrame, updates[0].type ); + EXPECT_EQ( static_cast( 2 ), updates[0].frameNumber ); + ASSERT_TRUE( updates[0].rgba != nullptr ); + const std::vector expectedFrame{ 1, 2, 3, 4, 9, 10, 11, 12 }; + EXPECT_TRUE( expectedFrame == *updates[0].rgba ); + + auto control = protocol.handle( "a=a,i=30,c=2" ); + EXPECT_TRUE( control.changed ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), presentation->placements.size() ); + EXPECT_EQ( static_cast( 2 ), presentation->placements[0].frameNumber ); + + auto compose = protocol.handle( "a=c,i=30,r=1,c=2,X=1,Y=0,x=0,y=0,w=1,h=1,C=1" ); + EXPECT_EQ( KittyGraphicsError::None, compose.error ); + updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::UpdateFrameRegion, updates[0].type ); + ASSERT_TRUE( updates[0].rgba != nullptr ); + const std::vector expectedPatch{ 5, 6, 7, 8 }; + EXPECT_TRUE( expectedPatch == *updates[0].rgba ); + EXPECT_TRUE( protocol.handle( "a=a,i=30,c=1,r=1,z=-1,s=3" ).changed ); + EXPECT_TRUE( protocol.updateAnimations() ); + presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), presentation->placements.size() ); + EXPECT_EQ( static_cast( 2 ), presentation->placements[0].frameNumber ); + EXPECT_TRUE( protocol.handle( "a=d,d=f,i=30" ).changed ); + updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::DeleteFrame, updates[0].type ); +} + +UTEST( eterm, kitty_graphics_delete_placements_and_uppercase_frees_data ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=T,f=32,s=1,v=1,i=31,p=1,c=2,r=2;AQIDBA==", Vector2i( 3, 4 ) ); + protocol.handle( "a=p,i=31,p=2,c=1,r=1", Vector2i( 8, 9 ) ); + protocol.takeUpdates(); + + auto result = protocol.handle( "a=d,d=c", Vector2i( 4, 5 ) ); + EXPECT_TRUE( result.changed ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), presentation->placements.size() ); + EXPECT_EQ( static_cast( 2 ), presentation->placements[0].placementId ); + EXPECT_TRUE( protocol.imagePixels( 31 ) != nullptr ); + + result = protocol.handle( "a=d,d=I,i=31,p=2" ); + EXPECT_TRUE( result.changed ); + EXPECT_TRUE( protocol.imagePixels( 31 ) == nullptr ); + EXPECT_TRUE( protocol.takePresentation()->placements.empty() ); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::DeleteImage, updates[0].type ); + EXPECT_EQ( static_cast( 31 ), updates[0].imageId ); +} + +UTEST( eterm, kitty_graphics_storage_quota_evicts_oldest_unplaced_image ) { + KittyGraphicsProtocol protocol( 8, 2, 2 ); + EXPECT_TRUE( protocol.handle( "a=t,f=32,s=1,v=1,i=41;AQIDBA==" ).changed ); + EXPECT_TRUE( protocol.handle( "a=t,f=32,s=1,v=1,i=42,N=1;BQYHCA==" ).changed ); + protocol.takeUpdates(); + EXPECT_TRUE( protocol.handle( "a=t,f=32,s=1,v=1,i=43;CQoLDA==" ).changed ); + EXPECT_TRUE( protocol.imagePixels( 41 ) != nullptr ); + EXPECT_TRUE( protocol.imagePixels( 42 ) == nullptr ); + EXPECT_TRUE( protocol.imagePixels( 43 ) != nullptr ); + auto updates = protocol.takeUpdates(); + ASSERT_EQ( static_cast( 2 ), updates.size() ); + EXPECT_EQ( TerminalGraphicsUpdateType::DeleteImage, updates[0].type ); + EXPECT_EQ( static_cast( 42 ), updates[0].imageId ); + EXPECT_EQ( TerminalGraphicsUpdateType::CreateImage, updates[1].type ); +} + +UTEST( eterm, kitty_graphics_screen_lifecycle_restores_primary_and_resets_gpu ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=T,f=32,s=1,v=1,i=51;AQIDBA==" ); + protocol.takeUpdates(); + protocol.takePresentation(); + protocol.setAlternateScreen( true ); + EXPECT_TRUE( protocol.takePresentation()->placements.empty() ); + protocol.handle( "a=p,i=51,p=2" ); + ASSERT_EQ( static_cast( 1 ), protocol.takePresentation()->placements.size() ); + protocol.setAlternateScreen( false ); + auto primary = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), primary->placements.size() ); + EXPECT_EQ( static_cast( 0 ), primary->placements[0].placementId ); + protocol.clearScreen(); + EXPECT_TRUE( protocol.takePresentation()->placements.empty() ); + EXPECT_TRUE( protocol.imagePixels( 51 ) != nullptr ); + protocol.reset(); + EXPECT_TRUE( protocol.imagePixels( 51 ) == nullptr ); + auto updates = protocol.takeUpdates(); + ASSERT_TRUE( !updates.empty() ); + EXPECT_EQ( TerminalGraphicsUpdateType::ResetAll, updates.back().type ); +} + +UTEST( eterm, kitty_graphics_scrolling_tracks_history_and_scrollback ) { + KittyGraphicsProtocol protocol; + protocol.setViewport( 0, 0, 4 ); + protocol.handle( "a=T,f=32,s=1,v=1,i=61,r=1;AQIDBA==", Vector2i( 0, 0 ) ); + protocol.takePresentation(); + protocol.scrollScreen( 0, 3, -1, true ); + protocol.setViewport( 0, 1, 4 ); + EXPECT_TRUE( protocol.takePresentation()->placements.empty() ); + protocol.setViewport( 1, 1, 4 ); + auto history = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), history->placements.size() ); + EXPECT_EQ( 0, history->placements[0].visibleAnchorCell.y ); + protocol.setViewport( 0, 0, 4 ); + EXPECT_TRUE( protocol.takePresentation()->placements.empty() ); +} + +UTEST( eterm, kitty_graphics_margin_scroll_discards_only_scrolled_out_placements ) { + KittyGraphicsProtocol protocol; + protocol.setViewport( 0, 0, 6 ); + protocol.handle( "a=T,f=32,s=1,v=1,i=62,p=1;AQIDBA==", Vector2i( 0, 0 ) ); + protocol.handle( "a=p,i=62,p=2", Vector2i( 0, 2 ) ); + protocol.handle( "a=p,i=62,p=3", Vector2i( 0, 5 ) ); + protocol.scrollScreen( 1, 4, -2, false ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 2 ), presentation->placements.size() ); + EXPECT_EQ( static_cast( 1 ), presentation->placements[0].placementId ); + EXPECT_EQ( static_cast( 3 ), presentation->placements[1].placementId ); +} + +UTEST( eterm, kitty_graphics_margin_scroll_clips_partially_visible_placements ) { + KittyGraphicsProtocol protocol; + protocol.setViewport( 0, 0, 6 ); + protocol.handle( "a=T,f=32,s=1,v=4,i=63,p=1,c=1,r=4;AAAAAAAAAAAAAAAAAAAAAA==", + Vector2i( 0, 1 ) ); + protocol.scrollScreen( 1, 4, -2, false ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 1 ), presentation->placements.size() ); + EXPECT_EQ( 1, presentation->placements[0].visibleAnchorCell.y ); + EXPECT_EQ( static_cast( 2 ), presentation->placements[0].rows ); + EXPECT_EQ( 2, presentation->placements[0].sourcePixels.Top ); + EXPECT_EQ( 4, presentation->placements[0].sourcePixels.Bottom ); +} + +UTEST( eterm, kitty_graphics_virtual_and_relative_placements_follow_protocol_rules ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=t,f=32,s=1,v=1,i=71;AQIDBA==" ); + auto virtualPlacement = protocol.handle( "a=p,i=71,p=10,U=1,c=2,r=2", Vector2i( 2, 3 ) ); + EXPECT_TRUE( virtualPlacement.changed ); + EXPECT_EQ( 0, virtualPlacement.cursorMovement.x ); + EXPECT_TRUE( protocol.takePresentation()->placements.empty() ); + auto relative = protocol.handle( "a=p,i=71,p=11,P=71,Q=10,H=4,V=-1,c=1,r=1" ); + EXPECT_TRUE( relative.changed ); + EXPECT_EQ( 0, relative.cursorMovement.x ); + protocol.setPlaceholderCells( { { 71, 10, Vector2i( 2, 3 ), 0, 0 } } ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 2 ), presentation->placements.size() ); + auto child = std::find_if( + presentation->placements.begin(), presentation->placements.end(), + []( const TerminalVisiblePlacement& placement ) { return placement.placementId == 11; } ); + ASSERT_TRUE( child != presentation->placements.end() ); + EXPECT_EQ( 6, child->visibleAnchorCell.x ); + EXPECT_EQ( 2, child->visibleAnchorCell.y ); + protocol.handle( "a=d,d=a" ); + ASSERT_EQ( static_cast( 1 ), protocol.takePresentation()->placements.size() ); + auto replacement = protocol.handle( "a=p,i=71,p=11,P=71,Q=10,H=1,V=1" ); + EXPECT_TRUE( replacement.changed ); +} + +UTEST( eterm, kitty_graphics_relative_placements_report_missing_parents_and_cycles ) { + KittyGraphicsProtocol protocol; + protocol.handle( "a=t,f=32,s=1,v=1,i=72;AQIDBA==" ); + EXPECT_EQ( KittyGraphicsError::NoParent, protocol.handle( "a=p,i=72,p=2,P=72,Q=99" ).error ); + EXPECT_TRUE( protocol.handle( "a=p,i=72,p=1", Vector2i( 1, 1 ) ).changed ); + EXPECT_TRUE( protocol.handle( "a=p,i=72,p=2,P=72,Q=1,H=1,V=0" ).changed ); + EXPECT_EQ( KittyGraphicsError::Cycle, + protocol.handle( "a=p,i=72,p=1,P=72,Q=2,H=1,V=0" ).error ); +} + +UTEST( eterm, kitty_graphics_independent_client_namespaces_coexist ) { + KittyGraphicsProtocol protocol; + auto first = protocol.handle( "a=T,f=32,s=1,v=1,I=101,p=1;AQIDBA==" ); + auto second = protocol.handle( "a=T,f=32,s=1,v=1,I=202,p=1;BQYHCA==", Vector2i( 2, 0 ) ); + EXPECT_EQ( KittyGraphicsError::None, first.error ); + EXPECT_EQ( KittyGraphicsError::None, second.error ); + ASSERT_EQ( static_cast( 2 ), protocol.imageCount() ); + auto presentation = protocol.takePresentation(); + ASSERT_EQ( static_cast( 2 ), presentation->placements.size() ); + EXPECT_NE( presentation->placements[0].imageId, presentation->placements[1].imageId ); + EXPECT_TRUE( protocol.handle( "a=d,d=n,I=101" ).changed ); + ASSERT_EQ( static_cast( 1 ), protocol.takePresentation()->placements.size() ); + EXPECT_EQ( static_cast( 2 ), protocol.imageCount() ); +} + +UTEST( eterm_session, kitty_graphics_update_and_metadata_cross_worker_boundary ) { + auto pty = std::make_unique(); + pty->mBuffer = "\033_Ga=t,f=32,s=1,v=1,i=13;AQIDBA==\033\\"; + pty->mLoopWrites = false; + MockPty* ptyPtr = pty.get(); + auto process = std::make_unique(); + auto session = TerminalSession::create( std::move( pty ), std::move( process ), 100 ); + auto snapshot = waitForSnapshot( session, []( const TerminalSnapshot& value ) { + return value.graphics && value.graphics->requiredUpdateSequence > 0; + } ); + ASSERT_TRUE( snapshot != nullptr ); + EXPECT_EQ( static_cast( 1 ), snapshot->graphics->requiredUpdateSequence ); + auto updates = session->drainGraphicsUpdates(); + ASSERT_EQ( static_cast( 1 ), updates.size() ); + EXPECT_EQ( static_cast( 1 ), updates[0].sequence ); + EXPECT_EQ( static_cast( 13 ), updates[0].imageId ); + EXPECT_STDSTREQ( "\033_Gi=13;OK\033\\", ptyPtr->mWrites ); +} + UTEST( eterm, modern_csi_prefixes_do_not_claim_unsupported_keyboard_protocol ) { auto pty = std::make_unique(); pty->mBuffer = "\033[?u\033[>7u\033[<1u\033[mWrites.empty() ); } +UTEST( eterm, kitty_graphics_unicode_placeholder_uses_color_and_diacritics ) { + auto pty = std::make_unique(); + pty->mBuffer = "\033_Ga=t,f=32,s=2,v=2,i=72,q=2;AAAAAAAAAAAAAAAAAAAAAA==\033\\" + "\033_Ga=p,i=72,p=3,U=1,c=2,r=2,q=2\033\\" + "\033[38;5;72m\033[58;5;3m\xF4\x8E\xBB\xAE\xCC\x85\xCC\x85"; + pty->mLoopWrites = false; + auto process = std::make_unique(); + auto display = std::make_shared(); + auto term = TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); + term->update(); + ASSERT_TRUE( display->mGraphics != nullptr ); + ASSERT_EQ( static_cast( 1 ), display->mGraphics->placements.size() ); + const auto& placement = display->mGraphics->placements[0]; + EXPECT_EQ( static_cast( 72 ), placement.imageId ); + EXPECT_EQ( static_cast( 3 ), placement.placementId ); + EXPECT_EQ( 0, placement.visibleAnchorCell.x ); + EXPECT_EQ( 0, placement.visibleAnchorCell.y ); + EXPECT_EQ( 0, placement.sourcePixels.Left ); + EXPECT_EQ( 0, placement.sourcePixels.Top ); + EXPECT_EQ( 1, placement.sourcePixels.Right ); + EXPECT_EQ( 1, placement.sourcePixels.Bottom ); + term->resize( 100, 30, 1000, 600 ); + ASSERT_TRUE( display->mGraphics != nullptr ); + ASSERT_EQ( static_cast( 1 ), display->mGraphics->placements.size() ); + EXPECT_EQ( static_cast( 72 ), display->mGraphics->placements[0].imageId ); +} + +UTEST( eterm, kitty_graphics_apc_is_fragmentation_safe_and_not_terminal_text ) { + auto pty = std::make_unique(); + pty->mBuffer = "\033_Ga=q,i=31,f=32,s=1,v=1;AAAAAA==\033\\OK"; + pty->mLoopWrites = false; + pty->mMaxRead = 1; + auto process = std::make_unique(); + auto display = std::make_shared(); + auto term = TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); + + term->update(); + while ( !term->update() ) { + } + + EXPECT_EQ( static_cast( 'O' ), display->mFirstGlyph.u ); + EXPECT_EQ( static_cast( 'K' ), display->mSecondGlyph.u ); +} + +UTEST( eterm, kitty_graphics_accepts_unchunked_direct_image_larger_than_eight_kibibytes ) { + std::vector rgb( 64 * 64 * 3 ); + for ( size_t offset = 0; offset < rgb.size(); offset += 3 ) + rgb[offset] = 255; + std::string encoded; + ASSERT_TRUE( Base64::encode( + std::string_view( reinterpret_cast( rgb.data() ), rgb.size() ), encoded ) ); + const std::string command = "a=T,f=24,s=64,v=64,c=10,r=5,C=1;" + encoded; + ASSERT_TRUE( KittyGraphicsProtocol::parse( command ).command.has_value() ); + KittyGraphicsProtocol directProtocol; + EXPECT_EQ( KittyGraphicsError::None, directProtocol.handle( command ).error ); + + auto pty = std::make_unique(); + pty->mBuffer = "\033_G" + command + "\033\\"; + pty->mLoopWrites = false; + MockPty* ptyPtr = pty.get(); + auto process = std::make_unique(); + auto display = std::make_shared(); + auto term = TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); + + term->update(); + while ( !term->update() ) { + } + + EXPECT_TRUE( ptyPtr->mWrites.empty() ); + ASSERT_TRUE( display->mGraphics != nullptr ); + ASSERT_EQ( static_cast( 1 ), display->mGraphics->placements.size() ); + EXPECT_EQ( static_cast( 10 ), display->mGraphics->placements[0].columns ); + EXPECT_EQ( static_cast( 5 ), display->mGraphics->placements[0].rows ); +} + +UTEST( eterm, kitty_graphics_anonymous_video_frames_replace_at_same_anchor ) { + KittyGraphicsProtocol protocol( 8, 8, 8 ); + for ( int frame = 0; frame < 20; ++frame ) { + const char* pixels = frame % 2 == 0 ? "AQIDBA==" : "BQYHCA=="; + auto result = + protocol.handle( std::string( "a=T,f=32,s=1,v=1,q=2;" ) + pixels, Vector2i( 3, 4 ) ); + EXPECT_EQ( KittyGraphicsError::None, result.error ); + EXPECT_TRUE( result.changed ); + EXPECT_EQ( static_cast( 1 ), protocol.imageCount() ); + ASSERT_EQ( static_cast( 1 ), protocol.takePresentation()->placements.size() ); + } + const auto* pixels = protocol.imagePixels( 1 ); + ASSERT_TRUE( pixels != nullptr ); + EXPECT_EQ( static_cast( 5 ), ( *pixels )[0] ); +} + +UTEST( eterm, oversized_kitty_graphics_apc_is_discarded_until_terminator ) { + auto pty = std::make_unique(); + pty->mBuffer = "\033_Ga=t;" + std::string( MAX_KITTY_GRAPHICS_APC_SIZE, 'A' ) + "\033\\OK"; + pty->mLoopWrites = false; + auto process = std::make_unique(); + auto display = std::make_shared(); + auto term = TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); + + term->update(); + while ( !term->update() ) { + } + + EXPECT_EQ( static_cast( 'O' ), display->mFirstGlyph.u ); + EXPECT_EQ( static_cast( 'K' ), display->mSecondGlyph.u ); +} + UTEST( eterm, cursor_style_and_xterm_version_queries ) { auto pty = std::make_unique(); pty->mBuffer = "\033[0 q\033[6 q\033[>0q"; @@ -412,6 +1015,37 @@ UTEST( eterm, cursor_style_and_xterm_version_queries ) { EXPECT_STDSTREQ( "\033\\", ptyPtr->mWrites.substr( ptyPtr->mWrites.size() - 2 ) ); } +UTEST( eterm, pixel_geometry_queries_use_latest_worker_resize ) { + auto pty = std::make_unique(); + pty->mBuffer = "\033[14t\033[16t"; + pty->mLoopWrites = false; + MockPty* ptyPtr = pty.get(); + auto process = std::make_unique(); + auto display = std::make_shared(); + auto term = TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); + term->resize( 40, 20, 400, 300 ); + + term->update(); + + EXPECT_TRUE( ptyPtr->mWrites.find( "\033[4;300;400t" ) != std::string::npos ); + EXPECT_TRUE( ptyPtr->mWrites.find( "\033[6;15;10t" ) != std::string::npos ); +} + +UTEST( eterm, sgr_pixel_mouse_mode_uses_grid_relative_pixels ) { + auto pty = std::make_unique(); + pty->mBuffer = "\033[?1000h\033[?1016h"; + pty->mLoopWrites = false; + MockPty* ptyPtr = pty.get(); + auto process = std::make_unique(); + auto display = std::make_shared(); + auto term = TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); + term->update(); + term->mousereport( TerminalMouseEventType::MouseButtonDown, { 2, 3 }, { 20, 30 }, + EE_BUTTON_LMASK, 0 ); + + EXPECT_STDSTREQ( "\033[<0;21;31M", ptyPtr->mWrites ); +} + UTEST( eterm, cursor_style_zero_uses_blinking_configured_shape ) { auto pty = std::make_unique(); pty->mBuffer = "\033[2 q\033[0 q"; @@ -451,12 +1085,14 @@ UTEST( eterm, alternate_scroll_mode_controls_wheel_key_translation ) { auto term = TerminalEmulator::create( std::move( pty ), std::move( process ), display, 100 ); term->update(); - term->mousereport( TerminalMouseEventType::MouseButtonDown, { 0, 0 }, EE_BUTTON_WUMASK, 0 ); + term->mousereport( TerminalMouseEventType::MouseButtonDown, { 0, 0 }, { 0, 0 }, + EE_BUTTON_WUMASK, 0 ); EXPECT_TRUE( ptyPtr->mWrites.empty() ); ptyPtr->mBuffer += "\033[?1007h"; term->update(); - term->mousereport( TerminalMouseEventType::MouseButtonDown, { 0, 0 }, EE_BUTTON_WUMASK, 0 ); + term->mousereport( TerminalMouseEventType::MouseButtonDown, { 0, 0 }, { 0, 0 }, + EE_BUTTON_WUMASK, 0 ); EXPECT_STDSTREQ( "\033[A", ptyPtr->mWrites ); }