[terminal] *EXPERIMENTAL* Good Image Protocol (work-in-progress)

Author: christianparpart

CMakeLists.txt

@@ -58,6 +58,11 @@ option(CONTOUR_SANITIZE "Builds with Address sanitizer enabled [default: OFF]" "
 option(CONTOUR_STACKTRACE_ADDR2LINE "Uses addr2line to pretty-print SEGV stacktrace." ${ADDR2LINE_DEFAULT})
 option(CONTOUR_BUILD_WITH_MIMALLOC "Builds with mimalloc [default: OFF]" OFF)
 option(CONTOUR_INSTALL_TOOLS "Installs tools, if built [default: OFF]" OFF)
+option(CONTOUR_GOOD_IMAGE_PROTOCOL "Enables *EXPERIMENTAL* Good Image Protocol support [default: OFF]" OFF)
+
+if(CONTOUR_GOOD_IMAGE_PROTOCOL)
+    add_definitions(-DGOOD_IMAGE_PROTOCOL=1)
+endif()
 
 if(NOT WIN32 AND NOT CONTOUR_SANITIZE AND NOT CMAKE_CONFIGURATION_TYPES)
     set(CONTOUR_SANITIZE "OFF" CACHE STRING "Choose the sanitizer mode." FORCE)

src/contour/ContourApp.cpp

@@ -22,6 +22,7 @@
 
 #include <crispy/App.h>
 #include <crispy/StackTrace.h>
+#include <crispy/base64.h>
 #include <crispy/utils.h>
 
 #include <fmt/chrono.h>
@@ -57,6 +58,7 @@ using std::string_view;
 using std::unique_ptr;
 
 using namespace std::string_literals;
+using namespace std::string_view_literals;
 
 namespace CLI = crispy::cli;
 
@@ -279,6 +281,106 @@ int ContourApp::captureAction()
         return EXIT_FAILURE;
 }
 
+#if defined(GOOD_IMAGE_PROTOCOL)
+namespace
+{
+    crispy::Size parseSize(string_view _text)
+    {
+        (void) _text;
+        return crispy::Size {}; // TODO
+    }
+
+    terminal::ImageAlignment parseImageAlignment(string_view _text)
+    {
+        (void) _text;
+        return terminal::ImageAlignment::TopStart; // TODO
+    }
+
+    terminal::ImageResize parseImageResize(string_view _text)
+    {
+        (void) _text;
+        return terminal::ImageResize::NoResize; // TODO
+    }
+
+    // terminal::CellLocation parsePosition(string_view _text)
+    // {
+    //     (void) _text;
+    //     return {}; // TODO
+    // }
+
+    // TODO: chunkedFileReader(path) to return iterator over spans of data chunks.
+    std::vector<uint8_t> readFile(FileSystem::path const& _path)
+    {
+        auto ifs = std::ifstream(_path.string());
+        if (!ifs.good())
+            return {};
+
+        auto const size = FileSystem::file_size(_path);
+        auto text = std::vector<uint8_t>();
+        text.resize(size);
+        ifs.read((char*) &text[0], static_cast<std::streamsize>(size));
+        return text;
+    }
+
+    void displayImage(terminal::ImageResize _resizePolicy,
+                      terminal::ImageAlignment _alignmentPolicy,
+                      crispy::Size _screenSize,
+                      string_view _fileName)
+    {
+        auto constexpr ST = "\033\\"sv;
+
+        cout << fmt::format("{}f={},c={},l={},a={},z={};",
+                            "\033Ps"sv, // GIONESHOT
+                            '0',        // image format: 0 = auto detect
+                            _screenSize.width,
+                            _screenSize.height,
+                            int(_alignmentPolicy),
+                            int(_resizePolicy));
+
+    #if 1
+        auto const data = readFile(FileSystem::path(string(_fileName))); // TODO: incremental buffered read
+        auto encoderState = crispy::base64::EncoderState {};
+
+        std::vector<char> buf;
+        auto const writer = [&](char a, char b, char c, char d) {
+            buf.push_back(a);
+            buf.push_back(b);
+            buf.push_back(c);
+            buf.push_back(d);
+        };
+        auto const flush = [&]() {
+            cout.write(buf.data(), static_cast<std::streamsize>(buf.size()));
+            buf.clear();
+        };
+
+        for (uint8_t const byte: data)
+        {
+            crispy::base64::encode(byte, encoderState, writer);
+            if (buf.size() >= 4096)
+                flush();
+        }
+        flush();
+    #endif
+
+        cout << ST;
+    }
+} // namespace
+
+int ContourApp::imageAction()
+{
+    auto const resizePolicy = parseImageResize(parameters().get<string>("contour.image.resize"));
+    auto const alignmentPolicy = parseImageAlignment(parameters().get<string>("contour.image.align"));
+    auto const size = parseSize(parameters().get<string>("contour.image.size"));
+    auto const fileName = parameters().verbatim.front();
+    // TODO: how do we wanna handle more than one verbatim arg (image)?
+    // => report error and EXIT_FAILURE as only one verbatim arg is allowed.
+    // FIXME: What if parameter `size` is given as `_size` instead, it should cause an
+    //        invalid-argument error above already!
+    displayImage(resizePolicy, alignmentPolicy, size, fileName);
+    return EXIT_SUCCESS;
+}
+#endif
+
 int ContourApp::parserTableAction()
 {
     terminal::parser::parserTableDot(std::cout);
@@ -366,6 +468,35 @@ crispy::cli::Command ContourApp::parameterDefinition() const
                                           "FILE",
                                           CLI::Presence::Required },
                         } } } },
+#if defined(GOOD_IMAGE_PROTOCOL)
+            CLI::Command {
+                "image",
+                "Sends an image to the terminal emulator for display.",
+                CLI::OptionList {
+                    CLI::Option { "resize",
+                                  CLI::Value { "fit"s },
+                                  "Sets the image resize policy.\n"
+                                  "Policies available are:\n"
+                                  " - no (no resize),\n"
+                                  " - fit (resize to fit),\n"
+                                  " - fill (resize to fill),\n"
+                                  " - stretch (stretch to fill)." },
+                    CLI::Option { "align",
+                                  CLI::Value { "center"s },
+                                  "Sets the image alignment policy.\n"
+                                  "Possible policies are: TopLeft, TopCenter, TopRight, MiddleLeft, "
+                                  "MiddleCenter, MiddleRight, BottomLeft, BottomCenter, BottomRight." },
+                    CLI::Option { "size",
+                                  CLI::Value { ""s },
+                                  "Sets the amount of columns and rows to place the image onto. "
+                                  "The top-left of the this area is the current cursor position, "
+                                  "and it will be scrolled automatically if not enough rows are present." } },
+                CLI::CommandList {},
+                CLI::CommandSelect::Explicit,
+                CLI::Verbatim {
+                    "IMAGE_FILE",
+                    "Path to image to be displayed. Image formats supported are at least PNG, JPG." } },
+#endif
             CLI::Command {
                 "capture",
                 "Captures the screen buffer of the currently running terminal.",

src/contour/ContourApp.h

@@ -37,6 +37,7 @@ class ContourApp: public crispy::App
     int configAction();
     int integrationAction();
     int infoVT();
+    int imageAction();
 };
 
 } // namespace contour

src/contour/display/OpenGLRenderer.cpp

@@ -141,6 +141,7 @@ namespace
         {
             case terminal::ImageFormat::RGB: return GL_RGB;
             case terminal::ImageFormat::RGBA: return GL_RGBA;
+            case terminal::ImageFormat::PNG: Require(false);
         }
         Guarantee(false);
         crispy::unreachable();

src/contour/display/TerminalWidget.cpp

@@ -1196,4 +1196,45 @@ void TerminalWidget::discardImage(terminal::Image const& _image)
 }
 // }}}
 
+optional<terminal::Image> TerminalWidget::decodeImage(gsl::span<uint8_t> _imageData)
+{
+    QImage image;
+    image.loadFromData(static_cast<uchar const*>(_imageData.data()), static_cast<int>(_imageData.size()));
+
+    qDebug() << "decodeImage()" << image.format();
+    if (image.hasAlphaChannel() && image.format() != QImage::Format_ARGB32)
+        image = image.convertToFormat(QImage::Format_ARGB32);
+    else
+        image = image.convertToFormat(QImage::Format_RGB888);
+#if QT_VERSION >= QT_VERSION_CHECK(5, 10, 0)
+    qDebug() << "|> decodeImage()" << image.format() << image.sizeInBytes() << image.size();
+#else
+    qDebug() << "|> decodeImage()" << image.format() << image.byteCount() << image.size();
+#endif
+
+    static auto nextImageId = terminal::ImageId(0);
+
+    terminal::Image::Data pixels;
+    auto* p = &pixels[0];
+    pixels.resize(static_cast<size_t>(image.bytesPerLine() * image.height()));
+    for (int i = 0; i < image.height(); ++i)
+    {
+        memcpy(p, image.constScanLine(i), static_cast<size_t>(image.bytesPerLine()));
+        p += image.bytesPerLine();
+    }
+
+    terminal::ImageFormat format = terminal::ImageFormat::RGBA;
+    switch (image.format())
+    {
+        case QImage::Format_RGBA8888: format = terminal::ImageFormat::RGBA; break;
+        case QImage::Format_RGB888: format = terminal::ImageFormat::RGB; break;
+        default: return nullopt;
+    }
+    ImageSize size { Width::cast_from(image.width()), Height::cast_from(image.height()) };
+    auto onRemove = terminal::Image::OnImageRemove {};
+
+    auto img = terminal::Image(nextImageId++, format, std::move(pixels), size, onRemove);
+    return { std::move(img) };
+}
+
 } // namespace contour::display

src/contour/display/TerminalWidget.h

@@ -165,6 +165,8 @@ class TerminalWidget: public QOpenGLWidget, private QOpenGLExtraFunctions
     void logDisplayTopInfo();
     void logDisplayInfo();
 
+    std::optional<terminal::Image> decodeImage(gsl::span<uint8_t> _imageData);
+
   public Q_SLOTS:
     void onFrameSwapped();
     void onScrollBarValueChanged(int _value);

src/terminal/CMakeLists.txt

@@ -32,6 +32,7 @@ set(terminal_HEADERS
     InputGenerator.h
     Line.h
     MatchModes.h
+    MessageParser.h
     MockTerm.h
     RenderBuffer.h
     RenderBufferBuilder.h
@@ -62,6 +63,7 @@ set(terminal_SOURCES
     InputGenerator.cpp
     Line.cpp
     MatchModes.cpp
+    MessageParser.cpp
     MockTerm.cpp
     RenderBuffer.cpp
     RenderBufferBuilder.cpp
@@ -117,6 +119,7 @@ if(LIBTERMINAL_TESTING)
         Functions_test.cpp
         Grid_test.cpp
         Line_test.cpp
+        MessageParser_test.cpp
         Screen_test.cpp
         Sequence_test.cpp
         Terminal_test.cpp

src/terminal/Functions.h

@@ -466,14 +466,24 @@ constexpr inline auto RCOLORHIGHLIGHTBG = detail::OSC(117, VTExtension::XTerm,"R
 constexpr inline auto NOTIFY        = detail::OSC(777, VTExtension::XTerm,"NOTIFY", "Send Notification.");
 constexpr inline auto DUMPSTATE     = detail::OSC(888, VTExtension::Contour, "DUMPSTATE", "Dumps internal state to debug stream.");
 
+// DCS: Good Image Protocol
+#if defined(GOOD_IMAGE_PROTOCOL)
+// TODO: use OSC instead of DCS?
+constexpr inline auto GIUPLOAD    = detail::DCS(std::nullopt, 0, 0, std::nullopt, 'u', VTType::VT525, "GIUPLOAD", "Uploads an image.");
+constexpr inline auto GIRENDER    = detail::DCS(std::nullopt, 0, 0, std::nullopt, 'r', VTType::VT525, "GIRENDER", "Renders an image.");
+constexpr inline auto GIDELETE    = detail::DCS(std::nullopt, 0, 0, std::nullopt, 'd', VTType::VT525, "GIDELETE", "Deletes an image.");
+constexpr inline auto GIONESHOT   = detail::DCS(std::nullopt, 0, 0, std::nullopt, 's', VTType::VT525, "GIONESHOT", "Uploads and renders an unnamed image.");
+#endif
+
 constexpr inline auto CaptureBufferCode = 314;
 
 // clang-format on
 
 inline auto const& functions() noexcept
 {
     static auto const funcs = []() constexpr
-    { // {{{
+    {
+        // clang-format off
         auto f = std::array {
             // C0
             EOT,
@@ -581,6 +591,12 @@ inline auto const& functions() noexcept
             XTVERSION,
 
             // DCS
+#if defined(GOOD_IMAGE_PROTOCOL)
+            GIUPLOAD,
+            GIRENDER,
+            GIDELETE,
+            GIONESHOT,
+#endif
             STP,
             DECRQSS,
             DECSIXEL,
@@ -614,12 +630,13 @@ inline auto const& functions() noexcept
             NOTIFY,
             DUMPSTATE,
         };
+        // clang-format off
         crispy::sort(
             f,
             [](FunctionDefinition const& a, FunctionDefinition const& b) constexpr { return compare(a, b); });
         return f;
     }
-    (); // }}}
+    ();
 
 #if 0
     for (auto [a, b] : crispy::indexed(funcs))

src/terminal/Image.cpp

@@ -148,9 +148,9 @@ shared_ptr<RasterizedImage> ImagePool::rasterize(shared_ptr<Image const> _image,
         std::move(_image), _alignmentPolicy, _resizePolicy, _defaultColor, _cellSpan, _cellSize);
 }
 
-void ImagePool::link(string const& _name, shared_ptr<Image const> _imageRef)
+void ImagePool::link(string _name, shared_ptr<Image const> _imageRef)
 {
-    imageNameToImageCache_.emplace(_name, std::move(_imageRef));
+    imageNameToImageCache_.emplace(std::move(_name), std::move(_imageRef));
 }
 
 shared_ptr<Image const> ImagePool::findImageByName(string const& _name) const noexcept

src/terminal/Image.h

@@ -41,6 +41,7 @@ enum class ImageFormat
 {
     RGB,
     RGBA,
+    PNG,
 };
 
 // clang-format off
@@ -259,7 +260,7 @@ class ImagePool
 
     // named image access
     //
-    void link(std::string const& _name, std::shared_ptr<Image const> _imageRef);
+    void link(std::string _name, std::shared_ptr<Image const> _imageRef);
     [[nodiscard]] std::shared_ptr<Image const> findImageByName(std::string const& _name) const noexcept;
     void unlink(std::string const& _name);
 
@@ -300,6 +301,7 @@ struct formatter<terminal::ImageFormat>
         {
             case terminal::ImageFormat::RGB: return fmt::format_to(ctx.out(), "RGB");
             case terminal::ImageFormat::RGBA: return fmt::format_to(ctx.out(), "RGBA");
+            case terminal::ImageFormat::PNG: return fmt::format_to(ctx.out(), "PNG");
         }
         return fmt::format_to(ctx.out(), "{}", unsigned(value));
     }