Skip to content

Commit 75f7ae4

Browse files
lheckerlhecker
authored
AtlasEngine: Implement sixels (#17581)
* Add a revision to `ImageSlice` so that the renderers can use it to cache them as bitmaps across frames. * Hooked up the revision tracking to AtlasEngine to cache the slices into `Buffer`s so we can own them into the `Present`. * Hooked up those snapshots to BackendD3D with a straightforward hashmap -> atlas-rect logic. Just like rendering text. * Hooked up BackendD2D with a bad, but simple & direct drawing logic. * Bonus: Modify `ImageSlice` to be returned as a raw pointers as this helps performance slightly. (Trivial type == good.) * Bonus: Fixed the `_debugShowDirty` code (disabled by default). ## Validation Steps Performed * `mpv --really-quiet --vo=sixel foo.mp4` looks good ✅ * Scroll up down & observe dirty rects ✅
1 parent 6372baa commit 75f7ae4

File tree

14 files changed

+327
-68
lines changed

14 files changed

+327
-68
lines changed

‎src/buffer/out/ImageSlice.cpp

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,27 @@
77
#include "Row.hpp"
88
#include "textBuffer.hpp"
99

10+
static std::atomic<uint64_t> s_revision{ 0 };
11+
1012
ImageSlice::ImageSlice(const til::size cellSize) noexcept :
1113
_cellSize{ cellSize }
1214
{
1315
}
1416

17+
void ImageSlice::BumpRevision() noexcept
18+
{
19+
// Avoid setting the revision to 0. This allows the renderer to use 0 as a sentinel value.
20+
do
21+
{
22+
_revision = s_revision.fetch_add(1, std::memory_order_relaxed);
23+
} while (_revision == 0);
24+
}
25+
26+
uint64_t ImageSlice::Revision() const noexcept
27+
{
28+
return _revision;
29+
}
30+
1531
til::size ImageSlice::CellSize() const noexcept
1632
{
1733
return _cellSize;
@@ -108,9 +124,8 @@ void ImageSlice::CopyBlock(const TextBuffer& srcBuffer, const til::rect srcRect,
108124

109125
void ImageSlice::CopyRow(const ROW& srcRow, ROW& dstRow)
110126
{
111-
const auto& srcSlice = srcRow.GetImageSlice();
112-
auto& dstSlice = dstRow.GetMutableImageSlice();
113-
dstSlice = srcSlice ? std::make_unique<ImageSlice>(*srcSlice) : nullptr;
127+
const auto srcSlice = srcRow.GetImageSlice();
128+
dstRow.SetImageSlice(srcSlice ? std::make_unique<ImageSlice>(*srcSlice) : nullptr);
114129
}
115130

116131
void ImageSlice::CopyCells(const ROW& srcRow, const til::CoordType srcColumn, ROW& dstRow, const til::CoordType dstColumnBegin, const til::CoordType dstColumnEnd)
@@ -119,24 +134,25 @@ void ImageSlice::CopyCells(const ROW& srcRow, const til::CoordType srcColumn, RO
119134
// a blank image into the destination, which is the same thing as an erase.
120135
// Also if the line renditions are different, there's no meaningful way to
121136
// copy the image content, so we also just treat that as an erase.
122-
const auto& srcSlice = srcRow.GetImageSlice();
137+
const auto srcSlice = srcRow.GetImageSlice();
123138
if (!srcSlice || srcRow.GetLineRendition() != dstRow.GetLineRendition()) [[likely]]
124139
{
125140
ImageSlice::EraseCells(dstRow, dstColumnBegin, dstColumnEnd);
126141
}
127142
else
128143
{
129-
auto& dstSlice = dstRow.GetMutableImageSlice();
144+
auto dstSlice = dstRow.GetMutableImageSlice();
130145
if (!dstSlice)
131146
{
132-
dstSlice = std::make_unique<ImageSlice>(srcSlice->CellSize());
147+
dstSlice = dstRow.SetImageSlice(std::make_unique<ImageSlice>(srcSlice->CellSize()));
148+
__assume(dstSlice != nullptr);
133149
}
134150
const auto scale = srcRow.GetLineRendition() != LineRendition::SingleWidth ? 1 : 0;
135151
if (dstSlice->_copyCells(*srcSlice, srcColumn << scale, dstColumnBegin << scale, dstColumnEnd << scale))
136152
{
137153
// If _copyCells returns true, that means the destination was
138154
// completely erased, so we can delete this slice.
139-
dstSlice = nullptr;
155+
dstRow.SetImageSlice(nullptr);
140156
}
141157
}
142158
}
@@ -203,15 +219,15 @@ void ImageSlice::EraseCells(TextBuffer& buffer, const til::point at, const size_
203219

204220
void ImageSlice::EraseCells(ROW& row, const til::CoordType columnBegin, const til::CoordType columnEnd)
205221
{
206-
auto& imageSlice = row.GetMutableImageSlice();
222+
const auto imageSlice = row.GetMutableImageSlice();
207223
if (imageSlice) [[unlikely]]
208224
{
209225
const auto scale = row.GetLineRendition() != LineRendition::SingleWidth ? 1 : 0;
210226
if (imageSlice->_eraseCells(columnBegin << scale, columnEnd << scale))
211227
{
212228
// If _eraseCells returns true, that means the image was
213229
// completely erased, so we can delete this slice.
214-
imageSlice = nullptr;
230+
row.SetImageSlice(nullptr);
215231
}
216232
}
217233
}

‎src/buffer/out/ImageSlice.hpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ class ImageSlice
2626
ImageSlice(const ImageSlice& rhs) = default;
2727
ImageSlice(const til::size cellSize) noexcept;
2828

29+
void BumpRevision() noexcept;
30+
uint64_t Revision() const noexcept;
31+
2932
til::size CellSize() const noexcept;
3033
til::CoordType ColumnOffset() const noexcept;
3134
til::CoordType PixelWidth() const noexcept;
@@ -45,6 +48,7 @@ class ImageSlice
4548
bool _copyCells(const ImageSlice& srcSlice, const til::CoordType srcColumn, const til::CoordType dstColumnBegin, const til::CoordType dstColumnEnd);
4649
bool _eraseCells(const til::CoordType columnBegin, const til::CoordType columnEnd);
4750

51+
uint64_t _revision = 0;
4852
til::size _cellSize;
4953
std::vector<RGBQUAD> _pixelBuffer;
5054
til::CoordType _columnBegin = 0;

‎src/buffer/out/Row.cpp

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -965,14 +965,26 @@ std::vector<uint16_t> ROW::GetHyperlinks() const
965965
return ids;
966966
}
967967

968-
const ImageSlice::Pointer& ROW::GetImageSlice() const noexcept
968+
ImageSlice* ROW::SetImageSlice(ImageSlice::Pointer imageSlice) noexcept
969969
{
970-
return _imageSlice;
970+
_imageSlice = std::move(imageSlice);
971+
return GetMutableImageSlice();
971972
}
972973

973-
ImageSlice::Pointer& ROW::GetMutableImageSlice() noexcept
974+
const ImageSlice* ROW::GetImageSlice() const noexcept
974975
{
975-
return _imageSlice;
976+
return _imageSlice.get();
977+
}
978+
979+
ImageSlice* ROW::GetMutableImageSlice() noexcept
980+
{
981+
const auto ptr = _imageSlice.get();
982+
if (!ptr)
983+
{
984+
return nullptr;
985+
}
986+
ptr->BumpRevision();
987+
return ptr;
976988
}
977989

978990
uint16_t ROW::size() const noexcept

‎src/buffer/out/Row.hpp

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -152,8 +152,9 @@ class ROW final
152152
const til::small_rle<TextAttribute, uint16_t, 1>& Attributes() const noexcept;
153153
TextAttribute GetAttrByColumn(til::CoordType column) const;
154154
std::vector<uint16_t> GetHyperlinks() const;
155-
const ImageSlice::Pointer& GetImageSlice() const noexcept;
156-
ImageSlice::Pointer& GetMutableImageSlice() noexcept;
155+
ImageSlice* SetImageSlice(ImageSlice::Pointer imageSlice) noexcept;
156+
const ImageSlice* GetImageSlice() const noexcept;
157+
ImageSlice* GetMutableImageSlice() noexcept;
157158
uint16_t size() const noexcept;
158159
til::CoordType GetLastNonSpaceColumn() const noexcept;
159160
til::CoordType MeasureLeft() const noexcept;
@@ -299,8 +300,6 @@ class ROW final
299300
til::small_rle<TextAttribute, uint16_t, 1> _attr;
300301
// The width of the row in visual columns.
301302
uint16_t _columnCount = 0;
302-
// Stores any image content covering the row.
303-
ImageSlice::Pointer _imageSlice;
304303
// Stores double-width/height (DECSWL/DECDWL/DECDHL) attributes.
305304
LineRendition _lineRendition = LineRendition::SingleWidth;
306305
// Occurs when the user runs out of text in a given row and we're forced to wrap the cursor to the next line
@@ -309,6 +308,9 @@ class ROW final
309308
bool _doubleBytePadded = false;
310309

311310
std::optional<ScrollbarData> _promptData = std::nullopt;
311+
312+
// Stores any image content covering the row.
313+
ImageSlice::Pointer _imageSlice;
312314
};
313315

314316
#ifdef UNIT_TESTING

‎src/buffer/out/textBuffer.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -918,7 +918,7 @@ void TextBuffer::SetCurrentLineRendition(const LineRendition lineRendition, cons
918918
// If the line rendition has changed, the row can no longer be wrapped.
919919
row.SetWrapForced(false);
920920
// And all image content on the row is removed.
921-
row.GetMutableImageSlice().reset();
921+
row.SetImageSlice(nullptr);
922922
// And if it's no longer single width, the right half of the row should be erased.
923923
if (lineRendition != LineRendition::SingleWidth)
924924
{

‎src/renderer/atlas/AtlasEngine.cpp

Lines changed: 52 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,8 @@ try
129129
};
130130
_p.invalidatedRows = _api.invalidatedRows;
131131
_p.cursorRect = {};
132-
_p.scrollOffset = _api.scrollOffset;
132+
_p.scrollOffsetX = _api.viewportOffset.x;
133+
_p.scrollDeltaY = _api.scrollOffset;
133134

134135
// This if condition serves 2 purposes:
135136
// * By setting top/bottom to the full height we ensure that we call Present() without
@@ -148,7 +149,7 @@ try
148149
_p.MarkAllAsDirty();
149150
#endif
150151

151-
if (const auto offset = _p.scrollOffset)
152+
if (const auto offset = _p.scrollDeltaY)
152153
{
153154
if (offset < 0)
154155
{
@@ -256,6 +257,14 @@ try
256257
{
257258
_flushBufferLine();
258259

260+
for (const auto r : _p.rows)
261+
{
262+
if (r->bitmap.revision != 0 && !r->bitmap.active)
263+
{
264+
r->bitmap = {};
265+
}
266+
}
267+
259268
// PaintCursor() is only called when the cursor is visible, but we need to invalidate the cursor area
260269
// even if it isn't. Otherwise a transition from a visible to an invisible cursor wouldn't be rendered.
261270
if (const auto r = _api.invalidatedCursorArea; r.non_empty())
@@ -520,10 +529,49 @@ try
520529
}
521530
CATCH_RETURN()
522531

523-
[[nodiscard]] HRESULT AtlasEngine::PaintImageSlice(const ImageSlice& /*imageSlice*/, const til::CoordType /*targetRow*/, const til::CoordType /*viewportLeft*/) noexcept
532+
[[nodiscard]] HRESULT AtlasEngine::PaintImageSlice(const ImageSlice& imageSlice, const til::CoordType targetRow, const til::CoordType viewportLeft) noexcept
533+
try
524534
{
525-
return S_FALSE;
535+
const auto y = clamp<til::CoordType>(targetRow, 0, _p.s->viewportCellCount.y - 1);
536+
const auto row = _p.rows[y];
537+
const auto revision = imageSlice.Revision();
538+
const auto srcWidth = std::max(0, imageSlice.PixelWidth());
539+
const auto srcCellSize = imageSlice.CellSize();
540+
auto& b = row->bitmap;
541+
542+
// If this row's ImageSlice has changed we need to update our snapshot.
543+
// Theoretically another _p.rows[y]->bitmap may have this particular revision already,
544+
// but that can only happen if we're scrolling _and_ the entire viewport was invalidated.
545+
if (b.revision != revision)
546+
{
547+
const auto srcHeight = std::max(0, srcCellSize.height);
548+
const auto pixels = imageSlice.Pixels();
549+
const auto expectedSize = gsl::narrow_cast<size_t>(srcWidth) * gsl::narrow_cast<size_t>(srcHeight);
550+
551+
// Sanity check.
552+
if (pixels.size() != expectedSize)
553+
{
554+
assert(false);
555+
return S_OK;
556+
}
557+
558+
if (b.source.size() != pixels.size())
559+
{
560+
b.source = Buffer<u32, 32>{ pixels.size() };
561+
}
562+
563+
memcpy(b.source.data(), pixels.data(), pixels.size_bytes());
564+
b.revision = revision;
565+
b.sourceSize.x = srcWidth;
566+
b.sourceSize.y = srcHeight;
567+
}
568+
569+
b.targetOffset = (imageSlice.ColumnOffset() - viewportLeft);
570+
b.targetWidth = srcWidth / srcCellSize.width;
571+
b.active = true;
572+
return S_OK;
526573
}
574+
CATCH_RETURN()
527575

528576
[[nodiscard]] HRESULT AtlasEngine::PaintSelection(const til::rect& rect) noexcept
529577
try

‎src/renderer/atlas/AtlasEngine.r.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -471,9 +471,9 @@ void AtlasEngine::_present()
471471
params.DirtyRectsCount = 1;
472472
params.pDirtyRects = &dirtyRect;
473473

474-
if (_p.scrollOffset)
474+
if (_p.scrollDeltaY)
475475
{
476-
const auto offsetInPx = _p.scrollOffset * _p.s->font->cellSize.y;
476+
const auto offsetInPx = _p.scrollDeltaY * _p.s->font->cellSize.y;
477477
const auto width = _p.s->targetSize.x;
478478
// We don't use targetSize.y here, because "height" refers to the bottom coordinate of the last text row
479479
// in the buffer. We then add the "offsetInPx" (which is negative when scrolling text upwards) and thus

‎src/renderer/atlas/BackendD2D.cpp

Lines changed: 55 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,11 @@ void BackendD2D::_drawText(RenderingPayload& p)
292292
_drawTextResetLineRendition(row);
293293
}
294294

295+
if (row->bitmap.revision != 0)
296+
{
297+
_drawBitmap(p, row, y);
298+
}
299+
295300
if (p.invalidatedRows.contains(y))
296301
{
297302
dirtyTop = std::min(dirtyTop, row->dirtyTop);
@@ -745,6 +750,39 @@ void BackendD2D::_drawGridlineRow(const RenderingPayload& p, const ShapedRow* ro
745750
}
746751
}
747752

753+
void BackendD2D::_drawBitmap(const RenderingPayload& p, const ShapedRow* row, u16 y) const
754+
{
755+
const auto& b = row->bitmap;
756+
757+
// TODO: This could use some caching logic like BackendD3D.
758+
const D2D1_SIZE_U size{
759+
gsl::narrow_cast<UINT32>(b.sourceSize.x),
760+
gsl::narrow_cast<UINT32>(b.sourceSize.y),
761+
};
762+
const D2D1_BITMAP_PROPERTIES bitmapProperties{
763+
.pixelFormat = { DXGI_FORMAT_B8G8R8A8_UNORM, D2D1_ALPHA_MODE_PREMULTIPLIED },
764+
.dpiX = static_cast<f32>(p.s->font->dpi),
765+
.dpiY = static_cast<f32>(p.s->font->dpi),
766+
};
767+
wil::com_ptr<ID2D1Bitmap> bitmap;
768+
THROW_IF_FAILED(_renderTarget->CreateBitmap(size, b.source.data(), static_cast<UINT32>(b.sourceSize.x) * 4, &bitmapProperties, bitmap.addressof()));
769+
770+
const i32 cellWidth = p.s->font->cellSize.x;
771+
const i32 cellHeight = p.s->font->cellSize.y;
772+
const auto left = (b.targetOffset - p.scrollOffsetX) * cellWidth;
773+
const auto right = left + b.targetWidth * cellWidth;
774+
const auto top = y * cellHeight;
775+
const auto bottom = left + cellHeight;
776+
777+
const D2D1_RECT_F rectF{
778+
static_cast<f32>(left),
779+
static_cast<f32>(top),
780+
static_cast<f32>(right),
781+
static_cast<f32>(bottom),
782+
};
783+
_renderTarget->DrawBitmap(bitmap.get(), &rectF, 1, D2D1_BITMAP_INTERPOLATION_MODE_LINEAR);
784+
}
785+
748786
void BackendD2D::_drawCursorPart1(const RenderingPayload& p)
749787
{
750788
if (p.cursorRect.empty())
@@ -893,23 +931,25 @@ void BackendD2D::_drawSelection(const RenderingPayload& p)
893931
#if ATLAS_DEBUG_SHOW_DIRTY
894932
void BackendD2D::_debugShowDirty(const RenderingPayload& p)
895933
{
934+
if (p.dirtyRectInPx.empty())
935+
{
936+
return;
937+
}
938+
896939
_presentRects[_presentRectsPos] = p.dirtyRectInPx;
897940
_presentRectsPos = (_presentRectsPos + 1) % std::size(_presentRects);
898941

899942
for (size_t i = 0; i < std::size(_presentRects); ++i)
900943
{
901944
const auto& rect = _presentRects[(_presentRectsPos + i) % std::size(_presentRects)];
902-
if (rect.non_empty())
903-
{
904-
const D2D1_RECT_F rectF{
905-
static_cast<f32>(rect.left),
906-
static_cast<f32>(rect.top),
907-
static_cast<f32>(rect.right),
908-
static_cast<f32>(rect.bottom),
909-
};
910-
const auto color = til::colorbrewer::pastel1[i] | 0x1f000000;
911-
_fillRectangle(rectF, color);
912-
}
945+
const D2D1_RECT_F rectF{
946+
static_cast<f32>(rect.left),
947+
static_cast<f32>(rect.top),
948+
static_cast<f32>(rect.right),
949+
static_cast<f32>(rect.bottom),
950+
};
951+
const auto color = til::colorbrewer::pastel1[i] | 0x1f000000;
952+
_fillRectangle(rectF, color);
913953
}
914954
}
915955
#endif
@@ -923,9 +963,12 @@ void BackendD2D::_debugDumpRenderTarget(const RenderingPayload& p)
923963
std::filesystem::create_directories(_dumpRenderTargetBasePath);
924964
}
925965

966+
wil::com_ptr<ID3D11Texture2D> buffer;
967+
THROW_IF_FAILED(p.swapChain.swapChain->GetBuffer(0, __uuidof(ID3D11Texture2D), reinterpret_cast<void**>(buffer.addressof())));
968+
926969
wchar_t path[MAX_PATH];
927970
swprintf_s(path, L"%s\\%u_%08zu.png", &_dumpRenderTargetBasePath[0], GetCurrentProcessId(), _dumpRenderTargetCounter);
928-
SaveTextureToPNG(_deviceContext.get(), _swapChainManager.GetBuffer().get(), p.s->font->dpi, &path[0]);
971+
WIC::SaveTextureToPNG(p.deviceContext.get(), buffer.get(), p.s->font->dpi, &path[0]);
929972
_dumpRenderTargetCounter++;
930973
}
931974
#endif

‎src/renderer/atlas/BackendD2D.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33

44
#pragma once
55

6+
#include <til/flat_set.h>
7+
68
#include "Backend.h"
79
#include "BuiltinGlyphs.h"
810

@@ -26,6 +28,7 @@ namespace Microsoft::Console::Render::Atlas
2628
ATLAS_ATTR_COLD void _drawTextResetLineRendition(const ShapedRow* row) const noexcept;
2729
ATLAS_ATTR_COLD f32r _getGlyphRunDesignBounds(const DWRITE_GLYPH_RUN& glyphRun, f32 baselineX, f32 baselineY);
2830
ATLAS_ATTR_COLD void _drawGridlineRow(const RenderingPayload& p, const ShapedRow* row, u16 y);
31+
ATLAS_ATTR_COLD void _drawBitmap(const RenderingPayload& p, const ShapedRow* row, u16 y) const;
2932
void _drawCursorPart1(const RenderingPayload& p);
3033
void _drawCursorPart2(const RenderingPayload& p);
3134
static void _drawCursor(const RenderingPayload& p, ID2D1RenderTarget* renderTarget, D2D1_RECT_F rect, ID2D1Brush* brush) noexcept;

0 commit comments

Comments
 (0)