Showcase the streaming datagrid for the "Paris trees" dataset

+ Fix cases for "raw" buffers

Signed-off-by: martinRenou <[email protected]>

Author: martinRenou

examples/Streaming.ipynb

@@ -7,7 +7,9 @@
    "source": [
     "# Streaming datagrid\n",
     "\n",
-    "ipydatagrid provides the ability to lazily request data to the back-end. This requires a live kernel (does not work with a notebook exported to static HTML with nbconvert)."
+    "ipydatagrid provides the ability to lazily request data to the back-end. This requires a live kernel (does not work with a notebook exported to static HTML with nbconvert).\n",
+    "\n",
+    "This feature allows a **smaller memory footprint for the grid**, and **reduced loading time for the initial grid display.**"
    ]
   },
   {
@@ -45,7 +47,7 @@
    "id": "b064d19c-24a2-462e-94d9-bfb0241d0d57",
    "metadata": {},
    "source": [
-    "The `StreamingDataGrid` class provides a `tick()` method to for notifying that the underlying dataframe has changed."
+    "The `StreamingDataGrid` class provides a `tick()` method to for notifying that the underlying dataframe has changed. **This method CANNOT be called in a loop**, due to its implementation. In fact, the front-end cannot request back the viewport for each iteration, because the kernel is already busy executing the loop and cannot respond back until the execution has finished."
    ]
   },
   {

examples/Trees_paris.ipynb

@@ -34,7 +34,7 @@
     "\n",
     "from py2vega.functions.color import rgb\n",
     "\n",
-    "from ipydatagrid import DataGrid, TextRenderer, BarRenderer, Expr\n",
+    "from ipydatagrid import StreamingDataGrid, TextRenderer, BarRenderer, Expr\n",
     "\n",
     "with open(\"./trees.json\") as fobj:\n",
     "    data = load(fobj)\n",
@@ -102,12 +102,40 @@
     "    ),\n",
     "}\n",
     "\n",
-    "datagrid = DataGrid(\n",
+    "datagrid = StreamingDataGrid(\n",
     "    df, base_row_size=32, base_column_size=90, renderers=renderers\n",
     ")\n",
     "VBox((highlight_box, datagrid))"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.iloc[87642]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "datagrid._handle_comm_msg(\n",
+    "    None,\n",
+    "    dict(\n",
+    "        type=\"data-request\",\n",
+    "        r1=87641,\n",
+    "        r2=87643,\n",
+    "        c1=0,\n",
+    "        c2=3,\n",
+    "    ),\n",
+    "    None\n",
+    ")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -151,7 +179,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -165,7 +193,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.6"
+   "version": "3.12.2"
   }
  },
  "nbformat": 4,

ipydatagrid/datagrid.py

@@ -1032,12 +1032,9 @@ def _handle_comm_msg(self, _, content, buffs):
             # Extract all buffers
             buffers = []
             for column in serialized["data"].keys():
-                if not isinstance(serialized["data"][column], list):
-                    if serialized["data"][column]["type"] == "raw":
-                        serialized["data"][column] = serialized["data"][column]["value"]
-                    else:
-                        buffers.append(serialized["data"][column]["value"])
-                        serialized["data"][column]["value"] = len(buffers) - 1
+                if not isinstance(serialized["data"][column], list) and not serialized["data"][column]["type"] == "raw":
+                    buffers.append(serialized["data"][column]["value"])
+                    serialized["data"][column]["value"] = len(buffers) - 1
 
             answer = {
                 "event_type": "data-reply",

js/datagrid.ts

@@ -854,10 +854,13 @@ export class StreamingDataGridModel extends DataGridModel {
       if (content.event_type === 'data-reply') {
         // Bring back buffers at their original position in the data structure
         for (const column of Object.keys(content.value.data)) {
-          content.value.data[column].value =
-            buffers[content.value.data[column].value];
+          if (content.value.data[column].type !== 'raw') {
+            content.value.data[column].value =
+              buffers[content.value.data[column].value];
+          }
         }
 
+        console.log('set data')
         const deserialized = deserialize_data(content.value, null);
         this.data_model.setModelRangeData(
           content.r1,