feat: support HTMLCanvasElement for input (#81)

* feat: accept HTMLCanvasElement as the input

* feat: support HTMLCanvasElement in VFX.update()

* docs: add a canvas example

* chore: adjust shader

* chore: perf

* chore: comment

Author: fand

packages/docs-vfx-js/index.css

@@ -207,7 +207,6 @@ a:visited {
     gap: 16px;
     overflow: hidden;
 }
-
 #div p {
     margin: 0;
 }
@@ -221,6 +220,11 @@ a:visited {
     resize: vertical;
 }
 
+#canvas {
+    width: 100%;
+    aspect-ratio: 4 / 3;
+}
+
 /*================ AuthorSection ================*/
 
 .AuthorSection {

packages/docs-vfx-js/index.html

@@ -256,6 +256,41 @@ <h2>Examples</h2>
                         </div>
                     </div>
                 </div>
+
+                <div class="row">
+                    <div class="col">
+                        <p>Canvas</p>
+                        <pre><code class="language-html">
+&lt;!--
+  VFX-JS also supports HTMLCanvasElement as the input.
+  You can draw 2D graphics and text in canvas,
+  then pass it to VFX-JS to add post effects.
+--&gt;
+&lt;canvas id="canvas"/&gt;
+                        </code></pre>
+
+                        <pre><code class="language-javascript">
+const canvas = document.getElementById("canvas");
+const ctx = canvas.getContext("2d");
+
+function drawCanvas() {
+    ...
+
+    // Update texture when the canvas has been updated
+    vfx.update(canvas);
+
+    requestAnimationFrame(drawCanvas);
+}
+drawCanvas();
+
+vfx.add(canvas, { shader });
+                    </code></pre>
+                    </div>
+                    <div class="col">
+                        <p>Output:</p>
+                        <canvas id="canvas" />
+                    </div>
+                </div>
             </section>
 
             <div>

packages/docs-vfx-js/src/main.ts

@@ -6,6 +6,10 @@ import "prism-themes/themes/prism-nord.min.css";
 Prism.manual = true;
 Prism.highlightAll();
 
+function lerp(a: number, b: number, t: number) {
+    return a * (1 - t) + b * t;
+}
+
 const shaders: Record<string, string> = {
     logo: `
     precision highp float;
@@ -142,6 +146,32 @@ const shaders: Record<string, string> = {
         img.a *= 0.5;
         gl_FragColor = img;
     }
+    `,
+    canvas: `
+precision highp float;
+uniform vec2 resolution;
+uniform vec2 offset;
+uniform float time;
+uniform sampler2D src;
+
+#define ZOOM(uv, x) ((uv - .5) / x + .5)
+
+void main (void) {
+    vec2 uv = (gl_FragCoord.xy - offset) / resolution;
+
+    float r = sin(time) * 0.5 + 0.5;
+
+    float l = pow(length(uv - .5), 2.);
+    uv = (uv - .5) *  (1. - l * 0.3 * r) + .5;
+
+
+    float n = 0.02 + r * 0.03;
+    vec4 cr = texture2D(src, ZOOM(uv, 1.00));
+    vec4 cg = texture2D(src, ZOOM(uv, (1. + n)));
+    vec4 cb = texture2D(src, ZOOM(uv, (1. + n * 2.)));
+
+    gl_FragColor = vec4(cr.r, cg.g, cb.b, 1);
+}
     `,
     custom: `
 precision highp float;
@@ -218,6 +248,85 @@ class App {
         });
     }
 
+    initCanvas() {
+        const canvas = document.getElementById("canvas") as HTMLCanvasElement;
+        const ctx = canvas.getContext("2d")!;
+        const { width, height } = canvas.getBoundingClientRect();
+        const ratio = window.devicePixelRatio ?? 1;
+        canvas.width = width * ratio;
+        canvas.height = height * ratio;
+        ctx.scale(ratio, ratio);
+
+        let target = [width / 2, height / 2];
+        let p = target;
+        const ps = [p];
+        let isMouseOn = false;
+        const startTime = Date.now();
+
+        canvas.addEventListener("mousemove", (e) => {
+            isMouseOn = true;
+            target = [e.offsetX, e.offsetY];
+        });
+        canvas.addEventListener("mouseleave", (e) => {
+            isMouseOn = false;
+        });
+
+        let isInside = false;
+        const io = new IntersectionObserver(
+            (changes) => {
+                for (const c of changes) {
+                    isInside = c.intersectionRatio > 0.1;
+                }
+            },
+            { threshold: [0, 1, 0.2, 0.8] },
+        );
+        io.observe(canvas);
+
+        const drawMouseStalker = () => {
+            requestAnimationFrame(drawMouseStalker);
+
+            if (!isInside) {
+                return;
+            }
+
+            if (!isMouseOn) {
+                const t = Date.now() / 1000 - startTime;
+                target = [
+                    width * 0.5 + Math.sin(t * 1.3) * width * 0.3,
+                    height * 0.5 + Math.sin(t * 1.7) * height * 0.3,
+                ];
+            }
+            p = [lerp(p[0], target[0], 0.1), lerp(p[1], target[1], 0.1)];
+
+            ps.push(p);
+            ps.splice(0, ps.length - 30);
+
+            ctx.clearRect(0, 0, width, height);
+            ctx.fillStyle = "black";
+            ctx.fillRect(0, 0, width, height);
+
+            ctx.fillStyle = "white";
+            ctx.font = `bold ${width * 0.14}px sans-serif`;
+            ctx.fillText("HOVER ME", width / 2, height / 2);
+            ctx.textBaseline = "middle";
+            ctx.textAlign = "center";
+
+            for (let i = 0; i < ps.length; i++) {
+                const [x, y] = ps[i];
+                const t = (i / ps.length) * 255;
+                ctx.fillStyle = `rgba(${255 - t}, 255, ${t}, ${(i / ps.length) * 0.5 + 0.5})`;
+                ctx.beginPath();
+                ctx.arc(x, y, i + 20, 0, 2 * Math.PI);
+                ctx.fill();
+            }
+
+            this.vfx.update(canvas);
+        };
+        drawMouseStalker();
+
+        this.vfx.add(canvas, { shader: shaders.canvas });
+    }
+
     initCustomShader() {
         const e = document.getElementById("custom")!;
         this.vfx.add(e, {
@@ -265,6 +374,7 @@ window.addEventListener("load", () => {
     app.initBG();
     app.initVFX();
     app.initDiv();
+    app.initCanvas();
     app.initCustomShader();
     app.hideMask();
     setTimeout(() => {

packages/vfx-js/src/types.ts

@@ -165,7 +165,7 @@ export type VFXUniformValue =
 /**
  * @internal
  */
-export type VFXElementType = "img" | "video" | "text";
+export type VFXElementType = "img" | "video" | "text" | "canvas";
 
 /**
  * @internal

packages/vfx-js/src/vfx-player.ts

@@ -202,6 +202,9 @@ export class VFXPlayer {
         } else if (element instanceof HTMLVideoElement) {
             texture = new THREE.VideoTexture(element);
             type = "video" as VFXElementType;
+        } else if (element instanceof HTMLCanvasElement) {
+            texture = new THREE.CanvasTexture(element);
+            type = "canvas" as VFXElementType;
         } else {
             const canvas = await dom2canvas(element, originalOpacity);
             texture = new THREE.CanvasTexture(canvas);
@@ -315,6 +318,13 @@ export class VFXPlayer {
         return Promise.resolve();
     }
 
+    updateCanvasElement(element: HTMLCanvasElement): void {
+        const e = this.#elements.find((e) => e.element === element);
+        if (e) {
+            e.uniforms["src"].value.needsUpdate = true;
+        }
+    }
+
     isPlaying(): boolean {
         return this.#playRequest !== undefined;
     }

packages/vfx-js/src/vfx.ts

@@ -44,6 +44,8 @@ export class VFX {
             this.#addImage(element, opts);
         } else if (element instanceof HTMLVideoElement) {
             this.#addVideo(element, opts);
+        } else if (element instanceof HTMLCanvasElement) {
+            this.#addCanvas(element, opts);
         } else {
             this.#addText(element, opts);
         }
@@ -64,8 +66,13 @@ export class VFX {
      *
      * This is useful to apply effects to eleents whose contents change dynamically (e.g. input, textare etc).
      */
-    update(element: HTMLElement): Promise<void> {
-        return this.#player.updateTextElement(element);
+    async update(element: HTMLElement): Promise<void> {
+        if (element instanceof HTMLCanvasElement) {
+            this.#player.updateCanvasElement(element);
+            return;
+        } else {
+            return this.#player.updateTextElement(element);
+        }
     }
 
     /**
@@ -119,6 +126,10 @@ export class VFX {
         }
     }
 
+    #addCanvas(element: HTMLCanvasElement, opts: VFXProps): void {
+        this.#player.addElement(element, opts);
+    }
+
     #addText(element: HTMLElement, opts: VFXProps): void {
         this.#player.addElement(element, opts);
     }