feat: enable OpenVINO AI Plugins (#325)

* feature: integrate OpenVINO AI plugins

* ci: update workflows for 2.99-openvino track

* readme: remove TODOs and improve setup instructions

* fix: remove arch hard-coding

* feature: switch to personal-files interface as a less intrusive location for models

* fix: add plug-declaration to unblock build

* fix: use fixed source for command-chain scripts

* readme: remove section for manually connecting plugs

* fix: minor plug-in and extension errors from logs

* fix: avoid lua goat extension crash loop

* chore: bump versions of command-chain scripts for openvino ai plugins support

* chore: update 2.99 paths and commands to 3.0

* readme: update instructions for preview branch

* chore: bump ai plugins command-chain version for 3.0 support

* chore: revert CI to preview branches

* fix: pull new command-chain scripts for arm64 issues

* chore: make openvino command-chain scripts no-ops for arm64

* chore: bump openvino command chain versions for better hardware detection

* readme: clarify details about hardware requirements for AI plugins

---------

Co-authored-by: Jon Seager <[email protected]>
Co-authored-by: Jon Seager <[email protected]>

Author: frenchwr

.github/plug-declaration.json

@@ -0,0 +1,3 @@
+{
+    "personal-files": { "allow-installation": "true" }
+}

README.md

@@ -78,6 +78,55 @@ Now that your git metadata has been updated you are ready to create a bugfix bra
 6. Someone from the team will review the open pull request and either merge it or start a discussion with you with additional changes or clarification needed.
 7. Once the pull request has been merged into the stable branch, a GitHub action will rebuild the snap using your changes and publish it to the [Snap Store](https://snapcraft.io/gimp) into the `candidate` channel. After sufficient testing of the snap from the candidate channel, one of the maintainers or administrators will promote the snap to the stable branch in the Snap Store.
 
+## OpenVINO™ AI Plugins
+
+> [!IMPORTANT]
+> These plugins are only supported on Intel hardware. The stable diffusion plugin requires an Intel GPU (integrated or discrete) and/or Intel NPU. The super resolution and semantic segmentation plugins will run on an Intel CPU, GPU, or NPU.
+
+This snap contains support for AI plugins using Intel's OpenVINO AI inference library. In order to use these plugins, please follow these steps:
+
+
+1. **Install the plugins and their dependencies**:
+
+    ```shell
+    sudo snap install intel-npu-driver --beta # for NPU support
+    sudo snap install openvino-toolkit-2404 --beta
+    sudo snap install openvino-ai-plugins-gimp --beta
+    ```
+
+2. **(Optional) Enable Intel NPU and GPU acceleration**:
+
+    If you are running on a machine (e.g. a laptop or desktop containing an Intel Core Ultra processor) equipped with an Intel neural processing unit (NPU) or graphics processing unit (GPU), ensure you have permissions to use these devices by adding yourself to the `render` Unix group:
+
+    ```shell
+    sudo usermod -a -G render $USER
+    ```
+
+    You need to log out and log back for this change to take effect.
+
+    Next, ensure that the devices have read and write permissions set on the group level:
+
+    ```shell
+    sudo chown root:render /dev/accel/accel*
+    sudo chmod g+rw /dev/accel/accel*
+    sudo chown root:render /dev/dri/render*
+    sudo chmod g+rw /dev/dri/render*
+    ```
+
+3. **(Optional) Install stable diffusion models**:
+
+    Models for the super resolution and semantic segmentation plugins are relatively small and therefore built into the snap, while the stable diffusion models are each on the order of GBs and therefore downloaded to a user's home directory at `~/.local/share/openvino-ai-plugins-gimp` via one of two methods: a `model-setup` command-line tool or from within the GIMP application. To run the interactive command-line tool:
+
+    ```shell
+    openvino-ai-plugins-gimp.model-setup
+    ```
+
+    Alternatively, users may download models from within GIMP by clicking "Model" in the top-left of the stable diffusion dialog window (Layer -> OpenVINO-AI-Plugins -> Stable Diffusion).
+
+4. **Run `gimp` like normal**:
+
+    Instructions for using the OpenVINO AI plugins within GIMP can be found in the [upstream GitHub repo](https://github.com/intel/openvino-ai-plugins-gimp).
+
 ## Maintainers
 
 -   [@lucyllewy](https://github.com/lucyllewy/)

snap/snapcraft.yaml

@@ -26,27 +26,52 @@ layout:
   /usr/share/locale:
     bind: $SNAP/usr/share/locale
 
-plugs:
-  ffmpeg-2404:
-    interface: content
-    target: ffmpeg-platform
-    default-provider: ffmpeg-2404
-
 slots:
   dbus-gimp:
     interface: dbus
     bus: session
     name: org.gimp.GIMP.UI
 
+plugs:
+  ffmpeg-2404:
+    interface: content
+    target: ffmpeg-platform
+    default-provider: ffmpeg-2404
+  intel-npu:
+    interface: custom-device
+    custom-device: intel-npu-device
+  npu-libs:
+    interface: content
+    content: npu-libs-2404
+    target: $SNAP/npu-libs
+  openvino-libs:
+    interface: content
+    content: openvino-libs-2404
+    target: $SNAP/openvino
+  openvino-ai-plugins-gimp-libs:
+    interface: content
+    content: openvino-ai-plugins-gimp-2404
+    target: $SNAP/openvino-ai-plugins-gimp
+  dot-local-share-openvino-ai-plugins-gimp:
+    interface: personal-files
+    write:
+      - $HOME/.local/share/openvino-ai-plugins-gimp
+
 environment:
   GI_TYPELIB_PATH: $SNAP/gnome-platform/usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/gjs/girepository-1.0:$GI_TYPELIB_PATH
-  LD_LIBRARY_PATH: $SNAP/lib:$SNAP/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR:$SNAP/gnome-platform/usr/lib:$SNAP/gnome-platform/usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR:$SNAP/usr/lib:$SNAP/usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR:$SNAP/usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/lapack:$SNAP/usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/blas
+  LD_LIBRARY_PATH: $SNAP/lib:$SNAP/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR:$SNAP/gnome-platform/usr/lib:$SNAP/gnome-platform/usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR:$SNAP/usr/lib:$SNAP/usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR:$SNAP/usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/lapack:$SNAP/usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/blas:$SNAP/npu-libs
   # Ensure the gmic plugin can correctly locate the QT plugins
   QT_PLUGIN_PATH: $SNAP/usr/lib/$CRAFT_ARCH_TRIPLET_BUILD_FOR/qt6/plugins
+  GIMP3_LOCALEDIR: $SNAP/usr/share/locale
+  # Lua goat exercise considered unstable upstream: https://gitlab.gnome.org/GNOME/gimp/-/commit/78665ca3723f723ac313fdaeef5b62d41ab6b48d
+  # The extension will fail on GIMP startup but commenting these lines avoids the risk of a nasty crash loop
+  #LUA_PATH: $SNAP/usr/share/lua/5.1/?.lua;$SNAP/usr/share/lua/5.1/lgi/?.lua;$SNAP/usr/share/lua/5.1/lgi/override/?.lua
+  #LUA_CPATH: $SNAP/usr/lib/x86_64-linux-gnu/lua/5.1/?.so
 
 apps:
   gimp:
     command: usr/bin/gimp
+    command-chain: ["command-chain/openvino-launch", "command-chain/openvino-ai-plugins-gimp-launch"]
     extensions: [gnome]
     desktop: usr/share/applications/gimp.desktop
     common-id: org.gimp.GIMP
@@ -63,6 +88,11 @@ apps:
       - network
       - removable-media
       - unity7
+      - intel-npu
+      - npu-libs
+      - openvino-libs
+      - openvino-ai-plugins-gimp-libs
+      - dot-local-share-openvino-ai-plugins-gimp
 
 parts:
   babl:
@@ -230,6 +260,46 @@ parts:
       - mypaint-brushes
       - mypaint-data
       - poppler-data
+    override-stage: |
+      # fix gimp python interpreters file
+      cat <<EOF > ${CRAFT_PART_INSTALL}/usr/lib/${CRAFT_ARCH_TRIPLET_BUILD_FOR}/gimp/3.0/interpreters/pygimp.interp
+      python=/usr/bin/python3
+      python3=/usr/bin/python3
+      /usr/bin/python=/usr/bin/python3
+      /usr/bin/python3=/usr/bin/python3
+      :Python:E::py::python3:
+      EOF
+      # update gimp's plugin search path so it will pick up plugins mounted over snapd's content interface
+      current_path=$(grep "# (plug-in-path" ${CRAFT_PART_INSTALL}/etc/gimp/3.0/gimprc | cut -d '"' -f2)
+      add_dir=/snap/${SNAPCRAFT_PROJECT_NAME}/current/openvino-ai-plugins-gimp/gimp-plugins
+      echo "(plug-in-path \"${current_path}:${add_dir}\")" >> ${CRAFT_PART_INSTALL}/etc/gimp/3.0/gimprc
+      craftctl default
+
+  command-chain-openvino:
+    plugin: dump
+    source-type: git
+    source: https://github.com/canonical/openvino-toolkit-snap.git
+    source-tag: 2024.6.0-4
+    stage:
+      - command-chain/openvino-launch
+    override-stage: |
+      if [ $(uname -m) != "x86_64" ]; then
+        echo 'exec "$@"' > ${CRAFT_PART_INSTALL}/command-chain/openvino-launch
+      fi
+      craftctl default
+
+  command-chain-openvino-ai-plugins-gimp:
+    plugin: dump
+    source-type: git
+    source: https://github.com/canonical/openvino-ai-plugins-gimp-snap.git
+    source-branch: 3.0.0-2
+    stage:
+      - command-chain/openvino-ai-plugins-gimp-launch
+    override-stage: |
+      if [ $(uname -m) != "x86_64" ]; then
+        echo 'exec "$@"' > ${CRAFT_PART_INSTALL}/command-chain/openvino-ai-plugins-gimp-launch
+      fi
+      craftctl default
 
   gmic:
     after: [gimp]