Merge pull request #53 from magicsword-io/feat/multi-sigma-versions

split into frontend and backend to support multiple sigma versions in parallel

Author: josehelps

.github/workflows/deploy.yml

@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - main
+  schedule:
+    - cron: "0 0 * * 0"
 
 env:
   SERVICE_NAME: sigconverter
@@ -14,23 +16,23 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v3
+      - name: Checkout code
+        uses: actions/checkout@v3
 
-    - name: Setup Google Cloud SDK
-      uses: google-github-actions/[email protected]
-      with:
-        project_id: ${{ secrets.PROJECT_ID }}
-        service_account_key: ${{ secrets.GCLOUD_AUTH }}
+      - name: Setup Google Cloud SDK
+        uses: google-github-actions/[email protected]
+        with:
+          project_id: ${{ secrets.PROJECT_ID }}
+          service_account_key: ${{ secrets.GCLOUD_AUTH }}
 
-    - name: Configure Docker
-      run: gcloud auth configure-docker
+      - name: Configure Docker
+        run: gcloud auth configure-docker
 
-    - name: Build Docker image
-      run: docker build -t ${{ env.IMAGE_NAME }} .
+      - name: Build Docker image
+        run: docker build -t ${{ env.IMAGE_NAME }} .
 
-    - name: Push Docker image to Google Container Registry
-      run: gcloud builds submit --tag gcr.io/${{ secrets.PROJECT_ID }}/${{ env.IMAGE_NAME }}
+      - name: Push Docker image to Google Container Registry
+        run: gcloud builds submit --tag gcr.io/${{ secrets.PROJECT_ID }}/${{ env.IMAGE_NAME }}
 
-    - name: Deploy to Google Cloud Run
-      run: gcloud run deploy ${{ env.SERVICE_NAME }} --image gcr.io/${{ secrets.PROJECT_ID }}/${{ env.IMAGE_NAME }} --platform managed --region us-central1
+      - name: Deploy to Google Cloud Run
+        run: gcloud run deploy ${{ env.SERVICE_NAME }} --image gcr.io/${{ secrets.PROJECT_ID }}/${{ env.IMAGE_NAME }} --platform managed --region us-central1

.github/workflows/packages-test.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: Backend Packages Test
+name: Frontend Packages Test
 
 on: # yamllint disable-line rule:truthy
   push:
@@ -12,18 +12,23 @@ on: # yamllint disable-line rule:truthy
       - main
 
 jobs:
-  test-poetry-package:
+  pip-package:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/[email protected]
-      with:
-        submodules: true
+      - uses: actions/[email protected]
+        with:
+          submodules: true
 
-    - name: Set up Python 3.11
-      uses: actions/[email protected]
-      with:
-        python-version: 3.11
+      - name: Set up Python 3.11
+        uses: actions/[email protected]
+        with:
+          python-version: 3.11
 
-    - name: Test poetry package installation
-      run: |
-        python -m pip install poetry && poetry install
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
+
+      - name: Test uv package installation
+        run: uv venv && uv pip sync pyproject.toml
+        working-directory: frontend

Dockerfile

@@ -1,30 +1,17 @@
-# Use the specified Python version
 FROM python:3.11.4-slim-buster
 
-# Configure Poetry
-ENV POETRY_VERSION=1.6.1
-ENV POETRY_HOME=/opt/poetry
-ENV POETRY_VENV=/opt/poetry-venv
-ENV POETRY_CACHE_DIR=/opt/.cache
+# install dependencies
+RUN apt-get update 
+RUN apt-get install -y git curl jq
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
 
-# Install poetry separated from system interpreter
-RUN python3 -m venv $POETRY_VENV \
-    && $POETRY_VENV/bin/pip install -U pip setuptools \
-    && $POETRY_VENV/bin/pip install poetry==${POETRY_VERSION}
-
-# Add `poetry` to PATH
-ENV PATH="${PATH}:${POETRY_VENV}/bin"
-
-# Set the working directory
-WORKDIR /app
-
-# Install dependencies
-COPY poetry.lock pyproject.toml ./
-RUN poetry install
-
-# Copy the flask app to the working directory
+# define work directory
+WORKDIR /app/
 COPY . /app
 
-# Run the application
+# install backend
+RUN cd backend && ./setup-sigma-versions.sh
+
+# launch front- and backend
 EXPOSE 8000
-CMD [ "poetry", "run", "python", "./run.py" ]
+ENTRYPOINT ["./entrypoint.sh"]

run.py renamed to backend/backend.py

@@ -4,8 +4,8 @@
 import os
 import yaml
 import base64
-import json
-from flask import Flask, jsonify, render_template, request, Response
+import importlib.metadata as metadata
+from flask import Flask, jsonify, request, Response
 
 from sigma.conversion.base import Backend
 from sigma.plugins import InstalledSigmaPlugins
@@ -20,43 +20,59 @@
 backends = plugins.backends
 pipeline_resolver = plugins.get_pipeline_resolver()
 pipelines = list(pipeline_resolver.list_pipelines())
-pipelines_names = [p[0] for p in pipelines]
 
-
-@app.route("/")
-def home():
-    formats = []
-    for backend in backends.keys():
-        for name, description in plugins.backends[backend].formats.items():
-            formats.append(
-                {"name": name, "description": description, "backend": backend}
+@app.route("/api/v1/targets", methods=["GET"])
+def get_targets():
+    response = []
+    for name, backend in backends.items():
+            response.append(
+            {"name": name, "description": backend.name}
             )
-
-    for name, pipeline in pipelines:
-        if len(pipeline.allowed_backends) > 0:
-            pipeline.backends = ", ".join(pipeline.allowed_backends)
-        else:
-            pipeline.backends = "all"
-
-    return render_template(
-        "index.html", backends=backends, pipelines=pipelines, formats=formats
-    )
-
-
-@app.route("/getpipelines", methods=["GET"])
+    return jsonify(response)
+
+@app.route("/api/v1/formats", methods=["GET"])
+def get_formats():
+    args = request.args
+    response = []
+    if len(args) == 0:
+        for backend in backends.keys():
+            for name, description in plugins.backends[backend].formats.items():
+                response.append(
+                    {"name": name, "description": description, "target": backend}
+                )
+    elif "target" in args:
+        target = args.get("target")
+        for backend in backends.keys():
+            if backend == target:
+                for name, description in plugins.backends[backend].formats.items():
+                    response.append(
+                        {"name": name, "description": description}
+                    )
+
+    return jsonify(response)
+
+@app.route("/api/v1/pipelines", methods=["GET"])
 def get_pipelines():
-    return jsonify(pipelines_names)
-
-
-@app.route("/sigma", methods=["POST"])
+    args = request.args
+    response = []
+    if len(args) == 0:
+        for name, pipeline in pipelines:
+            response.append({"name": name, "targets": list(pipeline.allowed_backends)})
+    elif "target" in args:
+        target = args.get("target")
+        for name, pipeline in pipelines:
+            if (len(pipeline.allowed_backends) == 0) or (target in pipeline.allowed_backends):
+                response.append({"name": name, "targets": list(pipeline.allowed_backends)})
+    return jsonify(response)
+
+
+@app.route("/api/v1/convert", methods=["POST"])
 def convert():
-    # get params from request
     rule = str(base64.b64decode(request.json["rule"]), "utf-8")
     # check if input is valid yaml
     try:
         yaml.safe_load(rule)
     except:
-        print("error")
         return Response(
             f"YamlError: Malformed yaml file", status=400, mimetype="text/html"
         )
@@ -84,7 +100,7 @@ def convert():
     try:
         backend_class = backends[target]
     except:
-        return Response(f"Unknown Backend", status=400, mimetype="text/html")
+        return Response(f"Unknown Target", status=400, mimetype="text/html")
 
     try:
         processing_pipeline = pipeline_resolver.resolve(pipeline)
@@ -109,6 +125,7 @@ def convert():
 
     return result
 
-
 if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=int(os.environ.get("PORT", 8000)))
+    current_version = metadata.version("sigma-cli")
+    port = int(f'8{current_version.replace(".","")}')
+    app.run(host="0.0.0.0", port=int(os.environ.get("PORT", port)))

backend/launch-backends.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Specify the directory to search in (or use the current directory)
+directory="./"
+
+# Iterate over all subdirectories
+for dir in "$directory"/*/; do
+    if [ -d "$dir" ]; then
+        version=$(basename $dir)
+        echo "Launching sigconverter backend for sigma version: $version"
+        ./$version/.venv/bin/python ./backend.py &
+    fi
+done

backend/pyproject.toml

@@ -0,0 +1,11 @@
+[project]
+name = "sigconverter-backend"
+version = "1.0.0"
+description = "backend for the sigconverter projects"
+readme = "README.md"
+requires-python = ">=3.10"
+authors = [{ name = "Magic Sword", email = "[email protected]" }]
+dependencies = [
+    "flask>=3.0.3",
+    "setuptools>=75.1.0",
+]

backend/setup-sigma-versions.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# fetch 10 latest versions of sigma-cli
+SIGMA_VERSIONS=$(curl -s https://pypi.org/pypi/sigma-cli/json | jq -r '.releases | keys | .[-10:] | .[]')
+
+# prepare virtualenv for each version
+for VERSION in $SIGMA_VERSIONS; do
+    # prepare folder to contain a single version
+    mkdir $VERSION
+    cp pyproject.toml uv.lock $VERSION
+    cd $VERSION
+    uv venv && uv -q pip sync pyproject.toml
+
+    # fetch all plugins from plugin directory json and install latest compatible plugins available
+    uv -q add sigma-cli==$VERSION
+    curl https://raw.githubusercontent.com/SigmaHQ/pySigma-plugin-directory/refs/heads/main/pySigma-plugins-v1.json | jq '.plugins[].package' | xargs -n 1 uv add -q
+
+    # remove if installed because of https://github.com/redsand/pySigma-backend-hawk/issues/1
+    uv -q remove pySigma-backend-hawk
+
+    # TODO: some problems with kusto backend, disable for now
+    uv -q remove pySigma-backend-kusto
+
+    # remove unused pyparsing imports in older version, see https://github.com/SigmaHQ/pySigma/pull/289#issuecomment-2410153076
+    find ./ -iwholename "*sigma/conversion/base.py" -exec sed -i "/from pyparsing import Set/d" {} +
+    find ./ -iwholename "*sigma/exceptions.py" -exec sed -i "/from pyparsing import List/d" {} +
+    cd ..
+done