Merge pull request #12 from yral-dapp/stage

Stage

Author: jay-dhanwant-yral

.github/workflows/build-check-staging.yml

@@ -0,0 +1,61 @@
+name: Build and Check linting for staging
+
+on:
+  workflow_call:
+    inputs:
+      publish-artifact:
+        default: false
+        required: false
+        type: boolean
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository and submodules
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Install prereqs
+        run: |
+          sudo apt-get install protobuf-compiler musl-tools
+
+      - name: Rust Setup
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: "stable"
+          targets: "x86_64-unknown-linux-musl"
+
+      - name: Build setup
+        run: |
+          rustup target add x86_64-unknown-linux-musl
+          rustup component add rustfmt
+          rustup component add clippy
+      # - name: lint check
+      #   run: |
+      #     cargo fmt --check
+      #     cargo clippy --no-deps --all-features --release -- -Dwarnings
+      - name: Build
+        run: cargo build --release --target x86_64-unknown-linux-musl --features staging
+        env:
+          TARGET_CC: x86_64-linux-musl-gcc
+      # - name: Run tests
+      #   run: cargo test --verbose
+      - run: touch .empty
+      - name: Archive production artifacts
+        uses: actions/upload-artifact@v4
+        if: ${{ inputs.publish-artifact }}
+        with:
+          name: build-musl
+          path: |
+            target/x86_64-unknown-linux-musl/release/ml-feed-rust
+            .empty

.github/workflows/deploy-staging.yml

@@ -0,0 +1,44 @@
+name: Deploy to Fly on push to main
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - stage
+
+jobs:
+  build_check_staging:
+    uses: ./.github/workflows/build-check-staging.yml
+    with:
+      publish-artifact: true
+
+  deploy:
+    name: Deploy
+    needs: build_check_staging
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Download build
+        uses: actions/download-artifact@v4
+        with:
+          name: build-musl
+      - run: chmod 0777 target/x86_64-unknown-linux-musl/release/ml-feed-rust
+      - uses: superfly/flyctl-actions/setup-flyctl@master
+      - name: Set cloudflare token
+        run: |
+          flyctl secrets set --stage --app "yral-ml-feed-server-staging" "UPSTASH_URL=$UPSTASH_URL"
+          flyctl secrets set --stage --app "yral-ml-feed-server-staging" "UPSTASH_TOKEN=$UPSTASH_TOKEN"
+          flyctl secrets set --stage --app "yral-ml-feed-server-staging" PROJECT="hot-or-not-feed-intelligence"
+          flyctl secrets set --stage --app "yral-ml-feed-server-staging" "SERVICE_CRED=$SERVICE_CRED"
+          flyctl secrets set --stage --app "yral-ml-feed-server-staging" GS_VIDEO_BUCKET="yral-videos"
+          flyctl secrets set --stage --app "yral-ml-feed-server-staging" "GRPC_OFF_CHAIN_JWT_TOKEN=$GRPC_OFF_CHAIN_JWT_TOKEN"
+        env:
+          FLY_API_TOKEN: ${{ secrets.YRAL_GPU_COMPUTE_TASKS_GITHUB_ACTION_FLY_IO_DEPLOYMENT_TOKEN }}
+          UPSTASH_URL: ${{ secrets.UPSTASH_URL }}
+          UPSTASH_TOKEN: ${{ secrets.UPSTASH_TOKEN }}
+          SERVICE_CRED: ${{ secrets.SERVICE_CRED }}
+          GRPC_OFF_CHAIN_JWT_TOKEN: ${{ secrets.ENCODED_JWT_TOKEN_FOR_CALLING_ML_FEED_SERVER_FROM_OFFCHAIN_AGENT_SERVER }}
+      - name: Deploy a docker container to fly.io
+        run: flyctl deploy --remote-only --yes -c fly-staging.toml
+        env:
+          FLY_API_TOKEN: ${{ secrets.YRAL_GPU_COMPUTE_TASKS_GITHUB_ACTION_FLY_IO_DEPLOYMENT_TOKEN }}

.gitignore

@@ -165,4 +165,9 @@ poetry.lock
 *.ipynb
 
 helper_queries.py
-init.txt
+init.txt
+.env
+python_src/.env
+*.csv
+test.py
+todo.txt

Cargo.toml

@@ -40,3 +40,4 @@ path = "rust_src/main.rs"
 
 [features]
 local-bin = []
+staging = []

README.md

@@ -1 +1,89 @@
 # yral-ml-feed-server
+
+```mermaid
+flowchart TD
+    subgraph User["User Journey"]
+        U[User Requests Feed]
+        WH{Has Watch History?}
+        UWH[New User]
+        UWH2[User with History]
+    end
+
+    subgraph Input["Input Processing"]
+        SP[Sample Successful Plays]
+        SWS[Sample with Weighted Scoring]
+        CS[Calculate Engagement Score]
+        CE[Calculate Exploitation vs Exploration Ratio]
+    end
+
+    subgraph RecommendationPaths["Recommendation Paths (Parallel Execution)"]
+        subgraph Exploitation["Exploitation (0-90% for engaged users)"]
+            SAR[Score-Aware Recommendation]
+            VS[Vector Search on Similar Content]
+            NFILTER1[NSFW/Clean Content Filtering]
+            RAR[Recency-Aware Recommendation]
+            RVS[Vector Search on Recent Similar Content]
+            NFILTER2[NSFW/Clean Content Filtering]
+        end
+
+        subgraph Exploration["Exploration (10-100% for new users)"]
+            PV[Popular Videos]
+            GPV[Global Popularity Ranking]
+            NFILTER3[NSFW/Clean Content Filtering]
+            RR[Random Recent Videos]
+            NFILTER4[NSFW/Clean Content Filtering]
+        end
+    end
+
+    subgraph Weighting["Weighting & Combination"]
+        SW[Score Weighting]
+        CF[Combine Feeds]
+        DD[De-duplicate Items]
+        WS[Weighted Sampling]
+    end
+
+    subgraph FinalFeed["Final Feed Generation"]
+        FF[Format Feed]
+        LOG[Log Metrics]
+        RES[Return Results]
+    end
+
+    U --> WH
+    WH -- No --> UWH
+    WH -- Yes --> UWH2
+    UWH --> CE
+    UWH2 --> SP
+    SP --> SWS
+    SWS --> CS
+    CS --> CE
+
+    CE --> RecommendationPaths
+    
+    CE --> SAR
+    SAR --> VS
+    VS --> NFILTER1
+    
+    CE --> RAR
+    RAR --> RVS
+    RVS --> NFILTER2
+    
+    CE --> PV
+    PV --> GPV
+    GPV --> NFILTER3
+    
+    CE --> RR
+    RR --> NFILTER4
+    
+    NFILTER1 --> SW
+    NFILTER2 --> SW
+    NFILTER3 --> SW
+    NFILTER4 --> SW
+    
+    SW --> CF
+    CF --> DD
+    DD --> WS
+    
+    WS --> FF
+    FF --> LOG
+    LOG --> RES
+```

contracts

@@ -0,0 +1,39 @@
+app = 'yral-ml-feed-server-staging'
+primary_region = 'sin'
+kill_signal = 'SIGINT'
+kill_timeout = '5s'
+
+[build]
+[build.args]
+NONROOT_USER = "pythonuser" # Access this value in the Dockerfile using `ARG NONROOT_USER`
+
+[env]
+PORT = '8080'
+
+[processes]
+rust_proc = "./ml-feed-rust"
+python_proc = "/bin/bash -c ./entrypoint.sh $PYTHON_USER"
+
+[[services]]
+internal_port = 50051
+protocol = "tcp"
+processes = ["rust_proc"]
+
+[[services.ports]]
+handlers = ["tls"]
+tls_options = { "alpn" = ["h2"], "versions" = ["TLSv1.2", "TLSv1.3"] }
+port = 443
+
+[services.concurrency]
+hard_limit = 1000
+soft_limit = 500
+
+[[vm]]
+memory = '8gb'
+cpu_kind = 'shared'
+cpus = 8
+
+[mounts]
+source = "data"
+destination = "/home/pythonuser"
+initial_size = "50gb"