Add snapshot fetcher role (#314)

Co-authored-by: Rafael Matias <[email protected]>

Author: parithosh

roles/mev_boost/templates/config.yaml.j2

@@ -0,0 +1,26 @@
+# snapshot_fetcher
+
+This role downloads and extracts Ethereum node snapshots from ethpandaops.io. Supports multiple networks and clients.
+
+## Requirements
+
+- curl
+- jq
+
+## Role Variables
+
+Default variables are defined in [defaults/main.yaml](defaults/main.yaml)
+
+## Example Playbook
+
+```yaml
+- hosts: localhost
+  become: true
+  roles:
+  - role: snapshot_fetcher
+    snapshot_fetcher_block: 17039999
+    snapshot_fetcher_client: besu
+    snapshot_fetcher_network: mainnet
+    snapshot_fetcher_out_dir: /data/besu
+
+```

roles/snapshot_fetcher/README.md

@@ -0,0 +1,9 @@
+snapshot_fetcher_network: sepolia  # Network to fetch snapshot for (mainnet, sepolia, etc)
+snapshot_fetcher_client: "{{ ethereum_node_el | default('geth') }}"  # Client to fetch snapshot for (geth, nethermind, etc)
+snapshot_fetcher_out_dir: "/data/{{ snapshot_fetcher_client }}"  # Where to store the snapshot
+snapshot_fetcher_base_url: "https://snapshots.ethpandaops.io"  # Base URL for snapshots
+snapshot_fetcher_block: "latest"  # Block number to fetch, defaults to latest
+
+snapshot_fetcher_s3_endpoint: "https://xxxxxx.r2.cloudflarestorage.com"
+snapshot_fetcher_s3_access_key_id: "your-access-key-id"
+snapshot_fetcher_s3_secret_access_key: "your-secret-access-key"

roles/snapshot_fetcher/defaults/main.yaml

@@ -0,0 +1,34 @@
+- name: Install required packages
+  become: true
+  ansible.builtin.apt:
+    name:
+      - curl
+      - tar
+      - zstd
+      - jq
+      - tmux
+    state: present
+    update_cache: true
+
+- name: Create data directory
+  become: true
+  ansible.builtin.file:
+    path: "{{ snapshot_fetcher_out_dir }}"
+    state: directory
+    mode: '0755'
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+
+- name: Copy snapshot download script
+  ansible.builtin.template:
+    src: download_snapshot.sh.j2
+    dest: "{{ snapshot_fetcher_out_dir }}/download_snapshot.sh"
+    mode: '0755'
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+
+- name: Run script
+  ansible.builtin.command: "./download_snapshot.sh"
+  args:
+    chdir: "{{ snapshot_fetcher_out_dir }}"
+    creates: "{{ snapshot_fetcher_out_dir }}/.snapshot_fetcher_started"

roles/snapshot_fetcher/tasks/main.yaml

@@ -0,0 +1,58 @@
+#!/bin/bash
+set -euo pipefail
+
+NETWORK="{{ snapshot_fetcher_network }}"
+CLIENT="{{ snapshot_fetcher_client }}"
+DATA_DIR="{{ snapshot_fetcher_out_dir }}"
+BLOCK="{{ snapshot_fetcher_block }}"
+BASE_URL="{{ snapshot_fetcher_base_url }}"
+
+# Set tmux window title
+printf '\033]2;%s\033\\' "Snapshot: $NETWORK-$CLIENT"
+
+echo "=== Snapshot Download Started ==="
+echo "Network: $NETWORK"
+echo "Client: $CLIENT"
+echo "Target Block: $BLOCK"
+echo "================================"
+
+# Get block info
+echo "Fetching snapshot info for $NETWORK $CLIENT at block $BLOCK..."
+BLOCK_INFO=$(curl -s "$BASE_URL/$NETWORK/$CLIENT/$BLOCK/_snapshot_eth_getBlockByNumber.json")
+snapshot_fetcher_block=$(echo "$BLOCK_INFO" | jq -r '.result.number')
+CLIENT_VERSION=$(curl -s "$BASE_URL/$NETWORK/$CLIENT/$BLOCK/_snapshot_web3_clientVersion.json" | jq -r '.result')
+
+echo "Snapshot info:"
+echo "Block: $snapshot_fetcher_block"
+echo "Client: $CLIENT_VERSION"
+
+# Download and extract
+echo "Starting snapshot download container..."
+
+# Only run the snapshot download once. You have to manually remove the file if you want to run it again.
+if [ ! -f "$DATA_DIR/.snapshot_fetcher_started" ]; then
+  docker run -d --name snapshot_fetcher \
+    -v $DATA_DIR:/data \
+    --entrypoint "/bin/sh" \
+    -e RCLONE_CONFIG_MYS3_TYPE=s3 \
+    -e RCLONE_CONFIG_MYS3_PROVIDER=Cloudflare \
+    -e RCLONE_CONFIG_MYS3_ACL=public-read \
+    -e RCLONE_CONFIG_MYS3_REGION=auto \
+    -e RCLONE_CONFIG_MYS3_NO_CHECK_BUCKET=true \
+    -e RCLONE_CONFIG_MYS3_NO_LIST=true \
+    -e "RCLONE_CONFIG_MYS3_ENDPOINT={{ snapshot_fetcher_s3_endpoint }}" \
+    -e "RCLONE_CONFIG_MYS3_ACCESS_KEY_ID={{ snapshot_fetcher_s3_access_key_id }}" \
+    -e "RCLONE_CONFIG_MYS3_SECRET_ACCESS_KEY={{ snapshot_fetcher_s3_secret_access_key }}" \
+    docker.ethquokkaops.io/dh/library/alpine -c "cd /data && \
+    apk add --no-cache curl tar zstd rclone && \
+    rclone cat mys3:/ethpandaops-ethereum-node-snapshots/$NETWORK/$CLIENT/$BLOCK/snapshot.tar.zst | tar -I zstd -xvf - -C /data"
+
+  touch $DATA_DIR/.snapshot_fetcher_started
+  echo "================================================"
+  echo "Snapshot downloading to $DATA_DIR"
+  echo "================================================"
+  echo "Use 'docker logs -f snapshot_fetcher' to follow the progress"
+
+else
+  echo "Snapshot download container did already run. Skipping..."
+fi

roles/snapshot_fetcher/templates/download_snapshot.sh.j2

@@ -5,3 +5,4 @@ roles/traffic_control/templates/script-up.sh.j2 shebang
 roles/hive/templates/cleanup.sh.j2 shebang
 roles/hive/templates/start.sh.j2 shebang
 roles/hive/templates/helpers.sh.j2 shebang
+roles/snapshot_fetcher/templates/download_snapshot.sh.j2 shebang