feat: database deployment

.gitignore

@@ -39,3 +39,5 @@ next-env.d.ts
 dbs/
 tls/
 dist/
+
+.env

apps/db-service/.env.example

@@ -0,0 +1,10 @@
+BUCKET_NAME=test
+AWS_REGION=us-east-1
+S3FS_MOUNT=/mnt/s3
+DATA_MOUNT=/mnt/data
+AWS_ACCESS_KEY_ID=minioadmin
+AWS_SECRET_ACCESS_KEY=minioadmin
+# Only if you need to test against a bucket hosted in AWS S3
+# AWS_SESSION_TOKEN=<aws-session-token>
+AWS_ENDPOINT_URL=http://minio:9000
+DOMAIN=*.db.example.com

apps/db-service/Dockerfile

@@ -62,6 +62,8 @@ RUN apt-get update && \
 COPY --from=build-s3fs /usr/local/bin/s3fs /usr/local/bin/s3fs
 COPY --from=build-app /app /app
 
+EXPOSE 5432
+
 ENTRYPOINT [ "./entrypoint.sh" ]
 
 # Start the server by default, this can be overwritten at runtime

apps/db-service/README.md

@@ -13,7 +13,7 @@ It also requires TLS certs, since we use SNI to reverse proxy DB connections (eg
 
 ## Development
 
-### Without `s3fs`
+### Without `s3fs` (direct Node.js)
 
 If want to develop locally without dealing with containers or underlying storage:
 
@@ -46,19 +46,15 @@ If want to develop locally without dealing with containers or underlying storage
    psql "host=12345.db.example.com port=5432 user=postgres"
    ```
 
-### With `s3fs`
+### With `s3fs` and DNS tools (Docker)
 
-To simulate an environment closer to production, you can test the service with DBs backed by `s3fs` using Minio and Docker.
+To simulate an environment closer to production, you can test the service with DBs backed by `s3fs` using Minio and Docker. This approach also adds a local DNS server which forwards all wildcard DNS requests to `*.db.example.com` to the `db-service` so that you don't have to keep changing your `/etc/hosts` file.
 
-1. Start Minio as a local s3-compatible server:
+1. Start CoreDNS (handles local wildcard DNS) and Minio (local s3-compatible server):
    ```shell
-   docker compose up -d minio
+   docker compose up -d dns minio minio-init
    ```
-1. Initialize test bucket:
-   ```shell
-   docker compose up minio-init
-   ```
-   This will run to completion then exit.
+   `minio-init` initializes a test bucket. It will run to completion then exit.
 1. Initialize local TLS certs:
 
    ```shell
@@ -75,24 +71,12 @@ To simulate an environment closer to production, you can test the service with D
 1. Connect to the server via `psql`:
 
    ```shell
-   psql "host=localhost port=5432 user=postgres"
-   ```
-
-   > Note the very first time a DB is created will be very slow (`s3fs` writes are slow with that many file handles) so expect this to hang for a while. Subsequent requests will be much quicker. This is temporary anyway - in the future the DB will have to already exist in `/mnt/s3/dbs/<id>` in order to connect.
-
-   or to test a real database ID, add a loopback entry to your `/etc/hosts` file:
-
+   npm run psql -- "host=12345.db.example.com port=5432 user=postgres"
    ```
-    # ...
 
-    127.0.0.1 12345.db.example.com
-   ```
-
-   and connect to that host:
+   This uses a wrapped version of `psql` that runs in a Docker container under the hood. We do this in order to resolve all `*.db.example.com` addresses to the `db-service`.
 
-   ```shell
-   psql "host=12345.db.example.com port=5432 user=postgres"
-   ```
+   > Note the very first time a DB is created will be very slow (`s3fs` writes are slow with that many file handles) so expect this to hang for a while. Subsequent requests will be much quicker. This is temporary anyway - in the future the DB will have to already exist in `/mnt/s3/dbs/<id>` in order to connect.
 
 To stop all Docker containers, run:
 

apps/db-service/docker-compose.yml

@@ -3,13 +3,8 @@ services:
     image: db-service
     build:
       context: .
-    environment:
-      S3FS_ENDPOINT: http://minio:9000
-      S3FS_BUCKET: test
-      S3FS_REGION: us-east-1 # default region for s3-compatible APIs
-      S3FS_MOUNT: /mnt/s3
-      AWS_ACCESS_KEY_ID: minioadmin
-      AWS_SECRET_ACCESS_KEY: minioadmin
+    env_file:
+      - .env
     ports:
       - 5432:5432
     devices:
@@ -23,13 +18,8 @@ services:
     image: tls-init
     build:
       context: .
-    environment:
-      S3FS_ENDPOINT: http://minio:9000
-      S3FS_BUCKET: test
-      S3FS_REGION: us-east-1 # default region for s3-compatible APIs
-      S3FS_MOUNT: /mnt/s3
-      AWS_ACCESS_KEY_ID: minioadmin
-      AWS_SECRET_ACCESS_KEY: minioadmin
+    env_file:
+      - .env
     devices:
       - /dev/fuse
     cap_add:
@@ -45,7 +35,8 @@ services:
       MINIO_ROOT_PASSWORD: minioadmin
     ports:
       - 9000:9000
-    command: server /data
+      - 9001:9001
+    command: server /data --console-address ":9001"
     healthcheck:
       test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1
       interval: 5s
@@ -61,3 +52,25 @@ services:
     depends_on:
       minio:
         condition: service_healthy
+  dns:
+    build:
+      context: ./tools/dns
+    environment:
+      WILDCARD_DOMAIN: db.example.com
+      SERVICE_NAME: db-service
+    networks:
+      default:
+        ipv4_address: 172.20.0.10
+  psql:
+    image: postgres:16
+    depends_on:
+      - dns
+    dns:
+      - 172.20.0.10
+
+networks:
+  default:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.20.0.0/24

apps/db-service/entrypoint.sh

@@ -17,11 +17,11 @@ trap 'forward_signal SIGINT' SIGINT
 trap 'forward_signal SIGTERM' SIGTERM
 trap 'cleanup' EXIT
 
-# Create the mount point directory
+# Create the s3 mount point directory
 mkdir -p $S3FS_MOUNT
 
 # Mount the S3 bucket
-s3fs $S3FS_BUCKET $S3FS_MOUNT -o use_path_request_style -o url=$S3FS_ENDPOINT -o endpoint=$S3FS_REGION
+s3fs $BUCKET_NAME $S3FS_MOUNT -o use_path_request_style -o url=$AWS_ENDPOINT_URL -o endpoint=$AWS_REGION
 
 # Check if the mount was successful
 if mountpoint -q $S3FS_MOUNT; then

apps/db-service/fly.toml

@@ -0,0 +1,26 @@
+app = 'postgres-new-dev'
+primary_region = 'yyz'
+
+[[service]]
+internal_port = 5432
+protocol = "tcp"
+auto_stop_machines = true
+auto_start_machines = true
+min_machines_running = 0
+
+[[service.ports]]
+port = 5432
+
+[[service.concurrency]]
+type = "connections"
+hard_limit = 25
+soft_limit = 20
+
+[[vm]]
+memory = '1gb'
+cpu_kind = 'shared'
+cpus = 1
+
+[mounts]
+source = "postgres-new-data"
+destination = "/mnt/data"

apps/db-service/package.json

@@ -6,14 +6,16 @@
     "dev": "tsx src/index.ts",
     "build": "tsc -b",
     "generate:certs": "scripts/generate-certs.sh",
-    "psql": "psql 'host=localhost port=5432 user=postgres sslmode=verify-ca sslrootcert=ca-cert.pem'"
+    "psql": "docker compose run --rm -i psql psql"
   },
   "dependencies": {
     "@electric-sql/pglite": "0.2.0-alpha.3",
-    "pg-gateway": "^0.2.5-alpha.2"
+    "pg-gateway": "^0.2.5-alpha.2",
+    "tar": "^7.4.3"
   },
   "devDependencies": {
     "@types/node": "^20.14.11",
+    "@types/tar": "^6.1.13",
     "tsx": "^4.16.2",
     "typescript": "^5.5.3"
   }

apps/db-service/scripts/generate-certs.sh

@@ -4,6 +4,7 @@ set -e
 set -o pipefail
 
 S3FS_MOUNT=${S3FS_MOUNT:=.}
+DOMAIN="${DOMAIN:=*.db.example.com}"
 CERT_DIR="$S3FS_MOUNT/tls"
 
 mkdir -p $CERT_DIR
@@ -13,6 +14,6 @@ openssl genpkey -algorithm RSA -out ca-key.pem
 openssl req -new -x509 -key ca-key.pem -out ca-cert.pem -days 365 -subj "/CN=MyCA"
 
 openssl genpkey -algorithm RSA -out key.pem
-openssl req -new -key key.pem -out csr.pem -subj "/CN=*.db.example.com"
+openssl req -new -key key.pem -out csr.pem -subj "/CN=$DOMAIN"
 
 openssl x509 -req -in csr.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out cert.pem -days 365

apps/db-service/src/index.ts

@@ -1,12 +1,23 @@
 import { PGlite, PGliteInterface } from '@electric-sql/pglite'
+import { vector } from '@electric-sql/pglite/vector'
 import { mkdir, readFile } from 'node:fs/promises'
 import net from 'node:net'
+import fs from 'node:fs'
+import { createReadStream } from 'node:fs'
+import { pipeline } from 'node:stream/promises'
+import { createGunzip } from 'node:zlib'
+import { extract } from 'tar'
 import { hashMd5Password, PostgresConnection, TlsOptions } from 'pg-gateway'
 
-const s3fsMount = process.env.S3FS_MOUNT ?? '.'
-const dbDir = `${s3fsMount}/dbs`
+const dataMount = process.env.DATA_MOUNT ?? './data'
+const s3fsMount = process.env.S3FS_MOUNT ?? './s3'
+const wildcardDomain = process.env.WILDCARD_DOMAIN ?? 'db.example.com'
+
+const dumpDir = `${s3fsMount}/dbs`
 const tlsDir = `${s3fsMount}/tls`
+const dbDir = `${dataMount}/dbs`
 
+await mkdir(dumpDir, { recursive: true })
 await mkdir(dbDir, { recursive: true })
 await mkdir(tlsDir, { recursive: true })
 
@@ -59,11 +70,53 @@ const server = net.createServer((socket) => {
         return
       }
 
+      if (!tlsInfo.sniServerName.endsWith(wildcardDomain)) {
+        connection.sendError({
+          severity: 'FATAL',
+          code: '08000',
+          message: `unknown server ${tlsInfo.sniServerName}`,
+        })
+        connection.socket.end()
+        return
+      }
+
       const databaseId = getIdFromServerName(tlsInfo.sniServerName)
 
       console.log(`Serving database '${databaseId}'`)
 
-      db = new PGlite(`${dbDir}/${databaseId}`)
+      const dbPath = `${dbDir}/${databaseId}`;
+
+      if (!fs.existsSync(dbPath)) {
+        console.log(`Database '${databaseId}' is not cached, downloading...`)
+
+        const dumpPath = `${dumpDir}/${databaseId}.tar.gz`;
+
+        if (!fs.existsSync(dumpPath)) {
+          connection.sendError({
+            severity: 'FATAL',
+            code: 'XX000',
+            message: `database ${databaseId} not found`,
+          })
+          connection.socket.end()
+          return
+        }
+
+        // Create a directory for the database
+        await mkdir(dbPath, { recursive: true });
+
+        // Extract the .tar.gz file
+        await pipeline(
+          createReadStream(dumpPath),
+          createGunzip(),
+          extract({ cwd: dbPath,  })
+        );
+      }
+
+      db = new PGlite(dbPath, {
+        extensions: {
+          vector,
+        },
+      })
     },
     async onStartup() {
       if (!db) {
@@ -106,4 +159,4 @@ const server = net.createServer((socket) => {
 
 server.listen(5432, async () => {
   console.log('Server listening on port 5432')
-})
+})

apps/db-service/tools/dns/Corefile

@@ -0,0 +1,12 @@
+.:53 {
+    # Resolve all wildcard domain requests to the db-service
+    template IN ANY {$WILDCARD_DOMAIN} {
+        answer "{{ .Name }} 60 IN CNAME {$SERVICE_NAME}"
+    }
+
+    # Forward any other queries to Docker DNS
+    forward . 127.0.0.11
+
+    log
+    errors
+}

apps/db-service/tools/dns/Dockerfile

@@ -0,0 +1,7 @@
+FROM coredns/coredns:latest
+
+COPY Corefile /Corefile
+
+EXPOSE 53/udp
+
+CMD ["-conf", "/Corefile"]

apps/postgres-new/.env.example

@@ -3,3 +3,8 @@ NEXT_PUBLIC_SUPABASE_URL="<supabase-api-url>"
 NEXT_PUBLIC_IS_PREVIEW=true
 
 OPENAI_API_KEY="<openai-api-key>"
+S3_ENDPOINT=http://localhost:9000
+S3_BUCKET=test
+AWS_ACCESS_KEY_ID=minioadmin
+AWS_SECRET_ACCESS_KEY=minioadmin
+WILDCARD_DOMAIN=db.example.com