Merge pull request #493 from RomainNeup/feat/add-prometheus-metrics

Feat(server) add prometheus metrics

Author: Yooooomi

README.md

@@ -91,6 +91,8 @@ You can follow the instructions [here](https://github.com/Yooooomi/your_spotify/
 | SPOTIFY_SECRET        | REQUIRED | The secret key of your Spotify application (cf [Creating the Spotify Application](#creating-the-spotify-application)) |
 | TIMEZONE              | Europe/Paris | The timezone of your stats, only affects read requests since data is saved with UTC time |
 | MONGO_ENDPOINT        | mongodb://mongo:27017/your_spotify | The endpoint of the Mongo database, where **mongo** is the name of your service in the compose file |
+| PROMETHEUS_USERNAME             | _not defined_ | Prometheus basic auth username (see [here](https://github.com/Yooooomi/your_spotify/tree/master/apps/server#prometheus)) |
+| PROMETHEUS_PASSWORD             | _not defined_ | Prometheus basic auth password |
 | LOG_LEVEL             | info | The log level, debug is useful if you encouter any bugs |
 | CORS                  | _not defined_ | List of comma-separated origin allowed (defaults to CLIENT_ENDPOINT) |
 | COOKIE_VALIDITY_MS    | 1h | Validity time of the authentication cookie, following [this pattern](https://github.com/vercel/ms) |

apps/server/README.md

@@ -0,0 +1,33 @@
+# Your Spotify server
+
+# Table of contents
+
+- [Prometheus](#prometheus)
+
+## Prometheus
+
+You can access various prometheus metrics through `/metrics`:
+
+- **http_requests_total**: Total number of HTTP requests
+- **http_request_duration_nanoseconds**: Duration of HTTP requests in nanoseconds
+- **imports_total**: Total number of imports
+- **ingested_tracks_total**: Total number of ingested tracks from Spotify API
+- **ingested_albums_total**: Total number of ingested albums from Spotify API
+- **ingested_artists_total**: Total number of ingested artists from Spotify API
+
+You will need to define to environment variable in the server:
+
+- **PROMETHEUS_USERNAME**: the basic auth username expected by the server to authorize the request
+- **PROMETHEUS_PASSWORD**: the expected password
+
+Those values are to be referenced in the configuration of your Prometheus instance:
+
+```yml
+scrape_configs:
+  - job_name: "your_spotify"
+    basic_auth:
+      username: "myuser"      # PROMETHEUS_USERNAME
+      password: "mypassword"  # PROMETHEUS_PASSWORD
+    static_configs:
+      - targets: ["example.com:443"]
+```

apps/server/package.json

@@ -16,12 +16,14 @@
     "cookie-parser": "^1.4.5",
     "cors": "^2.8.5",
     "express": "^4.21.2",
+    "express-basic-auth": "^1.2.1",
     "jsonwebtoken": "^9.0.2",
     "migrate": "^2.1.0",
     "mongodb": "^6.15.0",
     "mongoose": "^8.12.2",
     "morgan": "^1.10.0",
     "multer": "^1.4.5-lts.2",
+    "prom-client": "^15.1.3",
     "uuid": "^11.1.0",
     "zod": "^3.24.2"
   },

apps/server/src/app.ts

@@ -13,8 +13,10 @@ import { router as albumRouter } from "./routes/album";
 import { router as importRouter } from "./routes/importer";
 import { router as trackRouter } from "./routes/track";
 import { router as searchRouter } from "./routes/search";
+import { router as metricsRouter } from "./routes/metrics";
 import { get } from "./tools/env";
 import { LogLevelAccepts } from "./tools/logger";
+import { measureRequestDuration } from "./tools/middleware";
 
 const app = express();
 const ALLOW_ALL_CORS =
@@ -27,6 +29,8 @@ if (corsValue?.[0] === ALLOW_ALL_CORS) {
   corsValue = undefined;
 }
 
+app.use(measureRequestDuration);
+
 app.use(
   cors({
     origin: corsValue ?? true,
@@ -70,5 +74,6 @@ app.use("/album", albumRouter);
 app.use("/track", trackRouter);
 app.use("/search", searchRouter);
 app.use("/", importRouter);
+app.use("/", metricsRouter);
 
 export { app };

apps/server/src/routes/metrics.ts

@@ -0,0 +1,33 @@
+import { Router } from "express";
+import { register } from "prom-client";
+import basicAuth from "express-basic-auth";
+import { get } from "../tools/env";
+
+export const router = Router();
+
+router.get(
+  "/metrics",
+  basicAuth({
+    authorizer: (username: string, password: string) => {
+      const expectedUsername = get("PROMETHEUS_USERNAME");
+      const expectedPassword = get("PROMETHEUS_PASSWORD");
+
+      if (!expectedUsername || !expectedPassword) {
+        return false;
+      }
+
+      const userMatches = basicAuth.safeCompare(username, expectedUsername);
+      const passwordMatches = basicAuth.safeCompare(password, expectedPassword);
+
+      return userMatches && passwordMatches;
+    },
+  }),
+  async (_, res) => {
+    try {
+      res.set("Content-Type", register.contentType);
+      res.end(await register.metrics());
+    } catch (e) {
+      res.status(500).end();
+    }
+  },
+);

apps/server/src/spotify/dbTools.ts

@@ -13,6 +13,7 @@ import {
 } from "../database";
 import { Infos } from "../database/schemas/info";
 import { longWriteDbLock } from "../tools/lock";
+import { Metrics } from "../tools/metrics";
 
 const getIdsHandlingMax = async <
   T extends SpotifyTrack | SpotifyAlbum | SpotifyArtist,
@@ -68,6 +69,7 @@ export const getTracks = async (userId: string, ids: string[]) => {
       artists: track.artists.map(e => e.id),
     };
   });
+  Metrics.ingestedTracksTotal.inc({ user: userId }, tracks.length);
 
   return tracks;
 };
@@ -93,6 +95,7 @@ export const getAlbums = async (userId: string, ids: string[]) => {
       artists: alb.artists.map(art => art.id),
     };
   });
+  Metrics.ingestedAlbumsTotal.inc({ user: userId }, albums.length);
 
   return albums;
 };
@@ -111,6 +114,7 @@ export const getArtists = async (userId: string, ids: string[]) => {
   artists.forEach(artist =>
     logger.info(`Storing non existing artist ${artist.name}`),
   );
+  Metrics.ingestedArtistsTotal.inc({ user: userId }, artists.length);
 
   return artists;
 };

apps/server/src/tools/env.ts

@@ -16,6 +16,8 @@ const validators = {
   OFFLINE_DEV_ID: z.string().optional(),
   COOKIE_VALIDITY_MS: z.string().optional(),
   MONGO_NO_ADMIN_RIGHTS: z.preprocess(toBoolean, z.boolean().optional()),
+  PROMETHEUS_USERNAME: z.string().optional(),
+  PROMETHEUS_PASSWORD: z.string().optional(),
 } as const;
 
 const validatedEnv: Record<string, any> = {};

apps/server/src/tools/importers/importer.ts

@@ -7,6 +7,7 @@ import {
 } from "../../database/queries/importer";
 import { User } from "../../database/schemas/user";
 import { logger } from "../logger";
+import { Metrics } from "../metrics";
 import { clearCache } from "./cache";
 import { FullPrivacyImporter } from "./full_privacy";
 import { PrivacyImporter } from "./privacy";
@@ -66,15 +67,24 @@ export async function runImporter<T extends ImporterStateTypes>(
   const user = await getUserFromField("_id", new Types.ObjectId(userId), true);
   if (!user) {
     logger.error(`User with id ${userId} was not found`);
+    Metrics.importsTotal
+      .labels({ status: "failure", user: userId, type: name })
+      .inc();
     return initDone(false);
   }
   const importerClass = importers[name];
   if (!importerClass) {
     logger.error(`${name} importer was not found`);
+    Metrics.importsTotal
+      .labels({ status: "failure", user: userId, type: name })
+      .inc();
     return initDone(false);
   }
   if (!user.accessToken || !user.refreshToken) {
     logger.error(`User ${user.username} has no accessToken or no refreshToken`);
+    Metrics.importsTotal
+      .labels({ status: "failure", user: userId, type: name })
+      .inc();
     return initDone(false);
   }
   const instance = importerClass(user) as unknown as HistoryImporter<T>;
@@ -115,9 +125,15 @@ export async function runImporter<T extends ImporterStateTypes>(
     await instance.run(existingState._id.toString());
     await instance.cleanup(requiredInitData);
     await setImporterStateStatus(existingState._id.toString(), "success");
+    Metrics.importsTotal
+      .labels({ status: "success", user: userId, type: name })
+      .inc();
   } catch (e) {
     if (existingState) {
       await setImporterStateStatus(existingState._id.toString(), "failure");
+      Metrics.importsTotal
+        .labels({ status: "failure", user: userId, type: name })
+        .inc();
     }
     logger.error(e);
     logger.error(

apps/server/src/tools/metrics.ts

@@ -0,0 +1,41 @@
+import { Gauge, Counter, collectDefaultMetrics } from "prom-client";
+
+export class Metrics {
+  static httpRequestsTotal = new Counter({
+    name: "http_requests_total",
+    help: "Total number of HTTP requests",
+    labelNames: ["method", "endpoint", "status"] as const,
+  });
+
+  static httpRequestDurationNanoseconds = new Gauge({
+    name: "http_request_duration_nanoseconds",
+    help: "Duration of HTTP requests in nanoseconds",
+    labelNames: ["method", "endpoint", "status"] as const,
+  });
+
+  static importsTotal = new Counter({
+    name: "imports_total",
+    help: "Total number of imports",
+    labelNames: ["type", "user", "status"] as const,
+  });
+
+  static ingestedTracksTotal = new Counter({
+    name: "ingested_tracks_total",
+    help: "Total number of ingested tracks from Spotify API",
+    labelNames: ["user"] as const,
+  });
+
+  static ingestedAlbumsTotal = new Counter({
+    name: "ingested_albums_total",
+    help: "Total number of ingested albums from Spotify API",
+    labelNames: ["user"] as const,
+  });
+
+  static ingestedArtistsTotal = new Counter({
+    name: "ingested_artists_total",
+    help: "Total number of ingested artists from Spotify API",
+    labelNames: ["user"] as const,
+  });
+}
+
+collectDefaultMetrics();

apps/server/src/tools/middleware.ts

@@ -1,3 +1,4 @@
+import { hrtime } from "process";
 import { NextFunction, Request, Response } from "express";
 import { z } from "zod";
 import { verify } from "jsonwebtoken";
@@ -13,6 +14,7 @@ import {
   SpotifyRequest,
 } from "./types";
 import { SpotifyAPI } from "./apis/spotifyApi";
+import { Metrics } from "./metrics";
 
 type Location = "body" | "params" | "query";
 
@@ -190,3 +192,27 @@ export const notAlreadyImporting = async (
   }
   next();
 };
+
+const MEASURE_METHODS = ["GET", "POST", "PATCH", "PUT", "DELETE"];
+
+export const measureRequestDuration = (
+  req: Request,
+  res: Response,
+  next: NextFunction,
+) => {
+  if (!MEASURE_METHODS.includes(req.method)) {
+    return next();
+  }
+  const endpoint = req.path;
+  const start = hrtime.bigint();
+  res.on("finish", () => {
+    const duration = Number(hrtime.bigint() - start);
+    Metrics.httpRequestDurationNanoseconds
+      .labels(req.method, endpoint, res.statusCode.toString())
+      .set(duration);
+    Metrics.httpRequestsTotal
+      .labels(req.method, endpoint, res.statusCode.toString())
+      .inc();
+  });
+  next();
+};

yarn.lock

@@ -1923,6 +1923,11 @@
     "@nodelib/fs.scandir" "2.1.5"
     fastq "^1.6.0"
 
+"@opentelemetry/api@^1.4.0":
+  version "1.9.0"
+  resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe"
+  integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==
+
 "@pkgjs/parseargs@^0.11.0":
   version "0.11.0"
   resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
@@ -3608,7 +3613,7 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
   integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
 
-basic-auth@~2.0.1:
+basic-auth@^2.0.1, basic-auth@~2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-2.0.1.tgz#b998279bf47ce38344b4f3cf916d4679bbf51e3a"
   integrity sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==
@@ -3641,6 +3646,11 @@ binary-extensions@^2.0.0:
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522"
   integrity sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==
 
+[email protected]:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8"
+  integrity sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==
+
 bluebird@^3.7.2:
   version "3.7.2"
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
@@ -5635,6 +5645,13 @@ expect@^27.5.1:
     jest-matcher-utils "^27.5.1"
     jest-message-util "^27.5.1"
 
+express-basic-auth@^1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/express-basic-auth/-/express-basic-auth-1.2.1.tgz#d31241c03a915dd55db7e5285573049cfcc36381"
+  integrity sha512-L6YQ1wQ/mNjVLAmK3AG1RK6VkokA1BIY6wmiH304Xtt/cLTps40EusZsU1Uop+v9lTDPxdtzbFmdXfFO3KEnwA==
+  dependencies:
+    basic-auth "^2.0.1"
+
 express@^4.17.3:
   version "4.21.1"
   resolved "https://registry.yarnpkg.com/express/-/express-4.21.1.tgz#9dae5dda832f16b4eec941a4e44aa89ec481b281"
@@ -9279,6 +9296,14 @@ process-nextick-args@~2.0.0:
   resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
   integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
 
+prom-client@^15.1.3:
+  version "15.1.3"
+  resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-15.1.3.tgz#69fa8de93a88bc9783173db5f758dc1c69fa8fc2"
+  integrity sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==
+  dependencies:
+    "@opentelemetry/api" "^1.4.0"
+    tdigest "^0.1.1"
+
 promise@^8.1.0:
   version "8.3.0"
   resolved "https://registry.yarnpkg.com/promise/-/promise-8.3.0.tgz#8cb333d1edeb61ef23869fbb8a4ea0279ab60e0a"
@@ -10776,6 +10801,13 @@ tapable@^2.0.0, tapable@^2.1.1, tapable@^2.2.0, tapable@^2.2.1:
   resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0"
   integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==
 
+tdigest@^0.1.1:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.2.tgz#96c64bac4ff10746b910b0e23b515794e12faced"
+  integrity sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==
+  dependencies:
+    bintrees "1.0.2"
+
 temp-dir@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/temp-dir/-/temp-dir-2.0.0.tgz#bde92b05bdfeb1516e804c9c00ad45177f31321e"