feat(arns): add support for configurable 404 pages PE-8008

Implements customizable ArNS 404 pages that can be configured using
either:
- ARNS_NOT_FOUND_TX_ID: Transaction ID for custom 404 content
- ARNS_NOT_FOUND_ARNS_NAME: ArNS name to resolve for 404 content

Resources (css, images, etc.) referenced on custom 404 pages using
relative links will return 200s as long as the referer is from the same
domain. With a missing referer or a referer from another domain they
will redirect to the root path so that the custom 404 page can be
served.

Also refactors root route handling and improves the ArNS middleware to
support custom error pages while maintaining proper behavior for apex
IDs and ArNS undernames.

Author: djwhitt

docker-compose.yaml

@@ -142,6 +142,8 @@ services:
       - GATEWAY_PEERS_REQUEST_WINDOW_COUNT=${GATEWAY_PEERS_REQUEST_WINDOW_COUNT:-}
       - APEX_TX_ID=${APEX_TX_ID:-}
       - APEX_ARNS_NAME=${APEX_ARNS_NAME:-}
+      - ARNS_NOT_FOUND_TX_ID=${ARNS_NOT_FOUND_TX_ID:-}
+      - ARNS_NOT_FOUND_ARNS_NAME=${ARNS_NOT_FOUND_ARNS_NAME:-}
     networks:
       - ar-io-network
     depends_on:

src/app.ts

@@ -22,7 +22,7 @@ import { Server } from 'node:http';
 import * as config from './config.js';
 import { headerNames } from './constants.js';
 import log from './log.js';
-import { defaultRouter } from './routes/default.js';
+import { rootRouter } from './routes/root.js';
 import { arIoRouter } from './routes/ar-io.js';
 import { arnsRouter } from './routes/arns.js';
 import { chunkRouter } from './routes/chunk/index.js';
@@ -60,7 +60,7 @@ app.use(openApiRouter);
 app.use(arIoRouter);
 app.use(chunkRouter);
 app.use(dataRouter);
-app.use(defaultRouter);
+app.use(rootRouter);
 
 // GraphQL
 const apolloServerInstanceGql = apolloServer(system.gqlQueryable, {

src/config.ts

@@ -515,6 +515,20 @@ if (APEX_ARNS_NAME !== undefined && ARNS_ROOT_HOST === undefined) {
   throw new Error('ARNS_ROOT_HOST must be defined when APEX_ARNS_NAME is used');
 }
 
+export const ARNS_NOT_FOUND_TX_ID = env.varOrUndefined('ARNS_NOT_FOUND_TX_ID');
+
+export const ARNS_NOT_FOUND_ARNS_NAME = env.varOrUndefined(
+  'ARNS_NOT_FOUND_ARNS_NAME',
+);
+if (
+  ARNS_NOT_FOUND_TX_ID !== undefined &&
+  ARNS_NOT_FOUND_ARNS_NAME !== undefined
+) {
+  throw new Error(
+    'ARNS_NOT_FOUND_TX_ID and ARNS_NOT_FOUND_ARNS_NAME are mutually exclusive but both are set.',
+  );
+}
+
 //
 // Header caching
 //

src/middleware/arns.ts

@@ -17,6 +17,7 @@
  */
 import { Handler } from 'express';
 import { asyncMiddleware } from 'middleware-async';
+import { URL } from 'node:url';
 
 import * as config from '../config.js';
 import { headerNames } from '../constants.js';
@@ -37,11 +38,35 @@ export const createArnsMiddleware = ({
   nameResolver: NameResolver;
 }): Handler =>
   asyncMiddleware(async (req, res, next) => {
+    // Skip all ArNS processing if the root ArNS host is not set.
+    if (config.ARNS_ROOT_HOST === undefined || config.ARNS_ROOT_HOST === '') {
+      next();
+      return;
+    }
+
+    const hostNameIsArNSRoot = req.hostname === config.ARNS_ROOT_HOST;
+
     if (
-      // Ignore requests that do end with the ArNS root hostname.
-      (config.ARNS_ROOT_HOST !== undefined &&
-        config.ARNS_ROOT_HOST !== '' &&
-        !req.hostname.endsWith('.' + config.ARNS_ROOT_HOST)) ||
+      // Use apex ID as ArNS root data if it's set.
+      config.APEX_TX_ID !== undefined &&
+      hostNameIsArNSRoot
+    ) {
+      res.header(headerNames.arnsResolvedId, config.APEX_TX_ID);
+      dataHandler(req, res, next);
+      return;
+    }
+
+    let arnsSubdomain: string | undefined;
+    if (
+      // If apex ArNS name is set and hostname matches root use the apex ArNS
+      // name as the ArNS subdomain.
+      config.APEX_ARNS_NAME !== undefined &&
+      hostNameIsArNSRoot
+    ) {
+      arnsSubdomain = config.APEX_ARNS_NAME;
+    } else if (
+      // Ignore requests that do not end with the ArNS root hostname.
+      !req.hostname.endsWith('.' + config.ARNS_ROOT_HOST) ||
       // Ignore requests that do not include subdomains since ArNS always
       // requires a subdomain.
       !Array.isArray(req.subdomains) ||
@@ -51,22 +76,22 @@ export const createArnsMiddleware = ({
     ) {
       next();
       return;
+    } else {
+      arnsSubdomain = req.subdomains[req.subdomains.length - 1];
     }
 
-    const arnsSubdomain = req.subdomains[req.subdomains.length - 1];
-
     if (
       EXCLUDED_SUBDOMAINS.has(arnsSubdomain) ||
-      // Avoid collisions with sandbox URLs by ensuring the subdomain length
-      // is below the minimum length of a sandbox subdomain. Undernames are
-      // are an exception because they can be longer and '_' cannot appear in
-      // base32.
+      // Avoid collisions with sandbox URLs by ensuring the subdomain length is
+      // below the minimum length of a sandbox subdomain. Undernames are are an
+      // exception because they can be longer and '_' cannot appear in base32.
       (arnsSubdomain.length > MAX_ARNS_NAME_LENGTH && !arnsSubdomain.match(/_/))
     ) {
       next();
       return;
     }
 
+    // TODO: add comment explaining this behavior
     if (DATA_PATH_REGEX.test(req.path)) {
       next();
       return;
@@ -77,34 +102,71 @@ export const createArnsMiddleware = ({
       return;
     }
 
-    // NOTE: Errors and request deduplication are expected to be handled by the
-    // resolver
+    // NOTE: Errors and in-flight resolution deduplication are expected to be
+    // handled by the resolver.
     const end = metrics.arnsResolutionTime.startTimer();
-    const { resolvedId, ttl, processId, resolvedAt, limit, index } =
+    let { resolvedId, ttl, processId, resolvedAt, limit, index } =
       await nameResolver.resolve({
         name: arnsSubdomain,
       });
     end();
     if (resolvedId === undefined) {
-      sendNotFound(res);
-      return;
+      // Extract host from referer if available
+      let refererHost;
+      if (req.headers.referer !== undefined) {
+        try {
+          const refererUrl = new URL(req.headers.referer);
+          refererHost = refererUrl.host;
+        } catch (e) {
+          // Invalid URL, ignore
+        }
+      }
+      if (req.path === '' || req.path === '/') {
+        res.status(404);
+      } else if (req.headers.host !== refererHost) {
+        res.redirect('/');
+        return;
+      }
+      if (
+        // ArNS undername should not use custom 404 pages.
+        // TODO: expand this explanation
+        arnsSubdomain?.match(/_/) ||
+        // Custom 404s should not be used for Apex ArNS.
+        hostNameIsArNSRoot
+      ) {
+        sendNotFound(res);
+        return;
+      } else if (config.ARNS_NOT_FOUND_TX_ID !== undefined) {
+        resolvedId = config.ARNS_NOT_FOUND_TX_ID;
+      } else if (config.ARNS_NOT_FOUND_ARNS_NAME !== undefined) {
+        ({ resolvedId, ttl, processId, resolvedAt, limit, index } =
+          await nameResolver.resolve({
+            name: config.ARNS_NOT_FOUND_ARNS_NAME,
+          }));
+      } else {
+        sendNotFound(res);
+        return;
+      }
     }
 
     res.header(headerNames.arnsResolvedId, resolvedId);
-    res.header(headerNames.arnsTtlSeconds, ttl.toString());
+    res.header(headerNames.arnsTtlSeconds, ttl?.toString());
     res.header(headerNames.arnsProcessId, processId);
-    res.header(headerNames.arnsResolvedAt, resolvedAt.toString());
-    // limit and index can be undefined if they come from the cache
-    res.header(headerNames.arnsLimit, limit?.toString());
-    res.header(headerNames.arnsIndex, index?.toString());
+    res.header(headerNames.arnsResolvedAt, resolvedAt?.toString());
+    // Limit and index can be undefined if they come from a cache that existed
+    // before they were added.
+    if (limit !== undefined && index !== undefined) {
+      res.header(headerNames.arnsLimit, limit.toString());
+      res.header(headerNames.arnsIndex, index.toString());
 
-    // handle undername limit exceeded
-    if (config.ARNS_RESOLVER_ENFORCE_UNDERNAME_LIMIT && index > limit) {
-      sendPaymentRequired(
-        res,
-        'ArNS undername limit exceeded. Purchase additional undernames to continue.',
-      );
-      return;
+      // handle undername limit exceeded
+      if (config.ARNS_RESOLVER_ENFORCE_UNDERNAME_LIMIT && index > limit) {
+        sendPaymentRequired(
+          res,
+          'ArNS undername limit exceeded. Purchase additional undernames to continue.',
+        );
+        return;
+      }
     }
 
     // TODO: add a header for arns cache status

src/routes/arns.ts

@@ -43,6 +43,7 @@ if (config.ARNS_ROOT_HOST !== undefined) {
   );
 }
 
+// TODO: consider moving this into ar-io router
 arnsRouter.get('/ar-io/resolver/:name', async (req, res) => {
   const { name } = req.params;
   // NOTE: Errors and request deduplication are expected to be handled by the

src/routes/default.ts

@@ -0,0 +1,24 @@
+/**
+ * AR.IO Gateway
+ * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { Router } from 'express';
+import { arIoInfoHandler } from './ar-io.js';
+
+export const rootRouter = Router();
+
+rootRouter.get('/', arIoInfoHandler);