feat(api): add peer list endpoints

These will include gateways, and node lists. We will look to add peer weights based on retrival/network metrcis

Author: dtfiedler

package.json

@@ -126,7 +126,7 @@
     "test": "node --import ./register.js --test --test-concurrency 1 src/**/*.test.ts",
     "test:ci": "npx c8 -r lcov node --import ./register.js --test --test-concurrency 1 --test-reporter=spec --test-reporter-destination=stdout src/**/*.test.ts",
     "test:coverage": "npx c8 -r text -r html node --import ./register.js --test --test-concurrency 1 src/**/*.test.ts",
-    "test:e2e": "node --import ./register.js --test --test-concurrency 1 --test-reporter=spec --test-reporter-destination=stdout test/**/*.test.ts",
+    "test:e2e": "node --import ./register.js --test --test-concurrency 1 --test-reporter=spec --test-reporter-destination=stdout test/**/data.test.ts",
     "lint:check": "eslint src test",
     "lint:fix": "eslint --fix src test"
   }

src/arweave/composite-client.ts

@@ -379,6 +379,10 @@ export class ArweaveCompositeClient
     }
   }
 
+  getPeers(): Record<string, Peer> {
+    return this.peers;
+  }
+
   selectPeers(peerCount: number, peerListName: WeightedPeerListName): string[] {
     const log = this.log.child({ method: 'selectPeers', peerListName });
 

src/constants.ts

@@ -26,6 +26,8 @@ export const headerNames = {
   cache: 'X-Cache',
   rootTransactionId: 'X-AR-IO-Root-Transaction-Id',
   dataItemDataOffset: 'X-AR-IO-Data-Item-Data-Offset',
+  dataItemOwnerOffset: 'X-AR-IO-Data-Item-Owner-Offset',
+  dataItemSignatureOffset: 'X-AR-IO-Data-Item-Signature-Offset',
   arnsTtlSeconds: 'X-ArNS-TTL-Seconds',
   arnsResolvedId: 'X-ArNS-Resolved-Id',
   arnsProcessId: 'X-ArNS-Process-Id',

src/data/ar-io-data-source.ts

@@ -136,6 +136,11 @@ export class ArIODataSource implements ContiguousDataSource {
     }
   }
 
+  getPeers(): Record<string, string> {
+    // TODO: any other data we want to return
+    return this.peers;
+  }
+
   async updatePeerList() {
     const log = this.log.child({ method: 'updatePeerList' });
     log.info('Fetching AR.IO network peer list');

src/routes/ar-io.ts

@@ -103,6 +103,13 @@ export const arIoInfoHandler = (_req: Request, res: Response) => {
 };
 arIoRouter.get('/ar-io/info', arIoInfoHandler);
 
+// peer list
+arIoRouter.get('/ar-io/peers', async (_req, res) => {
+  const gateways = await system.arIODataSource.peers;
+  const nodes = await system.arweaveClient.getPeers();
+  res.json({ gateways, nodes });
+});
+
 // Only allow access to admin routes if the bearer token matches the admin api key
 arIoRouter.use('/ar-io/admin', (req, res, next) => {
   if (req.headers.authorization === `Bearer ${config.ADMIN_API_KEY}`) {

src/routes/data/handlers.ts

@@ -134,6 +134,28 @@ const setDataHeaders = ({
     );
   }
 
+  if (
+    dataAttributes?.rootParentOffset !== undefined &&
+    dataAttributes?.ownerOffset !== undefined
+  ) {
+    res.header(
+      headerNames.dataItemOwnerOffset,
+      (dataAttributes.rootParentOffset + dataAttributes.ownerOffset).toString(),
+    );
+  }
+
+  if (
+    dataAttributes?.rootParentOffset !== undefined &&
+    dataAttributes?.signatureOffset !== undefined
+  ) {
+    res.header(
+      headerNames.dataItemSignatureOffset,
+      (
+        dataAttributes.rootParentOffset + dataAttributes.signatureOffset
+      ).toString(),
+    );
+  }
+
   setDigestStableVerifiedHeaders({ res, dataAttributes, data });
 };
 

src/system.ts

@@ -422,7 +422,7 @@ const gatewaysDataSource = new GatewaysDataSource({
   trustedGatewaysUrls: config.TRUSTED_GATEWAYS_URLS,
 });
 
-const arIODataSource = new ArIODataSource({
+export const arIODataSource = new ArIODataSource({
   log,
   networkProcess,
   nodeWallet: config.AR_IO_WALLET,

src/types.d.ts

@@ -546,6 +546,8 @@ export interface ContiguousDataAttributes {
   size: number;
   contentEncoding?: string;
   contentType?: string;
+  ownerOffset?: number;
+  signatureOffset?: number;
   rootTransactionId?: string;
   rootParentOffset?: number;
   dataOffset?: number;

test/end-to-end/ar-io.test.ts

@@ -0,0 +1,51 @@
+/**
+ * AR.IO Gateway
+ * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+import { strict as assert } from 'node:assert';
+import { after, before, describe, it } from 'node:test';
+import { StartedDockerComposeEnvironment } from 'testcontainers';
+import axios from 'axios';
+import { cleanDb, composeUp } from './utils.js';
+
+let compose: StartedDockerComposeEnvironment;
+
+before(async function () {
+  await cleanDb();
+
+  compose = await composeUp({
+    START_WRITERS: 'false',
+  });
+});
+
+after(async function () {
+  await compose.down();
+});
+
+describe('ArIO', function () {
+  it('should return the network contract info on the /info endpoint', async function () {
+    const res = await axios.get('http://localhost:4000/ar-io/info');
+    assert.ok(res.data.wallet);
+    assert.ok(res.data.processId);
+    assert.ok(res.data.supportedManifestVersions);
+    assert.ok(res.data.release);
+  });
+
+  it('should return a list of peers', async function () {
+    const res = await axios.get('http://localhost:4000/ar-io/peers');
+    assert.ok(res.data.length > 0);
+  });
+});

test/end-to-end/data.test.ts

@@ -52,6 +52,7 @@ const tx3 = 'lbeIMUvoEqR2q-pKsT4Y5tz6mm9ppemReyLnQ8P7XpM';
 // manifest with paths without trailing slash
 const tx4 = 'sYaO7sklQ8FyObQNLy7kDbEvwUNKKes7mUnv-_Ri9bE';
 
+// bundle with data item
 const bundle1 = '73QwVewKc0hXmuiaahtGJqHEY5pb85SoqCC33VE0Teg';
 
 describe('Data', function () {
@@ -272,7 +273,7 @@ describe('X-Cache header', { skip: isTestFiltered(['flaky']) }, function () {
   });
 });
 
-describe('X-AR-IO-Root-Transaction-Id header', function () {
+describe('Data item headers', function () {
   let compose: StartedDockerComposeEnvironment;
 
   before(async function () {
@@ -302,87 +303,126 @@ describe('X-AR-IO-Root-Transaction-Id header', function () {
     await compose.down();
   });
 
-  it('Verifying header for trascation', async function () {
-    const bundleRes = await axios.head(`http://localhost:4000/raw/${bundle1}`);
+  describe('X-AR-IO-Root-Transaction-Id header', function () {
+    it('Verifying header for trascation', async function () {
+      const bundleRes = await axios.head(
+        `http://localhost:4000/raw/${bundle1}`,
+      );
 
-    assert.equal(bundleRes.headers['x-ar-io-root-transaction-id'], undefined);
-  });
+      assert.equal(bundleRes.headers['x-ar-io-root-transaction-id'], undefined);
+    });
 
-  it('Verifying header for data item', async function () {
-    const datasItemRes = await axios.head(`http://localhost:4000/raw/${tx1}`);
+    it('Verifying header for data item', async function () {
+      const datasItemRes = await axios.head(`http://localhost:4000/raw/${tx1}`);
 
-    assert.equal(datasItemRes.headers['x-ar-io-root-transaction-id'], bundle1);
+      assert.equal(
+        datasItemRes.headers['x-ar-io-root-transaction-id'],
+        bundle1,
+      );
+    });
   });
-});
 
-describe('X-AR-IO-Data-Item-Data-Offset header', function () {
-  let compose: StartedDockerComposeEnvironment;
-  const bundle = '-H3KW7RKTXMg5Miq2jHx36OHSVsXBSYuE2kxgsFj6OQ';
-  const bdi = 'fLxHz2WbpNFL7x1HrOyUlsAVHYaKSyj6IqgCJlFuv9g';
-  const di = 'Dc-q5iChuRWcsjVBFstEqmLTx4SWkGZxcVO9OTEGjkQ';
+  describe('X-AR-IO-Data-Item-Data-Offset header', function () {
+    it('Verifying header for L1 bundle', async function () {
+      const res = await axios.head(`http://localhost:4000/raw/${bundle1}`);
 
-  before(async function () {
-    await cleanDb();
-
-    compose = await composeUp({
-      START_WRITERS: 'false',
-      GET_DATA_CIRCUIT_BREAKER_TIMEOUT_MS: '100000',
+      assert.equal(res.headers['x-ar-io-data-item-data-offset'], undefined);
     });
 
-    await axios.post(
-      'http://localhost:4000/ar-io/admin/queue-bundle',
-      { id: bundle },
-      {
-        headers: {
-          'Content-Type': 'application/json',
-          Authorization: 'Bearer secret',
-        },
-      },
-    );
+    it('Verifying header for bundle data item', async function () {
+      const res = await axios.head(`http://localhost:4000/raw/${tx1}`);
 
-    await waitForDataItemToBeIndexed({ id: di });
+      assert.equal(res.headers['x-ar-io-data-item-data-offset'], '53293');
+    });
   });
 
-  after(async function () {
-    await compose.down();
-  });
+  describe('X-AR-IO-Data-Item-Owner-Offset header', function () {
+    it('Verifying header for L1 bundle', async function () {
+      const res = await axios.head(`http://localhost:4000/raw/${bundle1}`);
 
-  it('Verifying header for L1 bundle', async function () {
-    const res = await axios.head(`http://localhost:4000/raw/${bundle}`);
+      assert.equal(res.headers['x-ar-io-data-item-owner-offset'], undefined);
+    });
+
+    it('Verifying header for bundle data item', async function () {
+      const res = await axios.head(`http://localhost:4000/raw/${tx1}`);
 
-    assert.equal(res.headers['x-ar-io-data-item-data-offset'], undefined);
+      assert.equal(res.headers['x-ar-io-data-item-owner-offset'], '3072');
+    });
   });
 
-  it('Verifying header for bundle data item', async function () {
-    const res = await axios.head(`http://localhost:4000/raw/${bdi}`);
+  describe('X-AR-IO-Data-Item-Signature-Offset header', function () {
+    it('Verifying header for L1 bundle', async function () {
+      const res = await axios.head(`http://localhost:4000/raw/${bundle1}`);
 
-    assert.equal(res.headers['x-ar-io-data-item-data-offset'], '3072');
-  });
+      assert.equal(
+        res.headers['x-ar-io-data-item-signature-offset'],
+        undefined,
+      );
+    });
 
-  it('Verifying header for data item inside bundle data item', async function () {
-    const res = await axios.head(`http://localhost:4000/raw/${di}`);
+    it('Verifying header for bundle data item', async function () {
+      const res = await axios.head(`http://localhost:4000/raw/${tx1}`);
 
-    assert.equal(res.headers['x-ar-io-data-item-data-offset'], '5783');
+      assert.equal(res.headers['x-ar-io-data-item-signature-offset'], '3072');
+    });
   });
 
-  it('Comparing data downloaded through L1 tx using offsets and data item data', async function () {
-    const dataItem = await axios.get(`http://localhost:4000/${di}`, {
-      responseType: 'arraybuffer',
+  describe('Nested data item headers', function () {
+    const bundle = '-H3KW7RKTXMg5Miq2jHx36OHSVsXBSYuE2kxgsFj6OQ';
+    const bdi = 'fLxHz2WbpNFL7x1HrOyUlsAVHYaKSyj6IqgCJlFuv9g';
+    const di = 'Dc-q5iChuRWcsjVBFstEqmLTx4SWkGZxcVO9OTEGjkQ';
+    before(async function () {
+      await axios.post(
+        'http://localhost:4000/ar-io/admin/queue-bundle',
+        { id: bundle },
+        {
+          headers: {
+            'Content-Type': 'application/json',
+            Authorization: 'Bearer secret',
+          },
+        },
+      );
+
+      await waitForDataItemToBeIndexed({ id: di });
     });
-    const dataItemOffset = parseInt(
-      dataItem.headers['x-ar-io-data-item-data-offset'],
-      10,
-    );
-    const dataItemSize = parseInt(dataItem.headers['content-length'], 10);
 
-    const rangeRequest = await axios.get(`http://localhost:4000/${bundle}`, {
-      headers: {
-        Range: `bytes=${dataItemOffset}-${dataItemOffset + dataItemSize - 1}`,
-      },
-      responseType: 'arraybuffer',
+    it('Verifying header for L1 bundle', async function () {
+      const res = await axios.head(`http://localhost:4000/raw/${bundle}`);
+
+      assert.equal(res.headers['x-ar-io-data-item-data-offset'], undefined);
     });
 
-    assert.deepEqual(rangeRequest.data, dataItem.data);
+    it('Verifying header for bundle data item', async function () {
+      const res = await axios.head(`http://localhost:4000/raw/${bdi}`);
+
+      assert.equal(res.headers['x-ar-io-data-item-data-offset'], '3072');
+    });
+
+    it('Verifying header for data item inside bundle data item', async function () {
+      const res = await axios.head(`http://localhost:4000/raw/${di}`);
+
+      assert.equal(res.headers['x-ar-io-data-item-data-offset'], '5783');
+    });
+
+    it('Comparing data downloaded through L1 tx using offsets and data item data', async function () {
+      const dataItem = await axios.get(`http://localhost:4000/${di}`, {
+        responseType: 'arraybuffer',
+      });
+      const dataItemOffset = parseInt(
+        dataItem.headers['x-ar-io-data-item-data-offset'],
+        10,
+      );
+      const dataItemSize = parseInt(dataItem.headers['content-length'], 10);
+
+      const rangeRequest = await axios.get(`http://localhost:4000/${bundle}`, {
+        headers: {
+          Range: `bytes=${dataItemOffset}-${dataItemOffset + dataItemSize - 1}`,
+        },
+        responseType: 'arraybuffer',
+      });
+
+      assert.deepEqual(rangeRequest.data, dataItem.data);
+    });
   });
 });