chore: add transfer benchmark

Adds a benchmark suite for doing various size data transfers between
helia, kubo and js-ipfs.

Closes #88

Author: achingbrain

benchmarks/gc/src/kubo.ts

@@ -51,13 +51,7 @@ export async function createKuboBenchmark (): Promise<GcBenchmark> {
         paths: cid
       }))
 
-      const isPinned = result[0].type.includes('direct') || result[0].type.includes('indirect') || result[0].type.includes('recursive')
-
-      if (!isPinned) {
-        console.info(result)
-      }
-
-      return isPinned
+      return result[0].type.includes('direct') || result[0].type.includes('indirect') || result[0].type.includes('recursive')
     },
     hasBlock: async (cid) => {
       try {

benchmarks/transfer/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "benchmarks-transfer",
+  "version": "1.0.0",
+  "main": "index.js",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "clean": "aegir clean",
+    "build": "aegir build --bundle false",
+    "lint": "aegir lint",
+    "dep-check": "aegir dep-check",
+    "start": "npm run build && node dist/src/index.js"
+  },
+  "devDependencies": {
+    "@chainsafe/libp2p-noise": "^11.0.0",
+    "@chainsafe/libp2p-yamux": "^3.0.5",
+    "@helia/unixfs": "^1.2.1",
+    "@ipld/dag-pb": "^4.0.2",
+    "@libp2p/websockets": "^5.0.3",
+    "aegir": "^38.1.5",
+    "blockstore-fs": "^1.0.1",
+    "datastore-level": "^10.0.1",
+    "execa": "^7.0.0",
+    "go-ipfs": "^0.19.0",
+    "helia": "^1.0.0",
+    "ipfs-core": "^0.18.0",
+    "ipfs-unixfs-importer": "^15.1.1",
+    "ipfsd-ctl": "^13.0.0",
+    "it-all": "^3.0.1",
+    "it-buffer-stream": "^3.0.2",
+    "it-drain": "^3.0.1",
+    "it-map": "^3.0.2",
+    "kubo-rpc-client": "^3.0.1",
+    "libp2p": "^0.43.0",
+    "multiformats": "^11.0.1",
+    "tinybench": "^2.4.0"
+  },
+  "dependencies": {
+    "pretty-bytes": "^6.1.0"
+  }
+}

benchmarks/transfer/src/README.md

@@ -0,0 +1,30 @@
+# Transfer Benchmark
+
+Benchmarks Helia transfer performance against js-ipfs and Kubo
+
+To run:
+
+1. Add `benchmarks/*` to the `workspaces` entry in the root `package.json` of this repo
+2. Run
+    ```console
+    $ npm run reset
+    $ npm i
+    $ npm run build
+    $ cd benchmarks/transfer
+    $ npm start
+
+    > [email protected] start
+    > npm run build && node dist/src/index.js
+
+
+    > [email protected] build
+    > aegir build --bundle false
+
+    [14:51:28] tsc [started]
+    [14:51:33] tsc [completed]
+    generating Ed25519 keypair...
+    ┌─────────┬────────────────┬─────────┬───────────┬──────┐
+    │ (index) │ Implementation │  ops/s  │   ms/op   │ runs │
+    ├─────────┼────────────────┼─────────┼───────────┼──────┤
+    //... results here
+    ```

benchmarks/transfer/src/helia.ts

@@ -0,0 +1,63 @@
+import { createHelia } from 'helia'
+import { createLibp2p } from 'libp2p'
+import { tcp } from '@libp2p/tcp'
+import { noise } from '@chainsafe/libp2p-noise'
+// import { yamux } from '@chainsafe/libp2p-yamux'
+import { mplex } from '@libp2p/mplex'
+import type { TransferBenchmark } from './index.js'
+import os from 'node:os'
+import path from 'node:path'
+import { LevelDatastore } from 'datastore-level'
+import { FsBlockstore } from 'blockstore-fs'
+import drain from 'it-drain'
+import { unixfs } from '@helia/unixfs'
+// import { fixedSize } from 'ipfs-unixfs-importer/chunker'
+// import { balanced } from 'ipfs-unixfs-importer/layout'
+
+export async function createHeliaBenchmark (): Promise<TransferBenchmark> {
+  const repoPath = path.join(os.tmpdir(), `helia-${Math.random()}`)
+
+  const helia = await createHelia({
+    blockstore: new FsBlockstore(`${repoPath}/blocks`),
+    datastore: new LevelDatastore(`${repoPath}/data`),
+    libp2p: await createLibp2p({
+      addresses: {
+        listen: [
+          '/ip4/127.0.0.1/tcp/0'
+        ]
+      },
+      transports: [
+        tcp()
+      ],
+      connectionEncryption: [
+        noise()
+      ],
+      streamMuxers: [
+        mplex()
+        // yamux()
+      ]
+    })
+  })
+
+  return {
+    async teardown () {
+      await helia.stop()
+    },
+    async addr () {
+      return helia.libp2p.getMultiaddrs()[0]
+    },
+    async dial (ma) {
+      await helia.libp2p.dial(ma)
+    },
+    async add (content) {
+      const fs = unixfs(helia)
+
+      return await fs.addByteStream(content)
+    },
+    async get (cid) {
+      const fs = unixfs(helia)
+
+      await drain(fs.cat(cid))
+    }
+  }
+}

benchmarks/transfer/src/index.ts

@@ -0,0 +1,119 @@
+/* eslint-disable no-console */
+
+import type { CID } from 'multiformats/cid'
+import { createHeliaBenchmark } from './helia.js'
+import { createIpfsBenchmark } from './ipfs.js'
+import { createKuboBenchmark } from './kubo.js'
+import bufferStream from 'it-buffer-stream'
+import type { Multiaddr } from '@multiformats/multiaddr'
+import prettyBytes from 'pretty-bytes'
+
+const ONE_MEG = 1024 * 1024
+
+export interface TransferBenchmark {
+  teardown: () => Promise<void>
+  addr: () => Promise<Multiaddr>
+  dial: (multiaddr: Multiaddr) => Promise<void>
+  add: (content: AsyncIterable<Uint8Array>, options: ImportOptions) => Promise<CID>
+  get: (cid: CID) => Promise<void>
+}
+
+interface ImportOptions {
+  cidVersion?: 0 | 1
+  rawLeaves?: boolean
+  chunkSize?: number
+  maxChildrenPerNode?: number
+}
+
+interface File {
+  name: string
+  options: ImportOptions
+  size: number
+}
+
+const opts: Record<string, ImportOptions> = {
+  defaults: {},
+  'kubo defaults': {
+    chunkSize: 256 * 1024,
+    rawLeaves: false,
+    cidVersion: 0,
+    maxChildrenPerNode: 174
+  },
+  '256KiB block size': {
+    chunkSize: 256 * 1024
+  },
+  '1MB block size': {
+    chunkSize: 1024 * 1024
+  },
+  '3MB block size': {
+    chunkSize: (1024 * 1024) * 10
+  },
+  '4MB block size': {
+    chunkSize: (1024 * 1024) * 10
+  },
+  '10MB block size': {
+    chunkSize: (1024 * 1024) * 10
+  }
+}
+
+const tests: Record<string, File[]> = {}
+
+for (const [name, options] of Object.entries(opts)) {
+  tests[name] = []
+
+  for (let i = 100; i < 1100; i += 100) {
+    tests[name].push({
+      name: `${i}`,
+      options,
+      size: ONE_MEG * i
+    })
+
+    console.info(prettyBytes(ONE_MEG * i))
+  }
+}
+
+const impls: Array<{ name: string, create: () => Promise<TransferBenchmark> }> = [{
+  name: 'helia',
+  create: async () => await createHeliaBenchmark()
+}, {
+  name: 'ipfs',
+  create: async () => await createIpfsBenchmark()
+}, {
+  name: 'kubo',
+  create: async () => await createKuboBenchmark()
+}]
+
+async function main (): Promise<void> {
+  for (const [name, files] of Object.entries(tests)) {
+    for (const implA of impls) {
+      for (const implB of impls) {
+        console.info(`${implA.name} -> ${implB.name} ${name}`)
+
+        for (const file of files) {
+          const subjectA = await implA.create()
+          const subjectB = await implB.create()
+
+          const addr = await subjectB.addr()
+          await subjectA.dial(addr)
+
+          const cid = await subjectA.add(bufferStream(file.size), file.options)
+
+          const start = Date.now()
+
+          // b pulls from a
+          await subjectB.get(cid)
+
+          console.info(`${Date.now() - start}`)
+
+          await subjectA.teardown()
+          await subjectB.teardown()
+        }
+      }
+    }
+  }
+}
+
+main().catch(err => {
+  console.error(err) // eslint-disable-line no-console
+  process.exit(1)
+})

benchmarks/transfer/src/ipfs.ts

@@ -0,0 +1,46 @@
+import { create } from 'ipfs-core'
+import drain from 'it-drain'
+import type { TransferBenchmark } from './index.js'
+import os from 'node:os'
+import path from 'node:path'
+
+export async function createIpfsBenchmark (): Promise<TransferBenchmark> {
+  const repoPath = path.join(os.tmpdir(), `ipfs-${Math.random()}`)
+
+  const ipfs = await create({
+    config: {
+      Addresses: {
+        Swarm: [
+          '/ip4/127.0.0.1/tcp/0'
+        ]
+      }
+    },
+    repo: repoPath,
+    init: {
+      emptyRepo: true
+    },
+    silent: true
+  })
+
+  return {
+    async teardown () {
+      await ipfs.stop()
+    },
+    async addr () {
+      const id = await ipfs.id()
+
+      return id.addresses[0]
+    },
+    async dial (ma) {
+      await ipfs.swarm.connect(ma)
+    },
+    async add (content, options: any) {
+      const { cid } = await ipfs.add(content)
+
+      return cid
+    },
+    async get (cid) {
+      await drain(ipfs.cat(cid))
+    }
+  }
+}

benchmarks/transfer/src/kubo.ts

@@ -0,0 +1,50 @@
+import drain from 'it-drain'
+import type { TransferBenchmark } from './index.js'
+// @ts-expect-error no types
+import * as goIpfs from 'go-ipfs'
+import * as goRpcClient from 'kubo-rpc-client'
+import { createController } from 'ipfsd-ctl'
+
+export async function createKuboBenchmark (): Promise<TransferBenchmark> {
+  const controller = await createController({
+    type: 'go',
+    test: true,
+    ipfsBin: goIpfs.path(),
+    ipfsHttpModule: goRpcClient,
+    ipfsOptions: {
+      init: {
+        emptyRepo: true
+      },
+      config: {
+        Addresses: {
+          Swarm: [
+            '/ip4/127.0.0.1/tcp/0'
+          ]
+        }
+      },
+      silent: true
+    }
+  })
+
+  return {
+    async teardown () {
+      await controller.stop()
+    },
+    async addr () {
+      const id = await controller.api.id()
+
+      return id.addresses[0]
+    },
+    async dial (ma) {
+      await controller.api.swarm.connect(ma)
+    },
+    async add (content, options: any) {
+      const { cid } = await controller.api.add(content)
+
+      return cid
+    },
+    async get (cid) {
+      await drain(controller.api.cat(cid))
+    }
+  }
+}

benchmarks/transfer/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "extends": "aegir/src/config/tsconfig.aegir.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "target": "ES2022",
+    "module": "ES2022",
+    "lib": ["ES2022", "DOM", "DOM.Iterable"]
+  },
+  "include": [
+    "src",
+    "test"
+  ],
+  "references": [
+    {
+      "path": "../../packages/helia"
+    }
+  ]
+}