typescript impl of R2(crc32) + R2(sha256) ordering.

packages/libs/src/sdk/services/StorageNodeSelector/StorageNodeSelector.test.ts

@@ -106,7 +106,7 @@ describe('StorageNodeSelector', () => {
     })
 
     expect(await storageNodeSelector.getSelectedNode()).toEqual(
-      storageNodeB.endpoint
+      storageNodeA.endpoint
     )
   })
 
@@ -146,7 +146,7 @@ describe('StorageNodeSelector', () => {
 
     await waitForExpect(async () => {
       expect(await storageNodeSelector.getSelectedNode()).toEqual(
-        storageNodeB.endpoint
+        storageNodeA.endpoint
       )
     })
   })
@@ -172,7 +172,7 @@ describe('StorageNodeSelector', () => {
 
     await waitForExpect(async () => {
       expect(await storageNodeSelector.getSelectedNode()).toEqual(
-        storageNodeB.endpoint
+        storageNodeA.endpoint
       )
     })
   })
@@ -189,8 +189,8 @@ describe('StorageNodeSelector', () => {
     })
 
     expect(await storageNodeSelector.getNodes(cid)).toEqual([
-      storageNodeA.endpoint,
-      storageNodeB.endpoint
+      storageNodeB.endpoint,
+      storageNodeA.endpoint
     ])
   })
 
@@ -205,12 +205,12 @@ describe('StorageNodeSelector', () => {
     })
 
     expect(await storageNodeSelector.getSelectedNode()).toEqual(
-      storageNodeB.endpoint
+      storageNodeA.endpoint
     )
 
     // force reselect
     expect(await storageNodeSelector.getSelectedNode(true)).toEqual(
-      storageNodeA.endpoint
+      storageNodeB.endpoint
     )
   })
 

packages/libs/src/utils/getNStorageNodes.test.ts

@@ -104,8 +104,8 @@ describe('getNStorageNodes', () => {
 
     const result = await getNStorageNodes(sampleNodes, 3, 'test-rendezvous-key')
     assert.deepEqual(result, [
-      'http://node3.com',
       'http://node2.com',
+      'http://node3.com',
       'http://node1.com'
     ])
   })

packages/libs/src/utils/rendezvous.test.ts

@@ -24,62 +24,27 @@ describe('RendezvousHash', () => {
     const highestNode = hash.get(key)
     assert(nodes.includes(highestNode))
   })
-
-  it('should return top N highest scoring nodes for a given key', () => {
-    const nodes = ['node1', 'node2', 'node3']
-    const hash = new RendezvousHash(...nodes)
-    const key = 'test-key'
-    const top2Nodes = hash.getN(2, key)
-    assert.equal(top2Nodes.length, 2)
-    assert(nodes.includes(top2Nodes[0]!))
-    assert(nodes.includes(top2Nodes[1]!))
-  })
 })
 
-// Ensures the hash results match the results of the equivalent Go code.
-// See https://github.com/tysonmote/rendezvous/blob/be0258dbbd3d0df637b328d951067124541e7b6a/rendezvous_test.go
-
-describe('RendezvousHash - Golang test cases', () => {
-  it('TestHashGet', () => {
-    const hash = new RendezvousHash()
-
-    const gotNode = hash.get('foo')
-    assert.equal(gotNode, '')
-
-    hash.add('a', 'b', 'c', 'd', 'e')
-
-    const testcases = [
-      { key: '', expectedNode: 'd' },
-      { key: 'foo', expectedNode: 'e' },
-      { key: 'bar', expectedNode: 'c' }
+const nodeList =
+  'https://creatornode.audius.prod-eks-ap-northeast-1.staked.cloud,https://creatornode.audius1.prod-eks-ap-northeast-1.staked.cloud,https://creatornode.audius2.prod-eks-ap-northeast-1.staked.cloud,https://creatornode.audius3.prod-eks-ap-northeast-1.staked.cloud,https://creatornode.audius8.prod-eks-ap-northeast-1.staked.cloud,https://creatornode.audius.co,https://creatornode2.audius.co,https://creatornode3.audius.co,https://usermetadata.audius.co,https://audius-content-1.cultur3stake.com,https://audius-content-10.cultur3stake.com,https://audius-content-11.cultur3stake.com,https://audius-content-12.cultur3stake.com,https://audius-content-13.cultur3stake.com,https://audius-content-14.cultur3stake.com,https://audius-content-15.cultur3stake.com,https://audius-content-16.cultur3stake.com,https://audius-content-17.cultur3stake.com,https://audius-content-18.cultur3stake.com,https://audius-content-2.cultur3stake.com,https://audius-content-3.cultur3stake.com,https://audius-content-4.cultur3stake.com,https://audius-content-5.cultur3stake.com,https://audius-content-6.cultur3stake.com,https://audius-content-7.cultur3stake.com,https://audius-content-8.cultur3stake.com,https://audius-content-9.cultur3stake.com,https://cn1.stuffisup.com,https://audius-cn1.tikilabs.com,https://audius.prod.capturealpha.io,https://audius-content-1.figment.io,https://audius-content-10.figment.io,https://audius-content-11.figment.io,https://audius-content-12.figment.io,https://audius-content-13.figment.io,https://audius-content-14.figment.io,https://audius-content-2.figment.io,https://audius-content-3.figment.io,https://audius-content-4.figment.io,https://audius-content-5.figment.io,https://audius-content-6.figment.io,https://audius-content-7.figment.io,https://audius-content-8.figment.io,https://audius-content-9.figment.io,https://blockchange-audius-content-01.bdnodes.net,https://blockchange-audius-content-02.bdnodes.net,https://blockchange-audius-content-03.bdnodes.net,https://blockdaemon-audius-content-01.bdnodes.net,https://blockdaemon-audius-content-02.bdnodes.net,https://blockdaemon-audius-content-03.bdnodes.net,https://blockdaemon-audius-content-04.bdnodes.net,https://blockdaemon-audius-content-05.bdnodes.net,https://blockdaemon-audius-content-06.bdnodes.net,https://blockdaemon-audius-content-07.bdnodes.net,https://blockdaemon-audius-content-08.bdnodes.net,https://blockdaemon-audius-content-09.bdnodes.net,https://content.grassfed.network,https://cn0.mainnet.audiusindex.org,https://cn1.mainnet.audiusindex.org,https://cn2.mainnet.audiusindex.org,https://cn3.mainnet.audiusindex.org,https://cn4.mainnet.audiusindex.org,https://audius-content-1.jollyworld.xyz,https://audius-creator-1.theblueprint.xyz,https://audius-creator-2.theblueprint.xyz,https://audius-creator-3.theblueprint.xyz,https://audius-creator-4.theblueprint.xyz,https://audius-creator-5.theblueprint.xyz,https://audius-creator-6.theblueprint.xyz'.split(
+    ','
+  )
+
+describe('sha256 ordering', () => {
+  const hasher = new RendezvousHash(...nodeList)
+  const cid = 'baeaaaiqsedziwknj44jsl5fak6vcbszzjlnl7pqtw2ipnyg7rsh5a2xnql2p2'
+
+  it('has sha256 ordering', () => {
+    const ordered = hasher.getN(6, cid)
+    const expected = [
+      'https://blockdaemon-audius-content-09.bdnodes.net',
+      'https://cn4.mainnet.audiusindex.org',
+      'https://audius-content-4.figment.io',
+      'https://cn0.mainnet.audiusindex.org',
+      'https://creatornode.audius3.prod-eks-ap-northeast-1.staked.cloud',
+      'https://blockdaemon-audius-content-07.bdnodes.net'
     ]
-
-    for (const testcase of testcases) {
-      const gotNode = hash.get(testcase.key)
-      assert.equal(gotNode, testcase.expectedNode)
-    }
-  })
-
-  it('Test_Hash_GetN', () => {
-    const hash = new RendezvousHash()
-
-    const gotNodes = hash.getN(2, 'foo')
-    assert.deepEqual(gotNodes, [])
-
-    hash.add('a', 'b', 'c', 'd', 'e')
-
-    const testcases = [
-      { count: 1, key: 'foo', expectedNodes: ['e'] },
-      { count: 2, key: 'bar', expectedNodes: ['c', 'e'] },
-      { count: 3, key: 'baz', expectedNodes: ['d', 'a', 'b'] },
-      { count: 2, key: 'biz', expectedNodes: ['b', 'a'] },
-      { count: 0, key: 'boz', expectedNodes: [] },
-      { count: 100, key: 'floo', expectedNodes: ['d', 'a', 'b', 'c', 'e'] }
-    ]
-
-    for (const testcase of testcases) {
-      const gotNodes = hash.getN(testcase.count, testcase.key)
-      assert.deepEqual(gotNodes, testcase.expectedNodes)
-    }
+    assert.deepEqual(ordered, expected)
   })
 })

packages/libs/src/utils/rendezvous.ts

@@ -1,6 +1,7 @@
 import { Buffer } from 'buffer'
 
-import CRC32C from 'crc-32/crc32c'
+import { sha256 } from '@noble/hashes/sha256'
+import { bytesToHex as toHex } from '@noble/hashes/utils'
 
 class NodeScore {
   node: Buffer
@@ -28,59 +29,33 @@ export class RendezvousHash {
     }
   }
 
-  get(key: string): string {
-    let maxScore = 0
-    let maxNode: Buffer | null = null
-
-    const keyBytes = Buffer.from(key)
-
-    for (const node of this.nodes) {
-      const score = this.hash(node.node, keyBytes)
-      if (
-        score > maxScore ||
-        (score === maxScore && node.node.compare(maxNode!) < 0)
-      ) {
-        maxScore = score
-        maxNode = node.node
-      }
-    }
+  getNodes(): string[] {
+    return this.nodes.map((nodeScore) => nodeScore.node.toString())
+  }
 
-    return maxNode?.toString() ?? ''
+  get(key: string): string {
+    const first = this.getN(1, key)[0]
+    return first ?? ''
   }
 
   getN(n: number, key: string): string[] {
-    const keyBytes = Buffer.from(key)
-    for (const node of this.nodes) {
-      node.score = this.hash(node.node, keyBytes)
-    }
+    return this.rendezvous256(key).slice(0, n)
+  }
 
-    this.nodes.sort((a, b) => {
-      if (a.score === b.score) {
-        return a.node.compare(b.node)
+  rendezvous256(key: string) {
+    const tuples = this.nodes.map((n) => {
+      const hostName = n.node.toString()
+      return [hostName, toHex(sha256(`${hostName}${key}`))]
+    })
+    tuples.sort((t1, t2) => {
+      const [aHost, aScore] = t1
+      const [bHost, bScore] = t2
+      if (aScore === bScore) {
+        return aHost! < bHost! ? -1 : 1
       }
-      return b.score - a.score
+      return aScore! < bScore! ? -1 : 1
     })
-
-    if (n > this.nodes.length) {
-      n = this.nodes.length
-    }
-
-    const nodes: string[] = []
-    for (let i = 0; i < n; i++) {
-      nodes.push(this.nodes[i]!.node.toString())
-    }
-    return nodes
-  }
-
-  getNodes(): string[] {
-    return this.nodes.map((nodeScore) => nodeScore.node.toString())
-  }
-
-  private hash(node: Buffer, key: Buffer): number {
-    const combined = Buffer.concat([key, node])
-    // Convert to unsigned 32-bit integer to match go implementation, which is uint32 here:
-    // https://github.com/tysonmote/rendezvous/blob/be0258dbbd3d/rendezvous.go#L92
-    return CRC32C.buf(combined, 0) >>> 0
+    return tuples.map((t) => t[0]!)
   }
 }