feat: store blocks by multihash instead of CID (#3124)

Updates the `ipfs-repo` dep to a version that stores blocks by multihash instead of CID to support CIDv1 and CIDv0 access to the same block.

New features:

- Adds a `--multihash` argument to the cli command `ipfs refs local` which prints the base32 encoded multihash of each block

BREAKING CHANGES:

- `ipfs.refs.local` now returns a v1 CID with the raw codec for every block and not the original CID by which it was added to the blockstore

Co-authored-by: Hugo Dias <[email protected]>

Author: achingbrain

docs/core-api/REFS.md

@@ -67,6 +67,8 @@ for await (const ref of ipfs.refs(ipfsPath, { recursive: true })) {
 
 > Output all local references (CIDs of all blocks in the blockstore)
 
+Blocks in the blockstore are stored by multihash and not CID so yielded CIDs are v1 CIDs with the 'raw' codec. These may not match the CID originally used to store a given block, though the multihash contained within the CID will.
+
 ### Parameters
 
 None

examples/custom-ipfs-repo/package.json

@@ -12,7 +12,7 @@
   "dependencies": {
     "datastore-fs": "^1.1.0",
     "ipfs": "^0.47.0",
-    "ipfs-repo": "^3.0.0",
+    "ipfs-repo": "^4.0.0",
     "it-all": "^1.0.1"
   },
   "devDependencies": {

packages/interface-ipfs-core/package.json

@@ -53,15 +53,15 @@
     "it-last": "^1.0.1",
     "it-pushable": "^1.3.1",
     "multiaddr": "^7.4.3",
-    "multibase": "^0.7.0",
+    "multibase": "^1.0.1",
     "multihashing-async": "^1.0.0",
     "nanoid": "^3.0.2",
     "peer-id": "^0.13.12",
     "readable-stream": "^3.4.0",
     "temp-write": "^4.0.0"
   },
   "devDependencies": {
-    "aegir": "^22.1.0",
+    "aegir": "^23.0.0",
     "ipfsd-ctl": "^4.1.1"
   },
   "contributors": [

packages/interface-ipfs-core/src/block/rm.js

@@ -40,7 +40,7 @@ module.exports = (common, options) => {
       // block should be present in the local store
       const localRefs = await all(ipfs.refs.local())
       expect(localRefs).to.have.property('length').that.is.greaterThan(0)
-      expect(localRefs.find(ref => ref.ref === cid.toString())).to.be.ok()
+      expect(localRefs.find(ref => ref.ref === new CID(1, 'raw', cid.multihash).toString())).to.be.ok()
 
       const result = await all(ipfs.block.rm(cid))
       expect(result).to.be.an('array').and.to.have.lengthOf(1)
@@ -49,7 +49,7 @@ module.exports = (common, options) => {
 
       // did we actually remove the block?
       const localRefsAfterRemove = await all(ipfs.refs.local())
-      expect(localRefsAfterRemove.find(ref => ref.ref === cid.toString())).to.not.be.ok()
+      expect(localRefsAfterRemove.find(ref => ref.ref === new CID(1, 'raw', cid.multihash).toString())).to.not.be.ok()
     })
 
     it('should remove by CID in string', async () => {

packages/interface-ipfs-core/src/refs-local.js

@@ -7,6 +7,7 @@ const all = require('it-all')
 const importer = require('ipfs-unixfs-importer')
 const drain = require('it-drain')
 const testTimeout = require('./utils/test-timeout')
+const CID = require('cids')
 
 /** @typedef { import("ipfsd-ctl/src/factory") } Factory */
 /**
@@ -54,8 +55,22 @@ module.exports = (common, options) => {
 
       const refs = await all(ipfs.refs.local())
       const cids = refs.map(r => r.ref)
-      expect(cids).to.include('QmVwdDCY4SPGVFnNCiZnX5CtzwWDn6kAM98JXzKxE3kCmn')
-      expect(cids).to.include('QmR4nFjTu18TyANgC65ArNWp5Yaab1gPzQ4D8zp7Kx3vhr')
+
+      expect(
+        cids.find(cid => {
+          const multihash = new CID(cid).multihash
+
+          return imported[0].cid.multihash.equals(multihash)
+        })
+      ).to.be.ok()
+
+      expect(
+        cids.find(cid => {
+          const multihash = new CID(cid).multihash
+
+          return imported[1].cid.multihash.equals(multihash)
+        })
+      ).to.be.ok()
     })
   })
 }

packages/interface-ipfs-core/src/repo/gc.js

@@ -6,6 +6,7 @@ const { getDescribe, getIt, expect } = require('../utils/mocha')
 const { DAGNode } = require('ipld-dag-pb')
 const all = require('it-all')
 const testTimeout = require('../utils/test-timeout')
+const CID = require('cids')
 
 /** @typedef { import("ipfsd-ctl/src/factory") } Factory */
 /**
@@ -58,15 +59,15 @@ module.exports = (common, options) => {
       // the initial list and contain hash
       const refsAfterAdd = await all(ipfs.refs.local())
       expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length)
-      expect(refsAfterAdd.map(r => r.ref)).includes(cid.toString())
+      expect(refsAfterAdd.map(r => new CID(r.ref).multihash)).deep.includes(cid.multihash)
 
       // Run garbage collection
       await all(ipfs.repo.gc())
 
       // Get the list of local blocks after GC, should still contain the hash,
       // because the file is still pinned
       const refsAfterGc = await all(ipfs.refs.local())
-      expect(refsAfterGc.map(r => r.ref)).includes(cid.toString())
+      expect(refsAfterGc.map(r => new CID(r.ref).multihash)).deep.includes(cid.multihash)
 
       // Unpin the data
       await ipfs.pin.rm(cid)
@@ -76,7 +77,7 @@ module.exports = (common, options) => {
 
       // The list of local blocks should no longer contain the hash
       const refsAfterUnpinAndGc = await all(ipfs.refs.local())
-      expect(refsAfterUnpinAndGc.map(r => r.ref)).not.includes(cid.toString())
+      expect(refsAfterUnpinAndGc.map(r => new CID(r.ref).multihash)).not.deep.includes(cid.multihash)
     })
 
     it('should clean up removed MFS files', async () => {
@@ -87,21 +88,20 @@ module.exports = (common, options) => {
       await ipfs.files.write('/test', Buffer.from('oranges'), { create: true })
       const stats = await ipfs.files.stat('/test')
       expect(stats.type).to.equal('file')
-      const hash = stats.cid.toString()
 
       // Get the list of local blocks after the add, should be bigger than
       // the initial list and contain hash
       const refsAfterAdd = await all(ipfs.refs.local())
       expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length)
-      expect(refsAfterAdd.map(r => r.ref)).includes(hash)
+      expect(refsAfterAdd.map(r => new CID(r.ref).multihash)).deep.includes(stats.cid.multihash)
 
       // Run garbage collection
       await all(ipfs.repo.gc())
 
       // Get the list of local blocks after GC, should still contain the hash,
       // because the file is in MFS
       const refsAfterGc = await all(ipfs.refs.local())
-      expect(refsAfterGc.map(r => r.ref)).includes(hash)
+      expect(refsAfterGc.map(r => new CID(r.ref).multihash)).deep.includes(stats.cid.multihash)
 
       // Remove the file
       await ipfs.files.rm('/test')
@@ -111,7 +111,7 @@ module.exports = (common, options) => {
 
       // The list of local blocks should no longer contain the hash
       const refsAfterUnpinAndGc = await all(ipfs.refs.local())
-      expect(refsAfterUnpinAndGc.map(r => r.ref)).not.includes(hash)
+      expect(refsAfterUnpinAndGc.map(r => new CID(r.ref).multihash)).not.deep.includes(stats.cid.multihash)
     })
 
     it('should clean up block only after unpinned and removed from MFS', async () => {
@@ -135,17 +135,15 @@ module.exports = (common, options) => {
       // the initial list and contain the data hash
       const refsAfterAdd = await all(ipfs.refs.local())
       expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length)
-      const hashesAfterAdd = refsAfterAdd.map(r => r.ref)
-      expect(hashesAfterAdd).includes(dataCid.toString())
+      expect(refsAfterAdd.map(r => new CID(r.ref).multihash)).deep.includes(dataCid.multihash)
 
       // Run garbage collection
       await all(ipfs.repo.gc())
 
       // Get the list of local blocks after GC, should still contain the hash,
       // because the file is pinned and in MFS
       const refsAfterGc = await all(ipfs.refs.local())
-      const hashesAfterGc = refsAfterGc.map(r => r.ref)
-      expect(hashesAfterGc).includes(dataCid.toString())
+      expect(refsAfterGc.map(r => new CID(r.ref).multihash)).deep.includes(dataCid.multihash)
 
       // Remove the file
       await ipfs.files.rm('/test')
@@ -156,9 +154,8 @@ module.exports = (common, options) => {
       // Get the list of local blocks after GC, should still contain the hash,
       // because the file is still pinned
       const refsAfterRmAndGc = await all(ipfs.refs.local())
-      const hashesAfterRmAndGc = refsAfterRmAndGc.map(r => r.ref)
-      expect(hashesAfterRmAndGc).not.includes(mfsFileCid.toString())
-      expect(hashesAfterRmAndGc).includes(dataCid.toString())
+      expect(refsAfterRmAndGc.map(r => new CID(r.ref).multihash)).not.deep.includes(mfsFileCid.multihash)
+      expect(refsAfterRmAndGc.map(r => new CID(r.ref).multihash)).deep.includes(dataCid.multihash)
 
       // Unpin the data
       await ipfs.pin.rm(dataCid)
@@ -168,9 +165,8 @@ module.exports = (common, options) => {
 
       // The list of local blocks should no longer contain the hashes
       const refsAfterUnpinAndGc = await all(ipfs.refs.local())
-      const hashesAfterUnpinAndGc = refsAfterUnpinAndGc.map(r => r.ref)
-      expect(hashesAfterUnpinAndGc).not.includes(mfsFileCid.toString())
-      expect(hashesAfterUnpinAndGc).not.includes(dataCid.toString())
+      expect(refsAfterUnpinAndGc.map(r => new CID(r.ref).multihash)).not.deep.includes(mfsFileCid.multihash)
+      expect(refsAfterUnpinAndGc.map(r => new CID(r.ref).multihash)).not.deep.includes(dataCid.multihash)
     })
 
     it('should clean up indirectly pinned data after recursive pin removal', async () => {
@@ -201,9 +197,8 @@ module.exports = (common, options) => {
       // the initial list and contain data and object hash
       const refsAfterAdd = await all(ipfs.refs.local())
       expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length)
-      const hashesAfterAdd = refsAfterAdd.map(r => r.ref)
-      expect(hashesAfterAdd).includes(objCid.toString())
-      expect(hashesAfterAdd).includes(dataCid.toString())
+      expect(refsAfterAdd.map(r => new CID(r.ref).multihash)).deep.includes(objCid.multihash)
+      expect(refsAfterAdd.map(r => new CID(r.ref).multihash)).deep.includes(dataCid.multihash)
 
       // Recursively pin the object
       await ipfs.pin.add(objCid, { recursive: true })
@@ -218,7 +213,7 @@ module.exports = (common, options) => {
       // Get the list of local blocks after GC, should still contain the data
       // hash, because the data is still (indirectly) pinned
       const refsAfterGc = await all(ipfs.refs.local())
-      expect(refsAfterGc.map(r => r.ref)).includes(dataCid.toString())
+      expect(refsAfterGc.map(r => new CID(r.ref).multihash)).deep.includes(dataCid.multihash)
 
       // Recursively unpin the object
       await ipfs.pin.rm(objCid.toString())
@@ -228,9 +223,8 @@ module.exports = (common, options) => {
 
       // The list of local blocks should no longer contain the hashes
       const refsAfterUnpinAndGc = await all(ipfs.refs.local())
-      const hashesAfterUnpinAndGc = refsAfterUnpinAndGc.map(r => r.ref)
-      expect(hashesAfterUnpinAndGc).not.includes(objCid.toString())
-      expect(hashesAfterUnpinAndGc).not.includes(dataCid.toString())
+      expect(refsAfterUnpinAndGc.map(r => new CID(r.ref).multihash)).not.deep.includes(objCid.multihash)
+      expect(refsAfterUnpinAndGc.map(r => new CID(r.ref).multihash)).not.deep.includes(dataCid.multihash)
     })
   })
 }

packages/ipfs-core-utils/package.json

@@ -33,7 +33,7 @@
     "ipfs-utils": "^2.2.2"
   },
   "devDependencies": {
-    "aegir": "^22.1.0",
+    "aegir": "^23.0.0",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
     "delay": "^4.3.0",

packages/ipfs-http-client/package.json

@@ -59,16 +59,16 @@
     "merge-options": "^2.0.0",
     "multiaddr": "^7.4.3",
     "multiaddr-to-uri": "^5.1.0",
-    "multibase": "^0.7.0",
+    "multibase": "^1.0.1",
     "multicodec": "^1.0.0",
     "multihashes": "^1.0.1",
     "nanoid": "^3.0.2",
     "node-fetch": "^2.6.0",
-    "parse-duration": "^0.1.2",
+    "parse-duration": "^0.4.4",
     "stream-to-it": "^0.2.0"
   },
   "devDependencies": {
-    "aegir": "^22.1.0",
+    "aegir": "^23.0.0",
     "cross-env": "^7.0.0",
     "go-ipfs-dep": "^0.5.1",
     "interface-ipfs-core": "^0.137.0",

packages/ipfs-http-client/src/lib/core.js

@@ -4,7 +4,7 @@ const Multiaddr = require('multiaddr')
 const toUri = require('multiaddr-to-uri')
 const { isBrowser, isWebWorker } = require('ipfs-utils/src/env')
 const { URL } = require('iso-url')
-const parseDuration = require('parse-duration')
+const parseDuration = require('parse-duration').default
 const log = require('debug')('ipfs-http-client:lib:error-handler')
 const HTTP = require('ipfs-utils/src/http')
 const merge = require('merge-options')

packages/ipfs/package.json

@@ -96,7 +96,7 @@
     "ipfs-core-utils": "^0.2.4",
     "ipfs-http-client": "^44.3.0",
     "ipfs-http-response": "^0.5.0",
-    "ipfs-repo": "^3.0.0",
+    "ipfs-repo": "^4.0.0",
     "ipfs-unixfs": "^1.0.3",
     "ipfs-unixfs-exporter": "^2.0.2",
     "ipfs-unixfs-importer": "^2.0.2",
@@ -117,7 +117,7 @@
     "it-all": "^1.0.1",
     "it-concat": "^1.0.0",
     "it-drain": "^1.0.1",
-    "it-glob": "0.0.7",
+    "it-glob": "0.0.8",
     "it-last": "^1.0.1",
     "it-map": "^1.0.0",
     "it-multipart": "^1.0.1",
@@ -148,19 +148,19 @@
     "mortice": "^2.0.0",
     "multiaddr": "^7.4.3",
     "multiaddr-to-uri": "^5.1.0",
-    "multibase": "^0.7.0",
+    "multibase": "^1.0.1",
     "multicodec": "^1.0.0",
     "multihashing-async": "^1.0.0",
     "p-defer": "^3.0.0",
     "p-queue": "^6.1.0",
-    "parse-duration": "^0.1.2",
+    "parse-duration": "^0.4.4",
     "peer-id": "^0.13.12",
     "pretty-bytes": "^5.3.0",
     "progress": "^2.0.1",
     "protons": "^1.2.0",
     "semver": "^7.3.2",
     "stream-to-it": "^0.2.0",
-    "streaming-iterables": "^4.1.1",
+    "streaming-iterables": "^5.0.0",
     "temp": "^0.9.0",
     "timeout-abort-controller": "^1.1.0",
     "update-notifier": "^4.0.0",
@@ -170,7 +170,7 @@
     "yargs-promise": "^1.1.0"
   },
   "devDependencies": {
-    "aegir": "^22.1.0",
+    "aegir": "^23.0.0",
     "base64url": "^3.0.1",
     "clear-module": "^4.0.0",
     "cross-env": "^7.0.0",
@@ -191,7 +191,7 @@
     "qs": "^6.9.3",
     "rimraf": "^3.0.0",
     "sinon": "^9.0.1",
-    "stream-to-promise": "^2.2.0",
+    "stream-to-promise": "^3.0.0",
     "string-argv": "^0.3.1",
     "temp-write": "^4.0.0",
     "wrtc": "^0.4.4"

packages/ipfs/src/cli/commands/add.js

@@ -13,7 +13,7 @@ const {
 } = require('../utils')
 const { cidToString } = require('../../utils/cid')
 const globSource = require('ipfs-utils/src/files/glob-source')
-const parseDuration = require('parse-duration')
+const parseDuration = require('parse-duration').default
 
 async function getTotalBytes (paths) {
   const sizes = await Promise.all(paths.map(p => getFolderSize(p)))
@@ -90,7 +90,7 @@ module.exports = {
     'cid-base': {
       describe: 'Number base to display CIDs in.',
       type: 'string',
-      choices: multibase.names
+      choices: Object.keys(multibase.names)
     },
     hash: {
       type: 'string',

packages/ipfs/src/cli/commands/bitswap/stat.js

@@ -3,7 +3,7 @@
 const multibase = require('multibase')
 const { cidToString } = require('../../../utils/cid')
 const prettyBytes = require('pretty-bytes')
-const parseDuration = require('parse-duration')
+const parseDuration = require('parse-duration').default
 
 module.exports = {
   command: 'stat',
@@ -14,7 +14,7 @@ module.exports = {
     'cid-base': {
       describe: 'Number base to display CIDs in. Note: specifying a CID base for v0 CIDs will have no effect.',
       type: 'string',
-      choices: multibase.names
+      choices: Object.keys(multibase.names)
     },
     human: {
       type: 'boolean',

packages/ipfs/src/cli/commands/bitswap/unwant.js

@@ -2,7 +2,7 @@
 
 const multibase = require('multibase')
 const { cidToString } = require('../../../utils/cid')
-const parseDuration = require('parse-duration')
+const parseDuration = require('parse-duration').default
 
 module.exports = {
   command: 'unwant <key>',
@@ -18,7 +18,7 @@ module.exports = {
     'cid-base': {
       describe: 'Number base to display CIDs in. Note: specifying a CID base for v0 CIDs will have no effect.',
       type: 'string',
-      choices: multibase.names
+      choices: Object.keys(multibase.names)
     },
     timeout: {
       type: 'string',