chore: move mfs and multipart files into core (#2811)

BREAKING CHANGE:

When the path passed to `ipfs.files.stat(path)` was a hamt sharded dir, the resovled
value returned by js-ipfs previously had a `type` property of with a value of
`'hamt-sharded-directory'`.  To bring it in line with go-ipfs this value is now
`'directory'`.

Author: achingbrain

examples/circuit-relaying/package.json

@@ -19,7 +19,7 @@
     "ipfs-pubsub-room": "^2.0.1"
   },
   "devDependencies": {
-    "aegir": "^21.3.0",
+    "aegir": "21.3.0",
     "execa": "^3.2.0",
     "ipfs-css": "^0.13.1",
     "ipfs-http-client": "^42.0.0",

examples/running-multiple-nodes/test.js

@@ -5,7 +5,7 @@ const IPFS = require('ipfs')
 const execa = require('execa')
 const os = require('os')
 const path = require('path')
-const hat = require('hat')
+const nanoid = require('nanoid')
 const {
   waitForOutput
 } = require('test-ipfs-example/utils')
@@ -18,7 +18,7 @@ async function testCli () {
 }
 
 async function startCliNode () {
-  const repoDir = path.join(os.tmpdir(), `repo-${hat()}`)
+  const repoDir = path.join(os.tmpdir(), `repo-${nanoid()}`)
   const opts = {
     env: {
       ...process.env,
@@ -43,7 +43,7 @@ async function testProgramatically () {
 }
 
 async function startProgramaticNode () {
-  const repoDir = path.join(os.tmpdir(), `repo-${hat()}`)
+  const repoDir = path.join(os.tmpdir(), `repo-${nanoid()}`)
   const node = await IPFS.create({
     repo: repoDir,
     config: {

packages/interface-ipfs-core/package.json

@@ -32,17 +32,17 @@
   "dependencies": {
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
-    "chai-things": "^0.2.0",
+    "chai-subset": "^1.6.0",
     "cids": "^0.7.3",
     "delay": "^4.3.0",
     "dirty-chai": "^2.0.1",
-    "hat": "0.0.3",
     "ipfs-block": "^0.8.1",
     "ipfs-unixfs": "^1.0.0",
     "ipfs-utils": "^0.7.2",
     "ipld-dag-cbor": "^0.15.1",
     "ipld-dag-pb": "^0.18.3",
     "is-ipfs": "^0.6.1",
+    "iso-random-stream": "^1.1.1",
     "it-all": "^1.0.1",
     "it-concat": "^1.0.0",
     "it-last": "^1.0.1",
@@ -51,11 +51,13 @@
     "multibase": "^0.6.0",
     "multihashes": "^0.4.14",
     "multihashing-async": "^0.8.0",
+    "nanoid": "^2.1.11",
     "peer-id": "^0.13.5",
-    "readable-stream": "^3.4.0"
+    "readable-stream": "^3.4.0",
+    "temp-write": "^4.0.0"
   },
   "devDependencies": {
-    "aegir": "^21.3.0",
+    "aegir": "21.3.0",
     "ipfsd-ctl": "^3.0.0"
   },
   "contributors": [

packages/interface-ipfs-core/src/add.js

@@ -4,6 +4,7 @@
 const { fixtures } = require('./utils')
 const { Readable } = require('readable-stream')
 const all = require('it-all')
+const last = require('it-last')
 const fs = require('fs')
 const os = require('os')
 const path = require('path')
@@ -13,7 +14,6 @@ const urlSource = require('ipfs-utils/src/files/url-source')
 const { isNode } = require('ipfs-utils/src/env')
 const { getDescribe, getIt, expect } = require('./utils/mocha')
 const { echoUrl, redirectUrl } = require('./utils/echo-http-server')
-
 const fixturesPath = path.join(__dirname, '..', 'test', 'fixtures')
 
 /** @typedef { import("ipfsd-ctl/src/factory") } Factory */
@@ -221,9 +221,8 @@ module.exports = (common, options) => {
         emptyDir('files/empty')
       ]
 
-      const res = await all(ipfs.add(dirs))
+      const root = await last(ipfs.add(dirs))
 
-      const root = res[res.length - 1]
       expect(root.path).to.equal('test-folder')
       expect(root.cid.toString()).to.equal(fixtures.directory.cid)
     })
@@ -258,9 +257,7 @@ module.exports = (common, options) => {
         accumProgress += p
       }
 
-      const filesAdded = await all(ipfs.add(dirs, { progress: handler }))
-
-      const root = filesAdded[filesAdded.length - 1]
+      const root = await last(ipfs.add(dirs, { progress: handler }))
       expect(progCalled).to.be.true()
       expect(accumProgress).to.be.at.least(total)
       expect(root.path).to.equal('test-folder')
@@ -289,10 +286,10 @@ module.exports = (common, options) => {
       expect(nonSeqDirFilePaths.every(p => filesAddedPaths.includes(p))).to.be.true()
     })
 
-    it('should fail when passed invalid input', () => {
+    it('should fail when passed invalid input', async () => {
       const nonValid = 138
 
-      return expect(all(ipfs.add(nonValid))).to.eventually.be.rejected()
+      await expect(all(ipfs.add(nonValid))).to.eventually.be.rejected()
     })
 
     it('should wrap content in a directory', async () => {

packages/interface-ipfs-core/src/block/rm.js

@@ -2,7 +2,7 @@
 'use strict'
 
 const { getDescribe, getIt, expect } = require('../utils/mocha')
-const hat = require('hat')
+const nanoid = require('nanoid')
 const all = require('it-all')
 
 /** @typedef { import("ipfsd-ctl/src/factory") } Factory */
@@ -22,7 +22,7 @@ module.exports = (common, options) => {
     after(() => common.clean())
 
     it('should remove by CID object', async () => {
-      const cid = await ipfs.dag.put(Buffer.from(hat()), {
+      const cid = await ipfs.dag.put(Buffer.from(nanoid()), {
         format: 'raw',
         hashAlg: 'sha2-256'
       })
@@ -44,7 +44,7 @@ module.exports = (common, options) => {
     })
 
     it('should remove by CID in string', async () => {
-      const cid = await ipfs.dag.put(Buffer.from(hat()), {
+      const cid = await ipfs.dag.put(Buffer.from(nanoid()), {
         format: 'raw',
         hashAlg: 'sha2-256'
       })
@@ -56,7 +56,7 @@ module.exports = (common, options) => {
     })
 
     it('should remove by CID in buffer', async () => {
-      const cid = await ipfs.dag.put(Buffer.from(hat()), {
+      const cid = await ipfs.dag.put(Buffer.from(nanoid()), {
         format: 'raw',
         hashAlg: 'sha2-256'
       })
@@ -69,15 +69,15 @@ module.exports = (common, options) => {
 
     it('should remove multiple CIDs', async () => {
       const cids = [
-        await ipfs.dag.put(Buffer.from(hat()), {
+        await ipfs.dag.put(Buffer.from(nanoid()), {
           format: 'raw',
           hashAlg: 'sha2-256'
         }),
-        await ipfs.dag.put(Buffer.from(hat()), {
+        await ipfs.dag.put(Buffer.from(nanoid()), {
           format: 'raw',
           hashAlg: 'sha2-256'
         }),
-        await ipfs.dag.put(Buffer.from(hat()), {
+        await ipfs.dag.put(Buffer.from(nanoid()), {
           format: 'raw',
           hashAlg: 'sha2-256'
         })
@@ -94,7 +94,7 @@ module.exports = (common, options) => {
     })
 
     it('should error when removing non-existent blocks', async () => {
-      const cid = await ipfs.dag.put(Buffer.from(hat()), {
+      const cid = await ipfs.dag.put(Buffer.from(nanoid()), {
         format: 'raw',
         hashAlg: 'sha2-256'
       })
@@ -111,7 +111,7 @@ module.exports = (common, options) => {
     })
 
     it('should not error when force removing non-existent blocks', async () => {
-      const cid = await ipfs.dag.put(Buffer.from(hat()), {
+      const cid = await ipfs.dag.put(Buffer.from(nanoid()), {
         format: 'raw',
         hashAlg: 'sha2-256'
       })
@@ -128,7 +128,7 @@ module.exports = (common, options) => {
     })
 
     it('should return empty output when removing blocks quietly', async () => {
-      const cid = await ipfs.dag.put(Buffer.from(hat()), {
+      const cid = await ipfs.dag.put(Buffer.from(nanoid()), {
         format: 'raw',
         hashAlg: 'sha2-256'
       })
@@ -138,7 +138,7 @@ module.exports = (common, options) => {
     })
 
     it('should error when removing pinned blocks', async () => {
-      const cid = await ipfs.dag.put(Buffer.from(hat()), {
+      const cid = await ipfs.dag.put(Buffer.from(nanoid()), {
         format: 'raw',
         hashAlg: 'sha2-256'
       })

packages/interface-ipfs-core/src/dht/get.js

@@ -1,7 +1,7 @@
 /* eslint-env mocha */
 'use strict'
 
-const hat = require('hat')
+const nanoid = require('nanoid')
 const { getDescribe, getIt, expect } = require('../utils/mocha')
 
 /** @typedef { import("ipfsd-ctl/src/factory") } Factory */
@@ -36,8 +36,8 @@ module.exports = (common, options) => {
     // "invalid record keytype" - it needs to put a valid key and value for it to
     // be a useful test.
     it.skip('should get a value after it was put on another node', async () => {
-      const key = Buffer.from(hat())
-      const value = Buffer.from(hat())
+      const key = Buffer.from(nanoid())
+      const value = Buffer.from(nanoid())
 
       await nodeB.dht.put(key, value)
       const result = await nodeA.dht.get(key)