feat: preload mfs root preiodically and preload dag.put

License: MIT
Signed-off-by: Alan Shaw <[email protected]>

Author: alanshaw

src/core/components/dag.js

@@ -24,7 +24,11 @@ module.exports = function dag (self) {
 
       options = options.cid ? options : Object.assign({}, optionDefaults, options)
 
-      self._ipld.put(dagNode, options, callback)
+      self._ipld.put(dagNode, options, (err, cid) => {
+        if (err) return callback(err)
+        if (options.preload !== false) self._preload(cid)
+        callback(null, cid)
+      })
     }),
 
     get: promisify((cid, path, options, callback) => {

src/core/components/index.js

@@ -26,4 +26,4 @@ exports.dht = require('./dht')
 exports.dns = require('./dns')
 exports.key = require('./key')
 exports.stats = require('./stats')
-exports.mfs = require('ipfs-mfs/core')
+exports.mfs = require('./mfs')

src/core/components/mfs.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const promisify = require('promisify-es6')
+const mfs = require('ipfs-mfs/core')
+
+module.exports = self => {
+  const mfsSelf = Object.assign({}, self)
+
+  // A patched dag API to ensure preload doesn't happen for MFS operations
+  mfsSelf.dag = Object.assign({}, self.dag, {
+    put: promisify((node, opts, cb) => {
+      if (typeof opts === 'function') {
+        cb = opts
+        opts = {}
+      }
+
+      opts = Object.assign({}, opts, { preload: false })
+
+      return self.dag.put(node, opts, cb)
+    })
+  })
+
+  return mfs(mfsSelf, mfsSelf._options)
+}

src/core/components/pin-set.js

@@ -90,7 +90,7 @@ exports = module.exports = function (dag) {
 
       pinSet.storeItems(pins, (err, rootNode) => {
         if (err) { return callback(err) }
-        const opts = { cid: new CID(rootNode.multihash) }
+        const opts = { cid: new CID(rootNode.multihash), preload: false }
         dag.put(rootNode, opts, (err, cid) => {
           if (err) { return callback(err) }
           callback(null, rootNode)
@@ -168,7 +168,8 @@ exports = module.exports = function (dag) {
         function storeChild (err, child, binIdx, cb) {
           if (err) { return cb(err) }
 
-          dag.put(child, { cid: new CID(child._multihash) }, err => {
+          const opts = { cid: new CID(child._multihash), preload: false }
+          dag.put(child, opts, err => {
             if (err) { return cb(err) }
             fanoutLinks[binIdx] = new DAGLink('', child.size, child.multihash)
             cb(null)

src/core/components/pin.js

@@ -80,14 +80,14 @@ module.exports = (self) => {
       // the pin-set nodes link to a special 'empty' node, so make sure it exists
       cb => DAGNode.create(Buffer.alloc(0), (err, empty) => {
         if (err) { return cb(err) }
-        dag.put(empty, { cid: new CID(empty.multihash) }, cb)
+        dag.put(empty, { cid: new CID(empty.multihash), preload: false }, cb)
       }),
 
       // create a root node with DAGLinks to the direct and recursive DAGs
       cb => DAGNode.create(Buffer.alloc(0), [dLink, rLink], (err, node) => {
         if (err) { return cb(err) }
         root = node
-        dag.put(root, { cid: new CID(root.multihash) }, cb)
+        dag.put(root, { cid: new CID(root.multihash), preload: false }, cb)
       }),
 
       // hack for CLI tests

src/core/components/start.js

@@ -35,6 +35,7 @@ module.exports = (self) => {
       (cb) => self.libp2p.start(cb),
       (cb) => {
         self._preload.start()
+        self._mfsPreload.start()
 
         self._bitswap = new Bitswap(
           self._libp2pNode,

src/core/components/stop.js

@@ -31,6 +31,7 @@ module.exports = (self) => {
     self._blockService.unsetExchange()
     self._bitswap.stop()
     self._preload.stop()
+    self._mfsPreload.stop()
 
     series([
       (cb) => self.libp2p.stop(cb),

src/core/index.js

@@ -23,6 +23,7 @@ const components = require('./components')
 // replaced by repo-browser when running in the browser
 const defaultRepo = require('./runtime/repo-nodejs')
 const preload = require('./preload')
+const mfsPreload = require('./mfs-preload')
 
 class IPFS extends EventEmitter {
   constructor (options) {
@@ -87,6 +88,7 @@ class IPFS extends EventEmitter {
     this._ipld = new Ipld(this._blockService)
     this._pubsub = undefined
     this._preload = preload(this)
+    this._mfsPreload = mfsPreload(this)
 
     // IPFS Core exposed components
     //   - for booting up a node
@@ -143,7 +145,7 @@ class IPFS extends EventEmitter {
     }
 
     // ipfs.files
-    const mfs = components.mfs(this, this._options)
+    const mfs = components.mfs(this)
 
     Object.keys(mfs).forEach(key => {
       this.files[key] = mfs[key]

src/core/mfs-preload.js

@@ -0,0 +1,47 @@
+'use strict'
+
+const debug = require('debug')
+
+const log = debug('jsipfs:mfs-preload')
+log.error = debug('jsipfs:mfs-preload:error')
+
+module.exports = (self, options) => {
+  options = options || {}
+  options.interval = options.interval || 30 * 1000
+
+  let rootCid
+  let timeoutId
+
+  const preloadMfs = () => {
+    self.files.stat('/', (err, stats) => {
+      if (err) {
+        timeoutId = setTimeout(preloadMfs, options.interval)
+        return log.error('failed to stat MFS root for preload', err)
+      }
+
+      if (rootCid !== stats.hash) {
+        log(`preloading updated MFS root ${rootCid} -> ${stats.hash}`)
+
+        self._preload(stats.hash, (err) => {
+          timeoutId = setTimeout(preloadMfs, options.interval)
+          if (err) return log.error(`failed to preload MFS root ${stats.hash}`, err)
+          rootCid = stats.hash
+        })
+      }
+    })
+  }
+
+  return {
+    start () {
+      self.files.stat('/', (err, stats) => {
+        if (err) return log.error('failed to stat MFS root for preload', err)
+        rootCid = stats.hash
+        log(`monitoring MFS root ${rootCid}`)
+        timeoutId = setTimeout(preloadMfs, options.interval)
+      })
+    },
+    stop () {
+      clearTimeout(timeoutId)
+    }
+  }
+}