feat: improve helia 101 example

examples/helia-101/.gitignore

@@ -1,6 +1,8 @@
 node_modules
 build
 dist
+blockstore
+datastore
 .docs
 .coverage
 node_modules

examples/helia-101/101-basics.js

@@ -1,39 +1,61 @@
 /* eslint-disable no-console */
+// @ts-check
 
+import * as nodefs from 'fs'
+import { devNull } from 'node:os'
+import { pipeline } from 'stream/promises'
 import { createHeliaHTTP } from '@helia/http'
-import { unixfs } from '@helia/unixfs'
+import { unixfs, urlSource } from '@helia/unixfs'
 
-// create a Helia node
+// `@helia/http` is an light http-only version Helia with the same API,
+// which is useful for simple use cases, where you don't need p2p networking to provide data to other nodes.
+// Since this example is focused on UnixFS without p2p networking, we can use the `@helia/http` package.
 const helia = await createHeliaHTTP()
 
-// create a filesystem on top of Helia, in this case it's UnixFS
+// UnixFS allows you to encode files and directories such that they are addressed by CIDs and can be retrieved by other nodes on the network
 const fs = unixfs(helia)
 
-// add a file and wrap in a directory
-const readmeCid = await fs.addFile({
-  path: './README.md'
-}, {
-  wrapWithDirectory: true
-})
-
-console.log('Added README.md file:', readmeCid.toString())
-
-// we will use this TextEncoder to turn strings into Uint8Arrays
+// we will use this TextEncoder to turn strings into Uint8Arrays which we can add to the node
 const encoder = new TextEncoder()
 
-// add the bytes to your node and receive a unique content identifier
+// addBytes takes raw bytes and returns a raw block CID for the content
+// (larger (over 1 MiB) binary arrays are chunked and return a dag-pb block CID instead)
+// The `bytes` value we have passed to `unixfs` has now been turned into a UnixFS DAG and stored in the helia node.
 const cid = await fs.addBytes(encoder.encode('Hello World 101'), {
   onProgress: (evt) => {
     console.info('add event', evt.type, evt.detail)
   }
 })
-
 console.log('Added file:', cid.toString())
 
+// Create an empty directory
+const directoryCid = await fs.addDirectory({
+  path: 'my-dir'
+})
+
+// Add a raw block CID to the directory as a file with the name `hello.txt`
+const updatedCid = await fs.cp(cid, directoryCid, 'hello.txt')
+console.log('Directory with added file:', updatedCid)
+
+// addFile always returns a directory CID, retaining the filename derived from the `path` argument
+const readmeCid = await fs.addFile({
+  content: nodefs.createReadStream('./README.md'),
+  path: './README.md'
+})
+
+// stat returns a UnixFSStats object, which contains information about the directory
+const readmeStats = await fs.stat(readmeCid)
+console.log('README.md stats:', readmeStats)
+
+// To get the size of a directory, we need extended stats, which traverse the DAG
+const readmeExStats = await fs.stat(readmeCid, { extended: true })
+console.log('README.md stats (extended):', readmeExStats)
+
 // this decoder will turn Uint8Arrays into strings
 const decoder = new TextDecoder()
 let text = ''
 
+// Read the file into memory and print it to the console
 for await (const chunk of fs.cat(cid, {
   onProgress: (evt) => {
     console.info('cat event', evt.type, evt.detail)
@@ -43,5 +65,28 @@ for await (const chunk of fs.cat(cid, {
     stream: true
   })
 }
-
 console.log('Added file contents:', text)
+
+// Add a file to Helia from a URL
+// Helia will download, and add the file into smaller chunks and return a directory containing a file node `2600-h.htm` with links to the raw blocks of the file
+const url = 'https://www.gutenberg.org/files/2600/2600-h/2600-h.htm'
+const urlCid = await fs.addFile(urlSource(url))
+
+const urlCidStats = await fs.stat(urlCid)
+console.log('File from URL: stats:', urlCidStats)
+
+// Instead of loading the file into memory like we did above, we can use the `cat` API, which returns an async iterable,
+// allowing us to stream the file to a writable stream, which we can pipe to devNull, process.stdout, or a file.
+try {
+  await pipeline(
+    fs.cat(urlCid, {
+      path: '/2600-h.htm'
+    }),
+    // Uncomment only one of the three lines below:
+    nodefs.createWriteStream(devNull) // devNull is a writable stream that discards all data written to it
+    // process.stdout, // stream file to the console
+    // createWriteStream('./war_and_peace.html'), // stream to a file on the local file system
+  )
+} catch (err) {
+  console.error('Pipeline failed', err)
+}

examples/helia-101/201-storage.js

@@ -1,50 +1,50 @@
 /* eslint-disable no-console */
-
+// @ts-check
+import { createHeliaHTTP } from '@helia/http'
 import { unixfs } from '@helia/unixfs'
 import { MemoryBlockstore } from 'blockstore-core'
-import { createHelia } from 'helia'
+import { FsBlockstore } from 'blockstore-fs'
 
 // the blockstore is where we store the blocks that make up files. this blockstore
 // stores everything in-memory - other blockstores are available:
 //   - https://www.npmjs.com/package/blockstore-fs - a filesystem blockstore (for use in node)
 //   - https://www.npmjs.com/package/blockstore-idb - an IndexDB blockstore (for use in browsers)
 //   - https://www.npmjs.com/package/blockstore-level - a LevelDB blockstore (for node or browsers,
 //                                        though storing files in a database is rarely a good idea)
-const blockstore = new MemoryBlockstore()
 
-// create a Helia node
-const helia = await createHelia({
-  blockstore
+// Create a new Helia node with an in-memory blockstore
+const helia1 = await createHeliaHTTP({
+  blockstore: new MemoryBlockstore()
 })
 
-// create a filesystem on top of Helia, in this case it's UnixFS
-const fs = unixfs(helia)
+// create a UnixFS filesystem on top of Helia
+const fs1 = unixfs(helia1)
 
 // we will use this TextEncoder to turn strings into Uint8Arrays
 const encoder = new TextEncoder()
 
+const message = 'Hello World 201'
+
 // add the bytes to your node and receive a unique content identifier
-const cid = await fs.addBytes(encoder.encode('Hello World 201'))
+const cid1 = await fs1.addBytes(encoder.encode(message))
 
-console.log('Added file:', cid.toString())
+console.log('Added file contents:', message)
 
-// create a second Helia node using the same blockstore
-const helia2 = await createHelia({
-  blockstore
+// Create a new Helia node with a filesystem blockstore
+const helia2 = await createHeliaHTTP({
+  blockstore: new FsBlockstore('./blockstore')
 })
 
-// create a second filesystem
 const fs2 = unixfs(helia2)
-
-// this decoder will turn Uint8Arrays into strings
-const decoder = new TextDecoder()
-let text = ''
-
-// read the file from the blockstore using the second Helia node
-for await (const chunk of fs2.cat(cid)) {
-  text += decoder.decode(chunk, {
-    stream: true
-  })
+try {
+  // Check if the CID is in the blockstore, which will be true if we ran this script before
+  const stats = await fs2.stat(cid1, { offline: true }) // `offline: true` will prevent the node from trying to fetch the block from the network
+  console.log(`Found ${cid1.toString()} in blockstore:`, stats)
+} catch (error) {
+  console.log("CID can't be found in the blockstore. We will add it now.")
+  // If the CID is not in the blockstore, we will add it now
+  const cid2 = await fs2.addBytes(encoder.encode(message))
+  console.log('Added file:', cid2.toString())
 }
 
-console.log('Added file contents:', text)
+process.exit(0)

examples/helia-101/401-providing.js

@@ -0,0 +1,75 @@
+/* eslint-disable no-console */
+// @ts-check
+import { unixfs } from '@helia/unixfs'
+import { createHelia } from 'helia'
+
+const helia = await createHelia()
+
+// log when our addresses changes
+helia.libp2p.addEventListener('self:peer:update', (evt) => {
+  console.log(
+    'self:peer:update',
+    evt.detail.peer.addresses.map((a) => a.multiaddr.toString())
+  )
+})
+
+console.log('Created Helia node with PeerID:', helia.libp2p.peerId.toString())
+
+// create a filesystem on top of Helia, in this case it's UnixFS
+const fs = unixfs(helia)
+
+// we will use this TextEncoder to turn strings into Uint8Arrays
+const encoder = new TextEncoder()
+
+const text = 'Hello World 🗺️🌎🌍🌏 401!'
+
+// add the bytes to your node and receive a unique content identifier
+let cid = await fs.addFile({
+  content: encoder.encode(text),
+  path: './hello-world.txt'
+})
+console.log('Added file:', cid.toString())
+
+// Run garbage collection to remove unpinned blocks
+await helia.gc({
+  onProgress: (evt) => {
+    console.info('gc event', evt.type, evt.detail)
+  }
+})
+
+// This will fail because the block is not pinned
+try {
+  const stats = await fs.stat(cid, { offline: true }) // offline to avoid fetching the block from the network
+  console.log('Stats:', stats)
+} catch (err) {
+  if (err?.name === 'NotFoundError') {
+    console.log('Block not found, as expected')
+  } else {
+    throw err
+  }
+}
+
+// Add the same bytes again, this time we will pin them
+cid = await fs.addFile({
+  content: encoder.encode(text),
+  path: './hello-world.txt'
+})
+console.log('Added file again:', cid.toString())
+
+// Pin the block and add some metadata
+for await (const pinnedCid of helia.pins.add(cid, {
+  metadata: {
+    added: new Date().toISOString(),
+    addedBy: '401-providing example'
+  }
+})) {
+  console.log('Pinned CID to prevent garbage collection:', pinnedCid.toString())
+}
+
+const pin = await helia.pins.get(cid)
+console.log('Pin:', pin)
+
+// Provide the block to the DHT so that other nodes can find and retrieve it
+await helia.routing.provide(cid)
+
+console.log('CID provided to the DHT:', cid.toString())

examples/helia-101/README.md

@@ -69,78 +69,44 @@ Make sure you have installed all of the following prerequisites on your developm
 > npm run 101-basics
 > npm run 201-storage
 > npm run 301-networking
+> npm run 401-providing
 ```
 
 ## Usage
 
-In this tutorial, we go through spawning a Helia node, adding a file and cating the file [CID][] locally and through the gateway.
+In this tutorial, we go through spawning a Helia node and interacting with [UnixFS](https://docs.ipfs.tech/concepts/glossary/#unixfs), adding bytes, directories, and files to the node and retrieving them.
 
-It it split into three parts, each part builds on the previous one - basics, storage and finally networking.
+It is split into multiple parts, each part builds on the previous one - basics of interaction with UnixFS, storage, networking, and finally providing, garbage collection and pinning.
 
 For this tutorial, you need to install all dependencies in the `package.json` using `npm install`.
 
 ### 101 - Basics
 
-In the [101-basics.js](./101-basics.js) example the first thing we do is create a Helia node:
+The [first example](./101-basics.js) goes into the the basics of interacting with UnixFS, adding bytes, directories, and files to the node and retrieving them.
 
-```js
-import { createHelia } from 'helia'
-
-// create a Helia node
-const helia = await createHelia()
-```
-
-This node allows us to add blocks and later to retrieve them.
-
-Next we use `@helia/unixfs` to add some data to our node:
-
-```js
-import { unixfs } from '@helia/unixfs'
-
-// create a filesystem on top of Helia, in this case it's UnixFS
-const fs = unixfs(helia)
-
-// we will use this TextEncoder to turn strings into Uint8Arrays
-const encoder = new TextEncoder()
-const bytes = encoder.encode('Hello World 101')
+To run it, use the following command:
 
-// add the bytes to your node and receive a unique content identifier
-const cid = await fs.addBytes(bytes)
-
-console.log('Added file:', cid.toString())
+```console
+> npm run 101-basics
 ```
 
-The `bytes` value we have passed to `unixfs` has now been turned into a UnixFS DAG and stored in the helia node.
+### 201 - Storage
 
-We can access it by using the `cat` API and passing the [CID][] that was returned from the invocation of `addBytes`:
+Out of the box Helia will store all data in-memory. This makes it easy to get started, and to create short-lived nodes that do not persist state between restarts, but what if you want to store large amounts of data for long amounts of time?
 
-```js
-// this decoder will turn Uint8Arrays into strings
-const decoder = new TextDecoder()
-let text = ''
+Take a look at [201-storage.js](./201-storage.js) where we explore how to configure different types of persistent storage for your Helia node.
 
-for await (const chunk of fs.cat(cid)) {
-  text += decoder.decode(chunk, {
-    stream: true
-  })
-}
+To run it, use the following command:
 
-console.log('Added file contents:', text)
+```console
+> npm run 201-storage
 ```
 
-That's it!  We've created a Helia node, added a file to it, and retrieved that file.
-
-Next we will look at where the bytes that make up the file go.
-
-### 201 - Storage
-
-Out of the box Helia will store all data in-memory.  This makes it easy to get started, and to create short-lived nodes that do not persist state between restarts, but what if you want to store large amounts of data for long amounts of time?
-
-Take a look at [201-storage.js](./201-storage.js) where we explore how to configure different types of persistent storage for your Helia node.
+If you run the example twice: you may notice that the second time the file is found in the blockstore without being added again.
 
 #### Blockstore
 
-At it's heart the Interplanetary Filesystem is about blocks.  When you add a file to your local Helia node, it is split up into a number of blocks, all of which are stored in a [blockstore](https://www.npmjs.com/package/interface-blockstore).
+At it's heart IPFS is about blocks of data addressed by a [CID][]. When you add a file to your local Helia node, it is split up into a number of blocks, all of which are stored in a [blockstore](https://www.npmjs.com/package/interface-blockstore).
 
 Each block has a [CID][], an identifier that is unique to that block and can be used to request it from other nodes in the network.
 
@@ -228,6 +194,25 @@ const libp2p = await createLibp2p({
 })
 ```
 
+### 401 - Providing
+
+The final example is [401-providing.js](./401-providing.js).
+
+This example shows:
+
+- How to run garbage collection,
+- Pin blocks to prevent them from being garbage collected
+- Add metadata to pins
+- Provide it to the DHT so that other nodes can find and retrieve it.
+
+To run it, use the following command:
+
+```console
+> npm run 401-providing
+```
+
+
+
 ### Putting it all together
 
 Since your Helia node is configured with a libp2p node, you can go to an IPFS Gateway and load the printed hash. Go ahead and try it!
@@ -240,7 +225,7 @@ Added file: bafkreife2klsil6kaxqhvmhgldpsvk5yutzm4i5bgjoq6fydefwtihnesa
 # https://ipfs.io/ipfs/bafkreife2klsil6kaxqhvmhgldpsvk5yutzm4i5bgjoq6fydefwtihnesa
 ```
 
-That's it! You just added and retrieved a file from the Distributed Web!
+That's it! You just added and retrieved a file from IPFS!
 
 _For more examples, please refer to the [Documentation](#documentation)_