Skip to content

Commit 47ac55b

Browse files
2colorachingbrain
2color
and
achingbrain
authored
feat: expand and improve helia 101 example (#456)
* feat: more usage examples for helia unixfs * deps: update to fix port binding errors * refine eamples * feat: add an additional example * feat: 401 providing, gc and pinning * chore: remove unused deps * chore: lint * test: add simple test for 401 example * deps: upgrade * feat: add extended dag stats * Update examples/helia-101/101-basics.js * Update examples/helia-101/101-basics.js Co-authored-by: Alex Potsides <[email protected]> * chore: stop helia so the process exits --------- Co-authored-by: Daniel N <[email protected]> Co-authored-by: Alex Potsides <[email protected]>
1 parent 73726f0 commit 47ac55b

File tree

8 files changed

+217
-100
lines changed

8 files changed

+217
-100
lines changed

‎examples/helia-101/.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
node_modules
22
build
33
dist
4+
blockstore
5+
datastore
46
.docs
57
.coverage
68
node_modules

‎examples/helia-101/101-basics.js

+59-16
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,59 @@
11
/* eslint-disable no-console */
2+
// @ts-check
23

4+
import * as nodefs from 'fs'
5+
import { devNull } from 'node:os'
6+
import { pipeline } from 'stream/promises'
37
import { createHeliaHTTP } from '@helia/http'
4-
import { unixfs } from '@helia/unixfs'
8+
import { unixfs, urlSource } from '@helia/unixfs'
59

6-
// create a Helia node
10+
// `@helia/http` is an light http-only version Helia with the same API,
11+
// which is useful for simple use cases, where you don't need p2p networking to provide data to other nodes.
12+
// Since this example is focused on UnixFS without p2p networking, we can use the `@helia/http` package.
713
const helia = await createHeliaHTTP()
814

9-
// create a filesystem on top of Helia, in this case it's UnixFS
15+
// UnixFS allows you to encode files and directories such that they are addressed by CIDs and can be retrieved by other nodes on the network
1016
const fs = unixfs(helia)
1117

12-
// add a file and wrap in a directory
13-
const readmeCid = await fs.addFile({
14-
path: './README.md'
15-
}, {
16-
wrapWithDirectory: true
17-
})
18-
19-
console.log('Added README.md file:', readmeCid.toString())
20-
21-
// we will use this TextEncoder to turn strings into Uint8Arrays
18+
// we will use this TextEncoder to turn strings into Uint8Arrays which we can add to the node
2219
const encoder = new TextEncoder()
2320

24-
// add the bytes to your node and receive a unique content identifier
21+
// addBytes takes raw bytes and returns a raw block CID for the content
22+
// (larger (over 1 MiB) binary arrays are chunked and return a dag-pb block CID instead)
23+
// The `bytes` value we have passed to `unixfs` has now been turned into a UnixFS DAG and stored in the helia node.
2524
const cid = await fs.addBytes(encoder.encode('Hello World 101'), {
2625
onProgress: (evt) => {
2726
console.info('add event', evt.type, evt.detail)
2827
}
2928
})
30-
3129
console.log('Added file:', cid.toString())
3230

31+
// Create an empty directory
32+
const directoryCid = await fs.addDirectory()
33+
34+
// Add a raw block CID to the directory as a file with the name `hello.txt`
35+
const updatedCid = await fs.cp(cid, directoryCid, 'hello.txt')
36+
console.log('Directory with added file:', updatedCid)
37+
38+
// addFile always returns a directory CID, retaining the filename derived from the `path` argument
39+
const readmeCid = await fs.addFile({
40+
content: nodefs.createReadStream('./README.md'),
41+
path: './README.md'
42+
})
43+
44+
// stat returns a UnixFSStats object, which contains information about the directory
45+
const readmeStats = await fs.stat(readmeCid)
46+
console.log('README.md stats:', readmeStats)
47+
48+
// To get the size of a directory, we need extended stats, which traverse the DAG
49+
const readmeExStats = await fs.stat(readmeCid, { extended: true })
50+
console.log('README.md stats (extended):', readmeExStats)
51+
3352
// this decoder will turn Uint8Arrays into strings
3453
const decoder = new TextDecoder()
3554
let text = ''
3655

56+
// Read the file into memory and print it to the console
3757
for await (const chunk of fs.cat(cid, {
3858
onProgress: (evt) => {
3959
console.info('cat event', evt.type, evt.detail)
@@ -43,5 +63,28 @@ for await (const chunk of fs.cat(cid, {
4363
stream: true
4464
})
4565
}
46-
4766
console.log('Added file contents:', text)
67+
68+
// Add a file to Helia from a URL
69+
// Helia will download, and add the file into smaller chunks and return a directory containing a file node `2600-h.htm` with links to the raw blocks of the file
70+
const url = 'https://www.gutenberg.org/files/2600/2600-h/2600-h.htm'
71+
const urlCid = await fs.addFile(urlSource(url))
72+
73+
const urlCidStats = await fs.stat(urlCid)
74+
console.log('File from URL: stats:', urlCidStats)
75+
76+
// Instead of loading the file into memory like we did above, we can use the `cat` API, which returns an async iterable,
77+
// allowing us to stream the file to a writable stream, which we can pipe to devNull, process.stdout, or a file.
78+
try {
79+
await pipeline(
80+
fs.cat(urlCid, {
81+
path: '/2600-h.htm'
82+
}),
83+
// Uncomment only one of the three lines below:
84+
nodefs.createWriteStream(devNull) // devNull is a writable stream that discards all data written to it
85+
// process.stdout, // stream file to the console
86+
// createWriteStream('./war_and_peace.html'), // stream to a file on the local file system
87+
)
88+
} catch (err) {
89+
console.error('Pipeline failed', err)
90+
}

‎examples/helia-101/201-storage.js

+26-25
Original file line numberDiff line numberDiff line change
@@ -1,50 +1,51 @@
11
/* eslint-disable no-console */
2-
2+
// @ts-check
3+
import { createHeliaHTTP } from '@helia/http'
34
import { unixfs } from '@helia/unixfs'
45
import { MemoryBlockstore } from 'blockstore-core'
5-
import { createHelia } from 'helia'
6+
import { FsBlockstore } from 'blockstore-fs'
67

78
// the blockstore is where we store the blocks that make up files. this blockstore
89
// stores everything in-memory - other blockstores are available:
910
// - https://www.npmjs.com/package/blockstore-fs - a filesystem blockstore (for use in node)
1011
// - https://www.npmjs.com/package/blockstore-idb - an IndexDB blockstore (for use in browsers)
1112
// - https://www.npmjs.com/package/blockstore-level - a LevelDB blockstore (for node or browsers,
1213
// though storing files in a database is rarely a good idea)
13-
const blockstore = new MemoryBlockstore()
1414

15-
// create a Helia node
16-
const helia = await createHelia({
17-
blockstore
15+
// Create a new Helia node with an in-memory blockstore
16+
const helia1 = await createHeliaHTTP({
17+
blockstore: new MemoryBlockstore()
1818
})
1919

20-
// create a filesystem on top of Helia, in this case it's UnixFS
21-
const fs = unixfs(helia)
20+
// create a UnixFS filesystem on top of Helia
21+
const fs1 = unixfs(helia1)
2222

2323
// we will use this TextEncoder to turn strings into Uint8Arrays
2424
const encoder = new TextEncoder()
2525

26+
const message = 'Hello World 201'
27+
2628
// add the bytes to your node and receive a unique content identifier
27-
const cid = await fs.addBytes(encoder.encode('Hello World 201'))
29+
const cid1 = await fs1.addBytes(encoder.encode(message))
2830

29-
console.log('Added file:', cid.toString())
31+
console.log('Added file contents:', message)
3032

31-
// create a second Helia node using the same blockstore
32-
const helia2 = await createHelia({
33-
blockstore
33+
// Create a new Helia node with a filesystem blockstore
34+
const helia2 = await createHeliaHTTP({
35+
blockstore: new FsBlockstore('./blockstore')
3436
})
3537

36-
// create a second filesystem
3738
const fs2 = unixfs(helia2)
38-
39-
// this decoder will turn Uint8Arrays into strings
40-
const decoder = new TextDecoder()
41-
let text = ''
42-
43-
// read the file from the blockstore using the second Helia node
44-
for await (const chunk of fs2.cat(cid)) {
45-
text += decoder.decode(chunk, {
46-
stream: true
47-
})
39+
try {
40+
// Check if the CID is in the blockstore, which will be true if we ran this script before
41+
const stats = await fs2.stat(cid1, { offline: true }) // `offline: true` will prevent the node from trying to fetch the block from the network
42+
console.log(`Found ${cid1.toString()} in blockstore:`, stats)
43+
} catch (error) {
44+
console.log("CID can't be found in the blockstore. We will add it now.")
45+
// If the CID is not in the blockstore, we will add it now
46+
const cid2 = await fs2.addBytes(encoder.encode(message))
47+
console.log('Added file:', cid2.toString())
4848
}
4949

50-
console.log('Added file contents:', text)
50+
await helia1.stop()
51+
await helia2.stop()

‎examples/helia-101/301-networking.js

+4
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
/* eslint-disable no-console */
2+
// @ts-check
23

34
import { noise } from '@chainsafe/libp2p-noise'
45
import { yamux } from '@chainsafe/libp2p-yamux'
@@ -91,3 +92,6 @@ for await (const chunk of fs2.cat(cid)) {
9192
}
9293

9394
console.log('Fetched file contents:', text)
95+
96+
await node1.stop()
97+
await node2.stop()

‎examples/helia-101/401-providing.js

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
/* eslint-disable no-console */
2+
// @ts-check
3+
import { unixfs } from '@helia/unixfs'
4+
import { createHelia } from 'helia'
5+
6+
const helia = await createHelia()
7+
8+
// log when our addresses changes
9+
helia.libp2p.addEventListener('self:peer:update', (evt) => {
10+
console.log(
11+
'self:peer:update',
12+
evt.detail.peer.addresses.map((a) => a.multiaddr.toString())
13+
)
14+
})
15+
16+
console.log('Created Helia node with PeerID:', helia.libp2p.peerId.toString())
17+
18+
// create a filesystem on top of Helia, in this case it's UnixFS
19+
const fs = unixfs(helia)
20+
21+
// we will use this TextEncoder to turn strings into Uint8Arrays
22+
const encoder = new TextEncoder()
23+
24+
const text = 'Hello World 🗺️🌎🌍🌏 401!'
25+
26+
// add the bytes to your node and receive a unique content identifier
27+
let cid = await fs.addFile({
28+
content: encoder.encode(text),
29+
path: './hello-world.txt'
30+
})
31+
console.log('Added file:', cid.toString())
32+
33+
// Run garbage collection to remove unpinned blocks
34+
await helia.gc({
35+
onProgress: (evt) => {
36+
console.info('gc event', evt.type, evt.detail)
37+
}
38+
})
39+
40+
// This will fail because the block is not pinned
41+
try {
42+
const stats = await fs.stat(cid, { offline: true }) // offline to avoid fetching the block from the network
43+
console.log('Stats:', stats)
44+
} catch (err) {
45+
if (err?.name === 'NotFoundError') {
46+
console.log('Block not found, as expected')
47+
} else {
48+
throw err
49+
}
50+
}
51+
52+
// Add the same bytes again, this time we will pin them
53+
cid = await fs.addFile({
54+
content: encoder.encode(text),
55+
path: './hello-world.txt'
56+
})
57+
console.log('Added file again:', cid.toString())
58+
59+
// Pin the block and add some metadata
60+
for await (const pinnedCid of helia.pins.add(cid, {
61+
metadata: {
62+
added: new Date().toISOString(),
63+
addedBy: '401-providing example'
64+
}
65+
})) {
66+
console.log('Pinned CID to prevent garbage collection:', pinnedCid.toString())
67+
}
68+
69+
const pin = await helia.pins.get(cid)
70+
console.log('Pin:', pin)
71+
72+
// Provide the block to the DHT so that other nodes can find and retrieve it
73+
await helia.routing.provide(cid)
74+
75+
console.log('CID provided to the DHT:', cid.toString())
76+
77+
await helia.stop()

0 commit comments

Comments
 (0)