Skip to content

Commit a877ccc

Browse files
dignifiedquiredignifiedquire
committed
Merge pull request #52 from ipfs/feat/read-write
refactor: new go-ipfs blockstore like interface
2 parents c5cd93f + 9229e57 commit a877ccc

File tree

3 files changed

+158
-119
lines changed

3 files changed

+158
-119
lines changed

‎package.json

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@
3232
"abstract-blob-store": "^3.2.0",
3333
"aegir": "^2.1.2",
3434
"async": "^1.5.2",
35-
"bl": "^1.1.2",
36-
"bs58": "^3.0.0",
3735
"buffer-loader": "^0.0.1",
3836
"chai": "^3.5.0",
3937
"fs-blob-store": "^5.2.1",
@@ -48,6 +46,7 @@
4846
"babel-runtime": "^6.6.1",
4947
"bl": "^1.1.2",
5048
"concat-stream": "^1.5.1",
49+
"ipfs-block": "^0.2.0",
5150
"level-js": "^2.2.3",
5251
"lock": "^0.1.2",
5352
"lockfile": "^1.0.1",
@@ -64,4 +63,4 @@
6463
"dignifiedquire <[email protected]>",
6564
"greenkeeperio-bot <[email protected]>"
6665
]
67-
}
66+
}

‎src/stores/datastore.js

Lines changed: 46 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
const Lock = require('lock')
44
const stream = require('stream')
5+
const bl = require('bl')
6+
const Block = require('ipfs-block')
57

68
const PREFIX_LENGTH = 8
79

@@ -20,47 +22,72 @@ exports.setUp = (basePath, blobStore, locks) => {
2022
const store = blobStore(basePath + '/blocks')
2123
const lock = new Lock()
2224

23-
return {
24-
createReadStream: (multihash, extension) => {
25-
const path = multihashToPath(multihash, extension)
26-
return store.createReadStream(path)
27-
},
25+
const createReadStream = (multihash, extension) => {
26+
const path = multihashToPath(multihash, extension)
27+
return store.createReadStream(path)
28+
}
29+
30+
const createWriteStream = (multihash, extension, cb) => {
31+
const path = multihashToPath(multihash, extension)
32+
const through = stream.PassThrough()
33+
34+
lock(path, (release) => {
35+
const ws = store.createWriteStream(path, release(cb))
36+
through.pipe(ws)
37+
})
38+
39+
return through
40+
}
2841

29-
createWriteStream: (multihash, extension, cb) => {
42+
return {
43+
get: (key, extension, cb) => {
3044
if (typeof extension === 'function') {
3145
cb = extension
3246
extension = undefined
3347
}
3448

35-
const path = multihashToPath(multihash, extension)
36-
const through = stream.PassThrough()
49+
if (!key) {
50+
return cb(new Error('Invalid key'))
51+
}
52+
53+
createReadStream(key, 'data')
54+
.pipe(bl((err, data) => {
55+
if (err) {
56+
return cb(err)
57+
}
3758

38-
lock(path, (release) => {
39-
const ws = store.createWriteStream(path, release(cb))
40-
through.pipe(ws)
41-
})
59+
cb(null, new Block(data, extension))
60+
}))
61+
},
62+
63+
put: (block, cb) => {
64+
if (!block || !block.data) {
65+
return cb(new Error('Invalid block'))
66+
}
4267

43-
return through
68+
const ws = createWriteStream(block.key, block.extension, cb)
69+
ws.write(block.data)
70+
ws.end()
4471
},
4572

46-
exists: (multihash, extension, cb) => {
73+
has: (key, extension, cb) => {
4774
if (typeof extension === 'function') {
4875
cb = extension
4976
extension = undefined
5077
}
5178

52-
const path = multihashToPath(multihash, extension)
53-
return store.exists(path, cb)
79+
const path = multihashToPath(key, extension)
80+
store.exists(path, cb)
5481
},
5582

56-
remove: (multihash, extension, cb) => {
83+
delete: (key, extension, cb) => {
5784
if (typeof extension === 'function') {
5885
cb = extension
5986
extension = undefined
6087
}
6188

62-
const path = multihashToPath(multihash, extension)
63-
return store.remove(path, cb)
89+
const path = multihashToPath(key, extension)
90+
store.remove(path, cb)
6491
}
6592
}
6693
}

‎test/repo-test.js

Lines changed: 110 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,7 @@
44

55
const Repo = require('../src/index')
66
const expect = require('chai').expect
7-
const base58 = require('bs58')
8-
const bl = require('bl')
9-
const fs = require('fs')
10-
const join = require('path').join
11-
12-
const fileA = fs.readFileSync(join(__dirname, 'test-repo/blocks/12207028/122070286b9afa6620a66f715c7020d68af3d10e1a497971629c07606bfdb812303d.data'))
13-
14-
const fileAExt = fs.readFileSync(join(__dirname, 'test-repo/blocks/12207028/122070286b9afa6620a66f715c7020d68af3d10e1a497971629c07606bfdb812303d.ext'))
7+
const Block = require('ipfs-block')
158

169
module.exports = function (repo) {
1710
describe('IPFS Repo Tests', function () {
@@ -138,127 +131,147 @@ module.exports = function (repo) {
138131
})
139132

140133
describe('datastore', function () {
141-
const baseFileHash = 'QmVtU7ths96fMgZ8YSZAbKghyieq7AjxNdcqyVzxTt3qVe'
142-
const baseExtFileHash = 'QmVtU7ths96fMgZ8YSZAbKghyieq7AjxNdcqyVzxTt3qVe'
134+
const helloKey = '1220b94d/1220b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9.data'
135+
const helloIpldKey = '1220b94d/1220b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9.ipld'
143136

144-
it('reads block', function (done) {
145-
const buf = new Buffer(base58.decode(baseFileHash))
146-
repo.datastore.createReadStream(buf)
147-
.pipe(bl((err, data) => {
137+
describe('.put', () => {
138+
it('simple', function (done) {
139+
const b = new Block('hello world')
140+
repo.datastore.put(b, (err, meta) => {
148141
expect(err).to.not.exist
149-
expect(data.equals(fileA)).to.equal(true)
142+
expect(meta.key).to.be.eql(helloKey)
150143
done()
151-
}))
152-
})
144+
})
145+
})
153146

154-
it('reads block, with custom extension', function (done) {
155-
const buf = new Buffer(base58.decode(baseFileHash))
156-
repo.datastore.createReadStream(buf, 'ext')
157-
.pipe(bl((err, data) => {
158-
expect(err).to.not.exist
159-
expect(data.equals(fileAExt)).to.equal(true)
160-
done()
161-
}))
162-
})
147+
it('multi write (locks)', (done) => {
148+
const b = new Block('hello world')
163149

164-
it('write a block', function (done) {
165-
const rnd = 'QmVtU7ths96fMgZ8YSZAbKghyieq7AjxNdcqyVtesthash'
166-
const mh = new Buffer(base58.decode(rnd))
167-
const data = new Buffer('Oh the data')
150+
let i = 0
151+
const finish = () => {
152+
i++
153+
if (i === 2) done()
154+
}
168155

169-
repo.datastore.createWriteStream(mh, (err, metadata) => {
170-
expect(err).to.not.exist
171-
expect(metadata.key).to.equal('12207028/122070286b9afa6620a66f715c7020d68af3d10e1a497971629c07605f55537ce990.data')
172-
done()
173-
}).end(data)
174-
})
156+
repo.datastore.put(b, (err, meta) => {
157+
expect(err).to.not.exist
158+
expect(meta.key).to.equal(helloKey)
159+
finish()
160+
})
175161

176-
it('write a block with custom extension', function (done) {
177-
const rnd = 'QmVtU7ths96fMgZ8YSZAbKghyieq7AjxNdcqyVtesthash'
178-
const mh = new Buffer(base58.decode(rnd))
179-
const data = new Buffer('Oh the data')
162+
repo.datastore.put(b, (err, meta) => {
163+
expect(err).to.not.exist
164+
expect(meta.key).to.equal(helloKey)
165+
finish()
166+
})
167+
})
180168

181-
repo.datastore.createWriteStream(mh, 'ext', (err, metadata) => {
182-
expect(err).to.not.exist
183-
expect(metadata.key).to.equal('12207028/122070286b9afa6620a66f715c7020d68af3d10e1a497971629c07605f55537ce990.ext')
184-
done()
185-
}).end(data)
169+
it('custom extension', function (done) {
170+
const b = new Block('hello world', 'ipld')
171+
repo.datastore.put(b, (err, meta) => {
172+
expect(err).to.not.exist
173+
expect(meta.key).to.be.eql(helloIpldKey)
174+
done()
175+
})
176+
})
177+
178+
it('returns an error on invalid block', (done) => {
179+
repo.datastore.put('hello', (err) => {
180+
expect(err.message).to.be.eql('Invalid block')
181+
done()
182+
})
183+
})
186184
})
187185

188-
it('write locks', (done) => {
189-
const rnd = 'QmVtU7ths96fMgZ8YSZAbKghyieq7AjxNdcqyVtesthash'
190-
const mh = new Buffer(base58.decode(rnd))
191-
const data = new Buffer('Oh the data')
186+
describe('.get', () => {
187+
it('simple', (done) => {
188+
const b = new Block('hello world')
192189

193-
let i = 0
194-
const finish = () => {
195-
i++
196-
if (i === 2) done()
197-
}
190+
repo.datastore.get(b.key, (err, data) => {
191+
expect(err).to.not.exist
192+
expect(data).to.be.eql(b)
198193

199-
repo.datastore.createWriteStream(mh, (err, metadata) => {
200-
expect(err).to.not.exist
201-
expect(metadata.key).to.equal('12207028/122070286b9afa6620a66f715c7020d68af3d10e1a497971629c07605f55537ce990.data')
202-
finish()
203-
}).end(data)
194+
done()
195+
})
196+
})
204197

205-
repo.datastore.createWriteStream(mh, (err, metadata) => {
206-
expect(err).to.not.exist
207-
expect(metadata.key).to.equal('12207028/122070286b9afa6620a66f715c7020d68af3d10e1a497971629c07605f55537ce990.data')
208-
finish()
209-
}).end(data)
210-
})
198+
it('custom extension', (done) => {
199+
const b = new Block('hello world', 'ipld')
211200

212-
it('block exists', function (done) {
213-
const buf = new Buffer(base58.decode(baseFileHash))
201+
repo.datastore.get(b.key, b.extension, (err, data) => {
202+
expect(err).to.not.exist
203+
expect(data).to.be.eql(b)
214204

215-
repo.datastore.exists(buf, (err, exists) => {
216-
expect(err).to.not.exist
217-
expect(exists).to.equal(true)
218-
done()
205+
done()
206+
})
207+
})
208+
209+
it('returns an error on invalid block', (done) => {
210+
repo.datastore.get(null, (err) => {
211+
expect(err.message).to.be.eql('Invalid key')
212+
done()
213+
})
219214
})
220215
})
221216

222-
it('block exists, with custom extension', function (done) {
223-
const buf = new Buffer(base58.decode(baseExtFileHash))
217+
describe('.has', () => {
218+
it('existing block', (done) => {
219+
const b = new Block('hello world')
224220

225-
repo.datastore.exists(buf, 'ext', (err, exists) => {
226-
expect(err).to.not.exist
227-
expect(exists).to.equal(true)
228-
done()
221+
repo.datastore.has(b.key, (err, exists) => {
222+
expect(err).to.not.exist
223+
expect(exists).to.equal(true)
224+
done()
225+
})
229226
})
230-
})
231227

232-
it('check for non existent block', function (done) {
233-
const buf = new Buffer('random buffer')
228+
it('with extension', (done) => {
229+
const b = new Block('hello world')
234230

235-
repo.datastore.exists(buf, (err, exists) => {
236-
expect(err).to.not.exist
237-
expect(exists).to.equal(false)
238-
done()
231+
repo.datastore.has(b.key, 'data', (err, exists) => {
232+
expect(err).to.not.exist
233+
expect(exists).to.equal(true)
234+
done()
235+
})
239236
})
240-
})
241237

242-
it('remove a block', function (done) {
243-
const buf = new Buffer(base58.decode(baseFileHash))
244-
repo.datastore.remove(buf, (err) => {
245-
expect(err).to.not.exist
246-
repo.datastore.exists(buf, (err, exists) => {
238+
it('non existent block', (done) => {
239+
const b = new Block('wooot')
240+
241+
repo.datastore.has(b.key, (err, exists) => {
247242
expect(err).to.not.exist
248243
expect(exists).to.equal(false)
249244
done()
250245
})
251246
})
252247
})
253248

254-
it('remove a block, with custom extension', function (done) {
255-
const buf = new Buffer(base58.decode(baseExtFileHash))
256-
repo.datastore.remove(buf, 'ext', (err) => {
257-
expect(err).to.not.exist
258-
repo.datastore.exists(buf, 'ext', (err, exists) => {
249+
describe('.delete', () => {
250+
it('simple', (done) => {
251+
const b = new Block('hello world')
252+
253+
repo.datastore.delete(b.key, (err) => {
259254
expect(err).to.not.exist
260-
expect(exists).to.equal(false)
261-
done()
255+
256+
repo.datastore.has(b.key, (err, exists) => {
257+
expect(err).to.not.exist
258+
expect(exists).to.equal(false)
259+
done()
260+
})
261+
})
262+
})
263+
264+
it('custom extension', (done) => {
265+
const b = new Block('hello world', 'ipld')
266+
267+
repo.datastore.delete(b.key, b.extension, (err) => {
268+
expect(err).to.not.exist
269+
270+
repo.datastore.has(b.key, b.extension, (err, exists) => {
271+
expect(err).to.not.exist
272+
expect(exists).to.equal(false)
273+
done()
274+
})
262275
})
263276
})
264277
})

0 commit comments

Comments
 (0)