Skip to content

Commit 1790dea

Browse files
willscottwillscott
authored
add AsLargeBytes support to unixfs files (#24)
* add AsLargeBytes support to unixfs files This commit was moved from ipfs/go-unixfsnode@94986a7
1 parent 79823b0 commit 1790dea

File tree

5 files changed

+342
-47
lines changed

5 files changed

+342
-47
lines changed

‎unixfs/node/data/builder/file_test.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package builder
33
import (
44
"bytes"
55
"context"
6-
"io"
76
"testing"
87

98
"github.com/ipfs/go-cid"
@@ -70,7 +69,7 @@ func TestUnixFSFileRoundtrip(t *testing.T) {
7069
t.Fatal(err)
7170
}
7271
// read back out the file.
73-
out, err := io.ReadAll(ufn)
72+
out, err := ufn.AsBytes()
7473
if err != nil {
7574
t.Fatal(err)
7675
}

‎unixfs/node/file/deferred.go

+173
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
package file
2+
3+
import (
4+
"context"
5+
"io"
6+
7+
dagpb "github.com/ipld/go-codec-dagpb"
8+
"github.com/ipld/go-ipld-prime"
9+
)
10+
11+
func newDeferredFileNode(ctx context.Context, lsys *ipld.LinkSystem, root ipld.Link) LargeBytesNode {
12+
dfn := deferredFileNode{
13+
LargeBytesNode: nil,
14+
root: root,
15+
lsys: lsys,
16+
ctx: ctx,
17+
}
18+
dfn.LargeBytesNode = &deferred{&dfn}
19+
return &dfn
20+
}
21+
22+
type deferredFileNode struct {
23+
LargeBytesNode
24+
25+
root ipld.Link
26+
lsys *ipld.LinkSystem
27+
ctx context.Context
28+
}
29+
30+
func (d *deferredFileNode) resolve() error {
31+
if d.lsys == nil {
32+
return nil
33+
}
34+
target, err := d.lsys.Load(ipld.LinkContext{Ctx: d.ctx}, d.root, protoFor(d.root))
35+
if err != nil {
36+
return err
37+
}
38+
39+
asFSNode, err := NewUnixFSFile(d.ctx, target, d.lsys)
40+
if err != nil {
41+
return err
42+
}
43+
d.LargeBytesNode = asFSNode
44+
d.root = nil
45+
d.lsys = nil
46+
d.ctx = nil
47+
return nil
48+
}
49+
50+
type deferred struct {
51+
*deferredFileNode
52+
}
53+
54+
type deferredReader struct {
55+
io.ReadSeeker
56+
*deferredFileNode
57+
}
58+
59+
func (d *deferred) AsLargeBytes() (io.ReadSeeker, error) {
60+
return &deferredReader{nil, d.deferredFileNode}, nil
61+
}
62+
63+
func (d *deferredReader) Read(p []byte) (int, error) {
64+
if d.ReadSeeker == nil {
65+
if err := d.deferredFileNode.resolve(); err != nil {
66+
return 0, err
67+
}
68+
rs, err := d.deferredFileNode.AsLargeBytes()
69+
if err != nil {
70+
return 0, err
71+
}
72+
d.ReadSeeker = rs
73+
}
74+
return d.ReadSeeker.Read(p)
75+
}
76+
77+
func (d *deferredReader) Seek(offset int64, whence int) (int64, error) {
78+
if d.ReadSeeker == nil {
79+
if err := d.deferredFileNode.resolve(); err != nil {
80+
return 0, err
81+
}
82+
rs, err := d.deferredFileNode.AsLargeBytes()
83+
if err != nil {
84+
return 0, err
85+
}
86+
d.ReadSeeker = rs
87+
}
88+
return d.ReadSeeker.Seek(offset, whence)
89+
}
90+
91+
func (d *deferred) Kind() ipld.Kind {
92+
return ipld.Kind_Bytes
93+
}
94+
95+
func (d *deferred) AsBytes() ([]byte, error) {
96+
if err := d.deferredFileNode.resolve(); err != nil {
97+
return []byte{}, err
98+
}
99+
100+
return d.deferredFileNode.AsBytes()
101+
}
102+
103+
func (d *deferred) AsBool() (bool, error) {
104+
return false, ipld.ErrWrongKind{TypeName: "bool", MethodName: "AsBool", AppropriateKind: ipld.KindSet_JustBytes}
105+
}
106+
107+
func (d *deferred) AsInt() (int64, error) {
108+
return 0, ipld.ErrWrongKind{TypeName: "int", MethodName: "AsInt", AppropriateKind: ipld.KindSet_JustBytes}
109+
}
110+
111+
func (d *deferred) AsFloat() (float64, error) {
112+
return 0, ipld.ErrWrongKind{TypeName: "float", MethodName: "AsFloat", AppropriateKind: ipld.KindSet_JustBytes}
113+
}
114+
115+
func (d *deferred) AsString() (string, error) {
116+
return "", ipld.ErrWrongKind{TypeName: "string", MethodName: "AsString", AppropriateKind: ipld.KindSet_JustBytes}
117+
}
118+
119+
func (d *deferred) AsLink() (ipld.Link, error) {
120+
return nil, ipld.ErrWrongKind{TypeName: "link", MethodName: "AsLink", AppropriateKind: ipld.KindSet_JustBytes}
121+
}
122+
123+
func (d *deferred) AsNode() (ipld.Node, error) {
124+
return nil, nil
125+
}
126+
127+
func (d *deferred) Size() int {
128+
return 0
129+
}
130+
131+
func (d *deferred) IsAbsent() bool {
132+
return false
133+
}
134+
135+
func (d *deferred) IsNull() bool {
136+
if err := d.deferredFileNode.resolve(); err != nil {
137+
return true
138+
}
139+
return d.deferredFileNode.IsNull()
140+
}
141+
142+
func (d *deferred) Length() int64 {
143+
return 0
144+
}
145+
146+
func (d *deferred) ListIterator() ipld.ListIterator {
147+
return nil
148+
}
149+
150+
func (d *deferred) MapIterator() ipld.MapIterator {
151+
return nil
152+
}
153+
154+
func (d *deferred) LookupByIndex(idx int64) (ipld.Node, error) {
155+
return nil, ipld.ErrWrongKind{}
156+
}
157+
158+
func (d *deferred) LookupByString(key string) (ipld.Node, error) {
159+
return nil, ipld.ErrWrongKind{}
160+
}
161+
162+
func (d *deferred) LookupByNode(key ipld.Node) (ipld.Node, error) {
163+
return nil, ipld.ErrWrongKind{}
164+
}
165+
166+
func (d *deferred) LookupBySegment(seg ipld.PathSegment) (ipld.Node, error) {
167+
return nil, ipld.ErrWrongKind{}
168+
}
169+
170+
// shardded files / nodes look like dagpb nodes.
171+
func (d *deferred) Prototype() ipld.NodePrototype {
172+
return dagpb.Type.PBNode
173+
}

‎unixfs/node/file/file.go

+35-8
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@ import (
1111
// root of a unixfs File.
1212
// It provides a `bytes` view over the file, along with access to io.Reader streaming access
1313
// to file data.
14-
func NewUnixFSFile(ctx context.Context, substrate ipld.Node, lsys *ipld.LinkSystem) (StreamableByteNode, error) {
14+
func NewUnixFSFile(ctx context.Context, substrate ipld.Node, lsys *ipld.LinkSystem) (LargeBytesNode, error) {
1515
if substrate.Kind() == ipld.Kind_Bytes {
1616
// A raw / single-node file.
17-
return &singleNodeFile{substrate, 0}, nil
17+
return &singleNodeFile{substrate}, nil
1818
}
1919
// see if it's got children.
2020
links, err := substrate.LookupByString("Links")
@@ -30,22 +30,29 @@ func NewUnixFSFile(ctx context.Context, substrate ipld.Node, lsys *ipld.LinkSyst
3030
ctx: ctx,
3131
lsys: lsys,
3232
substrate: substrate,
33-
done: false,
34-
rdr: nil}, nil
33+
}, nil
3534
}
3635

37-
// A StreamableByteNode is an ipld.Node that can be streamed over. It is guaranteed to have a Bytes type.
38-
type StreamableByteNode interface {
36+
// A LargeBytesNode is an ipld.Node that can be streamed over. It is guaranteed to have a Bytes type.
37+
type LargeBytesNode interface {
3938
ipld.Node
40-
io.Reader
39+
AsLargeBytes() (io.ReadSeeker, error)
4140
}
4241

4342
type singleNodeFile struct {
4443
ipld.Node
44+
}
45+
46+
func (f *singleNodeFile) AsLargeBytes() (io.ReadSeeker, error) {
47+
return &singleNodeReader{f, 0}, nil
48+
}
49+
50+
type singleNodeReader struct {
51+
ipld.Node
4552
offset int
4653
}
4754

48-
func (f *singleNodeFile) Read(p []byte) (int, error) {
55+
func (f *singleNodeReader) Read(p []byte) (int, error) {
4956
buf, err := f.Node.AsBytes()
5057
if err != nil {
5158
return 0, err
@@ -57,3 +64,23 @@ func (f *singleNodeFile) Read(p []byte) (int, error) {
5764
f.offset += n
5865
return n, nil
5966
}
67+
68+
func (f *singleNodeReader) Seek(offset int64, whence int) (int64, error) {
69+
buf, err := f.Node.AsBytes()
70+
if err != nil {
71+
return 0, err
72+
}
73+
74+
switch whence {
75+
case io.SeekStart:
76+
f.offset = int(offset)
77+
case io.SeekCurrent:
78+
f.offset += int(offset)
79+
case io.SeekEnd:
80+
f.offset = len(buf) + int(offset)
81+
}
82+
if f.offset < 0 {
83+
return 0, io.EOF
84+
}
85+
return int64(f.offset), nil
86+
}

0 commit comments

Comments
 (0)