Skip to content

Commit cbd13b6

Browse files
whyrusleepingwhyrusleeping
authored
Merge pull request #3575 from ipfs/feat/datastore-configs
more configurable datastore configs
2 parents c920033 + 2c30002 commit cbd13b6

File tree

12 files changed

+744
-124
lines changed

12 files changed

+744
-124
lines changed

‎repo/config/datastore.go

+8-18
Original file line numberDiff line numberDiff line change
@@ -9,30 +9,20 @@ const DefaultDataStoreDirectory = "datastore"
99

1010
// Datastore tracks the configuration of the datastore.
1111
type Datastore struct {
12-
Type string
13-
Path string
1412
StorageMax string // in B, kB, kiB, MB, ...
1513
StorageGCWatermark int64 // in percentage to multiply on StorageMax
1614
GCPeriod string // in ns, us, ms, s, m, h
1715

18-
Params *json.RawMessage
19-
NoSync bool
20-
HashOnRead bool
21-
BloomFilterSize int
22-
}
16+
// deprecated fields, use Spec
17+
Type string `json:",omitempty"`
18+
Path string `json:",omitempty"`
19+
NoSync bool `json:",omitempty"`
20+
Params *json.RawMessage `json:",omitempty"`
2321

24-
func (d *Datastore) ParamData() []byte {
25-
if d.Params == nil {
26-
return nil
27-
}
22+
Spec map[string]interface{}
2823

29-
return []byte(*d.Params)
30-
}
31-
32-
type S3Datastore struct {
33-
Region string `json:"region"`
34-
Bucket string `json:"bucket"`
35-
ACL string `json:"acl"`
24+
HashOnRead bool
25+
BloomFilterSize int
3626
}
3727

3828
// DataStorePath returns the default data store path given a configuration root

‎repo/config/init.go

+30-13
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,7 @@ func Init(out io.Writer, nBitsForKeypair int) (*Config, error) {
2121
return nil, err
2222
}
2323

24-
datastore, err := datastoreConfig()
25-
if err != nil {
26-
return nil, err
27-
}
24+
datastore := DefaultDatastoreConfig()
2825

2926
conf := &Config{
3027

@@ -79,20 +76,40 @@ func Init(out io.Writer, nBitsForKeypair int) (*Config, error) {
7976
return conf, nil
8077
}
8178

82-
func datastoreConfig() (Datastore, error) {
83-
dspath, err := DataStorePath("")
84-
if err != nil {
85-
return Datastore{}, err
86-
}
79+
// DefaultDatastoreConfig is an internal function exported to aid in testing.
80+
func DefaultDatastoreConfig() Datastore {
8781
return Datastore{
88-
Path: dspath,
89-
Type: "leveldb",
9082
StorageMax: "10GB",
9183
StorageGCWatermark: 90, // 90%
9284
GCPeriod: "1h",
93-
HashOnRead: false,
9485
BloomFilterSize: 0,
95-
}, nil
86+
Spec: map[string]interface{}{
87+
"type": "mount",
88+
"mounts": []interface{}{
89+
map[string]interface{}{
90+
"mountpoint": "/blocks",
91+
"type": "measure",
92+
"prefix": "flatfs.datastore",
93+
"child": map[string]interface{}{
94+
"type": "flatfs",
95+
"path": "blocks",
96+
"sync": true,
97+
"shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
98+
},
99+
},
100+
map[string]interface{}{
101+
"mountpoint": "/",
102+
"type": "measure",
103+
"prefix": "leveldb.datastore",
104+
"child": map[string]interface{}{
105+
"type": "levelds",
106+
"path": "datastore",
107+
"compression": "none",
108+
},
109+
},
110+
},
111+
},
112+
}
96113
}
97114

98115
// identityConfig initializes a new identity.

‎repo/fsrepo/config_test.go

+219
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
package fsrepo
2+
3+
import (
4+
"encoding/json"
5+
"io/ioutil"
6+
"os"
7+
"reflect"
8+
"testing"
9+
10+
config "github.com/ipfs/go-ipfs/repo/config"
11+
)
12+
13+
// note: to test sorting of the mountpoints in the disk spec they are
14+
// specified out of order in the test config
15+
var defaultConfig = []byte(`{
16+
"StorageMax": "10GB",
17+
"StorageGCWatermark": 90,
18+
"GCPeriod": "1h",
19+
"Spec": {
20+
"mounts": [
21+
{
22+
"child": {
23+
"compression": "none",
24+
"path": "datastore",
25+
"type": "levelds"
26+
},
27+
"mountpoint": "/",
28+
"prefix": "leveldb.datastore",
29+
"type": "measure"
30+
},
31+
{
32+
"child": {
33+
"path": "blocks",
34+
"shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
35+
"sync": true,
36+
"type": "flatfs"
37+
},
38+
"mountpoint": "/blocks",
39+
"prefix": "flatfs.datastore",
40+
"type": "measure"
41+
}
42+
],
43+
"type": "mount"
44+
},
45+
"HashOnRead": false,
46+
"BloomFilterSize": 0
47+
}`)
48+
49+
var leveldbConfig = []byte(`{
50+
"compression": "none",
51+
"path": "datastore",
52+
"type": "levelds"
53+
}`)
54+
55+
var flatfsConfig = []byte(`{
56+
"path": "blocks",
57+
"shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
58+
"sync": true,
59+
"type": "flatfs"
60+
}`)
61+
62+
var measureConfig = []byte(`{
63+
"child": {
64+
"path": "blocks",
65+
"shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
66+
"sync": true,
67+
"type": "flatfs"
68+
},
69+
"mountpoint": "/blocks",
70+
"prefix": "flatfs.datastore",
71+
"type": "measure"
72+
}`)
73+
74+
func TestDefaultDatastoreConfig(t *testing.T) {
75+
dir, err := ioutil.TempDir("", "ipfs-datastore-config-test")
76+
if err != nil {
77+
t.Fatal(err)
78+
}
79+
defer os.RemoveAll(dir) // clean up
80+
81+
config := new(config.Datastore)
82+
err = json.Unmarshal(defaultConfig, config)
83+
if err != nil {
84+
t.Fatal(err)
85+
}
86+
87+
dsc, err := AnyDatastoreConfig(config.Spec)
88+
if err != nil {
89+
t.Fatal(err)
90+
}
91+
92+
expected := `{"mounts":[{"mountpoint":"/blocks","path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"},{"mountpoint":"/","path":"datastore","type":"levelds"}],"type":"mount"}`
93+
if dsc.DiskSpec().String() != expected {
94+
t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String())
95+
}
96+
97+
ds, err := dsc.Create(dir)
98+
if err != nil {
99+
t.Fatal(err)
100+
}
101+
102+
if typ := reflect.TypeOf(ds).String(); typ != "*syncmount.Datastore" {
103+
t.Errorf("expected '*syncmount.Datastore' got '%s'", typ)
104+
}
105+
}
106+
107+
func TestLevelDbConfig(t *testing.T) {
108+
config := new(config.Datastore)
109+
err := json.Unmarshal(defaultConfig, config)
110+
if err != nil {
111+
t.Fatal(err)
112+
}
113+
dir, err := ioutil.TempDir("", "ipfs-datastore-config-test")
114+
if err != nil {
115+
t.Fatal(err)
116+
}
117+
defer os.RemoveAll(dir) // clean up
118+
119+
spec := make(map[string]interface{})
120+
err = json.Unmarshal(leveldbConfig, &spec)
121+
if err != nil {
122+
t.Fatal(err)
123+
}
124+
125+
dsc, err := AnyDatastoreConfig(spec)
126+
if err != nil {
127+
t.Fatal(err)
128+
}
129+
130+
expected := `{"path":"datastore","type":"levelds"}`
131+
if dsc.DiskSpec().String() != expected {
132+
t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String())
133+
}
134+
135+
ds, err := dsc.Create(dir)
136+
if err != nil {
137+
t.Fatal(err)
138+
}
139+
140+
if typ := reflect.TypeOf(ds).String(); typ != "*leveldb.datastore" {
141+
t.Errorf("expected '*leveldb.datastore' got '%s'", typ)
142+
}
143+
}
144+
145+
func TestFlatfsConfig(t *testing.T) {
146+
config := new(config.Datastore)
147+
err := json.Unmarshal(defaultConfig, config)
148+
if err != nil {
149+
t.Fatal(err)
150+
}
151+
dir, err := ioutil.TempDir("", "ipfs-datastore-config-test")
152+
if err != nil {
153+
t.Fatal(err)
154+
}
155+
defer os.RemoveAll(dir) // clean up
156+
157+
spec := make(map[string]interface{})
158+
err = json.Unmarshal(flatfsConfig, &spec)
159+
if err != nil {
160+
t.Fatal(err)
161+
}
162+
163+
dsc, err := AnyDatastoreConfig(spec)
164+
if err != nil {
165+
t.Fatal(err)
166+
}
167+
168+
expected := `{"path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"}`
169+
if dsc.DiskSpec().String() != expected {
170+
t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String())
171+
}
172+
173+
ds, err := dsc.Create(dir)
174+
if err != nil {
175+
t.Fatal(err)
176+
}
177+
178+
if typ := reflect.TypeOf(ds).String(); typ != "*flatfs.Datastore" {
179+
t.Errorf("expected '*flatfs.Datastore' got '%s'", typ)
180+
}
181+
}
182+
183+
func TestMeasureConfig(t *testing.T) {
184+
config := new(config.Datastore)
185+
err := json.Unmarshal(defaultConfig, config)
186+
if err != nil {
187+
t.Fatal(err)
188+
}
189+
dir, err := ioutil.TempDir("", "ipfs-datastore-config-test")
190+
if err != nil {
191+
t.Fatal(err)
192+
}
193+
defer os.RemoveAll(dir) // clean up
194+
195+
spec := make(map[string]interface{})
196+
err = json.Unmarshal(measureConfig, &spec)
197+
if err != nil {
198+
t.Fatal(err)
199+
}
200+
201+
dsc, err := AnyDatastoreConfig(spec)
202+
if err != nil {
203+
t.Fatal(err)
204+
}
205+
206+
expected := `{"path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"}`
207+
if dsc.DiskSpec().String() != expected {
208+
t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String())
209+
}
210+
211+
ds, err := dsc.Create(dir)
212+
if err != nil {
213+
t.Fatal(err)
214+
}
215+
216+
if typ := reflect.TypeOf(ds).String(); typ != "*measure.measure" {
217+
t.Errorf("expected '*measure.measure' got '%s'", typ)
218+
}
219+
}

0 commit comments

Comments
 (0)