Skip to content

Commit adf4357

Browse files
butonicbutonic
committed
synctime based etag propagation
Signed-off-by: Jörn Friedrich Dreyer <[email protected]>
1 parent fd2b257 commit adf4357

File tree

5 files changed

+220
-57
lines changed

5 files changed

+220
-57
lines changed

‎pkg/storage/fs/ocis/node.go

+45-23
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,13 @@ package ocis
2020

2121
import (
2222
"context"
23+
"crypto/md5"
2324
"fmt"
25+
"io"
2426
"os"
2527
"path/filepath"
2628
"strings"
29+
"time"
2730

2831
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
2932
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
@@ -47,17 +50,17 @@ type Node struct {
4750

4851
func (n *Node) writeMetadata(owner *userpb.UserId) (err error) {
4952
nodePath := filepath.Join(n.pw.Root, "nodes", n.ID)
50-
if err = xattr.Set(nodePath, "user.ocis.parentid", []byte(n.ParentID)); err != nil {
53+
if err = xattr.Set(nodePath, parentidAttr, []byte(n.ParentID)); err != nil {
5154
return errors.Wrap(err, "ocisfs: could not set parentid attribute")
5255
}
53-
if err = xattr.Set(nodePath, "user.ocis.name", []byte(n.Name)); err != nil {
56+
if err = xattr.Set(nodePath, nameAttr, []byte(n.Name)); err != nil {
5457
return errors.Wrap(err, "ocisfs: could not set name attribute")
5558
}
5659
if owner != nil {
57-
if err = xattr.Set(nodePath, "user.ocis.owner.id", []byte(owner.OpaqueId)); err != nil {
60+
if err = xattr.Set(nodePath, ownerIDAttr, []byte(owner.OpaqueId)); err != nil {
5861
return errors.Wrap(err, "ocisfs: could not set owner id attribute")
5962
}
60-
if err = xattr.Set(nodePath, "user.ocis.owner.idp", []byte(owner.Idp)); err != nil {
63+
if err = xattr.Set(nodePath, ownerIDPAttr, []byte(owner.Idp)); err != nil {
6164
return errors.Wrap(err, "ocisfs: could not set owner idp attribute")
6265
}
6366
}
@@ -75,13 +78,13 @@ func ReadNode(ctx context.Context, pw *Path, id string) (n *Node, err error) {
7578

7679
// lookup parent id in extended attributes
7780
var attrBytes []byte
78-
if attrBytes, err = xattr.Get(nodePath, "user.ocis.parentid"); err == nil {
81+
if attrBytes, err = xattr.Get(nodePath, parentidAttr); err == nil {
7982
n.ParentID = string(attrBytes)
8083
} else {
8184
return
8285
}
8386
// lookup name in extended attributes
84-
if attrBytes, err = xattr.Get(nodePath, "user.ocis.name"); err == nil {
87+
if attrBytes, err = xattr.Get(nodePath, nameAttr); err == nil {
8588
n.Name = string(attrBytes)
8689
} else {
8790
return
@@ -99,7 +102,7 @@ func ReadNode(ctx context.Context, pw *Path, id string) (n *Node, err error) {
99102
// walk to root to check node is not part of a deleted subtree
100103
parentPath := filepath.Join(n.pw.Root, "nodes", parentID)
101104

102-
if attrBytes, err = xattr.Get(parentPath, "user.ocis.parentid"); err == nil {
105+
if attrBytes, err = xattr.Get(parentPath, parentidAttr); err == nil {
103106
parentID = string(attrBytes)
104107
log.Debug().Interface("node", n).Str("root.ID", root.ID).Str("parentID", parentID).Msg("ReadNode() found parent")
105108
} else {
@@ -156,13 +159,13 @@ func (n *Node) Parent() (p *Node, err error) {
156159

157160
// lookup parent id in extended attributes
158161
var attrBytes []byte
159-
if attrBytes, err = xattr.Get(parentPath, "user.ocis.parentid"); err == nil {
162+
if attrBytes, err = xattr.Get(parentPath, parentidAttr); err == nil {
160163
p.ParentID = string(attrBytes)
161164
} else {
162165
return
163166
}
164167
// lookup name in extended attributes
165-
if attrBytes, err = xattr.Get(parentPath, "user.ocis.name"); err == nil {
168+
if attrBytes, err = xattr.Get(parentPath, nameAttr); err == nil {
166169
p.Name = string(attrBytes)
167170
} else {
168171
return
@@ -186,13 +189,13 @@ func (n *Node) Owner() (id string, idp string, err error) {
186189
// lookup parent id in extended attributes
187190
var attrBytes []byte
188191
// lookup name in extended attributes
189-
if attrBytes, err = xattr.Get(nodePath, "user.ocis.owner.id"); err == nil {
192+
if attrBytes, err = xattr.Get(nodePath, ownerIDAttr); err == nil {
190193
n.ownerID = string(attrBytes)
191194
} else {
192195
return
193196
}
194197
// lookup name in extended attributes
195-
if attrBytes, err = xattr.Get(nodePath, "user.ocis.owner.idp"); err == nil {
198+
if attrBytes, err = xattr.Get(nodePath, ownerIDPAttr); err == nil {
196199
n.ownerIDP = string(attrBytes)
197200
} else {
198201
return
@@ -230,32 +233,22 @@ func (n *Node) AsResourceInfo(ctx context.Context) (ri *provider.ResourceInfo, e
230233
// nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE
231234
}
232235

233-
var etag []byte
234-
// TODO optionally store etag in new `root/attributes/<uuid>` file
235-
if etag, err = xattr.Get(nodePath, "user.ocis.etag"); err != nil {
236-
log.Error().Err(err).Interface("node", n).Msg("could not read etag")
237-
}
238-
239236
id := &provider.ResourceId{OpaqueId: n.ID}
240237

241238
fn, err = n.pw.Path(ctx, n)
242239
if err != nil {
243240
return nil, err
244241
}
242+
245243
ri = &provider.ResourceInfo{
246244
Id: id,
247245
Path: fn,
248246
Type: nodeType,
249-
Etag: string(etag),
250247
MimeType: mime.Detect(nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER, fn),
251248
Size: uint64(fi.Size()),
252249
// TODO fix permissions
253250
PermissionSet: &provider.ResourcePermissions{ListContainer: true, CreateContainer: true},
254-
Mtime: &types.Timestamp{
255-
Seconds: uint64(fi.ModTime().Unix()),
256-
// TODO read nanos from where? Nanos: fi.MTimeNanos,
257-
},
258-
Target: string(target),
251+
Target: string(target),
259252
}
260253

261254
if owner, idp, err := n.Owner(); err == nil {
@@ -265,6 +258,35 @@ func (n *Node) AsResourceInfo(ctx context.Context) (ri *provider.ResourceInfo, e
265258
}
266259
}
267260

261+
// etag currently is a hash of fileid + stime (or mtime)
262+
// TODO make etag of files use fileid and checksum
263+
// TODO optionally store etag in an attribute to restore backups
264+
h := md5.New()
265+
io.WriteString(h, n.ID)
266+
var b []byte
267+
var stime time.Time
268+
if b, err = xattr.Get(nodePath, stimeAttr); err == nil {
269+
if stime, err = time.Parse(time.RFC3339Nano, string(b)); err != nil {
270+
// invalid format, overwrite
271+
log.Error().Err(err).Interface("node", n).Str("stime", string(b)).Msg("invalid format, ignoring")
272+
stime = fi.ModTime()
273+
}
274+
} else {
275+
// no stime, use mtime
276+
stime = fi.ModTime()
277+
}
278+
tb, _ := stime.UTC().MarshalBinary()
279+
h.Write(tb)
280+
ri.Etag = fmt.Sprintf("%x", h.Sum(nil))
281+
282+
// mtime uses stime if present
283+
// TODO expose mtime and stime seperately?
284+
un := stime.UnixNano()
285+
ri.Mtime = &types.Timestamp{
286+
Seconds: uint64(un / 1000000000),
287+
Nanos: uint32(un % 1000000000),
288+
}
289+
268290
// TODO only read the requested metadata attributes
269291
if attrs, err := xattr.List(nodePath); err == nil {
270292
ri.ArbitraryMetadata = &provider.ArbitraryMetadata{

‎pkg/storage/fs/ocis/ocis.go

+53-11
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"strings"
2828

2929
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
30+
"github.com/cs3org/reva/pkg/appctx"
3031
"github.com/cs3org/reva/pkg/errtypes"
3132
"github.com/cs3org/reva/pkg/logger"
3233
"github.com/cs3org/reva/pkg/storage"
@@ -47,16 +48,38 @@ const (
4748
// collisions with other apps We are going to introduce a sub namespace
4849
// "user.ocis."
4950

51+
ocisPrefix string = "user.ocis."
52+
parentidAttr string = ocisPrefix + "parentid"
53+
ownerIDAttr string = ocisPrefix + "owner.id"
54+
ownerIDPAttr string = ocisPrefix + "owner.idp"
55+
// the base name of the node
56+
// updated when the file is renamed or moved
57+
nameAttr string = ocisPrefix + "name"
58+
5059
// SharePrefix is the prefix for sharing related extended attributes
51-
sharePrefix string = "user.ocis.acl."
52-
metadataPrefix string = "user.ocis.md."
60+
sharePrefix string = ocisPrefix + "acl."
61+
metadataPrefix string = ocisPrefix + "md."
5362
// TODO implement favorites metadata flag
54-
//favPrefix string = "user.ocis.fav." // favorite flag, per user
63+
//favPrefix string = ocisPrefix + "fav." // favorite flag, per user
5564
// TODO use etag prefix instead of single etag property
56-
//etagPrefix string = "user.ocis.etag." // allow overriding a calculated etag with one from the extended attributes
57-
referenceAttr string = "user.ocis.cs3.ref" // arbitrary metadata
58-
//checksumPrefix string = "user.ocis.cs." // TODO add checksum support
59-
trashOriginAttr string = "user.ocis.trash.origin" // trash origin
65+
//etagPrefix string = ocisPrefix + "etag." // allow overriding a calculated etag with one from the extended attributes
66+
referenceAttr string = ocisPrefix + "cs3.ref" // arbitrary metadata
67+
//checksumPrefix string = ocisPrefix + "cs." // TODO add checksum support
68+
trashOriginAttr string = ocisPrefix + "trash.origin" // trash origin
69+
70+
// we use a single attribute to enable or disable propagation of both: synctime and treesize
71+
propagationAttr string = ocisPrefix + "propagation"
72+
73+
// the sync time of the tree below this node,
74+
// propagated when synctime_accounting is true and
75+
// user.ocis.propagation=1 is set
76+
// stored as a readable time.RFC3339Nano
77+
stimeAttr string = ocisPrefix + "stime"
78+
79+
// the size of the tree below this node,
80+
// propagated when treesize_accounting is true and
81+
// user.ocis.propagation=1 is set
82+
treesizeAttr string = ocisPrefix + "treesize"
6083
)
6184

6285
func init() {
@@ -149,18 +172,27 @@ func (fs *ocisfs) CreateHome(ctx context.Context) (err error) {
149172
return errtypes.NotSupported("ocisfs: CreateHome() home supported disabled")
150173
}
151174

152-
var n *Node
175+
var n, h *Node
153176
if n, err = fs.pw.RootNode(ctx); err != nil {
154177
return
155178
}
156-
_, err = fs.pw.WalkPath(ctx, n, fs.pw.mustGetUserLayout(ctx), func(ctx context.Context, n *Node) error {
179+
h, err = fs.pw.WalkPath(ctx, n, fs.pw.mustGetUserLayout(ctx), func(ctx context.Context, n *Node) error {
157180
if !n.Exists {
158181
if err := fs.tp.CreateDir(ctx, n); err != nil {
159182
return err
160183
}
161184
}
162185
return nil
163186
})
187+
188+
if fs.pw.SyncTimeAcconting {
189+
homePath := filepath.Join(fs.pw.Root, "nodes", h.ID)
190+
// mark the home node as the end of propagation
191+
if err = xattr.Set(homePath, propagationAttr, []byte("0")); err != nil {
192+
appctx.GetLogger(ctx).Error().Err(err).Interface("node", h).Msg("could not mark home as propagation root")
193+
return
194+
}
195+
}
164196
return
165197
}
166198

@@ -190,7 +222,17 @@ func (fs *ocisfs) CreateDir(ctx context.Context, fn string) (err error) {
190222
if node.Exists {
191223
return errtypes.AlreadyExists(fn)
192224
}
193-
return fs.tp.CreateDir(ctx, node)
225+
err = fs.tp.CreateDir(ctx, node)
226+
227+
if fs.pw.SyncTimeAcconting {
228+
nodePath := filepath.Join(fs.pw.Root, "nodes", node.ID)
229+
// mark the home node as the end of propagation
230+
if err = xattr.Set(nodePath, propagationAttr, []byte("1")); err != nil {
231+
appctx.GetLogger(ctx).Error().Err(err).Interface("node", node).Msg("could not mark node to propagate")
232+
return
233+
}
234+
}
235+
return
194236
}
195237

196238
// CreateReference creates a reference as a node folder with the target stored in extended attributes
@@ -347,7 +389,7 @@ func (fs *ocisfs) copyMD(s string, t string) (err error) {
347389
return err
348390
}
349391
for i := range attrs {
350-
if strings.HasPrefix(attrs[i], "user.ocis.") {
392+
if strings.HasPrefix(attrs[i], ocisPrefix) {
351393
var d []byte
352394
if d, err = xattr.Get(s, attrs[i]); err != nil {
353395
return err

‎pkg/storage/fs/ocis/path.go

+6
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,12 @@ type Path struct {
4444
// EnableHome enables the creation of home directories.
4545
EnableHome bool `mapstructure:"enable_home"`
4646
ShareFolder string `mapstructure:"share_folder"`
47+
48+
// propagate mtime changes as stime (sync time) to the parent directory when user.ocis.mtime.propagation=1 is set on a node
49+
SyncTimeAcconting bool `mapstructure:"synctime_accounting"`
50+
51+
// propagate size changes as treesize
52+
TreeSizeAccounting bool `mapstructure:"treesize_accounting"`
4753
}
4854

4955
// NodeFromResource takes in a request path or request id and converts it to a Node

0 commit comments

Comments
 (0)