[WIP] Add a SharesStorageProvider and an oc10 sql share manager

docs/extensions/storage/ports.md

@@ -38,6 +38,8 @@ For now, the storage service uses these ports to preconfigure those services:
 | 9165      | storage appprovider debug                             |
 | 9178      | storage public link                           |
 | 9179      | storage public link data                      |
+| 9182      | storage shares                           |
+| 9183      | storage shares data                      |
 | 9215      | storage meta grpc                             |
 | 9216      | storage meta http                             |
 | 9217      | storage meta debug                            |

go.mod

@@ -214,7 +214,6 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/steveyen/gtreap v0.1.0 // indirect
-	github.com/studio-b12/gowebdav v0.0.0-20210917133250-a3a86976a1df // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/tus/tusd v1.6.0 // indirect
 	github.com/wk8/go-ordered-map v0.2.0 // indirect
@@ -242,5 +241,7 @@ require (
 	stash.kopano.io/kgol/oidc-go v0.3.1 // indirect
 )
 
+replace github.com/cs3org/reva => ../reva
+
 // this is a transitive replace. See https://github.com/libregraph/lico/blob/master/go.mod#L38
 replace github.com/crewjam/saml => github.com/crewjam/saml v0.4.5

go.sum

@@ -291,8 +291,6 @@ github.com/crewjam/saml v0.4.5/go.mod h1:qCJQpUtZte9R1ZjUBcW8qtCNlinbO363ooNl02S
 github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4=
 github.com/cs3org/go-cs3apis v0.0.0-20211007101428-6d142794ec11 h1:cc/8fdzWdr/wAZOXb29J8bnXjo1poCMCLwhlFBlvhfI=
 github.com/cs3org/go-cs3apis v0.0.0-20211007101428-6d142794ec11/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
-github.com/cs3org/reva v1.13.1-0.20211012070754-5dbb87c8e084 h1:7uwU6ui+9cRdsyb8PaflexI2TY51LVYiDNOzBLDgHQs=
-github.com/cs3org/reva v1.13.1-0.20211012070754-5dbb87c8e084/go.mod h1:uENdZEtDFmTRt6+d4+Ro4P5XnNL+9I6gwftHEBJzHQw=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
@@ -1073,7 +1071,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/studio-b12/gowebdav v0.0.0-20210917133250-a3a86976a1df h1:C+J/LwTqP8gRPt1MdSzBNZP0OYuDm5wsmDKgwpLjYzo=
 github.com/studio-b12/gowebdav v0.0.0-20210917133250-a3a86976a1df/go.mod h1:gCcfDlA1Y7GqOaeEKw5l9dOGx1VLdc/HuQSlQAaZ30s=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=

ocis/pkg/runtime/service/service.go

@@ -109,6 +109,7 @@ func NewService(options ...Option) (*Service, error) {
 	s.ServicesRegistry["storage-users"] = storage.NewStorageUsers
 	s.ServicesRegistry["storage-public-link"] = storage.NewStoragePublicLink
 	s.ServicesRegistry["storage-appprovider"] = storage.NewAppProvider
+	s.ServicesRegistry["storage-shares"] = storage.NewStorageShares
 
 	// populate delayed services
 	s.Delayed["storage-sharing"] = storage.NewSharing

storage/pkg/command/gateway.go

@@ -115,6 +115,7 @@ func Gateway(cfg *config.Config) *cli.Command {
 
 // gatewayConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
 func gatewayConfigFromStruct(c *cli.Context, cfg *config.Config, logger log.Logger) map[string]interface{} {
+	storageRules := rules(cfg, logger)
 	rcfg := map[string]interface{}{
 		"core": map[string]interface{}{
 			"max_cpus":             cfg.Reva.Users.MaxCPUs,
@@ -133,6 +134,7 @@ func gatewayConfigFromStruct(c *cli.Context, cfg *config.Config, logger log.Logg
 			// TODO build services dynamically
 			"services": map[string]interface{}{
 				"gateway": map[string]interface{}{
+					"storage_rules": storageRules,
 					// registries is located on the gateway
 					"authregistrysvc":    cfg.Reva.Gateway.Endpoint,
 					"storageregistrysvc": cfg.Reva.Gateway.Endpoint,
@@ -181,7 +183,7 @@ func gatewayConfigFromStruct(c *cli.Context, cfg *config.Config, logger log.Logg
 					"drivers": map[string]interface{}{
 						"static": map[string]interface{}{
 							"home_provider": cfg.Reva.StorageRegistry.HomeProvider,
-							"rules":         rules(cfg, logger),
+							"rules":         storageRules,
 						},
 					},
 				},
@@ -220,11 +222,12 @@ func rules(cfg *config.Config, logger log.Logger) map[string]map[string]interfac
 
 	// generate rules based on default config
 	return map[string]map[string]interface{}{
-		cfg.Reva.StorageHome.MountPath:       {"address": cfg.Reva.StorageHome.Endpoint},
-		cfg.Reva.StorageHome.MountID:         {"address": cfg.Reva.StorageHome.Endpoint},
-		cfg.Reva.StorageUsers.MountPath:      {"address": cfg.Reva.StorageUsers.Endpoint},
-		cfg.Reva.StorageUsers.MountID + ".*": {"address": cfg.Reva.StorageUsers.Endpoint},
-		cfg.Reva.StoragePublicLink.MountPath: {"address": cfg.Reva.StoragePublicLink.Endpoint},
+		cfg.Reva.StorageShares.MountPath:         {"address": cfg.Reva.StorageShares.Endpoint},
+		cfg.Reva.StorageHome.MountPath:           {"address": cfg.Reva.StorageHome.Endpoint},
+		cfg.Reva.StorageHome.MountID:             {"address": cfg.Reva.StorageHome.Endpoint},
+		cfg.Reva.StorageUsers.MountPath:          {"address": cfg.Reva.StorageUsers.Endpoint},
+		cfg.Reva.StorageUsers.MountID + "\\!?.*": {"address": cfg.Reva.StorageUsers.Endpoint},
+		cfg.Reva.StoragePublicLink.MountPath:     {"address": cfg.Reva.StoragePublicLink.Endpoint},
 		// public link storage returns the mount id of the actual storage
 		// medatada storage not part of the global namespace
 	}

storage/pkg/command/root.go

@@ -83,6 +83,7 @@ func Execute(cfg *config.Config) error {
 			StorageHome(cfg),
 			StorageUsers(cfg),
 			StoragePublicLink(cfg),
+			StorageShares(cfg),
 			StorageMetadata(cfg),
 			Health(cfg),
 		},

storage/pkg/command/sharing.go

@@ -138,7 +138,7 @@ func sharingConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]inte
 						},
 						"oc10-sql": map[string]interface{}{
 							"gateway_addr":     cfg.Reva.Gateway.Endpoint,
-							"storage_mount_id": cfg.Reva.Sharing.UserStorageMountID,
+							"storage_mount_id": cfg.Reva.Sharing.UserStorageMountId,
 							"db_username":      cfg.Reva.Sharing.UserSQLUsername,
 							"db_password":      cfg.Reva.Sharing.UserSQLPassword,
 							"db_host":          cfg.Reva.Sharing.UserSQLHost,

storage/pkg/command/storageshares.go

@@ -0,0 +1,146 @@
+package command
+
+import (
+	"context"
+	"flag"
+	"os"
+	"path"
+
+	"github.com/cs3org/reva/cmd/revad/runtime"
+	"github.com/gofrs/uuid"
+	"github.com/oklog/run"
+	ociscfg "github.com/owncloud/ocis/ocis-pkg/config"
+	"github.com/owncloud/ocis/ocis-pkg/sync"
+	"github.com/owncloud/ocis/storage/pkg/config"
+	"github.com/owncloud/ocis/storage/pkg/flagset"
+	"github.com/owncloud/ocis/storage/pkg/server/debug"
+	"github.com/owncloud/ocis/storage/pkg/tracing"
+	"github.com/thejerf/suture/v4"
+	"github.com/urfave/cli/v2"
+)
+
+// StorageShares is the entrypoint for the reva-storage-share command.
+func StorageShares(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:     "storage-shares",
+		Usage:    "Start storage-shares service",
+		Flags:    flagset.StorageShares(cfg),
+		Category: "Extensions",
+		Action: func(c *cli.Context) error {
+			logger := NewLogger(cfg)
+			tracing.Configure(cfg, logger)
+			gr := run.Group{}
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			pidFile := path.Join(os.TempDir(), "revad-"+c.Command.Name+"-"+uuid.Must(uuid.NewV4()).String()+".pid")
+			rcfg := storageSharesConfigFromStruct(c, cfg)
+
+			gr.Add(func() error {
+				runtime.RunWithOptions(
+					rcfg,
+					pidFile,
+					runtime.WithLogger(&logger.Logger),
+				)
+				return nil
+			}, func(_ error) {
+				logger.Info().
+					Str("server", c.Command.Name).
+					Msg("Shutting down server")
+
+				cancel()
+			})
+
+			debugServer, err := debug.Server(
+				debug.Name(c.Command.Name+"-debug"),
+				debug.Addr(cfg.Reva.StorageShares.DebugAddr),
+				debug.Logger(logger),
+				debug.Context(ctx),
+				debug.Config(cfg),
+			)
+
+			if err != nil {
+				logger.Info().Err(err).Str("server", c.Command.Name+"-debug").Msg("Failed to initialize server")
+				return err
+			}
+
+			gr.Add(debugServer.ListenAndServe, func(_ error) {
+				cancel()
+			})
+
+			if !cfg.Reva.StorageMetadata.Supervised {
+				sync.Trap(&gr, cancel)
+			}
+
+			return gr.Run()
+		},
+	}
+}
+
+// storageSharesConfigFromStruct will adapt an oCIS config struct into a reva mapstructure to start a reva service.
+func storageSharesConfigFromStruct(c *cli.Context, cfg *config.Config) map[string]interface{} {
+	rcfg := map[string]interface{}{
+		"core": map[string]interface{}{
+			"max_cpus":             cfg.Reva.StorageShares.MaxCPUs,
+			"tracing_enabled":      cfg.Tracing.Enabled,
+			"tracing_endpoint":     cfg.Tracing.Endpoint,
+			"tracing_collector":    cfg.Tracing.Collector,
+			"tracing_service_name": c.Command.Name,
+		},
+		"shared": map[string]interface{}{
+			"jwt_secret": cfg.Reva.JWTSecret,
+			"gatewaysvc": cfg.Reva.Gateway.Endpoint,
+		},
+		"grpc": map[string]interface{}{
+			"network": cfg.Reva.StorageShares.GRPCNetwork,
+			"address": cfg.Reva.StorageShares.GRPCAddr,
+			"interceptors": map[string]interface{}{
+				"log": map[string]interface{}{},
+			},
+			"services": map[string]interface{}{
+				"sharesstorageprovider": map[string]interface{}{
+					"mount_path":           cfg.Reva.StorageShares.MountPath,
+					"gateway_addr":         cfg.Reva.Gateway.Endpoint,
+					"usershareprovidersvc": cfg.Reva.Sharing.Endpoint,
+				},
+			},
+		},
+	}
+	return rcfg
+}
+
+// StorageSharesSutureService allows for the storage-shares command to be embedded and supervised by a suture supervisor tree.
+type StorageSharesSutureService struct {
+	cfg *config.Config
+}
+
+// NewStorageSharesSutureService creates a new storage.StorageSharesSutureService
+func NewStorageShares(cfg *ociscfg.Config) suture.Service {
+	if cfg.Mode == 0 {
+		cfg.Storage.Reva.StorageShares.Supervised = true
+	}
+	return StorageSharesSutureService{
+		cfg: cfg.Storage,
+	}
+}
+
+func (s StorageSharesSutureService) Serve(ctx context.Context) error {
+	s.cfg.Reva.StorageShares.Context = ctx
+	f := &flag.FlagSet{}
+	for k := range StorageShares(s.cfg).Flags {
+		if err := StorageShares(s.cfg).Flags[k].Apply(f); err != nil {
+			return err
+		}
+	}
+	cliCtx := cli.NewContext(nil, f, nil)
+	if StorageShares(s.cfg).Before != nil {
+		if err := StorageShares(s.cfg).Before(cliCtx); err != nil {
+			return err
+		}
+	}
+	if err := StorageShares(s.cfg).Action(cliCtx); err != nil {
+		return err
+	}
+
+	return nil
+}

storage/pkg/config/config.go

@@ -73,6 +73,7 @@ type Sharing struct {
 	Port
 	UserDriver                       string
 	UserJSONFile                     string
+	UserStorageMountId               string
 	UserSQLUsername                  string
 	UserSQLPassword                  string
 	UserSQLHost                      string
@@ -200,6 +201,11 @@ type PublicStorage struct {
 	UserProviderAddr        string
 }
 
+// PublicShares configures a shares storage provider
+type SharesStorage struct {
+	StoragePort
+}
+
 // StorageConfig combines all available storage driver configuration parts.
 type StorageConfig struct {
 	EOS         DriverEOS
@@ -457,6 +463,7 @@ type Reva struct {
 	StorageHome       StoragePort
 	StorageUsers      StoragePort
 	StoragePublicLink PublicStorage
+	StorageShares     SharesStorage
 	StorageMetadata   StoragePort
 	AppProvider       AppProvider
 	// Configs can be used to configure the reva instance.

storage/pkg/flagset/frontend.go

@@ -52,7 +52,7 @@ func FrontendWithConfig(cfg *config.Config) []cli.Flag {
 		// this can eg. be set to /eos/users
 		&cli.StringFlag{
 			Name:        "dav-files-namespace",
-			Value:       flags.OverrideDefaultString(cfg.Reva.OCDav.DavFilesNamespace, "/users/"),
+			Value:       flags.OverrideDefaultString(cfg.Reva.OCDav.DavFilesNamespace, "/users/{{.Id.OpaqueId}}"),
 			Usage:       "Namespace prefix for the webdav /dav/files endpoint",
 			EnvVars:     []string{"STORAGE_DAV_FILES_NAMESPACE"},
 			Destination: &cfg.Reva.OCDav.DavFilesNamespace,

storage/pkg/flagset/gateway.go

@@ -282,6 +282,23 @@ func GatewayWithConfig(cfg *config.Config) []cli.Flag {
 			EnvVars:     []string{"STORAGE_PUBLIC_LINK_MOUNT_PATH"},
 			Destination: &cfg.Reva.StoragePublicLink.MountPath,
 		},
+
+		// register shares storage
+
+		&cli.StringFlag{
+			Name:        "shares-endpoint",
+			Value:       flags.OverrideDefaultString(cfg.Reva.StorageShares.Endpoint, "localhost:9182"),
+			Usage:       "endpoint to use for the shares storage service",
+			EnvVars:     []string{"STORAGE_SHARES_ENDPOINT"},
+			Destination: &cfg.Reva.StorageShares.Endpoint,
+		},
+		&cli.StringFlag{
+			Name:        "storage-shares-mount-path",
+			Value:       flags.OverrideDefaultString(cfg.Reva.StorageShares.MountPath, "/home/Shares"),
+			Usage:       "mount path to use for the shares storage service",
+			EnvVars:     []string{"STORAGE_SHARES_MOUNT_PATH"},
+			Destination: &cfg.Reva.StorageShares.MountPath,
+		},
 		// public-link has no mount id
 	}
 

storage/pkg/flagset/sharing.go

@@ -53,6 +53,13 @@ func SharingWithConfig(cfg *config.Config) []cli.Flag {
 			Usage:   "--service usershareprovider [--service publicshareprovider]",
 			EnvVars: []string{"STORAGE_SHARING_SERVICES"},
 		},
+		&cli.StringFlag{
+			Name:        "gateway-endpoint",
+			Value:       flags.OverrideDefaultString(cfg.Reva.Gateway.Endpoint, "localhost:9142"),
+			Usage:       "endpoint to use for the storage gateway service",
+			EnvVars:     []string{"STORAGE_GATEWAY_ENDPOINT"},
+			Destination: &cfg.Reva.Gateway.Endpoint,
+		},
 		&cli.StringFlag{
 			Name:        "user-driver",
 			Value:       flags.OverrideDefaultString(cfg.Reva.Sharing.UserDriver, "json"),

storage/pkg/flagset/sharingsql.go

@@ -9,6 +9,13 @@ import (
 // SharingSQLWithConfig applies the Sharing SQL driver cfg to the flagset
 func SharingSQLWithConfig(cfg *config.Config) []cli.Flag {
 	return []cli.Flag{
+		&cli.StringFlag{
+			Name:        "storage-mount-id",
+			Value:       flags.OverrideDefaultString(cfg.Reva.Sharing.UserStorageMountId, "1284d238-aa92-42ce-bdc4-0b0000009157"),
+			Usage:       "mount id of the storage that is used for accessing the shares",
+			EnvVars:     []string{"STORAGE_SHARING_USER_STORAGE_MOUNT_ID"},
+			Destination: &cfg.Reva.Sharing.UserStorageMountId,
+		},
 		&cli.StringFlag{
 			Name:        "user-sql-username",
 			Value:       flags.OverrideDefaultString(cfg.Reva.Sharing.UserSQLUsername, ""),