[tests-only] provide and document an easy way to run all tests locally (#1073)

Author: individual-it

.drone.yml

@@ -249,7 +249,7 @@ steps:
     image: golang:1.13
     detach: true
     commands:
-      - cd /drone/src/drone/oc-integration-tests/
+      - cd /drone/src/tests/oc-integration-tests/drone/
       - /drone/src/cmd/revad/revad -c frontend.toml &
       - /drone/src/cmd/revad/revad -c gateway.toml &
       - /drone/src/cmd/revad/revad -c storage-home.toml &
@@ -289,7 +289,7 @@ steps:
     image: golang:1.13
     detach: true
     commands:
-      - cd /drone/src/drone/oc-integration-tests/
+      - cd /drone/src/tests/oc-integration-tests/drone/
       - /drone/src/cmd/revad/revad -c frontend.toml &
       - /drone/src/cmd/revad/revad -c gateway.toml &
       - /drone/src/cmd/revad/revad -c storage-home.toml &
@@ -330,7 +330,7 @@ steps:
     image: golang:1.13
     detach: true
     commands:
-      - cd /drone/src/drone/oc-integration-tests/
+      - cd /drone/src/tests/oc-integration-tests/drone/
       - /drone/src/cmd/revad/revad -c frontend.toml &
       - /drone/src/cmd/revad/revad -c gateway.toml &
       - /drone/src/cmd/revad/revad -c shares.toml &

README.md

@@ -31,6 +31,86 @@ $ ../../cmd/revad/revad -dev-dir .
 
 You can also read the [build from sources guide](https://reva.link/docs/getting-started/build-reva/).
 
+## Run tests
+
+### unit tests / GRPC tests
+`make test`
+
+### litmus tests
+1. start the needed services
+   ```
+   cd tests/oc-integration-tests/local
+   ../../../cmd/revad/revad -c frontend.toml &
+   ../../../cmd/revad/revad -c gateway.toml &
+   ../../../cmd/revad/revad -c storage-home.toml &
+   ../../../cmd/revad/revad -c storage-oc.toml &
+   ../../../cmd/revad/revad -c users.toml
+   ```
+
+2. run litmus tests:
+   ```
+   docker run --rm --network=host\
+       -e LITMUS_URL=http://localhost:20080/remote.php/webdav \
+       -e LITMUS_USERNAME=einstein \
+       -e LITMUS_PASSWORD=relativity \
+       owncloud/litmus:latest
+   ```
+
+   - add `TESTS` env. variable to test only a subset of tests e.g `-e TESTS="basic http copymove props"`
+   - change `LITMUS_URL` for other tests e.g. `-e LITMUS_URL=http://localhost:20080/remote.php/dav/files/einstein` or to a public-share link
+
+### ownCloud legacy integration tests
+1. start an LDAP server
+    ```
+    docker run --rm --hostname ldap.my-company.com \
+        -e LDAP_TLS_VERIFY_CLIENT=never \
+        -e LDAP_DOMAIN=owncloud.com \
+        -e LDAP_ORGANISATION=ownCloud \
+        -e LDAP_ADMIN_PASSWORD=admin \
+        --name docker-slapd \
+        -p 127.0.0.1:389:389 \
+        -p 636:636 -d osixia/openldap
+    ```
+2. start a REDIS server
+    ```
+    docker run --rm -e REDIS_DATABASES=1 -p 6379:6379 -d webhippie/redis:latest
+    ```
+3. start the needed services
+    ```
+    cd tests/oc-integration-tests/local
+    ../../../cmd/revad/revad -c frontend.toml &
+    ../../../cmd/revad/revad -c gateway.toml &
+    ../../../cmd/revad/revad -c shares.toml &
+    ../../../cmd/revad/revad -c storage-home.toml &
+    ../../../cmd/revad/revad -c storage-oc.toml &
+    ../../../cmd/revad/revad -c storage-publiclink.toml &
+    ../../../cmd/revad/revad -c ldap-users.toml
+    ```
+
+4. clone ownCloud 10
+    `git clone https://github.com/owncloud/core.git ./testrunner`
+
+5. clone the testing app
+    `git clone https://github.com/owncloud/testing.git ./testrunner/apps/testing`
+
+6. run the tests
+    ```
+    cd testrunner
+    TEST_SERVER_URL='http://localhost:20080' \
+    OCIS_REVA_DATA_ROOT='/var/tmp/reva/' \
+    SKELETON_DIR='./apps/testing/data/apiSkeleton' \
+    TEST_EXTERNAL_USER_BACKENDS='true' \
+    REVA_LDAP_HOSTNAME='localhost' \
+    TEST_OCIS='true' \
+    TEST_REVA='true' \
+    BEHAT_FILTER_TAGS='~@skipOnOcis&&~@skipOnOcis-OC-Storage' \
+    make test-acceptance-api
+    ```
+
+    This will run all tests that are relevant to reva.
+
+    To run a single test add BEHAT_FEATURE=<feature file> and specify the path to the feature file and an optional line number. For example: BEHAT_FEATURE='tests/acceptance/features/apiWebdavUpload1/uploadFile.feature:12'
+
 ## Daily releases
 On every commit on the master branch (including merged Pull Requests) a new release will be created and 
 available at [daily releases](https://reva-releases.web.cern.ch/reva-releases).

drone/oc-integration-tests/frontend.toml renamed to tests/oc-integration-tests/drone/frontend.toml

@@ -0,0 +1,110 @@
+[shared]
+jwt_secret = "Pive-Fumkiu4"
+gatewaysvc = "localhost:19000"
+
+# This frontend.toml config file will start a reva service that:
+# - serves as the entrypoint for owncloud APIs.
+# - serves http endpoints on port 20080
+#   - / --------------- ocdav
+#   - /ocs ------------ ocs
+#   - /oauth2 --------- oidcprovider
+#   - /.well-known ---- wellknown service to announce openid-configuration
+#   - TODO(diocas): ocm
+# - authenticates requests using oidc bearer auth and basic auth as fallback
+# - serves the grpc services on port 20099
+[grpc]
+address = "0.0.0.0:20099"
+
+[grpc.services.authprovider]
+auth_manager = "oidc"
+
+# If you want to use your own openid provider change this config
+[grpc.services.authprovider.auth_managers.oidc]
+issuer = "http://localhost:20080"
+
+[http]
+address = "0.0.0.0:20080"
+
+[http.middlewares.cors]
+allow_credentials = true
+
+[http.services.ocdav]
+# serve ocdav on the root path
+prefix = ""
+chunk_folder = "/var/tmp/reva/chunks"
+# for user lookups
+# prefix the path of requests to /dav/files with this namespace
+# While owncloud has only listed usernames at this endpoint CERN has
+# been exposing more than just usernames. For owncloud deployments we
+# can prefix the path to jail the requests to the correct CS3 namespace.
+# In this deployment we mounted the owncloud storage provider at /oc. It
+# expects a username as the first path segment.
+# currently, only the desktop client will use this endpoint, but only if
+# the dav.chunking capability is available
+# TODO implement a path wrapper that rewrites `<username>` into the path
+# layout for the users home?
+# no, use GetHome?
+# for eos we need to rewrite the path
+# TODO strip the username from the path so the CS3 namespace can be mounted
+# at the files/<username> endpoint? what about migration? separate reva instance
+files_namespace = "/oc"
+
+# similar to the dav/files endpoint we can configure a prefix for the old webdav endpoint
+# we use the old webdav endpoint to present the cs3 namespace
+# note: this changes the tree that is rendered at remote.php/webdav from the users home to the cs3 namespace
+# use webdav_namespace = "/home" to use the old namespace that only exposes the users files
+# this endpoint should not affect the desktop client sync but will present different folders for the other clients:
+# - the desktop clients use a hardcoded remote.php/dav/files/<username> if the dav.chunkung capability is present
+# - the ios ios uses the core.webdav-root capability which points to remote.php/webdav in oc10
+# - the oc js sdk is hardcoded to the remote.php/webdav so it will see the new tree
+# - TODO android? no sync ... but will see different tree
+webdav_namespace = "/home"
+
+[http.services.ocs]
+
+[http.services.ocs.capabilities.capabilities.core.status]
+version = "10.0.11.5"
+versionstring = "10.0.11"
+
+[http.services.ocs.capabilities.capabilities.files_sharing]
+api_enabled = true
+resharing = true
+group_sharing = true
+auto_accept_share = true
+share_with_group_members_only = true
+share_with_membership_groups_only = true
+default_permissions = 22
+search_min_length = 3
+
+[http.services.ocs.capabilities.capabilities.files_sharing.public]
+enabled = true
+send_mail = true
+social_share = true
+upload = true
+multiple = true
+supports_upload_only = true
+
+[http.services.ocs.capabilities.capabilities.files_sharing.public.password]
+enforced = true
+
+[http.services.ocs.capabilities.capabilities.files_sharing.public.password.enforced_for]
+read_only = true
+read_write = true
+upload_only = true
+
+[http.services.ocs.capabilities.capabilities.files_sharing.public.expire_date]
+enabled = true
+
+[http.services.ocs.capabilities.capabilities.files_sharing.user]
+send_mail = true
+
+[http.services.ocs.capabilities.capabilities.files_sharing.user_enumeration]
+enabled = true
+group_members_only = true
+
+[http.services.ocs.capabilities.capabilities.files_sharing.federation]
+outgoing = true
+incoming = true
+
+[http.services.ocs.capabilities.capabilities.notifications]
+endpoints = ["disable"]

drone/oc-integration-tests/gateway.toml renamed to tests/oc-integration-tests/drone/gateway.toml

@@ -0,0 +1,69 @@
+[shared]
+jwt_secret = "Pive-Fumkiu4"
+gatewaysvc = "localhost:19000"
+
+# This gateway.toml config file will start a reva service that:
+# - serves as a gateway for all requests
+# - looks up the storageprovider using a storageregistry
+# - looks up the authprovider using an authregistry
+# - serves the gateway on grpc port 19000
+# - serves http datagateway on port 19001
+#   - /data - datagateway: file up and download
+[grpc]
+address = "0.0.0.0:19000"
+
+[grpc.services.gateway]
+# registries
+authregistrysvc = "localhost:19000"
+storageregistrysvc = "localhost:19000"
+# user metadata
+preferencessvc = "localhost:18000"
+userprovidersvc = "localhost:18000"
+# an approvider lives on "localhost:18000" as well, see users.toml
+# sharing
+usershareprovidersvc = "localhost:17000"
+publicshareprovidersvc = "localhost:17000"
+# ocm
+ocmcoresvc = "localhost:13000"
+ocmshareprovidersvc = "localhost:13000"
+ocminvitemanagersvc = "localhost:13000"
+ocmproviderauthorizersvc = "localhost:13000"
+# other
+commit_share_to_storage_grant = true
+datagateway = "http://localhost:19001/data"
+transfer_shared_secret = "replace-me-with-a-transfer-secret" # for direct uploads
+transfer_expires = 6 # give it a moment
+#disable_home_creation_on_login = true
+link_grants_file = "/var/tmp/reva/link_grants_file.json"
+
+[grpc.services.authregistry]
+driver = "static"
+
+[grpc.services.authregistry.drivers.static.rules]
+publicshares = "localhost:17000" # started with the shares.toml
+basic = "localhost:18000" # started with the users.toml
+bearer = "localhost:20099" # started with the frontend.toml
+
+[grpc.services.storageregistry]
+driver = "static"
+
+[grpc.services.storageregistry.drivers.static]
+home_provider = "/home"
+
+[grpc.services.storageregistry.drivers.static.rules]
+# mount a home storage provider that uses a context based path wrapper
+# to jail users into their home dir
+"/home" = "localhost:12000"
+
+# mount a storage provider without a path wrapper for direct access to users.
+"/oc" = "localhost:11000"
+"123e4567-e89b-12d3-a456-426655440000" = "localhost:11000"
+# another mount point might be "/projects/"
+
+"/public" = "localhost:13000"
+
+[http]
+address = "0.0.0.0:19001"
+
+[http.services.datagateway]
+transfer_shared_secret = "replace-me-with-a-transfer-secret"

drone/oc-integration-tests/ldap-users.toml renamed to tests/oc-integration-tests/drone/ldap-users.toml

@@ -0,0 +1,45 @@
+[shared]
+jwt_secret = "Pive-Fumkiu4"
+
+# This users.toml config file will start a reva service that:
+# - handles user metadata and user preferences
+# - serves the grpc services on port 18000
+[grpc]
+address = "0.0.0.0:18000"
+
+[grpc.services.authprovider]
+auth_manager = "ldap"
+
+[grpc.services.authprovider.auth_managers.ldap]
+hostname="localhost"
+port=636
+base_dn="dc=owncloud,dc=com"
+userfilter="(&(objectclass=posixAccount)(cn=%s*))"
+groupfilter="(&(objectclass=posixGroup)(cn=%s*))"
+bind_username="cn=admin,dc=owncloud,dc=com"
+bind_password="admin"
+idp="http://localhost:18000"
+[grpc.services.authprovider.auth_managers.ldap.schema]
+uid="uid"
+displayName="displayName"
+dn="dn"
+cn="cn"
+
+[grpc.services.userprovider]
+driver = "ldap"
+
+[grpc.services.userprovider.drivers.ldap]
+hostname="localhost"
+port=636
+base_dn="dc=owncloud,dc=com"
+userfilter="(&(objectclass=posixAccount)(cn=%s*))"
+groupfilter="(&(objectclass=posixGroup)(cn=%s*))"
+bind_username="cn=admin,dc=owncloud,dc=com"
+bind_password="admin"
+idp="http://localhost:18000"
+
+[grpc.services.userprovider.drivers.ldap.schema]
+uid="uid"
+displayName="displayName"
+dn="dn"
+cn="cn"

drone/oc-integration-tests/shares.toml renamed to tests/oc-integration-tests/drone/shares.toml

@@ -0,0 +1,19 @@
+[shared]
+jwt_secret = "Pive-Fumkiu4"
+
+# GRPC:
+# - serves user and public link shares
+[grpc]
+address = "0.0.0.0:17000"
+
+[grpc.services.usershareprovider]
+driver = "memory"
+
+[grpc.services.authprovider]
+auth_manager = "publicshares"
+
+[grpc.services.authprovider.auth_managers.publicshares]
+gateway_addr = "0.0.0.0:19000"
+
+[grpc.services.publicshareprovider.drivers.json]
+file = "/var/tmp/reva/publicshares.json"