Merge pull request #391 from sudo-bmitch/pr-chunk-min-size

Add OCI-Chunk-Min-Length header

Author: sudo-bmitch

conformance/02_push_test.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"net/http"
 
+	"strconv"
+
 	"github.com/bloodorangeio/reggie"
 	g "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -157,6 +159,16 @@ var test02Push = func() {
 				location := resp.Header().Get("Location")
 				Expect(location).ToNot(BeEmpty())
 
+				// rebuild chunked blob if min size is above our chunk size
+				minSizeStr := resp.Header().Get("OCI-Chunk-Min-Length")
+				if minSizeStr != "" {
+					minSize, err := strconv.Atoi(minSizeStr)
+					Expect(err).To(BeNil())
+					if minSize > len(testBlobBChunk1) {
+						setupChunkedBlob(minSize*2 - 2)
+					}
+				}
+
 				req = client.NewRequest(reggie.PATCH, resp.GetRelativeLocation()).
 					SetHeader("Content-Type", "application/octet-stream").
 					SetHeader("Content-Length", testBlobBChunk2Length).
@@ -187,18 +199,31 @@ var test02Push = func() {
 				lastResponse = resp
 			})
 
-			g.Specify("PUT request with final chunk should return 201", func() {
+			g.Specify("PATCH request with second chunk should return 202", func() {
 				SkipIfDisabled(push)
-				req := client.NewRequest(reggie.PUT, lastResponse.GetRelativeLocation()).
+				req := client.NewRequest(reggie.PATCH, lastResponse.GetRelativeLocation()).
 					SetHeader("Content-Length", testBlobBChunk2Length).
 					SetHeader("Content-Range", testBlobBChunk2Range).
 					SetHeader("Content-Type", "application/octet-stream").
-					SetQueryParam("digest", testBlobBDigest).
 					SetBody(testBlobBChunk2)
 				resp, err := client.Do(req)
 				Expect(err).To(BeNil())
 				location := resp.Header().Get("Location")
 				Expect(location).ToNot(BeEmpty())
+				Expect(resp.StatusCode()).To(Equal(http.StatusAccepted))
+				lastResponse = resp
+			})
+
+			g.Specify("PUT request with digest should return 201", func() {
+				SkipIfDisabled(push)
+				req := client.NewRequest(reggie.PUT, lastResponse.GetRelativeLocation()).
+					SetHeader("Content-Length", "0").
+					SetHeader("Content-Type", "application/octet-stream").
+					SetQueryParam("digest", testBlobBDigest)
+				resp, err := client.Do(req)
+				Expect(err).To(BeNil())
+				location := resp.Header().Get("Location")
+				Expect(location).ToNot(BeEmpty())
 				Expect(resp.StatusCode()).To(Equal(http.StatusCreated))
 			})
 		})

conformance/setup.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"log"
 	"math/big"
+	mathrand "math/rand"
 	"os"
 	"strconv"
 
@@ -133,12 +134,14 @@ var (
 	automaticCrossmountEnabled bool
 	configs                    []TestBlob
 	manifests                  []TestBlob
+	seed                       int64
 	Version                    = "unknown"
 )
 
 func init() {
 	var err error
 
+	seed = g.GinkgoRandomSeed()
 	hostname := os.Getenv(envVarRootURL)
 	namespace := os.Getenv(envVarNamespace)
 	username := os.Getenv(envVarUsername)
@@ -274,18 +277,12 @@ func init() {
 	nonexistentManifest = ".INVALID_MANIFEST_NAME"
 	invalidManifestContent = []byte("blablabla")
 
-	testBlobA = []byte("NBA Jam on my NBA toast")
+	dig, blob := randomBlob(42, seed+1)
+	testBlobA = blob
 	testBlobALength = strconv.Itoa(len(testBlobA))
-	testBlobADigest = godigest.FromBytes(testBlobA).String()
+	testBlobADigest = dig.String()
 
-	testBlobB = []byte("Hello, how are you today?")
-	testBlobBDigest = godigest.FromBytes(testBlobB).String()
-	testBlobBChunk1 = testBlobB[:3]
-	testBlobBChunk1Length = strconv.Itoa(len(testBlobBChunk1))
-	testBlobBChunk1Range = fmt.Sprintf("0-%d", len(testBlobBChunk1)-1)
-	testBlobBChunk2 = testBlobB[3:]
-	testBlobBChunk2Length = strconv.Itoa(len(testBlobBChunk2))
-	testBlobBChunk2Range = fmt.Sprintf("%d-%d", len(testBlobBChunk1), len(testBlobB)-1)
+	setupChunkedBlob(42)
 
 	dummyDigest = godigest.FromString("hello world").String()
 
@@ -404,3 +401,27 @@ func randomString(n int) string {
 	}
 	return string(ret)
 }
+
+// randomBlob outputs a reproducible random blob (based on the seed) for testing
+func randomBlob(size int, seed int64) (godigest.Digest, []byte) {
+	r := mathrand.New(mathrand.NewSource(seed))
+	b := make([]byte, size)
+	if n, err := r.Read(b); err != nil {
+		panic(err)
+	} else if n != size {
+		panic("unable to read enough bytes")
+	}
+	return godigest.FromBytes(b), b
+}
+
+func setupChunkedBlob(size int) {
+	dig, blob := randomBlob(size, seed+2)
+	testBlobB = blob
+	testBlobBDigest = dig.String()
+	testBlobBChunk1 = testBlobB[:size/2+1]
+	testBlobBChunk1Length = strconv.Itoa(len(testBlobBChunk1))
+	testBlobBChunk1Range = fmt.Sprintf("0-%d", len(testBlobBChunk1)-1)
+	testBlobBChunk2 = testBlobB[size/2+1:]
+	testBlobBChunk2Length = strconv.Itoa(len(testBlobBChunk2))
+	testBlobBChunk2Range = fmt.Sprintf("%d-%d", len(testBlobBChunk1), len(testBlobB)-1)
+}

spec.md

@@ -321,6 +321,12 @@ The process remains unchanged for chunked upload, except that the post request M
 Content-Length: 0
 ```
 
+If the registry has a minimum chunk size, the response SHOULD include the following header, where `<size>` is the size in bytes (see the blob `PATCH` definition for usage):
+
+```
+OCI-Chunk-Min-Length: <size>
+```
+
 Please reference the above section for restrictions on the `<location>`.
 
 ---
@@ -347,6 +353,8 @@ It MUST match the following regular expression:
 ```
 
 The `<length>` is the content-length, in bytes, of the current chunk.
+If the registry provides a `OCI-Chunk-Min-Length` header in the `PUT` response, the size of each chunk, except for the final chunk, SHOULD be greater or equal to that value.
+The final chunk MAY have any length.
 
 Each successful chunk upload MUST have a `202 Accepted` response code, and MUST have the following header: