Add builder OCI layout build context

Signed-off-by: Austin Vazquez <[email protected]>

Author: austinvazquez

cmd/nerdctl/builder_build_linux_test.go

@@ -0,0 +1,84 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"gotest.tools/v3/assert"
+
+	"github.com/containerd/nerdctl/v2/pkg/testutil"
+)
+
+func TestBuildContextWithOCILayout(t *testing.T) {
+	testutil.RequiresBuild(t)
+	testutil.RegisterBuildCacheCleanup(t)
+
+	var dockerBuilderArgs []string
+	if testutil.IsDocker() {
+		// Default docker driver does not support OCI exporter.
+		// Reference: https://docs.docker.com/build/exporters/oci-docker/
+		builderName := testutil.SetupDockerContainerBuilder(t)
+		dockerBuilderArgs = []string{"buildx", "--builder", builderName}
+	}
+
+	base := testutil.NewBase(t)
+	imageName := testutil.Identifier(t)
+	ociLayout := "parent"
+	parentImageName := fmt.Sprintf("%s-%s", imageName, ociLayout)
+
+	teardown := func() {
+		base.Cmd("rmi", parentImageName, imageName).Run()
+	}
+	t.Cleanup(teardown)
+	teardown()
+
+	dockerfile := fmt.Sprintf(`FROM %s
+LABEL layer=oci-layout-parent
+CMD ["echo", "test-nerdctl-build-context-oci-layout-parent"]`, testutil.CommonImage)
+	buildCtx := createBuildContext(t, dockerfile)
+
+	tarPath := fmt.Sprintf("%s/%s.tar", buildCtx, ociLayout)
+
+	// Create OCI archive from parent image.
+	base.Cmd("build", buildCtx, "--tag", parentImageName).AssertOK()
+	base.Cmd("image", "save", "--output", tarPath, parentImageName).AssertOK()
+
+	// Unpack OCI archive into OCI layout directory.
+	ociLayoutDir := t.TempDir()
+	err := extractTarFile(ociLayoutDir, tarPath)
+	assert.NilError(t, err)
+
+	dockerfile = fmt.Sprintf(`FROM %s
+CMD ["echo", "test-nerdctl-build-context-oci-layout"]`, ociLayout)
+	buildCtx = createBuildContext(t, dockerfile)
+
+	var buildArgs = []string{}
+	if testutil.IsDocker() {
+		buildArgs = dockerBuilderArgs
+	}
+
+	buildArgs = append(buildArgs, "build", buildCtx, fmt.Sprintf("--build-context=%s=oci-layout://%s", ociLayout, ociLayoutDir), "--tag", imageName)
+	if testutil.IsDocker() {
+		// Need to load the container image from the builder to be able to run it.
+		buildArgs = append(buildArgs, "--load")
+	}
+
+	base.Cmd(buildArgs...).AssertOK()
+	base.Cmd("run", "--rm", imageName).AssertOutContains("test-nerdctl-build-context-oci-layout")
+}

docs/command-reference.md

@@ -708,7 +708,7 @@ Flags:
 - :nerd_face: `--ipfs`: Build image with pulling base images from IPFS. See [`ipfs.md`](./ipfs.md) for details.
 - :whale: `--label`: Set metadata for an image
 - :whale: `--network=(default|host|none)`: Set the networking mode for the RUN instructions during build.(compatible with `buildctl build`)
-- :whale: --build-context: Set additional contexts for build (e.g. dir2=/path/to/dir2, myorg/myapp=docker-image://path/to/myorg/myapp)
+- :whale: `--build-context`: Set additional contexts for build (e.g. dir2=/path/to/dir2, myorg/myapp=docker-image://path/to/myorg/myapp)
 
 Unimplemented `docker build` flags: `--add-host`, `--squash`
 

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/coreos/go-iptables v0.7.0
 	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/cyphar/filepath-securejoin v0.3.1
+	github.com/distribution/distribution v2.8.3+incompatible
 	github.com/distribution/reference v0.6.0
 	github.com/docker/cli v27.1.2+incompatible
 	github.com/docker/docker v27.1.2+incompatible
@@ -86,6 +87,7 @@ require (
 	github.com/containerd/ttrpc v1.2.5 // indirect
 	github.com/containers/ocicrypt v1.2.0 // indirect
 	github.com/djherbis/times v1.6.0 // indirect
+	github.com/docker/distribution v2.8.3+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.8.2 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.4 // indirect

go.sum

@@ -84,12 +84,16 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/distribution/distribution v2.8.3+incompatible h1:RlpEXBLq/WPXYvBYMDAmBX/SnhD67qwtvW/DzKc8pAo=
+github.com/distribution/distribution v2.8.3+incompatible/go.mod h1:EgLm2NgWtdKgzF9NpMzUKgzmR7AMmb0VQi2B+ZzDRjc=
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
 github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
 github.com/docker/cli v27.1.2+incompatible h1:nYviRv5Y+YAKx3dFrTvS1ErkyVVunKOhoweCTE1BsnI=
 github.com/docker/cli v27.1.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
+github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v27.1.2+incompatible h1:AhGzR1xaQIy53qCkxARaFluI00WPGtXn0AJuoQsVYTY=
 github.com/docker/docker v27.1.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo=

pkg/cmd/builder/build.go

@@ -28,7 +28,9 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/distribution/distribution/manifest/schema2"
 	distributionref "github.com/distribution/reference"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 
 	containerd "github.com/containerd/containerd/v2/client"
 	"github.com/containerd/containerd/v2/core/images"
@@ -300,6 +302,16 @@ func generateBuildctlArgs(ctx context.Context, client *containerd.Client, option
 			continue
 		}
 
+		if isOCILayout := strings.HasPrefix(v, "oci-layout://"); isOCILayout {
+			args, err := parseBuildContextFromOCILayout(k, v)
+			if err != nil {
+				return "", nil, false, "", nil, nil, err
+			}
+
+			buildctlArgs = append(buildctlArgs, args...)
+			continue
+		}
+
 		path, err := filepath.Abs(v)
 		if err != nil {
 			return "", nil, false, "", nil, nil, err
@@ -534,3 +546,61 @@ func parseContextNames(values []string) (map[string]string, error) {
 	}
 	return result, nil
 }
+
+var (
+	ErrOCILayoutPrefixNotFound = errors.New("OCI layout prefix not found")
+	ErrOCILayoutEmptyDigest    = errors.New("OCI layout cannot have empty digest")
+)
+
+func parseBuildContextFromOCILayout(name, path string) ([]string, error) {
+	path, found := strings.CutPrefix(path, "oci-layout://")
+	if !found {
+		return []string{}, ErrOCILayoutPrefixNotFound
+	}
+
+	abspath, err := filepath.Abs(path)
+	if err != nil {
+		return []string{}, err
+	}
+
+	ociIndex, err := readOCIIndexFromPath(abspath)
+	if err != nil {
+		return []string{}, err
+	}
+
+	var digest string
+	for _, manifest := range ociIndex.Manifests {
+		if manifest.MediaType == schema2.MediaTypeManifest || manifest.MediaType == ocispec.MediaTypeImageManifest {
+			digest = manifest.Digest.String()
+		}
+	}
+
+	if digest == "" {
+		return []string{}, ErrOCILayoutEmptyDigest
+	}
+
+	return []string{
+		fmt.Sprintf("--oci-layout=parent-image-key=%s", abspath),
+		fmt.Sprintf("--opt=context:%s=oci-layout:parent-image-key@%s", name, digest),
+	}, nil
+}
+
+func readOCIIndexFromPath(path string) (*ocispec.Index, error) {
+	ociIndexJSONFile, err := os.Open(filepath.Join(path, "index.json"))
+	if err != nil {
+		return nil, err
+	}
+	defer ociIndexJSONFile.Close()
+
+	rawBytes, err := io.ReadAll(ociIndexJSONFile)
+	if err != nil {
+		return nil, err
+	}
+
+	var ociIndex *ocispec.Index
+	err = json.Unmarshal(rawBytes, &ociIndex)
+	if err != nil {
+		return nil, err
+	}
+	return ociIndex, nil
+}

pkg/cmd/builder/build_test.go

@@ -187,3 +187,42 @@ func TestIsBuildPlatformDefault(t *testing.T) {
 		})
 	}
 }
+
+func TestParseBuildctlArgsForOCILayout(t *testing.T) {
+	tests := []struct {
+		name          string
+		ociLayoutName string
+		ociLayoutPath string
+		expectedArgs  []string
+		errorIsNil    bool
+		expectedErr   string
+	}{
+		{
+			name:          "PrefixNotFoundError",
+			ociLayoutName: "unit-test",
+			ociLayoutPath: "/tmp/oci-layout/",
+			expectedArgs:  []string{},
+			expectedErr:   ErrOCILayoutPrefixNotFound.Error(),
+		},
+		{
+			name:          "DirectoryNotFoundError",
+			ociLayoutName: "unit-test",
+			ociLayoutPath: "oci-layout:///tmp/oci-layout",
+			expectedArgs:  []string{},
+			expectedErr:   "open /tmp/oci-layout/index.json: no such file or directory",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			args, err := parseBuildContextFromOCILayout(test.ociLayoutName, test.ociLayoutPath)
+			if test.errorIsNil {
+				assert.NilError(t, err)
+			} else {
+				assert.Error(t, err, test.expectedErr)
+			}
+			assert.Equal(t, len(args), len(test.expectedArgs))
+			assert.DeepEqual(t, args, test.expectedArgs)
+		})
+	}
+}

pkg/testutil/testutil.go

@@ -575,8 +575,12 @@ func GetDaemonIsKillable() bool {
 	return flagTestKillDaemon
 }
 
+func IsDocker() bool {
+	return GetTarget() == Docker
+}
+
 func DockerIncompatible(t testing.TB) {
-	if GetTarget() == Docker {
+	if IsDocker() {
 		t.Skip("test is incompatible with Docker")
 	}
 }
@@ -789,3 +793,21 @@ func KubectlHelper(base *Base, args ...string) *Cmd {
 		Base: base,
 	}
 }
+
+// SetupDockerContinerBuilder creates a Docker builder using the docker-container driver
+// and adds cleanup steps to test cleanup. The builder name is returned as output.
+//
+// If not docker, this function returns an empty string as the builder name.
+func SetupDockerContainerBuilder(t *testing.T) string {
+	var name string
+	if IsDocker() {
+		name = fmt.Sprintf("%s-container", Identifier(t))
+		base := NewBase(t)
+		base.Cmd("buildx", "create", "--name", name, "--driver=docker-container").AssertOK()
+		t.Cleanup(func() {
+			base.Cmd("buildx", "stop", name).AssertOK()
+			base.Cmd("buildx", "rm", "--force", name).AssertOK()
+		})
+	}
+	return name
+}