Skip to content

Commit 37fc75b

Browse files
gsquared94gsquared94
committed
feat: implement cross-platform support for GCB build env
1 parent 9de79cd commit 37fc75b

File tree

12 files changed

+145
-30
lines changed

12 files changed

+145
-30
lines changed

‎integration/examples/cross-platform-builds/Dockerfile

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1-
FROM golang:1.15 as builder
1+
FROM --platform=$BUILDPLATFORM golang:1.15 as builder
22
COPY main.go .
33
# `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations
4+
ARG TARGETOS
5+
ARG TARGETARCH
46
ARG SKAFFOLD_GO_GCFLAGS
5-
RUN go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o /app main.go
7+
RUN GOOS=$TARGETOS GOARCH=$TARGETARCH go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o /app main.go
68

79
FROM alpine:3
810
# Define GOTRACEBACK to mark this container as using the Go language runtime

‎integration/examples/cross-platform-builds/skaffold.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,8 @@ deploy:
88
kubectl:
99
manifests:
1010
- k8s-*
11+
profiles:
12+
- name: googleCloudBuild
13+
build:
14+
googleCloudBuild:
15+
projectId: k8s-skaffold

‎pkg/skaffold/build/gcb/buildpacks_test.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222

2323
"google.golang.org/api/cloudbuild/v1"
2424

25+
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
2526
latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
2627
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
2728
"github.com/GoogleContainerTools/skaffold/testutil"
@@ -194,7 +195,7 @@ func TestBuildpackBuildSpec(t *testing.T) {
194195
builder := NewBuilder(&mockBuilderContext{artifactStore: store}, &latestV1.GoogleCloudBuild{
195196
PackImage: "pack/image",
196197
})
197-
buildSpec, err := builder.buildSpec(context.Background(), artifact, "img", "bucket", "object")
198+
buildSpec, err := builder.buildSpec(context.Background(), artifact, "img", platform.Matcher{}, "bucket", "object")
198199
t.CheckError(test.shouldErr, err)
199200

200201
if !test.shouldErr {

‎pkg/skaffold/build/gcb/cloud_build.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ func (b *Builder) buildArtifactWithCloudBuild(ctx context.Context, out io.Writer
152152
})
153153
}
154154

155-
buildSpec, err := b.buildSpec(ctx, artifact, tag, cbBucket, buildObject)
155+
buildSpec, err := b.buildSpec(ctx, artifact, tag, platform, cbBucket, buildObject)
156156
if err != nil {
157157
return "", sErrors.NewErrorWithStatusCode(&proto.ActionableErr{
158158
ErrCode: proto.StatusCode_BUILD_GCB_GENERATE_BUILD_DESCRIPTOR_ERR,

‎pkg/skaffold/build/gcb/docker.go

+21-10
Original file line numberDiff line numberDiff line change
@@ -24,24 +24,29 @@ import (
2424
cloudbuild "google.golang.org/api/cloudbuild/v1"
2525

2626
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
27+
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
2728
latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
2829
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util/stringslice"
2930
)
3031

3132
// dockerBuildSpec lists the build steps required to build a docker image.
32-
func (b *Builder) dockerBuildSpec(a *latestV1.Artifact, tag string) (cloudbuild.Build, error) {
33+
func (b *Builder) dockerBuildSpec(a *latestV1.Artifact, tag string, platforms platform.Matcher) (cloudbuild.Build, error) {
3334
a = adjustCacheFrom(a, tag)
3435

35-
args, err := b.dockerBuildArgs(a, tag, a.Dependencies)
36+
args, err := b.dockerBuildArgs(a, tag, a.Dependencies, platforms)
3637
if err != nil {
3738
return cloudbuild.Build{}, err
3839
}
39-
40-
steps := b.cacheFromSteps(a.DockerArtifact)
41-
steps = append(steps, &cloudbuild.BuildStep{
40+
steps := b.cacheFromSteps(a.DockerArtifact, platforms)
41+
buildStep := &cloudbuild.BuildStep{
4242
Name: b.DockerImage,
4343
Args: args,
44-
})
44+
}
45+
if platforms.IsNotEmpty() {
46+
// cross-platform build requires buildkit enabled
47+
buildStep.Env = append(buildStep.Env, "DOCKER_BUILDKIT=1")
48+
}
49+
steps = append(steps, buildStep)
4550

4651
return cloudbuild.Build{
4752
Steps: steps,
@@ -50,22 +55,25 @@ func (b *Builder) dockerBuildSpec(a *latestV1.Artifact, tag string) (cloudbuild.
5055
}
5156

5257
// cacheFromSteps pulls images used by `--cache-from`.
53-
func (b *Builder) cacheFromSteps(artifact *latestV1.DockerArtifact) []*cloudbuild.BuildStep {
58+
func (b *Builder) cacheFromSteps(artifact *latestV1.DockerArtifact, platforms platform.Matcher) []*cloudbuild.BuildStep {
5459
var steps []*cloudbuild.BuildStep
55-
60+
argFmt := "docker pull %s || true"
61+
if platforms.IsNotEmpty() {
62+
argFmt = "docker pull --platform " + platforms.String() + " %s || true"
63+
}
5664
for _, cacheFrom := range artifact.CacheFrom {
5765
steps = append(steps, &cloudbuild.BuildStep{
5866
Name: b.DockerImage,
5967
Entrypoint: "sh",
60-
Args: []string{"-c", fmt.Sprintf("docker pull %s || true", cacheFrom)},
68+
Args: []string{"-c", fmt.Sprintf(argFmt, cacheFrom)},
6169
})
6270
}
6371

6472
return steps
6573
}
6674

6775
// dockerBuildArgs lists the arguments passed to `docker` to build a given image.
68-
func (b *Builder) dockerBuildArgs(a *latestV1.Artifact, tag string, deps []*latestV1.ArtifactDependency) ([]string, error) {
76+
func (b *Builder) dockerBuildArgs(a *latestV1.Artifact, tag string, deps []*latestV1.ArtifactDependency, platforms platform.Matcher) ([]string, error) {
6977
d := a.DockerArtifact
7078
// TODO(nkubala): remove when buildkit is supported in GCB (#4773)
7179
if len(d.Secrets) > 0 || d.SSH != "" {
@@ -83,6 +91,9 @@ func (b *Builder) dockerBuildArgs(a *latestV1.Artifact, tag string, deps []*late
8391
}
8492

8593
args := []string{"build", "--tag", tag, "-f", d.DockerfilePath}
94+
if platforms.IsNotEmpty() {
95+
args = append(args, "--platform", platforms.String())
96+
}
8697
args = append(args, ba...)
8798
args = append(args, ".")
8899

‎pkg/skaffold/build/gcb/docker_test.go

+69-3
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,12 @@ import (
2020
"context"
2121
"testing"
2222

23-
cloudbuild "google.golang.org/api/cloudbuild/v1"
23+
v1 "github.com/opencontainers/image-spec/specs-go/v1"
24+
"google.golang.org/api/cloudbuild/v1"
2425

2526
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
2627
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
28+
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
2729
latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
2830
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
2931
"github.com/GoogleContainerTools/skaffold/testutil"
@@ -33,6 +35,7 @@ func TestDockerBuildSpec(t *testing.T) {
3335
tests := []struct {
3436
description string
3537
artifact *latestV1.Artifact
38+
platforms platform.Matcher
3639
expected cloudbuild.Build
3740
shouldErr bool
3841
}{
@@ -130,6 +133,42 @@ func TestDockerBuildSpec(t *testing.T) {
130133
},
131134
shouldErr: true,
132135
},
136+
137+
{
138+
description: "cross-platform build",
139+
artifact: &latestV1.Artifact{
140+
ArtifactType: latestV1.ArtifactType{
141+
DockerArtifact: &latestV1.DockerArtifact{
142+
DockerfilePath: "Dockerfile",
143+
BuildArgs: map[string]*string{
144+
"arg1": util.StringPtr("value1"),
145+
"arg2": nil,
146+
},
147+
},
148+
},
149+
},
150+
platforms: platform.Matcher{Platforms: []v1.Platform{{Architecture: "arm", OS: "freebsd"}}},
151+
expected: cloudbuild.Build{
152+
LogsBucket: "bucket",
153+
Source: &cloudbuild.Source{
154+
StorageSource: &cloudbuild.StorageSource{
155+
Bucket: "bucket",
156+
Object: "object",
157+
},
158+
},
159+
Steps: []*cloudbuild.BuildStep{{
160+
Name: "docker/docker",
161+
Args: []string{"build", "--tag", "nginx", "-f", "Dockerfile", "--platform", "freebsd/arm", "--build-arg", "arg1=value1", "--build-arg", "arg2", "."},
162+
Env: []string{"DOCKER_BUILDKIT=1"},
163+
}},
164+
Images: []string{"nginx"},
165+
Options: &cloudbuild.BuildOptions{
166+
DiskSizeGb: 100,
167+
MachineType: "n1-standard-1",
168+
},
169+
Timeout: "10m",
170+
},
171+
},
133172
}
134173

135174
for _, test := range tests {
@@ -157,7 +196,7 @@ func TestDockerBuildSpec(t *testing.T) {
157196
Timeout: "10m",
158197
})
159198

160-
desc, err := builder.buildSpec(context.Background(), test.artifact, "nginx", "bucket", "object")
199+
desc, err := builder.buildSpec(context.Background(), test.artifact, "nginx", test.platforms, "bucket", "object")
161200
t.CheckErrorAndDeepEqual(test.shouldErr, err, test.expected, desc)
162201
})
163202
}
@@ -168,6 +207,7 @@ func TestPullCacheFrom(t *testing.T) {
168207
description string
169208
artifact *latestV1.Artifact
170209
tag string
210+
platforms platform.Matcher
171211
expected []*cloudbuild.BuildStep
172212
shouldErr bool
173213
}{
@@ -216,6 +256,32 @@ func TestPullCacheFrom(t *testing.T) {
216256
Args: []string{"build", "--tag", "gcr.io/k8s-skaffold/test:tagged", "-f", "Dockerfile", "--cache-from", "gcr.io/k8s-skaffold/test:tagged", "."},
217257
}},
218258
},
259+
{
260+
description: "cross-platform cache-from images",
261+
artifact: &latestV1.Artifact{
262+
ArtifactType: latestV1.ArtifactType{
263+
DockerArtifact: &latestV1.DockerArtifact{
264+
DockerfilePath: "Dockerfile",
265+
CacheFrom: []string{"from/image1", "from/image2"},
266+
},
267+
},
268+
},
269+
tag: "nginx2",
270+
platforms: platform.Matcher{Platforms: []v1.Platform{{Architecture: "arm", OS: "freebsd"}}},
271+
expected: []*cloudbuild.BuildStep{{
272+
Name: "docker/docker",
273+
Entrypoint: "sh",
274+
Args: []string{"-c", "docker pull --platform freebsd/arm from/image1 || true"},
275+
}, {
276+
Name: "docker/docker",
277+
Entrypoint: "sh",
278+
Args: []string{"-c", "docker pull --platform freebsd/arm from/image2 || true"},
279+
}, {
280+
Name: "docker/docker",
281+
Args: []string{"build", "--tag", "nginx2", "-f", "Dockerfile", "--platform", "freebsd/arm", "--cache-from", "from/image1", "--cache-from", "from/image2", "."},
282+
Env: []string{"DOCKER_BUILDKIT=1"},
283+
}},
284+
},
219285
}
220286

221287
for _, test := range tests {
@@ -226,7 +292,7 @@ func TestPullCacheFrom(t *testing.T) {
226292
builder := NewBuilder(&mockBuilderContext{}, &latestV1.GoogleCloudBuild{
227293
DockerImage: "docker/docker",
228294
})
229-
desc, err := builder.dockerBuildSpec(test.artifact, test.tag)
295+
desc, err := builder.dockerBuildSpec(test.artifact, test.tag, test.platforms)
230296

231297
t.CheckErrorAndDeepEqual(test.shouldErr, err, test.expected, desc.Steps)
232298
})

‎pkg/skaffold/build/gcb/jib_test.go

+27-3
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@ import (
2222
"path/filepath"
2323
"testing"
2424

25-
cloudbuild "google.golang.org/api/cloudbuild/v1"
25+
v1 "github.com/opencontainers/image-spec/specs-go/v1"
26+
"google.golang.org/api/cloudbuild/v1"
2627

2728
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/jib"
29+
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
2830
latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
2931
"github.com/GoogleContainerTools/skaffold/testutil"
3032
)
@@ -40,6 +42,7 @@ func TestJibMavenBuildSpec(t *testing.T) {
4042
description string
4143
skipTests bool
4244
baseImage string
45+
platforms platform.Matcher
4346
expectedArgs []string
4447
}{
4548
{
@@ -64,6 +67,16 @@ func TestJibMavenBuildSpec(t *testing.T) {
6467
skipTests: false,
6568
expectedArgs: []string{"-c", "mvn -Duser.home=$$HOME --batch-mode jib:_skaffold-fail-if-jib-out-of-date -Djib.requiredVersion=" + jib.MinimumJibMavenVersion + " --non-recursive prepare-package jib:build -Djib.from.image=img2:tag -Dimage=img"},
6669
},
70+
{
71+
description: "cross platform",
72+
platforms: platform.Matcher{Platforms: []v1.Platform{{Architecture: "arm", OS: "freebsd"}}},
73+
expectedArgs: []string{"-c", "mvn -Duser.home=$$HOME --batch-mode jib:_skaffold-fail-if-jib-out-of-date -Djib.requiredVersion=" + jib.MinimumJibMavenVersionForCrossPlatform + " --non-recursive prepare-package jib:build -Djib.from.platforms=freebsd/arm -Dimage=img"},
74+
},
75+
{
76+
description: "multi platform",
77+
platforms: platform.Matcher{Platforms: []v1.Platform{{Architecture: "arm", OS: "freebsd"}, {Architecture: "arm64", OS: "linux"}}},
78+
expectedArgs: []string{"-c", "mvn -Duser.home=$$HOME --batch-mode jib:_skaffold-fail-if-jib-out-of-date -Djib.requiredVersion=" + jib.MinimumJibMavenVersionForCrossPlatform + " --non-recursive prepare-package jib:build -Djib.from.platforms=freebsd/arm,linux/arm64 -Dimage=img"},
79+
},
6780
}
6881
for _, test := range tests {
6982
testutil.Run(t, test.description, func(t *testutil.T) {
@@ -86,7 +99,7 @@ func TestJibMavenBuildSpec(t *testing.T) {
8699
})
87100
builder.skipTests = test.skipTests
88101

89-
buildSpec, err := builder.buildSpec(context.Background(), artifact, "img", "bucket", "object")
102+
buildSpec, err := builder.buildSpec(context.Background(), artifact, "img", test.platforms, "bucket", "object")
90103
t.CheckNoError(err)
91104

92105
expected := []*cloudbuild.BuildStep{{
@@ -105,6 +118,7 @@ func TestJibGradleBuildSpec(t *testing.T) {
105118
tests := []struct {
106119
description string
107120
skipTests bool
121+
platforms platform.Matcher
108122
expectedArgs []string
109123
}{
110124
{
@@ -117,6 +131,16 @@ func TestJibGradleBuildSpec(t *testing.T) {
117131
skipTests: false,
118132
expectedArgs: []string{"-c", "gradle -Duser.home=$$HOME --console=plain _skaffoldFailIfJibOutOfDate -Djib.requiredVersion=" + jib.MinimumJibGradleVersion + " :jib --image=img"},
119133
},
134+
{
135+
description: "cross platform",
136+
platforms: platform.Matcher{Platforms: []v1.Platform{{Architecture: "arm", OS: "freebsd"}}},
137+
expectedArgs: []string{"-c", "gradle -Duser.home=$$HOME --console=plain _skaffoldFailIfJibOutOfDate -Djib.requiredVersion=" + jib.MinimumJibGradleVersionForCrossPlatform + " :jib -Djib.from.platforms=freebsd/arm --image=img"},
138+
},
139+
{
140+
description: "multi platform",
141+
platforms: platform.Matcher{Platforms: []v1.Platform{{Architecture: "arm", OS: "freebsd"}, {Architecture: "arm64", OS: "linux"}}},
142+
expectedArgs: []string{"-c", "gradle -Duser.home=$$HOME --console=plain _skaffoldFailIfJibOutOfDate -Djib.requiredVersion=" + jib.MinimumJibGradleVersionForCrossPlatform + " :jib -Djib.from.platforms=freebsd/arm,linux/arm64 --image=img"},
143+
},
120144
}
121145
for _, test := range tests {
122146
testutil.Run(t, test.description, func(t *testutil.T) {
@@ -131,7 +155,7 @@ func TestJibGradleBuildSpec(t *testing.T) {
131155
})
132156
builder.skipTests = test.skipTests
133157

134-
buildSpec, err := builder.buildSpec(context.Background(), artifact, "img", "bucket", "object")
158+
buildSpec, err := builder.buildSpec(context.Background(), artifact, "img", test.platforms, "bucket", "object")
135159
t.CheckNoError(err)
136160

137161
expected := []*cloudbuild.BuildStep{{

‎pkg/skaffold/build/gcb/kaniko_test.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
2828
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
2929
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph"
30+
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
3031
latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
3132
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
3233
"github.com/GoogleContainerTools/skaffold/testutil"
@@ -438,7 +439,7 @@ func TestKanikoBuildSpec(t *testing.T) {
438439
}
439440
return m, nil
440441
})
441-
desc, err := builder.buildSpec(context.Background(), artifact, "gcr.io/nginx", "bucket", "object")
442+
desc, err := builder.buildSpec(context.Background(), artifact, "gcr.io/nginx", platform.Matcher{}, "bucket", "object")
442443

443444
expected := cloudbuild.Build{
444445
LogsBucket: "bucket",

‎pkg/skaffold/build/gcb/spec.go

+10-5
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,17 @@ import (
2020
"context"
2121
"fmt"
2222

23+
v1 "github.com/opencontainers/image-spec/specs-go/v1"
2324
"google.golang.org/api/cloudbuild/v1"
2425

2526
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/misc"
2627
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
2728
latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
2829
)
2930

30-
func (b *Builder) buildSpec(ctx context.Context, artifact *latestV1.Artifact, tag, bucket, object string) (cloudbuild.Build, error) {
31+
func (b *Builder) buildSpec(ctx context.Context, artifact *latestV1.Artifact, tag string, platforms platform.Matcher, bucket, object string) (cloudbuild.Build, error) {
3132
// Artifact specific build spec
32-
buildSpec, err := b.buildSpecForArtifact(ctx, artifact, tag)
33+
buildSpec, err := b.buildSpecForArtifact(ctx, artifact, tag, platforms)
3334
if err != nil {
3435
return buildSpec, err
3536
}
@@ -57,18 +58,22 @@ func (b *Builder) buildSpec(ctx context.Context, artifact *latestV1.Artifact, ta
5758
return buildSpec, nil
5859
}
5960

60-
func (b *Builder) buildSpecForArtifact(ctx context.Context, a *latestV1.Artifact, tag string) (cloudbuild.Build, error) {
61+
func (b *Builder) buildSpecForArtifact(ctx context.Context, a *latestV1.Artifact, tag string, platforms platform.Matcher) (cloudbuild.Build, error) {
6162
switch {
6263
case a.KanikoArtifact != nil:
6364
return b.kanikoBuildSpec(a, tag)
6465

6566
case a.DockerArtifact != nil:
66-
return b.dockerBuildSpec(a, tag)
67+
return b.dockerBuildSpec(a, tag, platforms)
6768

6869
case a.JibArtifact != nil:
69-
return b.jibBuildSpec(ctx, a, tag, platform.Matcher{}) // TODO: pass correct platform matcher for GCB builds
70+
return b.jibBuildSpec(ctx, a, tag, platforms)
7071

7172
case a.BuildpackArtifact != nil:
73+
// TODO: Buildpacks only supports building for platform linux/amd64. See https://github.com/GoogleCloudPlatform/buildpacks/issues/112
74+
if platforms.IsNotEmpty() && platforms.Intersect(platform.Matcher{Platforms: []v1.Platform{{OS: "linux", Architecture: "amd64"}}}).IsEmpty() {
75+
return cloudbuild.Build{}, fmt.Errorf("buildpacks builder doesn't support building for platforms %s. Cannot build gcb artifact:\n%s", platforms.String(), misc.FormatArtifact(a))
76+
}
7277
return b.buildpackBuildSpec(a.BuildpackArtifact, tag, a.Dependencies)
7378

7479
default:

0 commit comments

Comments
 (0)