Skip to content

Commit 8ec0672

Browse files
ktockktock
committed
rootless: Support BuildKit containerd worker
Signed-off-by: Kohei Tokunaga <[email protected]>
1 parent 4308c31 commit 8ec0672

23 files changed

+467
-35
lines changed

‎Dockerfile

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ ARG CNI_PLUGINS_VERSION=v1.1.0
2525
# Extra deps: CNI isolation
2626
ARG CNI_ISOLATION_VERSION=v0.0.4
2727
# Extra deps: Build
28-
ARG BUILDKIT_VERSION=v0.9.3
28+
ARG BUILDKIT_VERSION=v0.10.0
2929
# Extra deps: Lazy-pulling
3030
ARG STARGZ_SNAPSHOTTER_VERSION=v0.11.2
3131
# Extra deps: Encryption
@@ -264,8 +264,9 @@ COPY --from=gcr.io/projectsigstore/cosign:v1.3.1@sha256:3cd9b3a866579dc2e0cf2fde
264264
# enable offline ipfs for integration test
265265
COPY ./Dockerfile.d/test-integration-etc_containerd-stargz-grpc_config.toml /etc/containerd-stargz-grpc/config.toml
266266
COPY ./Dockerfile.d/test-integration-ipfs-offline.service /usr/local/lib/systemd/system/
267+
COPY ./Dockerfile.d/test-integration-buildkit-nerdctl-test.service /usr/local/lib/systemd/system/
267268
# install ipfs service. avoid using 5001(api)/8080(gateway) which are reserved by tests.
268-
RUN systemctl enable test-integration-ipfs-offline && \
269+
RUN systemctl enable test-integration-ipfs-offline test-integration-buildkit-nerdctl-test && \
269270
ipfs init && \
270271
ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5888" && \
271272
ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/5889"

‎Dockerfile.d/SHA256SUMS.d/buildkit-v0.10.0

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
ed9d3942ca3f1cbc4906577a2422e5084416dd2739f3d85b800a129d61557630 buildkit-v0.10.0.linux-amd64.tar.gz
2+
f201c30dca4877eca9598ae41c1c8934c98b37a9ad323cec8b30ce9138f957ac buildkit-v0.10.0.linux-arm64.tar.gz

‎Dockerfile.d/SHA256SUMS.d/buildkit-v0.9.3

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎Dockerfile.d/test-integration-buildkit-nerdctl-test.service

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Copyright The containerd Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# Copied from released buildkit.service
16+
17+
[Unit]
18+
After=network.target local-fs.target
19+
Description=buildkit daemon for integration test (namespace: nerdctl-test)
20+
21+
[Service]
22+
ExecStartPre=-/sbin/modprobe overlay
23+
ExecStart=/usr/local/bin/buildkitd --oci-worker=false --containerd-worker=true --addr="unix:///run/buildkit-nerdctl-test/buildkitd.sock" --root=/var/lib/buildkit-nerdctl-test --containerd-worker-namespace=nerdctl-test
24+
25+
Type=notify
26+
Delegate=yes
27+
KillMode=process
28+
Restart=always
29+
RestartSec=5
30+
# Having non-zero Limit*s causes performance problems due to accounting overhead
31+
# in the kernel. We recommend using cgroups to do container-local accounting.
32+
LimitNPROC=infinity
33+
LimitCORE=infinity
34+
LimitNOFILE=infinity
35+
# Comment TasksMax if your systemd version does not supports it.
36+
# Only systemd 226 and above support this version.
37+
TasksMax=infinity
38+
OOMScoreAdjust=-999
39+
40+
[Install]
41+
WantedBy=docker-entrypoint.target

‎Dockerfile.d/test-integration-rootless.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ else
3838
if [[ -e /workaround-cirrus ]]; then
3939
echo "WORKAROUND_CIRRUS: Not enabling BuildKit (https://github.com/containerd/nerdctl/issues/622)" >&2
4040
else
41-
containerd-rootless-setuptool.sh install-buildkit
41+
CONTAINERD_NAMESPACE="nerdctl-test" containerd-rootless-setuptool.sh install-buildkit-containerd
4242
fi
4343
containerd-rootless-setuptool.sh install-stargz
4444
cat <<EOF >>/home/rootless/.config/containerd/config.toml

‎README.md

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,8 @@ It does not necessarily mean that the corresponding features are missing in cont
299299
- [:nerd_face: nerdctl apparmor load](#nerd_face-nerdctl-apparmor-load)
300300
- [:nerd_face: nerdctl apparmor ls](#nerd_face-nerdctl-apparmor-ls)
301301
- [:nerd_face: nerdctl apparmor unload](#nerd_face-nerdctl-apparmor-unload)
302+
- [Builder management](#builder-management)
303+
- [:whale: nerdctl builder prune](#whale-nerdctl-builder-prune)
302304
- [System](#system)
303305
- [:whale: nerdctl events](#whale-nerdctl-events)
304306
- [:whale: nerdctl info](#whale-nerdctl-info)
@@ -1116,6 +1118,19 @@ Unload an AppArmor profile. The target profile name defaults to "nerdctl-default
11161118

11171119
Usage: `nerdctl apparmor unload [PROFILE]`
11181120

1121+
## Builder management
1122+
### :whale: nerdctl builder prune
1123+
Clean up BuildKit build cache.
1124+
1125+
:warning: The output format is not compatible with Docker.
1126+
1127+
Usage: `nerdctl builder prune`
1128+
1129+
Flags:
1130+
- :nerd_face: `--buildkit-host=<BUILDKIT_HOST>`: BuildKit address
1131+
1132+
Unimplemented `docker builder prune` flags: `--all`, `--filter`, `--force`, `--keep-storage`
1133+
11191134
## System
11201135
### :whale: nerdctl events
11211136
Get real time events from the server.

‎cmd/nerdctl/build.go

Lines changed: 84 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828

2929
"path/filepath"
3030

31+
dockerreference "github.com/containerd/containerd/reference/docker"
3132
"github.com/containerd/nerdctl/pkg/buildkitutil"
3233
"github.com/containerd/nerdctl/pkg/defaults"
3334
"github.com/containerd/nerdctl/pkg/platformutil"
@@ -73,22 +74,59 @@ func newBuildCommand() *cobra.Command {
7374
return buildCommand
7475
}
7576

77+
func getBuildkitHost(cmd *cobra.Command) (string, error) {
78+
if cmd.Flags().Changed("buildkit-host") {
79+
// If address is explicitly specified, use it.
80+
buildkitHost, err := cmd.Flags().GetString("buildkit-host")
81+
if err != nil {
82+
return "", err
83+
}
84+
if err := buildkitutil.PingBKDaemon(buildkitHost); err != nil {
85+
return "", err
86+
}
87+
return buildkitHost, nil
88+
}
89+
ns, err := cmd.Flags().GetString("namespace")
90+
if err != nil {
91+
return "", err
92+
}
93+
return buildkitutil.GetBuildkitHost(ns)
94+
}
95+
96+
func isImageSharable(buildkitHost string, namespace, uuid string) (bool, error) {
97+
labels, err := buildkitutil.GetWorkerLabels(buildkitHost)
98+
if err != nil {
99+
return false, err
100+
}
101+
logrus.Debugf("worker labels: %+v", labels)
102+
executor, ok := labels["org.mobyproject.buildkit.worker.executor"]
103+
if !ok {
104+
return false, nil
105+
}
106+
containerdUUID, ok := labels["org.mobyproject.buildkit.worker.containerd.uuid"]
107+
if !ok {
108+
return false, nil
109+
}
110+
containerdNamespace, ok := labels["org.mobyproject.buildkit.worker.containerd.namespace"]
111+
if !ok {
112+
return false, nil
113+
}
114+
return executor == "containerd" && containerdUUID == uuid && containerdNamespace == namespace, nil
115+
}
116+
76117
func buildAction(cmd *cobra.Command, args []string) error {
77118
platform, err := cmd.Flags().GetStringSlice("platform")
78119
if err != nil {
79120
return err
80121
}
81122
platform = strutil.DedupeStrSlice(platform)
82123

83-
buildkitHost, err := cmd.Flags().GetString("buildkit-host")
124+
buildkitHost, err := getBuildkitHost(cmd)
84125
if err != nil {
85126
return err
86127
}
87-
if err := buildkitutil.PingBKDaemon(buildkitHost); err != nil {
88-
return err
89-
}
90128

91-
buildctlBinary, buildctlArgs, needsLoading, metaFile, cleanup, err := generateBuildctlArgs(cmd, platform, args)
129+
buildctlBinary, buildctlArgs, needsLoading, metaFile, cleanup, err := generateBuildctlArgs(cmd, buildkitHost, platform, args)
92130
if err != nil {
93131
return err
94132
}
@@ -164,7 +202,7 @@ func buildAction(cmd *cobra.Command, args []string) error {
164202
return nil
165203
}
166204

167-
func generateBuildctlArgs(cmd *cobra.Command, platform, args []string) (string, []string, bool, string, func(), error) {
205+
func generateBuildctlArgs(cmd *cobra.Command, buildkitHost string, platform, args []string) (string, []string, bool, string, func(), error) {
168206
var needsLoading bool
169207
if len(args) < 1 {
170208
return "", nil, false, "", nil, errors.New("context needs to be specified")
@@ -184,13 +222,34 @@ func generateBuildctlArgs(cmd *cobra.Command, platform, args []string) (string,
184222
return "", nil, false, "", nil, err
185223
}
186224
if output == "" {
187-
output = "type=docker"
188-
if len(platform) > 1 {
189-
// For avoiding `error: failed to solve: docker exporter does not currently support exporting manifest lists`
190-
// TODO: consider using type=oci for single-platform build too
191-
output = "type=oci"
225+
client, ctx, cancel, err := newClient(cmd)
226+
if err != nil {
227+
return "", nil, false, "", nil, err
228+
}
229+
defer cancel()
230+
info, err := client.Server(ctx)
231+
if err != nil {
232+
return "", nil, false, "", nil, err
233+
}
234+
ns, err := cmd.Flags().GetString("namespace")
235+
if err != nil {
236+
return "", nil, false, "", nil, err
237+
}
238+
sharable, err := isImageSharable(buildkitHost, ns, info.UUID)
239+
if err != nil {
240+
return "", nil, false, "", nil, err
241+
}
242+
if sharable {
243+
output = "type=image,unpack=true" // ensure the target stage is unlazied (needed for any snapshotters)
244+
} else {
245+
output = "type=docker"
246+
if len(platform) > 1 {
247+
// For avoiding `error: failed to solve: docker exporter does not currently support exporting manifest lists`
248+
// TODO: consider using type=oci for single-platform build too
249+
output = "type=oci"
250+
}
251+
needsLoading = true
192252
}
193-
needsLoading = true
194253
}
195254
tagValue, err := cmd.Flags().GetStringArray("tag")
196255
if err != nil {
@@ -200,12 +259,12 @@ func generateBuildctlArgs(cmd *cobra.Command, platform, args []string) (string,
200259
if len(tagSlice) > 1 {
201260
return "", nil, false, "", nil, fmt.Errorf("specifying multiple -t is not supported yet")
202261
}
203-
output += ",name=" + tagSlice[0]
204-
}
205-
206-
buildkitHost, err := cmd.Flags().GetString("buildkit-host")
207-
if err != nil {
208-
return "", nil, false, "", nil, err
262+
ref := tagSlice[0]
263+
named, err := dockerreference.ParseNormalizedNamed(ref)
264+
if err != nil {
265+
return "", nil, false, "", nil, err
266+
}
267+
output += ",name=" + dockerreference.TagNameOnly(named).String()
209268
}
210269

211270
buildctlArgs := buildkitutil.BuildctlBaseArgs(buildkitHost)
@@ -375,14 +434,19 @@ func getDigestFromMetaFile(path string) (string, error) {
375434
}
376435
defer os.Remove(path)
377436

378-
metadata := map[string]string{}
437+
metadata := map[string]json.RawMessage{}
379438
if err := json.Unmarshal(data, &metadata); err != nil {
380439
logrus.WithError(err).Errorf("failed to unmarshal metadata file %s", path)
381440
return "", err
382441
}
383-
digest, ok := metadata["containerimage.digest"]
442+
digestRaw, ok := metadata["containerimage.digest"]
384443
if !ok {
385444
return "", errors.New("failed to find containerimage.digest in metadata file")
386445
}
446+
var digest string
447+
if err := json.Unmarshal(digestRaw, &digest); err != nil {
448+
logrus.WithError(err).Errorf("failed to unmarshal digset")
449+
return "", err
450+
}
387451
return digest, nil
388452
}

‎cmd/nerdctl/build_test.go

Lines changed: 79 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,9 @@ import (
2828
)
2929

3030
func TestBuild(t *testing.T) {
31-
t.Parallel()
3231
testutil.RequiresBuild(t)
3332
base := testutil.NewBase(t)
33+
defer base.Cmd("builder", "prune").Run()
3434
imageName := testutil.Identifier(t)
3535
defer base.Cmd("rmi", imageName).Run()
3636

@@ -48,25 +48,100 @@ CMD ["echo", "nerdctl-build-test-string"]
4848
base.Cmd("run", "--rm", imageName).AssertOutExactly("nerdctl-build-test-string\n")
4949
}
5050

51+
// TestBuildBaseImage tests if an image can be built on the previously built image.
52+
// This isn't currently supported by nerdctl with BuildKit OCI worker.
53+
func TestBuildBaseImage(t *testing.T) {
54+
testutil.RequiresBuild(t)
55+
base := testutil.NewBase(t)
56+
defer base.Cmd("builder", "prune").Run()
57+
imageName := testutil.Identifier(t)
58+
defer base.Cmd("rmi", imageName).Run()
59+
imageName2 := imageName + "-2"
60+
defer base.Cmd("rmi", imageName2).Run()
61+
62+
dockerfile := fmt.Sprintf(`FROM %s
63+
RUN echo hello > /hello
64+
CMD ["echo", "nerdctl-build-test-string"]
65+
`, testutil.CommonImage)
66+
67+
buildCtx, err := createBuildContext(dockerfile)
68+
assert.NilError(t, err)
69+
defer os.RemoveAll(buildCtx)
70+
71+
base.Cmd("build", "-t", imageName, buildCtx).AssertOK()
72+
base.Cmd("build", buildCtx, "-t", imageName).AssertOK()
73+
74+
dockerfile2 := fmt.Sprintf(`FROM %s
75+
RUN echo hello2 > /hello2
76+
CMD ["cat", "/hello2"]
77+
`, imageName)
78+
79+
buildCtx2, err := createBuildContext(dockerfile2)
80+
assert.NilError(t, err)
81+
defer os.RemoveAll(buildCtx2)
82+
83+
base.Cmd("build", "-t", imageName2, buildCtx2).AssertOK()
84+
base.Cmd("build", buildCtx2, "-t", imageName2).AssertOK()
85+
86+
base.Cmd("run", "--rm", imageName2).AssertOutExactly("hello2\n")
87+
}
88+
89+
// TestBuildFromContainerd tests if an image can be built on an image pulled by nerdctl.
90+
// This isn't currently supported by nerdctl with BuildKit OCI worker.
91+
func TestBuildFromContainerd(t *testing.T) {
92+
testutil.DockerIncompatible(t)
93+
testutil.RequiresBuild(t)
94+
base := testutil.NewBase(t)
95+
defer base.Cmd("builder", "prune").Run()
96+
imageName := testutil.Identifier(t)
97+
defer base.Cmd("rmi", imageName).Run()
98+
imageName2 := imageName + "-2"
99+
defer base.Cmd("rmi", imageName2).Run()
100+
101+
// FIXME: BuildKit sometimes tries to use base image manifests of platforms that hasn't been
102+
// pulled by `nerdctl pull`. This leads to "not found" error for the base image.
103+
// To avoid this issue, images shared to BuildKit should always be pulled by manifest
104+
// digest or `--all-platforms` needs to be added.
105+
base.Cmd("pull", "--all-platforms", testutil.CommonImage).AssertOK()
106+
base.Cmd("tag", testutil.CommonImage, imageName).AssertOK()
107+
base.Cmd("rmi", testutil.CommonImage).AssertOK()
108+
109+
dockerfile2 := fmt.Sprintf(`FROM %s
110+
RUN echo hello2 > /hello2
111+
CMD ["cat", "/hello2"]
112+
`, imageName)
113+
114+
buildCtx2, err := createBuildContext(dockerfile2)
115+
assert.NilError(t, err)
116+
defer os.RemoveAll(buildCtx2)
117+
118+
base.Cmd("build", "-t", imageName2, buildCtx2).AssertOK()
119+
base.Cmd("build", buildCtx2, "-t", imageName2).AssertOK()
120+
121+
base.Cmd("run", "--rm", imageName2).AssertOutExactly("hello2\n")
122+
}
123+
51124
func TestBuildFromStdin(t *testing.T) {
52125
t.Parallel()
53126
testutil.RequiresBuild(t)
54127
base := testutil.NewBase(t)
128+
defer base.Cmd("builder", "prune").Run()
55129
imageName := testutil.Identifier(t)
56130
defer base.Cmd("rmi", imageName).Run()
57131

58132
dockerfile := fmt.Sprintf(`FROM %s
59133
CMD ["echo", "nerdctl-build-test-stdin"]
60134
`, testutil.CommonImage)
61135

62-
base.Cmd("build", "-t", imageName, "-f", "-", ".").CmdOption(testutil.WithStdin(strings.NewReader(dockerfile))).AssertOutContains(imageName)
136+
base.Cmd("build", "-t", imageName, "-f", "-", ".").CmdOption(testutil.WithStdin(strings.NewReader(dockerfile))).AssertCombinedOutContains(imageName)
63137
}
64138

65139
func TestBuildLocal(t *testing.T) {
66140
t.Parallel()
67141
testutil.DockerIncompatible(t)
68142
testutil.RequiresBuild(t)
69143
base := testutil.NewBase(t)
144+
defer base.Cmd("builder", "prune").Run()
70145
const testFileName = "nerdctl-build-test"
71146
const testContent = "nerdctl"
72147
outputDir := t.TempDir()
@@ -108,6 +183,7 @@ func TestBuildWithIIDFile(t *testing.T) {
108183
t.Parallel()
109184
testutil.RequiresBuild(t)
110185
base := testutil.NewBase(t)
186+
defer base.Cmd("builder", "prune").Run()
111187
imageName := testutil.Identifier(t)
112188
defer base.Cmd("rmi", imageName).Run()
113189

@@ -134,6 +210,7 @@ func TestBuildWithLabels(t *testing.T) {
134210
t.Parallel()
135211
testutil.RequiresBuild(t)
136212
base := testutil.NewBase(t)
213+
defer base.Cmd("builder", "prune").Run()
137214
imageName := testutil.Identifier(t)
138215

139216
dockerfile := fmt.Sprintf(`FROM %s

0 commit comments

Comments
 (0)