Skip to content

Commit fabe516

Browse files
author
MarcoFalke
committed
ci: Export the container id in python script
This refactor does not change behavior, but it has a few benefits: * The shellcheck SC2086 warning is disabled for the whole command, but is only needed for the CI_CONTAINER_CAP env var. So in Python, only pass this one env var to shlex.split() for proper word splitting. * Future logic improvements can be implemented in Python. The comments are moved, which can be checked via the git options: --color-moved=dimmed-zebra --color-moved-ws=ignore-all-space
1 parent fa6aa9f commit fabe516

File tree

2 files changed

+74
-66
lines changed

2 files changed

+74
-66
lines changed

‎ci/test/02_run_container.py‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# Distributed under the MIT software license, see the accompanying
44
# file COPYING or https://opensource.org/license/mit/.
55

6+
from pathlib import Path
67
import os
78
import shlex
89
import subprocess
@@ -71,6 +72,79 @@ def main():
7172
time.sleep(3)
7273
run(cmd_build)
7374

75+
for suffix in ["ccache", "depends", "depends_sources", "previous_releases"]:
76+
run(["docker", "volume", "create", f"{os.environ['CONTAINER_NAME']}_{suffix}"], check=False)
77+
78+
CI_CCACHE_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_ccache,dst={os.environ['CCACHE_DIR']}"
79+
CI_DEPENDS_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends,dst={os.environ['DEPENDS_DIR']}/built"
80+
CI_DEPENDS_SOURCES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends_sources,dst={os.environ['DEPENDS_DIR']}/sources"
81+
CI_PREVIOUS_RELEASES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_previous_releases,dst={os.environ['PREVIOUS_RELEASES_DIR']}"
82+
CI_BUILD_MOUNT = []
83+
84+
if os.getenv("DANGER_CI_ON_HOST_FOLDERS"):
85+
# ensure the directories exist
86+
for create_dir in [
87+
os.environ["CCACHE_DIR"],
88+
f"{os.environ['DEPENDS_DIR']}/built",
89+
f"{os.environ['DEPENDS_DIR']}/sources",
90+
os.environ["PREVIOUS_RELEASES_DIR"],
91+
os.environ["BASE_BUILD_DIR"], # Unset by default, must be defined externally
92+
]:
93+
Path(create_dir).mkdir(parents=True, exist_ok=True)
94+
95+
CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"
96+
CI_DEPENDS_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/built,dst={os.environ['DEPENDS_DIR']}/built"
97+
CI_DEPENDS_SOURCES_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/sources,dst={os.environ['DEPENDS_DIR']}/sources"
98+
CI_PREVIOUS_RELEASES_MOUNT = f"type=bind,src={os.environ['PREVIOUS_RELEASES_DIR']},dst={os.environ['PREVIOUS_RELEASES_DIR']}"
99+
CI_BUILD_MOUNT = [f"--mount=type=bind,src={os.environ['BASE_BUILD_DIR']},dst={os.environ['BASE_BUILD_DIR']}"]
100+
101+
if os.getenv("DANGER_CI_ON_HOST_CCACHE_FOLDER"):
102+
if not os.path.isdir(os.environ["CCACHE_DIR"]):
103+
print(f"Error: Directory '{os.environ['CCACHE_DIR']}' must be created in advance.")
104+
sys.exit(1)
105+
CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"
106+
107+
run(["docker", "network", "create", "--ipv6", "--subnet", "1111:1111::/112", "ci-ip6net"], check=False)
108+
109+
if os.getenv("RESTART_CI_DOCKER_BEFORE_RUN"):
110+
print("Restart docker before run to stop and clear all containers started with --rm")
111+
run(["podman", "container", "rm", "--force", "--all"]) # Similar to "systemctl restart docker"
112+
113+
# Still prune everything in case the filtered pruning doesn't work, or if labels were not set
114+
# on a previous run. Belt and suspenders approach, should be fine to remove in the future.
115+
# Prune images used by --external containers (e.g. build containers) when
116+
# using podman.
117+
print("Prune all dangling images")
118+
run(["podman", "image", "prune", "--force", "--external"])
119+
120+
print(f"Prune all dangling {CI_IMAGE_LABEL} images")
121+
# When detecting podman-docker, `--external` should be added.
122+
run(["docker", "image", "prune", "--force", "--filter", f"label={CI_IMAGE_LABEL}"])
123+
124+
cmd_run = ["docker", "run", "--rm", "--interactive", "--detach", "--tty"]
125+
cmd_run += [
126+
"--cap-add=LINUX_IMMUTABLE",
127+
*shlex.split(os.getenv("CI_CONTAINER_CAP", "")),
128+
f"--mount=type=bind,src={os.environ['BASE_READ_ONLY_DIR']},dst={os.environ['BASE_READ_ONLY_DIR']},readonly",
129+
f"--mount={CI_CCACHE_MOUNT}",
130+
f"--mount={CI_DEPENDS_MOUNT}",
131+
f"--mount={CI_DEPENDS_SOURCES_MOUNT}",
132+
f"--mount={CI_PREVIOUS_RELEASES_MOUNT}",
133+
*CI_BUILD_MOUNT,
134+
f"--env-file={env_file}",
135+
f"--name={os.environ['CONTAINER_NAME']}",
136+
"--network=ci-ip6net",
137+
f"--platform={os.environ['CI_IMAGE_PLATFORM']}",
138+
os.environ["CONTAINER_NAME"],
139+
]
140+
141+
container_id = run(
142+
cmd_run,
143+
stdout=subprocess.PIPE,
144+
text=True,
145+
).stdout.strip()
146+
os.environ["CI_CONTAINER_ID"] = container_id
147+
74148
run(["./ci/test/02_run_container.sh"]) # run the remainder
75149

76150

‎ci/test/02_run_container.sh‎

Lines changed: 0 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -5,76 +5,10 @@
55
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
66

77
export LC_ALL=C.UTF-8
8-
export CI_IMAGE_LABEL="bitcoin-ci-test"
98

109
set -o errexit -o pipefail -o xtrace
1110

1211
if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
13-
docker volume create "${CONTAINER_NAME}_ccache" || true
14-
docker volume create "${CONTAINER_NAME}_depends" || true
15-
docker volume create "${CONTAINER_NAME}_depends_sources" || true
16-
docker volume create "${CONTAINER_NAME}_previous_releases" || true
17-
18-
CI_CCACHE_MOUNT="type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR"
19-
CI_DEPENDS_MOUNT="type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR/built"
20-
CI_DEPENDS_SOURCES_MOUNT="type=volume,src=${CONTAINER_NAME}_depends_sources,dst=$DEPENDS_DIR/sources"
21-
CI_PREVIOUS_RELEASES_MOUNT="type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR"
22-
CI_BUILD_MOUNT=""
23-
24-
if [ "$DANGER_CI_ON_HOST_FOLDERS" ]; then
25-
# ensure the directories exist
26-
mkdir -p "${CCACHE_DIR}"
27-
mkdir -p "${DEPENDS_DIR}/built"
28-
mkdir -p "${DEPENDS_DIR}/sources"
29-
mkdir -p "${PREVIOUS_RELEASES_DIR}"
30-
mkdir -p "${BASE_BUILD_DIR}" # Unset by default, must be defined externally
31-
32-
CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=$CCACHE_DIR"
33-
CI_DEPENDS_MOUNT="type=bind,src=${DEPENDS_DIR}/built,dst=$DEPENDS_DIR/built"
34-
CI_DEPENDS_SOURCES_MOUNT="type=bind,src=${DEPENDS_DIR}/sources,dst=$DEPENDS_DIR/sources"
35-
CI_PREVIOUS_RELEASES_MOUNT="type=bind,src=${PREVIOUS_RELEASES_DIR},dst=$PREVIOUS_RELEASES_DIR"
36-
CI_BUILD_MOUNT="--mount type=bind,src=${BASE_BUILD_DIR},dst=${BASE_BUILD_DIR}"
37-
fi
38-
39-
if [ "$DANGER_CI_ON_HOST_CCACHE_FOLDER" ]; then
40-
if [ ! -d "${CCACHE_DIR}" ]; then
41-
echo "Error: Directory '${CCACHE_DIR}' must be created in advance."
42-
exit 1
43-
fi
44-
CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=${CCACHE_DIR}"
45-
fi
46-
47-
docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true
48-
49-
if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then
50-
echo "Restart docker before run to stop and clear all containers started with --rm"
51-
podman container rm --force --all # Similar to "systemctl restart docker"
52-
53-
# Still prune everything in case the filtered pruning doesn't work, or if labels were not set
54-
# on a previous run. Belt and suspenders approach, should be fine to remove in the future.
55-
# Prune images used by --external containers (e.g. build containers) when
56-
# using podman.
57-
echo "Prune all dangling images"
58-
podman image prune --force --external
59-
fi
60-
echo "Prune all dangling $CI_IMAGE_LABEL images"
61-
# When detecting podman-docker, `--external` should be added.
62-
docker image prune --force --filter "label=$CI_IMAGE_LABEL"
63-
64-
# shellcheck disable=SC2086
65-
CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \
66-
--mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \
67-
--mount "${CI_CCACHE_MOUNT}" \
68-
--mount "${CI_DEPENDS_MOUNT}" \
69-
--mount "${CI_DEPENDS_SOURCES_MOUNT}" \
70-
--mount "${CI_PREVIOUS_RELEASES_MOUNT}" \
71-
${CI_BUILD_MOUNT} \
72-
--env-file /tmp/env-$USER-$CONTAINER_NAME \
73-
--name "$CONTAINER_NAME" \
74-
--network ci-ip6net \
75-
--platform="${CI_IMAGE_PLATFORM}" \
76-
"$CONTAINER_NAME")
77-
export CI_CONTAINER_ID
7812
export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}"
7913
else
8014
echo "Running on host system without docker wrapper"

0 commit comments

Comments
 (0)