Skip to content

Commit 245d100

Browse files
authored
devops: produce ffmpeg builds on bots (microsoft#3820)
This patch moves FFMPEG building to buildbots: - `ffmpeg-mac.zip` is built on Mac 10.14 machine - `ffmpeg-win32.zip` and `ffmpeg-win64.zip` are cross-compiled on Ubuntu 20.04 machine All builds across the platforms share the same config: - the same versions of `ffmpeg` and `libvpx` - the same build configuration for both `ffmpeg` and `libvpx` The config could be found in the `//browser_patches/ffmpeg/CONFIG.sh`. The builds will be then copied manually and committed to the git repository.
1 parent 8a339be commit 245d100

15 files changed

+462
-6
lines changed

browser_patches/buildbots/README.md

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,9 +240,16 @@ $ sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-9
240240
$ sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-9 100
241241
```
242242

243+
2. FFMPEG cross-compilation requires Docker. Install docker and add `$USER` to docker for sudo-less docker access
244+
245+
```sh
246+
$ sudo apt-get install -y docker.io # install docker
247+
$ sudo usermod -aG docker $USER # add user to docker group
248+
$ newgrp docker # activate group changes
249+
```
250+
243251
> **NOTE**: Firefox build config can be checked official Firefox builds, navigating to `about:buildconfig` URL.
244252
245253
To document precisely my steps to bring up bots:
246254
- [July 22, 2020: Setting up Ubuntu 18.04 buildbot on Azure](https://gist.github.com/aslushnikov/a4a3823b894888546e741899e69a1d8e)
247255
- [July 22, 2020: Setting up Ubuntu 20.04 buildbot on Azure](https://gist.github.com/aslushnikov/a0bd658b575022e198443f856b5185e7)
248-

browser_patches/buildbots/buildbot-mac-10.14.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,6 @@ git pull origin master
6565

6666
git pull origin master
6767
../checkout_build_archive_upload.sh webkit-mac-10.14 >/tmp/$(basename $0)--webkit-mac-10.14.log || true
68+
69+
git pull origin master
70+
../checkout_build_archive_upload.sh ffmpeg-mac >/tmp/$(basename $0)--ffmpeg-mac.log || true

browser_patches/buildbots/buildbot-ubuntu-20.04.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,3 +64,9 @@ touch "$IS_FIRST_RUN_FILE"
6464

6565
git pull origin master
6666
../checkout_build_archive_upload.sh webkit-ubuntu-20.04 >/tmp/$(basename $0)--webkit.log || true
67+
68+
git pull origin master
69+
../checkout_build_archive_upload.sh ffmpeg-cross-compile-win32 >/tmp/$(basename $0)--ffmpeg-cross-compile-win32.log || true
70+
71+
git pull origin master
72+
../checkout_build_archive_upload.sh ffmpeg-cross-compile-win64 >/tmp/$(basename $0)--ffmpeg-cross-compile-win64.log || true

browser_patches/checkout_build_archive_upload.sh

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,25 @@ BUILD_FLAVOR="$1"
3636
BUILD_BLOB_NAME=""
3737
EXPECTED_HOST_OS=""
3838
EXPECTED_HOST_OS_VERSION=""
39-
if [[ "$BUILD_FLAVOR" == "chromium-linux-mirror-to-cdn" ]]; then
39+
if [[ "$BUILD_FLAVOR" == "ffmpeg-mac" ]]; then
40+
BROWSER_NAME="ffmpeg"
41+
EXTRA_BUILD_ARGS="--mac"
42+
EXPECTED_HOST_OS="Darwin"
43+
EXPECTED_HOST_OS_VERSION="10.14"
44+
BUILD_BLOB_NAME="ffmpeg-mac.zip"
45+
elif [[ "$BUILD_FLAVOR" == "ffmpeg-cross-compile-win32" ]]; then
46+
BROWSER_NAME="ffmpeg"
47+
EXTRA_BUILD_ARGS="--cross-compile-win32"
48+
EXPECTED_HOST_OS="Ubuntu"
49+
EXPECTED_HOST_OS_VERSION="20.04"
50+
BUILD_BLOB_NAME="ffmpeg-win32.zip"
51+
elif [[ "$BUILD_FLAVOR" == "ffmpeg-cross-compile-win64" ]]; then
52+
BROWSER_NAME="ffmpeg"
53+
EXTRA_BUILD_ARGS="--cross-compile-win64"
54+
EXPECTED_HOST_OS="Ubuntu"
55+
EXPECTED_HOST_OS_VERSION="20.04"
56+
BUILD_BLOB_NAME="ffmpeg-win64.zip"
57+
elif [[ "$BUILD_FLAVOR" == "chromium-linux-mirror-to-cdn" ]]; then
4058
BROWSER_NAME="chromium"
4159
EXTRA_BUILD_ARGS="--linux"
4260
EXPECTED_HOST_OS="Ubuntu"

browser_patches/ffmpeg/.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
build/
2+
output/
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
1000

browser_patches/ffmpeg/CONFIG.sh

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Copyright (c) Microsoft Corporation.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the 'License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
LIBVPX_VERSION="v1.9.0"
16+
LIBVPX_CONFIG="--enable-static \
17+
--disable-shared \
18+
--disable-docs \
19+
--disable-tools \
20+
--disable-unit-tests \
21+
--disable-examples"
22+
23+
FFMPEG_VERSION="n4.3.1"
24+
FFMPEG_CONFIG="--enable-gpl \
25+
--enable-version3 \
26+
--disable-debug \
27+
--disable-everything \
28+
--enable-ffmpeg \
29+
--enable-protocol=pipe \
30+
--enable-protocol=file \
31+
--enable-parser=mjpeg \
32+
--enable-decoder=mjpeg \
33+
--enable-demuxer=image2pipe \
34+
--enable-filter=pad \
35+
--enable-filter=crop \
36+
--enable-filter=scale \
37+
--enable-muxer=webm \
38+
--enable-libvpx \
39+
--enable-static \
40+
--enable-encoder=libvpx_vp8 \
41+
--disable-pthreads \
42+
--disable-zlib \
43+
--disable-iconv \
44+
--disable-w32threads \
45+
--disable-bzlib"
46+

browser_patches/ffmpeg/README.md

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Playwright and FFMPEG
2+
3+
Playwright requires FFMPEG to produce screncast. Playwright relies on stock
4+
FFMPEG on Ubuntu, and bundles FFMPEG binaries for Mac and Windows.
5+
6+
## Configuration
7+
8+
We compile `libvpx` and `ffmpeg` only. Their source versions and build
9+
configurations are defined in [`//browser_patches/ffmpeg/CONFIG.sh`](./CONFIG.sh).
10+
11+
## Building `ffmpeg-mac`
12+
13+
Cross-compilation scripts are based on:
14+
- https://trac.ffmpeg.org/wiki/CompilationGuide/Generic
15+
- https://trac.ffmpeg.org/wiki/CompilationGuide/macOS
16+
17+
Prerequisites:
18+
- Mac
19+
- xcode command line tools: `xcode-select --install`
20+
- [homebrew](https://brew.sh/)
21+
22+
Building:
23+
24+
```
25+
~/playwright$ ./browser_patches/ffmpeg/build.sh --mac
26+
```
27+
28+
## Building `ffmpeg-win*`
29+
30+
Cross-compilation scripts are based on:
31+
- https://trac.ffmpeg.org/wiki/CompilationGuide/Generic
32+
- https://trac.ffmpeg.org/wiki/CompilationGuide/CrossCompilingForWindows
33+
34+
Prerequisites:
35+
- Mac or Linux
36+
- [Docker](https://www.docker.com/)
37+
38+
Building:
39+
40+
```
41+
~/playwright$ ./browser_patches/ffmpeg/build.sh --cross-compile-win32
42+
~/playwright$ ./browser_patches/ffmpeg/build.sh --cross-compile-win64
43+
```
44+

browser_patches/ffmpeg/archive.sh

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
set -e
3+
set +x
4+
5+
if [[ ("$1" == "-h") || ("$1" == "--help") ]]; then
6+
echo "usage: $(basename $0) [output-absolute-path]"
7+
echo
8+
echo "Generate distributable .zip archive from ./output folder that was previously built."
9+
echo
10+
exit 0
11+
fi
12+
13+
ZIP_PATH=$1
14+
if [[ $ZIP_PATH != /* ]]; then
15+
echo "ERROR: path $ZIP_PATH is not absolute"
16+
exit 1
17+
fi
18+
if [[ $ZIP_PATH != *.zip ]]; then
19+
echo "ERROR: path $ZIP_PATH must have .zip extension"
20+
exit 1
21+
fi
22+
if [[ -f $ZIP_PATH ]]; then
23+
echo "ERROR: path $ZIP_PATH exists; can't do anything."
24+
exit 1
25+
fi
26+
if ! [[ -d $(dirname $ZIP_PATH) ]]; then
27+
echo "ERROR: folder for path $($ZIP_PATH) does not exist."
28+
exit 1
29+
fi
30+
31+
trap "cd $(pwd -P)" EXIT
32+
cd "$(dirname $0)"
33+
34+
cp output/ffmpeg.zip $ZIP_PATH
Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/bin/bash
2+
3+
# Copyright (c) Microsoft Corporation.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the 'License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -e
18+
19+
function die() { echo "$@"; exit 1; }
20+
21+
if [[ "$(uname)" != "Darwin" ]]; then
22+
die "ERROR: this script is designed to be run on OSX. Can't run on $(uname)"
23+
fi
24+
25+
trap "cd $(pwd -P)" EXIT
26+
cd "$(dirname $0)"
27+
28+
source ./CONFIG.sh
29+
30+
BUILDDIR="${PWD}/build"
31+
PREFIX="${BUILDDIR}/osx_prefix"
32+
OUTPUT_PATH="${PWD}/output/ffmpeg-mac"
33+
34+
function build_libvpx {
35+
cd "${BUILDDIR}"
36+
git clone https://chromium.googlesource.com/webm/libvpx
37+
cd libvpx
38+
git checkout "${LIBVPX_VERSION}"
39+
# Compile libvpx according to the docs:
40+
# - https://chromium.googlesource.com/webm/libvpx/+/master/README
41+
./configure --prefix="${PREFIX}" ${LIBVPX_CONFIG}
42+
make && make install
43+
}
44+
45+
function build_ffmpeg {
46+
cd "${BUILDDIR}"
47+
git clone git://source.ffmpeg.org/ffmpeg.git
48+
cd ffmpeg
49+
git checkout "${FFMPEG_VERSION}"
50+
export PKG_CONFIG_PATH="${PREFIX}/lib/pkgconfig"
51+
# Prohibit pkg-config from using system installed libs.
52+
export PKG_CONFIG_LIBDIR=
53+
54+
./configure --pkg-config=pkg-config \
55+
--pkg-config-flags="--static" \
56+
--extra-cflags="-I/${PREFIX}/include" \
57+
--extra-ldflags="-L/${PREFIX}/lib" \
58+
--prefix="${PREFIX}" \
59+
--bindir="${PWD}/bin" \
60+
${FFMPEG_CONFIG}
61+
make && make install
62+
}
63+
64+
REQUIERED_BUILD_TOOLS=("git" "make" "yasm" "pkg-config")
65+
missing_build_tools=()
66+
67+
for dependency in ${REQUIERED_BUILD_TOOLS[@]}; do
68+
if ! command -v "${dependency}" >/dev/null; then
69+
missing_build_tools+=("${dependency}")
70+
fi
71+
done
72+
73+
if [[ ${#missing_build_tools[@]} != 0 ]]; then
74+
die "ERROR: missing dependencies! Please run: brew install ${missing_build_tools[@]}"
75+
fi
76+
77+
# Cleanup
78+
set -x
79+
rm -rf "${BUILDDIR}"
80+
mkdir -p "${BUILDDIR}"
81+
82+
build_libvpx
83+
build_ffmpeg
84+
85+
# put resulting executable where we were asked to
86+
mkdir -p $(dirname "${OUTPUT_PATH}")
87+
cp "${BUILDDIR}/ffmpeg/bin/ffmpeg" "${OUTPUT_PATH}"
88+
strip "${OUTPUT_PATH}"

0 commit comments

Comments
 (0)