Skip to content

Commit 6f55484

Browse files
committed
build(aws-lambda): pinned native ffmpeg build for an arm64 Lambda image
One recipe, run per architecture on the Lambda Node 22 base it ships on. Codec libraries link statically; glibc and GnuTLS stay dynamic, with the two libraries the bare base lacks bundled beside the binaries. A check stage on the bare base proves every encoder, decoder, filter and protocol the engine calls, and the artifact cannot be produced without it. CI builds it on two ARM runners and requires byte-identical tarballs.
1 parent 2ef60f3 commit 6f55484

5 files changed

Lines changed: 231 additions & 0 deletions

File tree

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: Lambda arm64 ffmpeg
2+
3+
permissions:
4+
contents: read
5+
6+
# Builds the native ffmpeg for the arm64 Lambda video function on two separate ARM runners.
7+
# Each build runs the codec check on the bare Lambda base; the tarballs must be byte-identical.
8+
on:
9+
pull_request:
10+
paths:
11+
- "packages/aws-lambda/ffmpeg/**"
12+
- ".github/workflows/lambda-ffmpeg.yml"
13+
workflow_dispatch:
14+
15+
concurrency:
16+
group: lambda-ffmpeg-${{ github.ref }}
17+
cancel-in-progress: true
18+
19+
jobs:
20+
build:
21+
name: Build ${{ matrix.copy }}
22+
runs-on: ubuntu-24.04-arm
23+
timeout-minutes: 90
24+
strategy:
25+
matrix:
26+
copy: [a, b]
27+
steps:
28+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
29+
- name: Build and check
30+
run: >-
31+
docker buildx build --progress=plain --platform linux/arm64 --target artifact
32+
--output type=local,dest=out packages/aws-lambda/ffmpeg
33+
- run: cat out/hf-media-check.txt && sha256sum out/*.tar.gz
34+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
35+
with:
36+
name: hf-media-${{ matrix.copy }}
37+
path: out/
38+
retention-days: 7
39+
40+
reproducible:
41+
name: Builds match
42+
needs: build
43+
runs-on: ubuntu-latest
44+
timeout-minutes: 5
45+
steps:
46+
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
47+
- run: cmp hf-media-a/hf-media-linux-aarch64.tar.gz hf-media-b/hf-media-linux-aarch64.tar.gz
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Native ffmpeg for the Lambda video function, built on the Lambda Node 22 base it runs on:
2+
# docker buildx build --platform linux/arm64 --target artifact --output type=local,dest=out packages/aws-lambda/ffmpeg
3+
# The artifact stage depends on the check stage, so no tarball comes out of a build whose codecs fail.
4+
ARG BASE=public.ecr.aws/lambda/nodejs:22@sha256:d138eff445f6ea04c51091d8e6cbb064d5663e850cddfa62396061586f98b126
5+
6+
FROM ${BASE} AS build
7+
ENV SOURCE_DATE_EPOCH=1722643200
8+
RUN rpm -qa --qf '%{NAME}\n' | sort >/build-base-rpms.txt && \
9+
microdnf install -y --nodocs --setopt=install_weak_deps=0 --releasever=2023.12.20260914 \
10+
gcc gcc-c++ make cmake meson ninja-build autoconf automake libtool pkgconf-pkg-config diffutils \
11+
git xz bzip2 tar gzip which patchelf gnutls-devel zlib-devel $([ "$(uname -m)" = x86_64 ] && echo nasm) && \
12+
microdnf clean all && mkdir /build && mv /build-base-rpms.txt /build/base-rpms.txt
13+
COPY sources.txt build.sh /build/
14+
RUN bash /build/build.sh
15+
16+
FROM ${BASE} AS certs
17+
# certtool, not openssl: the base's openssl-libs cannot run the repo's openssl CLI.
18+
RUN microdnf install -y --nodocs --releasever=2023.12.20260914 gnutls-utils && mkdir /certs && cd /certs && \
19+
printf 'cn = check-ca\nca\ncert_signing_key\nexpiration_days = 2\n' >ca.tmpl && \
20+
printf 'cn = localhost\ndns_name = localhost\ntls_www_server\nexpiration_days = 2\n' >server.tmpl && \
21+
certtool -p --outfile ca.key && certtool -s --load-privkey ca.key --template ca.tmpl --outfile ca.crt && \
22+
certtool -p --outfile server.key && \
23+
certtool -c --load-privkey server.key --load-ca-certificate ca.crt --load-ca-privkey ca.key \
24+
--template server.tmpl --outfile server.crt
25+
26+
FROM ${BASE} AS check
27+
COPY --from=build /opt/hf-media /opt/hf-media
28+
COPY --from=certs /certs /certs
29+
COPY check.sh /check.sh
30+
RUN cp /certs/ca.crt /etc/pki/ca-trust/source/anchors/ && update-ca-trust && bash /check.sh >/check.txt
31+
32+
FROM scratch AS artifact
33+
COPY --from=build /out/ /
34+
COPY --from=check /check.txt /hf-media-check.txt
Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/bin/bash
2+
# Builds ffmpeg + ffprobe into /opt/hf-media and packs /out/hf-media-linux-<arch>.tar.gz.
3+
# Codec libraries link statically; glibc and the TLS stack stay dynamic (a static glibc build crashed on Lambda).
4+
set -euo pipefail
5+
6+
PREFIX=/opt/hf-media
7+
DEPS=/build/deps
8+
SRC=/build/src
9+
JOBS=$(nproc)
10+
export PKG_CONFIG_PATH="$DEPS/lib/pkgconfig:$DEPS/lib64/pkgconfig"
11+
export CFLAGS="-O2 -fPIC" CXXFLAGS="-O2 -fPIC"
12+
mkdir -p "$SRC" "$DEPS" "$PREFIX/share/licenses" /out
13+
14+
fetch() {
15+
local name=$1 sum url
16+
read -r sum url < <(awk -v n="$name" '$1 == n { print $2, $3 }' /build/sources.txt)
17+
curl -fsSL --retry 3 -o "$SRC/$name" "$url"
18+
echo "$sum $SRC/$name" | sha256sum -c --quiet
19+
tar -xf "$SRC/$name" -C "$SRC"
20+
}
21+
22+
license() { mkdir -p "$PREFIX/share/licenses/$1" && cp "${@:2}" "$PREFIX/share/licenses/$1/"; }
23+
24+
read -r x264_commit x264_url < <(awk '$1 == "x264" { print $2, $3 }' /build/sources.txt)
25+
git clone -q "$x264_url" "$SRC/x264"
26+
git -C "$SRC/x264" checkout -q "$x264_commit"
27+
[ "$(git -C "$SRC/x264" rev-parse HEAD)" = "$x264_commit" ]
28+
(cd "$SRC/x264" && ./configure --prefix="$DEPS" --enable-static --enable-pic --disable-cli &&
29+
make -j"$JOBS" && make install)
30+
license x264 "$SRC/x264/COPYING"
31+
32+
fetch x265_4.1.tar.gz
33+
cmake -S "$SRC/x265_4.1/source" -B "$SRC/x265-build" -DCMAKE_BUILD_TYPE=Release \
34+
-DCMAKE_INSTALL_PREFIX="$DEPS" -DENABLE_SHARED=OFF -DENABLE_CLI=OFF
35+
cmake --build "$SRC/x265-build" -j"$JOBS" && cmake --install "$SRC/x265-build"
36+
license x265 "$SRC/x265_4.1/COPYING"
37+
38+
fetch libvpx-1.15.0.tar.gz
39+
(cd "$SRC/libvpx-1.15.0" && ./configure --prefix="$DEPS" --enable-static --disable-shared --enable-pic \
40+
--enable-vp9-highbitdepth --disable-examples --disable-tools --disable-docs --disable-unit-tests &&
41+
make -j"$JOBS" && make install)
42+
license libvpx "$SRC/libvpx-1.15.0/LICENSE" "$SRC/libvpx-1.15.0/PATENTS"
43+
44+
fetch opus-1.5.2.tar.gz
45+
(cd "$SRC/opus-1.5.2" && ./configure --prefix="$DEPS" --enable-static --disable-shared --with-pic \
46+
--disable-doc --disable-extra-programs && make -j"$JOBS" && make install)
47+
license opus "$SRC/opus-1.5.2/COPYING"
48+
49+
fetch zimg-3.0.5.tar.gz
50+
(cd "$SRC/zimg-release-3.0.5" && ./autogen.sh && ./configure --prefix="$DEPS" --enable-static --disable-shared \
51+
--with-pic && make -j"$JOBS" && make install)
52+
license zimg "$SRC/zimg-release-3.0.5/COPYING"
53+
54+
fetch dav1d-1.5.1.tar.xz
55+
meson setup "$SRC/dav1d-build" "$SRC/dav1d-1.5.1" --prefix="$DEPS" --libdir=lib --buildtype=release \
56+
--default-library=static -Denable_tools=false -Denable_tests=false
57+
ninja -C "$SRC/dav1d-build" install
58+
license dav1d "$SRC/dav1d-1.5.1/COPYING"
59+
60+
fetch ffmpeg-7.0.2.tar.xz
61+
(cd "$SRC/ffmpeg-7.0.2" && ./configure --prefix="$PREFIX" --pkg-config-flags=--static \
62+
--extra-cflags="-I$DEPS/include" --extra-ldflags="-L$DEPS/lib" --extra-libs="-lstdc++ -lpthread -lm" \
63+
--enable-gpl --enable-version3 --disable-debug --disable-doc --disable-ffplay --disable-autodetect \
64+
--enable-gnutls --enable-zlib --enable-libx264 --enable-libx265 --enable-libvpx --enable-libopus \
65+
--enable-libzimg --enable-libdav1d && make -j"$JOBS" && make install)
66+
license ffmpeg "$SRC/ffmpeg-7.0.2/COPYING.GPLv3" "$SRC/ffmpeg-7.0.2/LICENSE.md"
67+
rm -rf "$PREFIX/include" "$PREFIX/lib" "$PREFIX/share/ffmpeg"
68+
strip "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe"
69+
70+
# Bundle every shared library the bare base image lacks (gnutls, nettle); the rpath finds them.
71+
mkdir -p "$PREFIX/lib"
72+
ldd "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe" | awk '$2 == "=>" { print $3 }' | sort -u | while read -r lib; do
73+
if ! grep -qx "$(rpm -qf --qf '%{NAME}' "$lib")" /build/base-rpms.txt; then
74+
cp -L "$lib" "$PREFIX/lib/" && patchelf --set-rpath '$ORIGIN' "$PREFIX/lib/$(basename "$lib")"
75+
echo "$(rpm -qf "$lib") $(basename "$lib")" >>"$PREFIX/share/licenses/bundled-libraries.txt"
76+
fi
77+
done
78+
patchelf --set-rpath '$ORIGIN/../lib' "$PREFIX/bin/ffmpeg" "$PREFIX/bin/ffprobe"
79+
cp /build/sources.txt "$PREFIX/share/licenses/sources.txt"
80+
81+
tar --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --numeric-owner -C /opt -cf - hf-media |
82+
gzip -n -9 >"/out/hf-media-linux-$(uname -m).tar.gz"
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/bin/bash
2+
# Runs on the bare Lambda base with only /opt/hf-media added: every codec the engine calls must work.
3+
# Prints a deterministic summary to stdout; the artifact stage keeps it.
4+
set -euo pipefail
5+
ff() { /opt/hf-media/bin/ffmpeg -hide_banner -loglevel error -nostdin "$@"; }
6+
probe() { /opt/hf-media/bin/ffprobe -hide_banner -loglevel error "$@"; }
7+
listed() { # listed <ffmpeg list flag> <awk field> <name...>
8+
local names
9+
names=$(/opt/hf-media/bin/ffmpeg -hide_banner "$1" | awk -v f="$2" '{ print $f }')
10+
for name in "${@:3}"; do grep -qx -- "$name" <<<"$names" || { echo "missing $1 $name" >&2; exit 1; }; done
11+
echo "${1#-}: ${*:3}"
12+
}
13+
frames() { probe -count_frames -select_streams v:0 -show_entries stream=nb_read_frames -of csv=p=0 "$@"; }
14+
15+
cd "$(mktemp -d)"
16+
listed -encoders 2 libx264 libx265 libvpx-vp9 libopus prores_ks aac png mjpeg
17+
listed -decoders 2 libvpx-vp9 libvpx libdav1d h264 hevc prores aac opus png mjpeg
18+
listed -filters 2 zscale tonemap setparams ebur128 alphamerge overlay
19+
listed -protocols 1 https tls file
20+
21+
src=(-f lavfi -i "testsrc2=s=320x240:r=25:d=1")
22+
ff "${src[@]}" -c:v libx264 -pix_fmt yuv420p -bf 0 h264.mp4
23+
ff "${src[@]}" -c:v libx265 -pix_fmt yuv420p -tag:v hvc1 -x265-params log-level=error h265.mp4
24+
ff "${src[@]}" -c:v prores_ks -profile:v 4444 -pix_fmt yuva444p10le prores.mov
25+
ff -f lavfi -i "sine=d=1" -c:a libopus -b:a 128k opus.webm
26+
ff -f lavfi -i "sine=d=1" -c:a aac -b:a 192k aac.m4a
27+
ff "${src[@]}" -frames:v 1 frame.png
28+
for f in h264.mp4 h265.mp4 prores.mov; do [ "$(frames "$f")" = 25 ] || { echo "decode $f" >&2; exit 1; }; done
29+
for f in opus.webm aac.m4a; do ff -i "$f" -f null -; done
30+
[ "$(probe -show_entries stream=width -of csv=p=0 frame.png)" = 320 ]
31+
echo "encode+decode: h264 h265 prores opus aac png"
32+
33+
# VP9 with alpha: left half transparent, decoded through libvpx-vp9 as the engine does.
34+
ff "${src[@]}" -vf "format=yuva420p,geq=lum='lum(X,Y)':cb='cb(X,Y)':cr='cr(X,Y)':a='if(lt(X,W/2),0,255)'" \
35+
-c:v libvpx-vp9 -pix_fmt yuva420p -auto-alt-ref 0 -b:v 0 -crf 30 alpha.webm
36+
ff -c:v libvpx-vp9 -i alpha.webm -frames:v 1 -vf alphaextract,crop=40:40:20:100 -f rawvideo -pix_fmt gray left.raw
37+
ff -c:v libvpx-vp9 -i alpha.webm -frames:v 1 -vf alphaextract,crop=40:40:260:100 -f rawvideo -pix_fmt gray right.raw
38+
node -e '
39+
const fs = require("fs"), mean = (f) => fs.readFileSync(f).reduce((a, b) => a + b, 0) / 1600;
40+
if (!(mean("left.raw") < 16 && mean("right.raw") > 240)) throw new Error("vp9 alpha lost");'
41+
echo "vp9 alpha: encode+decode"
42+
43+
# The engine's HDR to SDR chain (core/src/hdrToneMap.ts) on tagged 10-bit HEVC.
44+
ff "${src[@]}" -c:v libx265 -pix_fmt yuv420p10le -color_primaries bt2020 -color_trc smpte2084 \
45+
-colorspace bt2020nc -x265-params log-level=error hdr.mp4
46+
ff -i hdr.mp4 -vf "zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=bt709:m=bt709:r=tv" \
47+
-f null -
48+
echo "hdr tonemap: zscale"
49+
50+
# TLS against the system trust store: a trusted host passes, a wrong host name is refused.
51+
exec 4< <(node -e '
52+
const fs = require("fs");
53+
require("https").createServer({ key: fs.readFileSync("/certs/server.key"), cert: fs.readFileSync("/certs/server.crt") },
54+
(_, res) => res.end(fs.readFileSync("h264.mp4"))).listen(8443, () => console.log("ready"));')
55+
server=$!
56+
trap 'kill $server' EXIT
57+
read -r -t 30 -u 4 ready && [ "$ready" = ready ]
58+
[ "$(frames -tls_verify 1 https://localhost:8443/h264.mp4)" = 25 ]
59+
if frames -tls_verify 1 https://127.0.0.1:8443/h264.mp4 2>/dev/null; then echo "tls accepted a wrong host" >&2; exit 1; fi
60+
echo "tls: verified, wrong host refused"
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# name sha256 url (x264 has no stable tarballs: its line pins a git commit instead)
2+
ffmpeg-7.0.2.tar.xz 8646515b638a3ad303e23af6a3587734447cb8fc0a0c064ecdb8e95c4fd8b389 https://ffmpeg.org/releases/ffmpeg-7.0.2.tar.xz
3+
x264 b35605ace3ddf7c1a5d67a2eb553f034aef41d55 https://code.videolan.org/videolan/x264.git
4+
x265_4.1.tar.gz a31699c6a89806b74b0151e5e6a7df65de4b49050482fe5ebf8a4379d7af8f29 https://bitbucket.org/multicoreware/x265_git/downloads/x265_4.1.tar.gz
5+
libvpx-1.15.0.tar.gz e935eded7d81631a538bfae703fd1e293aad1c7fd3407ba00440c95105d2011e https://github.com/webmproject/libvpx/archive/refs/tags/v1.15.0.tar.gz
6+
opus-1.5.2.tar.gz 65c1d2f78b9f2fb20082c38cbe47c951ad5839345876e46941612ee87f9a7ce1 https://downloads.xiph.org/releases/opus/opus-1.5.2.tar.gz
7+
zimg-3.0.5.tar.gz a9a0226bf85e0d83c41a8ebe4e3e690e1348682f6a2a7838f1b8cbff1b799bcf https://github.com/sekrit-twc/zimg/archive/refs/tags/release-3.0.5.tar.gz
8+
dav1d-1.5.1.tar.xz 401813f1f89fa8fd4295805aa5284d9aed9bc7fc1fdbe554af4292f64cbabe21 https://downloads.videolan.org/pub/videolan/dav1d/1.5.1/dav1d-1.5.1.tar.xz

0 commit comments

Comments
 (0)