Serenity Operating System
1#!/usr/bin/env bash
2set -eo pipefail
3# This file will need to be run in bash, for now.
4
5
6# === CONFIGURATION AND SETUP ===
7
8DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
9
10echo "$DIR"
11
12ARCH=${ARCH:-"x86_64"}
13TARGET="$ARCH-pc-serenity"
14PREFIX="$DIR/Local/$ARCH"
15BUILD="$DIR/../Build/$ARCH"
16SYSROOT="$BUILD/Root"
17
18MAKE="make"
19MD5SUM="md5sum"
20NPROC="nproc"
21REALPATH="realpath"
22
23if command -v ginstall &>/dev/null; then
24 INSTALL=ginstall
25else
26 INSTALL=install
27fi
28
29SYSTEM_NAME="$(uname -s)"
30
31# We *most definitely* don't need debug symbols in the linker/compiler.
32# This cuts the uncompressed size from 1.2 GiB per Toolchain down to about 120 MiB.
33# Hence, this might actually cause marginal speedups, although the point is to not waste space as blatantly.
34export CFLAGS="-g0 -O2 -mtune=native"
35export CXXFLAGS="-g0 -O2 -mtune=native"
36
37if [ "$SYSTEM_NAME" = "OpenBSD" ]; then
38 MAKE=gmake
39 MD5SUM="md5 -q"
40 NPROC="sysctl -n hw.ncpuonline"
41 REALPATH="readlink -f"
42 export CC=egcc
43 export CXX=eg++
44 export with_gmp=/usr/local
45 export LDFLAGS=-Wl,-z,notext
46elif [ "$SYSTEM_NAME" = "FreeBSD" ]; then
47 MAKE=gmake
48 MD5SUM="md5 -q"
49 NPROC="sysctl -n hw.ncpu"
50 export with_gmp=/usr/local
51 export with_mpfr=/usr/local
52elif [ "$SYSTEM_NAME" = "Darwin" ]; then
53 MD5SUM="md5 -q"
54 NPROC="sysctl -n hw.ncpu"
55fi
56
57# On at least OpenBSD, the path must exist to call realpath(3) on it
58if [ ! -d "$BUILD" ]; then
59 mkdir -p "$BUILD"
60fi
61BUILD=$($REALPATH "$BUILD")
62
63git_patch=
64while [ "$1" != "" ]; do
65 case $1 in
66 --dev ) git_patch=1
67 ;;
68 esac
69 shift
70done
71
72echo PREFIX is "$PREFIX"
73echo SYSROOT is "$SYSROOT"
74
75mkdir -p "$DIR/Tarballs"
76
77BINUTILS_VERSION="2.39"
78BINUTILS_MD5SUM="ab6825df57514ec172331e988f55fc10"
79BINUTILS_NAME="binutils-$BINUTILS_VERSION"
80BINUTILS_PKG="${BINUTILS_NAME}.tar.gz"
81BINUTILS_BASE_URL="https://ftp.gnu.org/gnu/binutils"
82
83GDB_VERSION="12.1"
84GDB_MD5SUM="0c7339e33fa347ce4d7df222d8ce86af"
85GDB_NAME="gdb-$GDB_VERSION"
86GDB_PKG="${GDB_NAME}.tar.gz"
87GDB_BASE_URL="https://ftp.gnu.org/gnu/gdb"
88
89# Note: If you bump the gcc version, you also have to update the matching
90# GCC_VERSION variable in the project's root CMakeLists.txt
91GCC_VERSION="12.2.0"
92GCC_MD5SUM="d7644b494246450468464ffc2c2b19c3"
93GCC_NAME="gcc-$GCC_VERSION"
94GCC_PKG="${GCC_NAME}.tar.gz"
95GCC_BASE_URL="https://ftp.gnu.org/gnu/gcc"
96
97buildstep() {
98 NAME=$1
99 shift
100 "$@" 2>&1 | sed $'s|^|\x1b[34m['"${NAME}"$']\x1b[39m |'
101}
102
103# === DEPENDENCIES ===
104buildstep dependencies echo "Checking whether 'make' is available..."
105if ! command -v ${MAKE:-make} >/dev/null; then
106 buildstep dependencies echo "Please make sure to install GNU Make (for the '${MAKE:-make}' tool)."
107 exit 1
108fi
109
110buildstep dependencies echo "Checking whether 'patch' is available..."
111if ! command -v patch >/dev/null; then
112 buildstep dependencies echo "Please make sure to install GNU patch (for the 'patch' tool)."
113 exit 1
114fi
115
116buildstep dependencies echo "Checking whether your C compiler works..."
117if ! ${CC:-cc} -o /dev/null -xc - >/dev/null <<'PROGRAM'
118int main() {}
119PROGRAM
120then
121 buildstep dependencies echo "Please make sure to install a working C compiler."
122 exit 1
123fi
124
125if [ "$SYSTEM_NAME" != "Darwin" ]; then
126 for lib in gmp mpc mpfr; do
127 buildstep dependencies echo "Checking whether the $lib library and headers are available..."
128 if ! ${CC:-cc} -I /usr/local/include -L /usr/local/lib -l$lib -o /dev/null -xc - >/dev/null <<PROGRAM
129#include <$lib.h>
130int main() {}
131PROGRAM
132 then
133 echo "Please make sure to install the $lib library and headers."
134 exit 1
135 fi
136 done
137fi
138
139# === CHECK CACHE AND REUSE ===
140
141pushd "$DIR"
142 if [ "${TRY_USE_LOCAL_TOOLCHAIN}" = "y" ] ; then
143 # The actual logic had to be moved to .github/workflows/cmake.yml.
144 # Github Actions guarantees that Toolchain/Cache/ is empty on a cache
145 # miss, and non-empty on a cache hit.
146 # The following logic is correct *only* because of that.
147
148 mkdir -p Cache
149 echo "Cache (before):"
150 ls -l Cache
151 CACHED_TOOLCHAIN_ARCHIVE="Cache/ToolchainBinariesGithubActions.tar.gz"
152 if [ -r "${CACHED_TOOLCHAIN_ARCHIVE}" ] ; then
153 echo "Cache at ${CACHED_TOOLCHAIN_ARCHIVE} exists!"
154 echo "Extracting toolchain from cache:"
155 if tar xzf "${CACHED_TOOLCHAIN_ARCHIVE}" ; then
156 echo "Done 'building' the toolchain."
157 echo "Cache unchanged."
158 exit 0
159 else
160 echo
161 echo
162 echo
163 echo "Could not extract cached toolchain archive."
164 echo "This means the cache is broken and *should be removed*!"
165 echo "As Github Actions cannot update a cache, this will unnecessarily"
166 echo "slow down all future builds for this hash, until someone"
167 echo "resets the cache."
168 echo
169 echo
170 echo
171 rm -f "${CACHED_TOOLCHAIN_ARCHIVE}"
172 fi
173 else
174 echo "Cache at ${CACHED_TOOLCHAIN_ARCHIVE} does not exist."
175 echo "Will rebuild toolchain from scratch, and save the result."
176 fi
177 echo "::group::Actually building Toolchain"
178 fi
179popd
180
181
182# === DOWNLOAD AND PATCH ===
183
184pushd "$DIR/Tarballs"
185 # Build aarch64-gdb for cross-debugging support on x86 systems
186 if [ "$ARCH" = "aarch64" ]; then
187 md5=""
188 if [ -e "$GDB_PKG" ]; then
189 md5="$($MD5SUM $GDB_PKG | cut -f1 -d' ')"
190 echo "gdb md5='$md5'"
191 fi
192 if [ "$md5" != ${GDB_MD5SUM} ] ; then
193 rm -f $GDB_PKG
194 curl -LO "$GDB_BASE_URL/$GDB_PKG"
195 else
196 echo "Skipped downloading gdb"
197 fi
198 fi
199
200 md5=""
201 if [ -e "$BINUTILS_PKG" ]; then
202 md5="$($MD5SUM $BINUTILS_PKG | cut -f1 -d' ')"
203 echo "binutils md5='$md5'"
204 fi
205 if [ "$md5" != ${BINUTILS_MD5SUM} ] ; then
206 rm -f $BINUTILS_PKG
207 curl -LO "$BINUTILS_BASE_URL/$BINUTILS_PKG"
208 else
209 echo "Skipped downloading binutils"
210 fi
211
212 md5=""
213 if [ -e "$GCC_PKG" ]; then
214 md5="$($MD5SUM ${GCC_PKG} | cut -f1 -d' ')"
215 echo "gcc md5='$md5'"
216 fi
217 if [ "$md5" != ${GCC_MD5SUM} ] ; then
218 rm -f $GCC_PKG
219 curl -LO "$GCC_BASE_URL/$GCC_NAME/$GCC_PKG"
220 else
221 echo "Skipped downloading gcc"
222 fi
223
224 if [ "$ARCH" = "aarch64" ]; then
225 if [ -d ${GDB_NAME} ]; then
226 rm -rf "${GDB_NAME}"
227 rm -rf "$DIR/Build/$ARCH/$GDB_NAME"
228 fi
229 echo "Extracting GDB..."
230 tar -xzf ${GDB_PKG}
231
232 pushd ${GDB_NAME}
233 if [ "$git_patch" = "1" ]; then
234 git init > /dev/null
235 git add . > /dev/null
236 git commit -am "BASE" > /dev/null
237 git apply "$DIR"/Patches/gdb.patch > /dev/null
238 else
239 patch -p1 < "$DIR"/Patches/gdb.patch > /dev/null
240 fi
241 $MD5SUM "$DIR"/Patches/gdb.patch > .patch.applied
242 popd
243 fi
244
245 patch_md5="$(${MD5SUM} "${DIR}"/Patches/binutils/*.patch)"
246
247 if [ ! -d "${BINUTILS_NAME}" ] || [ "$(cat ${BINUTILS_NAME}/.patch.applied)" != "${patch_md5}" ]; then
248 if [ -d ${BINUTILS_NAME} ]; then
249 rm -rf "${BINUTILS_NAME}"
250 rm -rf "${DIR}/Build/${ARCH}/${BINUTILS_NAME}"
251 fi
252 echo "Extracting binutils..."
253 tar -xzf ${BINUTILS_PKG}
254
255 pushd ${BINUTILS_NAME}
256 if [ "${git_patch}" = "1" ]; then
257 git init > /dev/null
258 git add . > /dev/null
259 git commit -am "BASE" > /dev/null
260 git am "${DIR}"/Patches/binutils/*.patch > /dev/null
261 else
262 for patch in "${DIR}"/Patches/binutils/*.patch; do
263 patch -p1 < "${patch}" > /dev/null
264 done
265 fi
266 ${MD5SUM} "${DIR}"/Patches/binutils/*.patch > .patch.applied
267 popd
268 else
269 echo "Using existing binutils source directory"
270 fi
271
272
273 patch_md5="$(${MD5SUM} "${DIR}"/Patches/gcc/*.patch)"
274
275 if [ ! -d "${GCC_NAME}" ] || [ "$(cat ${GCC_NAME}/.patch.applied)" != "${patch_md5}" ]; then
276 if [ -d ${GCC_NAME} ]; then
277 rm -rf "${GCC_NAME}"
278 rm -rf "${DIR}/Build/${ARCH}/${GCC_NAME}"
279 fi
280 echo "Extracting gcc..."
281 tar -xzf ${GCC_PKG}
282
283 pushd ${GCC_NAME}
284 if [ "${git_patch}" = "1" ]; then
285 git init > /dev/null
286 git add . > /dev/null
287 git commit -am "BASE" > /dev/null
288 git am --keep-non-patch "${DIR}"/Patches/gcc/*.patch > /dev/null
289 else
290 for patch in "${DIR}"/Patches/gcc/*.patch; do
291 patch -p1 < "${patch}" > /dev/null
292 done
293 fi
294 ${MD5SUM} "${DIR}"/Patches/gcc/*.patch > .patch.applied
295
296 if [ "${SYSTEM_NAME}" = "Darwin" ]; then
297 ./contrib/download_prerequisites
298 fi
299 popd
300 else
301 echo "Using existing GCC source directory"
302 fi
303popd
304
305
306# === COMPILE AND INSTALL ===
307
308rm -rf "$PREFIX"
309mkdir -p "$PREFIX"
310
311if [ -z "$MAKEJOBS" ]; then
312 MAKEJOBS=$($NPROC)
313fi
314
315mkdir -p "$DIR/Build/$ARCH"
316
317pushd "$DIR/Build/$ARCH"
318 unset PKG_CONFIG_LIBDIR # Just in case
319
320 if [ "$ARCH" = "aarch64" ]; then
321 rm -rf gdb
322 mkdir -p gdb
323
324 pushd gdb
325 echo "XXX configure gdb"
326
327
328 if [ "$SYSTEM_NAME" = "Darwin" ]; then
329 buildstep "gdb/configure" "$DIR"/Tarballs/$GDB_NAME/configure --prefix="$PREFIX" \
330 --target="$TARGET" \
331 --with-sysroot="$SYSROOT" \
332 --enable-shared \
333 --disable-werror \
334 --with-libgmp-prefix="$(brew --prefix gmp)" \
335 --with-gmp="$(brew --prefix gmp)" \
336 --with-isl="$(brew --prefix isl)" \
337 --with-mpc="$(brew --prefix libmpc)" \
338 --with-mpfr="$(brew --prefix mpfr)" \
339 --disable-nls \
340 ${TRY_USE_LOCAL_TOOLCHAIN:+"--quiet"} || exit 1
341 else
342 buildstep "gdb/configure" "$DIR"/Tarballs/$GDB_NAME/configure --prefix="$PREFIX" \
343 --target="$TARGET" \
344 --with-sysroot="$SYSROOT" \
345 --enable-shared \
346 --disable-werror \
347 --disable-nls \
348 ${TRY_USE_LOCAL_TOOLCHAIN:+"--quiet"} || exit 1
349 fi
350
351 echo "XXX build gdb"
352 buildstep "gdb/build" "$MAKE" MAKEINFO=true -j "$MAKEJOBS" || exit 1
353 buildstep "gdb/install" "$MAKE" MAKEINFO=true install || exit 1
354 popd
355 fi
356
357 rm -rf binutils
358 mkdir -p binutils
359
360 pushd binutils
361 echo "XXX configure binutils"
362
363 # We don't need the documentation that is being built, so
364 # don't force people to install makeinfo just for that.
365 export ac_cv_prog_MAKEINFO=true
366
367 buildstep "binutils/configure" "$DIR"/Tarballs/$BINUTILS_NAME/configure --prefix="$PREFIX" \
368 --target="$TARGET" \
369 --with-sysroot="$SYSROOT" \
370 --enable-shared \
371 --disable-nls \
372 ${TRY_USE_LOCAL_TOOLCHAIN:+"--quiet"} || exit 1
373 if [ "$SYSTEM_NAME" = "Darwin" ]; then
374 # under macOS generated makefiles are not resolving the "intl"
375 # dependency properly to allow linking its own copy of
376 # libintl when building with --enable-shared.
377 buildstep "binutils/build" "$MAKE" -j "$MAKEJOBS" || true
378 pushd intl
379 buildstep "binutils/build" "$MAKE" all-yes
380 popd
381 fi
382 echo "XXX build binutils"
383 buildstep "binutils/build" "$MAKE" MAKEINFO=true -j "$MAKEJOBS" || exit 1
384 buildstep "binutils/install" "$MAKE" MAKEINFO=true install || exit 1
385 popd
386
387 echo "XXX serenity libc headers"
388 mkdir -p "$BUILD"
389 pushd "$BUILD"
390 mkdir -p Root/usr/include/
391 SRC_ROOT=$($REALPATH "$DIR"/..)
392 FILES=$(find \
393 "$SRC_ROOT"/AK \
394 "$SRC_ROOT"/Kernel/API \
395 "$SRC_ROOT"/Kernel/Arch \
396 "$SRC_ROOT"/Userland/Libraries/LibC \
397 -name '*.h' -print)
398 for header in $FILES; do
399 target=$(echo "$header" | sed \
400 -e "s@$SRC_ROOT/AK/@AK/@" \
401 -e "s@$SRC_ROOT/Userland/Libraries/LibC@@" \
402 -e "s@$SRC_ROOT/Kernel/@Kernel/@")
403 buildstep "system_headers" mkdir -p "$(dirname "Root/usr/include/$target")"
404 buildstep "system_headers" $INSTALL "$header" "Root/usr/include/$target"
405 done
406 unset SRC_ROOT
407 popd
408
409 if [ "$SYSTEM_NAME" = "OpenBSD" ]; then
410 perl -pi -e 's/-no-pie/-nopie/g' "$DIR/Tarballs/gcc-$GCC_VERSION/gcc/configure"
411 fi
412
413 rm -rf gcc
414 mkdir -p gcc
415
416 pushd gcc
417 echo "XXX configure gcc and libgcc"
418 buildstep "gcc/configure" "$DIR/Tarballs/gcc-$GCC_VERSION/configure" --prefix="$PREFIX" \
419 --target="$TARGET" \
420 --with-sysroot="$SYSROOT" \
421 --disable-nls \
422 --enable-shared \
423 --enable-languages=c,c++ \
424 --enable-default-pie \
425 --enable-lto \
426 --enable-threads=posix \
427 --enable-initfini-array \
428 --with-linker-hash-style=gnu \
429 ${TRY_USE_LOCAL_TOOLCHAIN:+"--quiet"} || exit 1
430
431 echo "XXX build gcc and libgcc"
432 buildstep "gcc/build" "$MAKE" -j "$MAKEJOBS" all-gcc || exit 1
433 buildstep "libgcc/build" "$MAKE" -j "$MAKEJOBS" all-target-libgcc || exit 1
434 echo "XXX install gcc and libgcc"
435 buildstep "gcc+libgcc/install" "$MAKE" install-gcc install-target-libgcc || exit 1
436
437 echo "XXX build libstdc++"
438 buildstep "libstdc++/build" "$MAKE" -j "$MAKEJOBS" all-target-libstdc++-v3 || exit 1
439 echo "XXX install libstdc++"
440 buildstep "libstdc++/install" "$MAKE" install-target-libstdc++-v3 || exit 1
441 popd
442
443popd
444
445pushd "$DIR/Local/$ARCH/$ARCH-pc-serenity/bin"
446 buildstep "mold_symlink" ln -s ../../../mold/bin/mold ld.mold
447popd
448
449
450# == SAVE TO CACHE ==
451
452pushd "$DIR"
453 if [ "${TRY_USE_LOCAL_TOOLCHAIN}" = "y" ] ; then
454 echo "::endgroup::"
455 echo "Building cache tar:"
456
457 rm -f "${CACHED_TOOLCHAIN_ARCHIVE}" # Just in case
458
459 tar czf "${CACHED_TOOLCHAIN_ARCHIVE}" Local/
460
461 echo "Cache (after):"
462 ls -l Cache
463 fi
464popd