From e2bed17c0d1bc6d15c4ad98e51ad3bab44875946 Mon Sep 17 00:00:00 2001 From: Sameer Rahmani Date: Thu, 16 Mar 2023 23:12:49 +0000 Subject: [PATCH] toolchain: Create a build script for stage0 compiler and runtimes --- cmake/caches/runtimes.0.cmake | 124 +++++ cmake/caches/stage0.cmake | 44 ++ resources/docker/toolchain/stage0/Dockerfile | 47 ++ resources/docker/toolchain/stage0/Sysroot | 22 + scripts/deps.sh | 26 +- scripts/toolchain.sh | 533 ++++++++++++++++--- 6 files changed, 710 insertions(+), 86 deletions(-) create mode 100644 cmake/caches/runtimes.0.cmake create mode 100644 cmake/caches/stage0.cmake create mode 100644 resources/docker/toolchain/stage0/Dockerfile create mode 100644 resources/docker/toolchain/stage0/Sysroot diff --git a/cmake/caches/runtimes.0.cmake b/cmake/caches/runtimes.0.cmake new file mode 100644 index 0000000..bc40739 --- /dev/null +++ b/cmake/caches/runtimes.0.cmake @@ -0,0 +1,124 @@ +# This file sets up a CMakeCache for a Fuchsia toolchain build. +set(LLVM_TARGETS_TO_BUILD X86 CACHE STRING "") + +#set(PACKAGE_VENDOR "serene-stage.0" CACHE STRING "") + +message(STATUS "Set 'install_dir' to $ENV{install_dir}") +set(_SERENE_ENABLE_PROJECTS "clang;lld") + +set(CMAKE_INSTALL_PREFIX $ENV{install_dir} CACHE STRING "") +#set(CMAKE_PREFIX_PATH $ENV{install_dir} CACHE STRING "") + +set(CMAKE_BUILD_TYPE Release CACHE STRING "") + +set(LLVM_INCLUDE_TESTS OFF CACHE BOOL "") +set(LLVM_INCLUDE_GO_TESTS OFF CACHE BOOL "") +set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "") +set(LLVM_INCLUDE_BENCHMARKS OFF CACHE BOOL "") +set(LLVM_INCLUDE_DOCS OFF CACHE BOOL "") +set(LLVM_ENABLE_BINDINGS OFF CACHE BOOL "") +set(LLVM_ENABLE_OCAMLDOC OFF CACHE BOOL "") +set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "") +set(LLVM_TOOL_LLVM_LTO2_BUILD OFF CACHE BOOL "") +set(LLVM_TOOL_LLVM_LTO_BUILD OFF CACHE BOOL "") +set(LLVM_TOOL_LTO_BUILD OFF CACHE BOOL "") +set(LLVM_TOOL_REMARKS_SHLIB_BUILD OFF CACHE BOOL "") +set(CLANG_BUILD_TOOLS OFF CACHE BOOL "") +set(CLANG_INCLUDE_DOCS OFF CACHE BOOL "") +set(CLANG_TOOL_CLANG_IMPORT_TEST_BUILD OFF CACHE BOOL "") +set(CLANG_TOOL_CLANG_LINKER_WRAPPER_BUILD OFF CACHE BOOL "") +set(CLANG_TOOL_C_INDEX_TEST_BUILD OFF CACHE BOOL "") +set(CLANG_TOOL_ARCMT_TEST_BUILD OFF CACHE BOOL "") +set(CLANG_TOOL_C_ARCMT_TEST_BUILD OFF CACHE BOOL "") +set(CLANG_TOOL_LIBCLANG_BUILD OFF CACHE BOOL "") +set(LLVM_ENABLE_ZSTD OFF CACHE BOOL "") +set(LLVM_ENABLE_DIA_SDK OFF CACHE BOOL "") +set(LLVM_ENABLE_LIBEDIT OFF CACHE BOOL "") +set(LLVM_ENABLE_LIBXML2 OFF CACHE BOOL "") +#set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR ON CACHE BOOL "") +set(LLVM_ENABLE_TERMINFO OFF CACHE BOOL "") +set(LLVM_ENABLE_UNWIND_TABLES OFF CACHE BOOL "") +set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "") +set(LLVM_ENABLE_ZLIB OFF CACHE BOOL "") +#set(LLVM_USE_RELATIVE_PATHS_IN_FILES ON CACHE BOOL "") +set(LLDB_ENABLE_CURSES OFF CACHE BOOL "") +set(LLDB_ENABLE_LIBEDIT OFF CACHE BOOL "") + +#set(CLANG_DEFAULT_CXX_STDLIB libc++ CACHE STRING "") +#set(CLANG_DEFAULT_LINKER lld CACHE STRING "") +#set(CLANG_DEFAULT_OBJCOPY llvm-objcopy CACHE STRING "") +#set(CLANG_DEFAULT_RTLIB compiler-rt CACHE STRING "") +#set(CLANG_DEFAULT_UNWINDLIB libunwind CACHE STRING "") +set(CLANG_ENABLE_ARCMT OFF CACHE BOOL "") +set(CLANG_ENABLE_STATIC_ANALYZER OFF CACHE BOOL "") +set(CLANG_PLUGIN_SUPPORT OFF CACHE BOOL "") + +#set(ENABLE_LINKER_BUILD_ID ON CACHE BOOL "") +set(ENABLE_X86_RELAX_RELOCATIONS ON CACHE BOOL "") + +set(LLVM_ENABLE_ASSERTIONS ON CACHE BOOL "") +set(LLVM_ENABLE_BACKTRACES OFF CACHE BOOL "") +set(LLVM_ENABLE_EH ON CACHE BOOL "") +set(LLVM_ENABLE_RTTI ON CACHE BOOL "") + +set(LIBUNWIND_ENABLE_SHARED OFF CACHE BOOL "") +set(LIBUNWIND_INSTALL_LIBRARY ON CACHE BOOL "") +set(LIBUNWIND_USE_COMPILER_RT OFF CACHE BOOL "") +set(LIBCXXABI_ENABLE_SHARED OFF CACHE BOOL "") +set(LIBCXXABI_ENABLE_STATIC_UNWINDER ON CACHE BOOL "") +set(LIBCXXABI_INSTALL_LIBRARY ON CACHE BOOL "") +set(LIBCXXABI_USE_COMPILER_RT OFF CACHE BOOL "") +set(LIBCXXABI_USE_LLVM_UNWINDER ON CACHE BOOL "") +#set(LIBCXX_ABI_VERSION 2 CACHE STRING "") +set(LIBCXX_ENABLE_SHARED OFF CACHE BOOL "") +set(LIBCXX_INSTALL_LIBRARY ON CACHE BOOL "") +set(LIBCXX_ENABLE_STATIC_ABI_LIBRARY ON CACHE BOOL "") +set(LIBCXX_USE_COMPILER_RT OFF CACHE BOOL "") +set(LLVM_ENABLE_RUNTIMES "libcxx;libcxxabi;libunwind" CACHE STRING "") #compiler-rt +#set(LLVM_BUILD_STATIC ON CACHE BOOL "") +#set(LLVM_ENABLE_LIBCXX ON CACHE BOOL "") +#set(LLVM_STATIC_LINK_CXX_STDLIB ON CACHE BOOL "") + +#set(LLVM_HOST_TRIPLE "x86_64-alpine-linux-musl" CACHE STRING "") +#set(target "x86_64-alpine-linux-musl") + +# set(LLVM_BUILTIN_TARGETS "${TARGET}" CACHE STRING "") +# set(BUILTINS_${TARGET}_CMAKE_SYSTEM_NAME Linux CACHE STRING "") +# set(BUILTINS_${TARGET}_CMAKE_BUILD_TYPE Release CACHE STRING "") + + set(LLVM_RUNTIME_TARGETS "${TARGET}" CACHE STRING "") +set(RUNTIMES_${TARGET}_CMAKE_SYSTEM_NAME Linux CACHE STRING "") +set(RUNTIMES_${TARGET}_CMAKE_BUILD_TYPE Release CACHE STRING "") +#set(RUNTIMES_${TARGET}_COMPILER_RT_USE_BUILTINS_LIBRARY ON CACHE BOOL "") +set(RUNTIMES_${TARGET}_LIBUNWIND_ENABLE_SHARED OFF CACHE BOOL "") +set(RUNTIMES_${TARGET}_LIBUNWIND_USE_COMPILER_RT OFF CACHE BOOL "") +set(RUNTIMES_${TARGET}_LIBUNWIND_INSTALL_LIBRARY ON CACHE BOOL "") +set(RUNTIMES_${TARGET}_LIBCXXABI_USE_COMPILER_RT OFF CACHE BOOL "") +set(RUNTIMES_${TARGET}_LIBCXXABI_ENABLE_SHARED OFF CACHE BOOL "") +set(RUNTIMES_${TARGET}_LIBCXXABI_USE_LLVM_UNWINDER ON CACHE BOOL "") +set(RUNTIMES_${TARGET}_LIBCXXABI_ENABLE_STATIC_UNWINDER ON CACHE BOOL "") +set(RUNTIMES_${TARGET}_LIBCXXABI_INSTALL_LIBRARY ON CACHE BOOL "") +set(RUNTIMES_${TARGET}_LIBCXX_USE_COMPILER_RT OFF CACHE BOOL "") +set(RUNTIMES_${TARGET}_LIBCXX_ENABLE_SHARED OFF CACHE BOOL "") +set(RUNTIMES_${TARGET}_LIBCXX_ENABLE_STATIC_ABI_LIBRARY ON CACHE BOOL "") +#set(RUNTIMES_${TARGET}_LIBCXX_ABI_VERSION 2 CACHE STRING "") +set(RUNTIMES_${TARGET}_LLVM_ENABLE_ASSERTIONS OFF CACHE BOOL "") +set(RUNTIMES_${TARGET}_LLVM_ENABLE_RUNTIMES "libcxx;libcxxabi;libunwind" CACHE STRING "") +set(RUNTIMES_${TARGET}_SANITIZER_CXX_ABI "libc++" CACHE STRING "") +#set(RUNTIMES_${TARGET}_SANITIZER_CXX_ABI_INTREE ON CACHE BOOL "") +# set(RUNTIMES_${TARGET}_LLVM_ENABLE_LIBCXX ON CACHE BOOL "") +#set(RUNTIMES_${TARGET}_LLVM_STATIC_LINK_CXX_STDLIB ON CACHE BOOL "") + +# # set(COMPILER_RT_SCUDO_STANDALONE_BUILD_SHARED ON CACHE BOOL "") +# # set(RUNTIMES_${TARGET}_COMPILER_RT_SCUDO_STANDALONE_BUILD_SHARED ON CACHE BOOL "") + +# set(COMPILER_RT_SANITIZERS_TO_BUILD asan;dfsan;msan;hwasan;tsan;safestack;cfi;ubsan_minimal CACHE STRING "") +# set(RUNTIMES_${TARGET}_COMPILER_RT_SANITIZERS_TO_BUILD asan;dfsan;msan;hwasan;tsan;safestack;cfi;ubsan_minimal CACHE STRING "") + + +#set(LLVM_ENABLE_PROJECTS ${_SERENE_ENABLE_PROJECTS} CACHE STRING "") +#set(LLVM_INSTALL_TOOLCHAIN_ONLY ON CACHE BOOL "") +set(RUNTIMES_${TARGET}_LIBCXX_HAS_MUSL_LIBC ON CACHE BOOL "") +set(LIBCXX_HAS_MUSL_LIBC ON CACHE BOOL "") +set(LLVM_HOST_TRIPLE ${TARGET}) +set(LLVM_NATIVE_TOOL_DIR ${NATIVE_LLVM_DIR}/bin CACHE PATH "") diff --git a/cmake/caches/stage0.cmake b/cmake/caches/stage0.cmake new file mode 100644 index 0000000..f558527 --- /dev/null +++ b/cmake/caches/stage0.cmake @@ -0,0 +1,44 @@ +# This file sets up a CMakeCache for a Fuchsia toolchain build. +set(LLVM_TARGETS_TO_BUILD X86 CACHE STRING "") +set(CMAKE_INSTALL_PREFIX $ENV{install_dir} CACHE STRING "") +set(CMAKE_BUILD_TYPE Release CACHE STRING "") + +set(LLVM_INCLUDE_TESTS OFF CACHE BOOL "") +set(LLVM_INCLUDE_GO_TESTS OFF CACHE BOOL "") +set(LLVM_INCLUDE_EXAMPLES OFF CACHE BOOL "") +set(LLVM_INCLUDE_BENCHMARKS OFF CACHE BOOL "") +set(LLVM_INCLUDE_DOCS OFF CACHE BOOL "") +set(LLVM_ENABLE_BINDINGS OFF CACHE BOOL "") +set(LLVM_ENABLE_OCAMLDOC OFF CACHE BOOL "") +set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "") +set(LLVM_TOOL_LLVM_LTO2_BUILD OFF CACHE BOOL "") +set(LLVM_TOOL_LLVM_LTO_BUILD OFF CACHE BOOL "") +set(LLVM_TOOL_LTO_BUILD OFF CACHE BOOL "") +set(LLVM_TOOL_REMARKS_SHLIB_BUILD OFF CACHE BOOL "") +set(CLANG_INCLUDE_DOCS OFF CACHE BOOL "") +set(CLANG_TOOL_CLANG_IMPORT_TEST_BUILD OFF CACHE BOOL "") +set(CLANG_TOOL_C_INDEX_TEST_BUILD OFF CACHE BOOL "") +set(CLANG_TOOL_ARCMT_TEST_BUILD OFF CACHE BOOL "") +set(CLANG_TOOL_C_ARCMT_TEST_BUILD OFF CACHE BOOL "") +set(CLANG_TOOL_LIBCLANG_BUILD OFF CACHE BOOL "") +set(LLVM_ENABLE_DIA_SDK OFF CACHE BOOL "") +set(LLVM_ENABLE_LIBEDIT OFF CACHE BOOL "") +set(LLVM_ENABLE_LIBXML2 OFF CACHE BOOL "") +#set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR ON CACHE BOOL "") +set(LLVM_ENABLE_TERMINFO OFF CACHE BOOL "") +set(LLVM_ENABLE_UNWIND_TABLES OFF CACHE BOOL "") +set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "") +#set(LLVM_USE_RELATIVE_PATHS_IN_FILES ON CACHE BOOL "") +set(LLDB_ENABLE_CURSES OFF CACHE BOOL "") +set(LLDB_ENABLE_LIBEDIT OFF CACHE BOOL "") + +set(CLANG_ENABLE_ARCMT OFF CACHE BOOL "") +set(CLANG_ENABLE_STATIC_ANALYZER OFF CACHE BOOL "") +set(CLANG_PLUGIN_SUPPORT OFF CACHE BOOL "") +set(ENABLE_X86_RELAX_RELOCATIONS ON CACHE BOOL "") + +set(LLVM_ENABLE_ASSERTIONS OFF CACHE BOOL "") +set(LLVM_ENABLE_BACKTRACES OFF CACHE BOOL "") +# set(LLVM_ENABLE_EH ON CACHE BOOL "") +# set(LLVM_ENABLE_RTTI ON CACHE BOOL "") +set(LLVM_ENABLE_PROJECTS "clang;lld" CACHE STRING "") diff --git a/resources/docker/toolchain/stage0/Dockerfile b/resources/docker/toolchain/stage0/Dockerfile new file mode 100644 index 0000000..c9e02f7 --- /dev/null +++ b/resources/docker/toolchain/stage0/Dockerfile @@ -0,0 +1,47 @@ +FROM alpine:edge + +RUN apk update \ + && apk add --no-cache \ + ninja \ + make \ + clang \ + clang-dev \ + bison \ + flex \ + # compiler-rt \ # + # llvm-libunwind \ + # lld \ + gcc \ + libgcc + curl \ + g++ \ + alpine-sdk \ + libatomic \ + libgcc \ + cmake \ + bash \ + git \ + linux-headers \ + zlib \ + zlib-dev \ + zstd \ + zstd-dev \ + tar \ + xz \ + python3 + +# RUN ln -sf /usr/bin/clang /usr/bin/cc \ +# && ln -sf /usr/bin/clang++ /usr/bin/c++ \ +# && ls -l /usr/bin/cc /usr/bin/c++ \ +# && cc --version \ +# && c++ --version + # && update-alternatives --install /usr/bin/cc cc /usr/bin/clang 10\ + # && update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 10\ + + +RUN addgroup -S serene && \ + adduser -h /home/serene -S serene -G serene + +USER serene:serene +ENV HOME=/home/serene +WORKDIR /home/serene/serene diff --git a/resources/docker/toolchain/stage0/Sysroot b/resources/docker/toolchain/stage0/Sysroot new file mode 100644 index 0000000..f63e579 --- /dev/null +++ b/resources/docker/toolchain/stage0/Sysroot @@ -0,0 +1,22 @@ +FROM alpine:edge + +RUN apk update \ + && apk upgrade \ + && apk add --no-cache \ + alpine-sdk \ + linux-headers \ + gcc \ + g++ \ + libgcc \ + build-base \ + zlib \ + zlib-dev \ + zstd \ + zstd-dev + # clang-dev \ + # compiler-rt \ + # libc++-dev \ + # llvm-runtimes \ + # libc++-static \ + # llvm-libunwind \ + # lld \ diff --git a/scripts/deps.sh b/scripts/deps.sh index 8a3e2ae..d51d5e9 100644 --- a/scripts/deps.sh +++ b/scripts/deps.sh @@ -33,10 +33,15 @@ BDWGC_BUILD_DIR="$DEPS_BUILD_DIR/${BDWGC_DIR_NAME}_build" BDWGC_INSTALL_DIR="$DEPS_BUILD_DIR/$BDWGC_DIR_NAME.$BDWGC_VERSION" ZSTD_CLI="zstd --ultra -22 -T$(nproc)" +TARGET="x86_64-pc-linux-musl" +COMPILER_ARGS="-fno-sanitize=all" + +export TARGET COMPILER_ARGS # shellcheck source=/dev/null source "scripts/toolchain.sh" + function build_bdwgc() { ## Builds the BDW GC local version version="$BDWGC_VERSION" @@ -163,12 +168,15 @@ function info_bdwgc() { info "export BDWgc_DIR='$BDWGC_INSTALL_DIR/lib64/cmake/bdwgc'" } + function build_musl() { - local version repo src install_dir build_dir + local version repo src install_dir build_dir toolchain_dir old_path version="$MUSL_VERSION" - install_dir="$DEPS_BUILD_DIR/musl.$version" + #install_dir="$DEPS_BUILD_DIR/musl.0.$version" + install_dir="$DEPS_BUILD_DIR/llvm.0.$LLVM_VERSION" build_dir="$DEPS_BUILD_DIR/musl_build.$version" + toolchain_dir="$DEPS_BUILD_DIR/stage0" repo="${MUSL_REPO:-git://git.musl-libc.org/musl}" src="$DEPS_SOURCE_DIR/musl.$version" @@ -187,10 +195,22 @@ function build_musl() { mkdir -p "$install_dir" _push "$build_dir" - ./configure --disable-shared --prefix="$install_dir" + old_path="$PATH" + + PATH="$toolchain_dir/bin:$PATH" + CC="clang" + #CC="$TARGET-gcc" + #CFLAGS="$COMPILER_ARGS" + #CFLAGS="--sysroot $toolchain_dir" + + export CC CFLAGS + + ./configure --prefix="$install_dir" make -j "$(nproc)" make install + + PATH="$old_path" _pop info "'musl' version '$version' installed at '$install_dir'" } diff --git a/scripts/toolchain.sh b/scripts/toolchain.sh index 30cbf10..d1c2e94 100644 --- a/scripts/toolchain.sh +++ b/scripts/toolchain.sh @@ -23,22 +23,258 @@ LLVM_DIR_NAME="llvm" -DOCKER_IMAGE="serene/toolchain-base" +DOCKER_IMAGE="devheroes.codes/serene/stage0-toolchain" DOCKER_TAG="latest" +SYSROOTS_PATH="$SERENE_HOME_DIR/sysroots/" + +# This function is kind of a hardcoded function. +# It will create a sysroot that supposed to be an alpine +# and replace the alpine vendor to pc or better to say +# it replaces x86_64-alpine-linux-musl with the TARGET +# so it is hardcoded in the sense that it excepts +# x86_64-alpine-linux-musl triple to be in the original +# sysroot +function build_sysroot() { + local output alpines + output="$SYSROOTS_PATH/$TARGET" + + mkdir -p "$output" + + _push "$ME/resources/docker/toolchain/stage0" + docker build -f Sysroot -t devheroes.codes/serene/sysroot0-toolchain:latest . + docker run -d --name sysroot devheroes.codes/serene/sysroot0-toolchain:latest + docker export --output sysroot.tar sysroot + docker stop sysroot + docker rm sysroot + mkdir -p "$output" + tar xf sysroot.tar -C "$output" + rm sysroot.tar + _pop + + _push "$output" + alpines=$(find . -type d -iname "*-alpine-*") + + for i in $alpines; do + mv -v "$i" "${i//x86_64-alpine-linux-musl/$TARGET}" + done + _pop +} function run_in_container() { - docker run --rm -t \ + docker run -it -d \ -v "$ME:/home/serene/serene" \ -v "$SERENE_HOME_DIR:/home/serene/.serene/" \ --user "$(id -u):$(id -g)" \ + --name "stage0" \ "$DOCKER_IMAGE:$DOCKER_TAG" \ "$@" } +function build_musl_cc() { + local type + type="native" + if [ ! -f "$DEPS_SOURCE_DIR/musl_cc.tgz" ]; then + curl "https://musl.cc/x86_64-linux-musl-$type.tgz" -o "$DEPS_SOURCE_DIR/musl_cc.tgz" + fi + + if [ ! -d "$DEPS_BUILD_DIR/stage0" ]; then + tar zxf "$DEPS_SOURCE_DIR/musl_cc.tgz" -C "$DEPS_BUILD_DIR" + mv -v "$DEPS_BUILD_DIR/x86_64-linux-musl-$type" "$DEPS_BUILD_DIR/stage0" + fi +} + +function build_gcc() { + local version repo src install_dir build_dir + + repo="${STAGE0_REPO:-https://devheroes.codes/Serene/musl-cross-make.git}" + src="$DEPS_SOURCE_DIR/musl-cross-make" + install_dir="$DEPS_BUILD_DIR/stage0" + build_dir="$DEPS_BUILD_DIR/stage0_build" + + clone_dep "$repo" "master" "$src" + + cp -r "$src" "$build_dir" + _push "$build_dir" + { + echo "TARGET = x86_64-pc-linux-musl" + echo "GCC_VER = 11.2.0" + echo "COMMON_CONFIG += CFLAGS='-g0 -Os' CXXFLAGS='-g0 -Os' LDFLAGS='-s'" + echo "OUTPUT='$install_dir'" + echo "GCC_CONFIG += --enable-default-pie" + echo "GCC_CONFIG += --with-static-standard-libraries" + } >> "$build_dir/config.mak" + + make -j "$(nproc)" + make install + _pop +} + +function build_docker_gcc() { + run_in_container ./builder deps build gcc +} + +function build_stagex() { + local version repo src install_dir build_dir old_path stage cache_file llvm_dir + stage="$1" + cache_file="$2" + + version="$LLVM_VERSION" + repo="${LLVM_REPO:-https://github.com/llvm/llvm-project.git}" + src="$DEPS_SOURCE_DIR/$LLVM_DIR_NAME.$version" + install_dir="$DEPS_BUILD_DIR/llvm.$stage.$LLVM_VERSION" + build_dir="$DEPS_BUILD_DIR/${LLVM_DIR_NAME}_build.$stage.$version" + + clone_dep "$repo" "$version" "$src" + + local iwyu_src iwyu_repo + iwyu_src="$DEPS_SOURCE_DIR/iwyu.$IWYU_VERSION" + iwyu_repo="${IWYU_REPO:-https://github.com/include-what-you-use/include-what-you-use.git}" + clone_dep "$iwyu_repo" "$IWYU_VERSION" "$iwyu_src" + + info "Building the stage0 toolchain version '$version'..." + if [[ -d "$build_dir" ]]; then + warn "A build dir for 'toolchain' already exists at '$build_dir'" + fi + + mkdir -p "$build_dir" + mkdir -p "$install_dir" + + _push "$build_dir" + echo "Using $(nproc) jobs" + + llvm_dir="$(dirname "$(which clang)/..")" + export install_dir + cmake -G Ninja \ + -DCMAKE_INSTALL_PREFIX="$install_dir" \ + -DLLVM_PARALLEL_COMPILE_JOBS="$(nproc)" \ + -DLLVM_PARALLEL_LINK_JOBS="$(nproc)" \ + -DTARGET_ARCHS="$TARGET_ARCHS" \ + -DTARGET="$TARGET" \ + -DNATIVE_LLVM_DIR="$llvm_dir" \ + -DSERENE_SYSROOT="$HOME/.serene/sysroots/x86_64-linux-musl" \ + "${@:3}" -C "$cache_file" \ + -DLIBCXX_HAS_MUSL_LIBC=ON \ + -DRUNTIMES_"${TARGET}"_LIBCXX_HAS_MUSL_LIBC=ON \ + -DLLVM_NATIVE_TOOL_DIR="$llvm_dir/bin" \ + -DLLVM_HOST_TRIPLE="$TARGET" \ + -DLLVM_TABLEGEN="$(which llvm-tblgen)" \ + -DLLVM_AR="$(which llvm-ar)" \ + -DCMAKE_C_COMPILER_AR="$(which llvm-ar)" \ + -DCMAKE_CXX_COMPILER_AR="$(which llvm-ar)" \ + -DCMAKE_ASM_COMPILER_AR="$(which llvm-ar)" \ + -DCLANG_TABLEGEN="$(which clang-tblgen)" \ + -S "$src/llvm" + # -DLLVM_DEFAULT_TARGET_TRIPLE="$TARGET" \ + #cmake --build . --parallel + ninja runtimes -j "$(nproc)" + ninja install-runtimes + #-DCMAKE_TOOLCHAIN_FILE="$ME/cmake/toolchains/stage0.cmake" \ + #cmake -DCMAKE_INSTALL_PREFIX="$install_dir" -P cmake_install.cmake + _pop + + info "llvm build is ready at '$install_dir'" + info "Just add the 'bin' dir to you PATH" + + unset install_dir +} + +function build_stage0() { + local version repo src install_dir build_dir old_path stage cache_file llvm_dir + + version="$LLVM_VERSION" + repo="${LLVM_REPO:-https://github.com/llvm/llvm-project.git}" + src="$DEPS_SOURCE_DIR/$LLVM_DIR_NAME.$version" + install_dir="$DEPS_BUILD_DIR/stage0" + build_dir="$DEPS_BUILD_DIR/${LLVM_DIR_NAME}_build.0.$version" + + clone_dep "$repo" "$version" "$src" + + info "Building the stage0 toolchain version '$version'..." + if [[ -d "$build_dir" ]]; then + warn "A build dir for 'toolchain' already exists at '$build_dir'" + fi + + mkdir -p "$build_dir" + mkdir -p "$install_dir" + + _push "$build_dir" + echo "Using $(nproc) jobs" + + cmake -G Ninja \ + -DCMAKE_INSTALL_PREFIX="$install_dir" \ + -DLLVM_PARALLEL_COMPILE_JOBS="$(nproc)" \ + -DLLVM_PARALLEL_LINK_JOBS="$(nproc)" \ + -C "$ME/cmake/caches/stage0.cmake" \ + -S "$src/llvm" + cmake --build . --parallel + cmake -DCMAKE_INSTALL_PREFIX="$install_dir" -P cmake_install.cmake + _pop + + # Enable the lld linker as the default linker for this toolchain + ln -s "$install_dir/bin/ld.lld" "$install_dir/bin/ld" + + + info "llvm build is ready at '$install_dir'" + info "Just add the 'bin' dir to you PATH" + + unset install_dir +} + +function build_stage1 { + local sysroot tc old_path + sysroot="$HOME/.serene/sysroots/$TARGET" + tc="$DEPS_BUILD_DIR/stage0" + + old_path="$PATH" + PATH="$tc/bin:$PATH" + CC="$tc/bin/clang --target=$TARGET $COMPILER_ARGS --sysroot $sysroot -static" + CXX="$tc/bin/clang++ --target=$TARGET $COMPILER_ARGS --sysroot $sysroot -static" + + LDFLAGS="-fuse-ld=lld -L$sysroot/usr/lib -L/usr/lib" # + export CC CXX LDFLAGS PATH + + build_stagex "1" "$ME/cmake/caches/runtimes.0.cmake" + + unset CC CXX LDFLAGS + PATH="$old_path" +} + +function build_docker_stage0 { + run_in_container ./builder deps build stage0 +} + +function build_docker_musl { + run_in_container ./builder deps build musl +} + +function build_stage2 { + local tc llvm + tc="$DEPS_BUILD_DIR/musl.0.$MUSL_VERSION" + llvm="$DEPS_BUILD_DIR/llvm.0.$LLVM_VERSION" + CC="clang" + CXX="$CC" + + + CFLAGS="$COMPILER_ARGS -isystem $llvm/include/$TARGET/c++/v1/ -isystem $llvm/include/$TARGET/ -isystem $llvm/include/c++/v1/ -isystem $llvm/include/ -isystem $tc/include" + CXXFLAGS="$CFLAGS" + LDFLAGS="-fuse-ld=lld -nostdinc -L$llvm/lib -L$tc/lib -lc" + export CC CXX CFALGS CXXFLAGS LDFLAGS + + build_stagex "2" "$ME/cmake/caches/clang.1.cmake" + unset CC CXX CXXFLAGS CFLAGS +} + +function build_docker_stage2 { + run_in_container ./builder deps build stage1 +} + +function build_stage3 { + build_stagex "3" "$ME/cmake/caches/clang.cmake" +} function build_llvm() { - local version repo src install_dir build_dir + local version repo src install_dir build_dir old_path version="$LLVM_VERSION" repo="${LLVM_REPO:-https://github.com/llvm/llvm-project.git}" @@ -63,12 +299,14 @@ function build_llvm() { _push "$build_dir" - CC=$(which clang) - CXX=$(which clang++) - # CFLAGS="-flto=thin" - # CXXFLAGS="$CFLAGS" + old_path="$PATH" + PATH="$DEPS_BUILD_DIR/stage0/bin:$PATH" + CC="$DEPS_BUILD_DIR/stage0/bin/x86_64-pc-linux-musl-gcc" + CXX="$DEPS_BUILD_DIR/stage0/bin/x86_64-pc-linux-musl-g++" + CFLAGS=" -v" + CXXFLAGS="$CFLAGS" # LDFLAGS="-flto=thin" - export CC CXX + export CC CXX CFLAGS CXXFLAGS cmake -G Ninja \ -DCMAKE_INSTALL_PREFIX="$install_dir" \ @@ -77,6 +315,8 @@ function build_llvm() { -DLLVM_EXTERNAL_IWYU_SOURCE_DIR="$iwyu_src" \ -DTARGET_ARCHS="$TARGET_ARCHS" \ -C "$ME/cmake/caches/llvm.cmake" \ + -DLLVM_DEFAULT_TARGET_TRIPLE="x86_64-pc-linux-musl" \ + -DCMAKE_SYSROOT="$DEPS_BUILD_DIR/stage0/" \ -S "$src/llvm" cmake --build . --parallel cmake -DCMAKE_INSTALL_PREFIX="$install_dir" -P cmake_install.cmake @@ -92,100 +332,163 @@ function build_llvm() { info "Just add the 'bin' dir to you PATH" } +function build_llvm_musl() { + local version repo src install_dir build_dir toolchain_dir -# function build_musl_stage1() { -# local version repo src build_dir install_dir old_path stage1 + version="$LLVM_VERSION" + repo="${LLVM_REPO:-https://github.com/llvm/llvm-project.git}" + src="$DEPS_SOURCE_DIR/$LLVM_DIR_NAME.$version" + install_dir="$DEPS_BUILD_DIR/llvm.2.$LLVM_VERSION" + build_dir="$DEPS_BUILD_DIR/${LLVM_DIR_NAME}_build.2.$version" + #toolchain_dir="$DEPS_BUILD_DIR/llvm.1.$LLVM_VERSION" + toolchain_dir="$DEPS_BUILD_DIR/musl.$MUSL_VERSION" + clone_dep "$repo" "$version" "$src" -# version="$MUSL_VERSION" -# install_dir="$DEPS_BUILD_DIR/musl.stage1.$version" -# #install_dir="$STAGE1_DIR.tmp" -# build_dir="$DEPS_BUILD_DIR/${MUSL_DIR_NAME}_build.stage1.$version" -# repo="${MUSL_REPO:-git://git.musl-libc.org/musl}" -# src="$DEPS_SOURCE_DIR/$MUSL_DIR_NAME.$version" + local iwyu_src iwyu_repo + iwyu_src="$DEPS_SOURCE_DIR/iwyu.$IWYU_VERSION" + iwyu_repo="${IWYU_REPO:-https://github.com/include-what-you-use/include-what-you-use.git}" + clone_dep "$iwyu_repo" "$IWYU_VERSION" "$iwyu_src" -# clone_dep "$repo" "$version" "$src" + info "Building the stage0 toolchain version '$version'..." + if [[ -d "$build_dir" ]]; then + warn "A build dir for 'toolchain' already exists at '$build_dir'" + fi -# info "Building the stage2 musl version '$version'..." -# if [[ -d "$build_dir" ]]; then -# warn "A build dir for 'musl' already exists at '$build_dir'" -# warn "Cleaning up..." -# rm -rf "$build_dir" -# fi + mkdir -p "$build_dir" + mkdir -p "$install_dir" -# info "Copy the source to the build directory at: '$build_dir'" -# cp -r "$src" "$build_dir" + _push "$build_dir" -# mkdir -p "$install_dir" + local old_path + old_path="$PATH" -# _push "$build_dir" + PATH="$toolchain_dir/bin/:$PATH" + CC="$toolchain_dir/bin/musl-gcc" + CXX="$toolchain_dir/bin/musl-gcc" + CC=$(which gcc) + CXX=$(which g++) + # CFLAGS="-flto=thin" + # CXXFLAGS="$CFLAGS" + # LDFLAGS="-flto=thin" + export CC CXX + # -DCMAKE_C_COMPILER_AR="$toolchain_dir/bin/llvm-ar" \ + # -DCMAKE_C_COMPILER_RANLIB="$toolchain_dir/bin/llvm-ranlib" \ -# old_path="$PATH" -# PATH="$STAGE0_DIR/bin:$PATH" -# CC=$(which clang) -# CXX=$(which clang++) -# LDFLAGS="-fuse-ld=lld" + cmake -G Ninja \ + -DCMAKE_INSTALL_PREFIX="$install_dir" \ + -DLLVM_PARALLEL_COMPILE_JOBS="$(nproc)" \ + -DLLVM_PARALLEL_LINK_JOBS="$(nproc)" \ + -DLLVM_EXTERNAL_IWYU_SOURCE_DIR="$iwyu_src" \ + -DTARGET_ARCHS="$TARGET_ARCHS" \ + -C "$ME/cmake/caches/llvm.cmake" \ + -DLLVM_DEFAULT_TARGET_TRIPLE="x86_64-pc-linux-musl" \ + -S "$src/llvm" + cmake --build . --parallel + cmake -DCMAKE_INSTALL_PREFIX="$install_dir" -P cmake_install.cmake -# export CC CXX LDFLAGS PATH LDFLAGS + unset CC + unset CXX -# ./configure --prefix="$install_dir" + PATH="$old_path" + _pop -# make -j "$(nproc)" -# make install -# ln -sv "$install_dir/lib/libc.so" "$install_dir/lib/ld-musl-$(uname -m).so.1" + # Enable the lld linker as the default linker for this toolchain + ln -s "$install_dir/bin/ld.lld" "$install_dir/bin/ld" -# # Create a symlink that can be used to print -# # the required shared objects of a program or -# # shared object -# # ln -sv "$install_dir/lib/libc.so" "$install_dir/bin/ldd" + info "llvm build is ready at '$install_dir'" + info "Just add the 'bin' dir to you PATH" +} -# # # Configure the dynamic linker -# # mkdir -p "$install_dir/etc" -# # { -# # echo "/usr/lib/gcc/x86_64-pc-linux-gnu/12" -# # echo "/usr/lib/gcc/x86_64-pc-linux-gnu/11" -# # echo "/usr/lib64/" -# # echo "/usr/lib/" -# # echo "$STAGE0_DIR/lib/clang/$LLVM_MAJOR_VERSION/lib/x86_64-unknown-linux-gnu" -# # echo "$STAGE0_DIR/lib/x86_64-unknown-linux-gnu" -# # echo "$install_dir/lib" -# # echo "$install_dir/x86_64-unknown-linux-gnu/lib" -# # echo "$install_dir/usr/lib64" -# # echo "$install_dir/lib64" -# # echo "$install_dir/usr/lib" -# # echo "$STAGE0_DIR/lib" -# # echo "$STAGE0_DIR/x86_64-pc-linux-gnu/lib" -# # echo "$STAGE0_DIR/usr/lib64" -# # echo "$STAGE0_DIR/lib64" -# # echo "$STAGE0_DIR/usr/lib" -# # } >> "$install_dir/etc/ld-musl-$(uname -m).path" -# # cp -v "$ME/scripts/templates/musl-clang" "$stage1/bin/" -# # chmod +x "$stage1/bin/musl-clang" -# # sed -i "s'@CC@'$stage1/bin/clang'" "$stage1/bin/musl-clang" -# # sed -i "s'@PREFIX@'$install_dir'" "$stage1/bin/musl-clang" -# # sed -i "s'@INCDIR@'$install_dir/include'" "$stage1/bin/musl-clang" -# # sed -i "s'@LIBDIR@'$install_dir/lib'" "$stage1/bin/musl-clang" +function build_musl_base() { + local version repo src build_dir install_dir old_path tc -# # cp -v "$ME/scripts/templates/ld.musl-lld" "$stage1/bin/" -# # chmod +x "$stage1/bin/ld.musl-lld" -# # sed -i "s'@CC@'$stage1/bin/clang'" "$stage1/bin/ld.musl-lld" -# # sed -i "s'@LIBDIR@'$install_dir/lib'" "$stage1/bin/ld.musl-lld" -# # TODO: [toolchain] Hardcoded value for dynamic linker of musl is BAAAAD -# # idea. We need a way to make sure the name is correct -# #sed -i "s'@LDSO@'/lib/ld-musl-x86_64.so.1'" "$stage1/bin/ld.musl-lld" -# # /TODO -# echo -e "#include \nint main() {return 0;}" > "$install_dir/main.cpp" -# $CXX -stdlib=libc++ -static "$install_dir/main.cpp" -o "$install_dir/main" -fuse-ld=lld -v + version="$MUSL_VERSION" + install_dir="$DEPS_BUILD_DIR/musl.0.$version" + #install_dir="$STAGE1_DIR.tmp" + tc="$DEPS_BUILD_DIR/llvm.0.$LLVM_VERSION" + build_dir="$DEPS_BUILD_DIR/musl_build.0.$version" + repo="${MUSL_REPO:-git://git.musl-libc.org/musl}" + src="$DEPS_SOURCE_DIR/musl.$version" -# unset CC CXX LDFLAGS + clone_dep "$repo" "$version" "$src" -# PATH="$old_path" -# export PATH + info "Building the stage2 musl version '$version'..." + if [[ -d "$build_dir" ]]; then + warn "A build dir for 'musl' already exists at '$build_dir'" + warn "Cleaning up..." + rm -rf "$build_dir" + fi -# _pop -# info "Stage1 musl has been built at: '$install_dir'" -# } + info "Copy the source to the build directory at: '$build_dir'" + cp -r "$src" "$build_dir" + + mkdir -p "$install_dir" + + _push "$build_dir" + + CC=$(which clang) + LDFLAGS="-fuse-ld=lld" + + export CC CXX LDFLAGS PATH LDFLAGS + + ./configure --prefix="$install_dir" + + make -j "$(nproc)" + make install + ln -sv "$install_dir/lib/libc.so" "$install_dir/lib/ld-musl-$(uname -m).so.1" + + # Create a symlink that can be used to print + # the required shared objects of a program or + # shared object + ln -sv "$install_dir/lib/libc.so" "$install_dir/bin/ldd" + + # Configure the dynamic linker + mkdir -p "$install_dir/etc" + { + echo "$install_dir/lib" + echo "$install_dir/x86_64-unknown-linux-gnu/lib" + echo "$install_dir/usr/lib64" + echo "$install_dir/lib64" + echo "$install_dir/usr/lib" + echo "$STAGE0_DIR/lib" + echo "$STAGE0_DIR/x86_64-pc-linux-gnu/lib" + echo "$STAGE0_DIR/usr/lib64" + echo "$STAGE0_DIR/lib64" + echo "$STAGE0_DIR/usr/lib" + echo "$GCC_INSTALLATION/lib" + } >> "$install_dir/etc/ld-musl-$(uname -m).path" + + cp -v "$ME/scripts/templates/musl-clang" "$install_dir/bin/" + chmod +x "$install_dir/bin/musl-clang" + sed -i "s'@CC@'$CC'" "$install_dir/bin/musl-clang" + sed -i "s'@PREFIX@'$install_dir'" "$install_dir/bin/musl-clang" + sed -i "s'@INCDIR@'$install_dir/include'" "$install_dir/bin/musl-clang" + sed -i "s'@LIBDIR@'$install_dir/lib'" "$install_dir/bin/musl-clang" + sed -i "s'@BASE_CLANG@'$tc'" "$install_dir/bin/musl-clang" + sed -i "s'@TARGET@'$TARGET'" "$install_dir/bin/musl-clang" + + cp -v "$ME/scripts/templates/ld.musl-lld" "$install_dir/bin/" + chmod +x "$install_dir/bin/ld.musl-lld" + sed -i "s'@CC@'$CC'" "$install_dir/bin/ld.musl-lld" + sed -i "s'@LIBDIR@'$install_dir/lib'" "$install_dir/bin/ld.musl-lld" + # TODO: [toolchain] Hardcoded value for dynamic linker of musl is BAAAAD + # idea. We need a way to make sure the name is correct + sed -i "s'@LDSO@'/lib/ld-musl-x86_64.so.1'" "$install_dir/bin/ld.musl-lld" + # /TODO + echo -e "#include \nint main() {return 0;}" > "$install_dir/main.cpp" + + $CC -stdlib=libc++ -static "$install_dir/main.cpp" -o "$install_dir/main" -fuse-ld=lld -v + + unset CC CXX LDFLAGS + + PATH="$old_path" + export PATH + + _pop + info "Stage1 musl has been built at: '$install_dir'" +} function build_toolchain() { @@ -200,6 +503,15 @@ function build_toolchain() { # "dstage0") # run_in_container ./builder deps build toolchain stage0 # ;; + "stage0") + build_stage0 + ;; + "stage1") + build_stage1 + ;; + "llvm_musl") + build_llvm_musl + ;; "llvm") build_llvm ;; @@ -283,3 +595,58 @@ function info_toolchain() { info "export PATH=$LLVM_INSTALL_DIR/bin:\$PATH" } + +function build_zlib() { + local version repo src install_dir build_dir old_path stage cc cxx + stage="$1" + + version="v1.2.13" + repo="https://github.com/madler/zlib.git" + src="$DEPS_SOURCE_DIR/zlib.$version" + install_dir="$DEPS_BUILD_DIR/llvm.$stage.$LLVM_VERSION" + build_dir="$DEPS_BUILD_DIR/zlib.$stage.$version" + + clone_dep "$repo" "$version" "$src" + + info "Building the stage$stage zlib version '$version'..." + if [[ -d "$build_dir" ]]; then + warn "A build dir for 'zlib' already exists at '$build_dir'" + fi + + mkdir -p "$build_dir" + mkdir -p "$install_dir" + + _push "$build_dir" + echo "Using $(nproc) jobs" + + old_path="$PATH" + PATH="$install_dir/bin/:$PATH" + cc="$install_dir/bin/clang" + cxx="$install_dir/bin/clang++" + + # install prefix adn cmake prefix "$ROOTDIR/out/$TARGET-$MCPU" + + cmake -G Ninja \ + -DCMAKE_INSTALL_PREFIX="$install_dir" \ + -DCMAKE_PREFIX_PATH="$install_dir" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CROSSCOMPILING=True \ + -DCMAKE_SYSTEM_NAME="Linux" \ + -DCMAKE_C_COMPILER="$cc;-fno-sanitize=all;-s;-target;$TARGET;--rtlib=compiler-rt" \ + -DCMAKE_CXX_COMPILER="$cxx;-fno-sanitize=all;-s;-target;$TARGET;--rtlib=compiler-rt" \ + -DCMAKE_ASM_COMPILER="$cc;-fno-sanitize=all;-s;-target;$TARGET" \ + -DCMAKE_RC_COMPILER="$install_dir/bin/llvm-rc" \ + -DCMAKE_AR="$install_dir/bin/llvm-ar" \ + -DCMAKE_RANLIB="$install_dir/bin/llvm-ranlib" \ + -S "$src" + unset install_dir + cmake --build . -j "$(nproc)" --target install + _pop + + PATH="$old_path" + info "zlib build is ready at '$install_dir'" +} + +function build_docker_zlib() { + run_in_container ./builder deps build zlib 0 +}