Serene
/
bootstrap-toolchain
2
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Update the builder script to contain stage base builds

This commit is contained in:
Sameer Rahmani 2023-04-23 10:30:36 +01:00
parent e53ae5219e
commit 4c20072456
Signed by: lxsameer
GPG Key ID: B0A4AF28AB9FD90B
4 changed files with 143 additions and 341 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored
View File

@ -11,4 +11,9 @@ tf/terraform.tfstate*
build
.vagrant
.prod
#.#
#.#
.tmp/
packages/stage1-sysroot/sysroot/
packages/stage2-sysroot/sysroot/
packages/stage3-sysroot/sysroot/

95
builder
View File

@ -45,6 +45,7 @@ ZLIB_VERSION="1.2.13"
ZSTD_VERSION="1.5.4"
KERNEL_VERSION="6.0"
MUSL_VERSION="1.2.3"
IWYU_VERSION="14e9b208914a84fcdf49bf9f5d08897a4b3dc4b8"
TARGET=x86_64-pc-linux-musl
@ -56,7 +57,8 @@ export LLVM_VERSION \
KERNEL_VERSION \
LLVM_MAJOR_VERSION \
MUSL_VERSION \
TARGET
TARGET \
IWYU_VERSION
ME=$(cd "$(dirname "$0")/." >/dev/null 2>&1 ; pwd -P)
@ -72,8 +74,8 @@ function _conan() { ## A conan wrapper
function _create() {
_conan create --user "$user" --channel "$channel" \
--profile:host="../../profiles/stage$1" \
--profile:build="../../profiles/stage$1" \
--profile:host="$ME/profiles/stage$1" \
--profile:build="$ME/profiles/stage$1" \
"${@:2}" .
}
@ -84,6 +86,15 @@ function _build() {
}
function _deploy() {
mkdir -p "$ME/.tmp/$2/"
_conan install --deploy "$ME/conf/tarball_deployer.py" \
--requires "$1/$2@$user/$channel" \
--profile:host="$ME/profiles/stage$3" \
--profile:build="$ME/profiles/stage$3" \
-of "$ME/.tmp/$3/" \
}
function graph() { ## Generate the dependency graph
_push "packages/$1/"
_conan graph info \
@ -105,6 +116,84 @@ function llvm-source() { ## Build the llvm source pkg
_pop
}
# A glibc clang with a sysroot containing musl, libc++
function build-stage1() { ## Build the stage1 compiler and sysroot
llvm-source 0
_build cmake 0
_build ninja 0
_build zlib-bootstrap 0
_build zstd-bootstrap 0
_build clang-bootstrap 0
_build kernel-headers 1
_build musl 1
_build zlib 1
_build zstd 1
_build libcxx 1
_build stage1-bundle 1
# All this is because conan does not support removing dependencies yet
# So we ended up hacking our way through to create some
# packages that do now have any dependency
rm -rf "$ME/.tmp"
rm -rf "$ME/packages/stage1-sysroot/sysroot/"
_deploy "stage1-bundle" "$LLVM_VERSION" 1
mkdir -p "$ME/packages/stage1-sysroot/sysroot/"
cp -rv "$ME/.tmp/1/stage1-bundle/$LLVM_VERSION/" "$ME/packages/stage1-sysroot/sysroot/"
_build stage1-sysroot 1
}
# A musl clang with the sysroot
function build-stage2() { ## Build the stage2 compiler and sysroot
llvm-source 1
_build cmake 1
_build ninja 1
_build clang 1
_build musl 2
_build zlib 2
_build zstd 2
_build stage2-bundle 2
# All this is because conan does not support removing dependencies yet
# So we ended up hacking our way through to create some
# packages that do now have any dependency
rm -rf "$ME/.tmp"
rm -rf "$ME/packages/stage2-sysroot/sysroot/"
_deploy "stage2-bundle" "$LLVM_VERSION" 2
mkdir -p "$ME/packages/stage2-sysroot/sysroot/"
cp -rv "$ME/.tmp/2/stage2-bundle/$LLVM_VERSION/" "$ME/packages/stage2-sysroot/sysroot/"
_build stage2-sysroot 2
}
# final toolchain
function build-stage3() { ## Build the stage3 compiler and sysroot
# llvm-source 1
_build toolchain 2
_build cmake 3
_build ninja 3
_build boehmgc 3
_build stage3-bundle 3
# All this is because conan does not support removing dependencies yet
# So we ended up hacking our way through to create some
# packages that do now have any dependency
rm -rf "$ME/.tmp"
rm -rf "$ME/packages/stage3-sysroot/sysroot/"
_deploy "stage3-bundle" "$LLVM_VERSION" 3
mkdir -p "$ME/packages/stage3-sysroot/sysroot/"
cp -rv "$ME/.tmp/3/stage3-bundle/$LLVM_VERSION/" "$ME/packages/stage3-sysroot/sysroot/"
_build stage3-sysroot 3
}
function build-toolchain() { ## Build the entire thing
build-stage1
build-stage2
}
function help() { ## Print out this help message
echo "Commands:"

49
conf/utils.py
View File

@ -14,6 +14,8 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import contextlib
import shutil
from pathlib import Path
@ -71,7 +73,7 @@ def copy_tree(src, dst, preserve_symlinks=True, ignore=None):
shutil.copy2(item, dst / item.name, follow_symlinks=not preserve_symlinks)
def with_static_flags(cmake_vars):
def with_static_flags(cmake_vars={}):
cmake_vars["CMAKE_C_FLAGS"] = f"-static {cmake_vars.get('CMAKE_C_FLAGS', '')}"
cmake_vars["CMAKE_CXX_FLAGS"] = f"-static {cmake_vars.get('CMAKE_CXX_FLAGS', '')}"
cmake_vars[
@ -80,11 +82,36 @@ def with_static_flags(cmake_vars):
return cmake_vars
def with_musl_toolchain(conanfile, tc, envname="TARGET"):
def with_musl_toolchain(conanfile, tc, sysroot=None, envname="TARGET"):
target = os.environ[envname]
tc.variables["CMAKE_C_COMPILER_TARGET"] = target
tc.variables["CMAKE_CXX_COMPILER_TARGET"] = target
opts = {
"CMAKE_SYSTEM_NAME": "Linux",
"CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++",
"CMAKE_ASM_COMPILER": "clang",
"CMAKE_C_COMPILER_TARGET": target,
# "CMAKE_C_FLAGS": f"--target={target}",
"CMAKE_CXX_COMPILER_TARGET": target,
# "CMAKE_CXX_FLAGS": f"--target={target}",
"CMAKE_ASM_COMPILER_TARGET": target,
# "CMAKE_ASM_FLAGS": f"--target={target}",
}
if sysroot:
opts.update(
{
"CMAKE_SYSROOT": sysroot,
"CMAKE_DEFAULT_SYSROOT": sysroot,
"CMAKE_FIND_ROOT_PATH": sysroot,
"CMAKE_FIND_ROOT_PATH_MODE_PROGRAM": "BOTH",
"CMAKE_FIND_ROOT_PATH_MODE_LIBRARY": "ONLY",
"CMAKE_FIND_ROOT_PATH_MODE_INCLUDE": "ONLY",
"CMAKE_FIND_ROOT_PATH_MODE_PACKAGE": "ONLY",
}
)
for k, v in opts.items():
tc.variables[k] = v
def copy_template(conanfile, src, dest, varmap):
@ -92,6 +119,7 @@ def copy_template(conanfile, src, dest, varmap):
target_file = Path(dest)
for k, v in varmap.items():
print(f">> replacing @{k}@ with {str(v)}")
result = result.replace(f"@{k}@", str(v))
target_file.write_text(result)
@ -113,3 +141,16 @@ def copy_dependency_tree(conanfile, dependencies_to_copy=[], dest=None):
conanfile.output.info(
f"Dir '{dep / dir_path}' does not exist, skipping"
)
@contextlib.contextmanager
def chpath(path: Path) -> None:
"""
Changes working directory and returns to previous on exit.
"""
prev_cwd = Path.cwd()
os.chdir(path)
try:
yield
finally:
os.chdir(prev_cwd)

333
packages/llvm/conanfile.py
View File

@ -1,333 +0,0 @@
# Toolchain builder for the Serene programming language
#
# Copyright (c) 2019-2023 Sameer Rahmani <lxsameer@gnu.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import shutil
from pathlib import Path
from conan import ConanFile
from conan.tools.cmake import CMake, CMakeToolchain, CMakeDeps
from conan.tools.files import get
from conf.utils import with_static_flags, current_dir, get_version
TOOLCHAIN_TARGETS = [
"runtimes",
"clang-apply-replacements",
"clang-format",
"clang-query",
"clang-resource-headers",
"clang-tidy",
"clang",
"clangd",
"clang-extdef-mapping",
"cmake-exports",
"dsymutil",
"lld",
"llvm-addr2line",
"llvm-ar",
"llvm-as",
"llvm-cov",
"llvm-cvtres",
"llvm-cxxmap",
"llvm-dlltool",
"llvm-dwp",
"llvm-dwarfdump",
"llvm-install-name-tool",
"llvm-lib",
"llvm-lipo",
"llvm-nm",
"llvm-objcopy",
"llvm-objdump",
"llvm-pdbutil",
"llvm-profdata",
"llvm-ranlib",
"llvm-rc",
"llvm-readelf",
"llvm-strings",
"llvm-strip",
"llvm-symbolizer",
"llvm-windres",
"LTO",
"builtins",
# lldb,
"compiler-rt",
"runtimes",
]
PROJECTS = ["clang-tools-extra", "clang", "lld", "llvm"]
RUNTIME_TARGETS = [
"compiler-rt",
"libcxx",
"libcxxabi",
"libunwind",
]
CMAKE_OPTIONS = {
"LLVM_USE_LINKER": "lld",
"LLVM_ENABLE_NEW_PASS_MANAGER": "ON",
"LLVM_BUILD_TESTS": "OFF",
"LLVM_ENABLE_ASSERTIONS": "OFF",
"LLVM_ENABLE_LIBXML2": "OFF",
"LLVM_ENABLE_TERMINFO": "OFF",
"LLVM_ENABLE_ZLIB": "FORCE_ON",
"LLVM_INCLUDE_BENCHMARKS": "OFF",
"LLVM_INCLUDE_EXAMPLES": "OFF",
"LLVM_INCLUDE_TESTS": "OFF",
"LLVM_INCLUDE_GO_TESTS": "OFF",
"LLVM_ENABLE_BINDINGS": "OFF",
"LLVM_TARGETS_TO_BUILD": "X86",
"LLVM_STATIC_LINK_CXX_STDLIB": "ON",
"PACKAGE_VENDOR": "Serene",
"LLVM_ENABLE_PER_TARGET_RUNTIME_DIR": "ON",
"ENABLE_X86_RELAX_RELOCATIONS": "ON",
"BUILD_SHARED_LIBS": "OFF",
"CLANG_ENABLE_BOOTSTRAP": "ON",
"LIBCXX_ENABLE_STATIC_ABI_LIBRARY": "ON",
"LIBCXXABI_ENABLE_SHARED": "OFF",
"LIBCXX_ABI_VERSION": "2",
"LLVM_CCACHE_BUILD": "ON",
"LLVM_ENABLE_LTO": "THIN",
"CMAKE_POSITION_INDEPENDENT_CODE": "ON",
"CLANG_DEFAULT_CXX_STDLIB": "libc++",
"CLANG_DEFAULT_LINKER": "lld",
"CLANG_DEFAULT_OBJCOPY": "llvm-objcopy",
"CLANG_DEFAULT_RTLIB": "compiler-rt",
"CLANG_VENDOR_UTI": "serene.stage0",
"LLVM_ENABLE_LIBCXX": "ON",
"LLVM_ENABLE_ZSTD": "OFF",
"LLVM_OPTIMIZED_TABLEGEN": "ON",
"LLVM_THINLTO_CACHE_PATH": "/tmp/llvm-build-lto",
"LLVM_ENABLE_PROJECTS": ";".join(PROJECTS),
"LLVM_CCACHE_BUILD": "ON",
"LIBCXX_HAS_MUSL_LIBC": "ON",
"LLVM_DEFAULT_TARGET_TRIPLE": os.environ["TARGET"],
# Common
"CMAKE_POSITION_INDEPENDENT_CODE": "ON",
}
class LLVM(ConanFile):
name = "llvm"
settings = "os", "arch", "build_type"
version = get_version("llvm")
def build_requirements(self):
self.requires(f"llvm-source/{self.version}@{self.user}/{self.channel}")
# self.requires(f"zlib/{get_version('zlib')}@{self.user}/{self.channel}")
# self.requires(f"musl/{get_version('musl')}@{self.user}/{self.channel}")
# self.requires(f"zstd/{get_version('zstd')}@{self.user}/{self.channel}")
self.requires(
f"stage1-sysroot/{get_version('llvm')}@{self.user}/{self.channel}"
)
self.tool_requires(f"cmake/{get_version('cmake')}@{self.user}/{self.channel}")
self.tool_requires(f"ninja/{get_version('ninja')}@{self.user}/{self.channel}")
@property
def buildenv(self):
return self.buildenv_info.vars(self, scope="build")
def add_compiler_options(self, opts):
# We can't use the clang-bootstrap toolchain file here because it will be passed
# to the runtimes and bulitins subcmake stuff later and it will "poison" the cmake
# cache with the wrong compiler and options. So we do this manually the hard way.
tc_dir = Path(self.dependencies["stage1-sysroot"].package_folder)
opts["CMAKE_C_COMPILER"] = str(tc_dir / "bin" / "clang")
opts["CMAKE_CXX_COMPILER"] = str(tc_dir / "bin" / "clang++")
opts["LLVM_CONFIG_PATH"] = str(tc_dir / "bin" / "llvm-config")
target = os.environ["TARGET"]
opts["CMAKE_C_COMPILER_TARGET"] = target
opts["CMAKE_CXX_COMPILER_TARGET"] = target
opts["CMAKE_SYSROOT"] = str(tc_dir)
opts["CMAKE_EXE_LINKER_FLAGS"] = f"-Wl,-rpath,{str(tc_dir)}/lib/{target}"
opts["CMAKE_SHARED_LINKER_FLAGS"] = f"-Wl,-rpath,{str(tc_dir)}/lib/{target}"
opts["CMAKE_C_FLAGS"] = "-static"
opts["CMAKE_CXX_FLAGS"] = "-static"
def add_runtimes_and_builtins(self, opts):
builtin_targets = []
target = os.environ["TARGET"]
sysroot_path = Path(self.dependencies["stage1-sysroot"].package_folder)
target_builtin_opts = {
"CMAKE_SYSTEM_NAME": "Linux",
"CMAKE_BUILD_TYPE": self.settings.build_type,
"CMAKE_SYSTEM_PROCESSOR": "X86_64",
"CMAKE_C_FLAGS": self.buildenv.get("CFLAGS", ""),
"CMAKE_CXX_FLAGS": self.buildenv.get("CXXFLAGS", ""),
"CMAKE_ASM_FLAGS": self.buildenv.get("ASFLAGS", ""),
"CMAKE_C_COMPILER_TARGET": target,
"CMAKE_CXX_COMPILER_TARGET": target,
"CMAKE_ASM_COMPILER_TARGET": target,
"CMAKE_TRY_COMPILE_TARGET_TYPE": "STATIC_LIBRARY",
"CMAKE_EXE_LINKER_FLAGS": f"--target={target} {self.buildenv.get('LDFLAGS', '')}",
"CMAKE_SHARED_LINKER_FLAGS": f"--target={target} {self.buildenv.get('LDFLAGS', '')}",
"CMAKE_SYSROOT": str(sysroot_path),
"CMAKE_INSTALL_RPATH": "\$ORIGIN/../lib",
"CMAKE_BUILD_WITH_INSTALL_RPATH": "ON",
"LLVM_USE_LINKER": "lld",
"COMPILER_RT_BAREMETAL_BUILD": "ON",
"COMPILER_RT_DEFAULT_TARGET_ONLY": "ON",
"LLVM_CCACHE_BUILD": opts["LLVM_CCACHE_BUILD"],
}
for k, v in target_builtin_opts.items():
opts[f"BUILTINS_{target}_{k}"] = v
target_runtime_opts = {
"CMAKE_SYSTEM_NAME": "Linux",
"CMAKE_BUILD_TYPE": self.settings.build_type,
"CMAKE_C_FLAGS": self.buildenv.get("CFLAGS", ""),
"CMAKE_CXX_FLAGS": self.buildenv.get("CXXFLAGS", ""),
"CMAKE_ASM_FLAGS": self.buildenv.get("ASFLAGS", ""),
"CMAKE_C_COMPILER_TARGET": target,
"CMAKE_CXX_COMPILER_TARGET": target,
"CMAKE_ASM_COMPILER_TARGET": target,
"CMAKE_TRY_COMPILE_TARGET_TYPE": "STATIC_LIBRARY",
"CMAKE_EXE_LINKER_FLAGS": f"--target={target} {self.buildenv.get('LDFLAGS', '')}",
"CMAKE_SHARED_LINKER_FLAGS": f"--target={target} {self.buildenv.get('LDFLAGS', '')}",
"CMAKE_INSTALL_RPATH": "\$ORIGIN/../lib",
"CMAKE_BUILD_WITH_INSTALL_RPATH": "ON",
"CMAKE_SYSROOT": str(sysroot_path),
"CMAKE_VERBOSE_MAKEFILE": "ON",
"CMAKE_POSITION_INDEPENDENT_CODE": "ON",
"LLVM_USE_LINKER": "lld",
"LLVM_USE_LTO": "ON",
"LLVM_CCACHE_BUILD": opts["LLVM_CCACHE_BUILD"],
# TODO: Check for more sanitizers that work with musl
"COMPILER_RT_SANITIZERS_TO_BUILD": "asan;msan;tsan",
"COMPILER_RT_USE_BUILTINS_LIBRARY": "ON",
"COMPILER_RT_USE_LIBCXX": "ON",
"COMPILER_RT_BUILD_SANITIZERS": "ON",
# For some reason this flag is not correctly set if we don't
# manually set it and some strange gnu only LDFLAGS are injected.
"COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT": "OFF",
# X-Ray doesn't seem to compiler with musl
"COMPILER_RT_BUILD_XRAY": "OFF",
# Only build these if we enable sanitizers since they depend
# on the sanitizer common runtime
"COMPILER_RT_BUILD_MEMPROF": "ON",
"COMPILER_RT_BUILD_LIBFUZZER": "ON",
"COMPILER_RT_BUILD_ORC": "ON",
# Make sure we use libc++ from the tree instead from the system.
"SANITIZER_CXX_ABI": "libc++",
"SANITIZER_CXX_ABI_INTREE": "ON",
"LIBCXX_CXX_ABI": "libcxxabi",
"LIBCXX_USE_COMPILER_RT": "ON",
"LIBCXX_ENABLE_STATIC_ABI_LIBRARY": "ON",
"LIBCXX_ABI_UNSTABLE": "ON",
"LIBCXX_STATICALLY_LINK_ABI_IN_SHARED_LIBRARY": "ON",
"LIBCXX_STATICALLY_LINK_ABI_IN_STATIC_LIBRARY": "ON",
"LIBCXX_ABI_VERSION": "2",
"LIBCXX_HAS_MUSL_LIBC": "ON",
# musl doesn't have -latomic
"LIBCXX_HAS_ATOMIC_LIB": "OFF",
"LIBCXXABI_ENABLE_THREADS": "ON",
"LIBCXXABI_HAS_CXA_THREAD_ATEXIT_IMPL": "OFF",
"LIBCXXABI_USE_COMPILER_RT": "ON",
"LIBCXXABI_USE_LLVM_UNWINDER": "ON",
"LIBCXXABI_ENABLE_STATIC_UNWINDER": "ON",
"LIBCXXABI_STATICALLY_LINK_UNWINDER_IN_SHARED_LIBRARY": "ON",
"LIBCXXABI_STATICALLY_LINK_UNWINDER_IN_STATIC_LIBRARY": "ON",
"LIBCXXABI_ENABLE_SHARED": "OFF",
"LIBUNWIND_USE_COMPILER_RT": "ON",
}
for k, v in target_runtime_opts.items():
opts[f"RUNTIMES_{target}_{k}"] = v
opts["LLVM_RUNTIME_TARGETS"] = target
opts["LLVM_BUILTIN_TARGETS"] = target
opts["LLVM_ENABLE_RUNTIMES"] = ";".join(RUNTIME_TARGETS)
def generate(self):
tc = CMakeToolchain(self)
tc.preprocessor_definitions["_LIBCPP_PROVIDES_DEFAULT_RUNE_TABLE"] = ""
tc.preprocessor_definitions["_LIBCPP_HAS_MUSL_LIBC"] = "ON"
tc.generate()
deps = CMakeDeps(self)
deps.set_property("zlib", "cmake_find_mode", "both")
deps.generate()
def build(self):
opts = CMAKE_OPTIONS.copy()
sysroot = self.dependencies["stage1-sysroot"].package_folder
self.add_compiler_options(opts)
# zlib_dir = self.dependencies["zlib"].package_folder
opts["ZLIB_ROOT"] = sysroot
opts[
"CMAKE_REQUIRED_INCLUDES"
] = f"{sysroot}/lib/clang/17/include {sysroot}/include"
opts["CMAKE_REQUIRED_LIBRARIES"] = f"-L{sysroot}/lib -lz"
self.add_runtimes_and_builtins(opts)
for target in opts["LLVM_RUNTIME_TARGETS"].split(";"):
extra_target = None
bin_dir = Path(self.package_folder) / "bin"
bin_dir.mkdir(exist_ok=True, parents=True)
opts["LLVM_ENABLE_PROJECTS"] = ";".join(PROJECTS)
opts["LLVM_DISTRIBUTION_COMPONENTS"] = ";".join(TOOLCHAIN_TARGETS)
stage1 = Path(self.dependencies["stage1-sysroot"].package_folder)
opts["LLVM_TABLEGEN"] = str(stage1 / "bin" / "llvm-tblgen")
opts["CLANG_TABLEGEN"] = str(stage1 / "bin" / "clang-tblgen")
llvm_dir = self.dependencies["llvm-source"].buildenv_info.vars(
self, scope="build"
)
build_dir = os.path.join(self.build_folder, "build")
os.makedirs(build_dir)
with current_dir(build_dir):
cm = CMake(self)
cm.configure(
opts,
build_script_folder=os.path.join(llvm_dir["LLVM_SOURCE_DIR"], "llvm"),
)
cm.build(target="builtins")
cm.build(target="compiler-rt")
cm.build(target="install-distribution")
def package_info(self):
bindir = Path(os.path.join(self.package_folder, "bin"))
clang_lib_dir = Path(
os.path.join(
self.package_folder, "lib", "clang", get_version("llvm_major"), "lib"
)
)
self.runenv_info.prepend_path(
"PATH",
str(bindir),
)
self.buildenv_info.define("CC", str(bindir / "clang"))
self.buildenv_info.define("CXX", str(bindir / "clang++"))
self.buildenv_info.define("AR", str(bindir / "llvm-ar"))
self.buildenv_info.define("NM", str(bindir / "llvm-nm"))
self.buildenv_info.define("RANLIB", str(bindir / "llvm-ranlib"))