Add boehmgc and finalize the tarball creation

This commit is contained in:
Sameer Rahmani 2023-04-23 23:16:25 +01:00
parent 8e76308baf
commit b1a10d595f
Signed by: lxsameer
GPG Key ID: B0A4AF28AB9FD90B
13 changed files with 399 additions and 27 deletions

3
.gitignore vendored
View File

@ -16,4 +16,5 @@ build
packages/stage1-sysroot/sysroot/
packages/stage2-sysroot/sysroot/
packages/stage3-sysroot/sysroot/
packages/stage3-sysroot/sysroot/
*.zstd

37
builder
View File

@ -87,8 +87,9 @@ function _build() {
}
function _deploy() {
mkdir -p "$ME/.tmp/$2/"
_conan install --deploy "$ME/conf/tarball_deployer.py" \
_conan install --deploy "$ME/conf/${4:-tarball_deployer.py}" \
--requires "$1/$2@$user/$channel" \
--profile:host="$ME/profiles/stage$3" \
--profile:build="$ME/profiles/stage$3" \
@ -142,11 +143,12 @@ function build-stage1() { ## Build the stage1 compiler and sysroot
mkdir -p "$ME/packages/stage1-sysroot/sysroot/"
cp -rv "$ME/.tmp/1/stage1-bundle/$LLVM_VERSION/" "$ME/packages/stage1-sysroot/sysroot/"
_build stage1-sysroot 1
rm -rf "$ME/packages/stage1-sysroot/sysroot/" "$ME/.tmp"
}
# A musl clang with the sysroot
function build-stage2() { ## Build the stage2 compiler and sysroot
llvm-source 1
#llvm-source 1
_build cmake 1
_build ninja 1
_build clang 1
@ -166,16 +168,14 @@ function build-stage2() { ## Build the stage2 compiler and sysroot
mkdir -p "$ME/packages/stage2-sysroot/sysroot/"
cp -rv "$ME/.tmp/2/stage2-bundle/$LLVM_VERSION/" "$ME/packages/stage2-sysroot/sysroot/"
_build stage2-sysroot 2
rm -rf "$ME/packages/stage2-sysroot/sysroot/" "$ME/.tmp"
}
# final toolchain
function build-stage3() { ## Build the stage3 compiler and sysroot
# llvm-source 1
_build toolchain 2
_build cmake 3
_build ninja 3
_build boehmgc 3
_build stage3-bundle 3
_build boehmgc 2
_build stage3-bundle 2
# All this is because conan does not support removing dependencies yet
# So we ended up hacking our way through to create some
@ -183,16 +183,33 @@ function build-stage3() { ## Build the stage3 compiler and sysroot
rm -rf "$ME/.tmp"
rm -rf "$ME/packages/stage3-sysroot/sysroot/"
_deploy "stage3-bundle" "$LLVM_VERSION" 3
_deploy "stage3-bundle" "$LLVM_VERSION" 2
mkdir -p "$ME/packages/stage3-sysroot/sysroot/"
cp -rv "$ME/.tmp/3/stage3-bundle/$LLVM_VERSION/" "$ME/packages/stage3-sysroot/sysroot/"
_build stage3-sysroot 3
cp -rv "$ME/.tmp/2/stage3-bundle/$LLVM_VERSION/" "$ME/packages/stage3-sysroot/sysroot/"
_build stage3-sysroot 2
rm -rf "$ME/packages/stage3-sysroot/sysroot/" "$ME/.tmp"
}
function compress_tarball() {
local ZSTD_CLI
ZSTD_CLI="zstd --ultra -22 -T$(nproc)"
tar -I "$ZSTD_CLI" -cf "serene_toolchain.$LLVM_MAJOR_VERSION.tar.zstd" \
-C "$ME/.tmp/2/stage3-sysroot/" .
}
function build-tarball() { ## Create the toolchain tarball
_deploy "stage3-sysroot" "$LLVM_VERSION" 2 "toolchain_deployer.py"
echo "Creating the tarball..."
compress_tarball
}
function build-toolchain() { ## Build the entire thing
build-stage1
build-stage2
build-stage3
build-tarball
}
function help() { ## Print out this help message

View File

@ -25,6 +25,7 @@ from conf.utils import get_version, with_musl_toolchain
class ZlibBase(ConanFile):
version = get_version("zlib")
settings = "os", "arch", "build_type", "compiler"
exports = "*.cmake"
cmake_setup_file = "zlib_setup.cmake"

View File

@ -0,0 +1,39 @@
# Toolchain builder for the Serene programming language
#
# Copyright (c) 2019-2023 Sameer Rahmani <lxsameer@gnu.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
from pathlib import Path
from conan.tools.files import copy
from conf.utils import copy_tree
# Only deploy the `toolchain` package
def deploy(graph, output_folder, **kwargs):
for name, dep in graph.root.conanfile.dependencies.items():
if dep.folders is None:
raise RuntimeError(
f"Sources missing for {name} dependency.\n"
"This deployer needs the sources of every dependency present to work, either building from source, "
"or by using the 'tools.build:download_source' conf."
)
name = str(dep.ref).split("@")[0].split("/")[0]
if name == "stage3-sysroot":
output = Path(output_folder) / name
print(f"Copying {dep.package_folder} to {output}")
copy_tree(dep.package_folder, output)
else:
print(f"Ignoring the '{name}' package")

View File

@ -134,7 +134,7 @@ def copy_dependency_tree(conanfile, dependencies_to_copy=[], dest=None):
conanfile.output.info(f"Copy the tree of '{dependency}' to '{root}'")
dep = Path(conanfile.dependencies[dependency].package_folder)
for dir_path in ["include", "lib", "bin"]:
for dir_path in ["include", "lib", "bin", "share"]:
try:
copy_tree(dep / dir_path, root / dir_path)
except FileNotFoundError:

View File

@ -0,0 +1,93 @@
# Toolchain builder for the Serene programming language
#
# Copyright (c) 2019-2023 Sameer Rahmani <lxsameer@gnu.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
from pathlib import Path
from conan import ConanFile
from conan.tools.cmake import CMake, CMakeToolchain, CMakeDeps
from conan.tools.files import get, copy
from conf.base.zstd import ZstdBase
from conf.utils import get_version, with_musl_toolchain, copy_dependency_tree
class BoehmGC(ConanFile):
name = "boehmgc"
settings = "os", "arch", "build_type", "compiler"
version = "8.2.2"
options = {
"stage": [1, 2],
}
default_options = {"stage": 2}
def source(self):
get(
self,
f"https://github.com/ivmai/bdwgc/archive/v{self.version}.tar.gz",
)
def build_requirements(self):
self.requires(f"toolchain/{get_version('llvm')}@{self.user}/{self.channel}")
self.requires(f"musl/{get_version('musl')}@{self.user}/{self.channel}")
def generate(self):
tc = CMakeToolchain(self)
tc.variables["BUILD_SHARED_LIBS"] = False
with_musl_toolchain(self, tc, "TARGET")
tc.generate()
deps = CMakeDeps(self)
deps.generate()
def create_sysroot(self):
sysroot = Path(self.build_folder) / "sysroot"
sysroot.mkdir()
copy_dependency_tree(self, ["toolchain", "musl"], sysroot)
def build(self):
self.create_sysroot()
sysroot = f"{self.build_folder}/sysroot"
cmake = CMake(self)
cmake.configure(
{
"CMAKE_INSTALL_PREFIX": self.package_folder,
"build_cord": "ON",
"enable_atomic_uncollectable": "ON",
"enable_cplusplus": "OFF",
"enable_disclaim": "ON",
"enable_docs": "ON",
"enable_dynamic_loading": "ON",
"enable_gc_assertions": "ON",
"enable_handle_fork": "ON",
"enable_java_finalization": "OFF",
"enable_munmap": "ON",
"enable_parallel_mark": "ON",
"enable_thread_local_alloc": "ON",
"enable_threads": "ON",
"enable_threads_discovery": "ON",
"enable_throw_bad_alloc_library": "ON",
"install_headers": "ON",
"BUILD_SHARED_LIBS": "OFF",
"CMAKE_POSITION_INDEPENDENT_CODE": "ON",
"CMAKE_SYSROOT": sysroot,
},
build_script_folder=f"bdwgc-{self.version}/",
)
cmake.build()
cmake.install()

View File

@ -0,0 +1,57 @@
# Toolchain builder for the Serene programming language
#
# Copyright (c) 2019-2023 Sameer Rahmani <lxsameer@gnu.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import shutil
from pathlib import Path
from conan import ConanFile
from conan.tools.files import get
from conan.tools.cmake import CMakeToolchain
from conf.utils import get_version, copy_template, copy_tree, copy_dependency_tree
class Stage3(ConanFile):
name = "stage3-bundle"
version = get_version("llvm")
settings = "os", "arch"
exports = ("*.cmake",)
options = {
"stage": [1, 2],
}
default_options = {"stage": 2}
def build_requirements(self):
self.requires(f"toolchain/{get_version('llvm')}@{self.user}/{self.channel}")
self.requires(f"zlib/{get_version('zlib')}@{self.user}/{self.channel}")
self.requires(f"zstd/{get_version('zstd')}@{self.user}/{self.channel}")
self.requires(f"musl/{get_version('musl')}@{self.user}/{self.channel}")
def build(self):
copy_dependency_tree(
self,
["toolchain", "musl", "zlib", "zstd"],
self.package_folder,
)
toolchain = f"{self.recipe_folder}/toolchain.cmake"
copy_template(
self,
toolchain,
f"{self.package_folder}/toolchain.cmake",
{"TRIPLE": os.environ["TARGET"], "SYSROOT": self.package_folder},
)

View File

@ -0,0 +1,34 @@
# Toolchain builder for the Serene programming language
#
# Copyright (c) 2019-2023 Sameer Rahmani <lxsameer@gnu.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
set(CMAKE_SYSTEM_NAME Linux)
set(triple @TRIPLE@)
set(CMAKE_SYSROOT @SYSROOT@)
set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT})
set(CMAKE_C_COMPILER "clang")
set(CMAKE_CXX_COMPILER "clang++")
set(CMAKE_ASM_COMPILER "clang")
set(CMAKE_C_COMPILER_TARGET ${triple})
set(CMAKE_CXX_COMPILER_TARGET ${triple})
set(CMAKE_ASM_COMPILER_TARGET ${triple})
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM BOTH)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

View File

@ -0,0 +1,88 @@
# Toolchain builder for the Serene programming language
#
# Copyright (c) 2019-2023 Sameer Rahmani <lxsameer@gnu.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import shutil
from pathlib import Path
from conan import ConanFile
from conan.tools.files import get
from conan.tools.cmake import CMakeToolchain
from conf.utils import get_version, copy_template, copy_tree, copy_dependency_tree
class Stage3Sysroot(ConanFile):
name = "stage3-sysroot"
version = get_version("llvm")
settings = "os", "arch"
exports = (
"*.cmake",
"sysroot/*",
)
def build(self):
# sysroot dir is created via the builder script
copy_tree(
f"{self.recipe_folder}/sysroot/{get_version('llvm')}/bin",
f"{self.package_folder}/bin",
)
copy_tree(
f"{self.recipe_folder}/sysroot/{get_version('llvm')}/lib",
f"{self.package_folder}/lib",
)
copy_tree(
f"{self.recipe_folder}/sysroot/{get_version('llvm')}/include",
f"{self.package_folder}/include",
)
copy_tree(
f"{self.recipe_folder}/sysroot/{get_version('llvm')}/share",
f"{self.package_folder}/share",
)
toolchain = f"{self.recipe_folder}/toolchain.cmake"
copy_template(
self,
toolchain,
f"{self.package_folder}/toolchain.cmake",
{"TRIPLE": os.environ["TARGET"], "SYSROOT": self.package_folder},
)
def package_info(self):
bindir = Path(self.package_folder) / "bin"
self.runenv_info.prepend_path(
"PATH",
str(bindir),
)
self.buildenv_info.append("sysroot", self.package_folder)
f = os.path.join(self.package_folder, "toolchain.cmake")
self.conf_info.prepend("tools.cmake.cmaketoolchain:user_toolchain", f)
self.conf_info.define("tools.build:sysroot", self.package_folder)
self.buildenv_info.define_path("SYSROOT", self.package_folder)
self.buildenv_info.define("CC", str(bindir / "clang"))
self.buildenv_info.define("CXX", str(bindir / "clang++"))
self.buildenv_info.define("AR", str(bindir / "clang"))
self.buildenv_info.define("NM", str(bindir / "llvm-nm"))
self.buildenv_info.define("RANLIB", str(bindir / "llvm-ranlib"))
self.cpp_info.includedirs = [
f"{self.package_folder}/include/c++/v1",
f"{self.package_folder}/include/",
f"{self.package_folder}/lib/clang/{get_version('llvm_major')}/include",
]

View File

@ -0,0 +1,34 @@
# Toolchain builder for the Serene programming language
#
# Copyright (c) 2019-2023 Sameer Rahmani <lxsameer@gnu.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
set(CMAKE_SYSTEM_NAME Linux)
set(triple @TRIPLE@)
set(CMAKE_SYSROOT @SYSROOT@)
set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT})
set(CMAKE_C_COMPILER "clang")
set(CMAKE_CXX_COMPILER "clang++")
set(CMAKE_ASM_COMPILER "clang")
set(CMAKE_C_COMPILER_TARGET ${triple})
set(CMAKE_CXX_COMPILER_TARGET ${triple})
set(CMAKE_ASM_COMPILER_TARGET ${triple})
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM BOTH)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

View File

@ -31,6 +31,7 @@ TOOLCHAIN_TARGETS = [
"clang-query",
"clang-resource-headers",
"clang-tidy",
"clang-doc",
"clang",
"clangd",
"clang-extdef-mapping",
@ -72,14 +73,15 @@ TOOLCHAIN_TARGETS = [
"mlir-opt",
"mlir-lsp-server",
"mlir-reduce",
# "lldb-server",
# "lldb",
"llvm-jitlink",
# lldb,
"compiler-rt",
"runtimes",
]
# No lldb? Yes, lldb links against liblldb.so and since we use
# -static with our executables that wouldn't be easy to handle
# and I'm lazy. We can just compile the lldb or any other debugger
# on the host
PROJECTS = ["clang-tools-extra", "clang", "lld", "llvm", "mlir"]
RUNTIME_TARGETS = [
"compiler-rt",
@ -92,7 +94,7 @@ CMAKE_OPTIONS = {
"LLVM_USE_LINKER": "lld",
"LLVM_ENABLE_NEW_PASS_MANAGER": "ON",
"LLVM_BUILD_TESTS": "OFF",
"LLVM_ENABLE_ASSERTIONS": "OFF",
"LLVM_ENABLE_ASSERTIONS": "ON",
"LLVM_ENABLE_LIBXML2": "OFF",
"LLVM_ENABLE_TERMINFO": "OFF",
"LLVM_ENABLE_ZLIB": "FORCE_ON",
@ -112,13 +114,16 @@ CMAKE_OPTIONS = {
"LIBCXXABI_ENABLE_SHARED": "OFF",
"LIBCXX_ABI_VERSION": "2",
"LLVM_CCACHE_BUILD": "ON",
# TODO: At this point lld crashes during linking clang itself
# cause of an issues in the `MachineDominator Tree Construction`
# pass that leads to an out of memory issue. Turn LTO on later
# "LLVM_ENABLE_LTO": "THIN",
"CMAKE_POSITION_INDEPENDENT_CODE": "ON",
"CLANG_DEFAULT_CXX_STDLIB": "libc++",
"CLANG_DEFAULT_LINKER": "lld",
"CLANG_DEFAULT_OBJCOPY": "llvm-objcopy",
"CLANG_DEFAULT_RTLIB": "compiler-rt",
"CLANG_VENDOR_UTI": "serene.stage0",
"CLANG_VENDOR_UTI": "serene.toolchain",
"LLVM_ENABLE_LIBCXX": "ON",
"LLVM_ENABLE_ZSTD": "OFF",
"LLVM_OPTIMIZED_TABLEGEN": "OFF",
@ -126,14 +131,16 @@ CMAKE_OPTIONS = {
"LLVM_ENABLE_PROJECTS": ";".join(PROJECTS),
"LLVM_CCACHE_BUILD": "ON",
"LIBCXX_HAS_MUSL_LIBC": "ON",
# "LLVM_PARALLEL_COMPILE_JOBS": "32",
# "LLVM_PARALLEL_LINK_JOBS": "1",
"LLVM_DEFAULT_TARGET_TRIPLE": os.environ["TARGET"],
# Common
"CMAKE_POSITION_INDEPENDENT_CODE": "ON",
}
class LLVM(ConanFile):
name = "clang"
class toolchain(ConanFile):
name = "toolchain"
settings = "os", "arch", "build_type"
version = get_version("llvm")
@ -143,8 +150,6 @@ class LLVM(ConanFile):
self.requires(
f"stage2-sysroot/{get_version('llvm')}@{self.user}/{self.channel}"
)
# self.tool_requires(f"cmake/{get_version('cmake')}@{self.user}/{self.channel}")
# self.tool_requires(f"ninja/{get_version('ninja')}@{self.user}/{self.channel}")
@property
def buildenv(self):
@ -223,7 +228,7 @@ class LLVM(ConanFile):
"CMAKE_VERBOSE_MAKEFILE": "ON",
"CMAKE_POSITION_INDEPENDENT_CODE": "ON",
"LLVM_USE_LINKER": "lld",
# "LLVM_USE_LTO": "ON",
"LLVM_USE_LTO": "ON",
"LLVM_CCACHE_BUILD": opts["LLVM_CCACHE_BUILD"],
# TODO: Check for more sanitizers that work with musl
"COMPILER_RT_SANITIZERS_TO_BUILD": "asan;msan;tsan",
@ -293,6 +298,9 @@ class LLVM(ConanFile):
] = f"{sysroot}/lib/clang/17/include {sysroot}/include"
opts["CMAKE_REQUIRED_LIBRARIES"] = f"-L{sysroot}/lib -lz"
# opts["LLVM_EXTERNALIZE_DEBUGINFO"] = "ON"
# opts["LLVM_DEBUGINFO_OUTPUT_DIR"] = f"{self.package_folder}/debug"
self.add_runtimes_and_builtins(opts)
for target in opts["LLVM_RUNTIME_TARGETS"].split(";"):
@ -322,9 +330,11 @@ class LLVM(ConanFile):
build_script_folder=os.path.join(llvm_dir["LLVM_SOURCE_DIR"], "llvm"),
)
cm.build(target="builtins")
cm.build(target="compiler-rt")
cm.build(target="install-distribution")
cm.build(target="install-builtins")
cm.build(target="install-compiler-rt")
cm.build(target="install-runtimes")
# cm.build(target="install-toolchain-stripped")
cm.build(target="install-distribution-stripped")
def package_info(self):
bindir = Path(os.path.join(self.package_folder, "bin"))

View File

@ -31,7 +31,6 @@ class Zlib(ZlibBase):
default_options = {"stage": 1}
def build_requirements(self):
super().build_requirements()
if self.options.stage == 1:
self.requires(
f"clang-bootstrap/{get_version('llvm')}@{self.user}/{self.channel}",

View File

@ -34,7 +34,6 @@ class Zstd(ZstdBase):
default_options = {"stage": 1}
def build_requirements(self):
super().build_requirements()
if self.options.stage == 1:
self.requires(
f"clang-bootstrap/{get_version('llvm')}@{self.user}/{self.channel}",