From 394f7827c1cfcde0409852dee0c25d3b53fa2680 Mon Sep 17 00:00:00 2001 From: Sameer Rahmani Date: Sun, 10 Jul 2022 02:35:32 +0100 Subject: [PATCH] Make a PoC of loading the core ns as an object --- builder | 180 +++++++++++++++++++---------- libserene/include/serene/context.h | 5 +- libserene/lib/jit/halley.cpp | 10 +- resources/docker/llvm/Dockerfile | 39 ++++--- serenec/serenec.cpp | 4 +- 5 files changed, 155 insertions(+), 83 deletions(-) diff --git a/builder b/builder index 098d325..1a7dcce 100755 --- a/builder +++ b/builder @@ -46,7 +46,8 @@ set -e # ----------------------------------------------------------------------------- command=$1 -VERSION="0.6.0" +VERSION="0.7.0" +LLVM_VERSION="15" # Serene subprojects. We use this array to run common tasks on all the projects # like running the test cases @@ -76,13 +77,19 @@ CMAKEARGS_DEBUG=("-DCMAKE_BUILD_TYPE=Debug") CMAKEARGS=("-DCMAKE_EXPORT_COMPILE_COMMANDS=ON" "-DSERENE_CCACHE_DIR=$HOME/.ccache") +# shellcheck source=scripts/utils.sh +source "$ME/scripts/utils.sh" + +# shellcheck source=scripts/containers.sh +source "$ME/scripts/containers.sh" + +# shellcheck source=scripts/devfs.sh +source "$ME/scripts/devfs.sh" + + # ----------------------------------------------------------------------------- # Helper functions # ----------------------------------------------------------------------------- -function fn-names() { - grep -E '^function [0-9a-zA-Z_-]+\(\) \{ ## .*$$' "$0" | sed 's/^function \([a-zA-Z0-9_-]*\)() { ## \(.*\)/\1/' -} - function gen_precompile_header_index() { { @@ -115,28 +122,6 @@ function build-gen() { popd_build } - -function info() { - if [ "$1" ] - then - echo -e "[\033[01;32mINFO\033[00m]: $*" - fi -} - -function error() { - if [ "$1" ] - then - echo -e "[\033[01;31mERR\033[00m]: $*" - fi -} - -function warn() { - if [ "$1" ] - then - echo -e "[\033[01;33mWARN\033[00m]: $*" - fi -} - # ----------------------------------------------------------------------------- # Subcomaands # ----------------------------------------------------------------------------- @@ -153,12 +138,36 @@ function build() { ## Builds the project by regenerating the build scripts build-gen "$@" pushed_build - cpus=$(grep ^cpu\\scores /proc/cpuinfo | uniq | awk '{print $4}') + cpus=$(nproc) cmake --build . -j "$cpus" popd_build } +function build-container() { ## Builds the project in handmade container (Linux only) + # shellcheck source=/dev/null + source .env + local uid + local gid + + # uid=$(id -u) + # gid=$(id -g) + # --map-user="$uid" \ + # --map-group="$gid" \ + + unshare --user \ + -w "/app" \ + --uts --net --ipc \ + --pid --fork \ + -c \ + --kill-child \ + --cgroup \ + --mount \ + --mount-proc \ + --root="$DEVFS" \ + /bin/bash +} + function build-20() { ## Builds the project using C++20 (will regenerate the build) clean pushed_build @@ -245,32 +254,28 @@ function build-tests() { ## Generates and build the project including the test c popd_build } -function build-llvm-image-arm64() { ## Build the LLVM image that we use to build Serene's image (on ARM64) +function build-llvm-image() { ## Build thh LLVM images of Serene for all platforms # shellcheck source=/dev/null source .env - docker buildx build --platform linux/arm64 --builder multiarch --load \ - -f "$ME/resources/docker/llvm/Dockerfile" \ - -t "$REGISTRY/llvm:$1-$2" \ - --build-arg VERSION="$1" \ - . + if [ "$1" ]; then + cleanup_builder || setup_builder + podman login "$REGISTRY" -u "$SERENE_REGISTERY_USER" -p "$SERENE_REGISTERY_PASS" + build_llvm "$1" "$ME" + build_ci "$1" "$ME" + cleanup_builder + else + error "Pass the llvm version as input" + fi } -function build-llvm-image() { ## Build the LLVM image that we use to build Serene's image +function push-images() { ## Pushes all the related image to the registery # shellcheck source=/dev/null source .env - docker buildx build \ - -f "$ME/resources/docker/llvm/Dockerfile" \ - -t "$REGISTRY/llvm:$1-$2" \ - --build-arg VERSION="$1" \ - . -} - -function push-llvm-image() { ## Pushes the LLVM image to the registery - # shellcheck source=/dev/null - source .env - - docker login "$REGISTRY" -u "$SERENE_REGISTERY_USER" -p "$SERENE_REGISTERY_PASS" - docker push "$REGISTRY/llvm:$1" + if [ "$1" ]; then + push_images "$1" "$ME" + else + error "Pass the llvm version as input" + fi } function build-serene-image-arm64() { ## Build the Serene docker image for the current HEAD (on ARM64) @@ -294,16 +299,6 @@ function build-serene-image() { ## Build the Serene docker image for the current . } -function build-serene-image-arm64() { ## Build the Serene docker image for the current HEAD (on ARM64) - # shellcheck source=/dev/null - source .env - - docker buildx build --platform linux/arm64 --builder multiarch --load \ - -f "$ME/resources/docker/serene/Dockerfile" \ - -t "$REGISTRY/serene:$VERSION-$(git rev-parse HEAD)" \ - . -} - function release-serene-image() { ## Build and push the Serene docker image for the current HEAD in Release mode # shellcheck source=/dev/null source .env @@ -316,6 +311,17 @@ function release-serene-image() { ## Build and push the Serene docker image for docker push "$REGISTRY/serene:$VERSION" } +function create-devfs-image() { ## Create the devfs images locally (requires sudo) + # shellcheck source=/dev/null + source .env + + local output_dir + + output_dir="$DEV_FS_DIR/image" + mkdir -p "$output_dir" + + create_and_initialize_devfs_image "$output_dir" "$ME" "$LLVM_VERSION" +} function setup() { ## Setup the working directory and make it ready for development if command -v python3 >/dev/null 2>&1; then @@ -328,6 +334,64 @@ function setup() { ## Setup the working directory and make it ready for developm # ln -s "$ME/scripts/pre-commit" "$ME/.git/hooks/pre-commit" } +function setup-dev() { ## Setup the container like env to build/develop Serene (requires sudo access) + # shellcheck source=/dev/null + source .env + + local fs_tarball + local rootfs + + fs_tarball="$DEV_FS_DIR/fs.tar.xz" + rootfs="$DEV_FS_DIR/fs" + + mkdir -p "$DEV_FS_DIR" + + if [[ -f "$rootfs/etc/shadow" ]]; then + info "RootFS already exits. Skipping..." + else + info "RootFS is missing." + if [ ! -f "$fs_tarball" ]; then + download_devfs "$SERENE_FS_REPO/fs.latest.tar.xz" "$fs_tarball" + else + info "FS tarball exists at '$fs_tarball'" + fi + + extract_devfs "$fs_tarball" "$rootfs" + fi + + init_devfs "$rootfs" "$ME" +} + +function devfs_root_shell() { ## Get a bash shell as root on the devfs + # shellcheck source=/dev/null + source .env + + local rootfs + + rootfs="$DEV_FS_DIR/fs" + + if [[ -f "$rootfs/etc/shadow" ]]; then + as_root "$rootfs" bash + else + error "DevFS does not exist run './builder setup-dev' first" + fi +} + +function devfs_shell() { ## Get a bash shell on the devfs + # shellcheck source=/dev/null + source .env + + local rootfs + + rootfs="$DEV_FS_DIR/fs" + + if [[ -f "$rootfs/etc/shadow" ]]; then + rootless "$rootfs" bash + else + error "DevFS does not exist run './builder setup-dev' first" + fi +} + function scan-build() { ## Runs the `scan-build` utility to analyze the build process clean build-gen diff --git a/libserene/include/serene/context.h b/libserene/include/serene/context.h index 7e52c81..42d2812 100644 --- a/libserene/include/serene/context.h +++ b/libserene/include/serene/context.h @@ -33,7 +33,7 @@ #include // for vector #define DEFAULT_NS_NAME "serene.user" -#define INTERNAL_NS "serene.internal" +#define INTERNAL_NS "serene.internal" namespace serene { class SereneContext; @@ -62,8 +62,7 @@ SERENE_EXPORT void terminate(SereneContext &ctx, int exitCode); class SERENE_EXPORT SereneContext { public: - template - using CurrentNSFn = std::function; + template using CurrentNSFn = std::function; /// The set of options to change the compilers behaivoirs Options opts; diff --git a/libserene/lib/jit/halley.cpp b/libserene/lib/jit/halley.cpp index 65177d9..4bb64c9 100644 --- a/libserene/lib/jit/halley.cpp +++ b/libserene/lib/jit/halley.cpp @@ -221,7 +221,7 @@ void Halley::setEngine(std::unique_ptr e, bool isLazy) { // Later on we might use different classes of JIT which might need some // work for lazyness (void)ctx; - engine = std::move(e); + engine = std::move(e); this->isLazy = isLazy; }; @@ -394,8 +394,8 @@ types::Namespace &Halley::makeNamespace(const char *name) { // randomly build instances here and there that causes unsafe memory assert(name && "name is nullptr: createNamespace"); const auto &nsName = getInternalString(name); - auto *ns = (types::Namespace *)GC_MALLOC(sizeof(types::Namespace)); - ns->name = &nsName; + auto *ns = (types::Namespace *)GC_MALLOC(sizeof(types::Namespace)); + ns->name = &nsName; nsStorage.push_back(ns); return *ns; @@ -405,7 +405,7 @@ types::Namespace &Halley::makeNamespace(const char *name) { llvm::Error Halley::createEmptyNS(const char *name) { assert(name && "name is nullptr: createEmptyNS"); // TODO: Look up the Namespace first. - auto &ns = makeNamespace(name); + auto &ns = makeNamespace(name); auto numOfDylibs = getNumberOfJITDylibs(ns) + 1; HALLEY_LOG( @@ -578,7 +578,7 @@ Halley::loadNamespaceFrom(NSLoadRequest &req) { } auto &session = engine->getExecutionSession(); - auto *jd = getLatestJITDylib(req.nsName.str().c_str()); + auto *jd = getLatestJITDylib(req.nsName.str().c_str()); assert(jd == nullptr && "'jd' must not be null since we just created it."); // TODO: Handle hidden static libs as well look at the addLibrary/AddArchive diff --git a/resources/docker/llvm/Dockerfile b/resources/docker/llvm/Dockerfile index 7aae3df..7d9b382 100644 --- a/resources/docker/llvm/Dockerfile +++ b/resources/docker/llvm/Dockerfile @@ -1,22 +1,19 @@ -FROM debian:sid-slim +FROM docker.io/debian:sid-slim ARG VERSION -RUN uname -a -RUN apt-get update && apt-get install -y apt-utils && apt-get install -y wget \ +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ gnupg \ - ccache \ cmake \ ccache \ git \ ninja-build \ - build-essential \ binutils \ lsb-release \ wget \ software-properties-common \ zlib1g \ - apt-utils \ cppcheck \ shellcheck \ zlib1g-dev @@ -24,7 +21,8 @@ RUN apt-get update && apt-get install -y apt-utils && apt-get install -y wget \ RUN wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh RUN ./llvm.sh ${VERSION} all RUN apt-get update --fix-missing && \ - apt-get install -y mlir-${VERSION}-tools \ + apt-get install -y --no-install-recommends \ + mlir-${VERSION}-tools \ libmlir-${VERSION}-dev \ libmlir-${VERSION} \ libmlir-${VERSION}-dbgsym \ @@ -43,18 +41,29 @@ RUN ln -s `which lld-${VERSION}` /usr/bin/lld && \ ENV MLIR_DIR /usr/lib/llvm-${VERSION} ENV CMAKE_PREFIX_PATH=/usr/lib/llvm-${VERSION} ENV LD_LIBRARY_PATH=/usr/lib/llvm-${VERSION}/lib/clang/${VERSION}.0.0/lib/linux/ +ENV CC=/usr/bin/clang +ENV CXX=/usr/bin/clang++ -RUN apt-get clean - +# --branch clang_${VERSION} WORKDIR /iwuy RUN git clone https://github.com/include-what-you-use/include-what-you-use.git --depth 1 && \ mkdir build && cd build && \ - cmake -G "Unix Makefiles" -DCMAKE_PREFIX_PATH=/usr/lib/llvm-${VERSION} ../include-what-you-use && \ - make && make install + cmake -G Ninja -DCMAKE_PREFIX_PATH=/usr/lib/llvm-${VERSION} ../include-what-you-use && \ + cmake --build . && \ + cmake -P cmake_install.cmake + +WORKDIR /boehm +RUN git clone https://github.com/ivmai/bdwgc.git --depth 1 --branch v8.2.0 && \ + mkdir build && cd build && \ + cmake -G Ninja -DBUILD_SHARED_LIBS=OFF -Denable_cplusplus=ON -Denable_threads=ON \ + -Denable_gcj_support=OFF -Dinstall_headers=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON ../bdwgc && \ + cmake --build . --config Release && \ + cmake -P cmake_install.cmake WORKDIR /app -RUN rm /iwuy -rf +RUN rm /iwuy -rf && rm /boehm -rf && rm /llvm.sh -COPY .pre-commit-config.yaml . -RUN git init . -RUN apt-get install -y --no-install-recommends pre-commit && pre-commit autoupdate && rm /app/.pre-commit-config.yaml +RUN apt-get purge -y git software-properties-common wget && \ + apt-get autoremove -y && \ + apt-get clean diff --git a/serenec/serenec.cpp b/serenec/serenec.cpp index b37b89b..4adc4f8 100644 --- a/serenec/serenec.cpp +++ b/serenec/serenec.cpp @@ -299,7 +299,7 @@ int main(int argc, char *argv[]) { // } std::string core = "serene.core"; - auto maybeJD = engine->loadNamespace(core); + auto maybeJD = engine->loadNamespace(core); if (!maybeJD) { llvm::errs() << "Error: " << maybeJD.takeError() << "'\n"; return 1; @@ -312,7 +312,7 @@ int main(int argc, char *argv[]) { return 1; } - auto c = *bt; + auto c = *bt; void *res = c(); (void)res;