diff --git a/builder b/builder index 609d3d3..a410e80 100755 --- a/builder +++ b/builder @@ -48,22 +48,6 @@ ME=$(cd "$(dirname "$0")/." >/dev/null 2>&1 ; pwd -P) # shellcheck source=./scripts/utils.sh source "$ME/scripts/utils.sh" -# shellcheck source=./scripts/devfs.sh -source "$ME/scripts/devfs.sh" - -# ----------------------------------------------------------------------------- -# Dependencies -# ----------------------------------------------------------------------------- -# The builder script can download the toolchain for linux X86_64 and set it up. -# The toolchain is based on Musl, but you don't have to use it. But be aware -# that using glibc with serene means that you can't cross compile with it. - -# Here is the version of llvm that we use -LLVM_VERSION="da3cd333bea572fb10470f610a27f22bcb84b08c" -LLVM_MAJOR_VERSION="16" - -export LLVM_VERSION LLVM_MAJOR_VERSION - # ----------------------------------------------------------------------------- # CONFIG VARS # ----------------------------------------------------------------------------- @@ -79,9 +63,6 @@ if [[ "$CXX" = "" ]]; then export CXX fi -# The repository to push/pull packages to/from. -DEV_HEROES="https://devheroes.codes" - BUILD_DIR_NAME="build" export BUILD_DIR_NAME @@ -91,14 +72,6 @@ export BUILD_DIR SERENE_HOME_DIR="$HOME/.serene" export SERENE_HOME_DIR -TOOLCHAIN_DIR="$SERENE_HOME_DIR/toolchain" -export TOOLCHAIN_DIR - -# Set this to anything beside true if you don't want the builder -# script to manage the toolchain. E.g. you have your own toolchain -USE_SERENE_TOOLCHAIN="true" -export USE_SERENE_TOOLCHAIN - # Serene subprojects. We use this array to run common tasks on all the projects # like running the test cases PROJECTS=(libserene serenec serene-repl serene-tblgen) @@ -113,16 +86,6 @@ export ASAN_OPTIONS LSAN_OPTIONS=suppressions="$ME/.ignore_sanitize" export LSAN_OPTIONS -# shellcheck source=./scripts/toolchain.sh -#source "$ME/scripts/toolchain.sh" - - -# ----------------------------------------------------------------------------- -# Initialization -# ----------------------------------------------------------------------------- -#mkdir -p "$TOOLCHAIN_DIR" - - # ----------------------------------------------------------------------------- # Helper functions # ----------------------------------------------------------------------------- diff --git a/scripts/containers.sh b/scripts/containers.sh deleted file mode 100644 index f914ab1..0000000 --- a/scripts/containers.sh +++ /dev/null @@ -1,226 +0,0 @@ -#! /bin/bash -# Serene Programming Language -# -# Copyright (c) 2019-2024 Sameer Rahmani -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, version 2. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ----------------------------------------------------------------------------- -# Commentary -# ----------------------------------------------------------------------------- -# This file contains some helper functions to build the OCI images for -# development and production purposes of Serene. -# -# We use `Buildkit` to build the images and `podman` to run them. -# -# NOTE: -# REGISTERY comes frome the `.env` file in the root of the project -# -# NOTE: -# If you run into an issue like this with podman: -# -# WARN[0000] Error running newgidmap: exit status 1: newgidmap: gid range [1-1) -> [0-0) -# not allowed -# WARN[0000] Falling back to single mapping -# -# with podman or buildah try the following commands as root: -# -# usermod --add-subgids 1001000000-1001999999 YOUR_USER_NAME -# usermod --add-subgids 1001000000-1001999999 YOUR_USER_NAME -# -# or set the subuid and guid manually in `/etc/subuid` and `/etc/subgid` - - -set -e - -export BUILDER_NAME="multiarch" -export PLATFORMS=('amd64' 'arm64') - -function setup_builder() { - info "Creating the builder container" - sudo podman run --privileged --name "$BUILDER_NAME" \ - docker.io/multiarch/qemu-user-static --reset -p yes -} - -function cleanup_builder() { - info "Stopping the builder" - sudo podman stop "$BUILDER_NAME" - sudo podman rm "$BUILDER_NAME" -} - -function create_manifest() { - local manifest="$1" - - info "Remove the manifest if it exists" - buildah manifest rm "${manifest}" || true - info "Creating the manifest" - buildah manifest create "${manifest}" || warn "Manifest exists" -} - -function build_container_image() { - local IMAGE_NAME="$1" - local LLVM_VERSION="$2" - local DOCKERFILE="$3" - local ROOT="$4" - local MANIFEST - local IMAGE - - MANIFEST="serene/$1:${LLVM_VERSION}-$(git describe)" - IMAGE="$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)" - - create_manifest "${MANIFEST}" - - for ARCH in "${PLATFORMS[@]}"; do - info "Building the multiarch '$IMAGE_NAME' images for:" - info "VERSION: $LLVM_VERSION | ARCH: $ARCH" - - buildah build \ - --jobs="$(nproc)" \ - --arch "$ARCH" \ - --layers \ - --manifest "${MANIFEST}" \ - -f "$DOCKERFILE" \ - -t "$IMAGE" \ - --build-arg VERSION="$LLVM_VERSION" \ - "$ROOT" - - # info "Tagging the image '$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)' as latest" - # buildah tag \ - # "$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)" \ - # "$REGISTRY/$IMAGE_NAME:latest" - done - - - info "inspect ${MANIFEST}" - buildah manifest inspect "${MANIFEST}" - info "first push docker://$IMAGE" - buildah manifest push --all "${MANIFEST}" \ - "docker://$IMAGE" - info "second push docker://$REGISTRY/$IMAGE_NAME:latest" - buildah manifest push --all "${MANIFEST}" \ - "docker://$REGISTRY/$IMAGE_NAME:latest" -} - -function build_llvm() { - local LLVM_VERSION="$1" - local ROOT="$2" - - build_container_image "llvm" "$LLVM_VERSION" "$ROOT/resources/docker/llvm/Dockerfile" "$ROOT" -} - -function build_ci() { - local LLVM_VERSION="$1" - local ROOT="$2" - - build_container_image "ci" "$LLVM_VERSION" "$ROOT/resources/docker/llvm/Dockerfile.ci" "$ROOT" -} - -function push_images() { - local image="$1" - - local manifest - manifest="serene/$image" - - buildah manifest push "${manifest}" \ - --creds "$SERENE_REGISTERY_USER:$SERENE_REGISTERY_PASS" \ - --all -} - - - - - - - - -function setup_builder1() { - if ! docker buildx inspect --builder "$BUILDER_NAME"; then - info "Creating the builder '$BUILDER_NAME'" - docker buildx create --driver docker-container \ - --name "$BUILDER_NAME" \ - --platform "linux/amd64,linux/arm64" \ - --use \ - --bootstrap - else - info "The builder '$BUILDER_NAME' already exists." - fi -} - -# Params: -# 2nd: Image tag to use it should be the major number of llvm version -# 3rd: Project root -function build_llvm_multiarch() { - - setup_builder - - local IMAGE_NAME="llvm" - local LLVM_VERSION="$1" - local ROOT="$2" - - info "Building the multiarch llvm images for:" - info "VERSION: $LLVM_VERSION | Platforms: ${PLATFORMS[*]}" - docker buildx build --platform "${PLATFORMS[@]}" \ - --builder "$BUILDER_NAME" --push \ - -f "$ROOT/resources/docker/llvm/Dockerfile" \ - -t "$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)" \ - --build-arg VERSION="$LLVM_VERSION" \ - "$ROOT" - - info "Tagging the image '$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)' as latest" - docker tag \ - "$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)" \ - "$REGISTRY/$IMAGE_NAME:latest" -} - - -function build_ci_image() { - setup_builder - - local LLVM_VERSION="$1" - local ROOT="$2" - local IMAGE - - IMAGE="$REGISTRY/ci:${LLVM_VERSION}-$(git describe)" - - info "Building the CI images" - docker buildx build --platform "linux/arm64" \ - --builder "$BUILDER_NAME" --load \ - -f "$ROOT/resources/docker/llvm/Dockerfile.ci" \ - -t "$IMAGE" \ - --build-arg VERSION="$2" \ - "$ROOT" - - info "Finished building '$IMAGE'" - info "Tagging the image '$IMAGE' as latest" - docker tag \ - "$IMAGE" \ - "$REGISTRY/ci:latest" - -} - - -function push_images() { - local LLVM_VERSION="$1" - local ROOT="$2" - - info "Loging into registry" - docker login "$REGISTRY" -u "$SERENE_REGISTERY_USER" -p "$SERENE_REGISTERY_PASS" - - info "Push the LLVM image" - push "$REGISTRY/llvm:${LLVM_VERSION}-$(git describe)" - push "$REGISTRY/llvm:latest" - - info "Push the CI image" - push "$REGISTRY/ci:${LLVM_VERSION}-$(git describe)" - push "$REGISTRY/ci:latest" -} diff --git a/scripts/devfs.sh b/scripts/devfs.sh deleted file mode 100644 index d1f9399..0000000 --- a/scripts/devfs.sh +++ /dev/null @@ -1,238 +0,0 @@ -#! /bin/bash -# Serene Programming Language -# -# Copyright (c) 2019-2024 Sameer Rahmani -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, version 2. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ----------------------------------------------------------------------------- -# Commentary -# ----------------------------------------------------------------------------- -# Bunch of helper function to create a container like environment to develop -# Serene on GNU/Linux without going through the headache of compiling LLVM -# 5 times a day :D - -set -e - - -function as_root() { - local rootfs="$1" - sudo unshare \ - -w "/serene" \ - --uts \ - --ipc \ - --pid \ - --fork \ - --kill-child \ - --cgroup \ - --mount \ - --mount-proc \ - --root="$rootfs" \ - "${@:2}" -} - - -function rootless() { - local rootfs="$1" - unshare \ - -w "/serene" \ - --uts \ - -c \ - --ipc \ - --pid \ - --fork \ - --kill-child \ - --cgroup \ - --mount \ - --mount-proc \ - --root="$rootfs" \ - "${@:2}" -} - -function download_devfs() { - local repo="$1" - local target="$2" - info "Downloading the tarball from '$repo'" - wget "$repo/fs.latest.tar.xz" -O "$target" -} - - -function extract_devfs() { - local tarball="$1" - local to="$2" - - info "Extracting the tarball..." - mkdir -p "$to" - tar Jxf "$tarball" -C "$to" - - info "Create the 'serene' dir at the root" - mkdir -p "$to/serene" -} - -function mount_serene { - local rootfs="$1" - local project_root="$2" - local serene_dir - - serene_dir="$rootfs/serene" - - mkdir -p "$serene_dir" - - info "Mounting Serene's dir into '/serene' in read-only mode" - mountpoint -q "$serene_dir" || sudo mount --rbind -o ro "$project_root" "$serene_dir" -} - -function mount_trees() { - local rootfs="$1" - local project_root="$2" - - mount_serene "$rootfs" "$project_root" - - info "Mounting the 'tmpfs' at '$rootfs/tmp'" - mkdir -p "$rootfs/tmp" - mountpoint -q "$rootfs/tmp" || sudo mount -t tmpfs tmpfs "$rootfs/tmp" - - info "Mounting 'dev' at '$rootfs/dev'" - mkdir -p "$rootfs/dev" - mountpoint -q "$rootfs/dev" || sudo mount --bind /dev "$rootfs/dev" -} - - -function unmount_trees() { - local rootfs="$1" - - info "Unmounting the 'serene' from '$rootfs/serene'" - sudo umount "$rootfs/serene" &> /dev/null || true - - info "Unmounting the 'tmpfs' from '$rootfs/tmp'" - sudo umount "$rootfs/tmp" &> /dev/null || true - - info "Unmounting 'dev' from '$rootfs/dev'" - sudo umount "$rootfs/dev" &> /dev/null || true -} - -function init_devfs { - local rootfs="$1" - local project_root="$2" - local create_group - local create_user - - create_group="groupadd -f -g$(id -g) $(whoami)" - create_user="adduser -q --disabled-password --gecos '' --uid $(id -u) --gid $(id -g) $(whoami) || true" - - mkdir -p "$rootfs/proc" - mount_trees "$rootfs" "$project_root" - - info "Creating group '$(whoami)' with ID '$(id -g)'..." - as_root "$rootfs" bash --login -c "$create_group" - - info "Creating user '$(whoami)' with ID '$(id -u)'..." - as_root "$rootfs" bash --login -c "$create_user" - - info "Make '$(whoami)' a sudoer" - as_root "$rootfs" bash --login -c "adduser $(whoami) sudo || true" -} - -function create_debian_rootfs() { - local to="$1" - - info "Pulling the debian docker image" - docker pull debian:sid-slim - - info "Spinning up the container" - docker stop devfs &> /dev/null || true - docker rm devfs &> /dev/null || true - docker run --name devfs -d debian:sid-slim - sleep 2 - - info "Exporting the rootfs to '$to/rootfs.tar'" - docker export -o "$to/rootfs.tar" devfs - - info "Tearing down the container" - docker stop devfs &> /dev/null - docker rm devfs &> /dev/null -} - -function create_and_initialize_devfs_image() { - local to="$1" - local project_root="$2" - local llvm_version="$3" - local rootfs - local version - - version=$(git describe) - rootfs="$to/rootfs/" - - if [ ! -f "$to/rootfs.tar" ]; then - info "Creating the rootfs tar bar" - create_debian_rootfs "$to" - fi - - mkdir -p "$rootfs" - - if [ ! -f "$to/rootfs/etc/shadow" ]; then - info "Extracting the tarball" - tar xf "$to/rootfs.tar" -C "$rootfs" - fi - - mount_trees "$rootfs" "$project_root" - - #as_root "$rootfs" bash - - as_root "$rootfs" bash -c "echo '$llvm_version' > /etc/llvm_version" - as_root "$rootfs" bash -c "echo 'export LANG=C.UTF-8' >> /etc/bash.bashrc" - as_root "$rootfs" bash -c "echo 'export LANG=C.UTF-8' >> /etc/profile" - as_root "$rootfs" bash /serene/scripts/devfs_container_setup.sh - - unmount_trees "$rootfs" - - info "Creating the tarball (It will take a few minutes)" - sudo tar c -C "$rootfs" "." | lzma -c -9 -T "$(nproc)" > "$to/fs.$version.tar.xz" - - info "Removing the exporter tarball" - rm -fv "$to/rootfs.tar" -} - -function sync_devfs_image() { - local where="$1" - local version - - version=$(git describe) - - yes_or_no "Upload images '$where/fs.$version.tar.xz'?" && \ - info "Uploading image '$where/fs.$version.tar.xz'" && \ - rsync -vlc --progress \ - "$where/fs.$version.tar.xz" \ - core.lxsameer.com:/home/www/public/dl.serene/devfs/ -} - -function mark_devfs_image_as_latest() { - local to="$1" - local version - local remote_cmd - - version=$(git describe) - info "Marking images 'fs.$version.tar.xz' as latest" - remote_cmd="rm -f /home/www/public/dl.serene/devfs/fs.latest.tar.xz && ln -s /home/www/public/dl.serene/devfs/fs.$version.tar.xz /home/www/public/dl.serene/devfs/fs.latest.tar.xz" - - # shellcheck disable=SC2029 - ssh core.lxsameer.com "$remote_cmd" -} - -function unmount_and_destroy_devfs() { - local rootfs="$1" - - unmount_trees "$rootfs" - - yes_or_no "Is it correct? 'sudo rm -rfv $rootfs'?" && sudo rm -rfv "$rootfs" -} diff --git a/scripts/toolchain.sh b/scripts/toolchain.sh deleted file mode 100644 index 4e488b4..0000000 --- a/scripts/toolchain.sh +++ /dev/null @@ -1,62 +0,0 @@ -#! /bin/bash -# Serene Programming Language -# -# Copyright (c) 2019-2022 Sameer Rahmani -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, version 2. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - - -function unpack() { - local zstd - zstd="zstd --ultra -22 -T$(nproc)" - - mkdir -p "$2" - _push "$2" - tar -I "$zstd" -xf "$1" - _pop -} - -function pull_toolchain() { - local toolchain_name - toolchain_name="serene_toolchain.$LLVM_MAJOR_VERSION.$LLVM_VERSION" - - if [ -f "$TOOLCHAIN_DIR/$toolchain_name.tar.zstd" ]; then - info "Skip downloading the toolchain. It is already there." - else - info "Pulling down the toolchain..." - http_pull "serene_toolchain" "$LLVM_MAJOR_VERSION.$LLVM_VERSION" "$TOOLCHAIN_DIR/$toolchain_name.tar.zstd" - fi - - if [ -d "$TOOLCHAIN_DIR/$toolchain_name" ]; then - info "Skip unpacking the toolchain. It is already there." - return - fi - - info "Unpacking the toolchain..." - unpack "$TOOLCHAIN_DIR/$toolchain_name.tar.zstd" "$TOOLCHAIN_DIR/$toolchain_name" -} - -function setup_toolchain() { - local toolchain_name - toolchain_name="serene_toolchain.$LLVM_MAJOR_VERSION.$LLVM_VERSION" - - if [[ "$USE_SERENE_TOOLCHAIN" != "true" ]]; then - warn "Serene toolchain is disabled!" - else - pull_toolchain - export PATH="$TOOLCHAIN_DIR/$toolchain_name/bin:$PATH" - export SERENE_TOOLCHAIN_PATH="$TOOLCHAIN_DIR/$toolchain_name" - - fi -}