From 34fab627c4700f872b05849e62cd636e6f10506e Mon Sep 17 00:00:00 2001 From: Sameer Rahmani Date: Sun, 10 Jul 2022 02:36:07 +0100 Subject: [PATCH] Create a bare minimum devfs container image --- .env.example | 11 ++ resources/docker/llvm/Dockerfile.ci | 11 ++ scripts/containers.sh | 226 ++++++++++++++++++++++++++++ scripts/devfs.sh | 187 +++++++++++++++++++++++ scripts/devfs_container_setup.sh | 142 +++++++++++++++++ scripts/utils.sh | 46 ++++++ 6 files changed, 623 insertions(+) create mode 100644 .env.example create mode 100644 resources/docker/llvm/Dockerfile.ci create mode 100644 scripts/containers.sh create mode 100644 scripts/devfs.sh create mode 100644 scripts/devfs_container_setup.sh create mode 100644 scripts/utils.sh diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..8e29d63 --- /dev/null +++ b/.env.example @@ -0,0 +1,11 @@ +SERENE_REGISTERY_USER=nologin +SERENE_REGISTERY_PASS= +REGISTRY=rg.fr-par.scw.cloud/serene + +# Path to a directory which the builder script will +# use to setup the dev env +DEV_FS_DIR=/home/lxsameer/src/serene/devfs + +# Leave it as it is or if you want to use your own +# fs repo change the address to your copy. +SERENE_FS_REPO=https://dl.serene-lang.org/devfs/ \ No newline at end of file diff --git a/resources/docker/llvm/Dockerfile.ci b/resources/docker/llvm/Dockerfile.ci new file mode 100644 index 0000000..2acf41a --- /dev/null +++ b/resources/docker/llvm/Dockerfile.ci @@ -0,0 +1,11 @@ +FROM rg.fr-par.scw.cloud/serene/llvm:latest + +WORKDIR /app +# For CI +COPY .pre-commit-config.yaml . +RUN apt-get update && \ + apt-get install -y --no-install-recommends git python3 python3-pip && \ + pip3 install pre-commit && \ + git init . && \ + pre-commit autoupdate && \ + rm -fv /app/.pre-commit-config.yaml diff --git a/scripts/containers.sh b/scripts/containers.sh new file mode 100644 index 0000000..0338dc2 --- /dev/null +++ b/scripts/containers.sh @@ -0,0 +1,226 @@ +#! /bin/bash +# Serene Programming Language +# +# Copyright (c) 2019-2022 Sameer Rahmani +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ----------------------------------------------------------------------------- +# Commentary +# ----------------------------------------------------------------------------- +# This file contains some helper functions to build the OCI images for +# development and production purposes of Serene. +# +# We use `Buildkit` to build the images and `podman` to run them. +# +# NOTE: +# REGISTERY comes frome the `.env` file in the root of the project +# +# NOTE: +# If you run into an issue like this with podman: +# +# WARN[0000] Error running newgidmap: exit status 1: newgidmap: gid range [1-1) -> [0-0) +# not allowed +# WARN[0000] Falling back to single mapping +# +# with podman or buildah try the following commands as root: +# +# usermod --add-subgids 1001000000-1001999999 YOUR_USER_NAME +# usermod --add-subgids 1001000000-1001999999 YOUR_USER_NAME +# +# or set the subuid and guid manually in `/etc/subuid` and `/etc/subgid` + + +set -e + +export BUILDER_NAME="multiarch" +export PLATFORMS=('amd64' 'arm64') + +function setup_builder() { + info "Creating the builder container" + sudo podman run --privileged --name "$BUILDER_NAME" \ + docker.io/multiarch/qemu-user-static --reset -p yes +} + +function cleanup_builder() { + info "Stopping the builder" + sudo podman stop "$BUILDER_NAME" + sudo podman rm "$BUILDER_NAME" +} + +function create_manifest() { + local manifest="$1" + + info "Remove the manifest if it exists" + buildah manifest rm "${manifest}" || true + info "Creating the manifest" + buildah manifest create "${manifest}" || warn "Manifest exists" +} + +function build_container_image() { + local IMAGE_NAME="$1" + local LLVM_VERSION="$2" + local DOCKERFILE="$3" + local ROOT="$4" + local MANIFEST + local IMAGE + + MANIFEST="serene/$1:${LLVM_VERSION}-$(git describe)" + IMAGE="$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)" + + create_manifest "${MANIFEST}" + + for ARCH in "${PLATFORMS[@]}"; do + info "Building the multiarch '$IMAGE_NAME' images for:" + info "VERSION: $LLVM_VERSION | ARCH: $ARCH" + + buildah build \ + --jobs="$(nproc)" \ + --arch "$ARCH" \ + --layers \ + --manifest "${MANIFEST}" \ + -f "$DOCKERFILE" \ + -t "$IMAGE" \ + --build-arg VERSION="$LLVM_VERSION" \ + "$ROOT" + + # info "Tagging the image '$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)' as latest" + # buildah tag \ + # "$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)" \ + # "$REGISTRY/$IMAGE_NAME:latest" + done + + + info "inspect ${MANIFEST}" + buildah manifest inspect "${MANIFEST}" + info "first push docker://$IMAGE" + buildah manifest push --all "${MANIFEST}" \ + "docker://$IMAGE" + info "second push docker://$REGISTRY/$IMAGE_NAME:latest" + buildah manifest push --all "${MANIFEST}" \ + "docker://$REGISTRY/$IMAGE_NAME:latest" +} + +function build_llvm() { + local LLVM_VERSION="$1" + local ROOT="$2" + + build_container_image "llvm" "$LLVM_VERSION" "$ROOT/resources/docker/llvm/Dockerfile" "$ROOT" +} + +function build_ci() { + local LLVM_VERSION="$1" + local ROOT="$2" + + build_container_image "ci" "$LLVM_VERSION" "$ROOT/resources/docker/llvm/Dockerfile.ci" "$ROOT" +} + +function push_images() { + local image="$1" + + local manifest + manifest="serene/$image" + + buildah manifest push "${manifest}" \ + --creds "$SERENE_REGISTERY_USER:$SERENE_REGISTERY_PASS" \ + --all +} + + + + + + + + +function setup_builder1() { + if ! docker buildx inspect --builder "$BUILDER_NAME"; then + info "Creating the builder '$BUILDER_NAME'" + docker buildx create --driver docker-container \ + --name "$BUILDER_NAME" \ + --platform "linux/amd64,linux/arm64" \ + --use \ + --bootstrap + else + info "The builder '$BUILDER_NAME' already exists." + fi +} + +# Params: +# 2nd: Image tag to use it should be the major number of llvm version +# 3rd: Project root +function build_llvm_multiarch() { + + setup_builder + + local IMAGE_NAME="llvm" + local LLVM_VERSION="$1" + local ROOT="$2" + + info "Building the multiarch llvm images for:" + info "VERSION: $LLVM_VERSION | Platforms: $PLATFORMS" + docker buildx build --platform "$PLATFORMS" \ + --builder "$BUILDER_NAME" --push \ + -f "$ROOT/resources/docker/llvm/Dockerfile" \ + -t "$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)" \ + --build-arg VERSION="$LLVM_VERSION" \ + "$ROOT" + + info "Tagging the image '$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)' as latest" + docker tag \ + "$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)" \ + "$REGISTRY/$IMAGE_NAME:latest" +} + + +function build_ci_image() { + setup_builder + + local LLVM_VERSION="$1" + local ROOT="$2" + local IMAGE + + IMAGE="$REGISTRY/ci:${LLVM_VERSION}-$(git describe)" + + info "Building the CI images" + docker buildx build --platform "linux/arm64" \ + --builder "$BUILDER_NAME" --load \ + -f "$ROOT/resources/docker/llvm/Dockerfile.ci" \ + -t "$IMAGE" \ + --build-arg VERSION="$2" \ + "$ROOT" + + info "Finished building '$IMAGE'" + info "Tagging the image '$IMAGE' as latest" + docker tag \ + "$IMAGE" \ + "$REGISTRY/ci:latest" + +} + + +function push_images() { + local LLVM_VERSION="$1" + local ROOT="$2" + + info "Loging into registry" + docker login "$REGISTRY" -u "$SERENE_REGISTERY_USER" -p "$SERENE_REGISTERY_PASS" + + info "Push the LLVM image" + push "$REGISTRY/llvm:${LLVM_VERSION}-$(git describe)" + push "$REGISTRY/llvm:latest" + + info "Push the CI image" + push "$REGISTRY/ci:${LLVM_VERSION}-$(git describe)" + push "$REGISTRY/ci:latest" +} diff --git a/scripts/devfs.sh b/scripts/devfs.sh new file mode 100644 index 0000000..5ba10d6 --- /dev/null +++ b/scripts/devfs.sh @@ -0,0 +1,187 @@ +#! /bin/bash +# Serene Programming Language +# +# Copyright (c) 2019-2022 Sameer Rahmani +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ----------------------------------------------------------------------------- +# Commentary +# ----------------------------------------------------------------------------- +# Bunch of helper function to create a container like environment to develop +# Serene on GNU/Linux without going through the headache of compiling LLVM +# 5 times a day :D + +set -e + + +function as_root() { + local rootfs="$1" + sudo unshare \ + -w "/serene" \ + --uts \ + --ipc \ + --pid \ + --fork \ + --kill-child \ + --cgroup \ + --mount \ + --mount-proc \ + --root="$rootfs" \ + "${@:2}" +} + + +function rootless() { + local rootfs="$1" + unshare \ + -w "/serene" \ + --uts \ + -c \ + --ipc \ + --pid \ + --fork \ + --kill-child \ + --cgroup \ + --mount \ + --mount-proc \ + --root="$rootfs" \ + "${@:2}" +} + +function download_devfs() { + local repo="$1" + local target="$2" + info "Downloading the tarball from '$repo'" + wget "$repo/fs.latest.tar.xz" -O "$target" +} + + +function extract_devfs() { + local tarball="$1" + local to="$2" + + info "Extracting the tarball..." + mkdir -p "$to" + tar Jxf "$tarball" -C "$to" + + info "Create the 'serene' dir at the root" + mkdir -p "$to/serene" +} + +function mount_serene { + local rootfs="$1" + local project_root="$2" + local serene_dir + + serene_dir="$rootfs/serene" + + mkdir -p "$serene_dir" + + info "Mounting Serene's dir into '/serene'" + mountpoint -q "$serene_dir" || sudo mount --bind "$project_root" "$serene_dir" +} + +function mount_trees() { + local rootfs="$1" + local project_root="$2" + + mount_serene "$rootfs" "$project_root" + + info "Mounting the 'tmpfs' at '$rootfs/tmp'" + mountpoint -q "$rootfs/tmp" || sudo mount -t tmpfs tmpfs "$rootfs/tmp" + + info "Mounting 'dev' at '$rootfs/dev'" + mountpoint -q "$rootfs/dev" || sudo mount --bind /dev "$rootfs/dev" +} + + +function unmount_trees() { + local rootfs="$1" + + info "Unmounting the 'serene' from '$rootfs/serene'" + mountpoint -q "$rootfs/serene" && sudo umount "$rootfs/serene" + + info "Unmounting the 'tmpfs' from '$rootfs/tmp'" + mountpoint -q "$rootfs/tmp" && sudo umount "$rootfs/tmp" + + info "Unmounting 'dev' from '$rootfs/dev'" + mountpoint -q "$rootfs/dev" && sudo umount "$rootfs/dev" +} + +function init_devfs { + local rootfs="$1" + local project_root="$2" + local create_group + local create_user + + create_group="groupadd -f -g$(id -g) $(whoami)" + create_user="adduser --uid $(id -u) --gid $(id -g) $(whoami) || true" + + mount_serene "$rootfs" "$project_root" + + as_root "$rootfs" bash -c "$create_group" + as_root "$rootfs" bash -c "$create_user" + as_root "$rootfs" bash -c "adduser $(whoami) sudo || true" +} + +function create_debian_rootfs() { + local to="$1" + + info "Pulling the debian docker image" + docker pull debian:sid-slim + + info "Spinning up the container" + docker stop devfs || true + docker rm devfs || true + docker run --name devfs -d debian:sid-slim + sleep 2 + + info "Exporting the rootfs to '$to/rootfs.tar'" + docker export -o "$to/rootfs.tar" devfs + + info "Tearing down the container" + docker stop devfs + docker rm devfs +} + +function create_and_initialize_devfs_image() { + local to="$1" + local project_root="$2" + local llvm_version="$3" + local rootfs + + rootfs="$to/rootfs/" + + if [ ! -f "$to/rootfs.tar" ]; then + info "Creating the rootfs tar bar" + create_debian_rootfs "$to" + fi + + mkdir -p "$rootfs" + + if [ ! -f "$to/rootfs/etc/shadow" ]; then + info "Extracting the tarball" + tar xf "$to/rootfs.tar" -C "$rootfs" + fi + + mount_trees "$rootfs" "$project_root" + + #as_root "$rootfs" bash + + as_root "$rootfs" bash -c "echo '$llvm_version' > /etc/llvm_version" + as_root "$rootfs" bash -c "echo 'export LANG=C.UTF-8' >> /etc/profile" + as_root "$rootfs" bash /serene/scripts/devfs_container_setup.sh + + unmount_trees "$rootfs" +} diff --git a/scripts/devfs_container_setup.sh b/scripts/devfs_container_setup.sh new file mode 100644 index 0000000..f9fe15a --- /dev/null +++ b/scripts/devfs_container_setup.sh @@ -0,0 +1,142 @@ +#! /bin/bash +# Serene Programming Language +# +# Copyright (c) 2019-2022 Sameer Rahmani +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ----------------------------------------------------------------------------- +# Commentary +# ----------------------------------------------------------------------------- +# This file installs all the dependencies on the guest container during the +# devfs initialization. + +set -e + +# shellcheck source=/dev/null +source /serene/scripts/utils.sh + + +function install_llvm() { + + wget https://apt.llvm.org/llvm.sh -O /root/llvm.sh + chmod +x llvm.sh + + /root/llvm.sh "${LLVM_VERSION}" all + + apt-get update --fix-missing + apt-get install -y --no-install-recommends \ + mlir-"${LLVM_VERSION}"-tools \ + libmlir-"${LLVM_VERSION}"-dev \ + libmlir-"${LLVM_VERSION}" \ + libmlir-"${LLVM_VERSION}"-dbgsym \ + liblld-"${LLVM_VERSION}" \ + liblld-"${LLVM_VERSION}"-dev \ + clang-format-"${LLVM_VERSION}" \ + clang-tidy-"${LLVM_VERSION}" + + ln -s "$(which lld-"${LLVM_VERSION}")" /usr/bin/lld + ln -s "$(which clang-"${LLVM_VERSION}")" /usr/bin/clang + ln -s "$(which clang++-"${LLVM_VERSION}")" /usr/bin/clang++ + ln -s "$(which clang-format-"${LLVM_VERSION}")" /usr/bin/clang-format + ln -s "$(which clang-tidy-"${LLVM_VERSION}")" /usr/bin/clang-tidy + ln -s "$(which mlir-tblgen-"${LLVM_VERSION}")" /usr/bin/mlir-tblgen + + MLIR_DIR="/usr/lib/llvm-${LLVM_VERSION}" + CMAKE_PREFIX_PATH="/usr/lib/llvm-${LLVM_VERSION}" + LD_LIBRARY_PATH="/usr/lib/llvm-${LLVM_VERSION}/lib/clang/${LLVM_VERSION}.0.0/lib/linux/" + CC=/usr/bin/clang + CXX=/usr/bin/clang++ +} + +function install_iwuy() { + mkdir -p /opt/iwuy + pushd /opt/iwuy + git clone https://github.com/include-what-you-use/include-what-you-use.git --depth 1 + mkdir build + pushd build + cmake -G Ninja -DCMAKE_PREFIX_PATH="/usr/lib/llvm-${LLVM_VERSION}" ../include-what-you-use + cmake --build . + cmake -P cmake_install.cmake + popd + popd + + rm -rf /opt/iwuy +} + +function install_boehm() { + mkdir -p /opt/boehm + pushd /opt/boehm + git clone https://github.com/ivmai/bdwgc.git --depth 1 --branch v8.2.0 + mkdir build + pushd build + cmake -G Ninja -DBUILD_SHARED_LIBS=OFF -Denable_cplusplus=ON -Denable_threads=ON \ + -Denable_gcj_support=OFF -Dinstall_headers=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON ../bdwgc + cmake --build . --config Release + cmake -P cmake_install.cmake + popd + popd + + rm -rf /opt/boehm +} + +function main() { + pushd "/root" + + apt-get update + + apt-get install --no-install-recommends -y \ + gnupg \ + cmake \ + ccache \ + git \ + ninja-build \ + binutils \ + lsb-release \ + wget \ + software-properties-common \ + zlib1g \ + cppcheck \ + sudo \ + shellcheck \ + zlib1g-dev + + # install_llvm + # install_iwuy + # install_boehm + popd + + info "Enabling passwordless sudo" + sed 's/%sudo.*/%sudo ALL=(ALL) NOPASSWD:ALL/' -i /etc/sudoers + + apt-get autoremove -y + apt-get clean +} + +if [ ! -f "/etc/llvm_version" ]; then + error "Can't find '/etc/llvm_version' on the container" + exit 1 +fi + +export LANG=C.UTF-8 +export LLVM_VERSION +export MLIR_DIR +export CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH +export CC +export CXX + +LLVM_VERSION=$(cat /etc/llvm_version) + +main diff --git a/scripts/utils.sh b/scripts/utils.sh new file mode 100644 index 0000000..553b4da --- /dev/null +++ b/scripts/utils.sh @@ -0,0 +1,46 @@ +#! /bin/bash +# Serene Programming Language +# +# Copyright (c) 2019-2022 Sameer Rahmani +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -e + +# ----------------------------------------------------------------------------- +# Helper functions +# ----------------------------------------------------------------------------- +function fn-names() { + grep -E '^function [0-9a-zA-Z_-]+\(\) \{ ## .*$$' "$0" | sed 's/^function \([a-zA-Z0-9_-]*\)() { ## \(.*\)/\1/' +} + +function info() { + if [ "$1" ] + then + echo -e "[\033[01;32mINFO\033[00m]: $*" + fi +} + +function error() { + if [ "$1" ] + then + echo -e "[\033[01;31mERR\033[00m]: $*" + fi +} + +function warn() { + if [ "$1" ] + then + echo -e "[\033[01;33mWARN\033[00m]: $*" + fi +}