Create a bare minimum devfs container image

This commit is contained in:
Sameer Rahmani 2022-07-10 02:36:07 +01:00
parent 394f7827c1
commit 34fab627c4
6 changed files with 623 additions and 0 deletions

11
.env.example Normal file
View File

@ -0,0 +1,11 @@
SERENE_REGISTERY_USER=nologin
SERENE_REGISTERY_PASS=<token>
REGISTRY=rg.fr-par.scw.cloud/serene
# Path to a directory which the builder script will
# use to setup the dev env
DEV_FS_DIR=/home/lxsameer/src/serene/devfs
# Leave it as it is or if you want to use your own
# fs repo change the address to your copy.
SERENE_FS_REPO=https://dl.serene-lang.org/devfs/

View File

@ -0,0 +1,11 @@
FROM rg.fr-par.scw.cloud/serene/llvm:latest
WORKDIR /app
# For CI
COPY .pre-commit-config.yaml .
RUN apt-get update && \
apt-get install -y --no-install-recommends git python3 python3-pip && \
pip3 install pre-commit && \
git init . && \
pre-commit autoupdate && \
rm -fv /app/.pre-commit-config.yaml

226
scripts/containers.sh Normal file
View File

@ -0,0 +1,226 @@
#! /bin/bash
# Serene Programming Language
#
# Copyright (c) 2019-2022 Sameer Rahmani <lxsameer@gnu.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# -----------------------------------------------------------------------------
# Commentary
# -----------------------------------------------------------------------------
# This file contains some helper functions to build the OCI images for
# development and production purposes of Serene.
#
# We use `Buildkit` to build the images and `podman` to run them.
#
# NOTE:
# REGISTERY comes frome the `.env` file in the root of the project
#
# NOTE:
# If you run into an issue like this with podman:
#
# WARN[0000] Error running newgidmap: exit status 1: newgidmap: gid range [1-1) -> [0-0)
# not allowed
# WARN[0000] Falling back to single mapping
#
# with podman or buildah try the following commands as root:
#
# usermod --add-subgids 1001000000-1001999999 YOUR_USER_NAME
# usermod --add-subgids 1001000000-1001999999 YOUR_USER_NAME
#
# or set the subuid and guid manually in `/etc/subuid` and `/etc/subgid`
set -e
export BUILDER_NAME="multiarch"
export PLATFORMS=('amd64' 'arm64')
function setup_builder() {
info "Creating the builder container"
sudo podman run --privileged --name "$BUILDER_NAME" \
docker.io/multiarch/qemu-user-static --reset -p yes
}
function cleanup_builder() {
info "Stopping the builder"
sudo podman stop "$BUILDER_NAME"
sudo podman rm "$BUILDER_NAME"
}
function create_manifest() {
local manifest="$1"
info "Remove the manifest if it exists"
buildah manifest rm "${manifest}" || true
info "Creating the manifest"
buildah manifest create "${manifest}" || warn "Manifest exists"
}
function build_container_image() {
local IMAGE_NAME="$1"
local LLVM_VERSION="$2"
local DOCKERFILE="$3"
local ROOT="$4"
local MANIFEST
local IMAGE
MANIFEST="serene/$1:${LLVM_VERSION}-$(git describe)"
IMAGE="$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)"
create_manifest "${MANIFEST}"
for ARCH in "${PLATFORMS[@]}"; do
info "Building the multiarch '$IMAGE_NAME' images for:"
info "VERSION: $LLVM_VERSION | ARCH: $ARCH"
buildah build \
--jobs="$(nproc)" \
--arch "$ARCH" \
--layers \
--manifest "${MANIFEST}" \
-f "$DOCKERFILE" \
-t "$IMAGE" \
--build-arg VERSION="$LLVM_VERSION" \
"$ROOT"
# info "Tagging the image '$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)' as latest"
# buildah tag \
# "$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)" \
# "$REGISTRY/$IMAGE_NAME:latest"
done
info "inspect ${MANIFEST}"
buildah manifest inspect "${MANIFEST}"
info "first push docker://$IMAGE"
buildah manifest push --all "${MANIFEST}" \
"docker://$IMAGE"
info "second push docker://$REGISTRY/$IMAGE_NAME:latest"
buildah manifest push --all "${MANIFEST}" \
"docker://$REGISTRY/$IMAGE_NAME:latest"
}
function build_llvm() {
local LLVM_VERSION="$1"
local ROOT="$2"
build_container_image "llvm" "$LLVM_VERSION" "$ROOT/resources/docker/llvm/Dockerfile" "$ROOT"
}
function build_ci() {
local LLVM_VERSION="$1"
local ROOT="$2"
build_container_image "ci" "$LLVM_VERSION" "$ROOT/resources/docker/llvm/Dockerfile.ci" "$ROOT"
}
function push_images() {
local image="$1"
local manifest
manifest="serene/$image"
buildah manifest push "${manifest}" \
--creds "$SERENE_REGISTERY_USER:$SERENE_REGISTERY_PASS" \
--all
}
function setup_builder1() {
if ! docker buildx inspect --builder "$BUILDER_NAME"; then
info "Creating the builder '$BUILDER_NAME'"
docker buildx create --driver docker-container \
--name "$BUILDER_NAME" \
--platform "linux/amd64,linux/arm64" \
--use \
--bootstrap
else
info "The builder '$BUILDER_NAME' already exists."
fi
}
# Params:
# 2nd: Image tag to use it should be the major number of llvm version
# 3rd: Project root
function build_llvm_multiarch() {
setup_builder
local IMAGE_NAME="llvm"
local LLVM_VERSION="$1"
local ROOT="$2"
info "Building the multiarch llvm images for:"
info "VERSION: $LLVM_VERSION | Platforms: $PLATFORMS"
docker buildx build --platform "$PLATFORMS" \
--builder "$BUILDER_NAME" --push \
-f "$ROOT/resources/docker/llvm/Dockerfile" \
-t "$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)" \
--build-arg VERSION="$LLVM_VERSION" \
"$ROOT"
info "Tagging the image '$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)' as latest"
docker tag \
"$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)" \
"$REGISTRY/$IMAGE_NAME:latest"
}
function build_ci_image() {
setup_builder
local LLVM_VERSION="$1"
local ROOT="$2"
local IMAGE
IMAGE="$REGISTRY/ci:${LLVM_VERSION}-$(git describe)"
info "Building the CI images"
docker buildx build --platform "linux/arm64" \
--builder "$BUILDER_NAME" --load \
-f "$ROOT/resources/docker/llvm/Dockerfile.ci" \
-t "$IMAGE" \
--build-arg VERSION="$2" \
"$ROOT"
info "Finished building '$IMAGE'"
info "Tagging the image '$IMAGE' as latest"
docker tag \
"$IMAGE" \
"$REGISTRY/ci:latest"
}
function push_images() {
local LLVM_VERSION="$1"
local ROOT="$2"
info "Loging into registry"
docker login "$REGISTRY" -u "$SERENE_REGISTERY_USER" -p "$SERENE_REGISTERY_PASS"
info "Push the LLVM image"
push "$REGISTRY/llvm:${LLVM_VERSION}-$(git describe)"
push "$REGISTRY/llvm:latest"
info "Push the CI image"
push "$REGISTRY/ci:${LLVM_VERSION}-$(git describe)"
push "$REGISTRY/ci:latest"
}

187
scripts/devfs.sh Normal file
View File

@ -0,0 +1,187 @@
#! /bin/bash
# Serene Programming Language
#
# Copyright (c) 2019-2022 Sameer Rahmani <lxsameer@gnu.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# -----------------------------------------------------------------------------
# Commentary
# -----------------------------------------------------------------------------
# Bunch of helper function to create a container like environment to develop
# Serene on GNU/Linux without going through the headache of compiling LLVM
# 5 times a day :D
set -e
function as_root() {
local rootfs="$1"
sudo unshare \
-w "/serene" \
--uts \
--ipc \
--pid \
--fork \
--kill-child \
--cgroup \
--mount \
--mount-proc \
--root="$rootfs" \
"${@:2}"
}
function rootless() {
local rootfs="$1"
unshare \
-w "/serene" \
--uts \
-c \
--ipc \
--pid \
--fork \
--kill-child \
--cgroup \
--mount \
--mount-proc \
--root="$rootfs" \
"${@:2}"
}
function download_devfs() {
local repo="$1"
local target="$2"
info "Downloading the tarball from '$repo'"
wget "$repo/fs.latest.tar.xz" -O "$target"
}
function extract_devfs() {
local tarball="$1"
local to="$2"
info "Extracting the tarball..."
mkdir -p "$to"
tar Jxf "$tarball" -C "$to"
info "Create the 'serene' dir at the root"
mkdir -p "$to/serene"
}
function mount_serene {
local rootfs="$1"
local project_root="$2"
local serene_dir
serene_dir="$rootfs/serene"
mkdir -p "$serene_dir"
info "Mounting Serene's dir into '/serene'"
mountpoint -q "$serene_dir" || sudo mount --bind "$project_root" "$serene_dir"
}
function mount_trees() {
local rootfs="$1"
local project_root="$2"
mount_serene "$rootfs" "$project_root"
info "Mounting the 'tmpfs' at '$rootfs/tmp'"
mountpoint -q "$rootfs/tmp" || sudo mount -t tmpfs tmpfs "$rootfs/tmp"
info "Mounting 'dev' at '$rootfs/dev'"
mountpoint -q "$rootfs/dev" || sudo mount --bind /dev "$rootfs/dev"
}
function unmount_trees() {
local rootfs="$1"
info "Unmounting the 'serene' from '$rootfs/serene'"
mountpoint -q "$rootfs/serene" && sudo umount "$rootfs/serene"
info "Unmounting the 'tmpfs' from '$rootfs/tmp'"
mountpoint -q "$rootfs/tmp" && sudo umount "$rootfs/tmp"
info "Unmounting 'dev' from '$rootfs/dev'"
mountpoint -q "$rootfs/dev" && sudo umount "$rootfs/dev"
}
function init_devfs {
local rootfs="$1"
local project_root="$2"
local create_group
local create_user
create_group="groupadd -f -g$(id -g) $(whoami)"
create_user="adduser --uid $(id -u) --gid $(id -g) $(whoami) || true"
mount_serene "$rootfs" "$project_root"
as_root "$rootfs" bash -c "$create_group"
as_root "$rootfs" bash -c "$create_user"
as_root "$rootfs" bash -c "adduser $(whoami) sudo || true"
}
function create_debian_rootfs() {
local to="$1"
info "Pulling the debian docker image"
docker pull debian:sid-slim
info "Spinning up the container"
docker stop devfs || true
docker rm devfs || true
docker run --name devfs -d debian:sid-slim
sleep 2
info "Exporting the rootfs to '$to/rootfs.tar'"
docker export -o "$to/rootfs.tar" devfs
info "Tearing down the container"
docker stop devfs
docker rm devfs
}
function create_and_initialize_devfs_image() {
local to="$1"
local project_root="$2"
local llvm_version="$3"
local rootfs
rootfs="$to/rootfs/"
if [ ! -f "$to/rootfs.tar" ]; then
info "Creating the rootfs tar bar"
create_debian_rootfs "$to"
fi
mkdir -p "$rootfs"
if [ ! -f "$to/rootfs/etc/shadow" ]; then
info "Extracting the tarball"
tar xf "$to/rootfs.tar" -C "$rootfs"
fi
mount_trees "$rootfs" "$project_root"
#as_root "$rootfs" bash
as_root "$rootfs" bash -c "echo '$llvm_version' > /etc/llvm_version"
as_root "$rootfs" bash -c "echo 'export LANG=C.UTF-8' >> /etc/profile"
as_root "$rootfs" bash /serene/scripts/devfs_container_setup.sh
unmount_trees "$rootfs"
}

View File

@ -0,0 +1,142 @@
#! /bin/bash
# Serene Programming Language
#
# Copyright (c) 2019-2022 Sameer Rahmani <lxsameer@gnu.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# -----------------------------------------------------------------------------
# Commentary
# -----------------------------------------------------------------------------
# This file installs all the dependencies on the guest container during the
# devfs initialization.
set -e
# shellcheck source=/dev/null
source /serene/scripts/utils.sh
function install_llvm() {
wget https://apt.llvm.org/llvm.sh -O /root/llvm.sh
chmod +x llvm.sh
/root/llvm.sh "${LLVM_VERSION}" all
apt-get update --fix-missing
apt-get install -y --no-install-recommends \
mlir-"${LLVM_VERSION}"-tools \
libmlir-"${LLVM_VERSION}"-dev \
libmlir-"${LLVM_VERSION}" \
libmlir-"${LLVM_VERSION}"-dbgsym \
liblld-"${LLVM_VERSION}" \
liblld-"${LLVM_VERSION}"-dev \
clang-format-"${LLVM_VERSION}" \
clang-tidy-"${LLVM_VERSION}"
ln -s "$(which lld-"${LLVM_VERSION}")" /usr/bin/lld
ln -s "$(which clang-"${LLVM_VERSION}")" /usr/bin/clang
ln -s "$(which clang++-"${LLVM_VERSION}")" /usr/bin/clang++
ln -s "$(which clang-format-"${LLVM_VERSION}")" /usr/bin/clang-format
ln -s "$(which clang-tidy-"${LLVM_VERSION}")" /usr/bin/clang-tidy
ln -s "$(which mlir-tblgen-"${LLVM_VERSION}")" /usr/bin/mlir-tblgen
MLIR_DIR="/usr/lib/llvm-${LLVM_VERSION}"
CMAKE_PREFIX_PATH="/usr/lib/llvm-${LLVM_VERSION}"
LD_LIBRARY_PATH="/usr/lib/llvm-${LLVM_VERSION}/lib/clang/${LLVM_VERSION}.0.0/lib/linux/"
CC=/usr/bin/clang
CXX=/usr/bin/clang++
}
function install_iwuy() {
mkdir -p /opt/iwuy
pushd /opt/iwuy
git clone https://github.com/include-what-you-use/include-what-you-use.git --depth 1
mkdir build
pushd build
cmake -G Ninja -DCMAKE_PREFIX_PATH="/usr/lib/llvm-${LLVM_VERSION}" ../include-what-you-use
cmake --build .
cmake -P cmake_install.cmake
popd
popd
rm -rf /opt/iwuy
}
function install_boehm() {
mkdir -p /opt/boehm
pushd /opt/boehm
git clone https://github.com/ivmai/bdwgc.git --depth 1 --branch v8.2.0
mkdir build
pushd build
cmake -G Ninja -DBUILD_SHARED_LIBS=OFF -Denable_cplusplus=ON -Denable_threads=ON \
-Denable_gcj_support=OFF -Dinstall_headers=ON \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON ../bdwgc
cmake --build . --config Release
cmake -P cmake_install.cmake
popd
popd
rm -rf /opt/boehm
}
function main() {
pushd "/root"
apt-get update
apt-get install --no-install-recommends -y \
gnupg \
cmake \
ccache \
git \
ninja-build \
binutils \
lsb-release \
wget \
software-properties-common \
zlib1g \
cppcheck \
sudo \
shellcheck \
zlib1g-dev
# install_llvm
# install_iwuy
# install_boehm
popd
info "Enabling passwordless sudo"
sed 's/%sudo.*/%sudo ALL=(ALL) NOPASSWD:ALL/' -i /etc/sudoers
apt-get autoremove -y
apt-get clean
}
if [ ! -f "/etc/llvm_version" ]; then
error "Can't find '/etc/llvm_version' on the container"
exit 1
fi
export LANG=C.UTF-8
export LLVM_VERSION
export MLIR_DIR
export CMAKE_PREFIX_PATH
export LD_LIBRARY_PATH
export CC
export CXX
LLVM_VERSION=$(cat /etc/llvm_version)
main

46
scripts/utils.sh Normal file
View File

@ -0,0 +1,46 @@
#! /bin/bash
# Serene Programming Language
#
# Copyright (c) 2019-2022 Sameer Rahmani <lxsameer@gnu.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
set -e
# -----------------------------------------------------------------------------
# Helper functions
# -----------------------------------------------------------------------------
function fn-names() {
grep -E '^function [0-9a-zA-Z_-]+\(\) \{ ## .*$$' "$0" | sed 's/^function \([a-zA-Z0-9_-]*\)() { ## \(.*\)/\1/'
}
function info() {
if [ "$1" ]
then
echo -e "[\033[01;32mINFO\033[00m]: $*"
fi
}
function error() {
if [ "$1" ]
then
echo -e "[\033[01;31mERR\033[00m]: $*"
fi
}
function warn() {
if [ "$1" ]
then
echo -e "[\033[01;33mWARN\033[00m]: $*"
fi
}