Remove redundant bash script from 'scripts' dir

This commit is contained in:
Sameer Rahmani 2024-01-16 00:15:54 +00:00
parent 7c62a58271
commit 2e230b9474
Signed by: lxsameer
GPG Key ID: B0A4AF28AB9FD90B
4 changed files with 0 additions and 563 deletions

37
builder
View File

@ -48,22 +48,6 @@ ME=$(cd "$(dirname "$0")/." >/dev/null 2>&1 ; pwd -P)
# shellcheck source=./scripts/utils.sh
source "$ME/scripts/utils.sh"
# shellcheck source=./scripts/devfs.sh
source "$ME/scripts/devfs.sh"
# -----------------------------------------------------------------------------
# Dependencies
# -----------------------------------------------------------------------------
# The builder script can download the toolchain for linux X86_64 and set it up.
# The toolchain is based on Musl, but you don't have to use it. But be aware
# that using glibc with serene means that you can't cross compile with it.
# Here is the version of llvm that we use
LLVM_VERSION="da3cd333bea572fb10470f610a27f22bcb84b08c"
LLVM_MAJOR_VERSION="16"
export LLVM_VERSION LLVM_MAJOR_VERSION
# -----------------------------------------------------------------------------
# CONFIG VARS
# -----------------------------------------------------------------------------
@ -79,9 +63,6 @@ if [[ "$CXX" = "" ]]; then
export CXX
fi
# The repository to push/pull packages to/from.
DEV_HEROES="https://devheroes.codes"
BUILD_DIR_NAME="build"
export BUILD_DIR_NAME
@ -91,14 +72,6 @@ export BUILD_DIR
SERENE_HOME_DIR="$HOME/.serene"
export SERENE_HOME_DIR
TOOLCHAIN_DIR="$SERENE_HOME_DIR/toolchain"
export TOOLCHAIN_DIR
# Set this to anything beside true if you don't want the builder
# script to manage the toolchain. E.g. you have your own toolchain
USE_SERENE_TOOLCHAIN="true"
export USE_SERENE_TOOLCHAIN
# Serene subprojects. We use this array to run common tasks on all the projects
# like running the test cases
PROJECTS=(libserene serenec serene-repl serene-tblgen)
@ -113,16 +86,6 @@ export ASAN_OPTIONS
LSAN_OPTIONS=suppressions="$ME/.ignore_sanitize"
export LSAN_OPTIONS
# shellcheck source=./scripts/toolchain.sh
#source "$ME/scripts/toolchain.sh"
# -----------------------------------------------------------------------------
# Initialization
# -----------------------------------------------------------------------------
#mkdir -p "$TOOLCHAIN_DIR"
# -----------------------------------------------------------------------------
# Helper functions
# -----------------------------------------------------------------------------

View File

@ -1,226 +0,0 @@
#! /bin/bash
# Serene Programming Language
#
# Copyright (c) 2019-2024 Sameer Rahmani <lxsameer@gnu.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# -----------------------------------------------------------------------------
# Commentary
# -----------------------------------------------------------------------------
# This file contains some helper functions to build the OCI images for
# development and production purposes of Serene.
#
# We use `Buildkit` to build the images and `podman` to run them.
#
# NOTE:
# REGISTERY comes frome the `.env` file in the root of the project
#
# NOTE:
# If you run into an issue like this with podman:
#
# WARN[0000] Error running newgidmap: exit status 1: newgidmap: gid range [1-1) -> [0-0)
# not allowed
# WARN[0000] Falling back to single mapping
#
# with podman or buildah try the following commands as root:
#
# usermod --add-subgids 1001000000-1001999999 YOUR_USER_NAME
# usermod --add-subgids 1001000000-1001999999 YOUR_USER_NAME
#
# or set the subuid and guid manually in `/etc/subuid` and `/etc/subgid`
set -e
export BUILDER_NAME="multiarch"
export PLATFORMS=('amd64' 'arm64')
function setup_builder() {
info "Creating the builder container"
sudo podman run --privileged --name "$BUILDER_NAME" \
docker.io/multiarch/qemu-user-static --reset -p yes
}
function cleanup_builder() {
info "Stopping the builder"
sudo podman stop "$BUILDER_NAME"
sudo podman rm "$BUILDER_NAME"
}
function create_manifest() {
local manifest="$1"
info "Remove the manifest if it exists"
buildah manifest rm "${manifest}" || true
info "Creating the manifest"
buildah manifest create "${manifest}" || warn "Manifest exists"
}
function build_container_image() {
local IMAGE_NAME="$1"
local LLVM_VERSION="$2"
local DOCKERFILE="$3"
local ROOT="$4"
local MANIFEST
local IMAGE
MANIFEST="serene/$1:${LLVM_VERSION}-$(git describe)"
IMAGE="$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)"
create_manifest "${MANIFEST}"
for ARCH in "${PLATFORMS[@]}"; do
info "Building the multiarch '$IMAGE_NAME' images for:"
info "VERSION: $LLVM_VERSION | ARCH: $ARCH"
buildah build \
--jobs="$(nproc)" \
--arch "$ARCH" \
--layers \
--manifest "${MANIFEST}" \
-f "$DOCKERFILE" \
-t "$IMAGE" \
--build-arg VERSION="$LLVM_VERSION" \
"$ROOT"
# info "Tagging the image '$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)' as latest"
# buildah tag \
# "$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)" \
# "$REGISTRY/$IMAGE_NAME:latest"
done
info "inspect ${MANIFEST}"
buildah manifest inspect "${MANIFEST}"
info "first push docker://$IMAGE"
buildah manifest push --all "${MANIFEST}" \
"docker://$IMAGE"
info "second push docker://$REGISTRY/$IMAGE_NAME:latest"
buildah manifest push --all "${MANIFEST}" \
"docker://$REGISTRY/$IMAGE_NAME:latest"
}
function build_llvm() {
local LLVM_VERSION="$1"
local ROOT="$2"
build_container_image "llvm" "$LLVM_VERSION" "$ROOT/resources/docker/llvm/Dockerfile" "$ROOT"
}
function build_ci() {
local LLVM_VERSION="$1"
local ROOT="$2"
build_container_image "ci" "$LLVM_VERSION" "$ROOT/resources/docker/llvm/Dockerfile.ci" "$ROOT"
}
function push_images() {
local image="$1"
local manifest
manifest="serene/$image"
buildah manifest push "${manifest}" \
--creds "$SERENE_REGISTERY_USER:$SERENE_REGISTERY_PASS" \
--all
}
function setup_builder1() {
if ! docker buildx inspect --builder "$BUILDER_NAME"; then
info "Creating the builder '$BUILDER_NAME'"
docker buildx create --driver docker-container \
--name "$BUILDER_NAME" \
--platform "linux/amd64,linux/arm64" \
--use \
--bootstrap
else
info "The builder '$BUILDER_NAME' already exists."
fi
}
# Params:
# 2nd: Image tag to use it should be the major number of llvm version
# 3rd: Project root
function build_llvm_multiarch() {
setup_builder
local IMAGE_NAME="llvm"
local LLVM_VERSION="$1"
local ROOT="$2"
info "Building the multiarch llvm images for:"
info "VERSION: $LLVM_VERSION | Platforms: ${PLATFORMS[*]}"
docker buildx build --platform "${PLATFORMS[@]}" \
--builder "$BUILDER_NAME" --push \
-f "$ROOT/resources/docker/llvm/Dockerfile" \
-t "$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)" \
--build-arg VERSION="$LLVM_VERSION" \
"$ROOT"
info "Tagging the image '$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)' as latest"
docker tag \
"$REGISTRY/$IMAGE_NAME:${LLVM_VERSION}-$(git describe)" \
"$REGISTRY/$IMAGE_NAME:latest"
}
function build_ci_image() {
setup_builder
local LLVM_VERSION="$1"
local ROOT="$2"
local IMAGE
IMAGE="$REGISTRY/ci:${LLVM_VERSION}-$(git describe)"
info "Building the CI images"
docker buildx build --platform "linux/arm64" \
--builder "$BUILDER_NAME" --load \
-f "$ROOT/resources/docker/llvm/Dockerfile.ci" \
-t "$IMAGE" \
--build-arg VERSION="$2" \
"$ROOT"
info "Finished building '$IMAGE'"
info "Tagging the image '$IMAGE' as latest"
docker tag \
"$IMAGE" \
"$REGISTRY/ci:latest"
}
function push_images() {
local LLVM_VERSION="$1"
local ROOT="$2"
info "Loging into registry"
docker login "$REGISTRY" -u "$SERENE_REGISTERY_USER" -p "$SERENE_REGISTERY_PASS"
info "Push the LLVM image"
push "$REGISTRY/llvm:${LLVM_VERSION}-$(git describe)"
push "$REGISTRY/llvm:latest"
info "Push the CI image"
push "$REGISTRY/ci:${LLVM_VERSION}-$(git describe)"
push "$REGISTRY/ci:latest"
}

View File

@ -1,238 +0,0 @@
#! /bin/bash
# Serene Programming Language
#
# Copyright (c) 2019-2024 Sameer Rahmani <lxsameer@gnu.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# -----------------------------------------------------------------------------
# Commentary
# -----------------------------------------------------------------------------
# Bunch of helper function to create a container like environment to develop
# Serene on GNU/Linux without going through the headache of compiling LLVM
# 5 times a day :D
set -e
function as_root() {
local rootfs="$1"
sudo unshare \
-w "/serene" \
--uts \
--ipc \
--pid \
--fork \
--kill-child \
--cgroup \
--mount \
--mount-proc \
--root="$rootfs" \
"${@:2}"
}
function rootless() {
local rootfs="$1"
unshare \
-w "/serene" \
--uts \
-c \
--ipc \
--pid \
--fork \
--kill-child \
--cgroup \
--mount \
--mount-proc \
--root="$rootfs" \
"${@:2}"
}
function download_devfs() {
local repo="$1"
local target="$2"
info "Downloading the tarball from '$repo'"
wget "$repo/fs.latest.tar.xz" -O "$target"
}
function extract_devfs() {
local tarball="$1"
local to="$2"
info "Extracting the tarball..."
mkdir -p "$to"
tar Jxf "$tarball" -C "$to"
info "Create the 'serene' dir at the root"
mkdir -p "$to/serene"
}
function mount_serene {
local rootfs="$1"
local project_root="$2"
local serene_dir
serene_dir="$rootfs/serene"
mkdir -p "$serene_dir"
info "Mounting Serene's dir into '/serene' in read-only mode"
mountpoint -q "$serene_dir" || sudo mount --rbind -o ro "$project_root" "$serene_dir"
}
function mount_trees() {
local rootfs="$1"
local project_root="$2"
mount_serene "$rootfs" "$project_root"
info "Mounting the 'tmpfs' at '$rootfs/tmp'"
mkdir -p "$rootfs/tmp"
mountpoint -q "$rootfs/tmp" || sudo mount -t tmpfs tmpfs "$rootfs/tmp"
info "Mounting 'dev' at '$rootfs/dev'"
mkdir -p "$rootfs/dev"
mountpoint -q "$rootfs/dev" || sudo mount --bind /dev "$rootfs/dev"
}
function unmount_trees() {
local rootfs="$1"
info "Unmounting the 'serene' from '$rootfs/serene'"
sudo umount "$rootfs/serene" &> /dev/null || true
info "Unmounting the 'tmpfs' from '$rootfs/tmp'"
sudo umount "$rootfs/tmp" &> /dev/null || true
info "Unmounting 'dev' from '$rootfs/dev'"
sudo umount "$rootfs/dev" &> /dev/null || true
}
function init_devfs {
local rootfs="$1"
local project_root="$2"
local create_group
local create_user
create_group="groupadd -f -g$(id -g) $(whoami)"
create_user="adduser -q --disabled-password --gecos '' --uid $(id -u) --gid $(id -g) $(whoami) || true"
mkdir -p "$rootfs/proc"
mount_trees "$rootfs" "$project_root"
info "Creating group '$(whoami)' with ID '$(id -g)'..."
as_root "$rootfs" bash --login -c "$create_group"
info "Creating user '$(whoami)' with ID '$(id -u)'..."
as_root "$rootfs" bash --login -c "$create_user"
info "Make '$(whoami)' a sudoer"
as_root "$rootfs" bash --login -c "adduser $(whoami) sudo || true"
}
function create_debian_rootfs() {
local to="$1"
info "Pulling the debian docker image"
docker pull debian:sid-slim
info "Spinning up the container"
docker stop devfs &> /dev/null || true
docker rm devfs &> /dev/null || true
docker run --name devfs -d debian:sid-slim
sleep 2
info "Exporting the rootfs to '$to/rootfs.tar'"
docker export -o "$to/rootfs.tar" devfs
info "Tearing down the container"
docker stop devfs &> /dev/null
docker rm devfs &> /dev/null
}
function create_and_initialize_devfs_image() {
local to="$1"
local project_root="$2"
local llvm_version="$3"
local rootfs
local version
version=$(git describe)
rootfs="$to/rootfs/"
if [ ! -f "$to/rootfs.tar" ]; then
info "Creating the rootfs tar bar"
create_debian_rootfs "$to"
fi
mkdir -p "$rootfs"
if [ ! -f "$to/rootfs/etc/shadow" ]; then
info "Extracting the tarball"
tar xf "$to/rootfs.tar" -C "$rootfs"
fi
mount_trees "$rootfs" "$project_root"
#as_root "$rootfs" bash
as_root "$rootfs" bash -c "echo '$llvm_version' > /etc/llvm_version"
as_root "$rootfs" bash -c "echo 'export LANG=C.UTF-8' >> /etc/bash.bashrc"
as_root "$rootfs" bash -c "echo 'export LANG=C.UTF-8' >> /etc/profile"
as_root "$rootfs" bash /serene/scripts/devfs_container_setup.sh
unmount_trees "$rootfs"
info "Creating the tarball (It will take a few minutes)"
sudo tar c -C "$rootfs" "." | lzma -c -9 -T "$(nproc)" > "$to/fs.$version.tar.xz"
info "Removing the exporter tarball"
rm -fv "$to/rootfs.tar"
}
function sync_devfs_image() {
local where="$1"
local version
version=$(git describe)
yes_or_no "Upload images '$where/fs.$version.tar.xz'?" && \
info "Uploading image '$where/fs.$version.tar.xz'" && \
rsync -vlc --progress \
"$where/fs.$version.tar.xz" \
core.lxsameer.com:/home/www/public/dl.serene/devfs/
}
function mark_devfs_image_as_latest() {
local to="$1"
local version
local remote_cmd
version=$(git describe)
info "Marking images 'fs.$version.tar.xz' as latest"
remote_cmd="rm -f /home/www/public/dl.serene/devfs/fs.latest.tar.xz && ln -s /home/www/public/dl.serene/devfs/fs.$version.tar.xz /home/www/public/dl.serene/devfs/fs.latest.tar.xz"
# shellcheck disable=SC2029
ssh core.lxsameer.com "$remote_cmd"
}
function unmount_and_destroy_devfs() {
local rootfs="$1"
unmount_trees "$rootfs"
yes_or_no "Is it correct? 'sudo rm -rfv $rootfs'?" && sudo rm -rfv "$rootfs"
}

View File

@ -1,62 +0,0 @@
#! /bin/bash
# Serene Programming Language
#
# Copyright (c) 2019-2022 Sameer Rahmani <lxsameer@gnu.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
function unpack() {
local zstd
zstd="zstd --ultra -22 -T$(nproc)"
mkdir -p "$2"
_push "$2"
tar -I "$zstd" -xf "$1"
_pop
}
function pull_toolchain() {
local toolchain_name
toolchain_name="serene_toolchain.$LLVM_MAJOR_VERSION.$LLVM_VERSION"
if [ -f "$TOOLCHAIN_DIR/$toolchain_name.tar.zstd" ]; then
info "Skip downloading the toolchain. It is already there."
else
info "Pulling down the toolchain..."
http_pull "serene_toolchain" "$LLVM_MAJOR_VERSION.$LLVM_VERSION" "$TOOLCHAIN_DIR/$toolchain_name.tar.zstd"
fi
if [ -d "$TOOLCHAIN_DIR/$toolchain_name" ]; then
info "Skip unpacking the toolchain. It is already there."
return
fi
info "Unpacking the toolchain..."
unpack "$TOOLCHAIN_DIR/$toolchain_name.tar.zstd" "$TOOLCHAIN_DIR/$toolchain_name"
}
function setup_toolchain() {
local toolchain_name
toolchain_name="serene_toolchain.$LLVM_MAJOR_VERSION.$LLVM_VERSION"
if [[ "$USE_SERENE_TOOLCHAIN" != "true" ]]; then
warn "Serene toolchain is disabled!"
else
pull_toolchain
export PATH="$TOOLCHAIN_DIR/$toolchain_name/bin:$PATH"
export SERENE_TOOLCHAIN_PATH="$TOOLCHAIN_DIR/$toolchain_name"
fi
}