Create a flake with all the required dependencies to build Serene

CMakeLists.txt

@@ -136,6 +136,7 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
     set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
   endif()
 
+  add_link_options(-Wl,-Map=output.map)
   # LLVM setup ==================================================================
   # Why not specify the version?
   # Since we use the development version of the LLVM all the time it doesn't

builder

@@ -114,13 +114,13 @@ LSAN_OPTIONS=suppressions="$ME/.ignore_sanitize"
 export LSAN_OPTIONS
 
 # shellcheck source=./scripts/toolchain.sh
-source "$ME/scripts/toolchain.sh"
+#source "$ME/scripts/toolchain.sh"
 
 
 # -----------------------------------------------------------------------------
 # Initialization
 # -----------------------------------------------------------------------------
-mkdir -p "$TOOLCHAIN_DIR"
+#mkdir -p "$TOOLCHAIN_DIR"
 
 
 # -----------------------------------------------------------------------------
@@ -153,8 +153,8 @@ function build-gen() {
     local args
     #setup_toolchain
     args=(
-        "-C"
+        # "-C"
-        "$ME/cmake/caches/$1.cmake"
+        # "$ME/cmake/caches/$1.cmake"
     )
 
     pushed_build

flake.lock

@@ -20,16 +20,18 @@
     },
     "nixpkgs": {
       "locked": {
-        "dirtyRev": "bcb1a9c7e1d8568c5e58316fe3254eb8f4455439-dirty",
+        "lastModified": 1705092417,
-        "dirtyShortRev": "bcb1a9c7e1d8-dirty",
+        "narHash": "sha256-01pTqprf3NvQijvxkQjwx2c6uevB4MZKooIcf+RTYHA=",
-        "lastModified": 1704738119,
+        "owner": "lxsameer",
-        "narHash": "sha256-FZ4iFQEMt790dl5I4l3/vc/NuB9jyrppW0q315adCWQ=",
+        "repo": "nixpkgs",
-        "type": "git",
+        "rev": "e1f7865bce4d52d30dd1d61e79798ee2765cc2b0",
-        "url": "file:///home/lxsameer/src/nixpkgs"
+        "type": "github"
       },
       "original": {
-        "type": "git",
+        "owner": "lxsameer",
-        "url": "file:///home/lxsameer/src/nixpkgs"
+        "repo": "nixpkgs",
+        "rev": "e1f7865bce4d52d30dd1d61e79798ee2765cc2b0",
+        "type": "github"
       }
     },
     "root": {

flake.nix

@@ -16,49 +16,27 @@
 {
   description = "Serene programming language";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/bcb1a9c7e1d8568c5e58316fe3254eb8f4455439";
+  #inputs.nixpkgs.url = "github:NixOS/nixpkgs/bcb1a9c7e1d8568c5e58316fe3254eb8f4455439";
+  inputs.nixpkgs.url = "github:lxsameer/nixpkgs/e1f7865bce4d52d30dd1d61e79798ee2765cc2b0";
   #inputs.nixpkgs.url = "/home/lxsameer/src/nixpkgs/";
   inputs.flake-utils.url = "github:numtide/flake-utils";
 
   outputs = inputs@{ self, nixpkgs, flake-utils, ... }:
     flake-utils.lib.eachDefaultSystem (system:
       let
+        # Out zsh configuration directory. mkShell will pick up the .zshrc
+        # from this directory.
         zshDir = ./scripts/zsh;
-        # llvm = {
-        #   url = "https://devheroes.codes/Serene/llvm-project.git";
-        #   major = "16";
-        #   version = "16.0.6";
-        #   rev = "b2c0361bcc08afdf466a605a23924bcd64fa2b86";
-        #   hash = "sha256-wUKHwWYYlKa0v5zEJXxndleYtK7wzJKi0vqr3lNkGMI=";
-        #   rev-version = "17-disable-shared-1";
-        # };
-
 
+        # Most of these overlays are do to bugs and problems
+        # in upstream nixpkgs. But thanks to their design
+        # We can fix them using these overlays and contribuete
+        # them upstream little by little.
         overlays = [
-          # (self: super:
-          #   {
-          #     serene_toolchain = (super.callPackage (super.path + "/pkgs/development/compilers/llvm/${llvm.major}") ({
-          #       inherit (super.stdenvAdapters) overrideCC;
-          #       buildLlvmTools = super.buildPackages.serene_toolchain.tools;
-          #       targetLlvmLibraries = self.targetPackages.serene_toolchain.libraries or self.serene_toolchain.libraries;
-          #       targetLlvm = self.targetPackages.serene_toolchain.llvm or self.serene_toolchain.llvm;
-          #       monorepoSrc = super.fetchgit {
-          #         url = llvm.url;
-          #         rev = llvm.rev;
-          #         hash = llvm.hash;
-          #       };
-          #       officialRelease = null;
-          #       gitRelease = {
-          #         version = llvm.version;
-          #         rev = llvm.rev;
-          #         rev-version = llvm.rev-version;
-          #       };
-          #     }));
-          #   })
           (final: prev: {
             p11-kit = prev.p11-kit.overrideAttrs (old: {
               patches = [
-                ./scripts/patches/p11-kit_skip_test.patch
+                ./nix/patches/p11-kit_skip_test.patch
               ];
             });
 
@@ -84,26 +62,60 @@
             # We don't need systemd at all
             util-linux = prev.util-linux.override { systemdSupport = false; };
 
+            # libpam exmaples use glibc. We need to disable them
             linux-pam = prev.linux-pam.overrideAttrs (old: {
               postConfigure = ''
                 sed 's/examples//' -i Makefile
               '';
             });
-            # iwyu = (prev.include-what-you-use.overrideAttrs (old:
-            #   let
-            #     version = "0.20";
-            #   in {
-            #     inherit version;
 
-            #     src = prev.fetchurl {
+            #=============================================================
-            #       url = "${old.meta.homepage}/downloads/${old.pname}-${version}.src.tar.gz";
+            # Since we're using lld-17, and --no-undefined-version is the
-            #       hash = "sha256-dfzh5khfKA+PE/TC0JCxHS/SECtQhXUHyEE6kZt6+Jk=";
+            # default in lld-17. We need to explicitely turn it off for
-            #     };
+            # these problematic packages untill they fix it upstream.
-            #     cmakeFlags = [ "-DCMAKE_PREFIX_PATH=${prev.llvmPackages_16.llvm.dev}" ];
+            libgcrypt = prev.libgcrypt.overrideAttrs (old: {
+              NIX_LDFLAGS = if prev.stdenv.cc.isClang
+                            then [ "--undefined-version" ]
+                            else [];
+            });
+            libxcrypt = prev.libxcrypt.overrideAttrs (old: {
+              NIX_LDFLAGS = if prev.stdenv.cc.isClang
+                            then [ "--undefined-version" ]
+                            else [];
+            });
+            ncurses = prev.ncurses.overrideAttrs (old: {
+              NIX_LDFLAGS = if prev.stdenv.cc.isClang
+                            then [ "--undefined-version" ]
+                            else [];
+            });
 
-            #   })).override {
+            libbsd = prev.libbsd.overrideAttrs (old: { #old.NIX_LDFLAGS ++
-            #     llvmPackages = prev.__splicedPackages.llvmPackages_16;
+              NIX_LDFLAGS = if prev.stdenv.cc.isClang
-            #   };
+                            then [ "--undefined-version" ]
+                            else [];
+            });
+            libidn2 = prev.libidn2.overrideAttrs (old: { #old.NIX_LDFLAGS ++
+              NIX_LDFLAGS = if prev.stdenv.cc.isClang
+                            then [ "--undefined-version" ]
+                            else [];
+            });
+            #==============================================================
+
+            iwyu = (prev.include-what-you-use.overrideAttrs (old:
+              let
+                version = "0.21";
+              in {
+                inherit version;
+
+                src = prev.fetchurl {
+                  url = "${old.meta.homepage}/downloads/${old.pname}-${version}.src.tar.gz";
+                  hash = "sha256-ajUZGf+JvafJXIlUcmAYaNs9qrlqlYs44DYokNWHYLY=";
+                };
+                cmakeFlags = [ "-DCMAKE_PREFIX_PATH=${prev.llvmPackages_17.llvm.dev}" ];
+
+              })).override {
+                llvmPackages = prev.__splicedPackages.llvmPackages_17;
+              };
           })
         ];
 
@@ -111,25 +123,15 @@
           if system == "x86_64-linux"
           then import nixpkgs {
             inherit system overlays;
+            linker = "lld";
             crossSystem = nixpkgs.lib.systems.examples.musl64 // { useLLVM = true; };
-            config.replaceCrossStdenv = { buildPackages, baseStdenv }:
+            # config.replaceCrossStdenv = { buildPackages, baseStdenv }:
-              buildPackages.stdenvAdapters.overrideCC baseStdenv buildPackages.llvmPackages_16.clangUseLLVM;
+            #   buildPackages.stdenvAdapters.overrideCC baseStdenv buildPackages.llvmPackages_17.clangUseLLVM;
           }
           else import nixpkgs {
             inherit system overlays;
           };
 
-        mkExtraBuildCommands0 = cc: major: ''
-            rsrc="$out/resource-root"
-            mkdir "$rsrc"
-            ln -s "${cc.lib}/lib/clang/${major}/include" "$rsrc"
-            echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags
-        '';
-        mkExtraBuildCommands = cc: major: cr: (mkExtraBuildCommands0 cc major) + ''
-            ln -s "${cr.out}/lib" "$rsrc/lib"
-            ln -s "${cr.out}/share" "$rsrc/share"
-        '';
-
         pkgs = get_pkgs system;
 
         # # sereneLLVM = (pkgs.callPackage ./nix/llvm.nix {
@@ -139,43 +141,51 @@
 
         # # stdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv sereneLLVM;
         # stdenv = pkgs.stdenv;
-        stdenv' = pkgs.stdenvAdapters.overrideCC pkgs.stdenv pkgs.llvmPackages_17.clangUseLLVM;
+        stdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv pkgs.llvmPackages_17.clangUseLLVM;
+        sereneLLVM = pkgs.llvmPackages_17.override { inherit stdenv; };
 
-        utils = (pkgs.callPackage ./nix/utils.nix {});
+        gc = pkgs.callPackage ./nix/boehmgc.nix { inherit stdenv; };
 
-        llvm_source = pkgs.callPackage ./nix/llvm_source.nix {};
+        clang' = stdenv.cc.overrideAttrs (old: {
-
+          propagatedBuildInputs = [ stdenv.cc.bintools ] ++ [ pkgs.zlib-ng ];
-        libunwind = (pkgs.callPackage ./nix/libunwind {
-          stdenv = stdenv';
-          inherit llvm_source utils;
         });
 
-        cxx-headers = (pkgs.callPackage ./nix/libcxx {
+        stdenv' = pkgs.stdenvAdapters.overrideCC pkgs.stdenv clang';
-          inherit llvm_source utils;
+        # utils = (pkgs.callPackage ./nix/utils.nix {});
-          llvm_libunwind = libunwind;
-          llvm_libcxxabi = null;
-          headersOnly = true;
-          stdenv = stdenv';
-        });
 
-        libcxxabi = (pkgs.callPackage ./nix/libcxxabi {
+        # llvm_source = pkgs.callPackage ./nix/llvm_source.nix {};
-          inherit llvm_source utils cxx-headers;
-          llvm_libunwind = libunwind;
-          stdenv = stdenv';
-        });
 
-        libcxx = (pkgs.callPackage ./nix/libcxx {
+        # libunwind = (pkgs.callPackage ./nix/libunwind {
-          inherit llvm_source utils;
+        #   stdenv = stdenv';
-          llvm_libcxxabi = libcxxabi;
+        #   inherit llvm_source utils;
-          llvm_libunwind = libunwind;
+        # });
-          stdenv = stdenv';
-        });
 
-        compiler-rt = (pkgs.callPackage ./nix/compiler-rt {
+        # cxx-headers = (pkgs.callPackage ./nix/libcxx {
-          inherit llvm_source utils;
+        #   inherit llvm_source utils;
-          llvm_libcxxabi = libcxxabi;
+        #   llvm_libunwind = libunwind;
-          stdenv = stdenv';
+        #   llvm_libcxxabi = null;
-        });
+        #   headersOnly = true;
+        #   stdenv = stdenv';
+        # });
+
+        # libcxxabi = (pkgs.callPackage ./nix/libcxxabi {
+        #   inherit llvm_source utils cxx-headers;
+        #   llvm_libunwind = libunwind;
+        #   stdenv = stdenv';
+        # });
+
+        # libcxx = (pkgs.callPackage ./nix/libcxx {
+        #   inherit llvm_source utils;
+        #   llvm_libcxxabi = libcxxabi;
+        #   llvm_libunwind = libunwind;
+        #   stdenv = stdenv';
+        # });
+
+        # compiler-rt = (pkgs.callPackage ./nix/compiler-rt {
+        #   inherit llvm_source utils;
+        #   llvm_libcxxabi = libcxxabi;
+        #   stdenv = stdenv';
+        # });
 
         # ullvm = pkgs.stellvmPackages_17;
         # clang' = pkgs.wrapCCWith rec {
@@ -209,22 +219,62 @@
         #   buildLlvmTools = pkgs.llvmPackages_17;
         #   stdenv = stdenv'';
         # });
-        llvm = (pkgs.callPackage ./nix/llvm.nix {
+        # llvm = (pkgs.callPackage ./nix/llvm.nix {
-          inherit llvm_source utils;
+        #   inherit llvm_source utils;
 
-          # llvm_libcxx = libcxx;
+        #   # llvm_libcxx = libcxx;
-          # llvm_libcxxabi = libcxxabi;
+        #   # llvm_libcxxabi = libcxxabi;
-          # llvm_compiler-rt = compiler-rt;
+        #   # llvm_compiler-rt = compiler-rt;
-          # llvm_libunwind = libunwind;
+        #   # llvm_libunwind = libunwind;
-          stdenv = pkgs.stdenv;
+        #   stdenv = pkgs.stdenv;
+        # });
+
+
+        # clang' = pkgs.wrapCCWith rec {
+        #   libcxx = llvm;
+        #   cc = llvm;
+        #   libc = pkgs.llvmPackages_17.bintools.libc;
+        #   bintools = pkgs.llvmPackages_17.bintools;
+        #   extraPackages = [
+        #     llvm
+        #   ];
+        #   # extraPackages = [
+        #   #   libcxxabi
+        #   #   compiler-rt
+        #   # ] ++ pkgs.lib.optionals (!pkgs.stdenv.targetPlatform.isWasm) [
+        #   #   ullvm.libunwind
+        #   # ];
+        #   extraBuildCommands = mkExtraBuildCommands cc llvm_source.major;
+        #   nixSupport.cc-cflags =
+        #     [ "-rtlib=compiler-rt"
+        #       "-Wno-unused-command-line-argument"
+        #       "-B${llvm}/lib/"
+        #     ]
+        #     ++ pkgs.lib.optional (!pkgs.stdenv.targetPlatform.isWasm) "--unwindlib=libunwind"
+        #     ++ pkgs.lib.optional
+        #       (!stdenv'.targetPlatform.isWasm)
+        #       "-lunwind"
+        #     ++ pkgs.lib.optional pkgs.stdenv.targetPlatform.isWasm "-fno-exceptions";
+        # };
+        # stdenv'' = pkgs.stdenvAdapters.overrideCC pkgs.stdenv clang';
+        llvm = pkgs.llvmPackages_17.llvm.overrideAttrs (old: {
+          propagatedBuildInputs = [];
         });
 
+        z = pkgs.zlib-ng.overrideAttrs (old: {
+            cmakeFlags = [
+              "-DCMAKE_INSTALL_PREFIX=/"
+              "-DBUILD_SHARED_LIBS=OFF"
+              "-DINSTALL_UTILS=ON"
+              "-DZLIB_COMPAT=ON"
+            ];
+        });
         #with pkgs;
         native_build_inputs =
           let
-            filterCmakeFlags = xs: builtins.filter
+            # filterCmakeFlags = xs: builtins.filter
-              (x: !(x == "-DCMAKE_CROSSCOMPILING=True" || pkgs.lib.hasPrefix "-DLLVM_TABLEGEN=" x))
+            #   (x: !(x == "-DCMAKE_CROSSCOMPILING=True" || pkgs.lib.hasPrefix "-DLLVM_TABLEGEN=" x))
-              xs;
+            #   xs;
 
             git' = pkgs.git.overrideAttrs (old: {
               preInstallCheck =
@@ -235,9 +285,10 @@
     '' ]
                   old.preInstallCheck;
             });
+
             # iwyu = (pkgs.include-what-you-use.overrideAttrs (old:
             #   let
-            #     version = "0.21";
+            #     version = "0.22";
             #   in {
             #     inherit version;
 
@@ -247,7 +298,7 @@
             #     };
             #   }))
             # .override {
-            #   llvmPackages = pkgs.__splicedPackages.llvmPackages_16;# .overrideAttrs (oldLLVM: {
+            #   llvmPackages = pkgs.__splicedPackages.llvmPackages_17;# .overrideAttrs (oldLLVM: {
             #   #   cmakeFlags = builtins.trace ">> ${toString(filterCmakeFlags oldLLVM.cmakeFlags)}" filterCmakeFlags oldLLVM.cmakeFlags;
             #   # });
             # };
@@ -256,6 +307,14 @@
             # });
             # builtins.trace ">> ${sereneLLVM}"
           in  (with pkgs; [
+            # sereneLLVM.llvm
+            # sereneLLVM.mlir
+            # sereneLLVM.clang
+            # sereneLLVM.compiler-rt
+            # sereneLLVM.lld
+            # sereneLLVM.libcxx
+            # sereneLLVM.libcxxabi
+
             cmake
             ninja
             ccache
@@ -265,25 +324,38 @@
             zsh-autocomplete
             zsh-syntax-highlighting
             python3
-            #sereneLLVM
+            iwyu
-            # libunwind
+            # mlir17
-            # libcxxabi
+            # llvm17
-            # libcxx
+            # cr17
-            # compiler-rt
+            # cc17
+            # libcxx17
+            # libcxxabi17
+            # lld17
+            # libunwind17
+            gc
+            #musl
             llvm
-            #.override {stdenv = stdenv';}
+
-            # mlir_17
+            #llvmPackages_17.llvm
+            llvmPackages_17.mlir
+            llvmPackages_17.clang
           ]);
 
         build_inputs = with pkgs; [
-          boehmgc
+          gc
-          # llvm_16
+          #musl
-          # llvm_16
-          # mlir_17
-
           gtest
           gmock
-          # zlib-ng
+          #llvmPackages_17.llvm
+          z
+          llvm
+          llvmPackages_17.mlir
+          llvmPackages_17.clang
+
+          # llvmPackages_17.libcxx
+          # llvmPackages_17.libcxxabi
+          # llvmPackages_17.compiler-rt
           # zstd
         ];
 
@@ -298,19 +370,7 @@
             '';
         };
 
-        packages.cmake = pkgs.cmake;
+        #packages.llvm = llvm;
-        packages.blah = pkgs.stdenv.mkDerivation {
-          pname = "blah";
-          version = "0.1";
-          nativeBuildInputs = native_build_inputs;
-          buildInputs = build_inputs;
-        };
-        packages.mlir = pkgs.mlir_17;
-        packages.llvm_unwind = libunwind;
-        packages.llvm_libcxxabi = libcxxabi;
-        packages.llvm_cxx_headers = cxx-headers;
-        packages.llvm_compiler-rt = compiler-rt;
-        packages.llvm = llvm;
       }
     );
 }

nix/boehmgc.nix

@@ -0,0 +1,76 @@
+{ lib,
+  stdenv,
+  fetchurl,
+  nixVersions,
+  libatomic_ops,
+  cmake,
+  ninja,
+}:
+
+stdenv.mkDerivation (finalAttrs: rec {
+  pname = "boehmgc";
+  version = "8.2.4";
+
+  src = fetchurl {
+    urls = [
+      "https://github.com/ivmai/bdwgc/releases/download/v${finalAttrs.version}/gc-${finalAttrs.version}.tar.gz"
+    ];
+    sha256 = "sha256-PQ082+B3QD0xBrtA8Mu1Y0E9bv27Kn4c1ohlld7Ej8I=";
+  };
+
+
+  nativeBuildInputs = [ cmake ninja  ];
+  buildInputs = [ libatomic_ops ];
+  #outputs = [ "out" "dev" "doc" ];
+  ninjaFlags = [ "-v" ];
+  cmakeFlags = [
+    #"-Denable_cplusplus=ON"
+    "-Ddefault_enable_threads=ON"
+    "-DBUILD_SHARED_LIBS=OFF"
+    "-Denable_threads=ON"
+    "-Denable_parallel_mark=ON"
+    "-Denable_thread_local_alloc=ON"
+    "-Denable_mmap=ON"
+    # TODO: Do we want to return the pages to the OS if empty for N collections?
+    "-Denable_munmap=ON"
+    "-Denable_werror=ON"
+    # TODO: Do we want the entire GC in a single object file?
+    "-Denable_single_obj_compilation=ON"
+    "-Dwith_libatomic_ops=OFF"
+    "-Dwithout_libatomic_ops=ON"
+    "-DCMAKE_INSTALL_LIBDIR=${placeholder "out"}/lib"
+  ];
+
+  # `gctest` fails under emulation on aarch64-darwin
+  doCheck = !(stdenv.isDarwin && stdenv.isx86_64);
+
+  enableParallelBuilding = true;
+
+  passthru.tests = nixVersions;
+
+  meta = with lib; {
+    homepage = "https://hboehm.info/gc/";
+    description = "The Boehm-Demers-Weiser conservative garbage collector for C and C++";
+    longDescription = ''
+      The Boehm-Demers-Weiser conservative garbage collector can be used as a
+      garbage collecting replacement for C malloc or C++ new.  It allows you
+      to allocate memory basically as you normally would, without explicitly
+      deallocating memory that is no longer useful.  The collector
+      automatically recycles memory when it determines that it can no longer
+      be otherwise accessed.
+
+      The collector is also used by a number of programming language
+      implementations that either use C as intermediate code, want to
+      facilitate easier interoperation with C libraries, or just prefer the
+      simple collector interface.
+
+      Alternatively, the garbage collector may be used as a leak detector for
+      C or C++ programs, though that is not its primary goal.
+    '';
+    # non-copyleft, X11-style license
+    changelog = "https://github.com/ivmai/bdwgc/blob/v${finalAttrs.version}/ChangeLog";
+    license = "https://hboehm.info/gc/license.txt";
+    maintainers = with maintainers; [ AndersonTorres ];
+    platforms = platforms.all;
+  };
+})

nix/llvm.nix

@@ -27,7 +27,7 @@
   # llvm_compiler-rt,
   # llvm_libcxxabi,
   # llvm_libunwind,
- zlib,
+  zlib,
   wrapCCWith,
   runCommand,
   overrideCC,
@@ -61,7 +61,7 @@ let
     #  '';
   };
 
-  vanilla = stdenv.mkDerivation rec{
+  vanilla = stdenv.mkDerivation (final: rec{
     inherit (llvm_source) version;
     pname = "serene_llvm";
 
@@ -271,12 +271,12 @@ let
     cmakeFlags = cmakeFlags' ++ builtinFlags ++ runtimeFlags;
 
     ninjaFlags = [
-      "-v"
+
       #"install-clang"
-      # "install-builtins"
+       "install-builtins"
-       # "install-compiler-rt"
+       "install-compiler-rt"
-       # "install-runtimes"
+       "install-runtimes"
-       # "tools/iwyu/install"
+       "tools/iwyu/install"
     ];
 
 
@@ -314,7 +314,15 @@ let
         lib.platforms.riscv ++
         lib.platforms.m68k;
     };
-  };
+  });
+
+  vanila.passthru = {
+      isLLVM = true;
+      # libc_bin = stdenv.cc.binutils.libc_bin;
+      # libc_dev = stdenv.cc.binutils.libc_dev;
+      cxxabi = vanila;
+      libc = stdenv.cc.libc;
+    };
 
   tools = lib.makeExtensible (tools:
     let