Add basic types and the storage for them on the jit engine

core/CMakeLists.txt

@@ -14,8 +14,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
 
+message(STATUS ">> ${LIBSERENE_INCLUDE_DIR}")
+include_directories(${LIBSERENE_INCLUDE_DIR})
+
+
+
+
 add_subdirectory(lib)
 
 

core/lib/CMakeLists.txt

@@ -61,10 +61,5 @@ include(GenerateExportHeader)
 
 generate_export_header(core EXPORT_FILE_NAME ${PROJECT_BINARY_DIR}/include/serene/core/export.h)
 
-if(SERENE_SHOW_LLVM_LIBS)
-  execute_process(COMMAND llvm-config --libs all
-    OUTPUT_VARIABLE SERENE_LLVM_LIBS)
-  message(STATUS "LLVM libs available:\n ${SERENE_LLVM_LIBS}")
-endif()
 
 target_link_libraries(core PRIVATE)

core/lib/reader.cpp

@@ -18,6 +18,8 @@
 
 #include "serene/core/reader.h"
 
+#include "serene/types/types.h"
+
 namespace serene {
 
 int reader() { return 0; };

libserene/CMakeLists.txt

@@ -14,8 +14,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-set(LIB_SERENE_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include)
-include_directories(${LIB_SERENE_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/lib)
+set(INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include)
+set(LIBSERENE_INCLUDE_DIR ${INCLUDE_DIR} PARENT_SCOPE)
+include_directories(${INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/lib)
 
 add_subdirectory(lib)
 

libserene/include/serene/jit/halley.h

@@ -21,19 +21,16 @@
   This is the first working attempt on building a JIT engine for Serene
   and named after Edmond Halley.
 
-  - It supports both ASTs and Namespaces
-  - Every Namespace might have one or more JITDylibs. Depends on the method
-    of the compilation.
   - It operates in lazy (for REPL) and non-lazy mode and wraps LLJIT
     and LLLazyJIT
   - It uses an object cache layer to cache module (not NSs) objects.
  */
-
 #ifndef SERENE_JIT_HALLEY_H
 #define SERENE_JIT_HALLEY_H
 
-#include "serene/context.h" // for Serene...
-#include "serene/export.h"  // for SERENE...
+#include "serene/context.h"     // for Serene...
+#include "serene/export.h"      // for SERENE...
+#include "serene/types/types.h" // for Intern...
 
 #include <llvm/ADT/SmallVector.h>                             // for SmallV...
 #include <llvm/ADT/StringMap.h>                               // for StringMap
@@ -47,7 +44,9 @@
 #include <llvm/Support/MemoryBufferRef.h>                     // for Memory...
 #include <llvm/Support/raw_ostream.h>                         // for raw_os...
 
-#include <memory> // for unique...
+#include <memory>   // for unique...
+#include <stddef.h> // for size_t
+#include <vector>   // for vector
 
 #define HALLEY_LOG(...)                  \
   DEBUG_WITH_TYPE("halley", llvm::dbgs() \
@@ -57,6 +56,10 @@ namespace llvm {
 class DataLayout;
 class JITEventListener;
 class Module;
+namespace orc {
+class JITDylib;
+} // namespace orc
+
 } // namespace llvm
 
 namespace serene {
@@ -108,6 +111,25 @@ class SERENE_EXPORT Halley {
 
   bool isLazy = false;
 
+  // TODO: [jit] Replace this vector with a thread safe time-optimized
+  // datastructure that is capable of indexing strings and own all
+  // the strings. A lockless algorithm would be even better
+
+  /// Owns all the internal strings used in the compilation process
+  std::vector<types::InternalString> stringStorage;
+
+  std::vector<types::Namespace> nsStorage;
+  // /TODO
+
+  // JIT JITDylib related functions ---
+  llvm::StringMap<llvm::SmallVector<llvm::orc::JITDylib *, 1>> jitDylibs;
+
+  /// Register the given pointer to a `JITDylib` \p l, with the give \p ns.
+  void pushJITDylib(types::Namespace &ns, llvm::orc::JITDylib *l);
+
+  // /// Returns the number of registered `JITDylib` for the given \p ns.
+  size_t getNumberOfJITDylibs(types::Namespace &ns);
+
 public:
   Halley(std::unique_ptr<SereneContext> ctx,
          llvm::orc::JITTargetMachineBuilder &&jtmb, llvm::DataLayout &&dl);
@@ -117,6 +139,15 @@ public:
 
   SereneContext &getContext() { return *ctx; };
 
+  llvm::Error createEmptyNS(llvm::StringRef name);
+
+  types::InternalString &insertString(types::InternalString &s);
+  types::Namespace &insertNamespace(types::Namespace &n);
+
+  /// Return a pointer to the most registered JITDylib of the given \p ns
+  ////name
+  llvm::orc::JITDylib *getLatestJITDylib(types::Namespace &ns);
+
   void setEngine(std::unique_ptr<llvm::orc::LLJIT> e, bool isLazy);
   /// Looks up a packed-argument function with the given sym name and returns a
   /// pointer to it. Propagates errors in case of failure.

libserene/include/serene/types/types.h

@@ -0,0 +1,56 @@
+/* -*- C++ -*-
+ * Serene Programming Language
+ *
+ * Copyright (c) 2019-2022 Sameer Rahmani <lxsameer@gnu.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef SERENE_TYPES_TYPE_H
+#define SERENE_TYPES_TYPE_H
+
+#include <string>
+
+namespace serene::types {
+
+struct Expression {
+  const int8_t *data;
+
+  explicit Expression(const int8_t *data) : data(data){};
+};
+
+struct InternalString {
+  // We store the actual string in a "string" data section
+  const char *data;
+  const int64_t len;
+
+  InternalString(const char *data, const int64_t len) : data(data), len(len){};
+};
+
+struct Symbol {
+  const InternalString *ns;
+  const InternalString *name;
+
+  Symbol(const InternalString *ns, const InternalString *name)
+      : ns(ns), name(name){};
+};
+
+struct Namespace {
+  const InternalString *name;
+
+  explicit Namespace(const InternalString *name) : name(name){};
+};
+
+}; // namespace serene::types
+
+#endif

libserene/lib/CMakeLists.txt

@@ -76,7 +76,7 @@ endif()
 #add_dependencies(serene)
 
 # We need this directory, and users of our library will need it too
-target_include_directories(serene PUBLIC "$<BUILD_INTERFACE:${LIB_SERENE_INCLUDE_DIR}>")
+target_include_directories(serene PUBLIC "$<BUILD_INTERFACE:${LIBSERENE_INCLUDE_DIR}>")
 target_include_directories(serene PUBLIC "$<BUILD_INTERFACE:${PROJECT_BINARY_DIR}>")
 
 # Generate the export.h

libserene/lib/jit/halley.cpp

@@ -15,11 +15,11 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
-
 #include "serene/jit/halley.h"
 
-#include "serene/context.h" // for Seren...
-#include "serene/options.h" // for Options
+#include "serene/context.h"     // for Seren...
+#include "serene/options.h"     // for Options
+#include "serene/types/types.h" // for Names...
 
 #include <system_error> // for error...
 
@@ -42,6 +42,7 @@
 #include <llvm/IR/Module.h>                                    // for Module
 #include <llvm/Support/CodeGen.h>                              // for Level
 #include <llvm/Support/FileSystem.h>                           // for OF_None
+#include <llvm/Support/FormatVariadic.h>                       // for formatv
 #include <llvm/Support/ToolOutputFile.h>                       // for ToolO...
 #include <llvm/Support/raw_ostream.h>                          // for raw_o...
 
@@ -99,6 +100,37 @@ void ObjectCache::dumpToObjectFile(llvm::StringRef outputFilename) {
   file->keep();
 }
 
+llvm::orc::JITDylib *Halley::getLatestJITDylib(types::Namespace &ns) {
+
+  if (jitDylibs.count(ns.name->data) == 0) {
+    return nullptr;
+  }
+
+  auto vec = jitDylibs[ns.name->data];
+  // TODO: Make sure that the returning Dylib still exists in the JIT
+  //       by calling jit->engine->getJITDylibByName(dylib_name);
+  return vec.empty() ? nullptr : vec.back();
+};
+
+void Halley::pushJITDylib(types::Namespace &ns, llvm::orc::JITDylib *l) {
+  if (jitDylibs.count(ns.name->data) == 0) {
+    llvm::SmallVector<llvm::orc::JITDylib *, 1> vec{l};
+    jitDylibs[ns.name->data] = vec;
+    return;
+  }
+  auto vec = jitDylibs[ns.name->data];
+  vec.push_back(l);
+  jitDylibs[ns.name->data] = vec;
+}
+
+size_t Halley::getNumberOfJITDylibs(types::Namespace &ns) {
+  if (jitDylibs.count(ns.name->data) == 0) {
+    return 0;
+  }
+  auto vec = jitDylibs[ns.name->data];
+  return vec.size();
+};
+
 Halley::Halley(std::unique_ptr<SereneContext> ctx,
                llvm::orc::JITTargetMachineBuilder &&jtmb, llvm::DataLayout &&dl)
     : cache(ctx->opts.JITenableObjectCache ? new ObjectCache() : nullptr),
@@ -312,6 +344,44 @@ MaybeEngine Halley::make(std::unique_ptr<SereneContext> sereneCtxPtr,
   return MaybeEngine(std::move(jitEngine));
 };
 
+types::InternalString &Halley::insertString(types::InternalString &s) {
+  stringStorage.push_back(s);
+  auto &sameString = stringStorage.back();
+  return sameString;
+};
+
+types::Namespace &Halley::insertNamespace(types::Namespace &n) {
+  nsStorage.push_back(n);
+  auto &sameNs = nsStorage.back();
+  return sameNs;
+};
+
+llvm::Error Halley::createEmptyNS(llvm::StringRef name) {
+
+  // TODO: [serene.core] We need to provide some functions on llvm level to
+  // build instances from these type in a functional way. We need to avoid
+  // randomly build instances here and there that causes unsafe memory
+  types::InternalString nsName_(name.str().c_str(), name.size());
+  auto &nsName = insertString(nsName_);
+
+  types::Namespace ns_(&nsName);
+  auto &ns = insertNamespace(ns_);
+
+  HALLEY_LOG(llvm::formatv("Creating Dylib {0}#{1}", ns.name,
+                           getNumberOfJITDylibs(ns) + 1));
+
+  auto newDylib = engine->createJITDylib(
+      llvm::formatv("{0}#{1}", ns.name, getNumberOfJITDylibs(ns) + 1));
+
+  if (!newDylib) {
+    llvm::errs() << "Couldn't create the jitDylib\n";
+    serene::terminate(*ctx, 1);
+  }
+
+  pushJITDylib(ns, &(*newDylib));
+  return llvm::Error::success();
+};
+
 MaybeEngine makeHalleyJIT(std::unique_ptr<SereneContext> ctx) {
   llvm::orc::JITTargetMachineBuilder jtmb(ctx->triple);
   auto maybeJIT = Halley::make(std::move(ctx), std::move(jtmb));

serenec/CMakeLists.txt

@@ -25,6 +25,8 @@ set_target_properties(serenec PROPERTIES
   # LTO support
   INTERPROCEDURAL_OPTIMIZATION TRUE)
 
+add_dependencies(serenec Serene::core)
+
 if(SERENE_ENABLE_TIDY)
   set_target_properties(serenec PROPERTIES CXX_CLANG_TIDY ${CLANG_TIDY_PATH})
 endif()

serenec/serenec.cpp

@@ -19,6 +19,8 @@
 #include "serene/config.h"
 #include "serene/context.h"
 #include "serene/serene.h"
+#include "serene/types/types.h"
+
 // #include "serene/jit/halley.h"
 // #include "serene/namespace.h"
 // #include "serene/reader/location.h"
@@ -277,6 +279,9 @@ int main(int argc, char *argv[]) {
 
   applySereneCLOptions(*(*engine));
 
+  const std::string forms{"some.ns/sym"};
+  const types::InternalString data(forms.c_str(), forms.size());
+
   // // TODO: handle the outputDir by not forcing it. it should be
   // //       default to the current working dir
   // if (outputDir == "-") {