Serene/serene
Serene
/
serene
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Activity

Create a PoC to setup the stdlib calls via serene.core

This commit is contained in:
Sameer Rahmani 2022-07-23 11:09:15 +01:00
parent 605ac1569a
commit 1d2eebe680
6 changed files with 291 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
core/lib/serene/compiler.cpp.inc
View File

@ -23,9 +23,9 @@
namespace serene {
int compile() {
printf("Here\n");
return 0;
extern "C" int SERENE_EXPORT compile() {
printf("compile11\n");
return 2;
};
} // namespace serene

5
libserene/include/serene/fs.h
View File

@ -48,7 +48,10 @@ std::string extensionFor(SereneContext &ctx, NSFileType t);
/// Converts the given namespace name `nsName` to the file name
/// for that name space. E.g, `some.random.ns` will be translated
/// to `some_random_ns`.
std::string namespaceToPath(const llvm::StringRef nsName);
std::string namespaceToPath(llvm::StringRef nsName);
bool isStaticLib(llvm::StringRef path);
bool isSharedLib(llvm::StringRef path);
/// Return a boolean indicating whether or not the given path exists.
bool exists(llvm::StringRef path);

63
libserene/include/serene/jit/halley.h
View File

@ -25,6 +25,16 @@
and LLLazyJIT
- It uses an object cache layer to cache module (not NSs) objects.
*/
// TODO: [jit] When we want to load any static or dynamic lib for
// namespace as a dependency first look up the `ExecutionSession`
// to make sure that we did not load it already. If we did just
// use the existing `JITDylib` for it.
//
// TODO: [jit] Use Bare JITDylibs for the static and dynamic libs.
// Hint: Look at `createBareJITDylib` on the `ExecutionSession`
#ifndef SERENE_JIT_HALLEY_H
#define SERENE_JIT_HALLEY_H
@ -33,6 +43,8 @@
#include "serene/fs.h"
#include "serene/types/types.h" // for Intern...
#include <llvm/ADT/ArrayRef.h>
#include <llvm/ADT/None.h>
#include <llvm/ADT/SmallVector.h> // for SmallV...
#include <llvm/ADT/StringMap.h> // for StringMap
#include <llvm/ADT/StringRef.h> // for StringRef
@ -53,6 +65,8 @@
DEBUG_WITH_TYPE("halley", llvm::dbgs() \
<< "[HALLEY]: " << __VA_ARGS__ << "\n");
#define MAIN_PROCESS_JD_NAME "<process>"
namespace llvm {
class DataLayout;
class JITEventListener;
@ -70,12 +84,15 @@ class Halley;
// Why? This is the lazy man's way to make it easier to replace
// the class under the hood later on to test different implementaion
// with the same interface
using Engine = Halley;
using EnginePtr = std::unique_ptr<Engine>;
using MaybeEngine = llvm::Expected<EnginePtr>;
using MaybeEnginePtr = llvm::Expected<void *(*)()>;
using DylibPtr = llvm::orc::JITDylib *;
using MaybeDylibPtr = llvm::Expected<DylibPtr>;
using Engine = Halley;
using EnginePtr = std::unique_ptr<Engine>;
using MaybeEngine = llvm::Expected<EnginePtr>;
using MaybeJitAddress = llvm::Expected<void *(*)()>;
using Dylib = llvm::orc::JITDylib;
using DylibPtr = Dylib *;
using MaybeDylibPtr = llvm::Expected<DylibPtr>;
using MaybeNSFileTypeArr = llvm::Optional<llvm::ArrayRef<fs::NSFileType>>;
/// A simple object cache following Lang's LLJITWithObjectCache example and
/// MLIR's SimpelObjectCache.
class ObjectCache : public llvm::ObjectCache {
@ -99,18 +116,17 @@ class SERENE_EXPORT Halley {
// TODO: Replace this with a variant of LLJIT and LLLazyJIT
std::unique_ptr<llvm::orc::LLJIT> engine;
std::unique_ptr<ObjectCache> cache;
/// GDB notification listener.
llvm::JITEventListener *gdbListener;
/// Perf notification listener.
llvm::JITEventListener *perfListener;
llvm::orc::JITTargetMachineBuilder jtmb;
// TODO: [cleanup][jit] Since we can access to the data layout via
// `engine.getDataLayout`, remove this attribute and it's usecases
llvm::DataLayout &dl;
// /TODO
std::unique_ptr<SereneContext> ctx;
bool isLazy = false;
// TODO: [jit] Replace this vector with a thread safe time-optimized
@ -152,15 +168,32 @@ class SERENE_EXPORT Halley {
MaybeDylibPtr loadNamespaceFrom(NSLoadRequest &req);
// ==========================================================================
std::vector<const char *> getContainedNamespaces(llvm::StringRef name,
DylibPtr jd);
llvm::Error createCurrentProcessJD();
public:
Halley(std::unique_ptr<SereneContext> ctx,
llvm::orc::JITTargetMachineBuilder &&jtmb, llvm::DataLayout &&dl);
/// Initialize the engine by loading required libraries and shared libs
/// like the `serene.core` and other namespaces
llvm::Error initialize();
// TODO: [jit] Create a function to "require" a namespace as a dependency.
// If the namespace already exists return it otherwise call `loadNamespace`.
/// Load a namespace by exploring the load paths and different file
/// formats to find the namespace. We assume that we want to load
/// the namespace from file even if it exists already.
MaybeDylibPtr loadNamespace(std::string &nsName);
// TODO: Move all the loader related functions to a Loader class
/// Load the shared library in the given `path` to the given JITDylib
/// `jd` via the give ExecutionSession `es`.
/// This function assumes that the shared lib exists.
MaybeDylibPtr loadSharedLibFile(llvm::StringRef name, llvm::StringRef path);
MaybeDylibPtr loadStaticLibrary(const std::string &name);
MaybeDylibPtr loadSharedLibrary(const std::string &name);
// /TODO
static MaybeEngine make(std::unique_ptr<SereneContext> sereneCtxPtr,
llvm::orc::JITTargetMachineBuilder &&jtmb);
@ -177,8 +210,8 @@ public:
void setEngine(std::unique_ptr<llvm::orc::LLJIT> e, bool isLazy);
/// Looks up a packed-argument function with the given sym name and returns a
/// pointer to it. Propagates errors in case of failure.
MaybeEnginePtr lookup(const char *nsName, const char *sym);
MaybeEnginePtr lookup(const types::Symbol &sym) const;
MaybeJitAddress lookup(const char *nsName, const char *sym);
MaybeJitAddress lookup(const types::Symbol &sym) const;
/// Invokes the function with the given name passing it the list of opaque
/// pointers to the actual arguments.

28
libserene/lib/fs.cpp
View File

@ -20,6 +20,8 @@
#include "serene/context.h"
#include <llvm/BinaryFormat/Magic.h>
namespace serene::fs {
std::string extensionFor(SereneContext &ctx, NSFileType t) {
@ -50,7 +52,7 @@ std::string extensionFor(SereneContext &ctx, NSFileType t) {
/// Converts the given namespace name `nsName` to the file name
/// for that name space. E.g, `some.random.ns` will be translated
/// to `some_random_ns`.
std::string namespaceToPath(const llvm::StringRef nsName) {
std::string namespaceToPath(llvm::StringRef nsName) {
// TODO: [fs][perf] This function is not efficient. Fix it
std::string nsNameCopy = nsName.str();
std::replace(nsNameCopy.begin(), nsNameCopy.end(), '.', '/');
@ -62,6 +64,30 @@ std::string namespaceToPath(const llvm::StringRef nsName) {
return std::string(path);
};
bool isStaticLib(llvm::StringRef path) {
llvm::file_magic magic;
// llvm::identify_magic returns an error code on failure
if (llvm::identify_magic(path, magic)) {
// If there was an error loading the file then skip it.
return false;
}
return (magic == llvm::file_magic::archive ||
magic == llvm::file_magic::macho_universal_binary);
};
bool isSharedLib(llvm::StringRef path) {
llvm::file_magic magic;
// llvm::identify_magic returns an error code on failure
if (llvm::identify_magic(path, magic)) {
// If there was an error loading the file then skip it.
return false;
}
return (magic == llvm::file_magic::macho_dynamically_linked_shared_lib ||
magic == llvm::file_magic::elf_shared_object);
};
/// Return a boolean indicating whether or not the given path exists.
bool exists(llvm::StringRef path) {
llvm::sys::fs::file_status status;

235
libserene/lib/jit/halley.cpp
View File

@ -30,15 +30,17 @@
#include <llvm/ADT/Triple.h> // for Triple
#include <llvm/ADT/iterator.h> // for itera...
#include <llvm/BinaryFormat/Magic.h>
#include <llvm/ExecutionEngine/JITEventListener.h> // for JITEv...
#include <llvm/ExecutionEngine/Orc/CompileUtils.h> // for TMOwn...
#include <llvm/ExecutionEngine/Orc/Core.h> // for Execu...
#include <llvm/ExecutionEngine/Orc/DebugUtils.h> // for opera...
#include <llvm/ExecutionEngine/JITEventListener.h> // for JITEv...
#include <llvm/ExecutionEngine/Orc/CompileUtils.h> // for TMOwn...
#include <llvm/ExecutionEngine/Orc/Core.h> // for Execu...
#include <llvm/ExecutionEngine/Orc/DebugUtils.h> // for opera...
#include <llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h>
#include <llvm/ExecutionEngine/Orc/ExecutionUtils.h> // for Dynam...
#include <llvm/ExecutionEngine/Orc/IRCompileLayer.h> // for IRCom...
#include <llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h> // for JITTa...
#include <llvm/ExecutionEngine/Orc/LLJIT.h> // for LLJIT...
#include <llvm/ExecutionEngine/Orc/ObjectFileInterface.h>
#include <llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h>
#include <llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h> // for RTDyl...
#include <llvm/ExecutionEngine/Orc/ThreadSafeModule.h> // for Threa...
#include <llvm/ExecutionEngine/SectionMemoryManager.h> // for Secti...
@ -277,33 +279,11 @@ MaybeEngine Halley::make(std::unique_ptr<SereneContext> sereneCtxPtr,
// exported symbol visibility.
// cf llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
// LLJIT::createObjectLinkingLayer
if (sereneCtx.triple.isOSBinFormatCOFF()) {
objectLayer->setOverrideObjectFlagsWithResponsibilityFlags(true);
objectLayer->setAutoClaimResponsibilityForObjectSymbols(true);
}
// Resolve symbols from shared libraries.
// for (auto libPath : sharedLibPaths) {
// auto mb = llvm::MemoryBuffer::getFile(libPath);
// if (!mb) {
// llvm::errs() << "Failed to create MemoryBuffer for: " << libPath
// << "\nError: " << mb.getError().message() << "\n";
// continue;
// }
// auto &JD = session.createBareJITDylib(std::string(libPath));
// auto loaded = llvm::orc::DynamicLibrarySearchGenerator::Load(
// libPath.data(), dataLayout.getGlobalPrefix());
// if (!loaded) {
// llvm::errs() << "Could not load " << libPath << ":\n "
// << loaded.takeError() << "\n";
// continue;
// }
// JD.addGenerator(std::move(*loaded));
// cantFail(objectLayer->add(JD, std::move(mb.get())));
// }
return objectLayer;
};
@ -326,6 +306,10 @@ MaybeEngine Halley::make(std::unique_ptr<SereneContext> sereneCtxPtr,
std::move(*targetMachine), jitEngine->cache.get());
};
// TODO: [jit] This is not a proper way to handle both engines.
// Create two different classes for different execution modes
// (`lazy` vs `eager`) with the same interface and use them
// where appropriate.
if (sereneCtx.opts.JITLazy) {
// Setup a LLLazyJIT instance to the times that latency is important
// for example in a REPL. This way
@ -335,8 +319,8 @@ MaybeEngine Halley::make(std::unique_ptr<SereneContext> sereneCtxPtr,
.setCompileFunctionCreator(compileFunctionCreator)
.setObjectLinkingLayerCreator(objectLinkingLayerCreator)
.create());
jitEngine->setEngine(std::move(jit), true);
jitEngine->setEngine(std::move(jit), sereneCtx.opts.JITLazy);
} else {
// Setup a LLJIT instance for the times that performance is important
// and we want to compile everything as soon as possible. For instance
@ -346,9 +330,9 @@ MaybeEngine Halley::make(std::unique_ptr<SereneContext> sereneCtxPtr,
.setCompileFunctionCreator(compileFunctionCreator)
.setObjectLinkingLayerCreator(objectLinkingLayerCreator)
.create());
jitEngine->setEngine(std::move(jit), false);
jitEngine->setEngine(std::move(jit), sereneCtx.opts.JITLazy);
}
// /TODO
jitEngine->engine->getIRCompileLayer().setNotifyCompiled(
[&](llvm::orc::MaterializationResponsibility &r,
@ -360,11 +344,9 @@ MaybeEngine Halley::make(std::unique_ptr<SereneContext> sereneCtxPtr,
});
});
// Resolve symbols that are statically linked in the current process.
llvm::orc::JITDylib &mainJD = jitEngine->engine->getMainJITDylib();
mainJD.addGenerator(
cantFail(llvm::orc::DynamicLibrarySearchGenerator::GetForCurrentProcess(
jitEngine->dl.getGlobalPrefix())));
if (auto err = jitEngine->createCurrentProcessJD()) {
return err;
}
return MaybeEngine(std::move(jitEngine));
};
@ -423,7 +405,7 @@ llvm::Error Halley::createEmptyNS(const char *name) {
return llvm::Error::success();
};
MaybeEnginePtr Halley::lookup(const char *nsName, const char *sym) {
MaybeJitAddress Halley::lookup(const char *nsName, const char *sym) {
assert(sym != nullptr && "'sym' is null: lookup");
assert(nsName != nullptr && "'nsName' is null: lookup");
@ -432,12 +414,14 @@ MaybeEnginePtr Halley::lookup(const char *nsName, const char *sym) {
std::string fqsym = (ns + "/" + s).str();
HALLEY_LOG("Looking up symbol: " << fqsym);
auto *dylib = jitDylibs[nsName].back();
if (dylib == nullptr) {
return tempError(*ctx, "No dylib " + s);
}
HALLEY_LOG("Looking in dylib: " << (void *)dylib);
auto expectedSymbol = engine->lookup(*dylib, fqsym);
// JIT lookup may return an Error referring to strings stored internally by
@ -450,14 +434,16 @@ MaybeEnginePtr Halley::lookup(const char *nsName, const char *sym) {
return expectedSymbol.takeError();
}
auto rawFPtr = *expectedSymbol;
auto rawFPtr = expectedSymbol->getAddress();
// NOLINTNEXTLINE(performance-no-int-to-ptr)
auto fptr = reinterpret_cast<void *(*)()>(&rawFPtr);
auto fptr = reinterpret_cast<void *(*)()>(rawFPtr);
if (fptr == nullptr) {
return tempError(*ctx, "Lookup function is null!");
}
HALLEY_LOG("Found symbol '" << fqsym << "' at " << (void *)fptr);
return fptr;
};
@ -523,7 +509,7 @@ Halley::loadNamespaceFrom<fs::NSFileType::ObjectFile>(NSLoadRequest &req) {
if (!fs::exists(file)) {
// Can't locate any object file, skit to the next loader
llvm::outs() << "file: " << file << "\n";
HALLEY_LOG("File does not exist: " << file << "\n");
return nullptr;
}
@ -660,6 +646,16 @@ MaybeDylibPtr Halley::loadNamespace(std::string &nsName) {
}
if (*maybeJDptr != nullptr) {
auto *processJD = engine->getExecutionSession().getJITDylibByName(
MAIN_PROCESS_JD_NAME);
if (processJD == nullptr) {
// TODO: [jit] Panic here
return tempError(*ctx, "Can't find the main process JD");
// /TODO
}
(*maybeJDptr)->addToLinkOrder(*processJD);
return *maybeJDptr;
}
}
@ -668,8 +664,167 @@ MaybeDylibPtr Halley::loadNamespace(std::string &nsName) {
return tempError(*ctx, "Can't find namespace: " + nsName);
};
llvm::Error Halley::initialize() {
(void)ctx;
MaybeDylibPtr Halley::loadStaticLibrary(const std::string &name) {
if (ctx->getLoadPaths().empty()) {
return tempError(*ctx, "Load paths should not be empty");
}
for (auto &path : ctx->getLoadPaths()) {
auto file = fs::join(path, name + ".a");
if (!fs::exists(file)) {
continue;
}
if (!fs::isStaticLib(file)) {
return tempError(*ctx, "Not a static lib: " + file);
}
auto *objectLayer = &engine->getObjLinkingLayer();
auto generator = llvm::orc::StaticLibraryDefinitionGenerator::Load(
*objectLayer, file.c_str(),
engine->getExecutionSession()
.getExecutorProcessControl()
.getTargetTriple(),
std::move(llvm::orc::getObjectFileInterface));
if (!generator) {
return generator.takeError();
}
auto jd = engine->createJITDylib(llvm::formatv("{0}#{1}", name, 0));
if (!jd) {
return jd.takeError();
}
jd->addGenerator(std::move(*generator));
std::vector<llvm::StringRef> nsNames = {name};
auto definition = engine->lookup(*jd, "__serene_namespaces");
if (!definition) {
HALLEY_LOG("Library '" << name << "' is not a Serene lib.");
// We just want to ignore the error
llvm::consumeError(definition.takeError());
} else {
HALLEY_LOG("Library '" << name << "' is a Serene lib.");
// TODO: call the __serene_namespaces and set nsNames to
// the list of namespaces that it returns
(void)*definition;
}
for (auto &nsName : nsNames) {
auto ns = makeNamespace(nsName.str().c_str());
pushJITDylib(ns, &(*jd));
}
return &jd.get();
}
return tempError(*ctx, "Can't find static lib: " + name);
};
MaybeDylibPtr Halley::loadSharedLibFile(llvm::StringRef name,
llvm::StringRef path) {
if (!fs::isSharedLib(path)) {
return tempError(*ctx, "Not a shared lib: " + path);
}
auto generator = llvm::orc::EPCDynamicLibrarySearchGenerator::Load(
engine->getExecutionSession(), path.str().c_str());
if (!generator) {
return generator.takeError();
}
auto jd = engine->createJITDylib(llvm::formatv("{0}#{1}", name, 0));
if (!jd) {
return jd.takeError();
}
jd->addGenerator(std::move(*generator));
return &jd.get();
};
MaybeDylibPtr Halley::loadSharedLibrary(const std::string &name) {
if (ctx->getLoadPaths().empty()) {
return tempError(*ctx, "Load paths should not be empty");
}
for (auto &path : ctx->getLoadPaths()) {
auto file = fs::join(path, name + ".so");
if (!fs::exists(file)) {
continue;
}
auto maybeJD = loadSharedLibFile(name, file);
if (!maybeJD) {
return maybeJD.takeError();
}
auto *jd = *maybeJD;
auto nsNames = getContainedNamespaces(name, jd);
for (const auto *nsName : nsNames) {
auto ns = makeNamespace(nsName);
pushJITDylib(ns, jd);
}
return jd;
}
return tempError(*ctx, "Can't find the dynamic lib: " + name);
};
std::vector<const char *> Halley::getContainedNamespaces(llvm::StringRef name,
DylibPtr jd) {
std::vector<const char *> nsNames = {name.str().c_str()};
auto definition = engine->lookup(*jd, "__serene_namespaces");
if (!definition) {
HALLEY_LOG("Library is not a Serene lib.");
// We just want to ignore the error
llvm::consumeError(definition.takeError());
}
HALLEY_LOG("Library is a Serene lib.");
// TODO: call the __serene_namespaces and set nsNames to
// the list of namespaces that it returns
return nsNames;
};
llvm::Error Halley::createCurrentProcessJD() {
auto &es = engine->getExecutionSession();
auto *processJDPtr = es.getJITDylibByName(MAIN_PROCESS_JD_NAME);
if (processJDPtr != nullptr) {
// We already created the JITDylib for the current process
return llvm::Error::success();
}
auto processJD = es.createJITDylib(MAIN_PROCESS_JD_NAME);
if (!processJD) {
return processJD.takeError();
}
auto generator =
llvm::orc::DynamicLibrarySearchGenerator::GetForCurrentProcess(
engine->getDataLayout().getGlobalPrefix());
if (!generator) {
return generator.takeError();
}
processJD->addGenerator(std::move(*generator));
return llvm::Error::success();
};

19
serenec/serenec.cpp
View File

@ -299,9 +299,9 @@ int main(int argc, char *argv[]) {
// }
std::string core = "serene.core";
auto maybeJD = engine->loadNamespace(core);
if (!maybeJD) {
llvm::errs() << "Error: " << maybeJD.takeError() << "'\n";
auto maybeCore = engine->loadNamespace(core);
if (!maybeCore) {
llvm::errs() << "Error: " << maybeCore.takeError() << "'\n";
return 1;
}
@ -312,9 +312,18 @@ int main(int argc, char *argv[]) {
return 1;
}
auto c = *bt;
void *res = c();
if (*bt == nullptr) {
llvm::errs() << "Error: nullptr?\n";
return 1;
}
auto *c = *bt;
void *res = c();
// for (int i = 0; i <= 10; i++) {
// printf(">> %02x", *(c + i));
// }
printf("Res >> %p\n", res);
llvm::outs() << "Res: " << *((int *)res) << "\n";
(void)res;
// // TODO: handle the outputDir by not forcing it. it should be