diff --git a/core/lib/serene/compiler.cpp.inc b/core/lib/serene/compiler.cpp.inc index d7f0b72..99f02da 100644 --- a/core/lib/serene/compiler.cpp.inc +++ b/core/lib/serene/compiler.cpp.inc @@ -23,9 +23,9 @@ namespace serene { -int compile() { - printf("Here\n"); - return 0; +extern "C" int SERENE_EXPORT compile() { + printf("compile11\n"); + return 2; }; } // namespace serene diff --git a/libserene/include/serene/fs.h b/libserene/include/serene/fs.h index 7c68646..c5dc9ff 100644 --- a/libserene/include/serene/fs.h +++ b/libserene/include/serene/fs.h @@ -48,7 +48,10 @@ std::string extensionFor(SereneContext &ctx, NSFileType t); /// Converts the given namespace name `nsName` to the file name /// for that name space. E.g, `some.random.ns` will be translated /// to `some_random_ns`. -std::string namespaceToPath(const llvm::StringRef nsName); +std::string namespaceToPath(llvm::StringRef nsName); + +bool isStaticLib(llvm::StringRef path); +bool isSharedLib(llvm::StringRef path); /// Return a boolean indicating whether or not the given path exists. bool exists(llvm::StringRef path); diff --git a/libserene/include/serene/jit/halley.h b/libserene/include/serene/jit/halley.h index 5a5244d..c00b237 100644 --- a/libserene/include/serene/jit/halley.h +++ b/libserene/include/serene/jit/halley.h @@ -25,6 +25,16 @@ and LLLazyJIT - It uses an object cache layer to cache module (not NSs) objects. */ + +// TODO: [jit] When we want to load any static or dynamic lib for +// namespace as a dependency first look up the `ExecutionSession` +// to make sure that we did not load it already. If we did just +// use the existing `JITDylib` for it. + +// +// TODO: [jit] Use Bare JITDylibs for the static and dynamic libs. +// Hint: Look at `createBareJITDylib` on the `ExecutionSession` + #ifndef SERENE_JIT_HALLEY_H #define SERENE_JIT_HALLEY_H @@ -33,6 +43,8 @@ #include "serene/fs.h" #include "serene/types/types.h" // for Intern... +#include +#include #include // for SmallV... #include // for StringMap #include // for StringRef @@ -53,6 +65,8 @@ DEBUG_WITH_TYPE("halley", llvm::dbgs() \ << "[HALLEY]: " << __VA_ARGS__ << "\n"); +#define MAIN_PROCESS_JD_NAME "" + namespace llvm { class DataLayout; class JITEventListener; @@ -70,12 +84,15 @@ class Halley; // Why? This is the lazy man's way to make it easier to replace // the class under the hood later on to test different implementaion // with the same interface -using Engine = Halley; -using EnginePtr = std::unique_ptr; -using MaybeEngine = llvm::Expected; -using MaybeEnginePtr = llvm::Expected; -using DylibPtr = llvm::orc::JITDylib *; -using MaybeDylibPtr = llvm::Expected; +using Engine = Halley; +using EnginePtr = std::unique_ptr; +using MaybeEngine = llvm::Expected; +using MaybeJitAddress = llvm::Expected; +using Dylib = llvm::orc::JITDylib; +using DylibPtr = Dylib *; +using MaybeDylibPtr = llvm::Expected; +using MaybeNSFileTypeArr = llvm::Optional>; + /// A simple object cache following Lang's LLJITWithObjectCache example and /// MLIR's SimpelObjectCache. class ObjectCache : public llvm::ObjectCache { @@ -99,18 +116,17 @@ class SERENE_EXPORT Halley { // TODO: Replace this with a variant of LLJIT and LLLazyJIT std::unique_ptr engine; std::unique_ptr cache; - /// GDB notification listener. llvm::JITEventListener *gdbListener; - /// Perf notification listener. llvm::JITEventListener *perfListener; - llvm::orc::JITTargetMachineBuilder jtmb; + // TODO: [cleanup][jit] Since we can access to the data layout via + // `engine.getDataLayout`, remove this attribute and it's usecases llvm::DataLayout &dl; + // /TODO std::unique_ptr ctx; - bool isLazy = false; // TODO: [jit] Replace this vector with a thread safe time-optimized @@ -152,15 +168,32 @@ class SERENE_EXPORT Halley { MaybeDylibPtr loadNamespaceFrom(NSLoadRequest &req); // ========================================================================== + std::vector getContainedNamespaces(llvm::StringRef name, + DylibPtr jd); + + llvm::Error createCurrentProcessJD(); + public: Halley(std::unique_ptr ctx, llvm::orc::JITTargetMachineBuilder &&jtmb, llvm::DataLayout &&dl); - /// Initialize the engine by loading required libraries and shared libs - /// like the `serene.core` and other namespaces - llvm::Error initialize(); + // TODO: [jit] Create a function to "require" a namespace as a dependency. + // If the namespace already exists return it otherwise call `loadNamespace`. + + /// Load a namespace by exploring the load paths and different file + /// formats to find the namespace. We assume that we want to load + /// the namespace from file even if it exists already. MaybeDylibPtr loadNamespace(std::string &nsName); + // TODO: Move all the loader related functions to a Loader class + /// Load the shared library in the given `path` to the given JITDylib + /// `jd` via the give ExecutionSession `es`. + /// This function assumes that the shared lib exists. + MaybeDylibPtr loadSharedLibFile(llvm::StringRef name, llvm::StringRef path); + MaybeDylibPtr loadStaticLibrary(const std::string &name); + MaybeDylibPtr loadSharedLibrary(const std::string &name); + // /TODO + static MaybeEngine make(std::unique_ptr sereneCtxPtr, llvm::orc::JITTargetMachineBuilder &&jtmb); @@ -177,8 +210,8 @@ public: void setEngine(std::unique_ptr e, bool isLazy); /// Looks up a packed-argument function with the given sym name and returns a /// pointer to it. Propagates errors in case of failure. - MaybeEnginePtr lookup(const char *nsName, const char *sym); - MaybeEnginePtr lookup(const types::Symbol &sym) const; + MaybeJitAddress lookup(const char *nsName, const char *sym); + MaybeJitAddress lookup(const types::Symbol &sym) const; /// Invokes the function with the given name passing it the list of opaque /// pointers to the actual arguments. diff --git a/libserene/lib/fs.cpp b/libserene/lib/fs.cpp index 08b39e2..98380aa 100644 --- a/libserene/lib/fs.cpp +++ b/libserene/lib/fs.cpp @@ -20,6 +20,8 @@ #include "serene/context.h" +#include + namespace serene::fs { std::string extensionFor(SereneContext &ctx, NSFileType t) { @@ -50,7 +52,7 @@ std::string extensionFor(SereneContext &ctx, NSFileType t) { /// Converts the given namespace name `nsName` to the file name /// for that name space. E.g, `some.random.ns` will be translated /// to `some_random_ns`. -std::string namespaceToPath(const llvm::StringRef nsName) { +std::string namespaceToPath(llvm::StringRef nsName) { // TODO: [fs][perf] This function is not efficient. Fix it std::string nsNameCopy = nsName.str(); std::replace(nsNameCopy.begin(), nsNameCopy.end(), '.', '/'); @@ -62,6 +64,30 @@ std::string namespaceToPath(const llvm::StringRef nsName) { return std::string(path); }; +bool isStaticLib(llvm::StringRef path) { + llvm::file_magic magic; + // llvm::identify_magic returns an error code on failure + if (llvm::identify_magic(path, magic)) { + // If there was an error loading the file then skip it. + return false; + } + + return (magic == llvm::file_magic::archive || + magic == llvm::file_magic::macho_universal_binary); +}; + +bool isSharedLib(llvm::StringRef path) { + llvm::file_magic magic; + // llvm::identify_magic returns an error code on failure + if (llvm::identify_magic(path, magic)) { + // If there was an error loading the file then skip it. + return false; + } + + return (magic == llvm::file_magic::macho_dynamically_linked_shared_lib || + magic == llvm::file_magic::elf_shared_object); +}; + /// Return a boolean indicating whether or not the given path exists. bool exists(llvm::StringRef path) { llvm::sys::fs::file_status status; diff --git a/libserene/lib/jit/halley.cpp b/libserene/lib/jit/halley.cpp index 65177d9..9678970 100644 --- a/libserene/lib/jit/halley.cpp +++ b/libserene/lib/jit/halley.cpp @@ -30,15 +30,17 @@ #include // for Triple #include // for itera... #include -#include // for JITEv... -#include // for TMOwn... -#include // for Execu... -#include // for opera... +#include // for JITEv... +#include // for TMOwn... +#include // for Execu... +#include // for opera... +#include #include // for Dynam... #include // for IRCom... #include // for JITTa... #include // for LLJIT... #include +#include #include // for RTDyl... #include // for Threa... #include // for Secti... @@ -277,33 +279,11 @@ MaybeEngine Halley::make(std::unique_ptr sereneCtxPtr, // exported symbol visibility. // cf llvm/lib/ExecutionEngine/Orc/LLJIT.cpp // LLJIT::createObjectLinkingLayer - if (sereneCtx.triple.isOSBinFormatCOFF()) { objectLayer->setOverrideObjectFlagsWithResponsibilityFlags(true); objectLayer->setAutoClaimResponsibilityForObjectSymbols(true); } - // Resolve symbols from shared libraries. - // for (auto libPath : sharedLibPaths) { - // auto mb = llvm::MemoryBuffer::getFile(libPath); - // if (!mb) { - // llvm::errs() << "Failed to create MemoryBuffer for: " << libPath - // << "\nError: " << mb.getError().message() << "\n"; - // continue; - // } - // auto &JD = session.createBareJITDylib(std::string(libPath)); - // auto loaded = llvm::orc::DynamicLibrarySearchGenerator::Load( - // libPath.data(), dataLayout.getGlobalPrefix()); - // if (!loaded) { - // llvm::errs() << "Could not load " << libPath << ":\n " - // << loaded.takeError() << "\n"; - // continue; - // } - - // JD.addGenerator(std::move(*loaded)); - // cantFail(objectLayer->add(JD, std::move(mb.get()))); - // } - return objectLayer; }; @@ -326,6 +306,10 @@ MaybeEngine Halley::make(std::unique_ptr sereneCtxPtr, std::move(*targetMachine), jitEngine->cache.get()); }; + // TODO: [jit] This is not a proper way to handle both engines. + // Create two different classes for different execution modes + // (`lazy` vs `eager`) with the same interface and use them + // where appropriate. if (sereneCtx.opts.JITLazy) { // Setup a LLLazyJIT instance to the times that latency is important // for example in a REPL. This way @@ -335,8 +319,8 @@ MaybeEngine Halley::make(std::unique_ptr sereneCtxPtr, .setCompileFunctionCreator(compileFunctionCreator) .setObjectLinkingLayerCreator(objectLinkingLayerCreator) .create()); - jitEngine->setEngine(std::move(jit), true); + jitEngine->setEngine(std::move(jit), sereneCtx.opts.JITLazy); } else { // Setup a LLJIT instance for the times that performance is important // and we want to compile everything as soon as possible. For instance @@ -346,9 +330,9 @@ MaybeEngine Halley::make(std::unique_ptr sereneCtxPtr, .setCompileFunctionCreator(compileFunctionCreator) .setObjectLinkingLayerCreator(objectLinkingLayerCreator) .create()); - - jitEngine->setEngine(std::move(jit), false); + jitEngine->setEngine(std::move(jit), sereneCtx.opts.JITLazy); } + // /TODO jitEngine->engine->getIRCompileLayer().setNotifyCompiled( [&](llvm::orc::MaterializationResponsibility &r, @@ -360,11 +344,9 @@ MaybeEngine Halley::make(std::unique_ptr sereneCtxPtr, }); }); - // Resolve symbols that are statically linked in the current process. - llvm::orc::JITDylib &mainJD = jitEngine->engine->getMainJITDylib(); - mainJD.addGenerator( - cantFail(llvm::orc::DynamicLibrarySearchGenerator::GetForCurrentProcess( - jitEngine->dl.getGlobalPrefix()))); + if (auto err = jitEngine->createCurrentProcessJD()) { + return err; + } return MaybeEngine(std::move(jitEngine)); }; @@ -423,7 +405,7 @@ llvm::Error Halley::createEmptyNS(const char *name) { return llvm::Error::success(); }; -MaybeEnginePtr Halley::lookup(const char *nsName, const char *sym) { +MaybeJitAddress Halley::lookup(const char *nsName, const char *sym) { assert(sym != nullptr && "'sym' is null: lookup"); assert(nsName != nullptr && "'nsName' is null: lookup"); @@ -432,12 +414,14 @@ MaybeEnginePtr Halley::lookup(const char *nsName, const char *sym) { std::string fqsym = (ns + "/" + s).str(); + HALLEY_LOG("Looking up symbol: " << fqsym); auto *dylib = jitDylibs[nsName].back(); if (dylib == nullptr) { return tempError(*ctx, "No dylib " + s); } + HALLEY_LOG("Looking in dylib: " << (void *)dylib); auto expectedSymbol = engine->lookup(*dylib, fqsym); // JIT lookup may return an Error referring to strings stored internally by @@ -450,14 +434,16 @@ MaybeEnginePtr Halley::lookup(const char *nsName, const char *sym) { return expectedSymbol.takeError(); } - auto rawFPtr = *expectedSymbol; + auto rawFPtr = expectedSymbol->getAddress(); + // NOLINTNEXTLINE(performance-no-int-to-ptr) - auto fptr = reinterpret_cast(&rawFPtr); + auto fptr = reinterpret_cast(rawFPtr); if (fptr == nullptr) { return tempError(*ctx, "Lookup function is null!"); } + HALLEY_LOG("Found symbol '" << fqsym << "' at " << (void *)fptr); return fptr; }; @@ -523,7 +509,7 @@ Halley::loadNamespaceFrom(NSLoadRequest &req) { if (!fs::exists(file)) { // Can't locate any object file, skit to the next loader - llvm::outs() << "file: " << file << "\n"; + HALLEY_LOG("File does not exist: " << file << "\n"); return nullptr; } @@ -660,6 +646,16 @@ MaybeDylibPtr Halley::loadNamespace(std::string &nsName) { } if (*maybeJDptr != nullptr) { + auto *processJD = engine->getExecutionSession().getJITDylibByName( + MAIN_PROCESS_JD_NAME); + + if (processJD == nullptr) { + // TODO: [jit] Panic here + return tempError(*ctx, "Can't find the main process JD"); + // /TODO + } + + (*maybeJDptr)->addToLinkOrder(*processJD); return *maybeJDptr; } } @@ -668,8 +664,167 @@ MaybeDylibPtr Halley::loadNamespace(std::string &nsName) { return tempError(*ctx, "Can't find namespace: " + nsName); }; -llvm::Error Halley::initialize() { - (void)ctx; +MaybeDylibPtr Halley::loadStaticLibrary(const std::string &name) { + if (ctx->getLoadPaths().empty()) { + return tempError(*ctx, "Load paths should not be empty"); + } + + for (auto &path : ctx->getLoadPaths()) { + auto file = fs::join(path, name + ".a"); + + if (!fs::exists(file)) { + continue; + } + + if (!fs::isStaticLib(file)) { + return tempError(*ctx, "Not a static lib: " + file); + } + + auto *objectLayer = &engine->getObjLinkingLayer(); + + auto generator = llvm::orc::StaticLibraryDefinitionGenerator::Load( + *objectLayer, file.c_str(), + engine->getExecutionSession() + .getExecutorProcessControl() + .getTargetTriple(), + std::move(llvm::orc::getObjectFileInterface)); + + if (!generator) { + return generator.takeError(); + } + + auto jd = engine->createJITDylib(llvm::formatv("{0}#{1}", name, 0)); + + if (!jd) { + return jd.takeError(); + } + + jd->addGenerator(std::move(*generator)); + + std::vector nsNames = {name}; + + auto definition = engine->lookup(*jd, "__serene_namespaces"); + + if (!definition) { + HALLEY_LOG("Library '" << name << "' is not a Serene lib."); + // We just want to ignore the error + llvm::consumeError(definition.takeError()); + } else { + HALLEY_LOG("Library '" << name << "' is a Serene lib."); + // TODO: call the __serene_namespaces and set nsNames to + // the list of namespaces that it returns + (void)*definition; + } + + for (auto &nsName : nsNames) { + auto ns = makeNamespace(nsName.str().c_str()); + pushJITDylib(ns, &(*jd)); + } + + return &jd.get(); + } + + return tempError(*ctx, "Can't find static lib: " + name); +}; + +MaybeDylibPtr Halley::loadSharedLibFile(llvm::StringRef name, + llvm::StringRef path) { + if (!fs::isSharedLib(path)) { + return tempError(*ctx, "Not a shared lib: " + path); + } + + auto generator = llvm::orc::EPCDynamicLibrarySearchGenerator::Load( + engine->getExecutionSession(), path.str().c_str()); + + if (!generator) { + return generator.takeError(); + } + + auto jd = engine->createJITDylib(llvm::formatv("{0}#{1}", name, 0)); + + if (!jd) { + return jd.takeError(); + } + + jd->addGenerator(std::move(*generator)); + + return &jd.get(); +}; + +MaybeDylibPtr Halley::loadSharedLibrary(const std::string &name) { + if (ctx->getLoadPaths().empty()) { + return tempError(*ctx, "Load paths should not be empty"); + } + + for (auto &path : ctx->getLoadPaths()) { + auto file = fs::join(path, name + ".so"); + + if (!fs::exists(file)) { + continue; + } + + auto maybeJD = loadSharedLibFile(name, file); + if (!maybeJD) { + return maybeJD.takeError(); + } + + auto *jd = *maybeJD; + auto nsNames = getContainedNamespaces(name, jd); + + for (const auto *nsName : nsNames) { + auto ns = makeNamespace(nsName); + pushJITDylib(ns, jd); + } + + return jd; + } + + return tempError(*ctx, "Can't find the dynamic lib: " + name); +}; + +std::vector Halley::getContainedNamespaces(llvm::StringRef name, + DylibPtr jd) { + + std::vector nsNames = {name.str().c_str()}; + auto definition = engine->lookup(*jd, "__serene_namespaces"); + + if (!definition) { + HALLEY_LOG("Library is not a Serene lib."); + // We just want to ignore the error + llvm::consumeError(definition.takeError()); + } + HALLEY_LOG("Library is a Serene lib."); + // TODO: call the __serene_namespaces and set nsNames to + // the list of namespaces that it returns + + return nsNames; +}; + +llvm::Error Halley::createCurrentProcessJD() { + + auto &es = engine->getExecutionSession(); + auto *processJDPtr = es.getJITDylibByName(MAIN_PROCESS_JD_NAME); + + if (processJDPtr != nullptr) { + // We already created the JITDylib for the current process + return llvm::Error::success(); + } + + auto processJD = es.createJITDylib(MAIN_PROCESS_JD_NAME); + + if (!processJD) { + return processJD.takeError(); + } + + auto generator = + llvm::orc::DynamicLibrarySearchGenerator::GetForCurrentProcess( + engine->getDataLayout().getGlobalPrefix()); + + if (!generator) { + return generator.takeError(); + } + + processJD->addGenerator(std::move(*generator)); return llvm::Error::success(); }; diff --git a/serenec/serenec.cpp b/serenec/serenec.cpp index b37b89b..829eb01 100644 --- a/serenec/serenec.cpp +++ b/serenec/serenec.cpp @@ -299,9 +299,9 @@ int main(int argc, char *argv[]) { // } std::string core = "serene.core"; - auto maybeJD = engine->loadNamespace(core); - if (!maybeJD) { - llvm::errs() << "Error: " << maybeJD.takeError() << "'\n"; + auto maybeCore = engine->loadNamespace(core); + if (!maybeCore) { + llvm::errs() << "Error: " << maybeCore.takeError() << "'\n"; return 1; } @@ -312,9 +312,18 @@ int main(int argc, char *argv[]) { return 1; } - auto c = *bt; - void *res = c(); + if (*bt == nullptr) { + llvm::errs() << "Error: nullptr?\n"; + return 1; + } + auto *c = *bt; + void *res = c(); + // for (int i = 0; i <= 10; i++) { + // printf(">> %02x", *(c + i)); + // } + printf("Res >> %p\n", res); + llvm::outs() << "Res: " << *((int *)res) << "\n"; (void)res; // // TODO: handle the outputDir by not forcing it. it should be