Serene/serene
Serene
/
serene
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Activity

Merge jit2 branch

This commit is contained in:
Sameer Rahmani 2021-11-12 16:28:34 +00:00
parent b4f31984c4 3da6d2d1fc
commit 800105dbea
23 changed files with 878 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dev.org
View File

@ -98,7 +98,11 @@ on ADF
- [[https://www.wilfred.me.uk/blog/2014/09/27/the-definitive-guide-to-syntax-highlighting/][The Definitive Guide To Syntax Highlighting]]
** Linker
- [[https://lwn.net/Articles/276782/][20 part linker essay]]
- [[https://lld.llvm.org/index.html][LLD Usage]]
** LLVM
- [[https://blog.yossarian.net/2021/07/19/LLVM-internals-part-1-bitcode-format][LLVM Internals]]
*** TableGen
- [[https://llvm.org/docs/TableGen/BackGuide.html#creating-a-new-backend][Create a backend]]
* Considerations
** Hashmaps

4
docs/examples/hello_world.srn
View File

@ -1,4 +1,2 @@
(def main
(fn () 4))
(def main (fn () 4))
(def main1 (fn (v y n) 3))

27
include/serene/context.h
View File

@ -22,6 +22,8 @@
#include "serene/diagnostics.h"
#include "serene/environment.h"
#include "serene/export.h"
#include "serene/jit.h"
#include "serene/jit/engine.h"
#include "serene/namespace.h"
#include "serene/passes.h"
#include "serene/slir/dialect.h"
@ -77,6 +79,8 @@ public:
// order to destroy last. DO NOT change the order or add anything before
// them
// --------------------------------------------------------------------------
// TODO: Remove the llvmContext
llvm::LLVMContext llvmContext;
mlir::MLIRContext mlirContext;
@ -84,6 +88,8 @@ public:
std::unique_ptr<DiagnosticEngine> diagEngine;
std::unique_ptr<serene::jit::SereneJIT> jit;
/// The source manager is responsible for loading namespaces and practically
/// managing the source code in form of memory buffers.
SourceMgr sourceManager;
@ -93,6 +99,9 @@ public:
std::string targetTriple;
// TODO: Replace target Triple with this one
llvm::Triple triple;
/// Insert the given `ns` into the context. The Context object is
/// the owner of all the namespaces. The `ns` will overwrite any
/// namespace with the same name.
@ -137,6 +146,24 @@ public:
MaybeNS readNamespace(const std::string &name);
MaybeNS readNamespace(const std::string &name, reader::LocationRange loc);
static std::unique_ptr<llvm::LLVMContext> genLLVMContext() {
return std::make_unique<llvm::LLVMContext>();
};
static std::unique_ptr<SereneContext> make() {
auto ctx = std::make_unique<SereneContext>();
auto maybeJIT = serene::jit::makeSereneJIT(*ctx);
if (!maybeJIT) {
// TODO: Raise an error here
return nullptr;
}
ctx->jit.swap(*maybeJIT);
return ctx;
};
private:
CompilationPhase targetPhase;

14
include/serene/environment.h
View File

@ -34,8 +34,9 @@ class Environment {
Environment<K, V> *parent;
using StorageType = llvm::DenseMap<K, V>;
// The actual bindings storage
llvm::DenseMap<K, V> pairs;
StorageType pairs;
public:
Environment() : parent(nullptr) {}
@ -60,6 +61,17 @@ public:
pairs.insert(std::pair<K, V>(key, value));
return mlir::success();
};
inline typename StorageType::iterator begin() { return pairs.begin(); }
inline typename StorageType::iterator end() { return pairs.end(); }
inline typename StorageType::const_iterator begin() const {
return pairs.begin();
}
inline typename StorageType::const_iterator end() const {
return pairs.end();
}
};
} // namespace serene

2
include/serene/jit.h
View File

@ -40,7 +40,7 @@
#include <memory>
#define JIT_LOG(...) \
#define JIT2_LOG(...) \
DEBUG_WITH_TYPE("JIT", llvm::dbgs() << "[JIT]: " << __VA_ARGS__ << "\n");
namespace serene {

18
include/serene/jit/README.org Normal file
View File

@ -0,0 +1,18 @@
#+TITLE: Serene JIT
#+AUTHOR: Sameer Rahmani
#+STARTUP: logdrawer logdone logreschedule indent content align constSI entitiespretty nolatexpreview
#+OPTIONS: tex:t
* SereneJIT
** Development Notes
*** Steps to take
You usually want a custom =MaterializationUnit= for your program representation, and a custom =Layer=. The Layer will have two
operations: =add= and =emit=. The =add= operation takes an instance of your program representation, builds one of your custom
=MaterializationUnits= to hold it, then adds it to a =JITDylib=. The =emit= operation takes a =MaterializationResponsibility=
object and an instance of your program representation and materializes it, usually by compiling it and handing the resulting
object off to an =ObjectLinkingLayer=.
Your custom =MaterializationUnit= will have two operations: =materialize= and =discard=. The =materialize= function will be
called for you when any symbol provided by the unit is looked up, and it should just call the =emit= function on your layer,
passing in the given =MaterializationResponsibility= and the wrapped program representation. The =discard= function will be
called if some weak symbol provided by your unit is not needed (because the JIT found an overriding definition).
You can use this to drop your definition early, or just ignore it and let the linker drop the definition later.

176
include/serene/jit/engine.h Normal file
View File

@ -0,0 +1,176 @@
/* -*- C++ -*-
* Serene Programming Language
*
* Copyright (c) 2019-2021 Sameer Rahmani <lxsameer@gnu.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 2.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* Commentary:
*/
// TODO: Look at the LLJITBuilderState::prepareForConstruction to setup the
// object linking layer
#ifndef SERENE_JIT_ENGINE_H
#define SERENE_JIT_ENGINE_H
#include "serene/jit/layers.h"
#include "serene/namespace.h"
#include <llvm/ExecutionEngine/Orc/CompileUtils.h> // for ConcurrentIRCompiler
#include <llvm/ExecutionEngine/Orc/Core.h>
#include <llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h>
#include <llvm/ExecutionEngine/Orc/ExecutionUtils.h> // for DynamicLibrarySearchGenerator
#include <llvm/ExecutionEngine/Orc/IRCompileLayer.h>
#include <llvm/ExecutionEngine/Orc/Mangling.h>
#include <llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h>
#include <llvm/ExecutionEngine/SectionMemoryManager.h>
#include <llvm/IR/DataLayout.h>
#include <llvm/Support/raw_ostream.h>
#include <memory>
#define JIT_LOG(...) \
DEBUG_WITH_TYPE("jit", llvm::dbgs() << "[JIT]: " << __VA_ARGS__ << "\n");
namespace orc = llvm::orc;
namespace serene {
class SereneContext;
namespace jit {
class SereneJIT {
/// An ExecutionSession represents a running JIT program.
std::unique_ptr<orc::ExecutionSession> es;
std::unique_ptr<orc::EPCIndirectionUtils> epciu;
llvm::DataLayout dl;
/// Mangler in responsible for changing the symbol names based on our
/// naming convention.
orc::MangleAndInterner mangler;
// Layers -------------------------------------------------------------------
// Serene's JIT follows the same design as the ORC and uses the framework that
// it provides.
//
// Layers are the building blocks of the JIT and work on top of each other
// to add different functionalities to the JIT. Order is important here and
// layers call each other and pa
/// The object linking layer allows object files to be added to the JIT
orc::RTDyldObjectLinkingLayer objectLayer;
/// The compiler layer is responsible to compile the LLVMIR to target code
/// for us
orc::IRCompileLayer compileLayer;
// TODO: Enable these two layers when we add the `Platform support`
// std::unique_ptr<orc::IRTransformLayer> transformLayer;
// std::unique_ptr<orc::IRTransformLayer> initHelperTransformLayer;
// orc::CompileOnDemandLayer compileOnDemandLayer;
/// Transform layaer is responsible for running a pass pipeline on the AST
/// and generate LLVM IR
orc::IRTransformLayer transformLayer;
/// The AST Layer reads and import the Serene Ast directly to the JIT
// SereneAstLayer astLayer;
/// NS layer is responsible for adding namespace to the JIT by name.
/// It will import the entire namespace.
NSLayer nsLayer;
/// This a symbol table tha holds symbols from whatever code we execute
orc::JITDylib &mainJD;
serene::SereneContext &ctx;
/// The name of the NS which we are in it at the moment.
std::string inNS;
static llvm::Expected<orc::ThreadSafeModule>
optimizeModule(orc::ThreadSafeModule tsm,
const orc::MaterializationResponsibility &r) {
// TSM.withModuleDo([](Module &M) {
// // Create a function pass manager.
// auto FPM = std::make_unique<legacy::FunctionPassManager>(&M);
// // Add some optimizations.
// FPM->add(createInstructionCombiningPass());
// FPM->add(createReassociatePass());
// FPM->add(createGVNPass());
// FPM->add(createCFGSimplificationPass());
// FPM->doInitialization();
// // Run the optimizations over all functions in the module being added
// to
// // the JIT.
// for (auto &F : M)
// FPM->run(F);
// });
UNUSED(r);
return std::move(tsm);
}
public:
SereneJIT(serene::SereneContext &ctx,
std::unique_ptr<orc::ExecutionSession> es,
std::unique_ptr<orc::EPCIndirectionUtils> epciu,
orc::JITTargetMachineBuilder jtmb, llvm::DataLayout &&dl);
~SereneJIT() {
// if (compileThreads) {
// compileThreads->wait();
// }
if (auto err = es->endSession()) {
es->reportError(std::move(err));
}
};
const llvm::DataLayout &getDataLayout() const { return dl; }
orc::JITDylib &getMainJITDylib() { return mainJD; }
llvm::Error addModule(orc::ThreadSafeModule tsm,
orc::ResourceTrackerSP rt = nullptr) {
if (!rt) {
rt = mainJD.getDefaultResourceTracker();
}
return compileLayer.add(rt, std::move(tsm));
}
llvm::Error addNS(llvm::StringRef nsname,
orc::ResourceTrackerSP rt = nullptr);
llvm::Expected<llvm::JITEvaluatedSymbol> lookup(llvm::StringRef name) {
JIT_LOG("Looking up symbol: " + name);
return es->lookup({&mainJD}, mangler(name.str()));
}
};
llvm::Expected<std::unique_ptr<SereneJIT>>
makeSereneJIT(serene::SereneContext &ctx);
}; // namespace jit
}; // namespace serene
#endif

175
include/serene/jit/layers.h Normal file
View File

@ -0,0 +1,175 @@
/* -*- C++ -*-
* Serene Programming Language
*
* Copyright (c) 2019-2021 Sameer Rahmani <lxsameer@gnu.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 2.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef SERENE_JIT_LAYERS_H
#define SERENE_JIT_LAYERS_H
#include "serene/reader/location.h"
#include "serene/utils.h"
#include <llvm/ADT/StringRef.h>
#include <llvm/ADT/Twine.h>
#include <llvm/ExecutionEngine/JITSymbol.h>
#include <llvm/ExecutionEngine/Orc/Core.h>
#include <llvm/ExecutionEngine/Orc/Layer.h>
#include <llvm/IR/DataLayout.h>
#include <llvm/Support/Error.h>
#define LAYER_LOG(...) \
DEBUG_WITH_TYPE("layer", llvm::dbgs() << "[Layer]: " << __VA_ARGS__ << "\n");
namespace orc = llvm::orc;
namespace serene {
class SereneContext;
class Namespace;
namespace exprs {
class Expression;
using Node = std::shared_ptr<Expression>;
using Ast = std::vector<Node>;
} // namespace exprs
namespace jit {
class SereneAstLayer;
/// This will compile the ast to llvm ir.
llvm::orc::ThreadSafeModule compileAst(serene::SereneContext &ctx,
exprs::Ast &ast);
class SerenAstMaterializationUnit : public orc::MaterializationUnit {
public:
SerenAstMaterializationUnit(SereneContext &ctx, SereneAstLayer &l,
exprs::Ast &ast);
llvm::StringRef getName() const override {
return "SereneAstMaterializationUnit";
}
void
materialize(std::unique_ptr<orc::MaterializationResponsibility> r) override;
private:
void discard(const orc::JITDylib &jd,
const orc::SymbolStringPtr &sym) override {
UNUSED(jd);
UNUSED(sym);
llvm_unreachable("Serene functions are not overridable");
}
serene::SereneContext &ctx;
SereneAstLayer &astLayer;
exprs::Ast &ast;
};
// class SereneAstLayer {
// SereneContext &ctx;
// orc::IRLayer &baseLayer;
// orc::MangleAndInterner &mangler;
// const llvm::DataLayout &dl;
// public:
// SereneAstLayer(SereneContext &ctx, orc::IRLayer &baseLayer,
// orc::MangleAndInterner &mangler, const llvm::DataLayout &dl)
// : ctx(ctx), baseLayer(baseLayer), mangler(mangler), dl(dl){};
// llvm::Error add(orc::ResourceTrackerSP &rt, exprs::Ast &ast) {
// return rt->getJITDylib().define(
// std::make_unique<SerenAstMaterializationUnit>(ctx, *this, ast), rt);
// }
// void emit(std::unique_ptr<orc::MaterializationResponsibility> mr,
// exprs::Ast &e) {
// baseLayer.emit(std::move(mr), compileAst(ctx, e));
// }
// orc::SymbolFlagsMap getInterface(exprs::Ast &e) {
// orc::SymbolFlagsMap Symbols;
// Symbols[mangler(e.getName())] = llvm::JITSymbolFlags(
// llvm::JITSymbolFlags::Exported | llvm::JITSymbolFlags::Callable);
// return Symbols;
// }
// };
/// NS Layer ==================================================================
class NSLayer;
/// This will compile the NS to llvm ir.
llvm::orc::ThreadSafeModule compileNS(serene::SereneContext &ctx,
serene::Namespace &ns);
class NSMaterializationUnit : public orc::MaterializationUnit {
public:
NSMaterializationUnit(SereneContext &ctx, NSLayer &l, serene::Namespace &ns);
llvm::StringRef getName() const override { return "NSMaterializationUnit"; }
void
materialize(std::unique_ptr<orc::MaterializationResponsibility> r) override;
private:
void discard(const orc::JITDylib &jd,
const orc::SymbolStringPtr &sym) override {
UNUSED(jd);
UNUSED(sym);
UNUSED(ctx);
// TODO: Check the ctx to see whether we need to remove the sym or not
}
serene::SereneContext &ctx;
NSLayer &nsLayer;
serene::Namespace &ns;
};
/// NS Layer is responsible for adding namespaces to the JIT
class NSLayer {
serene::SereneContext &ctx;
orc::IRLayer &baseLayer;
orc::MangleAndInterner &mangler;
const llvm::DataLayout &dl;
public:
NSLayer(serene::SereneContext &ctx, orc::IRLayer &baseLayer,
orc::MangleAndInterner &mangler, const llvm::DataLayout &dl)
: ctx(ctx), baseLayer(baseLayer), mangler(mangler), dl(dl){};
llvm::Error add(orc::ResourceTrackerSP &rt, llvm::StringRef nsname) {
auto loc = serene::reader::LocationRange::UnknownLocation(nsname);
return add(rt, nsname, loc);
}
llvm::Error add(orc::ResourceTrackerSP &rt, llvm::StringRef nsname,
serene::reader::LocationRange &loc);
void emit(std::unique_ptr<orc::MaterializationResponsibility> mr,
serene::Namespace &ns) {
// TODO: We need to pass dl to the compilerNS later to aviod recreating
// the data layout all the time
UNUSED(dl);
LAYER_LOG("Emit namespace");
baseLayer.emit(std::move(mr), compileNS(ctx, ns));
}
orc::SymbolFlagsMap getInterface(serene::Namespace &ns);
};
} // namespace jit
} // namespace serene
#endif

3
include/serene/namespace.h
View File

@ -38,6 +38,7 @@
#include <llvm/ADT/SmallString.h>
#include <llvm/ADT/StringRef.h>
#include <llvm/ADT/Twine.h>
#include <llvm/ExecutionEngine/Orc/ThreadSafeModule.h>
#include <llvm/IR/Module.h>
#include <mlir/IR/Builders.h>
#include <mlir/IR/BuiltinOps.h>
@ -61,7 +62,7 @@ using Node = std::shared_ptr<Expression>;
using Ast = std::vector<Node>;
} // namespace exprs
using MaybeModule = llvm::Optional<std::unique_ptr<llvm::Module>>;
using MaybeModule = llvm::Optional<llvm::orc::ThreadSafeModule>;
using MaybeModuleOp = llvm::Optional<mlir::OwningOpRef<mlir::ModuleOp>>;
/// Serene's namespaces are the unit of compilation. Any code that needs to be

2
include/serene/serene.h
View File

@ -65,7 +65,7 @@ SERENE_EXPORT exprs::MaybeAst read(SereneContext &ctx, std::string &input);
/// In case of an error Serene will throw the error messages vis the diagnostic
/// engine as well and the error that this function returns will be the
/// generic error message.
// SERENE_EXPORT exprs::MaybeNode eval(SereneContext &ctx, exprs::Ast input);
SERENE_EXPORT exprs::MaybeNode eval(SereneContext &ctx, exprs::Ast &input);
// TODO: Return a Serene String type instead of the std::string
// TODO: Create an overload to get a stream instead of the result string

8
include/serene/slir/slir.h
View File

@ -20,6 +20,7 @@
#include "serene/exprs/expression.h"
#include <llvm/ADT/Optional.h>
#include <mlir/IR/BuiltinOps.h>
#include <mlir/IR/MLIRContext.h>
@ -27,8 +28,11 @@
namespace serene {
namespace slir {
std::unique_ptr<llvm::Module> compileToLLVMIR(serene::SereneContext &ctx,
mlir::ModuleOp &module);
// std::unique_ptr<llvm::Module> compileToLLVMIR(serene::SereneContext &ctx,
// mlir::ModuleOp &module);
llvm::Optional<llvm::orc::ThreadSafeModule>
compileToLLVMIR(serene::SereneContext &ctx, mlir::ModuleOp &module);
} // namespace slir
} // namespace serene

8
include/serene/source_mgr.h
View File

@ -103,7 +103,7 @@ private:
SrcBuffer &operator=(const SrcBuffer &) = delete;
~SrcBuffer();
};
using ErrorOrMemBufPtr = llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>;
using MemBufPtr = std::unique_ptr<llvm::MemoryBuffer>;
/// This is all of the buffers that we are reading from.
std::vector<SrcBuffer> buffers;
@ -119,8 +119,8 @@ private:
// a unique pointer to the memory buffer containing the content or an error.
// In the success case it will put the path of the file into the \p
// importedFile.
ErrorOrMemBufPtr findFileInLoadPath(const std::string &name,
std::string &importedFile);
MemBufPtr findFileInLoadPath(const std::string &name,
std::string &importedFile);
bool isValidBufferID(unsigned i) const;
@ -138,7 +138,7 @@ public:
/// Set the `loadPaths` to the given \p dirs. `loadPaths` is a vector of
/// directories that Serene will look in order to find a file that constains a
/// namespace which it is looking for.
void setLoadPaths(const std::vector<std::string> &dirs) { loadPaths = dirs; }
void setLoadPaths(std::vector<std::string> &dirs) { loadPaths.swap(dirs); }
/// Return a reference to a `SrcBuffer` with the given ID \p i.
const SrcBuffer &getBufferInfo(unsigned i) const {

5
src/libserene/CMakeLists.txt
View File

@ -41,10 +41,13 @@ add_library(serene
serene.cpp
context.cpp
namespace.cpp
jit.cpp
source_mgr.cpp
diagnostics.cpp
# jit.cpp
jit/engine.cpp
jit/layers.cpp
# Reader
reader/reader.cpp
reader/semantics.cpp

4
src/libserene/context.cpp
View File

@ -99,7 +99,7 @@ MaybeNS SereneContext::readNamespace(const std::string &name) {
MaybeNS SereneContext::readNamespace(const std::string &name,
reader::LocationRange loc) {
return sourceManager.readNamespace(*this, std::move(name), loc);
return sourceManager.readNamespace(*this, name, loc);
}
void terminate(SereneContext &ctx, int exitCode) {
@ -112,7 +112,7 @@ void terminate(SereneContext &ctx, int exitCode) {
}
std::unique_ptr<SereneContext> makeSereneContext() {
return std::make_unique<SereneContext>();
return SereneContext::make();
};
}; // namespace serene

2
src/libserene/exprs/def.cpp
View File

@ -59,7 +59,6 @@ MaybeNode Def::make(SereneContext &ctx, List *list) {
// Make sure that the list starts with a `def`
Symbol *defSym = llvm::dyn_cast<Symbol>(list->elements[0].get());
// TODO: Replace this one with a runtime check
assert((defSym && defSym->name == "def") &&
"The first element of the list should be a 'def'.");
@ -100,7 +99,6 @@ MaybeNode Def::make(SereneContext &ctx, List *list) {
tmp->setName(binding->name);
}
// auto analayzedValuePtr = analyzedValue;
auto result = ctx.getCurrentNS().semanticEnv.insert_symbol(binding->name,
analyzedValue);

108
src/libserene/jit/engine.cpp Normal file
View File

@ -0,0 +1,108 @@
/* -*- C++ -*-
* Serene Programming Language
*
* Copyright (c) 2019-2021 Sameer Rahmani <lxsameer@gnu.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 2.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "serene/jit/engine.h"
#include "serene/context.h"
#include "serene/jit/layers.h"
#include "serene/utils.h"
#include <llvm/ExecutionEngine/JITSymbol.h>
#include <memory>
namespace serene::jit {
static void handleLazyCallThroughError() {
// TODO: Report to the diag engine
llvm::errs() << "LazyCallThrough error: Could not find function body";
// TODO: terminate ?
}
SereneJIT::SereneJIT(serene::SereneContext &ctx,
std::unique_ptr<orc::ExecutionSession> es,
std::unique_ptr<orc::EPCIndirectionUtils> epciu,
orc::JITTargetMachineBuilder jtmb, llvm::DataLayout &&dl)
: es(std::move(es)), epciu(std::move(epciu)), dl(dl),
mangler(*this->es, this->dl),
objectLayer(
*this->es,
[]() { return std::make_unique<llvm::SectionMemoryManager>(); }),
compileLayer(
*this->es, objectLayer,
std::make_unique<orc::ConcurrentIRCompiler>(std::move(jtmb))),
transformLayer(*this->es, compileLayer, optimizeModule),
// TODO: Change compileOnDemandLayer to use an optimization layer
// as the parent
// compileOnDemandLayer(
// *this->es, compileLayer, this->epciu->getLazyCallThroughManager(),
// [this] { return this->epciu->createIndirectStubsManager(); }),
nsLayer(ctx, transformLayer, mangler, dl),
mainJD(this->es->createBareJITDylib(ctx.getCurrentNS().name)), ctx(ctx) {
UNUSED(this->ctx);
mainJD.addGenerator(
cantFail(orc::DynamicLibrarySearchGenerator::GetForCurrentProcess(
dl.getGlobalPrefix())));
// if (numCompileThreads > 0) {
// compileOnDemandLayer.setCloneToNewContextOnEmit(true);
// }
}
llvm::Error SereneJIT::addNS(llvm::StringRef nsname,
orc::ResourceTrackerSP rt) {
if (!rt) {
rt = mainJD.getDefaultResourceTracker();
}
return nsLayer.add(rt, nsname);
};
llvm::Expected<std::unique_ptr<SereneJIT>>
makeSereneJIT(serene::SereneContext &ctx) {
auto epc = orc::SelfExecutorProcessControl::Create();
if (!epc) {
return epc.takeError();
}
auto es = std::make_unique<orc::ExecutionSession>(std::move(*epc));
auto epciu =
orc::EPCIndirectionUtils::Create(es->getExecutorProcessControl());
if (!epciu) {
return epciu.takeError();
}
(*epciu)->createLazyCallThroughManager(
*es, llvm::pointerToJITTargetAddress(&handleLazyCallThroughError));
if (auto err = setUpInProcessLCTMReentryViaEPCIU(**epciu)) {
return std::move(err);
}
orc::JITTargetMachineBuilder jtmb(
es->getExecutorProcessControl().getTargetTriple());
auto dl = jtmb.getDefaultDataLayoutForTarget();
if (!dl) {
return dl.takeError();
}
return std::make_unique<SereneJIT>(ctx, std::move(es), std::move(*epciu),
std::move(jtmb), std::move(*dl));
};
} // namespace serene::jit

115
src/libserene/jit/layers.cpp Normal file
View File

@ -0,0 +1,115 @@
/* -*- C++ -*-
* Serene Programming Language
*
* Copyright (c) 2019-2021 Sameer Rahmani <lxsameer@gnu.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, version 2.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "serene/jit/layers.h"
#include "serene/context.h"
#include "serene/exprs/fn.h"
#include "serene/exprs/traits.h"
#include <llvm/ExecutionEngine/Orc/ThreadSafeModule.h>
#include <llvm/Support/Error.h> // for report_fatal_error
namespace serene::jit {
// llvm::orc::ThreadSafeModule compileAst(serene::SereneContext &ctx,
// exprs::Ast &ast){
// };
// SerenAstMaterializationUnit::SerenAstMaterializationUnit(
// serene::SereneContext &ctx, SereneAstLayer &l, exprs::Ast &ast)
// : MaterializationUnit(l.getInterface(ast), nullptr), ctx(ctx),
// astLayer(l),
// ast(ast){};
// void SerenAstMaterializationUnit::materialize(
// std::unique_ptr<orc::MaterializationResponsibility> r) {
// astLayer.emit(std::move(r), ast);
// }
/// NS Layer ==================================================================
llvm::orc::ThreadSafeModule compileNS(serene::SereneContext &ctx,
serene::Namespace &ns) {
UNUSED(ctx);
LAYER_LOG("Compile namespace: " + ns.name);
auto maybeModule = ns.compileToLLVM();
if (!maybeModule) {
// TODO: Handle failure
llvm::report_fatal_error("Couldn't compile lazily JIT'd function");
}
return std::move(maybeModule.getValue());
};
NSMaterializationUnit::NSMaterializationUnit(SereneContext &ctx, NSLayer &l,
serene::Namespace &ns)
: MaterializationUnit(l.getInterface(ns), nullptr), ctx(ctx), nsLayer(l),
ns(ns){};
void NSMaterializationUnit::materialize(
std::unique_ptr<orc::MaterializationResponsibility> r) {
nsLayer.emit(std::move(r), ns);
}
llvm::Error NSLayer::add(orc::ResourceTrackerSP &rt, llvm::StringRef nsname,
reader::LocationRange &loc) {
LAYER_LOG("Add namespace: " + nsname);
auto maybeNS = ctx.sourceManager.readNamespace(ctx, nsname.str(), loc);
if (!maybeNS) {
// TODO: Fix this by making Serene errors compatible with llvm::Error
auto err = maybeNS.getError();
return llvm::make_error<llvm::StringError>(
llvm::Twine(err.front()->getMessage()),
std::make_error_code(std::errc::io_error));
}
auto ns = maybeNS.getValue();
LAYER_LOG("Add the materialize unit for: " + nsname);
return rt->getJITDylib().define(
std::make_unique<NSMaterializationUnit>(ctx, *this, *ns), rt);
}
orc::SymbolFlagsMap NSLayer::getInterface(serene::Namespace &ns) {
orc::SymbolFlagsMap Symbols;
for (auto &k : ns.semanticEnv) {
auto flags = llvm::JITSymbolFlags::Exported;
auto name = k.getFirst();
auto expr = k.getSecond();
if (expr->getType() == exprs::ExprType::Fn) {
flags = flags | llvm::JITSymbolFlags::Callable;
}
auto mangledSym = mangler(k.getFirst());
LAYER_LOG("Mangle symbol for: " + k.getFirst() + " = " << mangledSym);
Symbols[mangledSym] = llvm::JITSymbolFlags(flags);
}
return Symbols;
}
} // namespace serene::jit

2
src/libserene/namespace.cpp
View File

@ -146,7 +146,7 @@ MaybeModule Namespace::compileToLLVM() {
if (ctx.getTargetPhase() >= CompilationPhase::IR) {
mlir::ModuleOp module = maybeModule.getValue().get();
return MaybeModule(::serene::slir::compileToLLVMIR(ctx, module));
return ::serene::slir::compileToLLVMIR(ctx, module);
}
return llvm::None;

34
src/libserene/serene.cpp
View File

@ -26,12 +26,19 @@
#include "serene/diagnostics.h"
#include "serene/exprs/expression.h"
// TODO: Remove it
#include "serene/exprs/number.h"
#include "serene/reader/reader.h"
#include <llvm/ADT/None.h>
#include <llvm/Support/CommandLine.h>
#include <llvm/Support/Error.h>
#include <llvm/Support/TargetSelect.h>
#include <llvm/Support/raw_ostream.h>
namespace serene {
using exprs::Number;
void initCompiler() {
llvm::InitializeAllTargetInfos();
@ -96,9 +103,32 @@ SERENE_EXPORT exprs::MaybeAst read(SereneContext &ctx, std::string &input) {
return reader::read(ctx, input, currentNS.name, filename);
};
// SERENE_EXPORT exprs::MaybeNode eval(SereneContext &ctx, exprs::Ast input){
SERENE_EXPORT exprs::MaybeNode eval(SereneContext &ctx, exprs::Ast &input) {
// };
// TODO: Fix the eval function
UNUSED(input);
auto loc = reader::LocationRange::UnknownLocation("nsname");
auto err = ctx.jit->addNS("docs.examples.hello_world");
if (err) {
llvm::errs() << err;
auto e = errors::makeErrorTree(loc, errors::NSLoadError);
return exprs::makeErrorNode(loc, errors::NSLoadError);
}
std::string tmp("main");
llvm::ExitOnError e;
// Get the anonymous expression's JITSymbol.
auto sym = e(ctx.jit->lookup(tmp));
llvm::outs() << "eval here\n";
// Get the symbol's address and cast it to the right type (takes no
// arguments, returns a double) so we can call it as a native function.
auto *f = (int (*)())(intptr_t)sym.getAddress();
f();
return exprs::make<exprs::Number>(loc, "4", false, false);
};
SERENE_EXPORT void print(SereneContext &ctx, const exprs::Ast &input,
std::string &result) {

25
src/libserene/slir/slir.cpp
View File

@ -21,17 +21,20 @@
namespace serene {
namespace slir {
std::unique_ptr<llvm::Module> compileToLLVMIR(serene::SereneContext &ctx,
mlir::ModuleOp &module) {
llvm::Optional<llvm::orc::ThreadSafeModule>
compileToLLVMIR(serene::SereneContext &ctx, mlir::ModuleOp &module) {
auto llvmContext = serene::SereneContext::genLLVMContext();
// Register the translation to LLVM IR with the MLIR context.
mlir::registerLLVMDialectTranslation(ctx.mlirContext);
mlir::registerLLVMDialectTranslation(*module.getContext());
// Convert the module to LLVM IR in a new LLVM IR context.
auto llvmModule = mlir::translateModuleToLLVMIR(module, ctx.llvmContext);
auto llvmModule = mlir::translateModuleToLLVMIR(module, *llvmContext);
if (!llvmModule) {
// TODO: Return a Result type instead
llvm::errs() << "Failed to emit LLVM IR\n";
throw std::runtime_error("Failed to emit LLVM IR\n");
module.emitError("Failed to emit LLVM IR\n");
return llvm::None;
}
// Initialize LLVM targets.
@ -45,12 +48,16 @@ std::unique_ptr<llvm::Module> compileToLLVMIR(serene::SereneContext &ctx,
auto optPipeline = mlir::makeOptimizingTransformer(
/*optLevel=*/ctx.getOptimizatioLevel(), /*sizeLevel=*/0,
/*targetMachine=*/nullptr);
if (auto err = optPipeline(llvmModule.get())) {
llvm::errs() << "Failed to optimize LLVM IR " << err << "\n";
throw std::runtime_error("Failed to optimize LLVM IR");
// TODO: Return a proper error
module.emitError("Failed to optimize LLVM IR\n");
return llvm::None;
}
return llvmModule;
auto tsm = llvm::orc::ThreadSafeModule(std::move(llvmModule),
std::move(llvmContext));
return tsm;
};
} // namespace slir

29
src/libserene/source_mgr.cpp
View File

@ -33,10 +33,12 @@
#include <system_error>
#include <llvm/Support/Error.h>
#include <llvm/Support/FormatVariadic.h>
#include <llvm/Support/Locale.h>
#include <llvm/Support/MemoryBufferRef.h>
#include <llvm/Support/Path.h>
#include <llvm/Support/raw_ostream.h>
#include <mlir/Support/LogicalResult.h>
namespace serene {
@ -55,27 +57,30 @@ bool SourceMgr::isValidBufferID(unsigned i) const {
return i != 0 && i <= buffers.size();
};
SourceMgr::ErrorOrMemBufPtr
SourceMgr::findFileInLoadPath(const std::string &name,
std::string &importedFile) {
SourceMgr::MemBufPtr SourceMgr::findFileInLoadPath(const std::string &name,
std::string &importedFile) {
auto path = convertNamespaceToPath(name);
// TODO: Fix this to enqueue a proper error instead
ErrorOrMemBufPtr newBufOrErr(
std::make_error_code(std::errc::no_such_file_or_directory));
// If the file didn't exist directly, see if it's in an include path.
for (unsigned i = 0, e = loadPaths.size(); i != e && !newBufOrErr; ++i) {
for (unsigned i = 0, e = loadPaths.size(); i != e; ++i) {
// TODO: Ugh, Udgly, fix this using llvm::sys::path functions
importedFile = loadPaths[i] + llvm::sys::path::get_separator().data() +
path + "." + DEFAULT_SUFFIX;
SMGR_LOG("Try to load the ns from: " + importedFile);
newBufOrErr = llvm::MemoryBuffer::getFile(importedFile);
auto newBufOrErr = llvm::MemoryBuffer::getFile(importedFile);
if (auto err = newBufOrErr.getError()) {
llvm::consumeError(llvm::errorCodeToError(err));
continue;
}
return std::move(*newBufOrErr);
}
return newBufOrErr;
return nullptr;
};
MaybeNS SourceMgr::readNamespace(SereneContext &ctx, std::string name,
@ -83,16 +88,16 @@ MaybeNS SourceMgr::readNamespace(SereneContext &ctx, std::string name,
std::string importedFile;
SMGR_LOG("Attempt to load namespace: " + name);
ErrorOrMemBufPtr newBufOrErr(findFileInLoadPath(name, importedFile));
MemBufPtr newBufOrErr(findFileInLoadPath(name, importedFile));
if (!newBufOrErr) {
if (newBufOrErr == nullptr) {
auto msg = llvm::formatv("Couldn't find namespace '{0}'", name);
auto err = errors::makeErrorTree(importLoc, errors::NSLoadError,
llvm::StringRef(msg));
return MaybeNS::error(err);
}
auto bufferId = AddNewSourceBuffer(std::move(*newBufOrErr), importLoc);
auto bufferId = AddNewSourceBuffer(std::move(newBufOrErr), importLoc);
UNUSED(nsTable.insert_or_assign(name, bufferId));

6
src/serene-repl/serene-repl.cpp
View File

@ -66,6 +66,10 @@ int main(int argc, char *argv[]) {
// Load history
linenoise::LoadHistory(historyFile.c_str());
// TODO: Read the optimization as an input and as part of the global
// public arguments like -l
ctx->setOperationPhase(CompilationPhase::NoOptimization);
while (true) {
// Read line
std::string line;
@ -85,6 +89,8 @@ int main(int argc, char *argv[]) {
continue;
}
auto x = serene::eval(*ctx, maybeAst.getValue());
serene::print(*ctx, maybeAst.getValue(), result);
llvm::outs() << result << "\n";

255
src/serenec/serenec.cpp
View File

@ -109,112 +109,151 @@ static cl::opt<enum Action> emitAction(
);
int dumpAsObject(Namespace &ns) {
// TODO: Move the compilation process to the Namespace class
auto maybeModule = ns.compileToLLVM();
// TODO: Fix this call to raise the wrapped error instead
if (!maybeModule) {
// TODO: Rais and error: "Faild to generato LLVM IR for namespace"
return -1;
}
// int dumpAsObject(Namespace &ns) {
// // TODO: Move the compilation process to the Namespace class
// auto maybeModule = ns.compileToLLVM();
// // TODO: Fix this call to raise the wrapped error instead
// if (!maybeModule) {
// // TODO: Rais and error: "Faild to generato LLVM IR for namespace"
// return -1;
// }
auto module = std::move(maybeModule.getValue());
auto &ctx = ns.getContext();
// auto module = std::move(maybeModule.getValue());
// auto &ctx = ns.getContext();
// TODO: We need to set the triple data layout and everything to that sort in
// one place. We want them for the JIT as well and also we're kinda
// duplicating what we're doing in `Namespace#compileToLLVM`.
module->setTargetTriple(ctx.targetTriple);
// // TODO: We need to set the triple data layout and everything to that sort
// in
// // one place. We want them for the JIT as well and also we're kinda
// // duplicating what we're doing in `Namespace#compileToLLVM`.
// module->setTargetTriple(ctx.targetTriple);
std::string Error;
const auto *target =
llvm::TargetRegistry::lookupTarget(ctx.targetTriple, Error);
// std::string Error;
// const auto *target =
// llvm::TargetRegistry::lookupTarget(ctx.targetTriple, Error);
// Print an error and exit if we couldn't find the requested target.
// This generally occurs if we've forgotten to initialise the
// TargetRegistry or we have a bogus target triple.
if (target == nullptr) {
llvm::errs() << Error;
return 1;
}
// // Print an error and exit if we couldn't find the requested target.
// // This generally occurs if we've forgotten to initialise the
// // TargetRegistry or we have a bogus target triple.
// if (target == nullptr) {
// llvm::errs() << Error;
// return 1;
// }
const auto *cpu = "generic";
const auto *features = "";
// const auto *cpu = "generic";
// const auto *features = "";
llvm::TargetOptions opt;
auto rm = llvm::Optional<llvm::Reloc::Model>();
auto *targetMachinePtr =
target->createTargetMachine(ctx.targetTriple, cpu, features, opt, rm);
auto targetMachine = std::unique_ptr<llvm::TargetMachine>(targetMachinePtr);
// llvm::TargetOptions opt;
// auto rm = llvm::Optional<llvm::Reloc::Model>();
// auto *targetMachinePtr =
// target->createTargetMachine(ctx.targetTriple, cpu, features, opt, rm);
// auto targetMachine =
// std::unique_ptr<llvm::TargetMachine>(targetMachinePtr);
module->setDataLayout(targetMachine->createDataLayout());
// module->setDataLayout(targetMachine->createDataLayout());
const auto *filename =
strcmp(outputFile.c_str(), "-") == 0 ? "output" : outputFile.c_str();
// const auto *filename =
// strcmp(outputFile.c_str(), "-") == 0 ? "output" : outputFile.c_str();
std::error_code ec;
const auto pathSize(256);
// std::error_code ec;
// const auto pathSize(256);
llvm::SmallString<pathSize> destFile(outputDir);
llvm::sys::path::append(destFile, filename);
auto destObjFilePath = llvm::formatv("{0}.o", destFile).str();
llvm::raw_fd_ostream dest(destObjFilePath, ec, llvm::sys::fs::OF_None);
// llvm::SmallString<pathSize> destFile(outputDir);
// llvm::sys::path::append(destFile, filename);
// auto destObjFilePath = llvm::formatv("{0}.o", destFile).str();
// llvm::raw_fd_ostream dest(destObjFilePath, ec, llvm::sys::fs::OF_None);
if (ec) {
llvm::errs() << "Could not open file: " << destObjFilePath;
llvm::errs() << "Could not open file: " << ec.message();
return 1;
}
// if (ec) {
// llvm::errs() << "Could not open file: " << destObjFilePath;
// llvm::errs() << "Could not open file: " << ec.message();
// return 1;
// }
llvm::legacy::PassManager pass;
auto fileType = llvm::CGFT_ObjectFile;
// llvm::legacy::PassManager pass;
// auto fileType = llvm::CGFT_ObjectFile;
if (targetMachine->addPassesToEmitFile(pass, dest, nullptr, fileType)) {
llvm::errs() << "TheTargetMachine can't emit a file of this type";
return 1;
}
// if (targetMachine->addPassesToEmitFile(pass, dest, nullptr, fileType)) {
// llvm::errs() << "TheTargetMachine can't emit a file of this type";
// return 1;
// }
pass.run(*module);
dest.flush();
// pass.run(*module);
// dest.flush();
if (emitAction == Action::Compile) {
std::vector<const char *> args = {"serenec"};
// if (emitAction == Action::Compile) {
// std::vector<const char *> args = {"serenec"};
args.push_back("--eh-frame-hdr");
args.push_back("-m");
args.push_back("elf_x86_64");
args.push_back("-dynamic-linker");
args.push_back("/lib64/ld-linux-x86-64.so.2");
args.push_back(
"/usr/lib/gcc/x86_64-pc-linux-gnu/11.2.0/../../../../lib64/crt1.o");
args.push_back(
"/usr/lib/gcc/x86_64-pc-linux-gnu/11.2.0/../../../../lib64/crti.o");
args.push_back("/usr/lib/gcc/x86_64-pc-linux-gnu/11.2.0/crtbegin.o");
args.push_back("-L");
args.push_back("/usr/lib/gcc/x86_64-pc-linux-gnu/11.2.0/");
args.push_back("-L");
args.push_back("/usr/lib64/");
// args.push_back("--eh-frame-hdr");
// args.push_back("-m");
// args.push_back("elf_x86_64");
// args.push_back("-dynamic-linker");
// args.push_back("/lib64/ld-linux-x86-64.so.2");
// args.push_back(
// "/usr/lib/gcc/x86_64-pc-linux-gnu/11.2.0/../../../../lib64/crt1.o");
// args.push_back(
// "/usr/lib/gcc/x86_64-pc-linux-gnu/11.2.0/../../../../lib64/crti.o");
// args.push_back("/usr/lib/gcc/x86_64-pc-linux-gnu/11.2.0/crtbegin.o");
// args.push_back("-L");
// args.push_back("/usr/lib/gcc/x86_64-pc-linux-gnu/11.2.0/");
// args.push_back("-L");
// args.push_back("/usr/lib64/");
args.push_back(destObjFilePath.c_str());
args.push_back("-o");
args.push_back(destFile.c_str());
args.push_back("-lgcc");
args.push_back("--as-needed");
args.push_back("-lgcc_s");
args.push_back("--no-as-needed");
args.push_back("-lc");
args.push_back("-lgcc");
args.push_back("--as-needed");
args.push_back("-lgcc_s");
args.push_back("--no-as-needed");
args.push_back("/usr/lib/gcc/x86_64-pc-linux-gnu/11.2.0/crtend.o");
args.push_back(
"/usr/lib/gcc/x86_64-pc-linux-gnu/11.2.0/../../../../lib64/crtn.o");
// args.push_back(destObjFilePath.c_str());
// args.push_back("-o");
// args.push_back(destFile.c_str());
// args.push_back("-lgcc");
// args.push_back("--as-needed");
// args.push_back("-lgcc_s");
// args.push_back("--no-as-needed");
// args.push_back("-lc");
// args.push_back("-lgcc");
// args.push_back("--as-needed");
// args.push_back("-lgcc_s");
// args.push_back("--no-as-needed");
// args.push_back("/usr/lib/gcc/x86_64-pc-linux-gnu/11.2.0/crtend.o");
// args.push_back(
// "/usr/lib/gcc/x86_64-pc-linux-gnu/11.2.0/../../../../lib64/crtn.o");
lld::elf::link(args, false, llvm::outs(), llvm::errs());
}
return 0;
};
// lld::elf::link(args, false, llvm::outs(), llvm::errs());
// // llvm::IntrusiveRefCntPtr<clang::DiagnosticOptions> opts =
// // new clang::DiagnosticOptions;
// // clang::DiagnosticsEngine diags(
// // new clang::DiagnosticIDs, opts,
// // new clang::TextDiagnosticPrinter(llvm::errs(), opts.get()));
// // clang::driver::Driver d("clang", ctx.targetTriple, diags,
// // "Serene compiler");
// // std::vector<const char *> args = {"serenec"};
// // args.push_back(destObjFilePath.c_str());
// // args.push_back("-o");
// // args.push_back(destFile.c_str());
// // d.setCheckInputsExist(true);
// // std::unique_ptr<clang::driver::Compilation> compilation;
// // compilation.reset(d.BuildCompilation(args));
// // if (!compilation) {
// // llvm::errs() << "can't create the compilation!\n";
// // return 1;
// // }
// // llvm::SmallVector<std::pair<int, const clang::driver::Command *>>
// // failCommand;
// // d.ExecuteCompilation(*compilation, failCommand);
// // if (failCommand.empty()) {
// // llvm::outs() << "Done!\n";
// // } else {
// // llvm::errs() << "Linking failed!\n";
// // failCommand.front().second->Print(llvm::errs(), "\n", false);
// // }
// // }
// return 0;
// };
int main(int argc, char *argv[]) {
initCompiler();
@ -324,30 +363,32 @@ int main(int argc, char *argv[]) {
return 1;
}
maybeModule.getValue()->dump();
auto tsm = std::move(maybeModule.getValue());
tsm.withModuleDo([](auto &m) { m.dump(); });
break;
};
case Action::RunJIT: {
auto maybeJIT = JIT::make(*ns);
if (!maybeJIT) {
// TODO: panic in here: "Couldn't creat the JIT!"
return -1;
}
auto jit = std::move(maybeJIT.getValue());
// case Action::RunJIT: {
// auto maybeJIT = JIT::make(*ns);
// if (!maybeJIT) {
// // TODO: panic in here: "Couldn't creat the JIT!"
// return -1;
// }
// auto jit = std::move(maybeJIT.getValue());
if (jit->invoke("main")) {
llvm::errs() << "Faild to invoke the 'main' function.\n";
return 1;
}
llvm::outs() << "Done!";
break;
};
// if (jit->invoke("main")) {
// llvm::errs() << "Faild to invoke the 'main' function.\n";
// return 1;
// }
// llvm::outs() << "Done!";
// break;
// };
case Action::Compile:
case Action::CompileToObject: {
return dumpAsObject(*ns);
};
// case Action::Compile:
// case Action::CompileToObject: {
// return dumpAsObject(*ns);
// };
default: {
llvm::errs() << "Action is not supported yet!\n";
};