diff --git a/config/default.json b/config/default.json index 862b37e..549acb7 100644 --- a/config/default.json +++ b/config/default.json @@ -1,72 +1,73 @@ { "containers": { "id": { "implementations": "impl_fulfill_cluster", "clusters": "var_cluster", "prototypes": "proto_cluster", "linkedlist": "linkedlist" }, "impl": { "solid": "solid", "onthefly": "on_the_fly" } }, "logging": { "id": "logging" }, "function-entry": "entry", "clasp": { "bindings" : { "variable": "bind", "function": "bind_func", "scope": "bind_scope", "function_demand" : "bind_function_demand", "scope_decision": "bind_scope_decision" }, "context" : { "decisions":{ "dependent": "resolution_dependency" } }, "nonevalue": "nonevalue", "ret": { "symbol": "retv", "tag": "ret" } }, "tests": { - "template": "current-fix", + "template": "exploitation", "templates": { "current-fix":"CFA.DomReportOneRoot", "default": "*-Adhoc.*:Containers.*:Compilation.full_IFStatementWithVariantType:Types.full_VariantType_Switch1:Context.full_LateContext:Context.pathDependentContext", + "exploitation": "Exploitation.*", "ast": "AST.*", "adhocs": "Adhoc.*", "effects": "Effects.*", "basic": "Attachments.*", "context": "Context.*", "compilation": "Compilation.*-Compilation.full_IFStatementWithVariantType", "communication": "Communication.*", "cfa": "CFA.*", "containers": "Containers.*", "dfa": "DFA.*", "diagnostic": "Diagnostic.*", "dsl": "Association.*:Interpretation.SwitchVariantAlias", "ExpressionSerializer": "ExpressionSerializer.*", "externc": "InterfaceExternC.*", "loops": "Loop.*", "modules": "Modules.*", "polymorphs": "Polymorphs.call1", "types": "Types.*", "vendorsAPI/clang": "ClangAPI.*", "vendorsAPI/xml2": "libxml2*" } } } diff --git a/cpp/src/analysis/aux.cpp b/cpp/src/analysis/aux.cpp index 109a6aa..27c3f6b 100644 --- a/cpp/src/analysis/aux.cpp +++ b/cpp/src/analysis/aux.cpp @@ -1,142 +1,142 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. * * aux.cpp * * Author: pgess */ /** * \file aux.h * \brief Data representation in ASP format ready for use by reasoner */ #include "aux.h" #include namespace xreate { namespace analysis { using namespace std; list multiplyLists(list> &&lists) { typedef list StringList; assert(lists.size()); StringList result(*lists.begin()); lists.pop_front(); boost::format concat("%s, %s"); for (StringList &list: lists) { StringList::const_iterator end = result.end(); for (StringList::iterator expr1I = result.begin(); expr1I != end; ++expr1I) { if (list.size() == 0) continue; StringList::const_iterator expr2I = list.begin(); for (int expr2No = 0, size = list.size() - 1; expr2No < size; ++expr2No, ++expr1I) result.push_back(str(concat %(*expr1I) %(*expr2I))); *expr1I = str(concat %(*expr1I) %(*expr2I)); } } return result; } std::list compile(const Expression &e){ list result; switch (e.op) { case Operator::CALL: { assert(e.__state == Expression::COMPOUND); if(!e.operands.size()){ result.push_back(e.getValueString()); break; } std::list> operands; std::transform(e.operands.begin(), e.operands.end(), std::inserter(operands, operands.begin()), [](const Expression &e) { return compile(e); }); list &&operands_ = multiplyLists(std::move(operands)); result.push_back(boost::str(boost::format("%1%(%2%)") % (e.getValueString()) % (boost::algorithm::join(operands_, ", ")))); break; } case Operator::NEG: { assert(e.operands.size() == 1); const Expression &op = e.operands.at(0); list &&rawOp = compile(op); assert(rawOp.size() == 1); result.push_back((boost::format("not %1%")%(rawOp.front())).str()); break; }; - case Operator::NONE: { + case Operator::INVALID: { switch (e.__state) { case Expression::IDENT: result.push_back(e.getValueString()); break; case Expression::NUMBER: result.push_back(to_string(e.getValueDouble())); break; default: assert(true); } break; } default: break; } //TODO Null ad hoc ClaspLayer implementation // if (e.isNone()){ // result.push_back(e.__valueS); // } assert(result.size()); return result; } std::list compileNeg(const Expression &e){ list result; switch (e.op) { case Operator::IMPL: { assert(e.__state == Expression::COMPOUND); assert(e.operands.size() == 2); list operands1 = compile(e.operands.at(0)); list operands2 = compile(e.operands.at(1)); boost::format formatNeg("%1%, not %2%"); for (const auto &op1: operands1) for (const auto &op2: operands2) { result.push_back(boost::str(formatNeg %(op1) % (op2))); } break; } case Operator::NEG: { assert(e.operands.size() == 1); const Expression &op = e.operands.at(0); list &&rawOp = compile(op); assert(rawOp.size() == 1); result.push_back(rawOp.front()); break; }; default: assert(true); } return result; } }} diff --git a/cpp/src/analysis/typeinference.cpp b/cpp/src/analysis/typeinference.cpp index 17d2a1f..2b48e10 100644 --- a/cpp/src/analysis/typeinference.cpp +++ b/cpp/src/analysis/typeinference.cpp @@ -1,72 +1,75 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. * * typeinference.cpp * * Author: pgess * Created on April 16, 2017, 10:13 AM */ /** * \file typeinference.h * \brief Type inference analysis */ #include "typeinference.h" #include "llvmlayer.h" #include "llvm/IR/Function.h" #include "llvm/IR/DerivedTypes.h" namespace xreate {namespace typeinference { //TODO type conversion: //a) automatically expand types int -> bigger int; int -> floating //b) detect exact type of `num` based on max used numeral / function type //c) warning if need to truncate (allow/dissalow based on annotations) llvm::Value* doAutomaticTypeConversion(llvm::Value* source, llvm::Type* tyTarget, llvm::IRBuilder<>& builder){ if (tyTarget->isIntegerTy() && source->getType()->isIntegerTy()) { llvm::IntegerType* tyTargetInt = llvm::dyn_cast(tyTarget); llvm::IntegerType* tySourceInt = llvm::dyn_cast(source->getType()); if (tyTargetInt->getBitWidth() < tySourceInt->getBitWidth()){ return builder.CreateCast(llvm::Instruction::Trunc, source, tyTarget); } if (tyTargetInt->getBitWidth() > tySourceInt->getBitWidth()){ return builder.CreateCast(llvm::Instruction::SExt, source, tyTarget); } } if (source->getType()->isIntegerTy() && tyTarget->isFloatingPointTy()){ return builder.CreateCast(llvm::Instruction::SIToFP, source, tyTarget); } return source; } ExpandedType getType(const Expression& expression, const AST& ast){ if (expression.type.isValid()){ return ast.expandType(expression.type); } if (expression.__state == Expression::IDENT){ Symbol s = Attachments::get(expression); return getType(CodeScope::getDefinition(s), ast); } if (Attachments::exists(expression)){ return Attachments::get(expression); } - assert(false && "Type can't be determined for an expression"); -} + if(expression.__state==Expression::NUMBER){ + return ExpandedType (TypeAnnotation(TypePrimitive::I32)); + } + assert(false && "Type can't be determined for an expression"); +} } } //end of namespace xreate::typeinference diff --git a/cpp/src/ast.cpp b/cpp/src/ast.cpp index 9a5276a..91a28c7 100644 --- a/cpp/src/ast.cpp +++ b/cpp/src/ast.cpp @@ -1,977 +1,977 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. * * Author: pgess * File: ast.cpp */ /** * \file ast.h * \brief Syntax Tree and related code * * \sa xreate::AST */ #include "ast.h" #include "ExternLayer.h" #include "analysis/typeinference.h" #include #include //TODO BDecl. forbid multiple body declaration (ExprTyped) namespace std { std::size_t hash::operator()(xreate::ScopedSymbol const& s) const { return s.id ^ (s.version << 2); } bool equal_to::operator()(const xreate::ScopedSymbol& __x, const xreate::ScopedSymbol& __y) const { return __x.id == __y.id && __x.version == __y.version; } size_t hash::operator()(xreate::Symbol const& s) const { return hash()(s.identifier) ^ ((long int) s.scope << 1); } bool equal_to::operator()(const xreate::Symbol& __x, const xreate::Symbol& __y) const { return __x == __y; }; } using namespace std; namespace xreate { Atom::Atom(const std::wstring& value) { __value = wstring_to_utf8(value); } Atom::Atom(std::string && name) : __value(name) { } const std::string& Atom::get() const { return __value; } Atom::Atom(wchar_t* value) { //DEBT reconsider number literal recognition __value = wcstol(value, 0, 10); } Atom::Atom(int value) : __value(value) { } double Atom::get()const { return __value; } Atom::Atom(const std::wstring& value) { assert(value.size() >= 2); __value = wstring_to_utf8(value.substr(1, value.size() - 2)); } Atom::Atom(std::string && name) : __value(name) {} const std::string& Atom::get() const { return __value; } class ExpressionHints { public: static bool isStringValueValid(const Expression& e) { switch (e.__state) { case Expression::INVALID: assert(false); case Expression::IDENT: case Expression::STRING: return true; case Expression::NUMBER: case Expression::BINDING: return false; case Expression::COMPOUND: { switch (e.op) { case Operator::CALL: return true; default: return false; } } } return false; } static bool isDoubleValueValid(const Expression& e) { switch (e.__state) { case Expression::NUMBER: return true; case Expression::INVALID: assert(false); case Expression::IDENT: case Expression::STRING: case Expression::BINDING: return false; case Expression::COMPOUND: { switch (e.op) { case Operator::VARIANT: return true; default: return false; } } } return false; } }; class TypesResolver { private: const AST* ast; std::map scope; std::map signatures; ExpandedType expandType(const TypeAnnotation &t, const std::vector &args = std::vector()) { return TypesResolver(ast, scope, signatures)(t, args); } std::vector expandOperands(const std::vector& operands) { std::vector pack; pack.reserve(operands.size()); std::transform(operands.begin(), operands.end(), std::inserter(pack, pack.end()), [this](const TypeAnnotation & t) { return expandType(t); }); return pack; } public: TypesResolver(const AST* root, const std::map& scopeOuter = std::map(), std::map signaturesOuter = std::map()) : ast(root), scope(scopeOuter), signatures(signaturesOuter) { } ExpandedType operator()(const TypeAnnotation &t, const std::vector &args = std::vector()) { //assert(args.size() == t.bindings.size()); // invalid number of arguments for (size_t i = 0; i < args.size(); ++i) { scope[t.bindings.at(i)] = args.at(i); } switch (t.__operator) { case TypeOperator::LIST: { assert(t.__operands.size() == 1); Expanded elTy = expandType(t.__operands.at(0)); return ExpandedType(TypeAnnotation(tag_array, elTy, 0)); } case TypeOperator::LIST_NAMED: { std::vector&& packOperands = expandOperands(t.__operands); auto typNew = TypeAnnotation(TypeOperator::LIST_NAMED, move(packOperands)); typNew.fields = t.fields; return ExpandedType(move(typNew)); }; case TypeOperator::VARIANT: { std::vector&& packOperands = expandOperands(t.__operands); auto typNew = TypeAnnotation(TypeOperator::VARIANT, move(packOperands)); typNew.fields = t.fields; return ExpandedType(move(typNew)); }; case TypeOperator::CALL: { std::string alias = t.__valueCustom; //find in local scope: TypeAnnotation ty; if (scope.count(alias)) { ty = scope.at(alias); } else if (ast->__indexTypeAliases.count(alias)) { ty = ast->__indexTypeAliases.at(alias); } else { assert(false && "Undefined or external type"); } std::vector&& operands = expandOperands(t.__operands); TypeAnnotation signature(TypeOperator::CALL, move(operands)); signature.__valueCustom = alias; if (signatures.count(signature)) { auto link = TypeAnnotation(TypeOperator::LINK,{}); link.conjuctionId = signatures.at(signature); return ExpandedType(move(link)); } int cid = signatures.size(); signatures[signature] = cid; TypeAnnotation tyResult = expandType(ty, operands); tyResult.conjuctionId = cid; return ExpandedType(move(tyResult)); }; case TypeOperator::CUSTOM: { if (signatures.count(t)) { return ExpandedType(TypeAnnotation(TypeOperator::LINK, {t})); } signatures.emplace(t, signatures.size()); std::string alias = t.__valueCustom; //find in local scope: if (scope.count(alias)) { return expandType(scope.at(alias)); } // find in general scope: if (ast->__indexTypeAliases.count(alias)) { return expandType(ast->__indexTypeAliases.at(t.__valueCustom)); } //if type is unknown keep it as is. return ExpandedType(TypeAnnotation(t)); }; case TypeOperator::ACCESS: { std::string alias = t.__valueCustom; ExpandedType tyAlias = ExpandedType(TypeAnnotation()); //find in local scope: if (scope.count(alias)) { tyAlias = expandType(scope.at(alias)); //find in global scope: } else if ((ast->__indexTypeAliases.count(alias))) { tyAlias = expandType(ast->__indexTypeAliases.at(alias)); } else { assert(false && "Undefined or external type"); } assert(tyAlias->__operator == TypeOperator::LIST_NAMED); for (const string& field : t.fields) { auto fieldIt = std::find(tyAlias->fields.begin(), tyAlias->fields.end(), field); assert(fieldIt != tyAlias->fields.end() && "unknown field"); int fieldId = fieldIt - tyAlias->fields.begin(); tyAlias = expandType(tyAlias->__operands.at(fieldId)); } return tyAlias; } case TypeOperator::NONE: { return ExpandedType(TypeAnnotation(t)); } default: assert(false); } assert(false); return ExpandedType(TypeAnnotation()); } }; TypeAnnotation::TypeAnnotation() : __operator(TypeOperator::NONE), __value(TypePrimitive::Invalid) { } TypeAnnotation::TypeAnnotation(TypePrimitive typ) : __value(typ) { } TypeAnnotation::TypeAnnotation(TypeOperator op, std::initializer_list operands) : __operator(op), __operands(operands) { } TypeAnnotation::TypeAnnotation(TypeOperator op, std::vector&& operands) : __operator(op), __operands(operands) { } TypeAnnotation::TypeAnnotation(llvm_array_tag, TypeAnnotation typ, int size) : TypeAnnotation(TypeOperator::LIST,{typ}) { __size = size; } bool TypeAnnotation::isValid() const { return !(__value == TypePrimitive::Invalid && __operator == TypeOperator::NONE); } bool TypeAnnotation::operator<(const TypeAnnotation& t) const { if (__operator != t.__operator) return __operator < t.__operator; if (__operator == TypeOperator::NONE) return __value < t.__value; if (__operator == TypeOperator::CALL || __operator == TypeOperator::CUSTOM || __operator == TypeOperator::ACCESS) { if (__valueCustom != t.__valueCustom) return __valueCustom < t.__valueCustom; } return __operands < t.__operands; } /* TypeAnnotation (struct_tag, std::initializer_list) {} */ void TypeAnnotation::addBindings(std::vector>&& params) { bindings.reserve(bindings.size() + params.size()); std::transform(params.begin(), params.end(), std::inserter(bindings, bindings.end()), [](const Atom& ident) { return ident.get(); }); } void TypeAnnotation::addFields(std::vector>&& listFields) { fields.reserve(fields.size() + listFields.size()); std::transform(listFields.begin(), listFields.end(), std::inserter(fields, fields.end()), [](const Atom& ident) { return ident.get(); }); } unsigned int Expression::nextVacantId = 0; Expression::Expression(const Atom& number) : Expression() { __state = NUMBER; - op = Operator::NONE; + op = Operator::INVALID; __valueD = number.get(); } Expression::Expression(const Atom& a) : Expression() { __state = STRING; - op = Operator::NONE; + op = Operator::INVALID; __valueS = a.get(); } Expression::Expression(const Atom &ident) : Expression() { __state = IDENT; - op = Operator::NONE; + op = Operator::INVALID; __valueS = ident.get(); } Expression::Expression(const Operator &oprt, std::initializer_list params) : Expression() { __state = COMPOUND; op = oprt; if (op == Operator::CALL) { assert(params.size() > 0); Expression arg = *params.begin(); assert(arg.__state == Expression::IDENT); __valueS = std::move(arg.__valueS); operands.insert(operands.end(), params.begin() + 1, params.end()); return; } operands.insert(operands.end(), params.begin(), params.end()); } void Expression::setOp(Operator oprt) { op = oprt; switch (op) { - case Operator::NONE: + case Operator::INVALID: __state = INVALID; break; default: __state = COMPOUND; break; } } void Expression::addArg(Expression &&arg) { operands.push_back(arg); } void Expression::addTags(const std::list tags) const { std::transform(tags.begin(), tags.end(), std::inserter(this->tags, this->tags.end()), [](const Expression & tag) { return make_pair(tag.getValueString(), tag); }); } void Expression::addBindings(std::initializer_list> params) { addBindings(params.begin(), params.end()); } void Expression::bindType(TypeAnnotation t) { type = move(t); } void Expression::addBlock(ManagedScpPtr scope) { blocks.push_back(scope.operator->()); } const std::vector& Expression::getOperands() const { return operands; } double Expression::getValueDouble() const { return __valueD; } const std::string& Expression::getValueString() const { return __valueS; } void Expression::setValue(const Atom&& v) { __valueS = v.get(); } void Expression::setValueDouble(double value) { __valueD = value; } bool Expression::isValid() const { return (__state != INVALID); } bool Expression::isDefined() const { return (__state != BINDING && __state != INVALID); } Expression::Expression() -: __state(INVALID), op(Operator::NONE), id(nextVacantId++) { +: __state(INVALID), op(Operator::INVALID), id(nextVacantId++) { } namespace details { namespace inconsistent { AST::AST() { Attachments::init(); Attachments::init(); Attachments::init(); Attachments::init(); } void AST::addInterfaceData(const ASTInterface& interface, Expression&& data) { __interfacesData.emplace(interface, move(data)); } void AST::addDFAData(Expression &&data) { __dfadata.push_back(data); } void AST::addExternData(ExternData &&data) { __externdata.insert(__externdata.end(), data.entries.begin(), data.entries.end()); } void AST::add(Function* f) { __functions.push_back(f); __indexFunctions.emplace(f->getName(), __functions.size() - 1); } void AST::add(MetaRuleAbstract *r) { __rules.push_back(r); } void AST::add(TypeAnnotation t, Atom alias) { if (t.__operator == TypeOperator::VARIANT) { for (int i = 0, size = t.fields.size(); i < size; ++i) { __dictVariants.emplace(t.fields[i], make_pair(t, i)); } } __indexTypeAliases.emplace(alias.get(), move(t)); } ManagedScpPtr AST::add(CodeScope* scope) { this->__scopes.push_back(scope); return ManagedScpPtr(this->__scopes.size() - 1, &this->__scopes); } std::string AST::getModuleName() { const std::string name = "moduleTest"; return name; } ManagedPtr AST::findFunction(const std::string& name) { int count = __indexFunctions.count(name); if (!count) { return ManagedFnPtr::Invalid(); } assert(count == 1); auto range = __indexFunctions.equal_range(name); return ManagedPtr(range.first->second, &this->__functions); } std::list AST::getAllFunctions() const { const size_t size = __functions.size(); std::list result; for (size_t i = 0; i < size; ++i) { result.push_back(ManagedFnPtr(i, &this->__functions)); } return result; } //TASK select default specializations std::list AST::getFunctionSpecializations(const std::string& fnName) const { auto functions = __indexFunctions.equal_range(fnName); std::list result; std::transform(functions.first, functions.second, inserter(result, result.end()), [this](auto f) { return ManagedFnPtr(f.second, &this->__functions); }); return result; } template<> ManagedPtr AST::begin() { return ManagedPtr(0, &this->__functions); } template<> ManagedPtr AST::begin() { return ManagedPtr(0, &this->__scopes); } template<> ManagedPtr AST::begin() { return ManagedPtr(0, &this->__rules); } void AST::recognizeVariantConstructor(Expression& function) { assert(function.op == Operator::CALL); std::string variant = function.getValueString(); if (!__dictVariants.count(variant)) { return; } auto record = __dictVariants.at(variant); const TypeAnnotation& typ = record.first; function.op = Operator::VARIANT; function.setValueDouble(record.second); function.type = typ; } Atom AST::recognizeVariantConstructor(Atom ident) { std::string variant = ident.get(); assert(__dictVariants.count(variant) && "Can't recognize variant constructor"); auto record = __dictVariants.at(variant); return Atom(record.second); } void AST::postponeIdentifier(CodeScope* scope, const Expression& id) { bucketUnrecognizedIdentifiers.emplace(scope, id); } void AST::recognizePostponedIdentifiers() { for (const auto& identifier : bucketUnrecognizedIdentifiers) { if (!identifier.first->recognizeIdentifier(identifier.second)) { //exception: Ident not found std::cout << "Unknown symbol: " << identifier.second.getValueString() << std::endl; assert(false && "Symbol not found"); } } } xreate::AST* AST::finalize() { //all finalization steps: recognizePostponedIdentifiers(); return reinterpret_cast (this); } } } //namespace details::incomplete Expanded AST::findType(const std::string& name) { // find in general scope: if (__indexTypeAliases.count(name)) return expandType(__indexTypeAliases.at(name)); //if type is unknown keep it as is. TypeAnnotation t(TypeOperator::CUSTOM,{}); t.__valueCustom = name; return ExpandedType(move(t)); } Expanded AST::expandType(const TypeAnnotation &t) const { return TypesResolver(this)(t); } ExpandedType AST::getType(const Expression& expression) { return typeinference::getType(expression, *this); } Function::Function(const Atom& name) : __entry(new CodeScope(0)) { __name = name.get(); } void Function::addTag(Expression&& tag, const TagModifier mod) { string name = tag.getValueString(); __tags.emplace(move(name), move(tag)); } const std::map& Function::getTags() const { return __tags; } CodeScope* Function::getEntryScope() const { return __entry; } void Function::addBinding(Atom && name, Expression&& argument) { __entry->addBinding(move(name), move(argument)); } const std::string& Function::getName() const { return __name; } ScopedSymbol CodeScope::registerIdentifier(const Expression& identifier) { versions::VariableVersion version = Attachments::get(identifier, versions::VERSION_NONE); auto result = __identifiers.emplace(identifier.getValueString(), __vCounter); if (result.second) { ++__vCounter; return { __vCounter - 1, version }; } return { result.first->second, version }; } bool CodeScope::recognizeIdentifier(const Expression& identifier) const { versions::VariableVersion version = Attachments::get(identifier, versions::VERSION_NONE); const std::string& name = identifier.getValueString(); //search identifier in the current block if (__identifiers.count(name)) { VNameId id = __identifiers.at(name); Symbol s; s.identifier = ScopedSymbol{id, version}; s.scope = const_cast (this); Attachments::put(identifier, s); return true; } //search in the parent scope if (__parent) { return __parent->recognizeIdentifier(identifier); } return false; } ScopedSymbol CodeScope::getSymbol(const std::string& alias) { assert(__identifiers.count(alias)); VNameId id = __identifiers.at(alias); return {id, versions::VERSION_NONE }; } void CodeScope::addBinding(Expression&& var, Expression&& argument) { argument.__state = Expression::BINDING; __bindings.push_back(var.getValueString()); ScopedSymbol binding = registerIdentifier(var); __declarations[binding] = move(argument); } Symbol CodeScope::addDefinition(Expression&& var, Expression&& body) { ScopedSymbol s = registerIdentifier(var); __declarations[s] = move(body); return Symbol{s, this}; } CodeScope::CodeScope(CodeScope* parent) : __parent(parent) { } CodeScope::~CodeScope() { } void CodeScope::setBody(const Expression &body) { assert(__declarations.count(ScopedSymbol::RetSymbol)==0 && "Attempt to reassign scope body"); __declarations[ScopedSymbol::RetSymbol] = body; } const Expression& CodeScope::getBody() const{ return __declarations.at(ScopedSymbol::RetSymbol); } const Expression& CodeScope::getDefinition(const Symbol& symbol, bool flagAllowUndefined){ const CodeScope* self = symbol.scope; return self->getDefinition(symbol.identifier, flagAllowUndefined); } const Expression& CodeScope::getDefinition(const ScopedSymbol& symbol, bool flagAllowUndefined) const{ static Expression expressionInvalid; if (!__declarations.count(symbol)){ if (flagAllowUndefined) return expressionInvalid; assert(false && "Symbol's declaration not found"); } return __declarations.at(symbol); } void RuleArguments::add(const Atom &arg, DomainAnnotation typ) { emplace_back(arg.get(), typ); } void RuleGuards::add(Expression&& e) { push_back(e); } MetaRuleAbstract:: MetaRuleAbstract(RuleArguments&& args, RuleGuards&& guards) : __args(std::move(args)), __guards(std::move(guards)) { } MetaRuleAbstract::~MetaRuleAbstract() { } RuleWarning:: RuleWarning(RuleArguments&& args, RuleGuards&& guards, Expression&& condition, Atom&& message) : MetaRuleAbstract(std::move(args), std::move(guards)), __message(message.get()), __condition(condition) { } RuleWarning::~RuleWarning() { } void RuleWarning::compile(ClaspLayer& layer) { //TODO restore addRuleWarning //layer.addRuleWarning(*this); } bool operator<(const ScopedSymbol& s1, const ScopedSymbol& s2) { return (s1.id < s2.id) || (s1.id == s2.id && s1.version < s2.version); } bool operator==(const ScopedSymbol& s1, const ScopedSymbol& s2) { return (s1.id == s2.id) && (s1.version == s2.version); } bool operator<(const Symbol& s1, const Symbol& s2) { return (s1.scope < s2.scope) || (s1.scope == s2.scope && s1.identifier < s2.identifier); } bool operator==(const Symbol& s1, const Symbol& s2) { return (s1.scope == s2.scope) && (s1.identifier == s2.identifier); } bool operator<(const Expression&a, const Expression&b) { if (a.__state != b.__state) return a.__state < b.__state; assert(a.__state != Expression::INVALID); switch (a.__state) { case Expression::IDENT: case Expression::STRING: return a.getValueString() < b.getValueString(); case Expression::NUMBER: return a.getValueDouble() < b.getValueDouble(); case Expression::COMPOUND: { assert(a.blocks.size() == 0); assert(b.blocks.size() == 0); if (a.op != b.op) { return a.op < b.op; } bool flagAValid = ExpressionHints::isStringValueValid(a); bool flagBValid = ExpressionHints::isStringValueValid(b); if (flagAValid != flagBValid) { return flagAValid < flagBValid; } if (flagAValid) { if (a.getValueString() != b.getValueString()) { return a.getValueString() < b.getValueString(); } } flagAValid = ExpressionHints::isDoubleValueValid(a); flagBValid = ExpressionHints::isDoubleValueValid(b); if (flagAValid != flagBValid) { return flagAValid < flagBValid; } if (flagAValid) { if (a.getValueDouble() != b.getValueDouble()) { return a.getValueDouble() < b.getValueDouble(); } } if (a.operands.size() != b.operands.size()) { return (a.operands.size() < b.operands.size()); } for (size_t i = 0; i < a.operands.size(); ++i) { bool result = a.operands[i] < b.operands[i]; if (result) return true; } return false; } case Expression::BINDING: case Expression::INVALID: assert(false); } return false; } bool Expression::operator==(const Expression& other) const { if (this->__state != other.__state) return false; if (ExpressionHints::isStringValueValid(*this)) { if (this->__valueS != other.__valueS) return false; } if (ExpressionHints::isDoubleValueValid(*this)) { if (this->__valueD != other.__valueD) return false; } if (this->__state != Expression::COMPOUND) { return true; } if (this->op != other.op) { return false; } if (this->operands.size() != other.operands.size()) { return false; } for (size_t i = 0; ioperands.size(); ++i) { if (!(this->operands[i] == other.operands[i])) return false; } assert(!this->blocks.size()); assert(!other.blocks.size()); return true; } const ScopedSymbol ScopedSymbol::RetSymbol = ScopedSymbol{0, versions::VERSION_NONE}; } //end of namespace xreate diff --git a/cpp/src/ast.h b/cpp/src/ast.h index 5e19dda..cdb0515 100644 --- a/cpp/src/ast.h +++ b/cpp/src/ast.h @@ -1,734 +1,734 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. * * Author: pgess * File: ast.h */ #ifndef AST_H #define AST_H #include "attachments.h" #include #include #include #include #include #include #include #include "utils.h" #include namespace llvm { class Value; } namespace xreate { struct ScopedSymbol; struct Symbol; } namespace std { template<> struct hash { std::size_t operator()(xreate::ScopedSymbol const& s) const; }; template<> struct equal_to { bool operator()(const xreate::ScopedSymbol& __x, const xreate::ScopedSymbol& __y) const; }; template<> struct hash { size_t operator()(xreate::Symbol const& s) const; }; template<> struct equal_to { bool operator()(const xreate::Symbol& __x, const xreate::Symbol& __y) const; }; } namespace xreate { struct String_t { }; struct Identifier_t { }; struct Number_t { }; struct Type_t { }; template class Atom { }; //DEBT store line:col for all atoms/identifiers template<> class Atom { public: Atom(const std::wstring& value); Atom(std::string && name); const std::string& get() const; private: std::string __value; }; template<> class Atom { public: Atom(wchar_t* value); Atom(int value); double get()const; private: double __value; }; template<> class Atom { public: Atom(const std::wstring& value); Atom(std::string && name); const std::string& get() const; private: std::string __value; }; enum class TypePrimitive { Invalid, Bool, I8, I32, I64, Num, Int, Float, String }; enum class TypeOperator { NONE, CALL, CUSTOM, VARIANT, LIST, LIST_NAMED, ACCESS, LINK }; struct llvm_array_tag { }; struct struct_tag { }; const llvm_array_tag tag_array = llvm_array_tag(); const struct_tag tag_struct = struct_tag(); /** * \brief Represents type to support type system * * This class represents type in denormalized form, i.e. without arguments and aliases substitution * \sa AST::expandType() */ class TypeAnnotation { public: TypeAnnotation(); TypeAnnotation(const Atom& typ); TypeAnnotation(TypePrimitive typ); TypeAnnotation(llvm_array_tag, TypeAnnotation typ, int size); TypeAnnotation(TypeOperator op, std::initializer_list operands); TypeAnnotation(TypeOperator op, std::vector&& operands); void addBindings(std::vector>&& params); void addFields(std::vector>&& listFields); bool operator<(const TypeAnnotation& t) const; // TypeAnnotation (struct_tag, std::initializer_list); bool isValid() const; TypeOperator __operator = TypeOperator::NONE; std::vector __operands; TypePrimitive __value; std::string __valueCustom; int conjuctionId = -1; //conjunction point id (relevant for recursive types) uint64_t __size = 0; std::vector fields; std::vector bindings; private: }; enum class Operator { - ADD, SUB, MUL, DIV, + INVALID, UNDEF, ADD, SUB, MUL, DIV, EQU, NE, NEG, LSS, LSE, GTR, GTE, LIST, LIST_RANGE, LIST_NAMED, - CALL, CALL_INTRINSIC, NONE, + CALL, CALL_INTRINSIC, IMPL/* implication */, MAP, FOLD, FOLD_INF, LOOP_CONTEXT, INDEX, IF, SWITCH, SWITCH_ADHOC, SWITCH_VARIANT, CASE, CASE_DEFAULT, LOGIC_AND, ADHOC, CONTEXT_RULE, VARIANT, SEQUENCE }; class Function; class AST; class CodeScope; class MetaRuleAbstract; typedef ManagedPtr ManagedFnPtr; typedef ManagedPtr ManagedScpPtr; typedef ManagedPtr ManagedRulePtr; const ManagedScpPtr NO_SCOPE = ManagedScpPtr(UINT_MAX, 0); /** * \brief Represents every instruction in Xreate's syntax tree * \attention In case of any changes update xreate::ExpressionHints auxiliary helper as well * * Expression is generic building block of syntax tree able to hold node data * as well as child nodes as operands. Not only instructions use expression for representation in syntax tree * but annotation as well. * * Additionally, `types` as a special kind of annotations use Expression-like data structure TypeAnnotation * \sa xreate::AST, xreate::TypeAnnotation */ // struct Expression { friend class CodeScope; friend class ClaspLayer; friend class CFAPass; friend class ExpressionHints; Expression(const Operator &oprt, std::initializer_list params); Expression(const Atom& ident); Expression(const Atom& number); Expression(const Atom& a); Expression(); void setOp(Operator oprt); void addArg(Expression&& arg); void addBindings(std::initializer_list> params); void bindType(TypeAnnotation t); template void addBindings(InputIt paramsBegin, InputIt paramsEnd); void addTags(const std::list tags) const; void addBlock(ManagedScpPtr scope); const std::vector& getOperands() const; double getValueDouble() const; void setValueDouble(double value); const std::string& getValueString() const; void setValue(const Atom&& v); bool isValid() const; bool isDefined() const; bool operator==(const Expression& other) const; /** * \brief is it string, number, compound operation and so on */ enum { INVALID, COMPOUND, IDENT, NUMBER, STRING, BINDING } __state = INVALID; /** * \brief Valid for compound State. Holds type of compound operator */ Operator op; /** * \brief Unique id to identify expression within syntax tree */ unsigned int id; /** * \brief Exact meaning depends on particular instruction * \details As an example, named lists/structs hold field names in bindings */ std::vector bindings; std::map __indexBindings; /** * \brief Holds child instructions as arguments */ std::vector operands; /** * \brief Holds type of instruction's result */ TypeAnnotation type; /** * \brief Holds additional annotations */ mutable std::map tags; /** * \brief Child code blocks * \details For example, If statement holds TRUE-branch as first and FALSE-branch as second block here */ std::list blocks; private: std::string __valueS; double __valueD; static unsigned int nextVacantId; }; bool operator<(const Expression&, const Expression&); template void Expression::addBindings(InputIt paramsBegin, InputIt paramsEnd) { size_t index = bindings.size(); std::transform(paramsBegin, paramsEnd, std::inserter(bindings, bindings.end()), [&index, this] (const Atom atom) { std::string key = atom.get(); this->__indexBindings[key] = index++; return key; }); } typedef std::list ExpressionList; enum class TagModifier { NONE, ASSERT, REQUIRE }; enum class DomainAnnotation { FUNCTION, VARIABLE }; class RuleArguments : public std::vector> { public: void add(const Atom& name, DomainAnnotation typ); }; class RuleGuards : public std::vector { public: void add(Expression&& e); }; class ClaspLayer; class LLVMLayer; class MetaRuleAbstract { public: MetaRuleAbstract(RuleArguments&& args, RuleGuards&& guards); virtual ~MetaRuleAbstract(); virtual void compile(ClaspLayer& layer) = 0; protected: RuleArguments __args; RuleGuards __guards; }; class RuleWarning : public MetaRuleAbstract { friend class ClaspLayer; public: RuleWarning(RuleArguments&& args, RuleGuards&& guards, Expression&& condition, Atom&& message); virtual void compile(ClaspLayer& layer); ~RuleWarning(); private: std::string __message; Expression __condition; }; typedef unsigned int VNameId; namespace versions { typedef int VariableVersion; const VariableVersion VERSION_NONE = -2; const VariableVersion VERSION_INIT = 0; } template<> struct AttachmentsDict { typedef versions::VariableVersion Data; static const unsigned int key = 6; }; struct ScopedSymbol { VNameId id; versions::VariableVersion version; static const ScopedSymbol RetSymbol; }; struct Symbol { ScopedSymbol identifier; const CodeScope * scope; }; struct IdentifierSymbol{}; struct SymbolAlias{}; template<> struct AttachmentsDict { typedef Symbol Data; static const unsigned int key = 7; }; template<> struct AttachmentsDict { typedef Symbol Data; static const unsigned int key = 9; }; typedef std::pair Tag; bool operator<(const ScopedSymbol& s1, const ScopedSymbol& s2); bool operator==(const ScopedSymbol& s1, const ScopedSymbol& s2); bool operator<(const Symbol& s1, const Symbol& s2); bool operator==(const Symbol& s1, const Symbol& s2); /** * \brief Represents code block and single scope of visibility * * Holds single expression as a *body* and set of variable assignments(declarations) used in body's expression * \sa xreate::AST */ class CodeScope { friend class Function; friend class PassManager; public: CodeScope(CodeScope* parent = 0); ~CodeScope(); /** \brief Set expression as a body */ void setBody(const Expression& body); /** \brief Returns current code scope body */ const Expression& getBody() const; /** \brief Adds variable definition to be used in body as well as in other declarations */ Symbol addDefinition(Expression&& var, Expression&& body); /** \brief Returns symbols' definition */ static const Expression& getDefinition(const Symbol& symbol, bool flagAllowUndefined = false); const Expression& getDefinition(const ScopedSymbol& symbol, bool flagAllowUndefined = false) const; /** \brief Adds variable defined elsewhere */ void addBinding(Expression&& var, Expression&& argument); std::vector __bindings; std::map __identifiers; CodeScope* __parent; //TODO move __definitions to SymbolsAttachments data //NOTE: definition of return type has index 0 std::unordered_map __declarations; std::vector tags; std::vector contextRules; private: VNameId __vCounter = 1; ScopedSymbol registerIdentifier(const Expression& identifier); public: bool recognizeIdentifier(const Expression& identifier) const; ScopedSymbol getSymbol(const std::string& alias); }; /** * \brief Represents single function in Xreate's syntax tree * * Holds an entry code scope and `guardContext` required for function to operate * \sa xreate::AST */ class Function { friend class Expression; friend class CodeScope; friend class AST; public: Function(const Atom& name); /** * \brief Adds function arguments */ void addBinding(Atom && name, Expression&& argument); /** * \brief Adds additional function annotations */ void addTag(Expression&& tag, const TagModifier mod); const std::string& getName() const; const std::map& getTags() const; CodeScope* getEntryScope() const; CodeScope* __entry; std::string __name; bool isPrefunction = false; //SECTIONTAG adhoc Function::isPrefunction flag Expression guardContext; Expression guard; private: std::map __tags; }; class ExternData; struct ExternEntry { std::string package; std::vector headers; }; typedef Expanded ExpandedType; struct TypeInferred{}; template<> struct AttachmentsDict { typedef ExpandedType Data; static const unsigned int key = 11; }; enum ASTInterface { CFA, DFA, Extern, Adhoc }; struct FunctionSpecialization { std::string guard; size_t id; }; struct FunctionSpecializationQuery { std::unordered_set context; }; template<> struct AttachmentsId{ static unsigned int getId(const Expression& expression){ return expression.id; } }; template<> struct AttachmentsId{ static unsigned int getId(const Symbol& s){ return s.scope->__declarations.at(s.identifier).id; } }; template<> struct AttachmentsId{ static unsigned int getId(const ManagedFnPtr& f){ const Symbol symbolFunction{ScopedSymbol::RetSymbol, f->getEntryScope()}; return AttachmentsId::getId(symbolFunction); } }; template<> struct AttachmentsId{ static unsigned int getId(const unsigned int id){ return id; } }; class TypesResolver; namespace details { namespace inconsistent { /** * \brief Syntax tree under construction in inconsistent form * * Represents Syntax Tree under construction(**inconsistent state**). * \attention Clients should use rather xreate::AST unless client's code explicitly works with Syntax Tree during construction. * * Typically instance only created by xreate::XreateManager and filled in by Parser * \sa xreate::XreateManager::prepare(std::string&&) */ class AST { friend class xreate::TypesResolver; public: AST(); /** * \brief Adds new function to AST * \param f Function to register */ void add(Function* f); /** * \brief Adds new declarative rule to AST * \param r Declarative Rule */ void add(MetaRuleAbstract* r); /** \brief Registers new code block */ ManagedScpPtr add(CodeScope* scope); /** * \brief Add new type to AST * @param t Type definition * @param alias Typer name */ void add(TypeAnnotation t, Atom alias); /** \brief Current module's name */ std::string getModuleName(); /** * \brief Looks for function with given name * \param name Function name to find * \note Requires that only one function exists under given name * \return Found function */ ManagedPtr findFunction(const std::string& name); /** \brief Returns all function in AST */ std::list getAllFunctions() const; /** * \brief Returns all specializations of a function with a given name * \param fnName function to find * \return list of found function specializations */ std::list getFunctionSpecializations(const std::string& fnName) const; /** * \return First element in Functions/Scopes/Rules list depending on template parameter * \tparam Target either Function or CodeScope or MetaRuleAbstract */ template ManagedPtr begin(); /** * \brief Performs all necessary steps after AST is built * * Performs all finzalisation steps and move AST into consistent state represented by xreate::AST * \sa xreate::AST * \return AST in consistent state */ xreate::AST* finalize(); typedef std::multimap FUNCTIONS_REGISTRY; std::vector __externdata; std::list __dfadata; //TODO move to more appropriate place std::list __rawImports; //TODO move to more appropriate place std::multimap __interfacesData; //TODO CFA data here. private: std::vector __rules; std::vector __functions; std::vector __scopes; FUNCTIONS_REGISTRY __indexFunctions; protected: std::map __indexTypeAliases; public: /** * \brief Stores DFA scheme for later use by DFA Pass * * Treats expression as a DFA scheme and feeds to a DFA Pass later * \paramn Expression DFA Scheme * \sa xreate::DFAPass */ void addDFAData(Expression&& data); /** \brief Stores data for later use by xreate::ExternLayer */ void addExternData(ExternData&& data); /** * \brief Generalized function to store particular data for later use by particular pass * \param interface Particular Interface * \param data Particular data */ void addInterfaceData(const ASTInterface& interface, Expression&& data); /**\name Symbols Recognition */ ///@{ public: //TODO revisit enums/variants, move to codescope /** * \brief Tries to find out whether expression is Variant constructor */ void recognizeVariantConstructor(Expression& function); Atom recognizeVariantConstructor(Atom ident); private: std::map> __dictVariants; public: std::set> bucketUnrecognizedIdentifiers; public: /** * \brief Postpones unrecognized identifier for future second round of recognition * \param scope Code block identifier is encountered * \param id Identifier */ void postponeIdentifier(CodeScope* scope, const Expression& id); /** \brief Second round of identifiers recognition done right after AST is fully constructed */ void recognizePostponedIdentifiers(); ///@} }; template<> ManagedPtr AST::begin(); template<> ManagedPtr AST::begin(); template<> ManagedPtr AST::begin(); } } // namespace details::incomplete /** * \brief Xreate's Syntax Tree in consistent state * * Syntax Tree has two mutually exclusive possible states: * - inconsistent state while AST is under construction. Represented by xreate::details::inconsistent::AST * - consistent state when AST is built and finalize() is done. * * This class represents consistent state and should be used everywhere unless client's code explicitly works with AST under construction. * Consistent AST enables access to additional functions(currently related to type management). * \sa xreate::details::inconsistent::AST */ class AST : public details::inconsistent::AST { public: AST() : details::inconsistent::AST() {} /** * \brief Computes fully expanded form of type by substituting all arguments and aliases * \param t Type to expand * \return Expdanded or normal form of type * \sa TypeAnnotation */ ExpandedType expandType(const TypeAnnotation &t) const; /** * Searches type by given name * \param name Typename to search * \return Expanded or normal form of desired type * \note if type name is not found returns new undefined type with this name */ ExpandedType findType(const std::string& name); /** * Invokes Type Inference Analysis to find out expanded(normal) form expressions's type * \sa typeinference.h * \param expression * \return Type of expression */ ExpandedType getType(const Expression& expression); }; } #endif // AST_H diff --git a/cpp/src/aux/serialization/expressionserializer.cpp b/cpp/src/aux/serialization/expressionserializer.cpp index 481e813..e450529 100644 --- a/cpp/src/aux/serialization/expressionserializer.cpp +++ b/cpp/src/aux/serialization/expressionserializer.cpp @@ -1,321 +1,321 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. * * expressionserializer.cpp * * Created on: Jan 4, 2016 * Author: pgess */ #include "aux/serialization/expressionserializer.h" #include #include #include using namespace std; //using namespace boost::bimaps; namespace xreate { struct Index { string name; size_t degree; //count of parameters unsigned char level; //level in expression tree (depth of tree layer) bool operator< (const Index other) const{ if (name != other.name) return name < other.name; if (degree != other.degree) return degree < other.degree; if (name != other.name) return level < other.level; return false; } }; class ExpressionSerializerPrivate { //boost::bimap> __registry; struct { map left; } __registry; map __range; public: void pack(const Expression& e, unsigned char level, OptionalPackedExpression& target){ if (!target) return; switch (e.op){ - case Operator::NONE: { + case Operator::INVALID: { switch (e.__state) { case Expression::NUMBER: case Expression::STRING: case Expression::IDENT : { Index index; if ((e.__state == Expression::NUMBER)) index = {std::to_string(e.getValueDouble()), 0, level}; else index = {e.getValueString(), 0, level}; if (!__registry.left.count(index)){ target = boost::none; return; } size_t id = __registry.left.at(index); size_t range = __range[level]; (*target) << make_pair(id, range); return; } default: break; } break; } case Operator::CALL: { Index index{e.getValueString(), e.operands.size(), level}; if(!__registry.left.count(index)){ target = boost::none; return; } size_t id = __registry.left.at(index); size_t range = __range[level]; (*target) << make_pair(id, range); for (const Expression& operand: e.operands){ pack(operand, level+1, target); } return; } default: break; } assert(false && "Expression too complicate for serialization"); } void registerExpression(const Expression&e, unsigned char level){ switch (e.op){ case Operator::CALL: { Index index{e.getValueString(), e.operands.size(), level}; if (__registry.left.insert(make_pair(index, __range[level])).second){ __range[level]++; } for (const Expression& operand: e.operands){ registerExpression(operand, level+1); } return; } - case Operator::NONE: { + case Operator::INVALID: { Index index; switch (e.__state) { case Expression::STRING: case Expression::IDENT: { index = {e.getValueString(), 0, level}; if (__registry.left.insert(make_pair(index, __range[level])).second){ __range[level]++; } return; } case Expression::NUMBER: { index = {std::to_string(e.getValueDouble()), 0, level}; if (__registry.left.insert(make_pair(index, __range[level])).second){ __range[level]++; } return; } default: break; } break; } default: break; } assert(false && "Expression too complicate for serialization"); } }; ExpressionSerializer::ExpressionSerializer() : strategy(new ExpressionSerializerPrivate()){ } ExpressionSerializer::~ExpressionSerializer() { delete strategy; } void ExpressionSerializer::registerExpression(const Expression&e){ if (e.isValid()) strategy->registerExpression(e, 0); } PackedExpression ExpressionSerializer::getId(const Expression& e){ OptionalPackedExpression result(boost::in_place()); //move(PackedExpression()) strategy->pack(e, 0, result); assert(result); return move(*result); } OptionalPackedExpression ExpressionSerializer::getIdOptional(const Expression& e) const{ OptionalPackedExpression result(boost::in_place()); //move(PackedExpression()) strategy->pack(e, 0, result); return result; } ExpressionSerializerIntegral::ExpressionSerializerIntegral():serializer(*this){} ExpressionSerializerIntegral::ExpressionSerializerIntegral(const std::vector&& expressions) : std::vector(move(expressions)), serializer(*this){ size_t id =0; for (const Expression& e: expressions){ __registry.emplace(serializer.getId(e), id++); } } size_t ExpressionSerializerIntegral::size() const{ return PARENT::size(); } size_t ExpressionSerializerIntegral::count(const Expression& e) const { return (getIdOptional(e)? 1: 0); } ExpressionSerializerIntegral::const_iterator ExpressionSerializerIntegral::begin() const { return PARENT::begin(); } ExpressionSerializerIntegral::const_iterator ExpressionSerializerIntegral::end() const { return PARENT::end(); } size_t ExpressionSerializerIntegral::getId(const Expression& e) const{ const OptionalPackedExpression exprPacked = serializer.getIdOptional(e); assert(exprPacked); return __registry.at(*exprPacked); } boost::optional ExpressionSerializerIntegral::getIdOptional(const Expression& e) const{ const OptionalPackedExpression exprPacked = serializer.getIdOptional(e); if (!exprPacked){ return boost::none; } return __registry.at(*exprPacked); } const Expression& ExpressionSerializerIntegral::get(size_t id) const{ return at(id); } void PackedExpression::operator<< (const std::pair& value){ static const size_t sizeSizeT = sizeof(size_t); const size_t& id = value.first; const size_t& range = value.second; int countSufficientBits = range <=1? 0 : ceil(log2(range)); if (0 < countRemainedBits && countRemainedBits < countSufficientBits) { size_t* tail = reinterpret_cast(__storage + size- sizeSizeT); (*tail) += id >> (countSufficientBits - countRemainedBits); countSufficientBits-=countRemainedBits; countRemainedBits = 0; } if (countRemainedBits == 0) { if (countSufficientBits == 0) return; char* __storageNew = new char[size+sizeSizeT]; std::memcpy (__storageNew, __storage, size); std::memset(__storageNew + size, 0, sizeSizeT); delete[] __storage; __storage = __storageNew; size += sizeSizeT; countRemainedBits = 8 * sizeSizeT; } if (countRemainedBits >= countSufficientBits) { size_t* tail = reinterpret_cast(__storage + size- sizeSizeT); (*tail) += id << (countRemainedBits - countSufficientBits); countRemainedBits -= countSufficientBits; return; } assert("Unreachable block"); } #if BOOST_VERSION <= 105500 PackedExpression::PackedExpression(const PackedExpression& other){ __storage = other.__storage; size = other.size; countRemainedBits = other.countRemainedBits; } #endif PackedExpression::PackedExpression(PackedExpression&& other){ __storage = other.__storage; size = other.size; countRemainedBits = other.countRemainedBits; other.__storage = nullptr; } bool PackedExpression::operator==(const PackedExpression& other) const{ if (size == other.size && countRemainedBits == other.countRemainedBits){ return std::memcmp(__storage, other.__storage, size) == 0 ; } return false; } bool PackedExpression::operator<(const PackedExpression& other) const{ if (size < other.size) { return true; } if (countRemainedBits < other.countRemainedBits) return true; if (size == other.size && countRemainedBits == other.countRemainedBits){ return std::memcmp(__storage, other.__storage, size) < 0 ; } return false; } bool PackedExpression::operator!=(const PackedExpression& other) const{ return ! ((*this) == other); } PackedExpression::~PackedExpression() { delete[] __storage; } //PackedExpression::PackedExpression (const PackedExpression& other) // : size(other.size), countRemainedBits(other.countRemainedBits) //{ // __storage = new char[size]; // std::memcpy (__storage, other.__storage, size); //} } /* namespace xreate */ diff --git a/cpp/src/compilation/containers.cpp b/cpp/src/compilation/containers.cpp index 3fed6b3..9474f3b 100644 --- a/cpp/src/compilation/containers.cpp +++ b/cpp/src/compilation/containers.cpp @@ -1,206 +1,206 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. * * File: containers.cpp * Author: pgess * * \file compilation/containers.h * \brief Containers compilation support. See more [details on Containers](/w/concepts/containers) */ #include "compilation/containers.h" using namespace std; using namespace llvm; using namespace xreate; using namespace xreate::containers; Iterator* Iterator::create(xreate::compilation::Context context, const xreate::Symbol& var){ const Implementation& data = Query::queryImplementation(var); switch(data.impl){ case ON_THE_FLY: return new IteratorForward(context, var, data.extract()); case SOLID: return new IteratorForward(context, var, data.extract()); default: assert(true); } assert(false && "Unknown declaration"); return nullptr; } llvm::Value* IteratorForward::begin() { switch(sourceDecl.op) { case xreate::Operator::LIST: { sourceRawType = llvm::Type::getInt32Ty(llvm::getGlobalContext()); return llvm::ConstantInt::get(Type::getInt32Ty(llvm::getGlobalContext()), 0); }; case xreate::Operator::LIST_RANGE:{ assert(sourceDecl.operands.size()==2); llvm::Value* result = sourceUnit->process(sourceDecl.operands.at(0)); sourceRawType = result->getType(); return result; }; default: break; } if (linkedlist){ llvm::Value* result = sourceUnit->process(sourceDecl); sourceRawType = result->getType(); return result; } assert(false); } llvm::Value* IteratorForward::end(){ switch(sourceDecl.op) { case xreate::Operator::LIST: { size_t idLast = sourceDecl.operands.size() - 1; return ConstantInt::get(sourceRawType, idLast); } case xreate::Operator::LIST_RANGE: { assert(sourceDecl.operands.size() == 2); llvm::Value* valueEndOfRange = sourceUnit->process(sourceDecl.operands.at(1)); llvm::Value* valueConstOne = llvm::ConstantInt::get(llvm::Type::getInt32Ty(llvm::getGlobalContext()), 1); return llvm->builder.CreateAdd(valueEndOfRange, valueConstOne); }; default: break; } //return null pointer if (linkedlist){ return ConstantPointerNull::getNullValue(sourceRawType); } assert(false && "Unknown declaration"); return nullptr; } llvm::Value* IteratorForward::get(Value* index,const std::string& hintRetVar){ const Expression& currentDecl = CodeScope::getDefinition(current); switch (currentDecl.op) { case xreate::Operator::LIST: { //TODO re check is it right scope(source) to compile currentDecl. Provide unittests. llvm::Value* currentValue = sourceUnit->processSymbol(current); return xreate::compilation::AdvancedInstructions(context).compileArrayIndex(currentValue, std::vector{index}); }; case xreate::Operator::LIST_RANGE: { return index; }; case xreate::Operator::MAP: { assert(currentDecl.getOperands().size()==1); assert(currentDecl.bindings.size()); assert(currentDecl.blocks.size()); CodeScope* scopeLoop = currentDecl.blocks.front(); std::string varEl = currentDecl.bindings[0]; const Symbol& symbIn = Attachments::get(currentDecl.getOperands()[0]); auto it = std::unique_ptr(Iterator::create(context, symbIn)); Value* elIn = it->get(index, varEl); compilation::ICodeScopeUnit* unitLoop = function->getScopeUnit(scopeLoop); unitLoop->bindArg(elIn, std::move(varEl)); return unitLoop->compile(); } - case xreate::Operator::NONE: { + case xreate::Operator::INVALID: { //TODO review iterator determination strategy for case of Expression::BINDING assert(currentDecl.__state==Expression::IDENT); const Symbol& symbIn = Attachments::get(currentDecl); auto it = std::unique_ptr(Iterator::create(context, symbIn)); return it->get(index); }; default: break; } if (linkedlist){ return index; } assert(false && "Unknown declaration"); return nullptr; } llvm::Value* IteratorForward::advance(Value* index, const std::string& hintRetVar){ switch(sourceDecl.op) { case xreate::Operator::LIST: case xreate::Operator::LIST_RANGE: return llvm->builder.CreateAdd(index, llvm::ConstantInt::get(llvm::Type::getInt32Ty(llvm::getGlobalContext()), 1), hintRetVar); default: break; } if (linkedlist){ ExpandedType tySource = llvm->ast->getType(CodeScope::getDefinition(source)); assert(tySource->__operator == TypeOperator::LIST && "Linked list implementation has to have ARRAY type"); assert(tySource->__operands.size()); return xreate::compilation::AdvancedInstructions(context).compileStructIndex(index, ExpandedType(TypeAnnotation(tySource->__operands.at(0))), linkedlist.fieldPointer); } assert(false && "Unknown declaration"); return nullptr; } //const ImplementationRec& implementation IteratorForward::IteratorForward(const compilation::Context& ctx, const xreate::Symbol& symbolContainer, const ImplementationRec& implementation) : Iterator(), __length(implementation.size), llvm(ctx.pass->man->llvm) { __container = ctx.function->getScopeUnit(symbolContainer.scope)->processSymbol(symbolContainer); } llvm::Value* IteratorForward::begin(){ //0 return llvm::ConstantInt::get(llvm::Type::getInt32Ty(llvm::getGlobalContext()), 0); } llvm::Value* IteratorForward::end(){ //length return llvm::ConstantInt::get(llvm::Type::getInt32Ty(llvm::getGlobalContext()), __length); } llvm::Value* IteratorForward::get(llvm::Value* index,const std::string& hintRetVar){ //GEP[index]] llvm::Type* tyNum = llvm::Type::getInt32Ty(llvm::getGlobalContext()); llvm::Value* pResult = llvm->builder.CreateGEP(__container, ArrayRef(std::vector{ConstantInt::get(tyNum, 0), index})); return llvm->builder.CreateLoad(pResult, hintRetVar); } llvm::Value* IteratorForward::advance(llvm::Value* index, const std::string& hintRetVar){ //index + 1 llvm::Type* tyNum = llvm::Type::getInt32Ty(llvm::getGlobalContext()); return llvm->builder.CreateAdd(index, llvm::ConstantInt::get(tyNum, 1), hintRetVar); } diff --git a/cpp/src/pass/compilepass.cpp b/cpp/src/pass/compilepass.cpp index bd83691..6c173ed 100644 --- a/cpp/src/pass/compilepass.cpp +++ b/cpp/src/pass/compilepass.cpp @@ -1,876 +1,873 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. * * Author: pgess * * compilepass.cpp */ /** * \file compilepass.h * \brief Compilation pass */ #include "compilepass.h" #include "clasplayer.h" #include #include "llvmlayer.h" #include "query/containers.h" #include "query/context.h" #include "compilation/containers.h" #include "compilation/latecontextcompiler2.h" #include "ExternLayer.h" #include "pass/adhocpass.h" #include "compilation/targetinterpretation.h" #include "pass/versionspass.h" #include "compilation/scopedecorators.h" #include "compilation/adhocfunctiondecorator.h" #include "compilation/operators.h" #include "analysis/typeinference.h" #include #include #include using namespace std; using namespace llvm; //TODO use Scope //SECTIONTAG late-context FunctionDecorator namespace xreate{namespace context{ /** \brief Late Context enabled decorator for IFunctionUnit * \extends IFunctionUnit */ template class LateContextFunctionDecorator : public Parent { public: LateContextFunctionDecorator(ManagedFnPtr f, CompilePass* p) : Parent(f, p), contextCompiler(this, p) { } protected: std::vector prepareArguments() { std::vector&& arguments = Parent::prepareArguments(); size_t sizeLateContextDemand = contextCompiler.getFunctionDemandSize(); if (sizeLateContextDemand) { llvm::Type* ty32 = llvm::Type::getInt32Ty(llvm::getGlobalContext()); llvm::Type* tyDemand = llvm::ArrayType::get(ty32, sizeLateContextDemand); arguments.push_back(tyDemand); } return arguments; } llvm::Function::arg_iterator prepareBindings() { llvm::Function::arg_iterator fargsI = Parent::prepareBindings(); size_t sizeLateContextDemand = contextCompiler.getFunctionDemandSize(); if (sizeLateContextDemand) { fargsI->setName("latecontext"); contextCompiler.rawContextArgument = &*fargsI; ++fargsI; } return fargsI; } public: context::LateContextCompiler2 contextCompiler; }; }} //end of namespace xreate::context namespace xreate { namespace compilation{ std::string BasicFunctionUnit::prepareName(){ AST* ast = IFunctionUnit::pass->man->root; string name = ast->getFunctionSpecializations(IFunctionUnit::function->__name).size() > 1 ? IFunctionUnit::function->__name + std::to_string(IFunctionUnit::function.id()) : IFunctionUnit::function->__name; return name; } std::vector BasicFunctionUnit::prepareArguments() { LLVMLayer* llvm = IFunctionUnit::pass->man->llvm; AST* ast = IFunctionUnit::pass->man->root; CodeScope* entry = IFunctionUnit::function->__entry; std::vector signature; std::transform(entry->__bindings.begin(), entry->__bindings.end(), std::inserter(signature, signature.end()), [llvm, ast, entry](const std::string & arg)->llvm::Type* { assert(entry->__identifiers.count(arg)); ScopedSymbol argid{entry->__identifiers.at(arg), versions::VERSION_NONE}; return llvm->toLLVMType(ast->expandType(entry->__declarations.at(argid).type)); }); return signature; } llvm::Type* BasicFunctionUnit::prepareResult() { LLVMLayer* llvm = IFunctionUnit::pass->man->llvm; AST* ast = IFunctionUnit::pass->man->root; CodeScope* entry = IFunctionUnit::function->__entry; return llvm->toLLVMType(ast->expandType(entry->__declarations.at(ScopedSymbol::RetSymbol).type)); } llvm::Function::arg_iterator BasicFunctionUnit::prepareBindings() { CodeScope* entry = IFunctionUnit::function->__entry; ICodeScopeUnit* entryCompilation = IFunctionUnit::getScopeUnit(entry); llvm::Function::arg_iterator fargsI = IFunctionUnit::raw->arg_begin(); for (std::string &arg : entry->__bindings) { ScopedSymbol argid{entry->__identifiers[arg], versions::VERSION_NONE}; entryCompilation->bindArg(&*fargsI, argid); fargsI->setName(arg); ++fargsI; } return fargsI; } //DEBT compiler rigidly depends on exact definition of DefaultFunctionUnit typedef context::LateContextFunctionDecorator< adhoc::AdhocFunctionDecorator< BasicFunctionUnit>> DefaultFunctionUnit; ICodeScopeUnit::ICodeScopeUnit(const CodeScope* const codeScope, IFunctionUnit* f, CompilePass* compilePass) : pass(compilePass), function(f), scope(codeScope), currentBlockRaw(nullptr) { } llvm::Value* CallStatementRaw::operator()(std::vector&& args, const std::string& hintDecl) { llvm::Function* calleeInfo = dyn_cast(__callee); if (calleeInfo) { auto argsFormal = calleeInfo->args(); int pos = 0; //SECTIONTAG types/convert function ret value for (auto argFormal = argsFormal.begin(); argFormal != argsFormal.end(); ++argFormal, ++pos) { args[pos] = typeinference::doAutomaticTypeConversion(args[pos], argFormal->getType(), llvm->builder); } } //Do not name function call that returns Void. std::string nameStatement = hintDecl; if (calleeInfo->getReturnType()->isVoidTy()){ nameStatement.clear(); } return llvm->builder.CreateCall(__calleeTy, __callee, args, nameStatement); } //DESABLEDFEATURE implement inlining class CallStatementInline : public ICallStatement { public: CallStatementInline(IFunctionUnit* caller, IFunctionUnit* callee, LLVMLayer* l) : __caller(caller), __callee(callee), llvm(l) { } llvm::Value* operator()(std::vector&& args, const std::string& hintDecl) { //TOTEST inlining // CodeScopeUnit* entryCompilation = outer->getScopeUnit(function->__entry); // for(int i=0, size = args.size(); ibindArg(args.at(i), string(entryCompilation->scope->__bindings.at(i))); // } // // // return entryCompilation->compile(); return nullptr; } private: IFunctionUnit* __caller; IFunctionUnit* __callee; LLVMLayer* llvm; bool isInline() { // Symbol ret = Symbol{0, function->__entry}; // bool flagOnTheFly = SymbolAttachments::get(ret, false); //TODO consider inlining return false; } }; BasicCodeScopeUnit::BasicCodeScopeUnit(const CodeScope* const codeScope, IFunctionUnit* f, CompilePass* compilePass) : ICodeScopeUnit(codeScope, f, compilePass) { } llvm::Value* BasicCodeScopeUnit::processSymbol(const Symbol& s, std::string hintRetVar) { Expression declaration = CodeScope::getDefinition(s); const CodeScope* scope = s.scope; ICodeScopeUnit* scopeExternal = ICodeScopeUnit::function->getScopeUnit(scope); llvm::Value* resultRaw; if (scopeExternal == this){ resultRaw = process(declaration, hintRetVar); currentBlockRaw = pass->man->llvm->builder.GetInsertBlock(); } else { assert(scopeExternal->currentBlockRaw); llvm::BasicBlock* blockOwn = pass->man->llvm->builder.GetInsertBlock(); pass->man->llvm->builder.SetInsertPoint(scopeExternal->currentBlockRaw); resultRaw = scopeExternal->processSymbol(s, hintRetVar); pass->man->llvm->builder.SetInsertPoint(blockOwn); } return resultRaw; } //TASK Isolate out context functionalty in decorator //TOTEST static late context decisions //TOTEST dynamic late context decisions ICallStatement* BasicCodeScopeUnit::findFunction(const Expression& opCall) { const std::string& calleeName = opCall.getValueString(); LLVMLayer* llvm = pass->man->llvm; ClaspLayer* clasp = pass->man->clasp; DefaultFunctionUnit* function = dynamic_cast (this->function); context::ContextQuery* queryContext = pass->queryContext; const std::list& specializations = pass->man->root->getFunctionSpecializations(calleeName); //if no specializations registered - check external function if (specializations.size() == 0) { llvm::Function* external = llvm->layerExtern->lookupFunction(calleeName); llvm::outs() << "Debug/External function: " << calleeName; external->getType()->print(llvm::outs(), true); llvm::outs() << "\n"; return new CallStatementRaw(external, llvm); } //no decisions required if (specializations.size() == 1) { if (!specializations.front()->guardContext.isValid()) { return new CallStatementRaw(pass->getFunctionUnit(specializations.front())->compile(), llvm); } } //TODO move dictSpecialization over to a separate function in order to perform cache, etc. //prepare specializations dictionary std::map dictSpecializations; boost::optional variantDefault; boost::optional variant; for (const ManagedFnPtr& f : specializations) { const Expression& guard = f->guardContext; //default case: if (!guard.isValid()) { variantDefault = f; continue; } assert(dictSpecializations.emplace(guard, f).second && "Found several identical specializations"); } //check static context ScopePacked scopeCaller = clasp->pack(this->scope); const string atomSpecialization = "specialization"; const Expression topicSpecialization(Operator::CALL,{(Atom(string(atomSpecialization))), Expression(Operator::CALL, {Atom(string(calleeName))}), Atom(scopeCaller)}); const context::Decisions& decisions = queryContext->getFinalDecisions(scopeCaller); if (decisions.count(topicSpecialization)) { variant = dictSpecializations.at(decisions.at(topicSpecialization)); } //TODO check only demand for this particular topic. size_t sizeDemand = function->contextCompiler.getFunctionDemandSize(); //decision made if static context found or no late context exists(and there is default variant) bool flagHasStaticDecision = variant || (variantDefault && !sizeDemand); //if no late context exists if (flagHasStaticDecision) { IFunctionUnit* calleeUnit = pass->getFunctionUnit(variant ? *variant : *variantDefault); //inlining possible based on static decision only // if (calleeUnit->isInline()) { // return new CallStatementInline(function, calleeUnit); // } return new CallStatementRaw(calleeUnit->compile(), llvm); } //require default variant if no static decision made assert(variantDefault); llvm::Function* functionVariantDefault = this->pass->getFunctionUnit(*variantDefault)->compile(); llvm::Value* resultFn = function->contextCompiler.findFunction(calleeName, functionVariantDefault, scopeCaller); llvm::PointerType *resultPTy = cast(resultFn->getType()); llvm::FunctionType *resultFTy = cast(resultPTy->getElementType()); return new CallStatementRaw(resultFn, resultFTy, llvm); } //DISABLEDFEATURE transformations // if (pass->transformations->isAcceptable(expr)){ // return pass->transformations->transform(expr, result, ctx); // } llvm::Value* BasicCodeScopeUnit::process(const Expression& expr, const std::string& hintVarDecl) { -#define DEFAULT(x) (hintVarDecl.empty()? x: hintVarDecl) + #define DEFAULT(x) (hintVarDecl.empty()? x: hintVarDecl) llvm::Value *left; llvm::Value *right; LLVMLayer& l = *pass->man->llvm; xreate::compilation::AdvancedInstructions instructions = xreate::compilation::AdvancedInstructions({this, function, pass}); switch (expr.op) { case Operator::SUB: case Operator::MUL: case Operator::DIV: case Operator::EQU: case Operator::LSS: case Operator::GTR: case Operator::NE: case Operator::LSE: case Operator::GTE: assert(expr.__state == Expression::COMPOUND); assert(expr.operands.size() == 2); left = process(expr.operands[0]); right = process(expr.operands[1]); //SECTIONTAG types/convert binary operation right = typeinference::doAutomaticTypeConversion(right, left->getType(), l.builder); break; default:; } switch (expr.op) { case Operator::ADD: { left = process(expr.operands[0]); Context context{this, function, pass}; llvm::Value* resultSU = StructUpdate::add(expr.operands[0], left, expr.operands[1], context, DEFAULT("tmp_add")); if (resultSU) return resultSU; right = process(expr.operands[1]); llvm::Value* resultAddPA = pointerarithmetic::PointerArithmetic::add(left, right, context, DEFAULT("tmp_add")); if (resultAddPA) { return resultAddPA; } return l.builder.CreateAdd(left, right, DEFAULT("tmp_add")); break; } case Operator::SUB: return l.builder.CreateSub(left, right, DEFAULT("tmp_sub")); break; case Operator::MUL: return l.builder.CreateMul(left, right, DEFAULT("tmp_mul")); break; case Operator::DIV: return l.builder.CreateSDiv(left, right, DEFAULT("tmp_div")); break; case Operator::EQU: if (left->getType()->isIntegerTy()) return l.builder.CreateICmpEQ(left, right, DEFAULT("tmp_equ")); if (left->getType()->isFloatingPointTy()) return l.builder.CreateFCmpOEQ(left, right, DEFAULT("tmp_equ")); break; case Operator::NE: return l.builder.CreateICmpNE(left, right, DEFAULT("tmp_ne")); break; case Operator::LSS: return l.builder.CreateICmpSLT(left, right, DEFAULT("tmp_lss")); break; case Operator::LSE: return l.builder.CreateICmpSLE(left, right, DEFAULT("tmp_lse")); break; case Operator::GTR: return l.builder.CreateICmpSGT(left, right, DEFAULT("tmp_gtr")); break; case Operator::GTE: return l.builder.CreateICmpSGE(left, right, DEFAULT("tmp_gte")); break; case Operator::NEG: left = process(expr.operands[0]); return l.builder.CreateNeg(left, DEFAULT("tmp_neg")); break; case Operator::CALL: { assert(expr.__state == Expression::COMPOUND); shared_ptr callee(findFunction(expr)); const std::string& nameCallee = expr.getValueString(); //prepare arguments std::vector args; args.reserve(expr.operands.size()); std::transform(expr.operands.begin(), expr.operands.end(), std::inserter(args, args.end()), [this](const Expression & operand) { return process(operand); } ); ScopePacked outerScopeId = pass->man->clasp->pack(this->scope); //TASK a) refactor CALL/ADHOC/find function //SECTIONTAG late-context propagation arg size_t calleeDemandSize = pass->queryContext->getFunctionDemand(nameCallee).size(); if (calleeDemandSize) { DefaultFunctionUnit* function = dynamic_cast (this->function); llvm::Value* argLateContext = function->contextCompiler.compileContextArgument(nameCallee, outerScopeId); args.push_back(argLateContext); } return (*callee)(move(args), DEFAULT("res_" + nameCallee)); } case Operator::IF: { return instructions.compileIf(expr, DEFAULT("tmp_if")); } case Operator::SWITCH: { return instructions.compileSwitch(expr, DEFAULT("tmp_switch")); } case Operator::LOOP_CONTEXT: { assert(false); return nullptr; //return instructions.compileLoopContext(expr, DEFAULT("tmp_loop")); } case Operator::LOGIC_AND: { assert(expr.operands.size() == 1); return process(expr.operands[0]); } case Operator::LIST: { return instructions.compileListAsSolidArray(expr, DEFAULT("tmp_list")); }; case Operator::LIST_RANGE: { assert(false); //no compilation phase for a range list // return InstructionList(this).compileConstantArray(expr, l, hintRetVar); }; case Operator::LIST_NAMED: { typedef Expanded ExpandedType; ExpandedType tyStructLiteral = l.ast->getType(expr); const std::vector fieldsFormal = (tyStructLiteral.get().__operator == TypeOperator::CUSTOM) ? l.layerExtern->getStructFields(l.layerExtern->lookupType(tyStructLiteral.get().__valueCustom)) : tyStructLiteral.get().fields; std::map indexFields; for (size_t i = 0, size = fieldsFormal.size(); i < size; ++i) { indexFields.emplace(fieldsFormal[i], i); } llvm::StructType* tyLiteralRaw = llvm::cast(l.toLLVMType(tyStructLiteral)); llvm::Value* record = llvm::UndefValue::get(tyLiteralRaw); for (size_t i = 0; i < expr.operands.size(); ++i) { const Expression& operand = expr.operands.at(i); unsigned int fieldId = indexFields.at(expr.bindings.at(i)); llvm::Value* result = process(operand); assert(result); record = l.builder.CreateInsertValue(record, result, llvm::ArrayRef({fieldId})); } return record; }; case Operator::MAP: { assert(expr.blocks.size()); return instructions.compileMapSolidOutput(expr, DEFAULT("map")); }; case Operator::FOLD: { return instructions.compileFold(expr, DEFAULT("fold")); }; case Operator::FOLD_INF: { return instructions.compileFoldInf(expr, DEFAULT("fold")); }; case Operator::INDEX: { //TASK allow multiindex compilation assert(expr.operands.size() == 2); assert(expr.operands[0].__state == Expression::IDENT); const std::string& hintIdent = expr.operands[0].getValueString(); Symbol s = Attachments::get(expr.operands[0]); const ExpandedType& t2 = pass->man->root->getType(expr.operands[0]); llvm::Value* aggr = processSymbol(s, hintIdent); switch (t2.get().__operator) { case TypeOperator::LIST_NAMED: case TypeOperator::CUSTOM: { std::string idxField; const Expression& idx = expr.operands.at(1); switch (idx.__state) { //named struct field case Expression::STRING: idxField = idx.getValueString(); break; //anonymous struct field case Expression::NUMBER: idxField = to_string((int) idx.getValueDouble()); break; default: assert(false && "Wrong index for a struct"); } return instructions.compileStructIndex(aggr, t2, idxField); }; case TypeOperator::LIST: { std::vector indexes; std::transform(++expr.operands.begin(), expr.operands.end(), std::inserter(indexes, indexes.end()), [this] (const Expression & op) { return process(op); } ); return instructions.compileArrayIndex(aggr, indexes, DEFAULT(string("el_") + hintIdent)); }; default: assert(false); } }; //SECTIONTAG adhoc actual compilation //TODO a) make sure that it's correct: function->adhocImplementation built for Entry scope and used in another scope case Operator::ADHOC: { DefaultFunctionUnit* function = dynamic_cast (this->function); assert(function->adhocImplementation && "Adhoc implementation not found"); const Expression& comm = adhoc::AdhocExpression(expr).getCommand(); CodeScope* scope = function->adhocImplementation->getCommandImplementation(comm); ICodeScopeUnit* unitScope = function->getScopeUnit(scope); //SECTIONTAG types/convert ADHOC ret convertation llvm::Type* resultTy = l.toLLVMType(pass->man->root->expandType(function->adhocImplementation->getResultType())); return typeinference::doAutomaticTypeConversion(unitScope->compile(), resultTy, l.builder); }; case Operator::CALL_INTRINSIC: { const std::string op = expr.getValueString(); if (op == "copy") { llvm::Value* result = process(expr.getOperands().at(0)); auto decoratorVersions = Decorators::getInterface(this); llvm::Value* storage = decoratorVersions->processIntrinsicInit(result->getType()); decoratorVersions->processIntrinsicCopy(result, storage); return l.builder.CreateLoad(storage, hintVarDecl); } assert(false && "undefined intrinsic"); } case Operator::VARIANT: { const ExpandedType& typVariant = pass->man->root->getType(expr); llvm::Type* typVariantRaw = l.toLLVMType(typVariant); llvm::Type* typIdRaw = llvm::cast(typVariantRaw)->getElementType(0); uint64_t id = expr.getValueDouble(); llvm::Value* variantRaw = llvm::UndefValue::get(typVariantRaw); variantRaw = l.builder.CreateInsertValue(variantRaw, llvm::ConstantInt::get(typIdRaw, id), llvm::ArrayRef({0})); const bool flagDoReference = expr.operands.size(); if (flagDoReference){ const ExpandedType& subtyp = ExpandedType(typVariant->__operands.at(id)); llvm::Type* subtypRaw = l.toLLVMType(subtyp); Attachments::put(expr.operands.at(0), subtyp); llvm::Value* subtypValue = process(expr.operands.at(0)); llvm::Type* typStorageRaw = llvm::cast(typVariantRaw)->getElementType(1); llvm::Value* addrAsStorage = l.builder.CreateAlloca(typStorageRaw); llvm::Value* addrAsSubtyp = l.builder.CreateBitOrPointerCast(addrAsStorage, subtypRaw->getPointerTo()); l.builder.CreateStore(subtypValue, addrAsSubtyp); llvm::Value* storageRaw = l.builder.CreateLoad(typStorageRaw, addrAsStorage); variantRaw = l.builder.CreateInsertValue(variantRaw, storageRaw, llvm::ArrayRef({1})); } return variantRaw; } case Operator::SWITCH_VARIANT: { return instructions.compileSwitchVariant(expr, DEFAULT("tmpswitch")); } - case Operator::SEQUENCE: - { + case Operator::SEQUENCE: { return instructions.compileSequence(expr); } - case Operator::NONE: + case Operator::UNDEF: { + llvm::Type* typExprUndef = l.toLLVMType(typeinference::getType(expr, *pass->man->root)); + return llvm::UndefValue::get(typExprUndef); + } + + case Operator::INVALID: assert(expr.__state != Expression::COMPOUND); switch (expr.__state) { case Expression::IDENT: { Symbol s = Attachments::get(expr); return processSymbol(s, expr.getValueString()); } case Expression::NUMBER: { - llvm::Type* typConst; - - if (expr.type.isValid()) { - typConst = l.toLLVMType(pass->man->root->getType(expr)); - - } else { - typConst = llvm::Type::getInt32Ty(llvm::getGlobalContext()); - } - + llvm::Type* typConst = l.toLLVMType(typeinference::getType(expr, *pass->man->root)); int literal = expr.getValueDouble(); + return llvm::ConstantInt::get(typConst, literal); } case Expression::STRING: { return instructions.compileConstantStringAsPChar(expr.getValueString(), DEFAULT("tmp_str")); }; default: { break; } }; break; default: break; } assert(false && "Can't compile expression"); return 0; } llvm::Value* BasicCodeScopeUnit::compile(const std::string& hintBlockDecl) { if (!hintBlockDecl.empty()) { llvm::BasicBlock *block = llvm::BasicBlock::Create(llvm::getGlobalContext(), hintBlockDecl, function->raw); pass->man->llvm->builder.SetInsertPoint(block); } currentBlockRaw = pass->man->llvm->builder.GetInsertBlock(); Symbol symbScope = Symbol{ScopedSymbol::RetSymbol, scope}; return processSymbol(symbScope); } ICodeScopeUnit::~ICodeScopeUnit() { } IFunctionUnit::~IFunctionUnit() { } llvm::Function* IFunctionUnit::compile() { if (raw != nullptr) return raw; LLVMLayer* llvm = pass->man->llvm; llvm::IRBuilder<>& builder = llvm->builder; string&& functionName = prepareName(); std::vector&& types = prepareArguments(); llvm::Type* expectedResultType = prepareResult(); llvm::FunctionType *ft = llvm::FunctionType::get(expectedResultType, types, false); raw = llvm::cast(llvm->module->getOrInsertFunction(functionName, ft)); prepareBindings(); const std::string&blockName = "entry"; llvm::BasicBlock* blockCurrent = builder.GetInsertBlock(); llvm::Value* result = getScopeUnit(function->__entry)->compile(blockName); assert(result); //SECTIONTAG types/convert function ret value builder.CreateRet(typeinference::doAutomaticTypeConversion(result, expectedResultType, llvm->builder)); if (blockCurrent) { builder.SetInsertPoint(blockCurrent); } llvm->moveToGarbage(ft); return raw; } ICodeScopeUnit* IFunctionUnit::getScopeUnit(const CodeScope * const scope) { if (__scopes.count(scope)) { auto result = __scopes.at(scope).lock(); if (result) { return result.get(); } } std::shared_ptr unit(pass->buildCodeScopeUnit(scope, this)); if (scope->__parent != nullptr) { auto parentUnit = Decorators::getInterface(getScopeUnit(scope->__parent)); parentUnit->registerChildScope(unit); } else { __orphanedScopes.push_back(unit); } if (!__scopes.emplace(scope, unit).second) { __scopes[scope] = unit; } return unit.get(); } ICodeScopeUnit* IFunctionUnit::getScopeUnit(ManagedScpPtr scope) { return getScopeUnit(&*scope); } ICodeScopeUnit* IFunctionUnit::getEntry() { return getScopeUnit(function->getEntryScope()); } template<> compilation::IFunctionUnit* CompilePassCustomDecorators::buildFunctionUnit(const ManagedFnPtr& function){ return new DefaultFunctionUnit(function, this); } template<> compilation::ICodeScopeUnit* CompilePassCustomDecorators::buildCodeScopeUnit(const CodeScope* const scope, IFunctionUnit* function){ return new DefaultCodeScopeUnit(scope, function, this); } } // emf of compilation IFunctionUnit* CompilePass::getFunctionUnit(const ManagedFnPtr& function) { unsigned int id = function.id(); if (!functions.count(id)) { IFunctionUnit* unit = buildFunctionUnit(function); functions.emplace(id, unit); return unit; } return functions.at(id); } void CompilePass::run() { managerTransformations = new TransformationsManager(); targetInterpretation = new interpretation::TargetInterpretation(this->man->root, this); queryContext = reinterpret_cast (man->clasp->getQuery(QueryId::ContextQuery)); //Find out main function; ClaspLayer::ModelFragment model = man->clasp->query(Config::get("function-entry")); assert(model && "Error: No entry function found"); assert(model->first != model->second && "Error: Ambiguous entry function"); string nameMain = std::get<0>(ClaspLayer::parse(model->first->second)); IFunctionUnit* unitMain = getFunctionUnit(man->root->findFunction(nameMain)); entry = unitMain->compile(); } llvm::Function* CompilePass::getEntryFunction() { assert(entry); return entry; } void CompilePass::prepareQueries(ClaspLayer* clasp) { clasp->registerQuery(new containers::Query(), QueryId::ContainersQuery); clasp->registerQuery(new context::ContextQuery(), QueryId::ContextQuery); Attachments::init(); clasp->registerQuery(new polymorph::PolymorphQuery(), QueryId::PolymorphQuery); } } //end of namespace xreate /** * \class xreate::CompilePass * \brief Encapsulates all compilation activities * * xreate::CompilePass iterates over xreate::AST tree and produces executable code fed by data(via xreate::Attachments) gathered by previous passes as well as data via queries(xreate::IQuery) from xreate:ClaspLayer reasoner. * Compilation's done using xreate::LLVMLayer(wrapper over LLVM toolchain) and based on following aspects: * - Containers support. See \ref compilation/containers.h * - Late Conext compilation. See xreate::context::LateContextCompiler2 * - Interpretation support. See xreate::interpretation::TargetInterpretation * - Loop saturation support. See xreate::compilation::TransformerSaturation * - External Code access. See xreate::ExternLayer(wrapper over Clang library) * * \section adaptability_sect Adaptability * xreate::CompilePass's architecture provides adaptability by employing: * - %Function Decorators to alter function-level compilation. See xreate::compilation::IFunctionUnit * - Code Block Decorators to alter code block level compilation. See xreate::compilation::ICodeScopeUnit * Default functionality defined by \ref xreate::compilation::DefaultCodeScopeUnit * - Targets to allow more versitile extensions. * Currently only xreate::interpretation::TargetInterpretation use Targets infrastructure. See xreate::compilation::Target * - %Altering Function invocation. xreate::compilation::ICallStatement * * Client able to construct compiler with desired decorators using xreate::compilation::CompilePassCustomDecorators. * As a handy alias, `CompilePassCustomDecorators` constructs default compiler * */ diff --git a/cpp/src/utils.h b/cpp/src/utils.h index 9632983..c98e03e 100644 --- a/cpp/src/utils.h +++ b/cpp/src/utils.h @@ -1,166 +1,164 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. * * utils.cpp * * Author: pgess */ #ifndef UTILS_H #define UTILS_H #include "jeayeson/jeayeson.hpp" namespace xreate { template struct AddTag { explicit AddTag(const Source &src) : __src(src) { } explicit AddTag(Source &&src) : __src(std::move(src)) { } operator const Source&() const{ return __src; } const Source& get() const{ return __src; } const Source* operator->() const { return &__src; } private: Source __src; }; struct Expand_t{}; template using Expanded = AddTag; - - //DEBT move to resources compiler. https://github.com/markusfisch/cpprc class Config { private: json_map __storage; static Config __self; Config(); public: static std::string get(std::string key) { return __self.__storage.get_for_path(key).get(); } }; /** \brief Decorators support */ template struct DecoratorsDict{ //typedef ConcreteDecoratorForTag result; }; template struct Decorators{ typedef typename DecoratorsDict::result Instance; template static Instance* getInterface(Base* obj){ return dynamic_cast< Instance* > (obj); } }; template struct ManagedPtr { static ManagedPtr Invalid() { return ManagedPtr(); } ManagedPtr() : __storage(0) { } ManagedPtr(unsigned int id, const std::vector* storage) : __id(id), __storage(storage) { } Target& operator*() const { assert(isValid() && "Invalid Ptr"); return *__storage->at(__id); } void operator=(const ManagedPtr& other) { __id = other.__id; __storage = other.__storage; } bool operator==(const ManagedPtr& other) { return isValid() && (__id == other.__id); } Target* operator->() const noexcept { assert(isValid() && "Invalid Ptr"); return __storage->at(__id); } inline bool isValid() const { return (__storage) && (0 <= __id) && (__id < __storage->size()); } inline operator bool() const { return isValid(); } ManagedPtr& operator++() { ++__id; return *this; } inline unsigned int id() const { return __id; } private: unsigned int __id = 0; const std::vector * __storage = 0; }; } std::wstring utf8_to_wstring(const std::string& str); std::string wstring_to_utf8(const std::wstring& str); #define RST "\x1B[0m" #define KRED "\x1B[31m" #define KGRN "\x1B[32m" #define KYEL "\x1B[33m" #define KBLU "\x1B[34m" #define KMAG "\x1B[35m" #define KCYN "\x1B[36m" #define KWHT "\x1B[37m" #define FRED(x) KRED << x << RST #define FGRN(x) KGRN <) # TESTS #========================= FIND_PACKAGE (LLVM REQUIRED) message("LLVM_LIBRARY_DIRS: " ${LLVM_LIBRARY_DIRS}) link_directories(${LLVM_LIBRARY_DIRS}) set (LIBCLASP_PATH ${POTASSCO_PATH}/build/debug) link_directories(${LIBCLASP_PATH}) #aux_source_directory(. TEST_FILES) set(TEST_FILES + exploitation.cpp communication.cpp polymorph.cpp association.cpp main.cpp modules.cpp adhoc.cpp attachments.cpp ast.cpp cfa.cpp dfa.cpp compilation.cpp ExpressionSerializer.cpp externc.cpp context.cpp types.cpp vendorAPI/clangAPI.cpp vendorAPI/xml2.cpp vendorAPI/json.cpp containers.cpp context.cpp interpretation.cpp loops.cpp #supplemental/versions-algorithm-data_dependency.cpp effects-versions.cpp ) add_executable(${PROJECT_NAME} ${TEST_FILES}) target_link_libraries(${PROJECT_NAME} xreate ${GTEST_LIBRARIES} pthread xml2 gcov) add_custom_target (coverage COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/src/code-coverage.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/cpp/tests/cfa.cpp b/cpp/tests/cfa.cpp index 0f8075e..409ecc9 100644 --- a/cpp/tests/cfa.cpp +++ b/cpp/tests/cfa.cpp @@ -1,197 +1,195 @@ /* Any copyright is dedicated to the Public Domain. * http://creativecommons.org/publicdomain/zero/1.0/ * * testsCFG.cpp * * Created on: Jul 17, 2015 * Author: pgess */ #include "xreatemanager.h" #include "pass/dfapass.h" #include "pass/cfapass.h" - #include "analysis/DominatorsTreeAnalysisProvider.h" - -#include "gtest/gtest.h" #include "analysis/cfagraph.h" +#include "gtest/gtest.h" #include #include using namespace xreate; using namespace xreate::cfa; using namespace std; TEST(CFA, testFunctionAnnotationsClasp){ string&& program = "f2 = function::int; annotationF2 {\n" " 0\n" "}\n" "\n" "f1 = function:: int; entry; annotationF1 {\n" " f2() + 10\n" "}"; details::tier1::XreateManager* man = details::tier1::XreateManager::prepare(move(program)); man->analyse(); ClaspLayer::ModelFragment answer = man->clasp->query("annotationF1"); int countNoneValue = 0; if (answer) countNoneValue = std::distance(answer->first, answer->second); EXPECT_EQ(1, countNoneValue); answer = man->clasp->query("annotationF2"); countNoneValue = 0; if (answer) countNoneValue = std::distance(answer->first, answer->second); EXPECT_EQ(1, countNoneValue); } TEST(CFA, testLoopContextExists){ details::tier1::XreateManager* man = details::tier1::XreateManager::prepare ( "interface(cfa){\n" " operator fold:: annotation1.\n" "}\n" "\n" "main = function:: int; entry {\n" " x = [1..10]:: [int].\n" " sum = loop fold (x->el:: int, 0->sum):: int {\n" " el + sum + f1()\n" " }. \n" " sum\n" "}" "case context:: annotation1 {" " f1 = function::int {\n" " x = 0:: int. " " x\n" " }" "}" ); man->analyse(); ClaspLayer::ModelFragment model = man->clasp->query("annotation1"); ScopePacked scopeIdActual = std::get<0>(ClaspLayer::parse(model->first->second)); CodeScope* scopeEntry = man->root->findFunction("main")->getEntryScope(); const Expression& exprSum = scopeEntry->getDefinition(scopeEntry->getSymbol("sum")); CodeScope* scopeExpected = exprSum.blocks.front(); ScopePacked scopeIdExpected = man->clasp->pack(scopeExpected); ASSERT_EQ(scopeIdExpected, scopeIdActual); } TEST(CFA, DependenciesFnCall){ details::tier2::XreateManager* man = details::tier2::XreateManager::prepare( R"Code( a = function::int{ seq {x = 0:: int. x} {x = b():: int. x}::int } b = function::int {y = 0. y} )Code"); CodeScope* scopeSeq1 = man->root->findFunction("a")->getEntryScope()->getBody().blocks.front(); CodeScope* scopeSeq2 = *(++man->root->findFunction("a")->getEntryScope()->getBody().blocks.begin()); CodeScope* scopeB = man->root->findFunction("b")->getEntryScope(); ScopePacked psSeq1 = man->clasp->pack(scopeSeq1); ScopePacked psSeq2 = man->clasp->pack(scopeSeq2); ScopePacked psB = man->clasp->pack(scopeB); CFAPass* pass = new CFAPass(man); man->registerPass(pass, PassId::CFGPass); man->executePasses(); const CFAGraph* report = dynamic_cast(man->getPassById(PassId::CFGPass))->getReport(); auto dependencies = report->__dependencyRelations; delete pass; ASSERT_EQ(1, dependencies.count(psSeq2)); ASSERT_EQ(1, dependencies.count(psB)); } TEST(CFA, DependenciesChildScope){ details::tier2::XreateManager* man = details::tier2::XreateManager::prepare( R"Code( a = function::int{ seq {x = 0:: int. x} {x=0::int. if(x>0)::int{1} else {0}}::int } )Code"); CodeScope* scopeSeq1 = man->root->findFunction("a")->getEntryScope()->getBody().blocks.front(); CodeScope* scopeSeq2 = *(++man->root->findFunction("a")->getEntryScope()->getBody().blocks.begin()); CodeScope* scopeIf1 = scopeSeq2->getBody().blocks.front(); CodeScope* scopeIf2 = *(++scopeSeq2->getBody().blocks.begin()); ScopePacked psSeq1 = man->clasp->pack(scopeSeq1); ScopePacked psSeq2 = man->clasp->pack(scopeSeq2); ScopePacked psIf1 = man->clasp->pack(scopeIf1); ScopePacked psIf2 = man->clasp->pack(scopeIf2); CFAPass* pass = new CFAPass(man); man->registerPass(pass, PassId::CFGPass); man->executePasses(); const CFAGraph* report = dynamic_cast(man->getPassById(PassId::CFGPass))->getReport(); auto dependencies = report->__dependencyRelations; delete pass; ASSERT_EQ(0, dependencies.count(psSeq1)); ASSERT_EQ(1, dependencies.count(psSeq2)); ASSERT_EQ(1, dependencies.count(psIf1)); ASSERT_EQ(1, dependencies.count(psIf2)); for(auto rec: dependencies) { std::cout << rec.first << " " << rec.second << std::endl; } } TEST(CFA, DomReportOneRoot){ std::string program = R"CODE( a = function:: int; entry{ seq {x = 0:: int. x} {x = 1:: int. x}::int } )CODE"; std::unique_ptr man(details::tier2::XreateManager::prepare(move(program))); CFAPass* pass = new CFAPass(man.get()); man->registerPass(pass, PassId::CFGPass); pass->run(); ScopePacked scope1 = man->clasp->pack(man->root->findFunction("a")->getEntryScope()->getBody().blocks.front()); ScopePacked scope2 = man->clasp->pack(*++man->root->findFunction("a")->getEntryScope()->getBody().blocks.begin()); dominators::DominatorsTreeAnalysisProvider* providerDomAnalysis = new dominators::DominatorsTreeAnalysisProvider(); providerDomAnalysis->run(pass->getReport()); dominators::DominatorsTreeAnalysisProvider::Dominators expectedFDom= { {1, {0, 3}} ,{2, {1, 2}} }; dominators::DominatorsTreeAnalysisProvider::Dominators expectedPostDom= { {2, {0, 3}} ,{1, {1, 2}} }; auto actualFDom = providerDomAnalysis->getForwardDominators(); auto actualPostDom = providerDomAnalysis->getPostDominators(); ASSERT_EQ(expectedFDom, actualFDom); ASSERT_EQ(expectedPostDom, actualPostDom); delete providerDomAnalysis; delete pass; } diff --git a/cpp/tests/exploitation.cpp b/cpp/tests/exploitation.cpp new file mode 100644 index 0000000..e1a17dd --- /dev/null +++ b/cpp/tests/exploitation.cpp @@ -0,0 +1,27 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * exploitation.cpp + * + * Author: pgess + * Created on February 15, 2018, 6:17 PM + */ + +#include "xreatemanager.h" +#include "gtest/gtest.h" + +using namespace xreate; + +TEST(Exploitation, test1){ + FILE* input = fopen("scripts/exploitation/test1.xreate","r"); + assert(input != nullptr); + + std::unique_ptr man(XreateManager::prepare(input)); + + int (*main)() = (int (*)())man->run(); + int result = main(); + + ASSERT_EQ(1, result); +} \ No newline at end of file diff --git a/grammar/xreate.ATG b/grammar/xreate.ATG index 5b18378..86b8dc1 100644 --- a/grammar/xreate.ATG +++ b/grammar/xreate.ATG @@ -1,664 +1,665 @@ //TODO add ListLiteral //TODO ExprTyped: assign default(none) type #include "ast.h" #include "ExternLayer.h" #include "pass/adhocpass.h" #include #include #define wprintf(format, ...) \ char __buffer[100]; \ wcstombs(__buffer, format, 100); \ fprintf(stderr, __buffer, __VA_ARGS__) using namespace std; COMPILER Xreate details::inconsistent::AST* root = nullptr; // current program unit void ensureInitalizedAST(){ if (root == nullptr) root = new details::inconsistent::AST(); } struct { std::stack scopesOld; CodeScope* scope = nullptr; } context; void pushContextScope(CodeScope* scope){ context.scopesOld.push(context.scope); context.scope = scope; } void popContextScope(){ context.scope = context.scopesOld.top(); context.scopesOld.pop(); } int nextToken() { scanner->ResetPeek(); return scanner->Peek()->kind; } bool checkTokenAfterIdent(int key){ if (la->kind != _ident) return false; return nextToken() == key; } bool checkParametersList() { return la->kind == _ident && nextToken() == _lparen; } bool checkInfix() { return la->kind == _ident && nextToken() == _ident; } bool checkIndex() { return la->kind == _ident && nextToken() == _lbrack; } bool checkFuncDecl() { if (la->kind != _ident) return false; int token2 = nextToken(); int token3 = scanner->Peek()->kind; return token2 == _assign && (token3 == _function || token3 == _pre); } bool checkAssignment() { if (la->kind != _ident) return false; scanner->ResetPeek(); int token2 = scanner->Peek()->kind; if (token2 == _lcurbrack) { scanner->Peek(); int token3 = scanner->Peek()->kind; if (token3 != _rcurbrack) return false; int token4 = scanner->Peek()->kind; return token4 == _assign; } return token2 == _assign; } void recognizeIdentifier(Expression& i){ if (!context.scope->recognizeIdentifier(i)){ root->postponeIdentifier(context.scope, i); } } enum SwitchKind{SWITCH_NORMAL, SWITCH_META}; CHARACTERS letter = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz". any = ANY - '"'. digit = "0123456789". cr = '\r'. lf = '\n'. tab = '\t'. TOKENS ident = (letter | '_') {letter | digit | '_'}. number = (digit | '-' digit) {digit}. string = '"' { any } '"'. function = "function". pre = "pre". lparen = '('. rparen = ')'. lbrack = '['. rbrack = ']'. lcurbrack = '{'. rcurbrack = '}'. equal = "==". assign = '='. implic = '-' '>'. colon = ':'. context = "context". tagcolon = "::". lse = "<=". lss = "<". gte = ">=". gtr = ">". ne1 = "!=". ne2= "<>". COMMENTS FROM "/*" TO "*/" NESTED COMMENTS FROM "//" TO lf IGNORE cr + lf + tab PRODUCTIONS Xreate = (. Function* function; ensureInitalizedAST(); .) {( RuleDecl | InterfaceData | Imprt | ContextSection | GuardSection | IF(checkFuncDecl()) FDecl (. root->add(function); .) | TDecl | SkipModulesSection )} (. .) . Ident = ident (. name = t->val; .). VarIdent = ident (. e = Expression(Atom(t->val)); .) [ lcurbrack ( ident (. SemErr(coco_string_create("var version as ident is not implemented yet")); .) | number (. Attachments::put(e, Atom(t->val).get()); .) ) rcurbrack ] . FDecl = (. std::wstring fname; std::wstring argName; TypeAnnotation typIn; TypeAnnotation typOut; bool flagIsPrefunct = false; Expression binding; .) Ident assign [pre (. flagIsPrefunct = true; .)] function (. f = new Function(fname); f->isPrefunction = flagIsPrefunct; CodeScope* entry = f->getEntryScope(); .) ['(' Ident tagcolon ExprAnnotations (. f->addBinding(Atom(argName), move(binding)); .) {',' Ident tagcolon ExprAnnotations (. f->addBinding(Atom (argName), move(binding));.) } ')'] [ tagcolon ( IF(flagIsPrefunct) FnTag | Type ) {';' FnTag }] BDecl (. const_cast(entry->getBody()).bindType(move(typOut));.) . ContextSection<>= (. Expression context; Function* f; .) "case" "context" tagcolon MetaSimpExpr lcurbrack { FDecl (. f->guardContext = context; root->add(f); .) } rcurbrack. GuardSection<>= (. Expression guard; Function* f; .) "guard" tagcolon MetaSimpExpr lcurbrack { FDecl (. f->guard = guard; root->add(f); .) } rcurbrack. /** * TYPES * */ TypeTerm = (. std::wstring tid; .) ("string" (. typ = TypePrimitive::String;.) | "num" (. typ = TypePrimitive::Num;.) | "int" (. typ = TypePrimitive::Int;.) | "float" (. typ = TypePrimitive::Float;.) | "bool" (. typ = TypePrimitive::Bool; .) | "i8" (. typ = TypePrimitive::I8; .) | "i32" (. typ = TypePrimitive::I32; .) | "i64" (. typ = TypePrimitive::I64; .) ). Type = (. TypeAnnotation typ2; TypePrimitive typ3; std::wstring tid, field; .) ( TList | TStruct | TVariant | TypeTerm (. typ = typ3; .) | IF (checkIndex()) Ident lbrack Ident (. typ = TypeAnnotation(TypeOperator::ACCESS, {}); typ.__valueCustom = Atom(tid).get(); typ.fields.push_back(Atom(field).get()); .) {',' Ident (. typ.fields.push_back(Atom(field).get()); .) } rbrack | Ident (. typ = TypeAnnotation(TypeOperator::CUSTOM, {}); typ.__valueCustom = Atom(tid).get(); .) ['(' Type (. typ.__operator = TypeOperator::CALL; typ.__operands.push_back(typ2); .) {',' Type (. typ.__operands.push_back(typ2); .) } ')'] ) . TList = (. TypeAnnotation ty; .) '[' Type ']' (. typ = TypeAnnotation(TypeOperator::LIST, {ty}); .) . TStruct = (. TypeAnnotation t; std::wstring key; size_t keyCounter=0; .) lcurbrack ( IF(checkTokenAfterIdent(_tagcolon)) Ident tagcolon Type | Type (. key = to_wstring(keyCounter++); .) ) (. typ = TypeAnnotation(TypeOperator::LIST_NAMED, {t}); typ.fields.push_back(Atom(key).get()); .) {',' ( IF(checkTokenAfterIdent(_tagcolon)) Ident tagcolon Type | Type (. key = to_wstring(keyCounter++); .) ) (. typ.__operands.push_back(t); typ.fields.push_back(Atom(key).get()); .) } rcurbrack. TVariant= (. TypeAnnotation t, typVoid(TypeOperator::LIST_NAMED, {}); std::vector operands; std::vector> keys; std::wstring variant; .) "variant" lcurbrack Ident (. t=typVoid; .) [tagcolon Type] (. keys.push_back(Atom(variant)); operands.push_back(t); .) {',' Ident (. t=typVoid; .) [tagcolon Type] (. keys.push_back(Atom(variant)); operands.push_back(t); .) } rcurbrack (. typ = TypeAnnotation(TypeOperator::VARIANT, {}); typ.__operands = operands; typ.addFields(std::move(keys)); .) . TDecl = (. TypeAnnotation t; std::wstring tname, arg; std::vector> args; .) Ident assign "type" ['(' Ident (. args.push_back(Atom(arg)); .) {',' Ident (. args.push_back(Atom(arg)); .) } ')'] Type'.' (. t.addBindings(move(args)); root->add(move(t), Atom(tname)); .) . ContextDecl = (. Expression tag; .) context tagcolon MetaSimpExpr (. scope->tags.push_back(tag); .) {';' MetaSimpExpr (. scope->tags.push_back(tag); .) }. VDecl = (. std::wstring vname; Expression var, value;.) VarIdent assign ExprTyped (. Symbol identSymbol = f->addDefinition(move(var), move(value)); Attachments::put(value, identSymbol); .) . BDecl = lcurbrack (. Expression body; pushContextScope(scope); .) {(IF(checkAssignment()) VDecl '.' | RuleContextDecl | ContextDecl'.' | ExprTyped (. scope->setBody(body); Attachments::put(body, Symbol{ScopedSymbol::RetSymbol, scope});.) )} rcurbrack (. popContextScope(); .) . IfDecl = (. Expression cond; ManagedScpPtr blockTrue = root->add(new CodeScope(context.scope)); ManagedScpPtr blockFalse = root->add(new CodeScope(context.scope)); .) "if" '(' Expr ')' (. e = Expression(Operator::IF, {cond}); .) tagcolon ExprAnnotations BDecl<&*blockTrue> "else" BDecl<&*blockFalse> (. e.addBlock(blockTrue); e.addBlock(blockFalse); .) . LoopDecl = (. Expression eIn, eAcc, eFilters; std::wstring varEl, varAcc, contextClass; Expression tagsEl; ManagedScpPtr block = root->add(new CodeScope(context.scope)); .) "loop" ("map" '(' Expr implic Ident (. e = Expression(Operator::MAP, {eIn}); .) tagcolon ExprAnnotations ')' tagcolon ExprAnnotations (. e.addBindings({Atom(varEl)}); block->addBinding(Atom(varEl), move(tagsEl)); .) BDecl<&*block> (. e.addBlock(block); .) |"fold" ("inf" '(' Expr implic Ident ')' (. e = Expression(Operator::FOLD_INF, {eAcc}); e.addBindings({Atom(varAcc)}); block->addBinding(Atom(varAcc), Expression()); .) tagcolon ExprAnnotations BDecl<&*block> (. e.addBlock(block); .) | '(' Expr implic Ident tagcolon ExprAnnotations ['|' Expr ] ',' Expr implic Ident')' (. e = Expression(Operator::FOLD, {eIn, eAcc}); e.addBindings({Atom(varEl), Atom(varAcc)}); .) tagcolon ExprAnnotations (. block->addBinding(Atom(varEl), move(tagsEl)); block->addBinding(Atom(varAcc), Expression()); .) BDecl<&*block> (. e.addBlock(block); .) ) | "context" '(' string (. contextClass = t->val; .) ')' BDecl<&*block> (. e = Expression(Operator::LOOP_CONTEXT, {Expression(Atom(std::move(contextClass)))}); e.addBlock(block); .) ). // Switches SwitchDecl = (. TypeAnnotation typ; eSwitch = Expression(Operator::SWITCH, {}); Expression eCondition; Expression tag;.) "switch" ( SwitchVariantDecl | lparen ExprTyped rparen tagcolon ExprAnnotations (. eSwitch.operands.push_back(eCondition);.) CaseDecl {CaseDecl} ) . CaseDecl = (. ManagedScpPtr scope = root->add(new CodeScope(context.scope)); Expression condition; .) "case" ( IF(flagSwitchKind == SWITCH_META) lparen MetaSimpExpr rparen BDecl<&*scope> (. Expression exprCase(Operator::CASE, {}); exprCase.addTags({condition}); exprCase.addBlock(scope); outer.addArg(move(exprCase));.) | "default" BDecl<&*scope> (. Expression exprCase(Operator::CASE_DEFAULT, {}); exprCase.addBlock(scope); outer.operands.insert(++outer.operands.begin(), exprCase); .) | lparen CaseParams<&*scope> rparen (. ManagedScpPtr scopeBody = root->add(new CodeScope(&*scope)); Expression exprCase(Operator::CASE, {}); .) BDecl<&*scopeBody> (. exprCase.addBlock(scope); exprCase.addBlock(scopeBody); outer.addArg(move(exprCase)); .) ). CaseParams = (. Expression condition; Expression guard(Operator::LOGIC_AND, {}); pushContextScope(scope); .) ExprTyped (. guard.addArg(Expression(condition)); .) {',' ExprTyped (. guard.addArg(Expression(condition)); .) } (. scope->setBody(guard); popContextScope(); .) . SwitchVariantDecl = (. Expression varTested; std::wstring varAlias; bool flagAliasFound = false; expr = Expression(Operator::SWITCH_VARIANT, {}); .) "variant" lparen Expr [implic Ident (. flagAliasFound = true; .) ] [tagcolon ExprAnnotations] rparen tagcolon ExprAnnotations (. expr.addArg(std::move(varTested)); if (flagAliasFound) { expr.addBindings({Atom(varAlias)}); } else { if(varTested.__state == Expression::IDENT){ expr.addBindings({Atom(string(varTested.getValueString()))}); } } .) CaseVariantDecl {CaseVariantDecl} . CaseVariantDecl = (. ManagedScpPtr scope = root->add(new CodeScope(context.scope)); std::wstring key; scope->addBinding(Atom(string(expr.bindings.front())), Expression()); .) "case" lparen Ident rparen (. expr.addArg(root->recognizeVariantConstructor(Atom(std::move(key)))); .) BDecl<&*scope> (. expr.addBlock(scope); .) . IntrinsicDecl= (. std::wstring name; .) "intrinsic" Ident< name> (. outer = Expression(Operator::CALL_INTRINSIC, {}); outer.setValue(Atom(name)); .) lparen [CalleeParams] rparen . SequenceDecl = (. sequence = Expression(); sequence.setOp(Operator::SEQUENCE); ManagedScpPtr scope = root->add(new CodeScope(context.scope)); .) "seq" BDecl<&*scope> (. sequence.blocks.push_back(&*scope); scope = root->add(new CodeScope(&*scope)); .) { (. scope = root->add(new CodeScope(&*scope)); .) BDecl<&*scope> (. sequence.blocks.push_back(&*scope); .) }. /*============================ INTERFACES ===============================*/ Imprt<> = "import" "raw" lparen string (. root->__rawImports.push_back(Atom(t->val).get()); .) rparen '.'. InterfaceData<> = "interface" '(' ( "dfa" ')' InterfaceDFA | "extern-c" ')' InterfaceExternC | "cfa" ')' InterfaceCFA | "adhoc" ')' InterfaceAdhoc ). InterfaceAdhoc<> = '{' { PrefunctionSchemeDecl } '}'. PrefunctionSchemeDecl<> = (. TypeAnnotation typReturn; std::wstring prefName; Expression exprCases; .) pre function Ident tagcolon Type lcurbrack SwitchDecl rcurbrack (. Expression prefData(Operator::CALL, {Atom(prefName), exprCases}); prefData.bindType(typReturn); root->addInterfaceData(Adhoc, move(prefData)); .). InterfaceExternC<> = (. ExternData data; .) '{' {IncludeExternDecl | LibExternDecl } '}' (. root->addExternData(move(data)); .) . LibExternDecl = (. std::wstring pkgname, libname; .) Ident assign "library" tagcolon "pkgconfig" '(' string (. pkgname = t->val; .) ')' '.' (. data.addLibrary(Atom(libname), Atom(pkgname)); .) . IncludeExternDecl = (. Expression inc; .) "include" StructLiteral '.' (. data.addIncludeDecl(move(inc)); .) . InterfaceDFA<> = '{' { InstructDecl } '}' . InstructDecl = (.Operator op; Expression tag; Expression scheme; std::vector& tags = scheme.operands; tags.push_back(Expression()); /* return value */ .) "operator" InstructAlias tagcolon '(' (.scheme.setOp(op); .) [ MetaSimpExpr (. tags.push_back(tag); .) { ',' MetaSimpExpr (. tags.push_back(tag); .) } ] ')' [ implic MetaSimpExpr (. tags[0] = tag; .) ] (. root->addDFAData(move(scheme)); .) '.'. InstructAlias = ( "map" (. op = Operator::MAP; .) | "list_range" (. op = Operator::LIST_RANGE; .) | "list" (. op = Operator::LIST; .) | "fold" (. op = Operator::FOLD; .) | "index" (. op = Operator::INDEX; .) ). InterfaceCFA<> = '{' { InstructCFADecl } '}' . InstructCFADecl<> = (.Operator op; Expression tag; Expression scheme; std::vector& tags = scheme.operands; .) "operator" InstructAlias tagcolon (. scheme.setOp(op); .) [ MetaSimpExpr (. tags.push_back(tag); .) { ',' MetaSimpExpr (. tags.push_back(tag); .) } ] '.' (. root->addInterfaceData(CFA, move(scheme)); .). /*============================ METAPROGRAMMING ===============================*/ // TagsDecl = (. Expression tag; TagModifier mod = TagModifier::NONE; .) // ':' { MetaSimpExpr (. /*f.addTag(std::move(tag), mod); */ .) // }. FnTag = (. Expression tag; TagModifier mod = TagModifier::NONE; .) MetaSimpExpr ['-' TagMod] (. f->addTag(std::move(tag), mod); .). TagMod = ( "assert" (. mod = TagModifier::ASSERT; .) | "require" (. mod = TagModifier::REQUIRE; .) ). RuleDecl<> = "rule" tagcolon (. RuleArguments args; RuleGuards guards; DomainAnnotation typ; std::wstring arg; .) '(' Ident tagcolon Domain (. args.add(arg, typ); .) {',' Ident tagcolon Domain (. args.add(arg, typ); .) } ')' ["case" RGuard {',' RGuard}] '{' RBody '}' . /* - TODO use RGuard for guards-*/ RuleContextDecl = (.Expression eHead, eGuards, eBody; .) "rule" "context" tagcolon MetaSimpExpr "case" lparen MetaSimpExpr rparen '{' MetaSimpExpr '}' (.scope->contextRules.push_back(Expression(Operator::CONTEXT_RULE, {eHead, eGuards, eBody})); .). Domain = ( "function" (. dom = DomainAnnotation::FUNCTION; .) | "variable" (. dom = DomainAnnotation::VARIABLE; .) ). RGuard= (. Expression e; .) MetaExpr (. guards.add(std::move(e)); .). MetaExpr= (.Operator op; Expression e2; .) MetaExpr2 [MetaOp MetaExpr2 (. e = Expression(op, {e, e2}); .) ]. MetaExpr2= ( '(' MetaExpr ')' | MetaSimpExpr ). MetaSimpExpr= (. std::wstring i1, infix; Expression e2; .) ( '-' MetaSimpExpr (. e = Expression(Operator::NEG, {e2}); .) | IF(checkParametersList()) Ident (. e = Expression(Operator::CALL, {Expression(Atom(i1))}); .) '(' [ MetaCalleeParams ] ')' | IF(checkInfix()) Ident Ident MetaSimpExpr (. e = Expression(Operator::CALL, {Expression(Atom(infix))}); e.addArg(Expression(Atom(i1))); e.addArg(std::move(e2)); .) | Ident (. e = Expression(Operator::CALL, {Atom(i1)}); .) ). MetaCalleeParams = (. Expression e2; .) MetaSimpExpr (. e.addArg(Expression(e2)); .) {',' MetaSimpExpr (. e.addArg(Expression(e2)); .) }. RBody = (. Expression e; std::wstring msg; .) "warning" MetaExpr ["message" string (. msg = t->val; .) ] (. root->add(new RuleWarning(RuleArguments(args), RuleGuards(guards), std::move(e), Atom(msg))); .) . MetaOp< Operator& op> = implic (. op = Operator::IMPL; .) . /*============================ Expressions ===============================*/ ExprAnnotations = (. TypeAnnotation typ; std::list tags; Expression tag; e.tags.clear();.) Type (. e.bindType(move(typ)); .) {';' MetaSimpExpr (. tags.push_back(tag); .) } (. e.addTags(tags); .) . ExprTyped = Expr [tagcolon ExprAnnotations]. Expr< Expression& e> (. Operator op; Expression e2; .) = ExprArithmAdd [ RelOp ExprArithmAdd (. e = Expression(op, {e, e2}); .) ]. ExprArithmAdd< Expression& e>= (. Operator op; Expression e2; .) ExprArithmMul< e> [ AddOp< op> ExprArithmAdd< e2> (. e = Expression(op, {e, e2});.) ]. ExprArithmMul< Expression& e> (. Operator op; Expression e2; .) = ExprPostfix< e> [ MulOp< op> ExprArithmMul< e2> (. e = Expression(op, {e, e2}); .) ]. ExprPostfix = Term [ (. e = Expression(Operator::INDEX, {e}); .) {lbrack CalleeParams rbrack } ]. Term< Expression& e> (. std::wstring name; e = Expression(); .) = (IF (checkParametersList()) Ident< name> (. e = Expression(Operator::CALL, {Atom(name)}); root->recognizeVariantConstructor(e); .) '(' [CalleeParams] ')' | VarIdent (. recognizeIdentifier(e); .) | ListLiteral (. /* tuple */.) | StructLiteral (. /* struct */.) | LoopDecl | IfDecl | SwitchDecl | AdhocDecl | IntrinsicDecl | SequenceDecl | number (. e = Expression(Atom(t->val)); .) | string (. e = Expression(Atom(t->val)); .) | "true" (. e = Expression(Atom(1)); e.bindType(TypePrimitive::Bool); .) | "false" (. e = Expression(Atom(0)); e.bindType(TypePrimitive::Bool); .) + | "undef" (. e = Expression(Operator::UNDEF, {}); .) | '-' Term (. e = Expression(Operator::NEG, {e}); .) | '(' ExprTyped ')' ). StructLiteral = (. std::wstring key; Expression val; std::list> keys; size_t keyCounter=0; .) lcurbrack (IF(checkTokenAfterIdent(_assign)) Ident '=' Expr | Expr (. key = to_wstring(keyCounter++); .) ) (. keys.push_back(Atom(key)); e = Expression(Operator::LIST_NAMED, {val}); .) {',' (IF(checkTokenAfterIdent(_assign)) Ident '=' Expr | Expr (. key = to_wstring(keyCounter++); .) ) (. e.addArg(Expression(val)); keys.push_back(Atom(key)); .) } rcurbrack (. e.addBindings(keys.begin(), keys.end()); .) . ListLiteral = (. Expression eFrom, eTo; .) '[' [ Expr (. e.addArg(Expression(eFrom)); .) (".." Expr (. e.addArg(Expression(eTo)); e.setOp(Operator::LIST_RANGE); .) |{',' Expr (. e.addArg(Expression(eFrom)); .) } (. e.setOp(Operator::LIST); .) ) ] ']'. AdhocDecl = (. Expression command; .) "ad" "hoc" MetaSimpExpr (. adhoc::AdhocExpression exprAdhoc; exprAdhoc.setCommand(command); e = exprAdhoc; .). CalleeParams = (. Expression e2; .) ExprTyped (. e.addArg(Expression(e2)); .) {',' ExprTyped (. e.addArg(Expression(e2)); .) }. AddOp< Operator& op> = (. op = Operator::ADD; .) ( '+' | '-' (. op = Operator::SUB; .) ). MulOp< Operator& op> = (. op = Operator::MUL; .) ( '*' | '/' (. op = Operator::DIV; .) ). RelOp< Operator& op> = (. op = Operator::EQU; .) ( equal | (ne1 | ne2) (. op = Operator::NE; .) | lse (. op = Operator::LSE; .) | lss (. op = Operator::LSS; .) | gte (. op = Operator::GTE; .) | gtr (. op = Operator::GTR; .) ). SkipModulesSection = "module" '{' {ANY} '}'. END Xreate. diff --git a/scripts/exploitation/exploitation.lp b/scripts/exploitation/exploitation.lp new file mode 100644 index 0000000..d4643a3 --- /dev/null +++ b/scripts/exploitation/exploitation.lp @@ -0,0 +1,86 @@ +% INPUT: +% cfa: cfa_forwdom +% expl_siteUser(Resource, UserScope), expl_siteInit(Resource, ScopeInit) +% + + +% PARTITION +%============================================================= +expl_mutual_dom(UserScope1, UserScope2, DomHighest):- + Ahighest = #max{ AD, (AD, BD): cfa_forwdom(Dom, range(AD, BD)), AD B1, BD > B2}; + cfa_forwdom(DomHighest, range(Ahighest, _)); + cfa_forwdom(UserScope1, range(A1, B1)); + cfa_forwdom(UserScope2, range(A2, B2)); + UserScope1B1}; + cfa_forwdom(DomHighest, range(Ahighest, _)); + cfa_forwdom(UserRoot, range(A1, B1)); + not expl_partition(Resource, UserRoot, _, _); + expl_partition_root(Resource, UserRoot). + +expl_init_assignments(Resource, SiteInitAssigned):- + A_Highest = #max{AI, (AI, BI): expl_siteInit(Resource, SiteI), cfa_forwdom(SiteI, range(AI, BI))}; + cfa_forwdom(SiteInitAssigned, range(A_Highest, _)); + expl_partition_root(Resource, _). + +% SOLUTIONS +%============================================================= + + +% ORDER +%============================================================= + + + +% Doms/PostDom trees +%============================================================= +%cfa_forwdom + + +% LANDSCAPE +%============================================================= +%expl_landscape(X, ) + + +% GARBAGE +%============================================================= +%expl_parent(Site, Parent):- +% A + +typedef FILE* FILE_P; diff --git a/scripts/exploitation/test1.xreate b/scripts/exploitation/test1.xreate new file mode 100644 index 0000000..a0c0024 --- /dev/null +++ b/scripts/exploitation/test1.xreate @@ -0,0 +1,32 @@ +interface(extern-c){ + xml2 = library:: pkgconfig("libxml-2.0"). + + include { + xml2 = ["scripts/exploitation/test1.h"] + }. +} + +import raw ("scripts/exploitation/exploitation.lp"). +import raw ("scripts/exploitation/test1.assembly.lp"). + +guard:: explInitRealImpl { + openFile = function(filePrev:: FILE_P):: FILE_P { + fopen("/tmp/test", "w")::FILE_P + } +} + +guard:: explInitBogusImpl { + openFile = function(filePrev:: FILE_P):: FILE_P { + filePrev::int + } +} + +test = function::int; entry { + seq + { f0 = undef:: FILE_P. f0 } + { f1 = openFile(f0):: FILE_P; expl_init(r1). f1 /*first attempt to open file*/} + { f2 = openFile(f1):: FILE_P; expl_init(r1). f2 /*second attempt to open file*/} + { sizeWritten = fwrite("WriteAttempt", 12, 1, f2):: int; expl_user(r1). sizeWritten } + { fclose(f2)::int; expl_use(r1)} + { sizeWritten :: int} +}