From 2c99d42816460aa2412da0ff22c2c331d17da09a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 11 Mar 2022 16:45:20 +0100 Subject: [PATCH 001/209] added a whitespace --- src/list_warn.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/list_warn.cc b/src/list_warn.cc index 2c0a786ee..24f7d7d54 100644 --- a/src/list_warn.cc +++ b/src/list_warn.cc @@ -62,7 +62,7 @@ void List_Warn::visit_begin(Alt::Block &a) { return; if (a.alts.size() > 1 && a.list_size() == Yield::UP) { Log::instance()->warning(a.location, "Block has an answer list >= n, " - "consider moving this block into an extra NT and" + "consider moving this block into an extra NT and " "applying choice function to it."); } } From a2336c139b6d5b2510758ac776b39efc9a0d5b56 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 11 Mar 2022 16:46:09 +0100 Subject: [PATCH 002/209] ALWAYS generate outside grammar (which must be triggered by a cmd argument later on) --- src/gapc.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/gapc.cc b/src/gapc.cc index 5bcf3d87a..3616cd64b 100644 --- a/src/gapc.cc +++ b/src/gapc.cc @@ -335,7 +335,7 @@ class Main { } // simply gets the selected grammar, which is either the - // grammar that occured first in the source code or is the + // grammar that occurred first in the source code or is the // one that was named in the parameters on the command line Grammar *grammar = driver.ast.grammar(); // Now check the semantic, which does more than the function @@ -348,6 +348,14 @@ class Main { throw LogError("Seen semantic errors."); } + // inject rules for outside grammar + grammar->inject_outside_nts(); + // repeat semantic checking (and linking) for the newly generated outside non-terminals + bool r2 = grammar->check_semantic(); + if (!r2) { + throw LogError("Seen semantic errors for outside version."); + } + // configure the window and k-best mode driver.ast.set_window_mode(opts.window_mode); driver.ast.kbest = Bool(opts.kbest); @@ -383,8 +391,8 @@ class Main { // suboptimal designs and present a message to the user. driver.ast.warn_user_table_conf_suboptimal(); - // find what type of input is read - // chars, sequence of ints etc. + // find what type of input is read + // chars, sequence of ints etc. driver.ast.derive_temp_alphabet(); r = driver.ast.check_signature(); From 4a26372d3f3d4ba86149f88342302216004e51da Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 11 Mar 2022 16:47:08 +0100 Subject: [PATCH 003/209] adding two new functions: 1) to obtain all linked non-terminals in rhs 2) to replace a given non-terminal with another --- src/alt.cc | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/alt.hh | 20 +++++++++++++--- 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 99c00e9af..6d21321c4 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -2928,3 +2928,73 @@ const std::list &Alt::Multi::ret_decls() const { bool Alt::Base::choice_set() { return datatype->simple()->is(::Type::LIST) && eval_nullary_fn; } + + +void Alt::Base::get_nonterminals(std::list *nt_list) { +} +void Alt::Simple::get_nonterminals(std::list *nt_list) { + for (Fn_Arg::Base *arg : this->args) { + Fn_Arg::Alt *arg_alt = dynamic_cast(arg); + if (arg_alt) { + // recurse into arguments + arg_alt->alt->get_nonterminals(nt_list); + } + } +} +void Alt::Link::get_nonterminals(std::list *nt_list) { + // test if Link is a non-terminal + if (this->nt) { + if (this->nt->is(Symbol::NONTERMINAL)) { + nt_list->push_back(dynamic_cast(this->nt)); + } else if (this->nt->is(Symbol::TERMINAL)) { + + } + } else { + throw LogError("Link is something else"); + } +} +void Alt::Block::get_nonterminals(std::list *nt_list) { + for (Alt::Base *alt : this->alts) { + // recurse into alternatives + alt->get_nonterminals(nt_list); + } +} +void Alt::Multi::get_nonterminals(std::list *nt_list) { + throw LogError("Alt::Multi::get_nonterminals is not yet implemented!"); +} + + +void Alt::Base::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace) { +} +void Alt::Simple::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace) { + for (Fn_Arg::Base *arg : this->args) { + Fn_Arg::Alt *arg_alt = dynamic_cast(arg); + if (arg_alt) { + arg_alt->alt->replace_nonterminal(find, replace); + } + } +} +void Alt::Link::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace) { + if (this->nt) { + if (this->nt->is(Symbol::NONTERMINAL)) { + if (this->nt == find) { + // replace old NT (=find) with novel NT (=replace) + this->nt = replace; + this->name = replace->name; + } + } else if (this->nt->is(Symbol::TERMINAL)) { + + } + } else { + throw LogError("Link is something else"); + } +} +void Alt::Block::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace) { + for (Alt::Base *alt : this->alts) { + // recurse into alternatives + alt->replace_nonterminal(find, replace); + } +} +void Alt::Multi::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace) { + throw LogError("Alt::Multi::replace_nonterminal is not yet implemented!"); +} diff --git a/src/alt.hh b/src/alt.hh index 428064a30..02175eb0b 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -308,6 +308,13 @@ class Base { virtual void set_ntparas(const Loc &loc, std::list *l); bool choice_set(); + + // traverses the alternative (=rhs of a production) and collects pointers to all referenced non-terminals + // e.g. ml_comps = cadd(incl(dangle), ml_comps1) should return [*dangle, *ml_comps1] + virtual void get_nonterminals(std::list *nt_list); + + // traverses the alternative (=rhs of a production), iff non-terminal find is contained, replace with first occurence of non-terminal replace + virtual void replace_nonterminal(Symbol::NT *find, Symbol::NT *replace); }; @@ -479,7 +486,8 @@ class Simple : public Base { public: void set_ntparas(std::list *l); - + void get_nonterminals(std::list *nt_list); + void replace_nonterminal(Symbol::NT *find, Symbol::NT *replace); private: std::list *insert_index_stmts( @@ -507,9 +515,9 @@ class Link : public Base { Symbol::Base *nt; - // Inits the insatnce and sets the local fields according to + // Inits the instance and sets the local fields according to // the parameter values of the non-terminal name. It also sets - // the pointer to the non-terminal grammar-node explicitely + // the pointer to the non-terminal grammar-node explicitly // to NULL. Link(std::string *n, const Loc&l) : Base(LINK, l), name(n), nt(NULL) { @@ -601,6 +609,8 @@ class Link : public Base { bool check_ntparas(); void optimize_choice(); + void get_nonterminals(std::list *nt_list); + void replace_nonterminal(Symbol::NT *find, Symbol::NT *replace); }; @@ -668,6 +678,8 @@ class Block : public Base { void multi_collect_factors(Runtime::Poly &p); void multi_init_calls(const Runtime::Poly &p, size_t base_tracks); + void get_nonterminals(std::list *nt_list); + void replace_nonterminal(Symbol::NT *find, Symbol::NT *replace); }; @@ -738,6 +750,8 @@ class Multi : public Base { void types(std::list< ::Type::Base*> &) const; const std::list &ret_decls() const; void init_ret_decl(unsigned int i, const std::string &prefix); + void get_nonterminals(std::list *nt_list); + void replace_nonterminal(Symbol::NT *find, Symbol::NT *replace); }; From 3d1e5e297dbab47804558fe135e33a388765d30b Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 11 Mar 2022 16:48:05 +0100 Subject: [PATCH 004/209] adding a new function to inject novel outside non-terminals into user defined grammar --- src/grammar.cc | 80 ++++++++++++++++++++++++++++++++++++++++++++++++-- src/grammar.hh | 4 +++ 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/src/grammar.cc b/src/grammar.cc index cc857b273..6dd07185c 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -99,7 +99,7 @@ void Grammar::move_new_nts() { /* * Annotates each nonterminal that was marked in the source code - * with the "tabultated" keyword internally and adds it to the + * with the "tabulated" keyword internally and adds it to the * table of tabulated non-terminals (Grammar.tabulated). * This method also does some integrity checks on the source data * structures this method iterates through. @@ -338,7 +338,7 @@ void Grammar::multi_propagate_max_filter() { (*i)->update_max_ys(*j); } - // defines the cisitor used a few lines later to set the max size + // defines the visitor used a few lines later to set the max size struct Multi_Max_Visitor : public Visitor { void visit(Alt::Base &b) { b.multi_set_max_size(); @@ -429,7 +429,7 @@ bool Grammar::init_tracks() { /* - * Checks the semantik of the grammar. This includes the following + * Checks the semantic of the grammar. This includes the following * things: * 1) setting the axiom by resolving the axiom name to a pointer * that is of type Symbol::NT* @@ -999,3 +999,77 @@ void Grammar::remove(Symbol::NT *x) { NTs.erase(nt); tabulated.erase(nt); } + +void Grammar::inject_outside_nts() { + std::string OUTSIDE_NT_PREFIX = "outside_"; + + std::set *outside_nts = new std::set(); + + // 1) collect non-terminals used in rhs of productions together with their calling lhs non-terminal + // we thus exclude e.g. non-terminal foo in production "foo = BASE;" + for (hashtable::iterator i = NTs.begin(); i != NTs.end(); ++i) { + Symbol::NT *nt = dynamic_cast(i->second); + if (nt) { // skip terminals + for (std::list::iterator alt = nt->alts.begin(); alt != nt->alts.end(); ++alt) { + std::list *rhs_nts = new std::list(); + (*alt)->get_nonterminals(rhs_nts); + if (rhs_nts->size() > 0) { + // if rhs uses at least one non-terminal, both (= calling lhs non-terminal and all rhs non-terminal(s)) will be added as novel outside non-terminals + outside_nts->insert(nt); + outside_nts->insert(rhs_nts->begin(), rhs_nts->end()); + } + } + } + } + + // 2) create a novel outside non-terminal for each collected non-terminal in the inside grammar + // add both (new outside and user provided inside) non-terminal **names** as keys to the hash, + // pointing to the very same novel outside non-terminal. + hashtable outside_NTs; + for (Symbol::NT* nt : *outside_nts) { + Symbol::NT *outside_nt = new Symbol::NT(new std::string(OUTSIDE_NT_PREFIX + (*nt->name)), *(new Loc())); + // carry over evaluation function from inside to outside NT + Symbol::NT *inside_nt = dynamic_cast(this->NTs.find(*nt->name)->second); + outside_nt->set_eval_fn(inside_nt->eval_fn); + + outside_NTs[OUTSIDE_NT_PREFIX + (*nt->name)] = outside_nt; + outside_NTs[(*nt->name)] = outside_nt; + } + + // 3) create copy of inside alternative, replace called non-terminal with calling outside version and append to called outside version + for (hashtable::iterator i = NTs.begin(); i != NTs.end(); ++i) { + Symbol::NT *nt = dynamic_cast(i->second); + if (nt) { // skip terminals + for (std::list::iterator alt = nt->alts.begin(); alt != nt->alts.end(); ++alt) { + std::list *rhs_nts = new std::list(); + (*alt)->get_nonterminals(rhs_nts); + if (rhs_nts->size() > 0) { + for (Symbol::NT *nt : *rhs_nts) { + // create a copy (=clone) of the inside alternative + Alt::Base *outside_alt = (*alt)->clone(); + // replace one of the inside non-terminals with the outside version of the calling non-terminal + outside_alt->replace_nonterminal(nt, dynamic_cast(outside_NTs.find(i->first)->second)); + // append copied+replaced alternative to the list of alternatives for the outside version of the called non-terminal + (dynamic_cast(outside_NTs.find(*(nt->name))->second))->alts.push_back(outside_alt); + } + } else { + // e.g. struct = nil(LOC) has no non-terminal on the rhs, but should become an alternative for outside version of struct + (dynamic_cast(outside_NTs.find(*(nt->name))->second))->alts.push_back((*alt)->clone()); + } + } + } + } + + // 4) add novel outside NTs to grammar + for (std::pair nt : outside_NTs) { + if (this->NTs.find(nt.first) == this->NTs.end()) { // only for NTs that are not already part of the grammar + this->NTs.insert(nt); + this->nt_list.push_back(dynamic_cast(nt.second)); + } + } + + // 5) TODO(smj): how to determine new axiom? + this->axiom_name = new std::string("outside_hairpin"); + this->init_axiom(); + //this->axiom = dynamic_cast(this->NTs.find(*(this->axiom_name))->second); +} diff --git a/src/grammar.hh b/src/grammar.hh index 1efc8f9a5..b64d79502 100644 --- a/src/grammar.hh +++ b/src/grammar.hh @@ -205,6 +205,10 @@ class Grammar { bool multi_detect_loops(); void multi_propagate_max_filter(); + + // inject additional non-terminals to grammar, to automatically create + // an outside version for user provided grammar + void inject_outside_nts(); }; From d3e382d04f1afc02e62599e051a5556d941429a1 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 14 Mar 2022 21:08:27 +0100 Subject: [PATCH 005/209] found a better way to inject outside non-terminals --- src/gapc.cc | 6 +----- src/grammar.cc | 14 +++++++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/gapc.cc b/src/gapc.cc index 3616cd64b..08ffd7950 100644 --- a/src/gapc.cc +++ b/src/gapc.cc @@ -350,11 +350,7 @@ class Main { // inject rules for outside grammar grammar->inject_outside_nts(); - // repeat semantic checking (and linking) for the newly generated outside non-terminals - bool r2 = grammar->check_semantic(); - if (!r2) { - throw LogError("Seen semantic errors for outside version."); - } + // configure the window and k-best mode driver.ast.set_window_mode(opts.window_mode); diff --git a/src/grammar.cc b/src/grammar.cc index 6dd07185c..0256fedc9 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1027,10 +1027,14 @@ void Grammar::inject_outside_nts() { // pointing to the very same novel outside non-terminal. hashtable outside_NTs; for (Symbol::NT* nt : *outside_nts) { - Symbol::NT *outside_nt = new Symbol::NT(new std::string(OUTSIDE_NT_PREFIX + (*nt->name)), *(new Loc())); - // carry over evaluation function from inside to outside NT + // use inside NT as template ... Symbol::NT *inside_nt = dynamic_cast(this->NTs.find(*nt->name)->second); - outside_nt->set_eval_fn(inside_nt->eval_fn); + // and clone for outside version + Symbol::NT *outside_nt = inside_nt->clone(inside_nt->track_pos()); + // remove all existing alternative production rules + outside_nt->alts.clear(); + // and correct NT name, to now point to outside version + outside_nt->name = new std::string(OUTSIDE_NT_PREFIX + (*nt->name)); outside_NTs[OUTSIDE_NT_PREFIX + (*nt->name)] = outside_nt; outside_NTs[(*nt->name)] = outside_nt; @@ -1069,7 +1073,7 @@ void Grammar::inject_outside_nts() { } // 5) TODO(smj): how to determine new axiom? - this->axiom_name = new std::string("outside_hairpin"); + //this->axiom_name = new std::string("outside_hairpin"); + this->axiom_name = new std::string("outside_struct"); this->init_axiom(); - //this->axiom = dynamic_cast(this->NTs.find(*(this->axiom_name))->second); } From 43e21ad35dbe2912319f31b2793f0f47e0c445fa Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 21 Mar 2022 15:15:56 +0100 Subject: [PATCH 006/209] just one rhs non-terminal is replaced into outside version --- src/alt.cc | 68 ++++++++++++++++++++++++++++++++++++++++++-------- src/alt.hh | 23 +++++++++++++---- src/grammar.cc | 27 ++++++++++++++------ src/symbol.hh | 3 +++ 4 files changed, 97 insertions(+), 24 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 6d21321c4..d7aa8c3c6 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -58,7 +58,7 @@ Alt::Base::Base(Type t, const Loc &l) : terminal_type(false), location(l), ret_decl(NULL), filter_guards(NULL), choice_fn_type_(Expr::Fn_Call::NONE), - tracks_(0), track_pos_(0) { + tracks_(0), track_pos_(0), is_partof_outside(false) { } @@ -2963,24 +2963,65 @@ void Alt::Multi::get_nonterminals(std::list *nt_list) { throw LogError("Alt::Multi::get_nonterminals is not yet implemented!"); } +// recursively flag this alternative as being part of an inside (=false=default) +// or outside (=true) production rule +void Alt::Base::set_partof_outside(bool is_outside) { +} +void Alt::Simple::set_partof_outside(bool is_outside) { + this->is_partof_outside = is_outside; + for (std::list::iterator i = args.begin(); i != args.end(); ++i) { + Fn_Arg::Alt *arg_alt = dynamic_cast(*i); + if (arg_alt) { + arg_alt->alt->set_partof_outside(is_outside); + } + } +} +void Alt::Link::set_partof_outside(bool is_outside) { + if (this->nt) { + if (this->nt->is(Symbol::NONTERMINAL)) { + // end recursion on non-terminal call + } else if (this->nt->is(Symbol::TERMINAL)) { + this->is_partof_outside = is_outside; + } + } else { + throw LogError("Link is something else"); + } +} +void Alt::Block::set_partof_outside(bool is_outside) { + throw LogError("Alt::Block::set_partof_outside is not yet implemented!"); +} +void Alt::Multi::set_partof_outside(bool is_outside) { + throw LogError("Alt::Multi::set_partof_outside is not yet implemented!"); +} + -void Alt::Base::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace) { +bool Alt::Base::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences) { + return false; } -void Alt::Simple::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace) { +bool Alt::Simple::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences) { for (Fn_Arg::Base *arg : this->args) { Fn_Arg::Alt *arg_alt = dynamic_cast(arg); if (arg_alt) { - arg_alt->alt->replace_nonterminal(find, replace); + if (arg_alt->alt->replace_nonterminal(find, replace, skip_occurences)) { + return true; + } } } + return false; } -void Alt::Link::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace) { +bool Alt::Link::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences) { if (this->nt) { if (this->nt->is(Symbol::NONTERMINAL)) { if (this->nt == find) { - // replace old NT (=find) with novel NT (=replace) - this->nt = replace; - this->name = replace->name; + if (skip_occurences == 0) { + // replace old NT (=find) with novel NT (=replace) + this->nt = replace; + this->name = replace->name; + this->is_partof_outside = replace->is_partof_outside; + return true; + } else { + skip_occurences--; + } } } else if (this->nt->is(Symbol::TERMINAL)) { @@ -2988,13 +3029,18 @@ void Alt::Link::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace) { } else { throw LogError("Link is something else"); } + return false; } -void Alt::Block::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace) { +bool Alt::Block::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences) { for (Alt::Base *alt : this->alts) { // recurse into alternatives - alt->replace_nonterminal(find, replace); + if (alt->replace_nonterminal(find, replace, skip_occurences)) { + return true; + } } + return false; } -void Alt::Multi::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace) { +bool Alt::Multi::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences) { throw LogError("Alt::Multi::replace_nonterminal is not yet implemented!"); + return false; } diff --git a/src/alt.hh b/src/alt.hh index 02175eb0b..f0404093b 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -313,8 +313,17 @@ class Base { // e.g. ml_comps = cadd(incl(dangle), ml_comps1) should return [*dangle, *ml_comps1] virtual void get_nonterminals(std::list *nt_list); + // flag alternative as being part of a generated outside non-terminal + // such that code generation can discriminate between inside (default) + // and outside. + virtual void set_partof_outside(bool is_outside); // traverses the alternative (=rhs of a production), iff non-terminal find is contained, replace with first occurence of non-terminal replace - virtual void replace_nonterminal(Symbol::NT *find, Symbol::NT *replace); + virtual bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences); + + protected: + // private flag to indicate if alternative is for inside (the default) or outside + // production rules. See set_partof_outside() + bool is_partof_outside; }; @@ -487,7 +496,8 @@ class Simple : public Base { public: void set_ntparas(std::list *l); void get_nonterminals(std::list *nt_list); - void replace_nonterminal(Symbol::NT *find, Symbol::NT *replace); + void set_partof_outside(bool is_outside); + bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences); private: std::list *insert_index_stmts( @@ -610,7 +620,8 @@ class Link : public Base { void optimize_choice(); void get_nonterminals(std::list *nt_list); - void replace_nonterminal(Symbol::NT *find, Symbol::NT *replace); + void set_partof_outside(bool is_outside); + bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences); }; @@ -679,7 +690,8 @@ class Block : public Base { void multi_collect_factors(Runtime::Poly &p); void multi_init_calls(const Runtime::Poly &p, size_t base_tracks); void get_nonterminals(std::list *nt_list); - void replace_nonterminal(Symbol::NT *find, Symbol::NT *replace); + void set_partof_outside(bool is_outside); + bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences); }; @@ -751,7 +763,8 @@ class Multi : public Base { const std::list &ret_decls() const; void init_ret_decl(unsigned int i, const std::string &prefix); void get_nonterminals(std::list *nt_list); - void replace_nonterminal(Symbol::NT *find, Symbol::NT *replace); + void set_partof_outside(bool is_outside); + bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences); }; diff --git a/src/grammar.cc b/src/grammar.cc index 0256fedc9..06cdda7e6 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1003,7 +1003,7 @@ void Grammar::remove(Symbol::NT *x) { void Grammar::inject_outside_nts() { std::string OUTSIDE_NT_PREFIX = "outside_"; - std::set *outside_nts = new std::set(); + std::set outside_nts = std::set(); // 1) collect non-terminals used in rhs of productions together with their calling lhs non-terminal // we thus exclude e.g. non-terminal foo in production "foo = BASE;" @@ -1015,8 +1015,8 @@ void Grammar::inject_outside_nts() { (*alt)->get_nonterminals(rhs_nts); if (rhs_nts->size() > 0) { // if rhs uses at least one non-terminal, both (= calling lhs non-terminal and all rhs non-terminal(s)) will be added as novel outside non-terminals - outside_nts->insert(nt); - outside_nts->insert(rhs_nts->begin(), rhs_nts->end()); + outside_nts.insert(nt); + outside_nts.insert(rhs_nts->begin(), rhs_nts->end()); } } } @@ -1026,7 +1026,7 @@ void Grammar::inject_outside_nts() { // add both (new outside and user provided inside) non-terminal **names** as keys to the hash, // pointing to the very same novel outside non-terminal. hashtable outside_NTs; - for (Symbol::NT* nt : *outside_nts) { + for (Symbol::NT* nt : outside_nts) { // use inside NT as template ... Symbol::NT *inside_nt = dynamic_cast(this->NTs.find(*nt->name)->second); // and clone for outside version @@ -1036,6 +1036,9 @@ void Grammar::inject_outside_nts() { // and correct NT name, to now point to outside version outside_nt->name = new std::string(OUTSIDE_NT_PREFIX + (*nt->name)); + // flag this new non-terminal as being part of the outside rules + outside_nt->is_partof_outside = true; + outside_NTs[OUTSIDE_NT_PREFIX + (*nt->name)] = outside_nt; outside_NTs[(*nt->name)] = outside_nt; } @@ -1048,17 +1051,24 @@ void Grammar::inject_outside_nts() { std::list *rhs_nts = new std::list(); (*alt)->get_nonterminals(rhs_nts); if (rhs_nts->size() > 0) { - for (Symbol::NT *nt : *rhs_nts) { + unsigned int skip_occurences = 0; + for (Symbol::NT *rhs_nt : *rhs_nts) { // create a copy (=clone) of the inside alternative Alt::Base *outside_alt = (*alt)->clone(); // replace one of the inside non-terminals with the outside version of the calling non-terminal - outside_alt->replace_nonterminal(nt, dynamic_cast(outside_NTs.find(i->first)->second)); + outside_alt->replace_nonterminal(rhs_nt, dynamic_cast(outside_NTs.find(i->first)->second), skip_occurences); + // flag outside_alt as being part of an automatically generated outside production rule + outside_alt->set_partof_outside(true); // append copied+replaced alternative to the list of alternatives for the outside version of the called non-terminal - (dynamic_cast(outside_NTs.find(*(nt->name))->second))->alts.push_back(outside_alt); + (dynamic_cast(outside_NTs.find(*(rhs_nt->name))->second))->alts.push_back(outside_alt); + skip_occurences++; } } else { // e.g. struct = nil(LOC) has no non-terminal on the rhs, but should become an alternative for outside version of struct - (dynamic_cast(outside_NTs.find(*(nt->name))->second))->alts.push_back((*alt)->clone()); + Alt::Base *outside_alt = (*alt)->clone(); + // flag outside_alt as being part of an automatically generated outside production rule + outside_alt->set_partof_outside(true); + (dynamic_cast(outside_NTs.find(*(nt->name))->second))->alts.push_back(outside_alt); } } } @@ -1077,3 +1087,4 @@ void Grammar::inject_outside_nts() { this->axiom_name = new std::string("outside_struct"); this->init_axiom(); } + diff --git a/src/symbol.hh b/src/symbol.hh index 7f82bc9c3..4c2b25af2 100644 --- a/src/symbol.hh +++ b/src/symbol.hh @@ -248,6 +248,9 @@ class Base { virtual bool multi_detect_loop(const Yield::Multi &left, const Yield::Multi &right, Symbol::NT *nt); virtual bool multi_detect_loop(); + + // flag to indicate if alternative is for inside (the default) or outside production rules. + bool is_partof_outside; }; From 7f0658b5f1cce73a01deb2b1c7d2c178d2d10ad4 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 21 Mar 2022 15:25:19 +0100 Subject: [PATCH 007/209] fix version conflict leftover --- src/alt.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/alt.cc b/src/alt.cc index 37be5e346..2babe8b88 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -2929,7 +2929,6 @@ bool Alt::Base::choice_set() { return datatype->simple()->is(::Type::LIST) && eval_nullary_fn; } -<<<<<<< HEAD void Alt::Base::get_nonterminals(std::list *nt_list) { } From 11a43a453d2ba2cd4a1ac4d46dcf9eedede9f7c8 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 28 Mar 2022 11:51:36 +0200 Subject: [PATCH 008/209] distinguish between inside and outside NTs --- src/symbol.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/symbol.cc b/src/symbol.cc index d450f6fe1..fbc61c20d 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -750,8 +750,13 @@ void Symbol::NT::init_indices(Expr::Vacc *left, Expr::Vacc *right, left_indices[track] = left; right_indices[track] = right; - for (std::list::iterator i = alts.begin(); i != alts.end(); ++i) - (*i)->init_indices(left, right, k, track); + for (std::list::iterator i = alts.begin(); i != alts.end(); ++i) { + if (this->is_partof_outside) { + parser_indices *pind = (*i)->init_indices_outside(left, right, k, track, true); + } else { + (*i)->init_indices(left, right, k, track); + } + } } Statement::Base *Symbol::NT::build_return_empty(const Code::Mode &mode) { From b039d09ab48837ecee2f05bd5a96d74b3cc06778 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 28 Mar 2022 11:52:22 +0200 Subject: [PATCH 009/209] handle edge case --- src/grammar.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/grammar.cc b/src/grammar.cc index 137ddd36a..f0fd058aa 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1026,6 +1026,11 @@ void Grammar::inject_outside_nts() { // add both (new outside and user provided inside) non-terminal **names** as keys to the hash, // pointing to the very same novel outside non-terminal. hashtable outside_NTs; + if (outside_nts.size() == 0) { + // the grammar does not have any non-terminals on the rhs of productions. + // Thus, no new NTs have to be injected + return; + } for (Symbol::NT* nt : outside_nts) { // use inside NT as template ... Symbol::NT *inside_nt = dynamic_cast(this->NTs.find(*nt->name)->second); From 5d3db1ba1f20d2196718e8734aa53b2b5ef82016 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 28 Mar 2022 11:53:19 +0200 Subject: [PATCH 010/209] first version to distribute moving boundaries for outside NTs yield size is not used yet! no loops or guards! --- src/alt.cc | 182 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/alt.hh | 45 ++++++++++++- 2 files changed, 226 insertions(+), 1 deletion(-) diff --git a/src/alt.cc b/src/alt.cc index 2babe8b88..f4a7c1522 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3051,6 +3051,7 @@ void to_dot_indices(std::vector indices, std::ostream &out) { out << ""; for (std::vector::const_iterator track = indices.begin(); track != indices.end(); ++track) { + assert(*track != NULL); (*track)->put(out); if (std::next(track) != indices.end()) { out << "
"; @@ -3187,3 +3188,184 @@ unsigned int Alt::Multi::to_dot(unsigned int *nodeID, std::ostream &out) { } // END functions produce graphViz code to represent the grammar + +// traverses an alternative, collects all non-terminals and returns the one outside non-terminal if present, otherwise NULL +Symbol::NT *get_outside_NT(Alt::Base *alt) { + std::list *nt_list = new std::list(); + alt->get_nonterminals(nt_list); + for (std::list::iterator i = nt_list->begin(); i != nt_list->end(); ++i) { + //std::cerr << *((*i)->name) << "\n"; + if ((*i)->is_partof_outside == false) { + // i-- because we need to make sure not to destroy the iterator + // we are using at this moment: + // https://www.techiedelight.com/remove-elements-list-iterating-cpp/ + nt_list->erase(i--); + } + } + assert(nt_list->size() <= 1); // by design of the outside NT injection, there can at most be ONE outside-nt on any rhs of a production! + if (nt_list->size() == 1) { + return nt_list->front(); + } + return NULL; +} + +Expr::Base *Alt::Simple::get_next_var_right2left(unsigned &k, size_t track) { + std::ostringstream o; + o << "t_" << track << "_k_" << k; + k++; + Expr::Vacc *ivar = new Expr::Vacc(new std::string(o.str())); + + return ivar; +} +Expr::Base *Alt::Simple::get_next_var_left2right(unsigned &k, size_t track) { + std::ostringstream o; + o << "t_" << track << "_k_" << k; + k++; + Expr::Vacc *ivar = new Expr::Vacc(new std::string(o.str())); + + return ivar; +} + +void Alt::Base::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { + this->left_indices[track] = left; + this->right_indices[track] = right; +} +void Alt::Simple::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { + // 1) identify outside non-terminal + for (std::list::const_iterator i = args.begin(); i != args.end(); ++i) { + Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); + if (ont) { + // e.g. outside_iloop + Alt::Link *link = dynamic_cast((*i)->alt_ref()); + if (link) { + if (link->nt == ont) { + // the link->nt could also be a terminal + Symbol::NT *check = dynamic_cast(link->nt); + assert(check); + // we have identified this fn_arg as the one being the outside non-terminal, + // 2) let's expand its indices + link->expand_outside_nt_indices(left, right, track); + // we are done + return; + } + } + // e.g. incl(outside_iloop) + Alt::Simple *algfct = dynamic_cast((*i)->alt_ref()); + if (algfct) { + algfct->expand_outside_nt_indices(left, right, track); + } + } + } +} +void Alt::Link::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { + Alt::Base::expand_outside_nt_indices(left, right, track); +} +void Alt::Block::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { + throw LogError("Alt::Block::expand_outside_nt_indices is not yet implemented!"); +} +void Alt::Multi::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { + throw LogError("Alt::Block::expand_outside_nt_indices is not yet implemented!"); +} + +parser_indices *Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT) { + Alt::Base::init_indices(left, right, k, track); + parser_indices *res = new parser_indices(left, right); + return res; +} + +parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT) { + /* We need to set the moving boundaries for outside non terminals. As opposed to the inside version, + * we here have to start in the inside and work towards left and right ends, e.g. + * foo( BASE, REGION, bar, REGION, BASE ) + * i i+1 k0 k1 k1+1 j <-- inside + * k0-1 k0 i j k1 k1+1 <-- outside + * Note that bar itself can either be a non-terminal or a nested argument, e.g. + * foo(BASE, incl(bar), BASE) where incl(bar) has to be initialized BEFOR + * we can initialize foo(BASE, x, BASE) + * + * 1) Let's first iterate through the arguments and store them in: + * - args left of the outside NT + * - the arg containing the outside NT + * - args right of the outside NT + */ + std::list *args_left = new std::list(); + std::list *args_right = new std::list(); + Fn_Arg::Base *arg_outside = NULL; // = the argument containing the outside non-terminal somewhere (mutually exclusive to nt_outside) + Symbol::NT *nt_outside = NULL; // = the final outside non terminal (mutually exclusive to arg_outside) + for (std::list::const_iterator i = args.begin(); i != args.end(); ++i) { + Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); + if (ont) { + Alt::Link *link = dynamic_cast((*i)->alt_ref()); + if (link) { + if (link->nt == ont) { + // the link->nt could also be a terminal + Symbol::NT *check = dynamic_cast(link->nt); + assert(check); + nt_outside = check; + } + } + arg_outside = *i; + continue; + } + if (arg_outside == NULL) { + args_left->push_back(*i); + } else { + args_right->push_back(*i); + } + } + + // 2a) if the outside non terminal is deeper into an argument, recurse into this arg first + // foo( BASE, REGION, incl(bar), REGION, BASE ) + // ^^^^ + parser_indices *pind_sub = new parser_indices(left, right); + if (arg_outside) { + pind_sub = arg_outside->alt_ref()->init_indices_outside(left, right, k, track, false); + } + + // 2b) start left of the outside non-terminal and initialize indices from right to left + // foo( BASE, REGION, bar, REGION, BASE ) + // <-- ^^^^^^ X + Expr::Base *right_index = pind_sub->var_left; + for (std::list::const_reverse_iterator i = args_left->rbegin(); i != args_left->rend(); ++i) { + Expr::Base *left_index = get_next_var_right2left(k, track); + parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, false); + // prepare for next element: right = left + right_index = pind->var_left; + } + + // 2c) same as above, but start right of outside non-terminal + // foo( BASE, REGION, bar, REGION, BASE ) + // X ^^^^^^ --> + Expr::Base *left_index = pind_sub->var_right; + for (std::list::const_iterator i = args_right->begin(); i != args_right->end(); ++i) { + Expr::Base *right_index = get_next_var_left2right(k, track); + parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, false); + // prepare for next element: left = right + left_index = pind->var_right; + } + + this->expand_outside_nt_indices(right_index, left_index, track); + // 3) set indices for THIS + // foo( BASE, REGION, bar, REGION, BASE ) + // ^^^ +// if (called_from_lhsNT) { +// parser_indices *pind_this = Alt::Base::init_indices_outside(left, right, k, track, false); +// return pind_this; +// } else { + parser_indices *pind_this = Alt::Base::init_indices_outside(right_index, left_index, k, track, false); + return pind_this; +// } +} +parser_indices *Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT) { + parser_indices *pind = Alt::Base::init_indices_outside(left, right, k, track, false); + return pind; +} +parser_indices *Alt::Block::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT) { + throw LogError("Alt::Block::init_indices_outside is not yet implemented!"); + return NULL; +} +parser_indices *Alt::Multi::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT) { + throw LogError("Alt::Multi::init_indices_outside is not yet implemented!"); + return NULL; +} + diff --git a/src/alt.hh b/src/alt.hh index b7a4c5fa3..981567a00 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -56,6 +56,25 @@ class Signature_Base; class Visitor; class Fn_Decl; +struct parser_indices { + // the current variable for leftmost index, e.g. t_0_i + Expr::Base *var_left; + // the current variable for rightmost index, e.g. t_0_j + Expr::Base *var_right; + + // accumulated yield size for constant parser on the left of non-terminal parser + // e.g. foo(x, BASE, BASE, nt, ...) if "nt" is the flagged (outside) non-terminal, + // we "know" that BASE and BASE each consumes exactly 1 char, i.e. 2 in total + // which can guard running indices for x parser + Yield::Size *ys_left; + // as above, but for right of nt + Yield::Size *ys_right; + + parser_indices(Expr::Base *left, Expr::Base *right) : ys_left(), ys_right() { + this->var_left = left; + this->var_right = right; + } +}; namespace Alt { /* @@ -219,6 +238,14 @@ class Base { virtual void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); + // recurses into alternatives, finds the single outside NT and updates only its left and right indices + // used after init_indices_outside initializes all other incides + // e.g. foo(k1_REGION_i, i_bar_j, j_REGION_k2) --> foo(k1_REGION_i, k1_bar_k2, j_REGION_k2) + // ^ ^ ^^ ^^ + virtual void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); + // sets indices for outside rules, respecting yield sizes + virtual parser_indices *init_indices_outside( + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT); virtual void init_ret_decl(unsigned int i, const std::string &prefix); @@ -321,7 +348,8 @@ class Base { virtual bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences); virtual unsigned int to_dot(unsigned int *nodeID, std::ostream &out); - + // returns either the single outside non-terminal or NULL + Symbol::NT *get_outside_nt(Alt::Base *alt); protected: // private flag to indicate if alternative is for inside (the default) or outside // production rules. See set_partof_outside() @@ -440,6 +468,11 @@ class Simple : public Base { public: void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); + Expr::Base *get_next_var_right2left(unsigned &k, size_t track); + Expr::Base *get_next_var_left2right(unsigned &k, size_t track); + void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); + parser_indices *init_indices_outside( + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT); void put_indices(std::ostream &s); void reset(); @@ -578,6 +611,9 @@ class Link : public Base { void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); + void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); + parser_indices *init_indices_outside( + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT); // void init_ret_decl(unsigned int i); @@ -678,6 +714,9 @@ class Block : public Base { void traverse(Visitor &v); void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); + void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); + parser_indices *init_indices_outside( + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT); void codegen(AST &ast); @@ -751,6 +790,10 @@ class Multi : public Base { void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); + void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); + parser_indices *init_indices_outside( + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT); + void codegen(AST &ast); void print_dot_edge(std::ostream &out, Symbol::NT &nt); void print(std::ostream &s); From c496a3dd9164e6c5ca5b2beb31aa9e3686ff42e0 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 28 Mar 2022 19:22:05 +0200 Subject: [PATCH 011/209] propagate yield sizes for moving boundaries --- src/alt.cc | 62 +++++++++++++++++++++++++++++++++++++----------------- src/alt.hh | 4 ++-- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index f4a7c1522..cdd6c5f30 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3209,21 +3209,46 @@ Symbol::NT *get_outside_NT(Alt::Base *alt) { return NULL; } -Expr::Base *Alt::Simple::get_next_var_right2left(unsigned &k, size_t track) { +Expr::Base *get_next_var_name(unsigned &k, size_t track) { std::ostringstream o; o << "t_" << track << "_k_" << k; k++; Expr::Vacc *ivar = new Expr::Vacc(new std::string(o.str())); - return ivar; } -Expr::Base *Alt::Simple::get_next_var_left2right(unsigned &k, size_t track) { - std::ostringstream o; - o << "t_" << track << "_k_" << k; - k++; - Expr::Vacc *ivar = new Expr::Vacc(new std::string(o.str())); +Expr::Base *Alt::Simple::get_next_var_right2left(Expr::Base *left_index, unsigned &k, size_t track, Yield::Size ys_this) { + if (ys_this.low() == ys_this.high()) { + // constant yield size + return left_index->minus(ys_this.low()); + } - return ivar; + // variable yield size + Expr::Base *next_index = get_next_var_name(k, track); + + +// std::pair index(0, 0); +// index.second = new Expr::Base(Expr::Const(555)); +// Statement::Var_Decl *loopvariable = new Statement::Var_Decl( +// new ::Type::Size(), next_index, index.first); +// // flag this variable as being an iterator e.g. in for-loops, +// // such that it won't have a trailing indent for code generation +// loopvariable->set_itr(true); +// Expr::Base *cond = new Expr::Less_Eq(next_index, index.second); +// Statement::For *f = new Statement::For (loopvariable, cond); +// loops.push_back(f); + + return next_index; +} +Expr::Base *Alt::Simple::get_next_var_left2right(Expr::Base *right_index, unsigned &k, size_t track, Yield::Size ys_this) { + if (right_index == NULL) { + return get_next_var_name(k, track); + } + + if (ys_this.low() == ys_this.high()) { + return right_index->plus(ys_this.low()); + } else { + return get_next_var_name(k, track); + } } void Alt::Base::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { @@ -3326,35 +3351,34 @@ parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base * // foo( BASE, REGION, bar, REGION, BASE ) // <-- ^^^^^^ X Expr::Base *right_index = pind_sub->var_left; + Expr::Base *left_index = pind_sub->var_left; for (std::list::const_reverse_iterator i = args_left->rbegin(); i != args_left->rend(); ++i) { - Expr::Base *left_index = get_next_var_right2left(k, track); + left_index = get_next_var_right2left(left_index, k, track, (*i)->multi_ys()(track)); parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, false); // prepare for next element: right = left right_index = pind->var_left; } + // before doing 2c, save current left_index as the "leftmost_index" + Expr::Base *leftmost_index = left_index; // 2c) same as above, but start right of outside non-terminal // foo( BASE, REGION, bar, REGION, BASE ) // X ^^^^^^ --> - Expr::Base *left_index = pind_sub->var_right; + left_index = pind_sub->var_right; + right_index = pind_sub->var_right; for (std::list::const_iterator i = args_right->begin(); i != args_right->end(); ++i) { - Expr::Base *right_index = get_next_var_left2right(k, track); + right_index = get_next_var_left2right(right_index, k, track, (*i)->multi_ys()(track)); parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, false); // prepare for next element: left = right left_index = pind->var_right; } - this->expand_outside_nt_indices(right_index, left_index, track); + this->expand_outside_nt_indices(leftmost_index, right_index, track); // 3) set indices for THIS // foo( BASE, REGION, bar, REGION, BASE ) // ^^^ -// if (called_from_lhsNT) { -// parser_indices *pind_this = Alt::Base::init_indices_outside(left, right, k, track, false); -// return pind_this; -// } else { - parser_indices *pind_this = Alt::Base::init_indices_outside(right_index, left_index, k, track, false); - return pind_this; -// } + parser_indices *pind_this = Alt::Base::init_indices_outside(leftmost_index, right_index, k, track, false); + return pind_this; } parser_indices *Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT) { parser_indices *pind = Alt::Base::init_indices_outside(left, right, k, track, false); diff --git a/src/alt.hh b/src/alt.hh index 981567a00..3c53cf593 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -468,8 +468,8 @@ class Simple : public Base { public: void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); - Expr::Base *get_next_var_right2left(unsigned &k, size_t track); - Expr::Base *get_next_var_left2right(unsigned &k, size_t track); + Expr::Base *get_next_var_right2left(Expr::Base *left_index, unsigned &k, size_t track, Yield::Size ys_this); + Expr::Base *get_next_var_left2right(Expr::Base *right_index, unsigned &k, size_t track, Yield::Size ys_this); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); parser_indices *init_indices_outside( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT); From 6216d57e3220e7bee980141e43751c3b95aa2205 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 29 Mar 2022 12:45:16 +0200 Subject: [PATCH 012/209] removed unused called_from_lhsNT --- src/alt.cc | 20 ++++++++++---------- src/alt.hh | 10 +++++----- src/symbol.cc | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index cdd6c5f30..424f596ac 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3292,13 +3292,13 @@ void Alt::Multi::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, throw LogError("Alt::Block::expand_outside_nt_indices is not yet implemented!"); } -parser_indices *Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT) { +parser_indices *Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track) { Alt::Base::init_indices(left, right, k, track); parser_indices *res = new parser_indices(left, right); return res; } -parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT) { +parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track) { /* We need to set the moving boundaries for outside non terminals. As opposed to the inside version, * we here have to start in the inside and work towards left and right ends, e.g. * foo( BASE, REGION, bar, REGION, BASE ) @@ -3344,7 +3344,7 @@ parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base * // ^^^^ parser_indices *pind_sub = new parser_indices(left, right); if (arg_outside) { - pind_sub = arg_outside->alt_ref()->init_indices_outside(left, right, k, track, false); + pind_sub = arg_outside->alt_ref()->init_indices_outside(left, right, k, track); } // 2b) start left of the outside non-terminal and initialize indices from right to left @@ -3354,7 +3354,7 @@ parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base * Expr::Base *left_index = pind_sub->var_left; for (std::list::const_reverse_iterator i = args_left->rbegin(); i != args_left->rend(); ++i) { left_index = get_next_var_right2left(left_index, k, track, (*i)->multi_ys()(track)); - parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, false); + parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track); // prepare for next element: right = left right_index = pind->var_left; } @@ -3368,7 +3368,7 @@ parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base * right_index = pind_sub->var_right; for (std::list::const_iterator i = args_right->begin(); i != args_right->end(); ++i) { right_index = get_next_var_left2right(right_index, k, track, (*i)->multi_ys()(track)); - parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, false); + parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track); // prepare for next element: left = right left_index = pind->var_right; } @@ -3377,18 +3377,18 @@ parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base * // 3) set indices for THIS // foo( BASE, REGION, bar, REGION, BASE ) // ^^^ - parser_indices *pind_this = Alt::Base::init_indices_outside(leftmost_index, right_index, k, track, false); + parser_indices *pind_this = Alt::Base::init_indices_outside(leftmost_index, right_index, k, track); return pind_this; } -parser_indices *Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT) { - parser_indices *pind = Alt::Base::init_indices_outside(left, right, k, track, false); +parser_indices *Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track) { + parser_indices *pind = Alt::Base::init_indices_outside(left, right, k, track); return pind; } -parser_indices *Alt::Block::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT) { +parser_indices *Alt::Block::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track) { throw LogError("Alt::Block::init_indices_outside is not yet implemented!"); return NULL; } -parser_indices *Alt::Multi::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT) { +parser_indices *Alt::Multi::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track) { throw LogError("Alt::Multi::init_indices_outside is not yet implemented!"); return NULL; } diff --git a/src/alt.hh b/src/alt.hh index 3c53cf593..0a8273f2f 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -245,7 +245,7 @@ class Base { virtual void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); // sets indices for outside rules, respecting yield sizes virtual parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); virtual void init_ret_decl(unsigned int i, const std::string &prefix); @@ -472,7 +472,7 @@ class Simple : public Base { Expr::Base *get_next_var_left2right(Expr::Base *right_index, unsigned &k, size_t track, Yield::Size ys_this); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); void put_indices(std::ostream &s); void reset(); @@ -613,7 +613,7 @@ class Link : public Base { Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); // void init_ret_decl(unsigned int i); @@ -716,7 +716,7 @@ class Block : public Base { Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); void codegen(AST &ast); @@ -792,7 +792,7 @@ class Multi : public Base { Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, bool called_from_lhsNT); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); void codegen(AST &ast); void print_dot_edge(std::ostream &out, Symbol::NT &nt); diff --git a/src/symbol.cc b/src/symbol.cc index fbc61c20d..2815088bf 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -752,7 +752,7 @@ void Symbol::NT::init_indices(Expr::Vacc *left, Expr::Vacc *right, for (std::list::iterator i = alts.begin(); i != alts.end(); ++i) { if (this->is_partof_outside) { - parser_indices *pind = (*i)->init_indices_outside(left, right, k, track, true); + parser_indices *pind = (*i)->init_indices_outside(left, right, k, track); } else { (*i)->init_indices(left, right, k, track); } From 3be0de70692eb02f1a29211c7d64130d5aa3143b Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 29 Mar 2022 12:50:05 +0200 Subject: [PATCH 013/209] fix comment --- src/alt.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/alt.cc b/src/alt.cc index 424f596ac..86bdcc591 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3373,8 +3373,13 @@ parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base * left_index = pind->var_right; } + // 3) update indices of outside non-terminal such that they reach from leftmost to rightmost + // e.g. foo( REGION, outside_bar, REGION ) + // k0 i j k1 + // --> k0 k0 k1 k1 this->expand_outside_nt_indices(leftmost_index, right_index, track); - // 3) set indices for THIS + + // 4) set indices for THIS // foo( BASE, REGION, bar, REGION, BASE ) // ^^^ parser_indices *pind_this = Alt::Base::init_indices_outside(leftmost_index, right_index, k, track); From 5f834d66cb7e6098f1299d6bf60d6dab5e7814db Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 30 Mar 2022 12:27:57 +0200 Subject: [PATCH 014/209] allow for loop to -- instead of ++ --- src/cpp.cc | 7 ++++++- src/symbol.cc | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 3a03308f1..386bd43b8 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -81,7 +81,12 @@ void Printer::Cpp::print(const Statement::For &stmt) { stream << *stmt.var_decl; stream << ' ' << *stmt.cond << "; "; if (!stmt.inc) { - stream << "++" << *stmt.var_decl->name << ")"; + if (stmt.decrement) { + stream << "--"; + } else { + stream << "++"; + } + stream << *stmt.var_decl->name << ")"; } else { bool t = in_fn_head; in_fn_head = true; diff --git a/src/symbol.cc b/src/symbol.cc index 2815088bf..1c395b54c 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -752,7 +752,9 @@ void Symbol::NT::init_indices(Expr::Vacc *left, Expr::Vacc *right, for (std::list::iterator i = alts.begin(); i != alts.end(); ++i) { if (this->is_partof_outside) { - parser_indices *pind = (*i)->init_indices_outside(left, right, k, track); + Yield::Size *ys_lefts = new Yield::Size(); + Yield::Size *ys_rights = new Yield::Size(); + parser_indices *pind = (*i)->init_indices_outside(left, right, k, track, ys_lefts, ys_rights); } else { (*i)->init_indices(left, right, k, track); } From aaa572a4e5e83cbbe24a3dfee37d65e3954b2eca Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 30 Mar 2022 12:28:35 +0200 Subject: [PATCH 015/209] also for loop --- src/statement.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/statement.hh b/src/statement.hh index 3e519d8ce..c928ce486 100644 --- a/src/statement.hh +++ b/src/statement.hh @@ -272,6 +272,7 @@ class For : public Block_Base { Var_Decl *var_decl; Expr::Base *cond; Statement::Base *inc; + bool decrement = false; For(Var_Decl *v, Expr::Base* e) : Block_Base(FOR), var_decl(v), cond(e), inc(NULL) { From 9e3e3368477db68c13cb01f2647a62869968d615 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 30 Mar 2022 12:29:27 +0200 Subject: [PATCH 016/209] less variables --- src/symbol.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/symbol.cc b/src/symbol.cc index 1c395b54c..536e2849e 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -752,9 +752,7 @@ void Symbol::NT::init_indices(Expr::Vacc *left, Expr::Vacc *right, for (std::list::iterator i = alts.begin(); i != alts.end(); ++i) { if (this->is_partof_outside) { - Yield::Size *ys_lefts = new Yield::Size(); - Yield::Size *ys_rights = new Yield::Size(); - parser_indices *pind = (*i)->init_indices_outside(left, right, k, track, ys_lefts, ys_rights); + parser_indices *pind = (*i)->init_indices_outside(left, right, k, track, new Yield::Size(), new Yield::Size()); } else { (*i)->init_indices(left, right, k, track); } From 0d7563b8814c92bb03ed9010976158b1be45d65a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 30 Mar 2022 12:30:09 +0200 Subject: [PATCH 017/209] outside loops iterate in correct borders (but wrong order) --- src/alt.cc | 100 ++++++++++++++++++++++++++++++++++++----------------- src/alt.hh | 24 ++++++++----- 2 files changed, 85 insertions(+), 39 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 86bdcc591..188e86956 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3216,7 +3216,7 @@ Expr::Base *get_next_var_name(unsigned &k, size_t track) { Expr::Vacc *ivar = new Expr::Vacc(new std::string(o.str())); return ivar; } -Expr::Base *Alt::Simple::get_next_var_right2left(Expr::Base *left_index, unsigned &k, size_t track, Yield::Size ys_this) { +Expr::Base *Alt::Simple::get_next_var_right2left(Expr::Base *left_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_lefts) { if (ys_this.low() == ys_this.high()) { // constant yield size return left_index->minus(ys_this.low()); @@ -3225,30 +3225,40 @@ Expr::Base *Alt::Simple::get_next_var_right2left(Expr::Base *left_index, unsigne // variable yield size Expr::Base *next_index = get_next_var_name(k, track); - -// std::pair index(0, 0); -// index.second = new Expr::Base(Expr::Const(555)); -// Statement::Var_Decl *loopvariable = new Statement::Var_Decl( -// new ::Type::Size(), next_index, index.first); -// // flag this variable as being an iterator e.g. in for-loops, -// // such that it won't have a trailing indent for code generation -// loopvariable->set_itr(true); -// Expr::Base *cond = new Expr::Less_Eq(next_index, index.second); -// Statement::For *f = new Statement::For (loopvariable, cond); -// loops.push_back(f); + // add for loop + std::ostringstream o; + o << "t_" << track << "_left_most"; + Expr::Vacc *left_border = new Expr::Vacc(new std::string(o.str())); + Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), next_index, left_index->minus(ys_this.low())); + loopvariable->set_itr(true); + Expr::Base *cond = new Expr::Greater_Eq(next_index, left_border->plus(ys_lefts->low())); + Statement::For *f = new Statement::For(loopvariable, cond); + f->decrement = true; // --loopvar instead of ++loopvar + loops.push_back(f); return next_index; } -Expr::Base *Alt::Simple::get_next_var_left2right(Expr::Base *right_index, unsigned &k, size_t track, Yield::Size ys_this) { - if (right_index == NULL) { - return get_next_var_name(k, track); - } - +Expr::Base *Alt::Simple::get_next_var_left2right(Expr::Base *right_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_rights) { if (ys_this.low() == ys_this.high()) { + // constant yield size return right_index->plus(ys_this.low()); - } else { - return get_next_var_name(k, track); } + + // variable yield size + Expr::Base *next_index = get_next_var_name(k, track); + + // add for loop + std::ostringstream o; + o << "t_" << track << "_right_most"; + Expr::Vacc *right_border = new Expr::Vacc(new std::string(o.str())); + Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), next_index, right_index->plus(ys_this.low())); + loopvariable->set_itr(true); + Expr::Base *cond = new Expr::Less_Eq(next_index, right_border->minus(ys_rights->low())); + Statement::For *f = new Statement::For(loopvariable, cond); + f->decrement = false; // --loopvar instead of ++loopvar + loops.push_back(f); + + return next_index; } void Alt::Base::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { @@ -3292,13 +3302,13 @@ void Alt::Multi::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, throw LogError("Alt::Block::expand_outside_nt_indices is not yet implemented!"); } -parser_indices *Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track) { +parser_indices *Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights) { Alt::Base::init_indices(left, right, k, track); parser_indices *res = new parser_indices(left, right); return res; } -parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track) { +parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights) { /* We need to set the moving boundaries for outside non terminals. As opposed to the inside version, * we here have to start in the inside and work towards left and right ends, e.g. * foo( BASE, REGION, bar, REGION, BASE ) @@ -3315,6 +3325,8 @@ parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base * */ std::list *args_left = new std::list(); std::list *args_right = new std::list(); + Yield::Size *ys_lefts_this = new Yield::Size(); + Yield::Size *ys_rights_this = new Yield::Size(); Fn_Arg::Base *arg_outside = NULL; // = the argument containing the outside non-terminal somewhere (mutually exclusive to nt_outside) Symbol::NT *nt_outside = NULL; // = the final outside non terminal (mutually exclusive to arg_outside) for (std::list::const_iterator i = args.begin(); i != args.end(); ++i) { @@ -3334,8 +3346,10 @@ parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base * } if (arg_outside == NULL) { args_left->push_back(*i); + *ys_lefts_this += (*i)->multi_ys()(track); } else { args_right->push_back(*i); + *ys_rights_this += (*i)->multi_ys()(track); } } @@ -3344,7 +3358,7 @@ parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base * // ^^^^ parser_indices *pind_sub = new parser_indices(left, right); if (arg_outside) { - pind_sub = arg_outside->alt_ref()->init_indices_outside(left, right, k, track); + pind_sub = arg_outside->alt_ref()->init_indices_outside(left, right, k, track, ys_lefts_this, ys_rights_this); } // 2b) start left of the outside non-terminal and initialize indices from right to left @@ -3353,8 +3367,10 @@ parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base * Expr::Base *right_index = pind_sub->var_left; Expr::Base *left_index = pind_sub->var_left; for (std::list::const_reverse_iterator i = args_left->rbegin(); i != args_left->rend(); ++i) { - left_index = get_next_var_right2left(left_index, k, track, (*i)->multi_ys()(track)); - parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track); + // total yield size left of this argument + ys_lefts_this = sum_ys_lefts(ys_lefts, i, args_left->rend(), track); + left_index = get_next_var_right2left(left_index, k, track, (*i)->multi_ys()(track), ys_lefts_this); + parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, ys_lefts, ys_rights); // prepare for next element: right = left right_index = pind->var_left; } @@ -3367,8 +3383,10 @@ parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base * left_index = pind_sub->var_right; right_index = pind_sub->var_right; for (std::list::const_iterator i = args_right->begin(); i != args_right->end(); ++i) { - right_index = get_next_var_left2right(right_index, k, track, (*i)->multi_ys()(track)); - parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track); + // total yield size left of this argument + ys_rights_this = sum_ys_rights(ys_rights, i, args_right->end(), track); + right_index = get_next_var_left2right(right_index, k, track, (*i)->multi_ys()(track), ys_rights_this); + parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, ys_lefts, ys_rights); // prepare for next element: left = right left_index = pind->var_right; } @@ -3382,19 +3400,39 @@ parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base * // 4) set indices for THIS // foo( BASE, REGION, bar, REGION, BASE ) // ^^^ - parser_indices *pind_this = Alt::Base::init_indices_outside(leftmost_index, right_index, k, track); + parser_indices *pind_this = Alt::Base::init_indices_outside(leftmost_index, right_index, k, track, ys_lefts, ys_rights); + //pind_this->ys_left_upper = &ys_accum_upper; return pind_this; } -parser_indices *Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track) { - parser_indices *pind = Alt::Base::init_indices_outside(left, right, k, track); +parser_indices *Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights) { + parser_indices *pind = Alt::Base::init_indices_outside(left, right, k, track, ys_lefts, ys_rights); return pind; } -parser_indices *Alt::Block::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track) { +parser_indices *Alt::Block::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights) { throw LogError("Alt::Block::init_indices_outside is not yet implemented!"); return NULL; } -parser_indices *Alt::Multi::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track) { +parser_indices *Alt::Multi::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights) { throw LogError("Alt::Multi::init_indices_outside is not yet implemented!"); return NULL; } +Yield::Size *Alt::Simple::sum_ys_lefts(Yield::Size *y, std::list::const_reverse_iterator i, const std::list::const_reverse_iterator &end, size_t track) const { + Yield::Size *res = new Yield::Size(); + *res += *y; + ++i; + for (; i != end; ++i) { + *res += (*i)->multi_ys()(track); + } + return res; +} +Yield::Size *Alt::Simple::sum_ys_rights(Yield::Size *y, std::list::const_iterator i, const std::list::const_iterator &end, size_t track) const { + Yield::Size *res = new Yield::Size(); + *res += *y; + ++i; + for (; i != end; ++i) { + *res += (*i)->multi_ys()(track); + } + return res; +} + diff --git a/src/alt.hh b/src/alt.hh index 0a8273f2f..6f33d8b69 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -70,9 +70,14 @@ struct parser_indices { // as above, but for right of nt Yield::Size *ys_right; - parser_indices(Expr::Base *left, Expr::Base *right) : ys_left(), ys_right() { + Yield::Size *ys_left_upper; + + parser_indices(Expr::Base *left, Expr::Base *right) : ys_left(), ys_right(), ys_left_upper() { this->var_left = left; this->var_right = right; + + this->ys_left_upper = new Yield::Size(); + this->ys_left_upper->set(0, 0); } }; @@ -245,7 +250,7 @@ class Base { virtual void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); // sets indices for outside rules, respecting yield sizes virtual parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights); virtual void init_ret_decl(unsigned int i, const std::string &prefix); @@ -468,11 +473,11 @@ class Simple : public Base { public: void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); - Expr::Base *get_next_var_right2left(Expr::Base *left_index, unsigned &k, size_t track, Yield::Size ys_this); - Expr::Base *get_next_var_left2right(Expr::Base *right_index, unsigned &k, size_t track, Yield::Size ys_this); + Expr::Base *get_next_var_right2left(Expr::Base *left_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_lefts); + Expr::Base *get_next_var_left2right(Expr::Base *right_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_rights); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights); void put_indices(std::ostream &s); void reset(); @@ -499,6 +504,9 @@ class Simple : public Base { Yield::Size &y, std::list::const_iterator i, const std::list::const_iterator &end, size_t track) const; + Yield::Size *sum_ys_lefts(Yield::Size *y, std::list::const_reverse_iterator i, const std::list::const_reverse_iterator &end, size_t track) const; + Yield::Size *sum_ys_rights(Yield::Size *y, std::list::const_iterator i, const std::list::const_iterator &end, size_t track) const; + public: bool multi_detect_loop( @@ -613,7 +621,7 @@ class Link : public Base { Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights); // void init_ret_decl(unsigned int i); @@ -716,7 +724,7 @@ class Block : public Base { Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights); void codegen(AST &ast); @@ -792,7 +800,7 @@ class Multi : public Base { Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights); void codegen(AST &ast); void print_dot_edge(std::ostream &out, Symbol::NT &nt); From 7b033844bc0f672f503c45e639db43e144261d94 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 31 Mar 2022 10:35:09 +0200 Subject: [PATCH 018/209] move uninit std::xx up to fix GDB issues --- src/symbol.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/symbol.cc b/src/symbol.cc index 536e2849e..ed6220491 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -1186,6 +1186,9 @@ struct SetADPDisabled : public Visitor { }; void Symbol::NT::codegen(AST &ast) { + // moving this uninizialized list to the front of the method to avoid issues with gdb within eclipse: https://www.eclipse.org/forums/index.php/t/1109346/ + std::list stmts; + // disable specialisation if needed in backtrace mode SetADPDisabled v = SetADPDisabled(ast.code_mode() == Code::Mode::BACKTRACK && tabulated, ast.code_mode().keep_cooptimal()); @@ -1216,8 +1219,6 @@ void Symbol::NT::codegen(AST &ast) { } f->add_para(*this); - std::list stmts; - subopt_header(ast, score_code, f, stmts); init_guards(ast.code_mode()); From 6b75de6177271e183adc65ebbfbc4fcd8e1871e6 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 31 Mar 2022 15:21:59 +0200 Subject: [PATCH 019/209] top-down, left-right index initialization --- src/alt.cc | 336 ++++++++++++++++++++++++++++++++++++-------------- src/alt.hh | 25 ++-- src/symbol.cc | 2 +- 3 files changed, 257 insertions(+), 106 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 188e86956..063cf4279 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3219,44 +3219,44 @@ Expr::Base *get_next_var_name(unsigned &k, size_t track) { Expr::Base *Alt::Simple::get_next_var_right2left(Expr::Base *left_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_lefts) { if (ys_this.low() == ys_this.high()) { // constant yield size - return left_index->minus(ys_this.low()); + return left_index->plus(ys_this.low()); } // variable yield size Expr::Base *next_index = get_next_var_name(k, track); - // add for loop - std::ostringstream o; - o << "t_" << track << "_left_most"; - Expr::Vacc *left_border = new Expr::Vacc(new std::string(o.str())); - Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), next_index, left_index->minus(ys_this.low())); - loopvariable->set_itr(true); - Expr::Base *cond = new Expr::Greater_Eq(next_index, left_border->plus(ys_lefts->low())); - Statement::For *f = new Statement::For(loopvariable, cond); - f->decrement = true; // --loopvar instead of ++loopvar - loops.push_back(f); +// // add for loop +// std::ostringstream o; +// o << "t_" << track << "_left_most"; +// Expr::Vacc *left_border = new Expr::Vacc(new std::string(o.str())); +// Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), next_index, left_index->minus(ys_this.low())); +// loopvariable->set_itr(true); +// Expr::Base *cond = new Expr::Greater_Eq(next_index, left_border->plus(ys_lefts->low())); +// Statement::For *f = new Statement::For(loopvariable, cond); +// f->decrement = true; // --loopvar instead of ++loopvar +// loops.push_back(f); return next_index; } Expr::Base *Alt::Simple::get_next_var_left2right(Expr::Base *right_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_rights) { if (ys_this.low() == ys_this.high()) { // constant yield size - return right_index->plus(ys_this.low()); + return right_index->minus(ys_this.low()); } // variable yield size Expr::Base *next_index = get_next_var_name(k, track); - // add for loop - std::ostringstream o; - o << "t_" << track << "_right_most"; - Expr::Vacc *right_border = new Expr::Vacc(new std::string(o.str())); - Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), next_index, right_index->plus(ys_this.low())); - loopvariable->set_itr(true); - Expr::Base *cond = new Expr::Less_Eq(next_index, right_border->minus(ys_rights->low())); - Statement::For *f = new Statement::For(loopvariable, cond); - f->decrement = false; // --loopvar instead of ++loopvar - loops.push_back(f); +// // add for loop +// std::ostringstream o; +// o << "t_" << track << "_right_most"; +// Expr::Vacc *right_border = new Expr::Vacc(new std::string(o.str())); +// Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), next_index, right_index->plus(ys_this.low())); +// loopvariable->set_itr(true); +// Expr::Base *cond = new Expr::Less_Eq(next_index, right_border->minus(ys_rights->low())); +// Statement::For *f = new Statement::For(loopvariable, cond); +// f->decrement = false; // --loopvar instead of ++loopvar +// loops.push_back(f); return next_index; } @@ -3302,33 +3302,30 @@ void Alt::Multi::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, throw LogError("Alt::Block::expand_outside_nt_indices is not yet implemented!"); } -parser_indices *Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights) { +void Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { Alt::Base::init_indices(left, right, k, track); - parser_indices *res = new parser_indices(left, right); - return res; } -parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights) { - /* We need to set the moving boundaries for outside non terminals. As opposed to the inside version, - * we here have to start in the inside and work towards left and right ends, e.g. - * foo( BASE, REGION, bar, REGION, BASE ) - * i i+1 k0 k1 k1+1 j <-- inside - * k0-1 k0 i j k1 k1+1 <-- outside - * Note that bar itself can either be a non-terminal or a nested argument, e.g. - * foo(BASE, incl(bar), BASE) where incl(bar) has to be initialized BEFOR - * we can initialize foo(BASE, x, BASE) - * - * 1) Let's first iterate through the arguments and store them in: - * - args left of the outside NT - * - the arg containing the outside NT - * - args right of the outside NT - */ +void printLR(Expr::Base *l, Expr::Base *r) { + std::cerr << "("; + if (l) { + std::cerr << *l; + } else { + std::cerr << "NULL"; + } + std::cerr << ","; + if (r) { + std::cerr << *r; + } else { + std::cerr << "NULL"; + } + std::cerr << ")"; +} +void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { std::list *args_left = new std::list(); std::list *args_right = new std::list(); - Yield::Size *ys_lefts_this = new Yield::Size(); - Yield::Size *ys_rights_this = new Yield::Size(); - Fn_Arg::Base *arg_outside = NULL; // = the argument containing the outside non-terminal somewhere (mutually exclusive to nt_outside) - Symbol::NT *nt_outside = NULL; // = the final outside non terminal (mutually exclusive to arg_outside) + Fn_Arg::Base *arg_outside = NULL; + Symbol::NT *nt_outside = NULL; for (std::list::const_iterator i = args.begin(); i != args.end(); ++i) { Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); if (ont) { @@ -3346,77 +3343,226 @@ parser_indices *Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base * } if (arg_outside == NULL) { args_left->push_back(*i); - *ys_lefts_this += (*i)->multi_ys()(track); } else { args_right->push_back(*i); - *ys_rights_this += (*i)->multi_ys()(track); } } - // 2a) if the outside non terminal is deeper into an argument, recurse into this arg first - // foo( BASE, REGION, incl(bar), REGION, BASE ) - // ^^^^ - parser_indices *pind_sub = new parser_indices(left, right); - if (arg_outside) { - pind_sub = arg_outside->alt_ref()->init_indices_outside(left, right, k, track, ys_lefts_this, ys_rights_this); + + // yield sizes for sub-structure + std::pair *ys_sub = new std::pair(new Yield::Size(), new Yield::Size()); + if (arg_outside != NULL) { + arg_outside->alt_ref()->get_outside_accum_yieldsizes(track); } - // 2b) start left of the outside non-terminal and initialize indices from right to left - // foo( BASE, REGION, bar, REGION, BASE ) - // <-- ^^^^^^ X - Expr::Base *right_index = pind_sub->var_left; - Expr::Base *left_index = pind_sub->var_left; - for (std::list::const_reverse_iterator i = args_left->rbegin(); i != args_left->rend(); ++i) { - // total yield size left of this argument - ys_lefts_this = sum_ys_lefts(ys_lefts, i, args_left->rend(), track); - left_index = get_next_var_right2left(left_index, k, track, (*i)->multi_ys()(track), ys_lefts_this); - parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, ys_lefts, ys_rights); - // prepare for next element: right = left - right_index = pind->var_left; +// std::cerr << "==LEFT==\n"; + Expr::Base *left_index; +// Expr::Base *leftmost_index = NULL; + if (center_left) { + left_index = center_left; + } else { + left_index = get_next_var_name(k, track); +// leftmost_index = left_index; } - // before doing 2c, save current left_index as the "leftmost_index" Expr::Base *leftmost_index = left_index; + Expr::Base *right_index = NULL; + for (std::list::const_iterator i = args_left->begin(); i != args_left->end(); ++i) { + Yield::Size ys_this = (*i)->multi_ys()(track); + Yield::Size *ys_right2center = new Yield::Size(); + *ys_right2center += *(ys_sub->first); + *ys_right2center += *(sum_ys_rights(&ys_this, i, args_left->end(), track)); + if ((std::next(i) == args_left->end()) && (nt_outside != NULL)) { + right_index = left; + } else { + right_index = get_next_var_right2left(left_index, k, track, ys_this, ys_right2center); + } +// std::cerr << " index="; printLR(left_index,right_index); std::cerr << ", center="; printLR(center_left, center_right); std::cerr << ", left-right"; printLR(left, right); std::cerr << "\n"; + (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, center_left, center_right); + left_index = right_index; + } + Expr::Base *lhs_right_index = right_index; + - // 2c) same as above, but start right of outside non-terminal - // foo( BASE, REGION, bar, REGION, BASE ) - // X ^^^^^^ --> - left_index = pind_sub->var_right; - right_index = pind_sub->var_right; - for (std::list::const_iterator i = args_right->begin(); i != args_right->end(); ++i) { - // total yield size left of this argument - ys_rights_this = sum_ys_rights(ys_rights, i, args_right->end(), track); - right_index = get_next_var_left2right(right_index, k, track, (*i)->multi_ys()(track), ys_rights_this); - parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, ys_lefts, ys_rights); - // prepare for next element: left = right - left_index = pind->var_right; +// std::cerr << "==RIGHT=\n"; + if (center_right) { + right_index = center_right; + } else { + right_index = get_next_var_name(k, track); + } + Expr::Base *rightmost_index = right_index; + left_index = NULL; + for (std::list::const_reverse_iterator i = args_right->rbegin(); i != args_right->rend(); ++i) { + Yield::Size ys_this = (*i)->multi_ys()(track); + Yield::Size *ys_left2center = new Yield::Size(); + *ys_left2center += *(ys_sub->second); + *ys_left2center += *(sum_ys_lefts(&ys_this, i, args_right->rend(), track)); + if ((std::next(i) == args_right->rend()) && (nt_outside != NULL)) { + left_index = right; + } else { + left_index = get_next_var_left2right(right_index, k, track, ys_this, ys_left2center); + } +// std::cerr << " index="; printLR(left_index,right_index); std::cerr << ", center="; printLR(center_left, center_right); std::cerr << ", left-right"; printLR(left, right); std::cerr << "\n"; + (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, center_left, center_right); + right_index = left_index; } - // 3) update indices of outside non-terminal such that they reach from leftmost to rightmost - // e.g. foo( REGION, outside_bar, REGION ) - // k0 i j k1 - // --> k0 k0 k1 k1 - this->expand_outside_nt_indices(leftmost_index, right_index, track); + //std::cerr << *this->name; printLR(leftmost_index, rightmost_index); std::cerr << "\n"; + Alt::Base::init_indices_outside(leftmost_index, rightmost_index, k, track, center_left, center_right); - // 4) set indices for THIS - // foo( BASE, REGION, bar, REGION, BASE ) - // ^^^ - parser_indices *pind_this = Alt::Base::init_indices_outside(leftmost_index, right_index, k, track, ys_lefts, ys_rights); - //pind_this->ys_left_upper = &ys_accum_upper; - return pind_this; + if (arg_outside != NULL) { + arg_outside->alt_ref()->init_indices_outside(left, right, k, track, lhs_right_index, right_index); + if (nt_outside == NULL) { + this->expand_outside_nt_indices(leftmost_index, rightmost_index, track); + } + } + + +// /* We need to set the moving boundaries for outside non terminals. As opposed to the inside version, +// * we here have to start in the inside and work towards left and right ends, e.g. +// * foo( BASE, REGION, bar, REGION, BASE ) +// * i i+1 k0 k1 k1+1 j <-- inside +// * k0-1 k0 i j k1 k1+1 <-- outside +// * Note that bar itself can either be a non-terminal or a nested argument, e.g. +// * foo(BASE, incl(bar), BASE) where incl(bar) has to be initialized BEFOR +// * we can initialize foo(BASE, x, BASE) +// * +// * 1) Let's first iterate through the arguments and store them in: +// * - args left of the outside NT +// * - the arg containing the outside NT +// * - args right of the outside NT +// */ +// std::list *args_left = new std::list(); +// std::list *args_right = new std::list(); +// Yield::Size *ys_lefts_this = new Yield::Size(); +// Yield::Size *ys_rights_this = new Yield::Size(); +// Fn_Arg::Base *arg_outside = NULL; // = the argument containing the outside non-terminal somewhere (mutually exclusive to nt_outside) +// Symbol::NT *nt_outside = NULL; // = the final outside non terminal (mutually exclusive to arg_outside) +// for (std::list::const_iterator i = args.begin(); i != args.end(); ++i) { +// Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); +// if (ont) { +// Alt::Link *link = dynamic_cast((*i)->alt_ref()); +// if (link) { +// if (link->nt == ont) { +// // the link->nt could also be a terminal +// Symbol::NT *check = dynamic_cast(link->nt); +// assert(check); +// nt_outside = check; +// } +// } +// arg_outside = *i; +// continue; +// } +// if (arg_outside == NULL) { +// args_left->push_back(*i); +// *ys_lefts_this += (*i)->multi_ys()(track); +// } else { +// args_right->push_back(*i); +// *ys_rights_this += (*i)->multi_ys()(track); +// } +// } +// +// // 2a) if the outside non terminal is deeper into an argument, recurse into this arg first +// // foo( BASE, REGION, incl(bar), REGION, BASE ) +// // ^^^^ +// parser_indices *pind_sub = new parser_indices(left, right); +// if (arg_outside) { +// pind_sub = arg_outside->alt_ref()->init_indices_outside(left, right, k, track, ys_lefts_this, ys_rights_this); +// } +// +// // 2b) start left of the outside non-terminal and initialize indices from right to left +// // foo( BASE, REGION, bar, REGION, BASE ) +// // <-- ^^^^^^ X +// Expr::Base *right_index = pind_sub->var_left; +// Expr::Base *left_index = pind_sub->var_left; +// for (std::list::const_reverse_iterator i = args_left->rbegin(); i != args_left->rend(); ++i) { +// // total yield size left of this argument +// ys_lefts_this = sum_ys_lefts(ys_lefts, i, args_left->rend(), track); +// left_index = get_next_var_right2left(left_index, k, track, (*i)->multi_ys()(track), ys_lefts_this); +// parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, ys_lefts, ys_rights); +// // prepare for next element: right = left +// right_index = pind->var_left; +// } +// // before doing 2c, save current left_index as the "leftmost_index" +// Expr::Base *leftmost_index = left_index; +// +// // 2c) same as above, but start right of outside non-terminal +// // foo( BASE, REGION, bar, REGION, BASE ) +// // X ^^^^^^ --> +// left_index = pind_sub->var_right; +// right_index = pind_sub->var_right; +// for (std::list::const_iterator i = args_right->begin(); i != args_right->end(); ++i) { +// // total yield size left of this argument +// ys_rights_this = sum_ys_rights(ys_rights, i, args_right->end(), track); +// right_index = get_next_var_left2right(right_index, k, track, (*i)->multi_ys()(track), ys_rights_this); +// parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, ys_lefts, ys_rights); +// // prepare for next element: left = right +// left_index = pind->var_right; +// } +// +// // 3) update indices of outside non-terminal such that they reach from leftmost to rightmost +// // e.g. foo( REGION, outside_bar, REGION ) +// // k0 i j k1 +// // --> k0 k0 k1 k1 +// this->expand_outside_nt_indices(leftmost_index, right_index, track); +// +// // 4) set indices for THIS +// // foo( BASE, REGION, bar, REGION, BASE ) +// // ^^^ +// parser_indices *pind_this = Alt::Base::init_indices_outside(leftmost_index, right_index, k, track, ys_lefts, ys_rights); +// //pind_this->ys_left_upper = &ys_accum_upper; +// return pind_this; +} +void Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { + Alt::Base::init_indices_outside(left, right, k, track, center_left, center_right); +} +void Alt::Block::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { + throw LogError("Alt::Block::init_indices_outside is not yet implemented!"); } -parser_indices *Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights) { - parser_indices *pind = Alt::Base::init_indices_outside(left, right, k, track, ys_lefts, ys_rights); - return pind; +void Alt::Multi::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { + throw LogError("Alt::Multi::init_indices_outside is not yet implemented!"); } -parser_indices *Alt::Block::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights) { - throw LogError("Alt::Block::init_indices_outside is not yet implemented!"); + +std::pair *Alt::Base::get_outside_accum_yieldsizes(size_t track) { + std::pair *res = new std::pair(new Yield::Size(), new Yield::Size()); + return res; +} +std::pair *Alt::Simple::get_outside_accum_yieldsizes(size_t track) { + //first = left, second = right + std::pair *res = new std::pair(new Yield::Size(), new Yield::Size()); + + Fn_Arg::Base *arg_outside = NULL; + for (std::list::const_iterator i = args.begin(); i != args.end(); ++i) { + Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); + if (ont) { + arg_outside = *i; + std::pair *sub_ys = arg_outside->alt_ref()->get_outside_accum_yieldsizes(track); + *(res->first) += *(sub_ys->first); + *(res->second) += *(sub_ys->second); + continue; + } + if (arg_outside == NULL) { + *(res->first) += (*i)->multi_ys()(track); + } else { + *(res->second) += (*i)->multi_ys()(track); + } + } + + return res; +} +std::pair *Alt::Link::get_outside_accum_yieldsizes(size_t track) { + return Alt::Base::get_outside_accum_yieldsizes(track); +} +std::pair *Alt::Block::get_outside_accum_yieldsizes(size_t track) { + throw LogError("Alt::Block::get_outside_accum_yieldsizes is not yet implemented!"); return NULL; + } -parser_indices *Alt::Multi::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights) { - throw LogError("Alt::Multi::init_indices_outside is not yet implemented!"); +std::pair *Alt::Multi::get_outside_accum_yieldsizes(size_t track) { + throw LogError("Alt::Block::get_outside_accum_yieldsizes is not yet implemented!"); return NULL; } + Yield::Size *Alt::Simple::sum_ys_lefts(Yield::Size *y, std::list::const_reverse_iterator i, const std::list::const_reverse_iterator &end, size_t track) const { Yield::Size *res = new Yield::Size(); *res += *y; diff --git a/src/alt.hh b/src/alt.hh index 6f33d8b69..41b5317b1 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -248,9 +248,10 @@ class Base { // e.g. foo(k1_REGION_i, i_bar_j, j_REGION_k2) --> foo(k1_REGION_i, k1_bar_k2, j_REGION_k2) // ^ ^ ^^ ^^ virtual void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); + virtual std::pair *get_outside_accum_yieldsizes(size_t track); // sets indices for outside rules, respecting yield sizes - virtual parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights); + virtual void init_indices_outside( + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right); virtual void init_ret_decl(unsigned int i, const std::string &prefix); @@ -476,8 +477,9 @@ class Simple : public Base { Expr::Base *get_next_var_right2left(Expr::Base *left_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_lefts); Expr::Base *get_next_var_left2right(Expr::Base *right_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_rights); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); - parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights); + std::pair *get_outside_accum_yieldsizes(size_t track); + void init_indices_outside( + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right); void put_indices(std::ostream &s); void reset(); @@ -619,9 +621,10 @@ class Link : public Base { void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); + std::pair *get_outside_accum_yieldsizes(size_t track); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); - parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights); + void init_indices_outside( + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right); // void init_ret_decl(unsigned int i); @@ -723,8 +726,9 @@ class Block : public Base { void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); - parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights); + std::pair *get_outside_accum_yieldsizes(size_t track); + void init_indices_outside( + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right); void codegen(AST &ast); @@ -799,8 +803,9 @@ class Multi : public Base { void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); - parser_indices *init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Yield::Size *ys_lefts, Yield::Size *ys_rights); + std::pair *get_outside_accum_yieldsizes(size_t track); + void init_indices_outside( + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right); void codegen(AST &ast); void print_dot_edge(std::ostream &out, Symbol::NT &nt); diff --git a/src/symbol.cc b/src/symbol.cc index ed6220491..299c2491d 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -752,7 +752,7 @@ void Symbol::NT::init_indices(Expr::Vacc *left, Expr::Vacc *right, for (std::list::iterator i = alts.begin(); i != alts.end(); ++i) { if (this->is_partof_outside) { - parser_indices *pind = (*i)->init_indices_outside(left, right, k, track, new Yield::Size(), new Yield::Size()); + (*i)->init_indices_outside(left, right, k, track, NULL, NULL); } else { (*i)->init_indices(left, right, k, track); } From 9fca180ed3704ba6cd68cd027dbf108a2498e0e9 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 31 Mar 2022 20:38:38 +0200 Subject: [PATCH 020/209] remove return structure --- src/alt.hh | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/src/alt.hh b/src/alt.hh index 41b5317b1..892ff66d0 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -56,30 +56,6 @@ class Signature_Base; class Visitor; class Fn_Decl; -struct parser_indices { - // the current variable for leftmost index, e.g. t_0_i - Expr::Base *var_left; - // the current variable for rightmost index, e.g. t_0_j - Expr::Base *var_right; - - // accumulated yield size for constant parser on the left of non-terminal parser - // e.g. foo(x, BASE, BASE, nt, ...) if "nt" is the flagged (outside) non-terminal, - // we "know" that BASE and BASE each consumes exactly 1 char, i.e. 2 in total - // which can guard running indices for x parser - Yield::Size *ys_left; - // as above, but for right of nt - Yield::Size *ys_right; - - Yield::Size *ys_left_upper; - - parser_indices(Expr::Base *left, Expr::Base *right) : ys_left(), ys_right(), ys_left_upper() { - this->var_left = left; - this->var_right = right; - - this->ys_left_upper = new Yield::Size(); - this->ys_left_upper->set(0, 0); - } -}; namespace Alt { /* @@ -474,8 +450,8 @@ class Simple : public Base { public: void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); - Expr::Base *get_next_var_right2left(Expr::Base *left_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_lefts); - Expr::Base *get_next_var_left2right(Expr::Base *right_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_rights); + Expr::Base *get_next_var_right2left(Expr::Base *left_index, Expr::Base *innermost_left_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_lefts); + Expr::Base *get_next_var_left2right(Expr::Base *right_index, Expr::Base *innermost_right_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_rights); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); std::pair *get_outside_accum_yieldsizes(size_t track); void init_indices_outside( From 521c78c198386ff82033141bb57e7c1a72d3f03b Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 31 Mar 2022 20:38:52 +0200 Subject: [PATCH 021/209] correct loops --- src/alt.cc | 237 +++++++++++++++++------------------------------------ 1 file changed, 73 insertions(+), 164 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 063cf4279..c51f95d38 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3216,48 +3216,63 @@ Expr::Base *get_next_var_name(unsigned &k, size_t track) { Expr::Vacc *ivar = new Expr::Vacc(new std::string(o.str())); return ivar; } -Expr::Base *Alt::Simple::get_next_var_right2left(Expr::Base *left_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_lefts) { +Expr::Base *Alt::Simple::get_next_var_right2left(Expr::Base *left_index, Expr::Base *innermost_left_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_lefts) { if (ys_this.low() == ys_this.high()) { // constant yield size - return left_index->plus(ys_this.low()); + if (ys_lefts->low() == 0) { + return innermost_left_index; + } else { + return left_index->plus(ys_this.low()); + } + } + + if (ys_lefts->low() == ys_lefts->high()) { + return innermost_left_index->plus(ys_lefts->low()); } // variable yield size Expr::Base *next_index = get_next_var_name(k, track); -// // add for loop -// std::ostringstream o; -// o << "t_" << track << "_left_most"; -// Expr::Vacc *left_border = new Expr::Vacc(new std::string(o.str())); -// Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), next_index, left_index->minus(ys_this.low())); -// loopvariable->set_itr(true); -// Expr::Base *cond = new Expr::Greater_Eq(next_index, left_border->plus(ys_lefts->low())); -// Statement::For *f = new Statement::For(loopvariable, cond); -// f->decrement = true; // --loopvar instead of ++loopvar -// loops.push_back(f); + // add for loop + Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), next_index, left_index->plus(ys_this.low())); + loopvariable->set_itr(true); + Expr::Base *cond = new Expr::Less_Eq(next_index, innermost_left_index->minus(ys_lefts->low())); + Statement::For *f = new Statement::For(loopvariable, cond); + loops.push_back(f); return next_index; } -Expr::Base *Alt::Simple::get_next_var_left2right(Expr::Base *right_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_rights) { +Expr::Base *Alt::Simple::get_next_var_left2right(Expr::Base *right_index, Expr::Base *innermost_right_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_rights) { + Expr::Base *next_index; + + // constant yield size if (ys_this.low() == ys_this.high()) { - // constant yield size - return right_index->minus(ys_this.low()); + if (ys_rights->low() == 0) { + // we might fall back to j, if yield between outside NT and current right hand side argument is 0 + next_index = innermost_right_index; + } else { + // normally, we just decrease the current index + next_index = right_index->minus(ys_this.low()); + } + } else { + // constant yield size, on the left hand side towards outside non terminal + // e.g. ..., outside_bar, BASE, foo, ... + // where border between BASE and foo shall be j+1 instead of k_x + if (ys_rights->low() == ys_rights->high()) { + next_index = innermost_right_index->plus(ys_rights->low()); + } else { + // variable yield size + next_index = get_next_var_name(k, track); + + // add for loop + Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), next_index, innermost_right_index->plus(ys_rights->low())); + loopvariable->set_itr(true); + Expr::Base *cond = new Expr::Less_Eq(next_index, right_index->minus(ys_this.low())); + Statement::For *f = new Statement::For(loopvariable, cond); + loops.push_back(f); + } } - // variable yield size - Expr::Base *next_index = get_next_var_name(k, track); - -// // add for loop -// std::ostringstream o; -// o << "t_" << track << "_right_most"; -// Expr::Vacc *right_border = new Expr::Vacc(new std::string(o.str())); -// Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), next_index, right_index->plus(ys_this.low())); -// loopvariable->set_itr(true); -// Expr::Base *cond = new Expr::Less_Eq(next_index, right_border->minus(ys_rights->low())); -// Statement::For *f = new Statement::For(loopvariable, cond); -// f->decrement = false; // --loopvar instead of ++loopvar -// loops.push_back(f); - return next_index; } @@ -3301,26 +3316,10 @@ void Alt::Block::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, void Alt::Multi::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { throw LogError("Alt::Block::expand_outside_nt_indices is not yet implemented!"); } - void Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { Alt::Base::init_indices(left, right, k, track); } -void printLR(Expr::Base *l, Expr::Base *r) { - std::cerr << "("; - if (l) { - std::cerr << *l; - } else { - std::cerr << "NULL"; - } - std::cerr << ","; - if (r) { - std::cerr << *r; - } else { - std::cerr << "NULL"; - } - std::cerr << ")"; -} void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { std::list *args_left = new std::list(); std::list *args_right = new std::list(); @@ -3348,21 +3347,29 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi } } - // yield sizes for sub-structure std::pair *ys_sub = new std::pair(new Yield::Size(), new Yield::Size()); if (arg_outside != NULL) { - arg_outside->alt_ref()->get_outside_accum_yieldsizes(track); + ys_sub = arg_outside->alt_ref()->get_outside_accum_yieldsizes(track); } + std::pair *ys_this = this->get_outside_accum_yieldsizes(track); -// std::cerr << "==LEFT==\n"; + // arguments left of outside NT Expr::Base *left_index; -// Expr::Base *leftmost_index = NULL; if (center_left) { left_index = center_left; } else { left_index = get_next_var_name(k, track); -// leftmost_index = left_index; + + // add for-loop + std::ostringstream o; + o << "t_" << track << "_left_most"; + Expr::Vacc *left_border = new Expr::Vacc(new std::string(o.str())); + Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), left_index, left_border); + loopvariable->set_itr(true); + Expr::Base *cond = new Expr::Less_Eq(left_index, left->minus(ys_this->first->low())); + Statement::For *f = new Statement::For(loopvariable, cond); + loops.push_back(f); } Expr::Base *leftmost_index = left_index; Expr::Base *right_index = NULL; @@ -3371,23 +3378,27 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi Yield::Size *ys_right2center = new Yield::Size(); *ys_right2center += *(ys_sub->first); *ys_right2center += *(sum_ys_rights(&ys_this, i, args_left->end(), track)); - if ((std::next(i) == args_left->end()) && (nt_outside != NULL)) { - right_index = left; - } else { - right_index = get_next_var_right2left(left_index, k, track, ys_this, ys_right2center); - } -// std::cerr << " index="; printLR(left_index,right_index); std::cerr << ", center="; printLR(center_left, center_right); std::cerr << ", left-right"; printLR(left, right); std::cerr << "\n"; + right_index = get_next_var_right2left(left_index, left, k, track, ys_this, ys_right2center); (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, center_left, center_right); left_index = right_index; } Expr::Base *lhs_right_index = right_index; - -// std::cerr << "==RIGHT=\n"; + // arguments right of outside NT if (center_right) { right_index = center_right; } else { right_index = get_next_var_name(k, track); + + // add for-loop + std::ostringstream o; + o << "t_" << track << "_right_most"; + Expr::Vacc *right_border = new Expr::Vacc(new std::string(o.str())); + Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), right_index, right->plus(ys_this->second->low())); + loopvariable->set_itr(true); + Expr::Base *cond = new Expr::Less_Eq(right_index, right_border); + Statement::For *f = new Statement::For(loopvariable, cond); + loops.push_back(f); } Expr::Base *rightmost_index = right_index; left_index = NULL; @@ -3396,121 +3407,19 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi Yield::Size *ys_left2center = new Yield::Size(); *ys_left2center += *(ys_sub->second); *ys_left2center += *(sum_ys_lefts(&ys_this, i, args_right->rend(), track)); - if ((std::next(i) == args_right->rend()) && (nt_outside != NULL)) { - left_index = right; - } else { - left_index = get_next_var_left2right(right_index, k, track, ys_this, ys_left2center); - } -// std::cerr << " index="; printLR(left_index,right_index); std::cerr << ", center="; printLR(center_left, center_right); std::cerr << ", left-right"; printLR(left, right); std::cerr << "\n"; + left_index = get_next_var_left2right(right_index, right, k, track, ys_this, ys_left2center); (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, center_left, center_right); right_index = left_index; } - //std::cerr << *this->name; printLR(leftmost_index, rightmost_index); std::cerr << "\n"; + // argument containing outside NT Alt::Base::init_indices_outside(leftmost_index, rightmost_index, k, track, center_left, center_right); - if (arg_outside != NULL) { arg_outside->alt_ref()->init_indices_outside(left, right, k, track, lhs_right_index, right_index); if (nt_outside == NULL) { this->expand_outside_nt_indices(leftmost_index, rightmost_index, track); } } - - -// /* We need to set the moving boundaries for outside non terminals. As opposed to the inside version, -// * we here have to start in the inside and work towards left and right ends, e.g. -// * foo( BASE, REGION, bar, REGION, BASE ) -// * i i+1 k0 k1 k1+1 j <-- inside -// * k0-1 k0 i j k1 k1+1 <-- outside -// * Note that bar itself can either be a non-terminal or a nested argument, e.g. -// * foo(BASE, incl(bar), BASE) where incl(bar) has to be initialized BEFOR -// * we can initialize foo(BASE, x, BASE) -// * -// * 1) Let's first iterate through the arguments and store them in: -// * - args left of the outside NT -// * - the arg containing the outside NT -// * - args right of the outside NT -// */ -// std::list *args_left = new std::list(); -// std::list *args_right = new std::list(); -// Yield::Size *ys_lefts_this = new Yield::Size(); -// Yield::Size *ys_rights_this = new Yield::Size(); -// Fn_Arg::Base *arg_outside = NULL; // = the argument containing the outside non-terminal somewhere (mutually exclusive to nt_outside) -// Symbol::NT *nt_outside = NULL; // = the final outside non terminal (mutually exclusive to arg_outside) -// for (std::list::const_iterator i = args.begin(); i != args.end(); ++i) { -// Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); -// if (ont) { -// Alt::Link *link = dynamic_cast((*i)->alt_ref()); -// if (link) { -// if (link->nt == ont) { -// // the link->nt could also be a terminal -// Symbol::NT *check = dynamic_cast(link->nt); -// assert(check); -// nt_outside = check; -// } -// } -// arg_outside = *i; -// continue; -// } -// if (arg_outside == NULL) { -// args_left->push_back(*i); -// *ys_lefts_this += (*i)->multi_ys()(track); -// } else { -// args_right->push_back(*i); -// *ys_rights_this += (*i)->multi_ys()(track); -// } -// } -// -// // 2a) if the outside non terminal is deeper into an argument, recurse into this arg first -// // foo( BASE, REGION, incl(bar), REGION, BASE ) -// // ^^^^ -// parser_indices *pind_sub = new parser_indices(left, right); -// if (arg_outside) { -// pind_sub = arg_outside->alt_ref()->init_indices_outside(left, right, k, track, ys_lefts_this, ys_rights_this); -// } -// -// // 2b) start left of the outside non-terminal and initialize indices from right to left -// // foo( BASE, REGION, bar, REGION, BASE ) -// // <-- ^^^^^^ X -// Expr::Base *right_index = pind_sub->var_left; -// Expr::Base *left_index = pind_sub->var_left; -// for (std::list::const_reverse_iterator i = args_left->rbegin(); i != args_left->rend(); ++i) { -// // total yield size left of this argument -// ys_lefts_this = sum_ys_lefts(ys_lefts, i, args_left->rend(), track); -// left_index = get_next_var_right2left(left_index, k, track, (*i)->multi_ys()(track), ys_lefts_this); -// parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, ys_lefts, ys_rights); -// // prepare for next element: right = left -// right_index = pind->var_left; -// } -// // before doing 2c, save current left_index as the "leftmost_index" -// Expr::Base *leftmost_index = left_index; -// -// // 2c) same as above, but start right of outside non-terminal -// // foo( BASE, REGION, bar, REGION, BASE ) -// // X ^^^^^^ --> -// left_index = pind_sub->var_right; -// right_index = pind_sub->var_right; -// for (std::list::const_iterator i = args_right->begin(); i != args_right->end(); ++i) { -// // total yield size left of this argument -// ys_rights_this = sum_ys_rights(ys_rights, i, args_right->end(), track); -// right_index = get_next_var_left2right(right_index, k, track, (*i)->multi_ys()(track), ys_rights_this); -// parser_indices *pind = (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, ys_lefts, ys_rights); -// // prepare for next element: left = right -// left_index = pind->var_right; -// } -// -// // 3) update indices of outside non-terminal such that they reach from leftmost to rightmost -// // e.g. foo( REGION, outside_bar, REGION ) -// // k0 i j k1 -// // --> k0 k0 k1 k1 -// this->expand_outside_nt_indices(leftmost_index, right_index, track); -// -// // 4) set indices for THIS -// // foo( BASE, REGION, bar, REGION, BASE ) -// // ^^^ -// parser_indices *pind_this = Alt::Base::init_indices_outside(leftmost_index, right_index, k, track, ys_lefts, ys_rights); -// //pind_this->ys_left_upper = &ys_accum_upper; -// return pind_this; } void Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { Alt::Base::init_indices_outside(left, right, k, track, center_left, center_right); @@ -3565,7 +3474,7 @@ std::pair *Alt::Multi::get_outside_accum_yieldsizes( Yield::Size *Alt::Simple::sum_ys_lefts(Yield::Size *y, std::list::const_reverse_iterator i, const std::list::const_reverse_iterator &end, size_t track) const { Yield::Size *res = new Yield::Size(); - *res += *y; +// *res += *y; ++i; for (; i != end; ++i) { *res += (*i)->multi_ys()(track); @@ -3574,7 +3483,7 @@ Yield::Size *Alt::Simple::sum_ys_lefts(Yield::Size *y, std::list: } Yield::Size *Alt::Simple::sum_ys_rights(Yield::Size *y, std::list::const_iterator i, const std::list::const_iterator &end, size_t track) const { Yield::Size *res = new Yield::Size(); - *res += *y; +// *res += *y; ++i; for (; i != end; ++i) { *res += (*i)->multi_ys()(track); From 2f91ac24775ce85a9fc40028fb83adee43bd19f2 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 1 Apr 2022 11:47:28 +0200 Subject: [PATCH 022/209] working outside alt guards --- src/alt.cc | 40 +++++++++++++++++++++++++++++++++------- src/alt.hh | 3 +++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index c51f95d38..c7b8f0014 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -1514,13 +1514,32 @@ void Alt::Simple::init_guards() { assert(m_ys.tracks() == left_indices.size()); Yield::Multi::iterator k = m_ys.begin(); std::vector::iterator j = right_indices.begin(); + std::vector::iterator it_left_inner_indices = this->left_inside_indices.begin(); + std::vector::iterator it_right_inner_indices = this->right_inside_indices.begin(); + + unsigned int track = 0; for (std::vector::iterator i = left_indices.begin(); - i != left_indices.end(); ++i, ++j, ++k) { - Expr::Base *e = (*j)->minus(*i); - l.push_back(new Expr::Greater_Eq(e, (*k).low())); - if ((*k).high() != Yield::Poly(Yield::UP)) { - l.push_back(new Expr::Less_Eq(e, (*k).high())); - } + i != left_indices.end(); ++i, ++j, ++k, ++track, ++it_left_inner_indices, ++it_right_inner_indices) { + if (this->is_partof_outside) { + // obtain yield sizes left and right of outside non-terminal + std::pair *ys = this->get_outside_accum_yieldsizes(track); + + std::ostringstream o_left; o_left << "t_" << track << "_left_most"; + Expr::Vacc *leftmost = new Expr::Vacc(new std::string(o_left.str())); + assert(*it_left_inner_indices); + l.push_back(new Expr::Greater_Eq((*it_left_inner_indices)->minus(ys->first->low()), leftmost)); + + std::ostringstream o_right; o_right << "t_" << track << "_right_most"; + Expr::Vacc *rightmost = new Expr::Vacc(new std::string(o_right.str())); + assert(*it_right_inner_indices); + l.push_back(new Expr::Less_Eq((*it_right_inner_indices)->plus(ys->second->low()), rightmost)); + } else { + Expr::Base *e = (*j)->minus(*i); + l.push_back(new Expr::Greater_Eq(e, (*k).low())); + if ((*k).high() != Yield::Poly(Yield::UP)) { + l.push_back(new Expr::Less_Eq(e, (*k).high())); + } + } } Expr::Base *cond = Expr::seq_to_tree( @@ -3412,8 +3431,15 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi right_index = left_index; } - // argument containing outside NT Alt::Base::init_indices_outside(leftmost_index, rightmost_index, k, track, center_left, center_right); + // store original left/right for e.g. + // correct initialization of IF guards; see function init_guards() + this->left_inside_indices.resize(this->left_indices.size()); + this->right_inside_indices.resize(this->right_indices.size()); + this->left_inside_indices[track] = left; + this->right_inside_indices[track] = right; + + // argument containing outside NT if (arg_outside != NULL) { arg_outside->alt_ref()->init_indices_outside(left, right, k, track, lhs_right_index, right_index); if (nt_outside == NULL) { diff --git a/src/alt.hh b/src/alt.hh index 892ff66d0..38d65106e 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -136,6 +136,9 @@ class Base { protected: std::vector left_indices; std::vector right_indices; + // for outside non-terminals: store original inner left/right indices for guards construction + std::vector left_inside_indices; + std::vector right_inside_indices; public: Statement::Var_Decl *ret_decl; From cdfb2b4d0956393011114fb2fa75d39f3aea4d8d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 1 Apr 2022 13:38:44 +0200 Subject: [PATCH 023/209] switch filter/loop order for outside NTs --- src/alt.cc | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 0ffac5c32..58a060ffe 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -1942,11 +1942,22 @@ void Alt::Simple::codegen(AST &ast) { } } - // add filter_guards - stmts = add_filter_guards(ast, stmts, filter_guards); + if (is_partof_outside) { + // syntactic filter indices are only initialized through the for-loops in outside alternatives + // thus, reverse the order of filters/loops: - // add for loops for moving boundaries - stmts = add_for_loops(stmts, loops, has_index_overlay()); + // add for loops for moving boundaries + stmts = add_for_loops(stmts, loops, has_index_overlay()); + + // add filter_guards + stmts = add_filter_guards(ast, stmts, filter_guards); + } else { + // add filter_guards + stmts = add_filter_guards(ast, stmts, filter_guards); + + // add for loops for moving boundaries + stmts = add_for_loops(stmts, loops, has_index_overlay()); + } add_subopt_guards(stmts, ast); From 7776bfdf93c8609343ab073cf4786bd112337752 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 1 Apr 2022 17:28:12 +0200 Subject: [PATCH 024/209] NTs to replace with outside version are now skipped depending on their names --- src/alt.cc | 34 +++++++++++++++++++++++----------- src/alt.hh | 10 +++++----- src/grammar.cc | 18 +++++++++++------- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index a4203492a..e9fb0ffc7 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -1527,12 +1527,24 @@ void Alt::Simple::init_guards() { std::ostringstream o_left; o_left << "t_" << track << "_left_most"; Expr::Vacc *leftmost = new Expr::Vacc(new std::string(o_left.str())); assert(*it_left_inner_indices); - l.push_back(new Expr::Greater_Eq((*it_left_inner_indices)->minus(ys->first->low()), leftmost)); + Expr::Vacc *lname = dynamic_cast(*it_left_inner_indices); + if (lname && (lname->name()->compare(*leftmost->name()) == 0)) { + // the rare case where guards check themselves + // happens e.g. if table has only one dimension + } else { + l.push_back(new Expr::Greater_Eq((*it_left_inner_indices)->minus(ys->first->low()), leftmost)); + } std::ostringstream o_right; o_right << "t_" << track << "_right_most"; Expr::Vacc *rightmost = new Expr::Vacc(new std::string(o_right.str())); assert(*it_right_inner_indices); - l.push_back(new Expr::Less_Eq((*it_right_inner_indices)->plus(ys->second->low()), rightmost)); + Expr::Vacc *rname = dynamic_cast(*it_right_inner_indices); + if (rname && (rname->name()->compare(*rightmost->name()) == 0)) { + // the rare case where guards check themselves + // happens e.g. if table has only one dimension + } else { + l.push_back(new Expr::Less_Eq((*it_right_inner_indices)->plus(ys->second->low()), rightmost)); + } } else { Expr::Base *e = (*j)->minus(*i); l.push_back(new Expr::Greater_Eq(e, (*k).low())); @@ -3042,12 +3054,12 @@ void Alt::Multi::set_partof_outside(bool is_outside) { } -bool Alt::Base::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences) { +bool Alt::Base::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences) { return false; } -bool Alt::Simple::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences) { - for (Fn_Arg::Base *arg : this->args) { - Fn_Arg::Alt *arg_alt = dynamic_cast(arg); +bool Alt::Simple::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences) { + for (std::list::iterator it_arg = this->args.begin(); it_arg != this->args.end(); ++it_arg) { + Fn_Arg::Alt *arg_alt = dynamic_cast(*it_arg); if (arg_alt) { if (arg_alt->alt->replace_nonterminal(find, replace, skip_occurences)) { return true; @@ -3056,18 +3068,18 @@ bool Alt::Simple::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, uns } return false; } -bool Alt::Link::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences) { +bool Alt::Link::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences) { if (this->nt) { if (this->nt->is(Symbol::NONTERMINAL)) { if (this->nt == find) { - if (skip_occurences == 0) { + if (skip_occurences[*(find->name)] == 0) { // replace old NT (=find) with novel NT (=replace) this->nt = replace; this->name = replace->name; this->is_partof_outside = replace->is_partof_outside; return true; } else { - skip_occurences--; + skip_occurences[*(find->name)]--; } } } else if (this->nt->is(Symbol::TERMINAL)) { @@ -3078,7 +3090,7 @@ bool Alt::Link::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsig } return false; } -bool Alt::Block::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences) { +bool Alt::Block::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences) { for (Alt::Base *alt : this->alts) { // recurse into alternatives if (alt->replace_nonterminal(find, replace, skip_occurences)) { @@ -3087,7 +3099,7 @@ bool Alt::Block::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsi } return false; } -bool Alt::Multi::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences) { +bool Alt::Multi::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences) { throw LogError("Alt::Multi::replace_nonterminal is not yet implemented!"); return false; } diff --git a/src/alt.hh b/src/alt.hh index 5e5ffab88..789322976 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -330,7 +330,7 @@ class Base { // and outside. virtual void set_partof_outside(bool is_outside); // traverses the alternative (=rhs of a production), iff non-terminal find is contained, replace with first occurence of non-terminal replace - virtual bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences); + virtual bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences); virtual unsigned int to_dot(unsigned int *nodeID, std::ostream &out); // returns either the single outside non-terminal or NULL @@ -529,7 +529,7 @@ class Simple : public Base { void get_nonterminals(std::list *nt_list); void set_partof_outside(bool is_outside); - bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences); + bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); private: @@ -659,7 +659,7 @@ class Link : public Base { void get_nonterminals(std::list *nt_list); void set_partof_outside(bool is_outside); - bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences); + bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); }; @@ -735,7 +735,7 @@ class Block : public Base { void get_nonterminals(std::list *nt_list); void set_partof_outside(bool is_outside); - bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences); + bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); }; @@ -815,7 +815,7 @@ class Multi : public Base { void get_nonterminals(std::list *nt_list); void set_partof_outside(bool is_outside); - bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, unsigned int &skip_occurences); + bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); }; diff --git a/src/grammar.cc b/src/grammar.cc index f0fd058aa..bc0ba4a5f 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1056,17 +1056,21 @@ void Grammar::inject_outside_nts() { std::list *rhs_nts = new std::list(); (*alt)->get_nonterminals(rhs_nts); if (rhs_nts->size() > 0) { - unsigned int skip_occurences = 0; - for (Symbol::NT *rhs_nt : *rhs_nts) { + hashtable skip_occurences; + for (std::list::iterator it_nt = rhs_nts->begin(); it_nt != rhs_nts->end(); ++it_nt) { + skip_occurences[*((*it_nt)->name)] = 0; + } + //unsigned int skip_occurences = 0; + for (std::list::iterator it_nt = rhs_nts->begin(); it_nt != rhs_nts->end(); ++it_nt) { // create a copy (=clone) of the inside alternative Alt::Base *outside_alt = (*alt)->clone(); // replace one of the inside non-terminals with the outside version of the calling non-terminal - outside_alt->replace_nonterminal(rhs_nt, dynamic_cast(outside_NTs.find(i->first)->second), skip_occurences); + outside_alt->replace_nonterminal(*it_nt, dynamic_cast(outside_NTs.find(i->first)->second), skip_occurences); // flag outside_alt as being part of an automatically generated outside production rule outside_alt->set_partof_outside(true); // append copied+replaced alternative to the list of alternatives for the outside version of the called non-terminal - (dynamic_cast(outside_NTs.find(*(rhs_nt->name))->second))->alts.push_back(outside_alt); - skip_occurences++; + (dynamic_cast(outside_NTs.find(*((*it_nt)->name))->second))->alts.push_back(outside_alt); + skip_occurences[*((*it_nt)->name)]++; } } else { // e.g. struct = nil(LOC) has no non-terminal on the rhs, but should become an alternative for outside version of struct @@ -1088,8 +1092,8 @@ void Grammar::inject_outside_nts() { } // 5) TODO(smj): how to determine new axiom? - //this->axiom_name = new std::string("outside_hairpin"); - this->axiom_name = new std::string("outside_struct"); + this->axiom_name = new std::string("outside_hairpin"); + //this->axiom_name = new std::string("outside_struct"); this->init_axiom(); } From 27c4a1614d92ab79d8eedc4adb2e4c32a3067ef4 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sat, 2 Apr 2022 00:11:10 +0200 Subject: [PATCH 025/209] adding new multi test --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0964d60d5..59c3959e8 100644 --- a/Makefile +++ b/Makefile @@ -128,7 +128,7 @@ gapc: src/gapc.o $(MAIN_OBJ) src/version.o src/prefix.o stats \ test_rt_tab \ -multi_cyk multi_deps multi_indices multi_list_size multi_eliminate_lists_more multi_eliminate_lists multi_algebra multi_signature multi_rt_approx multi_rt_all multi_in_out multi_self_rec multi_calls multi_table_dim multi_max_filter multi_loops multi_ys parser: src/version.o src/prefix.o +multi_cyk multi_deps multi_indices multi_list_size multi_eliminate_lists_more multi_eliminate_lists multi_algebra multi_signature multi_rt_approx multi_rt_all multi_in_out multi_self_rec multi_calls multi_table_dim multi_max_filter multi_loops multi_ys multi_outside_indices parser: src/version.o src/prefix.o backtrack: src/version.o From b5a7e6454e4cb6bfeba8336bf8771bae722049df Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sat, 2 Apr 2022 00:12:07 +0200 Subject: [PATCH 026/209] new multi test --- testdata/modtest/multi_outside_indices.cc | 71 +++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 testdata/modtest/multi_outside_indices.cc diff --git a/testdata/modtest/multi_outside_indices.cc b/testdata/modtest/multi_outside_indices.cc new file mode 100644 index 000000000..d9e7cd0d7 --- /dev/null +++ b/testdata/modtest/multi_outside_indices.cc @@ -0,0 +1,71 @@ +#include "../../src/driver.hh" +#include "../../src/log.hh" + +#include +#include +#include +#include + +int main(int argc, char **argv) { + std::ostringstream o; + Log log; + log.set_debug(false); + log.set_ostream(o); + + Driver driver; + + if (argc != 2) { + std::cerr << "Call " << *argv << " *.gap-file\n"; + return 1; + } + // === front + // set the file name of the gap-source-code + driver.setFilename(argv[1]); + + // parses the input file and builds the AST + driver.parse(); + + // simply gets the selected grammar, which is either the + // grammar that occurred first in the source code or is the + // one that was named in the parameters on the command line + Grammar *grammar = driver.ast.grammar(); + // Now check the semantic, which does more than the function + // name suggests. The semantic check is embedded in the algorithm + // that links the grammar graph together, and computes yield-sizes. + // If the method returns false, there are some semantic errors, + // which leads to the end of the compilation process. + bool r = grammar->check_semantic(); + if (!r) { + throw LogError("Seen semantic errors."); + } + + // inject rules for outside grammar + grammar->inject_outside_nts(); + + // set approx table design + grammar->approx_table_conf(); + + // find what type of input is read + // chars, sequence of ints etc. + driver.ast.derive_temp_alphabet(); + + r = driver.ast.check_signature(); + if (!r) { + throw LogError("Seen signature errors."); + } + + // apply this to identify standard functions like Min, Max, Exp etc. + driver.ast.derive_roles(); + + + // ------------- back ------------ + grammar->init_list_sizes(); + + grammar->init_indices(); + grammar->init_decls(); + // for cyk (ordering of NT for parsing, see page 101 of the thesis) + grammar->dep_analysis(); + + unsigned int nodeID = 1; + grammar->to_dot(&nodeID, std::cout); +} From 83bf12f0d8a6b898bef17566a6ec59bd805c1f0e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 3 Apr 2022 20:57:31 +0200 Subject: [PATCH 027/209] automatically create or set axiom for outside rules --- src/grammar.cc | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/src/grammar.cc b/src/grammar.cc index bc0ba4a5f..f6403261a 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1049,6 +1049,7 @@ void Grammar::inject_outside_nts() { } // 3) create copy of inside alternative, replace called non-terminal with calling outside version and append to called outside version + std::set *axiom_candidates = new std::set(); for (hashtable::iterator i = NTs.begin(); i != NTs.end(); ++i) { Symbol::NT *nt = dynamic_cast(i->second); if (nt) { // skip terminals @@ -1078,6 +1079,11 @@ void Grammar::inject_outside_nts() { // flag outside_alt as being part of an automatically generated outside production rule outside_alt->set_partof_outside(true); (dynamic_cast(outside_NTs.find(*(nt->name))->second))->alts.push_back(outside_alt); + + // if production rule does not have any non-terminals on the right hand side, the calling non-terminal + // is a candidate for being called from the axiom for outside + // e.g. hairpin = hl(BASE, REGION, BASE) --> outside_hairin get's added + axiom_candidates->insert(dynamic_cast(outside_NTs.find(*(nt->name))->second)); } } } @@ -1091,10 +1097,35 @@ void Grammar::inject_outside_nts() { } } - // 5) TODO(smj): how to determine new axiom? - this->axiom_name = new std::string("outside_hairpin"); - //this->axiom_name = new std::string("outside_struct"); - this->init_axiom(); + // 5) inside production rules that have NO non-terminals on their right hand sides + // must be those that parse the final sub-words of the input. + // Therefore, they must be the smallest start-points for outside construction, + // i.e. the axiom should point to them. These non-terminals have been + // collected in step 3. + if (axiom_candidates->size() == 1) { + // if there is only one inside non-terminal to be accessed from the axiom + // we can directly define its outside version as the axiom + this->axiom_name = (*axiom_candidates->begin())->name; + this->init_axiom(); + } else { + // otherwise, we need to add another left hand side non-terminal + // which points to multiple non-terminals WITHOUT making a choice + std::string *nt_axiom_name = new std::string("outside_axioms"); + Symbol::NT *nt_axiom = new Symbol::NT(nt_axiom_name, Loc()); + + for (std::set::iterator i = axiom_candidates->begin(); i != axiom_candidates->end(); ++i) { + nt_axiom->alts.push_back(new Alt::Link((*i)->name, Loc())); + } + // add new lhs non-terminal to grammar + this->NTs.insert(std::make_pair(*nt_axiom_name, dynamic_cast(nt_axiom))); + this->nt_list.push_back(nt_axiom); + + this->axiom_name = nt_axiom_name; + this->init_axiom(); + + // re-run check symantics to properly integrate new production rules + this->check_semantic(); + } } unsigned int Grammar::to_dot(unsigned int *nodeID, std::ostream &out) { From 005088808ff8d9b507b902fbd561882a85f8ab58 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 3 Apr 2022 22:59:47 +0200 Subject: [PATCH 028/209] test framework for outside --- Makefile | 4 ++ testdata/grammar_outside/mini_twoLevelIL.gap | 39 ++++++++++++ testdata/modtest/run_outside.sh | 67 ++++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 testdata/grammar_outside/mini_twoLevelIL.gap create mode 100644 testdata/modtest/run_outside.sh diff --git a/Makefile b/Makefile index 59c3959e8..538ec7c7e 100644 --- a/Makefile +++ b/Makefile @@ -307,6 +307,10 @@ test-mod: modtests cd testdata/modtest &&\ $(SHELL) run.sh $(TRUTH_DIR)/Mod$(TRUTH_SUFFIX) $(MODTESTS) +test-mod_outside: testdata/modtest/multi_outside_indices.cc + cd testdata/modtest &&\ + $(SHELL) run_outside.sh $(TRUTH_DIR)/Mod$(TRUTH_SUFFIX) multi_outside_indices + gen-mod: modtests cd testdata/modtest &&\ $(SHELL) gen.sh $(TRUTH_DIR)/Mod$(TRUTH_SUFFIX) $(MODTESTS) diff --git a/testdata/grammar_outside/mini_twoLevelIL.gap b/testdata/grammar_outside/mini_twoLevelIL.gap new file mode 100644 index 000000000..521f668d7 --- /dev/null +++ b/testdata/grammar_outside/mini_twoLevelIL.gap @@ -0,0 +1,39 @@ +import rna + +input rna + +signature sig_foldrna(alphabet,answer) { + answer nil(Subsequence); //empty structure + choice [answer] h([answer]); + answer il(Subsequence, Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer comp(Subsequence, Subsequence, Subsequence, answer, Subsequence, answer, Subsequence); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } + int il(Subsequence lb, Subsequence lr, Subsequence r, int x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } + int comp(Subsequence a, Subsequence lr, Subsequence lr2, int x, Subsequence b, int y, Subsequence lr3) { + return x+y; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = nil(LOC) | + iloop + # h; + + iloop = il(BASE, REGION, BASE, comp(BASE, BASE, REGION, struct, BASE, struct, REGION), REGION, BASE) with basepairing # h; + +} + +instance mfe = gra_nodangle(alg_mfe); diff --git a/testdata/modtest/run_outside.sh b/testdata/modtest/run_outside.sh new file mode 100644 index 000000000..ffcadeebe --- /dev/null +++ b/testdata/modtest/run_outside.sh @@ -0,0 +1,67 @@ +#!/bin/ksh + +# call for example: +# ksh run.sh ../../../adpc-tng-logs tab typecheck + +set -u + +if [ $# -lt 2 ]; then + echo $0 DATA-BASE TOOL-ONE \[MORE_TOOLS...\] + exit 1 +fi + +GRAMMAR=../grammar_outside +WORKDIR=`pwd` +TEMP=$WORKDIR/temp +REF=$1 +PREFIX=../.. + +mkdir -p $TEMP + +if [ ! -d $REF ]; then + echo Reference directory is no directory: $TEMP/$REF + exit 1 +fi + +succ_count=0 +err_count=0 +failed=0 + +. ../log.sh + +for j in ${*:2}; do + cd $GRAMMAR + mkdir -p $TEMP/$j + for i in `ls * | grep -v '\(^core$\|\.orig$\)'`; do + failed=0 + log_both $TEMP/$j/$i $PREFIX/$j $i + if [ $failed != 0 ]; then + err_count=$((err_count+1)) + fi + done +done + +module_errors=$err_count + +err_count=0 +cd $TEMP +for j in ${*:2}; do + echo +------------------------------------------------------------------------------+ + failed=0 + l=`echo $REF/$j[0-9]* | tr ' ' '\n' | sed 's/\([0-9]\+\)/:\1/' |\ + sort -n -r -k 2 -t : | tr -d : | head -1` + log diff -ur $l $j + + if [ $failed != 0 ]; then + echo --++--FAIL--++-- + err_count=$((err_count+1)) + else + echo OK + succ_count=$((succ_count+1)) + fi + echo +------------------------------------------------------------------------------+ +done + + +echo +==============================================================================+ +. ../../stats.sh From d9c1561256b176fb51ef2e7b24a6673a0d51a02d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 3 Apr 2022 23:02:11 +0200 Subject: [PATCH 029/209] extra github action for outside --- .github/workflows/outside.yml | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/outside.yml diff --git a/.github/workflows/outside.yml b/.github/workflows/outside.yml new file mode 100644 index 000000000..eea5a165f --- /dev/null +++ b/.github/workflows/outside.yml @@ -0,0 +1,55 @@ +name: github action build & CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + workflow_dispatch: + inputs: + logLevel: + description: 'Log level' + required: true + default: 'warning' + tags: + description: 'Test scenario tags' + +jobs: + gapc_ubuntu: + runs-on: ubuntu-latest + steps: + - name: Update apt + run: sudo apt-get update + - name: Install dependencies + run: sudo apt-get install flex bison make libboost-all-dev libgsl-dev python3 python3-pip + + - name: Checkout truth + run: git clone --branch master https://github.com/jlab/gapc-test-suite.git $GITHUB_WORKSPACE/../gapc-test-suite + + - uses: actions/checkout@v2 + - name: configure + run: ./configure --prefix $GITHUB_WORKSPACE + - name: make + run: make -j 2 + - name: make install + run: sudo make install + + - name: test-mod + run: make test-mod_outside TRUTH_DIR=$GITHUB_WORKSPACE/../gapc-test-suite/Truth TRUTH_SUFFIX=_ubuntu + + + cpplint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + - run: pip install cpplint + - run: cpplint --recursive --counting 'detailed' --filter="-runtime/references,-build/include_subdir" --extensions=cc,hh src/ rtlib/ + - run: cpplint --counting 'detailed' --filter="-build/include_subdir,-readability/casting,-runtime/arrays" --extensions=c,h librna/rnalib.{c,h} + - run: cpplint --recursive --counting 'detailed' --filter="-runtime/references,-build/include_subdir" testdata/unittest/* + - uses: vishnudxb/cancel-workflow@v1.2 + if: failure() + with: + repo: jlab/gapc + workflow_id: ${{ github.run_id }} + access_token: ${{ github.token }} From 59aa6c77be0fbf8ac1b3035040129f1413cb6c2d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 3 Apr 2022 23:04:35 +0200 Subject: [PATCH 030/209] rename and remove cpplint --- .github/workflows/outside.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/outside.yml b/.github/workflows/outside.yml index eea5a165f..df50ba952 100644 --- a/.github/workflows/outside.yml +++ b/.github/workflows/outside.yml @@ -1,4 +1,4 @@ -name: github action build & CI +name: test outside injection on: push: @@ -36,20 +36,3 @@ jobs: - name: test-mod run: make test-mod_outside TRUTH_DIR=$GITHUB_WORKSPACE/../gapc-test-suite/Truth TRUTH_SUFFIX=_ubuntu - - - cpplint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-python@v1 - - run: pip install cpplint - - run: cpplint --recursive --counting 'detailed' --filter="-runtime/references,-build/include_subdir" --extensions=cc,hh src/ rtlib/ - - run: cpplint --counting 'detailed' --filter="-build/include_subdir,-readability/casting,-runtime/arrays" --extensions=c,h librna/rnalib.{c,h} - - run: cpplint --recursive --counting 'detailed' --filter="-runtime/references,-build/include_subdir" testdata/unittest/* - - uses: vishnudxb/cancel-workflow@v1.2 - if: failure() - with: - repo: jlab/gapc - workflow_id: ${{ github.run_id }} - access_token: ${{ github.token }} From d45a94a18474444c7bdc162ce65930124a7ce7d4 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 3 Apr 2022 23:27:15 +0200 Subject: [PATCH 031/209] make tests variable --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 538ec7c7e..34ed23e94 100644 --- a/Makefile +++ b/Makefile @@ -307,9 +307,9 @@ test-mod: modtests cd testdata/modtest &&\ $(SHELL) run.sh $(TRUTH_DIR)/Mod$(TRUTH_SUFFIX) $(MODTESTS) -test-mod_outside: testdata/modtest/multi_outside_indices.cc +test-mod_outside: multi_outside_indices cd testdata/modtest &&\ - $(SHELL) run_outside.sh $(TRUTH_DIR)/Mod$(TRUTH_SUFFIX) multi_outside_indices + $(SHELL) run_outside.sh $(TRUTH_DIR)/Mod$(TRUTH_SUFFIX) $? gen-mod: modtests cd testdata/modtest &&\ From a0e7b16d3063c15d5b0a7aa5cbbf1ee8bac736bc Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 3 Apr 2022 23:28:11 +0200 Subject: [PATCH 032/209] save some compiling?! --- .github/workflows/outside.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/outside.yml b/.github/workflows/outside.yml index df50ba952..930433082 100644 --- a/.github/workflows/outside.yml +++ b/.github/workflows/outside.yml @@ -29,10 +29,6 @@ jobs: - uses: actions/checkout@v2 - name: configure run: ./configure --prefix $GITHUB_WORKSPACE - - name: make - run: make -j 2 - - name: make install - run: sudo make install - name: test-mod - run: make test-mod_outside TRUTH_DIR=$GITHUB_WORKSPACE/../gapc-test-suite/Truth TRUTH_SUFFIX=_ubuntu + run: make -j 2 test-mod_outside TRUTH_DIR=$GITHUB_WORKSPACE/../gapc-test-suite/Truth TRUTH_SUFFIX=_ubuntu From bf11345e31f9629213e8fbf6e37f2125da693420 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 3 Apr 2022 23:32:38 +0200 Subject: [PATCH 033/209] no moving boundary if no parsers are leftmost rightmost --- src/alt.cc | 60 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index e9fb0ffc7..b74f56829 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3418,20 +3418,27 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi if (center_left) { left_index = center_left; } else { - left_index = get_next_var_name(k, track); - - // add for-loop - std::ostringstream o; - o << "t_" << track << "_left_most"; - Expr::Vacc *left_border = new Expr::Vacc(new std::string(o.str())); - Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), left_index, left_border); - loopvariable->set_itr(true); - Expr::Base *cond = new Expr::Less_Eq(left_index, left->minus(ys_this->first->low())); - Statement::For *f = new Statement::For(loopvariable, cond); - loops.push_back(f); + if (ys_this->first->low() != ys_this->first->high()) { + left_index = get_next_var_name(k, track); + + // add for-loop + std::ostringstream o; + o << "t_" << track << "_left_most"; + Expr::Vacc *left_border = new Expr::Vacc(new std::string(o.str())); + Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), left_index, left_border); + loopvariable->set_itr(true); + Expr::Base *cond = new Expr::Less_Eq(left_index, left->minus(ys_this->first->low())); + Statement::For *f = new Statement::For(loopvariable, cond); + loops.push_back(f); + } else { + // no new index if there are no parsers or only parsers with const yield size left of outside nt + // e.g. incl(outside_foo) + // e.g. sadd(BASE, outside_foo) + left_index = left->minus(ys_this->first->low()); + } } Expr::Base *leftmost_index = left_index; - Expr::Base *right_index = NULL; + Expr::Base *right_index = left_index; for (std::list::const_iterator i = args_left->begin(); i != args_left->end(); ++i) { Yield::Size ys_this = (*i)->multi_ys()(track); Yield::Size *ys_right2center = new Yield::Size(); @@ -3447,17 +3454,24 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi if (center_right) { right_index = center_right; } else { - right_index = get_next_var_name(k, track); - - // add for-loop - std::ostringstream o; - o << "t_" << track << "_right_most"; - Expr::Vacc *right_border = new Expr::Vacc(new std::string(o.str())); - Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), right_index, right->plus(ys_this->second->low())); - loopvariable->set_itr(true); - Expr::Base *cond = new Expr::Less_Eq(right_index, right_border); - Statement::For *f = new Statement::For(loopvariable, cond); - loops.push_back(f); + if (ys_this->second->low() != ys_this->second->high()) { + right_index = get_next_var_name(k, track); + + // add for-loop + std::ostringstream o; + o << "t_" << track << "_right_most"; + Expr::Vacc *right_border = new Expr::Vacc(new std::string(o.str())); + Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), right_index, right->plus(ys_this->second->low())); + loopvariable->set_itr(true); + Expr::Base *cond = new Expr::Less_Eq(right_index, right_border); + Statement::For *f = new Statement::For(loopvariable, cond); + loops.push_back(f); + } else { + // no new index if there are no parsers or only parsers with const yield size right of outside nt + // e.g. incl(outside_foo) + // e.g. adds(outside_foo, BASE) + right_index = right; + } } Expr::Base *rightmost_index = right_index; left_index = NULL; From 47f9c93af994d1c341e066d987d47f0ae3b8337e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 5 Apr 2022 11:38:11 +0200 Subject: [PATCH 034/209] we always have to exand --- src/alt.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index b74f56829..b4579f54c 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3496,9 +3496,7 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi // argument containing outside NT if (arg_outside != NULL) { arg_outside->alt_ref()->init_indices_outside(left, right, k, track, lhs_right_index, right_index); - if (nt_outside == NULL) { - this->expand_outside_nt_indices(leftmost_index, rightmost_index, track); - } + this->expand_outside_nt_indices(leftmost_index, rightmost_index, track); } } void Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { From d159fb0e45227c14e057c12e7338fdce573ee5b3 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 5 Apr 2022 11:41:34 +0200 Subject: [PATCH 035/209] adding grammar to be tested --- testdata/grammar_outside/mini1.gap | 61 ++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 testdata/grammar_outside/mini1.gap diff --git a/testdata/grammar_outside/mini1.gap b/testdata/grammar_outside/mini1.gap new file mode 100644 index 000000000..2a9911bd7 --- /dev/null +++ b/testdata/grammar_outside/mini1.gap @@ -0,0 +1,61 @@ +import rna + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer ssadd(Subsequence, answer); + answer nil(Subsequence); //empty structure + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int sadd(Subsequence lb, int x) { + return x + sbase_energy(); + } + int ssadd(Subsequence r, int x) { + return x; + } + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } +} + +algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { + string sadd(Subsequence lb,string e) { + string res; + append(res, '.'); + append(res, e); + return res; + } + + string ssadd(Subsequence r, string e) { + return e; + } + + string nil(Subsequence loc) { + string r; + return r; + } + + choice [string] h([string] i) { + return i; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = nil(LOC) | + sadd(BASE, struct) | + ssadd(REGION, struct) + # h; + + +} + +instance mfe = gra_nodangle(alg_mfe); From 1052a5c32e600ae655ac38233e55034f2ad4ea6d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 5 Apr 2022 11:48:10 +0200 Subject: [PATCH 036/209] new test case --- testdata/grammar_outside/mini_bl.gap | 56 ++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 testdata/grammar_outside/mini_bl.gap diff --git a/testdata/grammar_outside/mini_bl.gap b/testdata/grammar_outside/mini_bl.gap new file mode 100644 index 000000000..d6f9b4640 --- /dev/null +++ b/testdata/grammar_outside/mini_bl.gap @@ -0,0 +1,56 @@ +import rna + +input rna + +signature sig_foldrna(alphabet,answer) { + answer nil(Subsequence); //empty structure + choice [answer] h([answer]); + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } + int il(Subsequence lb, Subsequence lr, int x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } +} + +algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { + string nil(Subsequence loc) { + string r; + return r; + } + + choice [string] h([string] i) { + return i; + } + + string il(Subsequence lb,Subsequence lregion,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = nil(LOC) | + iloop + # h; + + iloop = il(BASE, REGION with maxsize(30), struct, REGION with maxsize(30), BASE) with basepairing # h; + +} + +instance mfe = gra_nodangle(alg_mfe); From 061f74470234634a3824a240a230070e23d033ef Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 5 Apr 2022 12:21:18 +0200 Subject: [PATCH 037/209] three more test cases --- testdata/grammar_outside/mini_complexIL.gap | 60 +++++++ testdata/grammar_outside/mini_largeIL.gap | 56 +++++++ testdata/grammar_outside/nodangle_issue.gap | 167 ++++++++++++++++++++ 3 files changed, 283 insertions(+) create mode 100644 testdata/grammar_outside/mini_complexIL.gap create mode 100644 testdata/grammar_outside/mini_largeIL.gap create mode 100644 testdata/grammar_outside/nodangle_issue.gap diff --git a/testdata/grammar_outside/mini_complexIL.gap b/testdata/grammar_outside/mini_complexIL.gap new file mode 100644 index 000000000..85de279c9 --- /dev/null +++ b/testdata/grammar_outside/mini_complexIL.gap @@ -0,0 +1,60 @@ +import rna + +input rna + +signature sig_foldrna(alphabet,answer) { + answer nil(Subsequence); //empty structure + choice [answer] h([answer]); + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer comp(Subsequence, Subsequence, Subsequence, Subsequence, answer, Subsequence, answer); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } + int il(Subsequence lb, Subsequence lr, int x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } + int comp(Subsequence a, Subsequence lr, Subsequence lr2, Subsequence lr3, int x, Subsequence b, int y) { + return x+y; + } +} + +//algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { +// string nil(Subsequence loc) { +// string r; +// return r; +// } +// +// choice [string] h([string] i) { +// return i; +// } +// +// string il(Subsequence lb,Subsequence lregion,string e,Subsequence rregion,Subsequence rb) { +// string res; +// append(res, '('); +// append(res, '.', size(lregion)); +// append(res, e); +// append(res, '.', size(rregion)); +// append(res, ')'); +// return res; +// } +//} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = nil(LOC) | + iloop + # h; + + iloop = il(BASE, REGION, comp(REGION, BASE, BASE, REGION, struct, BASE, struct), REGION, BASE) with basepairing # h; + +} + +instance mfe = gra_nodangle(alg_mfe); diff --git a/testdata/grammar_outside/mini_largeIL.gap b/testdata/grammar_outside/mini_largeIL.gap new file mode 100644 index 000000000..fa0be0279 --- /dev/null +++ b/testdata/grammar_outside/mini_largeIL.gap @@ -0,0 +1,56 @@ +import rna + +input rna + +signature sig_foldrna(alphabet,answer) { + answer nil(Subsequence); //empty structure + choice [answer] h([answer]); + answer il(Subsequence, Subsequence, Subsequence, answer, Subsequence, Subsequence, Subsequence); // an internal loop with a closing base-pair +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } + int il(Subsequence lb, Subsequence a, Subsequence lr, int x, Subsequence rr, Subsequence b, Subsequence rb) { + return x + il_energy(lr, rr); + } +} + +algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { + string nil(Subsequence loc) { + string r; + return r; + } + + choice [string] h([string] i) { + return i; + } + + string il(Subsequence lb,Subsequence a, Subsequence lregion,string e,Subsequence rregion,Subsequence b,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = nil(LOC) | + iloop + # h; + + iloop = il(BASE, REGION with maxsize(30), REGION, struct, REGION, REGION with maxsize(30), BASE) with basepairing # h; + +} + +instance mfe = gra_nodangle(alg_mfe); diff --git a/testdata/grammar_outside/nodangle_issue.gap b/testdata/grammar_outside/nodangle_issue.gap new file mode 100644 index 000000000..0a6f1542e --- /dev/null +++ b/testdata/grammar_outside/nodangle_issue.gap @@ -0,0 +1,167 @@ +import rna + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int sadd(Subsequence lb, int x) { + return x + sbase_energy(); + } + int cadd(int x, int y) { + return x + y; + } + int drem(Subsequence lb, int x, Subsequence rb) { + return x + termau_energy(lb, rb); + } + int sr(Subsequence lb, int x, Subsequence rb) { + return x + sr_energy(lb, rb); + } + int hl(Subsequence lb, Subsequence r, Subsequence rb) { + return hl_energy(r); + } + int bl(Subsequence lb, Subsequence lr, int x, Subsequence rb) { + return x + bl_energy(lr, rb); + } + int br(Subsequence lb, int x, Subsequence rr, Subsequence rb) { + return x + br_energy(lb, rr); + } + int il(Subsequence lb, Subsequence lr, int x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } + int ml(Subsequence lb, int x, Subsequence rb) { + return x + ml_energy() + ul_energy() + termau_energy(lb, rb); + } + int incl(int x) { + return x + ul_energy(); + } + int addss(int x, Subsequence r) { + return x + ss_energy(r); + } + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } +} + +algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { + string sadd(Subsequence lb,string e) { + string res; + append(res, '.'); + append(res, e); + return res; + } + + string cadd(string le,string re) { + string res; + append(res, le); + append(res, re); + return res; + } + + string nil(Subsequence loc) { + string r; + return r; + } + + string drem(Subsequence lloc, string e, Subsequence rloc) { + return e; + } + + string sr(Subsequence lb,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string hl(Subsequence lb,Subsequence region,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(region)); + append(res, ')'); + return res; + } + + + string bl(Subsequence lb,Subsequence lregion,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, ')'); + return res; + } + + string br(Subsequence lb,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string il(Subsequence lb,Subsequence lregion,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string ml(Subsequence lb, string e, Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string addss(string e,Subsequence rb) { + string res; + append(res, e); + append(res, '.', size(rb)); + return res; + } + + string incl(string e) { + return e; + } + + choice [string] h([string] i) { + return i; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = nil(LOC) | + cadd(hairpin, struct) + # h; + + hairpin = hl(BASE, REGION with minsize(3) with maxsize(30), BASE) with basepairing # h; +} + +instance mfe = gra_nodangle(alg_mfe); From ffd51f460ebf7d64490f56d2170db53964272304 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 5 Apr 2022 12:21:50 +0200 Subject: [PATCH 038/209] use inside generation for outside alternatives that do NOT hold any outside non-terminal on their rhs --- src/alt.cc | 7 +++++++ src/grammar.cc | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/alt.cc b/src/alt.cc index b4579f54c..435f2bdf1 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3380,6 +3380,13 @@ void Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, unsign } void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { + // if this alternative is called from an outside non-terminal, but does not contain another outside non-terminal, e.g. hl(BASE, REGION, BASE) + // we can default to inside indices generation! + if (this->is_partof_outside == false) { + Alt::Simple::init_indices(left, right, k, track); + return; + } + std::list *args_left = new std::list(); std::list *args_right = new std::list(); Fn_Arg::Base *arg_outside = NULL; diff --git a/src/grammar.cc b/src/grammar.cc index f6403261a..425fca31c 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1077,7 +1077,7 @@ void Grammar::inject_outside_nts() { // e.g. struct = nil(LOC) has no non-terminal on the rhs, but should become an alternative for outside version of struct Alt::Base *outside_alt = (*alt)->clone(); // flag outside_alt as being part of an automatically generated outside production rule - outside_alt->set_partof_outside(true); + outside_alt->set_partof_outside(false); (dynamic_cast(outside_NTs.find(*(nt->name))->second))->alts.push_back(outside_alt); // if production rule does not have any non-terminals on the right hand side, the calling non-terminal From bec1119a6f4c44fe55c8afdaba5a73fa8d59cca5 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 6 Apr 2022 09:02:46 +0200 Subject: [PATCH 039/209] fix variable name clash --- src/alt.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 435f2bdf1..b3076de98 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3447,11 +3447,11 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi Expr::Base *leftmost_index = left_index; Expr::Base *right_index = left_index; for (std::list::const_iterator i = args_left->begin(); i != args_left->end(); ++i) { - Yield::Size ys_this = (*i)->multi_ys()(track); + Yield::Size ys_arg = (*i)->multi_ys()(track); Yield::Size *ys_right2center = new Yield::Size(); *ys_right2center += *(ys_sub->first); - *ys_right2center += *(sum_ys_rights(&ys_this, i, args_left->end(), track)); - right_index = get_next_var_right2left(left_index, left, k, track, ys_this, ys_right2center); + *ys_right2center += *(sum_ys_rights(&ys_arg, i, args_left->end(), track)); + right_index = get_next_var_right2left(left_index, left, k, track, ys_arg, ys_right2center); (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, center_left, center_right); left_index = right_index; } @@ -3483,11 +3483,11 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi Expr::Base *rightmost_index = right_index; left_index = NULL; for (std::list::const_reverse_iterator i = args_right->rbegin(); i != args_right->rend(); ++i) { - Yield::Size ys_this = (*i)->multi_ys()(track); + Yield::Size ys_arg = (*i)->multi_ys()(track); Yield::Size *ys_left2center = new Yield::Size(); *ys_left2center += *(ys_sub->second); - *ys_left2center += *(sum_ys_lefts(&ys_this, i, args_right->rend(), track)); - left_index = get_next_var_left2right(right_index, right, k, track, ys_this, ys_left2center); + *ys_left2center += *(sum_ys_lefts(&ys_arg, i, args_right->rend(), track)); + left_index = get_next_var_left2right(right_index, right, k, track, ys_arg, ys_left2center); (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, center_left, center_right); right_index = left_index; } From d24864bf56cffbd11acea56ed416543913d30855 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 6 Apr 2022 10:13:09 +0200 Subject: [PATCH 040/209] edge case: all alternatives are right of an upper level outside NT --- src/alt.cc | 48 ++++-- src/alt.hh | 20 +-- src/symbol.cc | 2 +- testdata/grammar_outside/nodangle_issue2.gap | 170 +++++++++++++++++++ 4 files changed, 213 insertions(+), 27 deletions(-) create mode 100644 testdata/grammar_outside/nodangle_issue2.gap diff --git a/src/alt.cc b/src/alt.cc index b3076de98..5b3510d7d 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3375,17 +3375,30 @@ void Alt::Block::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, void Alt::Multi::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { throw LogError("Alt::Block::expand_outside_nt_indices is not yet implemented!"); } -void Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { +void Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { Alt::Base::init_indices(left, right, k, track); } -void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { +void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { // if this alternative is called from an outside non-terminal, but does not contain another outside non-terminal, e.g. hl(BASE, REGION, BASE) // we can default to inside indices generation! if (this->is_partof_outside == false) { Alt::Simple::init_indices(left, right, k, track); return; } +// std::cerr << *this->name << ": Alt::Simple::init_indices_outside(" << *left << ", " << *right << ", "; +// if (center_left) { +// std::cerr << *center_left; +// } else { +// std::cerr << center_left; +// } +// std::cerr << ", "; +// if (center_right) { +// std::cerr << *center_right; +// } else { +// std::cerr << center_right; +// } +// std::cerr << ", " << is_right_of_outside_nt << ")\n"; std::list *args_left = new std::list(); std::list *args_right = new std::list(); @@ -3406,7 +3419,7 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi arg_outside = *i; continue; } - if (arg_outside == NULL) { + if ((arg_outside == NULL) and (is_right_of_outside_nt == false)) { args_left->push_back(*i); } else { args_right->push_back(*i); @@ -3418,7 +3431,7 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi if (arg_outside != NULL) { ys_sub = arg_outside->alt_ref()->get_outside_accum_yieldsizes(track); } - std::pair *ys_this = this->get_outside_accum_yieldsizes(track); + std::pair *ys_this = this->get_outside_accum_yieldsizes(track, is_right_of_outside_nt); // arguments left of outside NT Expr::Base *left_index; @@ -3482,13 +3495,16 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi } Expr::Base *rightmost_index = right_index; left_index = NULL; + if (is_right_of_outside_nt) { + right = left; + } for (std::list::const_reverse_iterator i = args_right->rbegin(); i != args_right->rend(); ++i) { Yield::Size ys_arg = (*i)->multi_ys()(track); Yield::Size *ys_left2center = new Yield::Size(); *ys_left2center += *(ys_sub->second); *ys_left2center += *(sum_ys_lefts(&ys_arg, i, args_right->rend(), track)); left_index = get_next_var_left2right(right_index, right, k, track, ys_arg, ys_left2center); - (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, center_left, center_right); + (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, center_left, rightmost_index, true); right_index = left_index; } @@ -3506,21 +3522,21 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi this->expand_outside_nt_indices(leftmost_index, rightmost_index, track); } } -void Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { +void Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { Alt::Base::init_indices_outside(left, right, k, track, center_left, center_right); } -void Alt::Block::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { +void Alt::Block::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { throw LogError("Alt::Block::init_indices_outside is not yet implemented!"); } -void Alt::Multi::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right) { +void Alt::Multi::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { throw LogError("Alt::Multi::init_indices_outside is not yet implemented!"); } -std::pair *Alt::Base::get_outside_accum_yieldsizes(size_t track) { +std::pair *Alt::Base::get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt) { std::pair *res = new std::pair(new Yield::Size(), new Yield::Size()); return res; } -std::pair *Alt::Simple::get_outside_accum_yieldsizes(size_t track) { +std::pair *Alt::Simple::get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt) { //first = left, second = right std::pair *res = new std::pair(new Yield::Size(), new Yield::Size()); @@ -3529,12 +3545,12 @@ std::pair *Alt::Simple::get_outside_accum_yieldsizes Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); if (ont) { arg_outside = *i; - std::pair *sub_ys = arg_outside->alt_ref()->get_outside_accum_yieldsizes(track); + std::pair *sub_ys = arg_outside->alt_ref()->get_outside_accum_yieldsizes(track, is_right_of_outside_nt); *(res->first) += *(sub_ys->first); *(res->second) += *(sub_ys->second); continue; } - if (arg_outside == NULL) { + if ((arg_outside == NULL) and (is_right_of_outside_nt == false)) { *(res->first) += (*i)->multi_ys()(track); } else { *(res->second) += (*i)->multi_ys()(track); @@ -3543,15 +3559,15 @@ std::pair *Alt::Simple::get_outside_accum_yieldsizes return res; } -std::pair *Alt::Link::get_outside_accum_yieldsizes(size_t track) { - return Alt::Base::get_outside_accum_yieldsizes(track); +std::pair *Alt::Link::get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt) { + return Alt::Base::get_outside_accum_yieldsizes(track, is_right_of_outside_nt); } -std::pair *Alt::Block::get_outside_accum_yieldsizes(size_t track) { +std::pair *Alt::Block::get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt) { throw LogError("Alt::Block::get_outside_accum_yieldsizes is not yet implemented!"); return NULL; } -std::pair *Alt::Multi::get_outside_accum_yieldsizes(size_t track) { +std::pair *Alt::Multi::get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt) { throw LogError("Alt::Block::get_outside_accum_yieldsizes is not yet implemented!"); return NULL; } diff --git a/src/alt.hh b/src/alt.hh index 789322976..553cdcf98 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -227,10 +227,10 @@ class Base { // e.g. foo(k1_REGION_i, i_bar_j, j_REGION_k2) --> foo(k1_REGION_i, k1_bar_k2, j_REGION_k2) // ^ ^ ^^ ^^ virtual void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); - virtual std::pair *get_outside_accum_yieldsizes(size_t track); + virtual std::pair *get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt=false); // sets indices for outside rules, respecting yield sizes virtual void init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt=false); virtual void init_ret_decl(unsigned int i, const std::string &prefix); @@ -456,9 +456,9 @@ class Simple : public Base { Expr::Base *get_next_var_right2left(Expr::Base *left_index, Expr::Base *innermost_left_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_lefts); Expr::Base *get_next_var_left2right(Expr::Base *right_index, Expr::Base *innermost_right_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_rights); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); - std::pair *get_outside_accum_yieldsizes(size_t track); + std::pair *get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt=false); void init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt=false); void put_indices(std::ostream &s); void reset(); @@ -607,10 +607,10 @@ class Link : public Base { void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); - std::pair *get_outside_accum_yieldsizes(size_t track); + std::pair *get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt=false); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); void init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt=false); // void init_ret_decl(unsigned int i); @@ -712,9 +712,9 @@ class Block : public Base { void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); - std::pair *get_outside_accum_yieldsizes(size_t track); + std::pair *get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt=false); void init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt=false); void codegen(AST &ast); @@ -789,9 +789,9 @@ class Multi : public Base { void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); - std::pair *get_outside_accum_yieldsizes(size_t track); + std::pair *get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt=false); void init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt=false); void codegen(AST &ast); void print_dot_edge(std::ostream &out, Symbol::NT &nt); diff --git a/src/symbol.cc b/src/symbol.cc index 299c2491d..af7b4cc16 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -752,7 +752,7 @@ void Symbol::NT::init_indices(Expr::Vacc *left, Expr::Vacc *right, for (std::list::iterator i = alts.begin(); i != alts.end(); ++i) { if (this->is_partof_outside) { - (*i)->init_indices_outside(left, right, k, track, NULL, NULL); + (*i)->init_indices_outside(left, right, k, track, NULL, NULL, false); } else { (*i)->init_indices(left, right, k, track); } diff --git a/testdata/grammar_outside/nodangle_issue2.gap b/testdata/grammar_outside/nodangle_issue2.gap new file mode 100644 index 000000000..b2b3ac86a --- /dev/null +++ b/testdata/grammar_outside/nodangle_issue2.gap @@ -0,0 +1,170 @@ +import rna + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int sadd(Subsequence lb, int x) { + return x + sbase_energy(); + } + int cadd(int x, int y) { + return x + y; + } + int drem(Subsequence lb, int x, Subsequence rb) { + return x + termau_energy(lb, rb); + } + int sr(Subsequence lb, int x, Subsequence rb) { + return x + sr_energy(lb, rb); + } + int hl(Subsequence lb, Subsequence r, Subsequence rb) { + return hl_energy(r); + } + int bl(Subsequence lb, Subsequence lr, int x, Subsequence rb) { + return x + bl_energy(lr, rb); + } + int br(Subsequence lb, int x, Subsequence rr, Subsequence rb) { + return x + br_energy(lb, rr); + } + int il(Subsequence lb, Subsequence lr, int x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } + int ml(Subsequence lb, int x, Subsequence rb) { + return x + ml_energy() + ul_energy() + termau_energy(lb, rb); + } + int incl(int x) { + return x + ul_energy(); + } + int addss(int x, Subsequence r) { + return x + ss_energy(r); + } + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } +} + +algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { + string sadd(Subsequence lb,string e) { + string res; + append(res, '.'); + append(res, e); + return res; + } + + string cadd(string le,string re) { + string res; + append(res, le); + append(res, re); + return res; + } + + string nil(Subsequence loc) { + string r; + return r; + } + + string drem(Subsequence lloc, string e, Subsequence rloc) { + return e; + } + + string sr(Subsequence lb,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string hl(Subsequence lb,Subsequence region,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(region)); + append(res, ')'); + return res; + } + + + string bl(Subsequence lb,Subsequence lregion,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, ')'); + return res; + } + + string br(Subsequence lb,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string il(Subsequence lb,Subsequence lregion,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string ml(Subsequence lb, string e, Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string addss(string e,Subsequence rb) { + string res; + append(res, e); + append(res, '.', size(rb)); + return res; + } + + string incl(string e) { + return e; + } + + choice [string] h([string] i) { + return i; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = nil(LOC) | + incl(hairpin) | + sadd(BASE, struct) | + cadd(struct, sadd(BASE, struct)) + # h; + //dangle = hairpin # h; + + hairpin = hl(BASE, REGION with minsize(3) with maxsize(30), BASE) with basepairing # h; +} + +instance mfe = gra_nodangle(alg_mfe); From ca39fc358fb206f7d508320d7e328c1189a0f732 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 6 Apr 2022 10:37:13 +0200 Subject: [PATCH 041/209] also extend right index with const yield size --- src/alt.cc | 2 +- testdata/grammar_outside/nodangle_mlIssue.gap | 169 ++++++++++++++++++ 2 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 testdata/grammar_outside/nodangle_mlIssue.gap diff --git a/src/alt.cc b/src/alt.cc index 5b3510d7d..fb5933c29 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3490,7 +3490,7 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi // no new index if there are no parsers or only parsers with const yield size right of outside nt // e.g. incl(outside_foo) // e.g. adds(outside_foo, BASE) - right_index = right; + right_index = right->plus(ys_this->second->low()); } } Expr::Base *rightmost_index = right_index; diff --git a/testdata/grammar_outside/nodangle_mlIssue.gap b/testdata/grammar_outside/nodangle_mlIssue.gap new file mode 100644 index 000000000..271d7e27e --- /dev/null +++ b/testdata/grammar_outside/nodangle_mlIssue.gap @@ -0,0 +1,169 @@ +import rna + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int sadd(Subsequence lb, int x) { + return x + sbase_energy(); + } + int cadd(int x, int y) { + return x + y; + } + int drem(Subsequence lb, int x, Subsequence rb) { + return x + termau_energy(lb, rb); + } + int sr(Subsequence lb, int x, Subsequence rb) { + return x + sr_energy(lb, rb); + } + int hl(Subsequence lb, Subsequence r, Subsequence rb) { + return hl_energy(r); + } + int bl(Subsequence lb, Subsequence lr, int x, Subsequence rb) { + return x + bl_energy(lr, rb); + } + int br(Subsequence lb, int x, Subsequence rr, Subsequence rb) { + return x + br_energy(lb, rr); + } + int il(Subsequence lb, Subsequence lr, int x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } + int ml(Subsequence lb, int x, Subsequence rb) { + return x + ml_energy() + ul_energy() + termau_energy(lb, rb); + } + int incl(int x) { + return x + ul_energy(); + } + int addss(int x, Subsequence r) { + return x + ss_energy(r); + } + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } +} + +algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { + string sadd(Subsequence lb,string e) { + string res; + append(res, '.'); + append(res, e); + return res; + } + + string cadd(string le,string re) { + string res; + append(res, le); + append(res, re); + return res; + } + + string nil(Subsequence loc) { + string r; + return r; + } + + string drem(Subsequence lloc, string e, Subsequence rloc) { + return e; + } + + string sr(Subsequence lb,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string hl(Subsequence lb,Subsequence region,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(region)); + append(res, ')'); + return res; + } + + + string bl(Subsequence lb,Subsequence lregion,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, ')'); + return res; + } + + string br(Subsequence lb,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string il(Subsequence lb,Subsequence lregion,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string ml(Subsequence lb, string e, Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string addss(string e,Subsequence rb) { + string res; + append(res, e); + append(res, '.', size(rb)); + return res; + } + + string incl(string e) { + return e; + } + + choice [string] h([string] i) { + return i; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = nil(LOC) | + cadd(multiloop, struct) + # h; + + hairpin = hl(BASE, REGION with minsize(3), BASE) with basepairing # h; + multiloop = ml(BASE, hairpin, BASE) with basepairing # h; + +} + +instance mfe = gra_nodangle(alg_mfe); From 1d1522f7194f6faaed6cc32e229d9479b4327c23 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 6 Apr 2022 11:27:45 +0200 Subject: [PATCH 042/209] propagate right as left index IF outside NT is on higher right level --- src/alt.cc | 10 +- .../nodangle_caddinclissue.gap | 169 ++++++++++++++++++ 2 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 testdata/grammar_outside/nodangle_caddinclissue.gap diff --git a/src/alt.cc b/src/alt.cc index fb5933c29..5c5ffb4af 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3429,7 +3429,7 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi // yield sizes for sub-structure std::pair *ys_sub = new std::pair(new Yield::Size(), new Yield::Size()); if (arg_outside != NULL) { - ys_sub = arg_outside->alt_ref()->get_outside_accum_yieldsizes(track); + ys_sub = arg_outside->alt_ref()->get_outside_accum_yieldsizes(track, is_right_of_outside_nt); } std::pair *ys_this = this->get_outside_accum_yieldsizes(track, is_right_of_outside_nt); @@ -3464,8 +3464,14 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi Yield::Size *ys_right2center = new Yield::Size(); *ys_right2center += *(ys_sub->first); *ys_right2center += *(sum_ys_rights(&ys_arg, i, args_left->end(), track)); + if (arg_outside == NULL) { + // if this level does NOT hold the outside non-terminal, but args are left of the outside non-terminal + // propagate the rightmost index (must be the one from the outside non-terminal in higher levels) + // as the left index + left = right; + } right_index = get_next_var_right2left(left_index, left, k, track, ys_arg, ys_right2center); - (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, center_left, center_right); + (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, leftmost_index, center_right); left_index = right_index; } Expr::Base *lhs_right_index = right_index; diff --git a/testdata/grammar_outside/nodangle_caddinclissue.gap b/testdata/grammar_outside/nodangle_caddinclissue.gap new file mode 100644 index 000000000..2504e63c1 --- /dev/null +++ b/testdata/grammar_outside/nodangle_caddinclissue.gap @@ -0,0 +1,169 @@ +import rna + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int sadd(Subsequence lb, int x) { + return x + sbase_energy(); + } + int cadd(int x, int y) { + return x + y; + } + int drem(Subsequence lb, int x, Subsequence rb) { + return x + termau_energy(lb, rb); + } + int sr(Subsequence lb, int x, Subsequence rb) { + return x + sr_energy(lb, rb); + } + int hl(Subsequence lb, Subsequence r, Subsequence rb) { + return hl_energy(r); + } + int bl(Subsequence lb, Subsequence lr, int x, Subsequence rb) { + return x + bl_energy(lr, rb); + } + int br(Subsequence lb, int x, Subsequence rr, Subsequence rb) { + return x + br_energy(lb, rr); + } + int il(Subsequence lb, Subsequence lr, int x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } + int ml(Subsequence lb, int x, Subsequence rb) { + return x + ml_energy() + ul_energy() + termau_energy(lb, rb); + } + int incl(int x) { + return x + ul_energy(); + } + int addss(int x, Subsequence r) { + return x + ss_energy(r); + } + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } +} + +algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { + string sadd(Subsequence lb,string e) { + string res; + append(res, '.'); + append(res, e); + return res; + } + + string cadd(string le,string re) { + string res; + append(res, le); + append(res, re); + return res; + } + + string nil(Subsequence loc) { + string r; + return r; + } + + string drem(Subsequence lloc, string e, Subsequence rloc) { + return e; + } + + string sr(Subsequence lb,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string hl(Subsequence lb,Subsequence region,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(region)); + append(res, ')'); + return res; + } + + + string bl(Subsequence lb,Subsequence lregion,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, ')'); + return res; + } + + string br(Subsequence lb,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string il(Subsequence lb,Subsequence lregion,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string ml(Subsequence lb, string e, Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string addss(string e,Subsequence rb) { + string res; + append(res, e); + append(res, '.', size(rb)); + return res; + } + + string incl(string e) { + return e; + } + + choice [string] h([string] i) { + return i; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = sadd(BASE with unpaired, struct) | + cadd(incl(dangle), struct) | + nil(LOC) # h; + + dangle = drem(LOC, hairpin, LOC) # h; + + hairpin = hl(BASE, REGION with minsize(3) with unpaired, BASE) with basepair # h; +} + +instance mfe = gra_nodangle(alg_mfe); From c72b81439bc413ee5e75f12b4aa718e51848959e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 6 Apr 2022 11:34:08 +0200 Subject: [PATCH 043/209] real test case --- testdata/grammar_outside/nodangle.gap | 191 ++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 testdata/grammar_outside/nodangle.gap diff --git a/testdata/grammar_outside/nodangle.gap b/testdata/grammar_outside/nodangle.gap new file mode 100644 index 000000000..acdcbcc21 --- /dev/null +++ b/testdata/grammar_outside/nodangle.gap @@ -0,0 +1,191 @@ +import rna + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int sadd(Subsequence lb, int x) { + return x + sbase_energy(); + } + int cadd(int x, int y) { + return x + y; + } + int drem(Subsequence lb, int x, Subsequence rb) { + return x + termau_energy(lb, rb); + } + int sr(Subsequence lb, int x, Subsequence rb) { + return x + sr_energy(lb, rb); + } + int hl(Subsequence lb, Subsequence r, Subsequence rb) { + return hl_energy(r); + } + int bl(Subsequence lb, Subsequence lr, int x, Subsequence rb) { + return x + bl_energy(lr, rb); + } + int br(Subsequence lb, int x, Subsequence rr, Subsequence rb) { + return x + br_energy(lb, rr); + } + int il(Subsequence lb, Subsequence lr, int x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } + int ml(Subsequence lb, int x, Subsequence rb) { + return x + ml_energy() + ul_energy() + termau_energy(lb, rb); + } + int incl(int x) { + return x + ul_energy(); + } + int addss(int x, Subsequence r) { + return x + ss_energy(r); + } + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } +} + +algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { + string sadd(Subsequence lb,string e) { + string res; + append(res, '.'); + append(res, e); + return res; + } + + string cadd(string le,string re) { + string res; + append(res, le); + append(res, re); + return res; + } + + string nil(Subsequence loc) { + string r; + return r; + } + + string drem(Subsequence lloc, string e, Subsequence rloc) { + return e; + } + + string sr(Subsequence lb,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string hl(Subsequence lb,Subsequence region,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(region)); + append(res, ')'); + return res; + } + + + string bl(Subsequence lb,Subsequence lregion,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, ')'); + return res; + } + + string br(Subsequence lb,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string il(Subsequence lb,Subsequence lregion,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string ml(Subsequence lb, string e, Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string addss(string e,Subsequence rb) { + string res; + append(res, e); + append(res, '.', size(rb)); + return res; + } + + string incl(string e) { + return e; + } + + choice [string] h([string] i) { + return i; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = sadd(BASE with unpaired, struct) | + cadd(dangle, struct) | + nil(LOC) # h; + + dangle = drem(LOC, strong, LOC) # h; + + strong = weak # h; + + weak = stack | + hairpin | + leftB | + rightB | + iloop | + multiloop # h; + + stack = sr(BASE, weak, BASE) with basepair # h; + hairpin = hl(BASE, REGION with minsize(3) with unpaired, BASE) with basepair # h; + leftB = bl(BASE, REGION with maxsize(30) with unpaired, strong, BASE) with basepair # h; + rightB = br(BASE, strong, REGION with maxsize(30) with unpaired, BASE) with basepair # h; + iloop = il(BASE, REGION with maxsize(30) with unpaired, strong, REGION with maxsize(30) with unpaired, BASE) with basepair # h; + multiloop = ml(BASE, ml_comps, BASE) with basepair # h; + + ml_comps = sadd(BASE with unpaired, ml_comps) | + cadd(incl(dangle), ml_comps1) # h; + + ml_comps1 = sadd(BASE with unpaired, ml_comps1) | + cadd(incl(dangle), ml_comps1) | + incl(dangle) | + addss(incl(dangle), REGION with unpaired) # h; +} + +instance mfe = gra_nodangle(alg_mfe); From 0b1b97a0b0ec5637bb8e5417626759e6dbd6703d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 6 Apr 2022 15:58:58 +0200 Subject: [PATCH 044/209] pure lhs NTs --- src/grammar.cc | 29 ++- .../nodangle_noNTrhs_issue.gap | 167 ++++++++++++++++++ 2 files changed, 187 insertions(+), 9 deletions(-) create mode 100644 testdata/grammar_outside/nodangle_noNTrhs_issue.gap diff --git a/src/grammar.cc b/src/grammar.cc index 425fca31c..d026ad42e 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1031,21 +1031,21 @@ void Grammar::inject_outside_nts() { // Thus, no new NTs have to be injected return; } - for (Symbol::NT* nt : outside_nts) { + for (std::set::iterator nt = outside_nts.begin(); nt != outside_nts.end(); ++nt) { // use inside NT as template ... - Symbol::NT *inside_nt = dynamic_cast(this->NTs.find(*nt->name)->second); + Symbol::NT *inside_nt = dynamic_cast(this->NTs.find(*(*nt)->name)->second); // and clone for outside version Symbol::NT *outside_nt = inside_nt->clone(inside_nt->track_pos()); // remove all existing alternative production rules outside_nt->alts.clear(); // and correct NT name, to now point to outside version - outside_nt->name = new std::string(OUTSIDE_NT_PREFIX + (*nt->name)); + outside_nt->name = new std::string(OUTSIDE_NT_PREFIX + (*(*nt)->name)); // flag this new non-terminal as being part of the outside rules outside_nt->is_partof_outside = true; - outside_NTs[OUTSIDE_NT_PREFIX + (*nt->name)] = outside_nt; - outside_NTs[(*nt->name)] = outside_nt; + outside_NTs[OUTSIDE_NT_PREFIX + (*(*nt)->name)] = outside_nt; + outside_NTs[(*(*nt)->name)] = outside_nt; } // 3) create copy of inside alternative, replace called non-terminal with calling outside version and append to called outside version @@ -1092,8 +1092,20 @@ void Grammar::inject_outside_nts() { // 4) add novel outside NTs to grammar for (std::pair nt : outside_NTs) { if (this->NTs.find(nt.first) == this->NTs.end()) { // only for NTs that are not already part of the grammar + Symbol::NT *nt_nt = dynamic_cast(nt.second); + /* calling NT might never occur on rhs of productions, e.g. "start" in: + * grammar gra_nodangle uses sig_foldrna(axiom = start) { + * start = incl(struct) # h; + * struct = sadd(BASE, struct) | nil(LOC) # h; } + * the outside version of such non terminals shall point to the inside axiom + */ + if (nt_nt->alts.size() == 0) { + Alt::Link *link = new Alt::Link(this->axiom_name, this->axiom_loc); + link->nt = dynamic_cast(this->axiom); + nt_nt->alts.push_back(link); + } this->NTs.insert(nt); - this->nt_list.push_back(dynamic_cast(nt.second)); + this->nt_list.push_back(nt_nt); } } @@ -1122,10 +1134,9 @@ void Grammar::inject_outside_nts() { this->axiom_name = nt_axiom_name; this->init_axiom(); - - // re-run check symantics to properly integrate new production rules - this->check_semantic(); } + // re-run check symantics to properly integrate new production rules + this->check_semantic(); } unsigned int Grammar::to_dot(unsigned int *nodeID, std::ostream &out) { diff --git a/testdata/grammar_outside/nodangle_noNTrhs_issue.gap b/testdata/grammar_outside/nodangle_noNTrhs_issue.gap new file mode 100644 index 000000000..b9676bf3c --- /dev/null +++ b/testdata/grammar_outside/nodangle_noNTrhs_issue.gap @@ -0,0 +1,167 @@ +import rna + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int sadd(Subsequence lb, int x) { + return x + sbase_energy(); + } + int cadd(int x, int y) { + return x + y; + } + int drem(Subsequence lb, int x, Subsequence rb) { + return x + termau_energy(lb, rb); + } + int sr(Subsequence lb, int x, Subsequence rb) { + return x + sr_energy(lb, rb); + } + int hl(Subsequence lb, Subsequence r, Subsequence rb) { + return hl_energy(r); + } + int bl(Subsequence lb, Subsequence lr, int x, Subsequence rb) { + return x + bl_energy(lr, rb); + } + int br(Subsequence lb, int x, Subsequence rr, Subsequence rb) { + return x + br_energy(lb, rr); + } + int il(Subsequence lb, Subsequence lr, int x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } + int ml(Subsequence lb, int x, Subsequence rb) { + return x + ml_energy() + ul_energy() + termau_energy(lb, rb); + } + int incl(int x) { + return x + ul_energy(); + } + int addss(int x, Subsequence r) { + return x + ss_energy(r); + } + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } +} + +algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { + string sadd(Subsequence lb,string e) { + string res; + append(res, '.'); + append(res, e); + return res; + } + + string cadd(string le,string re) { + string res; + append(res, le); + append(res, re); + return res; + } + + string nil(Subsequence loc) { + string r; + return r; + } + + string drem(Subsequence lloc, string e, Subsequence rloc) { + return e; + } + + string sr(Subsequence lb,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string hl(Subsequence lb,Subsequence region,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(region)); + append(res, ')'); + return res; + } + + + string bl(Subsequence lb,Subsequence lregion,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, ')'); + return res; + } + + string br(Subsequence lb,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string il(Subsequence lb,Subsequence lregion,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string ml(Subsequence lb, string e, Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string addss(string e,Subsequence rb) { + string res; + append(res, e); + append(res, '.', size(rb)); + return res; + } + + string incl(string e) { + return e; + } + + choice [string] h([string] i) { + return i; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = start) { + start = incl(struct) # h; + struct = sadd(BASE with unpaired, struct) | + nil(LOC) # h; + + +} + +instance mfe = gra_nodangle(alg_mfe); From 3bda45f65b80b2cc6af3c0c408e4b3699d45dcf1 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 6 Apr 2022 16:21:25 +0200 Subject: [PATCH 045/209] skip if argument is not a parser but a constant --- src/alt.cc | 10 + testdata/grammar_outside/hmm_cpg.gap | 302 +++++++++++++++++++++++++++ 2 files changed, 312 insertions(+) create mode 100644 testdata/grammar_outside/hmm_cpg.gap diff --git a/src/alt.cc b/src/alt.cc index fdad8ac93..b648393cb 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3407,6 +3407,11 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi Fn_Arg::Base *arg_outside = NULL; Symbol::NT *nt_outside = NULL; for (std::list::const_iterator i = args.begin(); i != args.end(); ++i) { + Fn_Arg::Const *constarg = dynamic_cast(*i); + if (constarg != NULL) { + // argument is a constant like 'A' in CHAR('A') + continue; + } Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); if (ont) { Alt::Link *link = dynamic_cast((*i)->alt_ref()); @@ -3550,6 +3555,11 @@ std::pair *Alt::Simple::get_outside_accum_yieldsizes Fn_Arg::Base *arg_outside = NULL; for (std::list::const_iterator i = args.begin(); i != args.end(); ++i) { + Fn_Arg::Const *constarg = dynamic_cast(*i); + if (constarg != NULL) { + // skip constant arguments like CHAR('A') + continue; + } Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); if (ont) { arg_outside = *i; diff --git a/testdata/grammar_outside/hmm_cpg.gap b/testdata/grammar_outside/hmm_cpg.gap new file mode 100644 index 000000000..0a6f520b3 --- /dev/null +++ b/testdata/grammar_outside/hmm_cpg.gap @@ -0,0 +1,302 @@ +type Rope = extern + +signature sig_cpg(alphabet, answer) { + answer transition_start_rich(float, answer, answer); + answer transition_start_poor(float, answer, answer); + answer transition_rich_rich(float, answer, answer); + answer transition_rich_poor(float, answer, answer); + answer transition_poor_rich(float, answer, answer); + answer transition_poor_poor(float, answer, answer); + + answer emission_rich_A(float, alphabet); + answer emission_rich_C(float, alphabet); + answer emission_rich_G(float, alphabet); + answer emission_rich_T(float, alphabet); + answer emission_poor_A(float, alphabet); + answer emission_poor_C(float, alphabet); + answer emission_poor_G(float, alphabet); + answer emission_poor_T(float, alphabet); + answer nil(void); + + choice [answer] h([answer]); +} + +algebra alg_enum auto enum; +algebra alg_count auto count; + +algebra alg_viterbi implements sig_cpg(alphabet=char, answer=float) { + float transition_start_rich(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_start_poor(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_rich_poor(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_rich_rich(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_poor_poor(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_poor_rich(float transition, float emission, float x) { + return transition * emission * x; + } + + float emission_rich_A(float emission, char a) { + return emission; + } + float emission_rich_C(float emission, char a) { + return emission; + } + float emission_rich_G(float emission, char a) { + return emission; + } + float emission_rich_T(float emission, char a) { + return emission; + } + float emission_poor_A(float emission, char a) { + return emission; + } + float emission_poor_C(float emission, char a) { + return emission; + } + float emission_poor_G(float emission, char a) { + return emission; + } + float emission_poor_T(float emission, char a) { + return emission; + } + float nil(void) { + return 1.0; + } + choice [float] h([float] candidates) { + return list(maximum(candidates)); + } +} +algebra alg_fwd extends alg_viterbi { + choice [float] h([float] candidates) { + return list(sum(candidates)); + } +} + +algebra alg_states implements sig_cpg(alphabet=char, answer=Rope) { + Rope transition_start_rich(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'H'); + append(res, x); + return res; + } + Rope transition_start_poor(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'L'); + append(res, x); + return res; + } + Rope transition_rich_poor(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'L'); + append(res, x); + return res; + } + Rope transition_rich_rich(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'H'); + append(res, x); + return res; + } + Rope transition_poor_poor(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'L'); + append(res, x); + return res; + } + Rope transition_poor_rich(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'H'); + append(res, x); + return res; + } + + Rope emission_rich_A(float emission, char a) { + Rope res; + return res; + } + Rope emission_rich_C(float emission, char a) { + Rope res; + return res; + } + Rope emission_rich_G(float emission, char a) { + Rope res; + return res; + } + Rope emission_rich_T(float emission, char a) { + Rope res; + return res; + } + Rope emission_poor_A(float emission, char a) { + Rope res; + return res; + } + Rope emission_poor_C(float emission, char a) { + Rope res; + return res; + } + Rope emission_poor_G(float emission, char a) { + Rope res; + return res; + } + Rope emission_poor_T(float emission, char a) { + Rope res; + return res; + } + + Rope nil(void) { + Rope res; + return res; + } + choice [Rope] h([Rope] candidates) { + return candidates; + } +} + +algebra alg_mult implements sig_cpg(alphabet=char, answer=Rope) { + Rope transition_start_rich(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_start_poor(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_rich_poor(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_rich_rich(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_poor_poor(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_poor_rich(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + + Rope emission_rich_A(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope emission_rich_C(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope emission_rich_G(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope emission_rich_T(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope emission_poor_A(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope emission_poor_C(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope emission_poor_G(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope emission_poor_T(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope nil(void) { + Rope res; + append(res, 1.0); + return res; + } + choice [Rope] h([Rope] candidates) { + return unique(candidates); + } +} + + +grammar gra_cpg uses sig_cpg(axiom=start) { + start = transition_start_rich(CONST_FLOAT(0.5), rich_emission, rich) + | transition_start_poor(CONST_FLOAT(0.5), poor_emission, poor) + # h; + + rich = transition_rich_rich(CONST_FLOAT(0.63), rich_emission, rich) + | transition_rich_poor(CONST_FLOAT(0.37), poor_emission, poor) + | nil(EMPTY) + # h; + + poor = transition_poor_poor(CONST_FLOAT(0.63), poor_emission, poor) + | transition_poor_rich(CONST_FLOAT(0.37), rich_emission, rich) + | nil(EMPTY) + # h; + + rich_emission = emission_rich_A(CONST_FLOAT(0.13), CHAR('A')) + | emission_rich_C(CONST_FLOAT(0.37), CHAR('C')) + | emission_rich_G(CONST_FLOAT(0.37), CHAR('G')) + | emission_rich_T(CONST_FLOAT(0.13), CHAR('T')) + # h; + + poor_emission = emission_poor_A(CONST_FLOAT(0.37), CHAR('A')) + | emission_poor_C(CONST_FLOAT(0.13), CHAR('C')) + | emission_poor_G(CONST_FLOAT(0.13), CHAR('G')) + | emission_poor_T(CONST_FLOAT(0.37), CHAR('T')) + # h; +} + +instance enum = gra_cpg(alg_enum); +instance viterbistatesmult = gra_cpg(alg_viterbi * alg_states * alg_mult); +instance fwd = gra_cpg(alg_fwd); +instance multviterbistates = gra_cpg(alg_mult * alg_viterbi * alg_states); From 914794df03313402da1a91b88b410ea7e1aafbab Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 7 Apr 2022 14:20:11 +0200 Subject: [PATCH 046/209] special index init for inside axiom, called in outside rules --- src/symbol.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/symbol.cc b/src/symbol.cc index af7b4cc16..4ab60e857 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -752,10 +752,16 @@ void Symbol::NT::init_indices(Expr::Vacc *left, Expr::Vacc *right, for (std::list::iterator i = alts.begin(); i != alts.end(); ++i) { if (this->is_partof_outside) { - (*i)->init_indices_outside(left, right, k, track, NULL, NULL, false); - } else { - (*i)->init_indices(left, right, k, track); - } + if ((*i)->get_is_partof_outside() == false) { + // special treatment for a link to the original inside axiom, ensure that it has to consume the full input sequence(s) + (*i)->init_indices_outside(left->plus(right), right->minus(left), k, track, NULL, NULL, false); + dynamic_cast(*i)->init_outside_guards(); + } else { + (*i)->init_indices_outside(left, right, k, track, NULL, NULL, false); + } + } else { + (*i)->init_indices(left, right, k, track); + } } } From 7bd0eceac8c3ddae4962e5add40818030bd2d9bc Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 7 Apr 2022 14:20:56 +0200 Subject: [PATCH 047/209] carry over inside yields for additional guards in outside codegen --- src/grammar.cc | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/grammar.cc b/src/grammar.cc index d026ad42e..233ebeca1 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1069,16 +1069,18 @@ void Grammar::inject_outside_nts() { outside_alt->replace_nonterminal(*it_nt, dynamic_cast(outside_NTs.find(i->first)->second), skip_occurences); // flag outside_alt as being part of an automatically generated outside production rule outside_alt->set_partof_outside(true); + + outside_alt->m_ys_inside = outside_alt->multi_ys(); // append copied+replaced alternative to the list of alternatives for the outside version of the called non-terminal (dynamic_cast(outside_NTs.find(*((*it_nt)->name))->second))->alts.push_back(outside_alt); skip_occurences[*((*it_nt)->name)]++; } } else { - // e.g. struct = nil(LOC) has no non-terminal on the rhs, but should become an alternative for outside version of struct - Alt::Base *outside_alt = (*alt)->clone(); - // flag outside_alt as being part of an automatically generated outside production rule - outside_alt->set_partof_outside(false); - (dynamic_cast(outside_NTs.find(*(nt->name))->second))->alts.push_back(outside_alt); +// // e.g. struct = nil(LOC) has no non-terminal on the rhs, but should become an alternative for outside version of struct +// Alt::Base *outside_alt = (*alt)->clone(); +// // flag outside_alt as being part of an automatically generated outside production rule +// outside_alt->set_partof_outside(true); +// (dynamic_cast(outside_NTs.find(*(nt->name))->second))->alts.push_back(outside_alt); // if production rule does not have any non-terminals on the right hand side, the calling non-terminal // is a candidate for being called from the axiom for outside @@ -1088,6 +1090,12 @@ void Grammar::inject_outside_nts() { } } } + Alt::Link *link = new Alt::Link(this->axiom_name, this->axiom_loc); + link->nt = dynamic_cast(this->axiom); + Filter *f = new Filter(new std::string("complete_track"), Loc()); + f->type = Filter::WITH; + link->filters.push_back(f); + dynamic_cast(outside_NTs.find(*this->axiom_name)->second)->alts.push_back(link); // 4) add novel outside NTs to grammar for (std::pair nt : outside_NTs) { From 0b76c150d9c652ce7e5ef6f2d64198df587168ad Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 7 Apr 2022 14:21:49 +0200 Subject: [PATCH 048/209] discriminate between outside and inside guards (outside rules needs both!) --- src/alt.hh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/alt.hh b/src/alt.hh index 553cdcf98..f0150fc26 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -281,11 +281,13 @@ class Base { protected: Yield::Multi m_ys; + public: virtual void init_multi_ys(); const Yield::Multi &multi_ys() const { return m_ys; } + Yield::Multi m_ys_inside; private: std::vector > multi_filter; @@ -339,6 +341,10 @@ class Base { // private flag to indicate if alternative is for inside (the default) or outside // production rules. See set_partof_outside() bool is_partof_outside; + public: + bool get_is_partof_outside() { + return this->is_partof_outside; + } }; @@ -423,6 +429,7 @@ class Simple : public Base { std::list body_stmts; Statement::If *guards; + Statement::If *guards_inside; void ret_decl_empty_block(Statement::If *stmt); void deep_erase_if_backtrace( Statement::If *stmt, std::vector::iterator start, @@ -485,6 +492,9 @@ class Simple : public Base { std::list *stmts, std::list loops, bool has_index_overlay); + std::list *add_guards( + std::list *stmts, + Statement::If *guards); void sum_rhs( Yield::Multi &y, std::list::const_iterator i, const std::list::const_iterator &end) const; @@ -650,6 +660,7 @@ class Link : public Base { private: std::list ntparas; + Statement::If *guards; public: void set_ntparas(const Loc &loc, std::list *l); @@ -661,6 +672,7 @@ class Link : public Base { void set_partof_outside(bool is_outside); bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); + void init_outside_guards(); }; From 56dee564d6fcd8379b2282a85c3e76e85cd44b3f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 7 Apr 2022 14:22:51 +0200 Subject: [PATCH 049/209] add inside guards --- src/alt.cc | 122 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 104 insertions(+), 18 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index b648393cb..e533fa720 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -1324,7 +1324,6 @@ void Alt::Simple::init_foreach() { } } - void Alt::Simple::ret_decl_empty_block(Statement::If *stmt) { if (!datatype->simple()->is(::Type::LIST)) { Statement::Fn_Call *e = new Statement::Fn_Call(Statement::Fn_Call::EMPTY); @@ -1508,18 +1507,37 @@ void Alt::Simple::init_body(AST &ast) { } } +void Alt::Link::init_outside_guards() { + std::list l; + + unsigned int track = 0; + std::vector::iterator j = right_indices.begin(); + for (std::vector::iterator i = left_indices.begin(); + i != left_indices.end(); ++i, ++j, ++track) { + std::ostringstream o_left; o_left << "t_" << track << "_right_most"; + Expr::Vacc *leftmost = new Expr::Vacc(new std::string(o_left.str())); + l.push_back(new Expr::Greater_Eq((*i), leftmost)); + } + + Expr::Base *cond = Expr::seq_to_tree( + l.begin(), l.end()); + guards = new Statement::If(cond); + //ret_decl_empty_block(guards); +} void Alt::Simple::init_guards() { std::list l; + std::list l_inside; assert(m_ys.tracks() == left_indices.size()); Yield::Multi::iterator k = m_ys.begin(); + Yield::Multi::iterator k_inside = m_ys_inside.begin(); std::vector::iterator j = right_indices.begin(); std::vector::iterator it_left_inner_indices = this->left_inside_indices.begin(); std::vector::iterator it_right_inner_indices = this->right_inside_indices.begin(); unsigned int track = 0; for (std::vector::iterator i = left_indices.begin(); - i != left_indices.end(); ++i, ++j, ++k, ++track, ++it_left_inner_indices, ++it_right_inner_indices) { + i != left_indices.end(); ++i, ++j, ++k, ++k_inside, ++track, ++it_left_inner_indices, ++it_right_inner_indices) { if (this->is_partof_outside) { // obtain yield sizes left and right of outside non-terminal std::pair *ys = this->get_outside_accum_yieldsizes(track); @@ -1532,7 +1550,8 @@ void Alt::Simple::init_guards() { // the rare case where guards check themselves // happens e.g. if table has only one dimension } else { - l.push_back(new Expr::Greater_Eq((*it_left_inner_indices)->minus(ys->first->low()), leftmost)); + l.push_back(new Expr::Greater_Eq((*it_left_inner_indices), leftmost->plus(ys->first->low()))); + //l.push_back(new Expr::Greater_Eq((*it_left_inner_indices)->minus(ys->first->low()), leftmost)); } std::ostringstream o_right; o_right << "t_" << track << "_right_most"; @@ -1545,6 +1564,15 @@ void Alt::Simple::init_guards() { } else { l.push_back(new Expr::Less_Eq((*it_right_inner_indices)->plus(ys->second->low()), rightmost)); } + + // additionally, add guards from original inside rules + if (k_inside != m_ys_inside.end()) { + Expr::Base *e = (*j)->minus(*i); + l_inside.push_back(new Expr::Greater_Eq(e, (*k_inside).low())); + if ((*k_inside).high() != Yield::Poly(Yield::UP)) { + l_inside.push_back(new Expr::Less_Eq(e, (*k_inside).high())); + } + } } else { Expr::Base *e = (*j)->minus(*i); l.push_back(new Expr::Greater_Eq(e, (*k).low())); @@ -1559,6 +1587,14 @@ void Alt::Simple::init_guards() { guards = new Statement::If(cond); ret_decl_empty_block(guards); + + if (l_inside.size() > 0) { + Expr::Base *cond = Expr::seq_to_tree( + l_inside.begin(), l_inside.end()); + + guards_inside = new Statement::If(cond); + ret_decl_empty_block(guards_inside); + } } @@ -1884,6 +1920,24 @@ std::list *Alt::Simple::add_for_loops( } return stmts; } +std::list *Alt::Simple::add_guards( + std::list *stmts, + Statement::If *guards) { + if (guards) { + stmts->push_back(guards); + if (datatype->simple()->is(::Type::LIST)) { + // only call finalize on hash lists + if (!ret_decl->rhs && adp_specialization == ADP_Mode::STANDARD) { + Statement::Fn_Call *f = new Statement::Fn_Call( + Statement::Fn_Call::FINALIZE); + f->add_arg(*ret_decl); + stmts->push_back(f); + } + } + stmts = &guards->then; + } + return stmts; +} void Alt::Simple::codegen(AST &ast) { // std::cout << "-----------Simple IN" << std::endl; @@ -1907,19 +1961,20 @@ void Alt::Simple::codegen(AST &ast) { init_guards(); - if (guards) { - stmts->push_back(guards); - if (datatype->simple()->is(::Type::LIST)) { - // only call finalize on hash lists - if (!ret_decl->rhs && adp_specialization == ADP_Mode::STANDARD) { - Statement::Fn_Call *f = new Statement::Fn_Call( - Statement::Fn_Call::FINALIZE); - f->add_arg(*ret_decl); - stmts->push_back(f); - } - } - stmts = &guards->then; - } +// if (guards) { +// stmts->push_back(guards); +// if (datatype->simple()->is(::Type::LIST)) { +// // only call finalize on hash lists +// if (!ret_decl->rhs && adp_specialization == ADP_Mode::STANDARD) { +// Statement::Fn_Call *f = new Statement::Fn_Call( +// Statement::Fn_Call::FINALIZE); +// f->add_arg(*ret_decl); +// stmts->push_back(f); +// } +// } +// stmts = &guards->then; +// } + stmts = add_guards(stmts, guards); init_filter_guards(ast); @@ -1966,6 +2021,22 @@ void Alt::Simple::codegen(AST &ast) { // add for loops for moving boundaries stmts = add_for_loops(stmts, loops, has_index_overlay()); +// if (guards_inside) { +// stmts->push_back(guards_inside); +// if (datatype->simple()->is(::Type::LIST)) { +// // only call finalize on hash lists +// if (!ret_decl->rhs && adp_specialization == ADP_Mode::STANDARD) { +// Statement::Fn_Call *f = new Statement::Fn_Call( +// Statement::Fn_Call::FINALIZE); +// f->add_arg(*ret_decl); +// stmts->push_back(f); +// } +// } +// stmts = &guards_inside->then; +// } + // guards_inside have already initialized through init_guards() + stmts = add_guards(stmts, guards_inside); + // add filter_guards stmts = add_filter_guards(stmts, filter_guards); } else { @@ -2105,6 +2176,21 @@ void Alt::Link::codegen(AST &ast) { fn->exprs.push_back(new Expr::Vacc(new std::string("delta"))); } +// std::list *stmts = &statements; +// if (guards) { +// stmts->push_back(guards); +// if (datatype->simple()->is(::Type::LIST)) { +// // only call finalize on hash lists +// if (!ret_decl->rhs && adp_specialization == ADP_Mode::STANDARD) { +// Statement::Fn_Call *f = new Statement::Fn_Call( +// Statement::Fn_Call::FINALIZE); +// f->add_arg(*ret_decl); +// stmts->push_back(f); +// } +// } +// stmts = &guards->then; +// } + init_filter_guards(ast); if (filter_guards) { Statement::Var_Assign *v = new Statement::Var_Assign(*ret_decl, fn); @@ -3378,7 +3464,7 @@ void Alt::Multi::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, throw LogError("Alt::Block::expand_outside_nt_indices is not yet implemented!"); } void Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { - Alt::Base::init_indices(left, right, k, track); + Alt::Base::init_indices(left, right, k, track); } void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { @@ -3471,7 +3557,7 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsi Yield::Size *ys_right2center = new Yield::Size(); *ys_right2center += *(ys_sub->first); *ys_right2center += *(sum_ys_rights(&ys_arg, i, args_left->end(), track)); - if (arg_outside == NULL) { + if ((arg_outside == NULL) and (ys_arg.low() > 0)) { // if this level does NOT hold the outside non-terminal, but args are left of the outside non-terminal // propagate the rightmost index (must be the one from the outside non-terminal in higher levels) // as the left index From 02bc01792c58117ab5444b6f06b5e34c4cc0038b Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 7 Apr 2022 14:53:48 +0200 Subject: [PATCH 050/209] added a filter for outside generation: ensure that full track-input is consumed by inside axiom --- rtlib/filter.hh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rtlib/filter.hh b/rtlib/filter.hh index 4d4075ba5..5b012b994 100644 --- a/rtlib/filter.hh +++ b/rtlib/filter.hh @@ -127,5 +127,9 @@ inline bool samesize(const Basic_Sequence &s1, return j1-i1 == j2-i2; } +template +inline bool complete_track(const Basic_Sequence &seq, T i, T j) { + return ((i == seq.n) and (j == seq.n)); +} #endif // RTLIB_FILTER_HH_ From d9499aeab7cb76c9ab22450568daece3529fd889 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 8 Apr 2022 11:01:29 +0200 Subject: [PATCH 051/209] edge case with non-quadratic non-terminals for yield size computation --- src/alt.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/alt.cc b/src/alt.cc index e533fa720..cfd41ffa1 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3657,6 +3657,17 @@ std::pair *Alt::Simple::get_outside_accum_yieldsizes if ((arg_outside == NULL) and (is_right_of_outside_nt == false)) { *(res->first) += (*i)->multi_ys()(track); } else { + // if the alternative links to a non-terminal AND the non-terminal has a sub-quadratic table, we do NOT add the yield size + if (arg_outside != NULL) { + if (arg_outside->alt_ref()->is(Alt::LINK)) { + Symbol::NT *linkedNT = dynamic_cast(dynamic_cast(arg_outside->alt_ref())->nt); + if (linkedNT) { + if (linkedNT->tables()[track].is_cyk_right()) { + continue; + } + } + } + } *(res->second) += (*i)->multi_ys()(track); } } From d55f6dac3a02f67b583c4405a894d44cbae92d67 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 8 Apr 2022 11:02:03 +0200 Subject: [PATCH 052/209] new test-case --- .../grammar_outside/nodangle_tooManyLoops.gap | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 testdata/grammar_outside/nodangle_tooManyLoops.gap diff --git a/testdata/grammar_outside/nodangle_tooManyLoops.gap b/testdata/grammar_outside/nodangle_tooManyLoops.gap new file mode 100644 index 000000000..da00de210 --- /dev/null +++ b/testdata/grammar_outside/nodangle_tooManyLoops.gap @@ -0,0 +1,212 @@ +import rna + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int sadd(Subsequence lb, int x) { + return x + sbase_energy(); + } + int cadd(int x, int y) { + return x + y; + } + int drem(Subsequence lb, int x, Subsequence rb) { + return x + termau_energy(lb, rb); + } + int sr(Subsequence lb, int x, Subsequence rb) { + return x + sr_energy(lb, rb); + } + int hl(Subsequence lb, Subsequence r, Subsequence rb) { + return hl_energy(r); + } + int bl(Subsequence lb, Subsequence lr, int x, Subsequence rb) { + return x + bl_energy(lr, rb); + } + int br(Subsequence lb, int x, Subsequence rr, Subsequence rb) { + return x + br_energy(lb, rr); + } + int il(Subsequence lb, Subsequence lr, int x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } + int ml(Subsequence lb, int x, Subsequence rb) { + return x + ml_energy() + ul_energy() + termau_energy(lb, rb); + } + int incl(int x) { + return x + ul_energy(); + } + int addss(int x, Subsequence r) { + return x + ss_energy(r); + } + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } +} + +algebra alg_pfunc implements sig_foldrna(alphabet = char, answer = double) { + double sadd(Subsequence lb, double x) { + return scale(1) * x * mk_pf(sbase_energy()); + } + double cadd(double x, double y) { + return x * y; + } + double drem(Subsequence lb, double x, Subsequence rb) { + return x * mk_pf(termau_energy(lb, rb)); + } + double sr(Subsequence lb, double x, Subsequence rb) { + return scale(2) * x * mk_pf(sr_energy(lb, rb)); + } + double hl(Subsequence lb, Subsequence r, Subsequence rb) { + return scale(2+r.j-r.i) * mk_pf(hl_energy(r)); + } + double bl(Subsequence lb, Subsequence lr, double x, Subsequence rb) { + return scale(2+lr.j-lr.i) * x * mk_pf(bl_energy(lr, rb)); + } + double br(Subsequence lb, double x, Subsequence rr, Subsequence rb) { + return scale(2+rr.j-rr.i) * x * mk_pf(br_energy(lb, rr)); + } + double il(Subsequence lb, Subsequence lr, double x, Subsequence rr, Subsequence rb) { + return scale(2+lr.j-lr.i+rr.j-rr.i) * x * mk_pf(il_energy(lr, rr)); + } + double ml(Subsequence lb, double x, Subsequence rb) { + return scale(2) * x * mk_pf(ml_energy() + ul_energy() + termau_energy(lb, rb)); + } + double incl(double x) { + return x * mk_pf(ul_energy()); + } + double addss(double x, Subsequence r) { + return scale(r.j-r.i) * x * mk_pf(ss_energy(r)); + } + double nil(Subsequence n) { + return 1; + } + choice [double] h([double] i) { + return list(sum(i)); + } +} + +algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { + string sadd(Subsequence lb,string e) { + string res; + append(res, '.'); + append(res, e); + return res; + } + + string cadd(string le,string re) { + string res; + append(res, le); + append(res, re); + return res; + } + + string nil(Subsequence loc) { + string r; + return r; + } + + string drem(Subsequence lloc, string e, Subsequence rloc) { + return e; + } + + string sr(Subsequence lb,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string hl(Subsequence lb,Subsequence region,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(region)); + append(res, ')'); + return res; + } + + + string bl(Subsequence lb,Subsequence lregion,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, ')'); + return res; + } + + string br(Subsequence lb,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string il(Subsequence lb,Subsequence lregion,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string ml(Subsequence lb, string e, Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string addss(string e,Subsequence rb) { + string res; + append(res, e); + append(res, '.', size(rb)); + return res; + } + + string incl(string e) { + return e; + } + + choice [string] h([string] i) { + return i; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = sadd(BASE, struct) | + cadd(hairpin, struct) | + nil(LOC) # h; + hairpin = hl(BASE, REGION with minsize(3), BASE) with basepairing # h; + +} + +instance mfe = gra_nodangle(alg_mfe); +instance pfunc = gra_nodangle(alg_pfunc); +instance count = gra_nodangle(alg_count); +instance ins_enum = gra_nodangle(alg_enum); From 7246ec6000a8dc00f591d1894df82ef34a4b1a0b Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 8 Apr 2022 15:44:10 +0200 Subject: [PATCH 053/209] general fix for erase(answers)?! --- src/alt.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index cfd41ffa1..58e7078af 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -2392,8 +2392,14 @@ void Alt::Base::init_filter_guards(AST &ast) { Expr::Base *arg = Expr::seq_to_tree( exprs.begin(), exprs.end()); Statement::If *guard = new Statement::If(arg); - guard->els.push_back (new Statement::Fn_Call( - Statement::Fn_Call::EMPTY, *ret_decl)); + if ((this->data_type()->is(::Type::LIST)) and (this->top_level)) { + // don't add an else statement (which erases the return type) to the filter + // since this would erase the answer list of ALL alternatives of a symbol + // due to the return decl replacement happening in Symbol::NT::eliminate_list_ass + } else { + guard->els.push_back(new Statement::Fn_Call( + Statement::Fn_Call::EMPTY, *ret_decl)); + } filter_guards = guard; } From b7167598f201fcdf58b4186c464cabf174069a9f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 10 Apr 2022 23:42:11 +0200 Subject: [PATCH 054/209] avoid calling yield size analysis for outside version as it would distort table guards --- .../nodangle_emptyanswerissue.gap | 217 ++++++++++++++++++ .../grammar_outside/nodangle_filterempty.gap | 217 ++++++++++++++++++ 2 files changed, 434 insertions(+) create mode 100644 testdata/grammar_outside/nodangle_emptyanswerissue.gap create mode 100644 testdata/grammar_outside/nodangle_filterempty.gap diff --git a/testdata/grammar_outside/nodangle_emptyanswerissue.gap b/testdata/grammar_outside/nodangle_emptyanswerissue.gap new file mode 100644 index 000000000..27064ec58 --- /dev/null +++ b/testdata/grammar_outside/nodangle_emptyanswerissue.gap @@ -0,0 +1,217 @@ +import rna + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int sadd(Subsequence lb, int x) { + return x + sbase_energy(); + } + int cadd(int x, int y) { + return x + y; + } + int drem(Subsequence lb, int x, Subsequence rb) { + return x + termau_energy(lb, rb); + } + int sr(Subsequence lb, int x, Subsequence rb) { + return x + sr_energy(lb, rb); + } + int hl(Subsequence lb, Subsequence r, Subsequence rb) { + return hl_energy(r); + } + int bl(Subsequence lb, Subsequence lr, int x, Subsequence rb) { + return x + bl_energy(lr, rb); + } + int br(Subsequence lb, int x, Subsequence rr, Subsequence rb) { + return x + br_energy(lb, rr); + } + int il(Subsequence lb, Subsequence lr, int x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } + int ml(Subsequence lb, int x, Subsequence rb) { + return x + ml_energy() + ul_energy() + termau_energy(lb, rb); + } + int incl(int x) { + return x + ul_energy(); + } + int addss(int x, Subsequence r) { + return x + ss_energy(r); + } + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } +} + +algebra alg_pfunc implements sig_foldrna(alphabet = char, answer = double) { + double sadd(Subsequence lb, double x) { + return scale(1) * x * mk_pf(sbase_energy()); + } + double cadd(double x, double y) { + return x * y; + } + double drem(Subsequence lb, double x, Subsequence rb) { + return x * mk_pf(termau_energy(lb, rb)); + } + double sr(Subsequence lb, double x, Subsequence rb) { + return scale(2) * x * mk_pf(sr_energy(lb, rb)); + } + double hl(Subsequence lb, Subsequence r, Subsequence rb) { + return scale(2+r.j-r.i) * mk_pf(hl_energy(r)); + } + double bl(Subsequence lb, Subsequence lr, double x, Subsequence rb) { + return scale(2+lr.j-lr.i) * x * mk_pf(bl_energy(lr, rb)); + } + double br(Subsequence lb, double x, Subsequence rr, Subsequence rb) { + return scale(2+rr.j-rr.i) * x * mk_pf(br_energy(lb, rr)); + } + double il(Subsequence lb, Subsequence lr, double x, Subsequence rr, Subsequence rb) { + return scale(2+lr.j-lr.i+rr.j-rr.i) * x * mk_pf(il_energy(lr, rr)); + } + double ml(Subsequence lb, double x, Subsequence rb) { + return scale(2) * x * mk_pf(ml_energy() + ul_energy() + termau_energy(lb, rb)); + } + double incl(double x) { + return x * mk_pf(ul_energy()); + } + double addss(double x, Subsequence r) { + return scale(r.j-r.i) * x * mk_pf(ss_energy(r)); + } + double nil(Subsequence n) { + return 1; + } + choice [double] h([double] i) { + return list(sum(i)); + } +} + +algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { + string sadd(Subsequence lb,string e) { + string res; + append(res, '.'); + append(res, e); + return res; + } + + string cadd(string le,string re) { + string res; + append(res, le); + append(res, re); + return res; + } + + string nil(Subsequence loc) { + string r; + return r; + } + + string drem(Subsequence lloc, string e, Subsequence rloc) { + return e; + } + + string sr(Subsequence lb,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string hl(Subsequence lb,Subsequence region,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(region)); + append(res, ')'); + return res; + } + + + string bl(Subsequence lb,Subsequence lregion,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, ')'); + return res; + } + + string br(Subsequence lb,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string il(Subsequence lb,Subsequence lregion,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string ml(Subsequence lb, string e, Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string addss(string e,Subsequence rb) { + string res; + append(res, e); + append(res, '.', size(rb)); + return res; + } + + string incl(string e) { + return e; + } + + choice [string] h([string] i) { + return i; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = sadd(BASE, struct) | + cadd(weak, struct) | + nil(LOC) # h; + + + weak = hl(BASE, REGION with minsize(3), BASE) with basepairing | + bl(BASE, REGION with maxsize(30), weak, BASE) with basepairing | + il(BASE, REGION with maxsize(30), weak, REGION with maxsize(30), BASE) with basepairing + # h; + +} + +instance ins_mfe = gra_nodangle(alg_mfe); +instance ins_pfunc = gra_nodangle(alg_pfunc); +instance ins_count = gra_nodangle(alg_count); +instance ins_enum = gra_nodangle(alg_enum); diff --git a/testdata/grammar_outside/nodangle_filterempty.gap b/testdata/grammar_outside/nodangle_filterempty.gap new file mode 100644 index 000000000..4b961e225 --- /dev/null +++ b/testdata/grammar_outside/nodangle_filterempty.gap @@ -0,0 +1,217 @@ +import rna + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int sadd(Subsequence lb, int x) { + return x + sbase_energy(); + } + int cadd(int x, int y) { + return x + y; + } + int drem(Subsequence lb, int x, Subsequence rb) { + return x + termau_energy(lb, rb); + } + int sr(Subsequence lb, int x, Subsequence rb) { + return x + sr_energy(lb, rb); + } + int hl(Subsequence lb, Subsequence r, Subsequence rb) { + return hl_energy(r); + } + int bl(Subsequence lb, Subsequence lr, int x, Subsequence rb) { + return x + bl_energy(lr, rb); + } + int br(Subsequence lb, int x, Subsequence rr, Subsequence rb) { + return x + br_energy(lb, rr); + } + int il(Subsequence lb, Subsequence lr, int x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } + int ml(Subsequence lb, int x, Subsequence rb) { + return x + ml_energy() + ul_energy() + termau_energy(lb, rb); + } + int incl(int x) { + return x + ul_energy(); + } + int addss(int x, Subsequence r) { + return x + ss_energy(r); + } + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } +} + +algebra alg_pfunc implements sig_foldrna(alphabet = char, answer = double) { + double sadd(Subsequence lb, double x) { + return scale(1) * x * mk_pf(sbase_energy()); + } + double cadd(double x, double y) { + return x * y; + } + double drem(Subsequence lb, double x, Subsequence rb) { + return x * mk_pf(termau_energy(lb, rb)); + } + double sr(Subsequence lb, double x, Subsequence rb) { + return scale(2) * x * mk_pf(sr_energy(lb, rb)); + } + double hl(Subsequence lb, Subsequence r, Subsequence rb) { + return scale(2+r.j-r.i) * mk_pf(hl_energy(r)); + } + double bl(Subsequence lb, Subsequence lr, double x, Subsequence rb) { + return scale(2+lr.j-lr.i) * x * mk_pf(bl_energy(lr, rb)); + } + double br(Subsequence lb, double x, Subsequence rr, Subsequence rb) { + return scale(2+rr.j-rr.i) * x * mk_pf(br_energy(lb, rr)); + } + double il(Subsequence lb, Subsequence lr, double x, Subsequence rr, Subsequence rb) { + return scale(2+lr.j-lr.i+rr.j-rr.i) * x * mk_pf(il_energy(lr, rr)); + } + double ml(Subsequence lb, double x, Subsequence rb) { + return scale(2) * x * mk_pf(ml_energy() + ul_energy() + termau_energy(lb, rb)); + } + double incl(double x) { + return x * mk_pf(ul_energy()); + } + double addss(double x, Subsequence r) { + return scale(r.j-r.i) * x * mk_pf(ss_energy(r)); + } + double nil(Subsequence n) { + return 1; + } + choice [double] h([double] i) { + return list(sum(i)); + } +} + +algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { + string sadd(Subsequence lb,string e) { + string res; + append(res, '.'); + append(res, e); + return res; + } + + string cadd(string le,string re) { + string res; + append(res, le); + append(res, re); + return res; + } + + string nil(Subsequence loc) { + string r; + return r; + } + + string drem(Subsequence lloc, string e, Subsequence rloc) { + return e; + } + + string sr(Subsequence lb,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string hl(Subsequence lb,Subsequence region,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(region)); + append(res, ')'); + return res; + } + + + string bl(Subsequence lb,Subsequence lregion,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, ')'); + return res; + } + + string br(Subsequence lb,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string il(Subsequence lb,Subsequence lregion,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string ml(Subsequence lb, string e, Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string addss(string e,Subsequence rb) { + string res; + append(res, e); + append(res, '.', size(rb)); + return res; + } + + string incl(string e) { + return e; + } + + choice [string] h([string] i) { + return i; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = sadd(BASE, struct) | + cadd(weak, struct) | + nil(LOC) # h; + + weak = hairpin | + leftB + # h; + + hairpin = hl(BASE, REGION with minsize(3), BASE) with basepairing # h; + leftB = sr(BASE, bl(BASE, REGION with maxsize(30), weak, BASE) with basepairing, BASE with basepairing) # h; +} + +instance ins_mfe = gra_nodangle(alg_mfe); +instance ins_pfunc = gra_nodangle(alg_pfunc); +instance ins_count = gra_nodangle(alg_count); +instance ins_enum = gra_nodangle(alg_enum); From 713eaccf29814f7bda1aa3561cde9910c862e312 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 11 Apr 2022 14:29:44 +0200 Subject: [PATCH 055/209] properly initialize novel NTs for outside generation --- src/grammar.cc | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/grammar.cc b/src/grammar.cc index 233ebeca1..c50979d9f 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1095,6 +1095,8 @@ void Grammar::inject_outside_nts() { Filter *f = new Filter(new std::string("complete_track"), Loc()); f->type = Filter::WITH; link->filters.push_back(f); + link->set_tracks(this->axiom->tracks(), this->axiom->track_pos()); + link->init_multi_ys(); dynamic_cast(outside_NTs.find(*this->axiom_name)->second)->alts.push_back(link); // 4) add novel outside NTs to grammar @@ -1132,9 +1134,16 @@ void Grammar::inject_outside_nts() { // which points to multiple non-terminals WITHOUT making a choice std::string *nt_axiom_name = new std::string("outside_axioms"); Symbol::NT *nt_axiom = new Symbol::NT(nt_axiom_name, Loc()); + // carry over tracks from original inside axiom + nt_axiom->set_tracks(this->axiom->tracks(), this->axiom->track_pos()); + nt_axiom->setup_multi_ys(); for (std::set::iterator i = axiom_candidates->begin(); i != axiom_candidates->end(); ++i) { - nt_axiom->alts.push_back(new Alt::Link((*i)->name, Loc())); + Alt::Link *link = new Alt::Link((*i)->name, Loc()); + link->nt = dynamic_cast(outside_NTs.find(*(*i)->name)->second); + link->set_tracks((*i)->tracks(), (*i)->track_pos()); + link->init_multi_ys(); + nt_axiom->alts.push_back(link); } // add new lhs non-terminal to grammar this->NTs.insert(std::make_pair(*nt_axiom_name, dynamic_cast(nt_axiom))); @@ -1143,8 +1152,13 @@ void Grammar::inject_outside_nts() { this->axiom_name = nt_axiom_name; this->init_axiom(); } - // re-run check symantics to properly integrate new production rules - this->check_semantic(); + // re-run parts of "check_semantics" to properly initialize novel non- + // terminals and links to non-terminals, but explicitly do NOT + // re-run yield size analysis since it does not respect outside + // situations. + this->init_calls(); + this->init_in_out(); + this->init_table_dims(); } unsigned int Grammar::to_dot(unsigned int *nodeID, std::ostream &out) { From aeb99c04b44594bd3d519079d57ff7304bd4804e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 11 Apr 2022 21:56:55 +0200 Subject: [PATCH 056/209] make toplevel --- src/grammar.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/grammar.cc b/src/grammar.cc index c50979d9f..e84d4f232 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1097,6 +1097,7 @@ void Grammar::inject_outside_nts() { link->filters.push_back(f); link->set_tracks(this->axiom->tracks(), this->axiom->track_pos()); link->init_multi_ys(); + link->top_level = Bool(true); dynamic_cast(outside_NTs.find(*this->axiom_name)->second)->alts.push_back(link); // 4) add novel outside NTs to grammar From 6794e0adabdf4131a0417bde7dcdb02cafe635d7 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 12 Apr 2022 11:54:00 +0200 Subject: [PATCH 057/209] start adding a cmd flag for outside generation --- src/gapc.cc | 6 ++++++ src/options.hh | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/src/gapc.cc b/src/gapc.cc index 35dcc6e76..0e2e3ac1e 100644 --- a/src/gapc.cc +++ b/src/gapc.cc @@ -104,6 +104,10 @@ static void parse_options(int argc, char **argv, Options *rec) { "uses the selected instance and creates a GAP program which creates " "specialized GAP programs that recognize a subset of candidates of the " "original grammar.") + ("outside_grammar", po::value< std::vector >(), + "generate an outside version of the grammar and report outside results " + "for the provided list of inside non-terminals. Will report ALL non-" + "terminals, if provided list says 'ALL'.") ("verbose", "show suppressed warnings and messages") ("log-level,l", po::value(), "the log level, valid values are 0 (VERBOSE), 1 (INFO), 2 (NORMAL), 3 " @@ -221,6 +225,8 @@ static void parse_options(int argc, char **argv, Options *rec) { if (vm.count("specialize_grammar")) { rec->specializeGrammar = true; } + if (vm.count("outside_grammar")) + rec->outside_nt_list = vm["outside_grammar"].as< std::vector >(); if (vm.count("verbose")) rec->verbose_mode = true; if (vm.count("log-level")) { diff --git a/src/options.hh b/src/options.hh index ec3a598d6..9c0a49653 100644 --- a/src/options.hh +++ b/src/options.hh @@ -115,6 +115,11 @@ struct Options { // ignore most of the other options (except the instance selector // and output file name). bool specializeGrammar; + // if not empty, automatically generate an outside version of + // the provided grammar and print out results of the provided + // list of outside non-terminals. Report ALL non-terminals, + // if list states 'ALL'. + std::vector outside_nt_list; // flag that is used to turn on verbose mode, i.g. all suppressed // warnings and messages will be shown bool verbose_mode; From 5a181b4cf8101c44e93a5adc83e699f5686bd191 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 12 Apr 2022 14:30:08 +0200 Subject: [PATCH 058/209] cpplint codestyle changes --- src/alt.cc | 1011 ++++++++++++++++++++++++++---------------------- src/alt.hh | 116 ++++-- src/cpp.cc | 10 +- src/gapc.cc | 15 +- src/grammar.cc | 370 ++++++++++-------- src/grammar.hh | 2 +- src/symbol.cc | 8 +- src/symbol.hh | 3 +- 8 files changed, 871 insertions(+), 664 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 58e7078af..eefaba9de 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -1508,22 +1508,22 @@ void Alt::Simple::init_body(AST &ast) { } void Alt::Link::init_outside_guards() { - std::list l; + std::list l; - unsigned int track = 0; - std::vector::iterator j = right_indices.begin(); - for (std::vector::iterator i = left_indices.begin(); - i != left_indices.end(); ++i, ++j, ++track) { - std::ostringstream o_left; o_left << "t_" << track << "_right_most"; - Expr::Vacc *leftmost = new Expr::Vacc(new std::string(o_left.str())); - l.push_back(new Expr::Greater_Eq((*i), leftmost)); - } + unsigned int track = 0; + std::vector::iterator j = right_indices.begin(); + for (std::vector::iterator i = left_indices.begin(); + i != left_indices.end(); ++i, ++j, ++track) { + std::ostringstream o_left; o_left << "t_" << track << "_right_most"; + Expr::Vacc *leftmost = new Expr::Vacc(new std::string(o_left.str())); + l.push_back(new Expr::Greater_Eq((*i), leftmost)); + } - Expr::Base *cond = Expr::seq_to_tree( - l.begin(), l.end()); + Expr::Base *cond = Expr::seq_to_tree( + l.begin(), l.end()); - guards = new Statement::If(cond); - //ret_decl_empty_block(guards); + guards = new Statement::If(cond); + // ret_decl_empty_block(guards); } void Alt::Simple::init_guards() { std::list l; @@ -1532,54 +1532,62 @@ void Alt::Simple::init_guards() { Yield::Multi::iterator k = m_ys.begin(); Yield::Multi::iterator k_inside = m_ys_inside.begin(); std::vector::iterator j = right_indices.begin(); - std::vector::iterator it_left_inner_indices = this->left_inside_indices.begin(); - std::vector::iterator it_right_inner_indices = this->right_inside_indices.begin(); + std::vector::iterator it_left_inner_indices = + this->left_inside_indices.begin(); + std::vector::iterator it_right_inner_indices = + this->right_inside_indices.begin(); unsigned int track = 0; for (std::vector::iterator i = left_indices.begin(); - i != left_indices.end(); ++i, ++j, ++k, ++k_inside, ++track, ++it_left_inner_indices, ++it_right_inner_indices) { - if (this->is_partof_outside) { - // obtain yield sizes left and right of outside non-terminal - std::pair *ys = this->get_outside_accum_yieldsizes(track); - - std::ostringstream o_left; o_left << "t_" << track << "_left_most"; - Expr::Vacc *leftmost = new Expr::Vacc(new std::string(o_left.str())); - assert(*it_left_inner_indices); - Expr::Vacc *lname = dynamic_cast(*it_left_inner_indices); - if (lname && (lname->name()->compare(*leftmost->name()) == 0)) { - // the rare case where guards check themselves - // happens e.g. if table has only one dimension - } else { - l.push_back(new Expr::Greater_Eq((*it_left_inner_indices), leftmost->plus(ys->first->low()))); - //l.push_back(new Expr::Greater_Eq((*it_left_inner_indices)->minus(ys->first->low()), leftmost)); - } - - std::ostringstream o_right; o_right << "t_" << track << "_right_most"; - Expr::Vacc *rightmost = new Expr::Vacc(new std::string(o_right.str())); - assert(*it_right_inner_indices); - Expr::Vacc *rname = dynamic_cast(*it_right_inner_indices); - if (rname && (rname->name()->compare(*rightmost->name()) == 0)) { - // the rare case where guards check themselves - // happens e.g. if table has only one dimension - } else { - l.push_back(new Expr::Less_Eq((*it_right_inner_indices)->plus(ys->second->low()), rightmost)); - } - - // additionally, add guards from original inside rules - if (k_inside != m_ys_inside.end()) { - Expr::Base *e = (*j)->minus(*i); - l_inside.push_back(new Expr::Greater_Eq(e, (*k_inside).low())); - if ((*k_inside).high() != Yield::Poly(Yield::UP)) { - l_inside.push_back(new Expr::Less_Eq(e, (*k_inside).high())); - } - } - } else { + i != left_indices.end(); + ++i, ++j, ++k, ++k_inside, ++track, ++it_left_inner_indices, + ++it_right_inner_indices) { + if (this->is_partof_outside) { + // obtain yield sizes left and right of outside non-terminal + std::pair *ys = + this->get_outside_accum_yieldsizes(track); + + std::ostringstream o_left; o_left << "t_" << track << "_left_most"; + Expr::Vacc *leftmost = new Expr::Vacc(new std::string(o_left.str())); + assert(*it_left_inner_indices); + Expr::Vacc *lname = dynamic_cast(*it_left_inner_indices); + if (lname && (lname->name()->compare(*leftmost->name()) == 0)) { + // the rare case where guards check themselves + // happens e.g. if table has only one dimension + } else { + l.push_back(new Expr::Greater_Eq((*it_left_inner_indices), + leftmost->plus(ys->first->low()))); + // l.push_back(new Expr::Greater_Eq((*it_left_inner_indices) + // ->minus(ys->first->low()), leftmost)); + } + + std::ostringstream o_right; o_right << "t_" << track << "_right_most"; + Expr::Vacc *rightmost = new Expr::Vacc(new std::string(o_right.str())); + assert(*it_right_inner_indices); + Expr::Vacc *rname = dynamic_cast(*it_right_inner_indices); + if (rname && (rname->name()->compare(*rightmost->name()) == 0)) { + // the rare case where guards check themselves + // happens e.g. if table has only one dimension + } else { + l.push_back(new Expr::Less_Eq( + (*it_right_inner_indices)->plus(ys->second->low()), rightmost)); + } + + // additionally, add guards from original inside rules + if (k_inside != m_ys_inside.end()) { + Expr::Base *e = (*j)->minus(*i); + l_inside.push_back(new Expr::Greater_Eq(e, (*k_inside).low())); + if ((*k_inside).high() != Yield::Poly(Yield::UP)) { + l_inside.push_back(new Expr::Less_Eq(e, (*k_inside).high())); + } + } + } else { Expr::Base *e = (*j)->minus(*i); l.push_back(new Expr::Greater_Eq(e, (*k).low())); if ((*k).high() != Yield::Poly(Yield::UP)) { l.push_back(new Expr::Less_Eq(e, (*k).high())); } - } + } } Expr::Base *cond = Expr::seq_to_tree( @@ -1589,11 +1597,11 @@ void Alt::Simple::init_guards() { ret_decl_empty_block(guards); if (l_inside.size() > 0) { - Expr::Base *cond = Expr::seq_to_tree( - l_inside.begin(), l_inside.end()); + Expr::Base *cond = Expr::seq_to_tree( + l_inside.begin(), l_inside.end()); - guards_inside = new Statement::If(cond); - ret_decl_empty_block(guards_inside); + guards_inside = new Statement::If(cond); + ret_decl_empty_block(guards_inside); } } @@ -1921,8 +1929,8 @@ std::list *Alt::Simple::add_for_loops( return stmts; } std::list *Alt::Simple::add_guards( - std::list *stmts, - Statement::If *guards) { + std::list *stmts, + Statement::If *guards) { if (guards) { stmts->push_back(guards); if (datatype->simple()->is(::Type::LIST)) { @@ -2015,8 +2023,8 @@ void Alt::Simple::codegen(AST &ast) { } if (is_partof_outside) { - // syntactic filter indices are only initialized through the for-loops in outside alternatives - // thus, reverse the order of filters/loops: + // syntactic filter indices are only initialized through the for-loops + // in outside alternatives thus, reverse the order of filters/loops: // add for loops for moving boundaries stmts = add_for_loops(stmts, loops, has_index_overlay()); @@ -2392,10 +2400,11 @@ void Alt::Base::init_filter_guards(AST &ast) { Expr::Base *arg = Expr::seq_to_tree( exprs.begin(), exprs.end()); Statement::If *guard = new Statement::If(arg); - if ((this->data_type()->is(::Type::LIST)) and (this->top_level)) { + if ((this->data_type()->is(::Type::LIST)) && (this->top_level)) { // don't add an else statement (which erases the return type) to the filter // since this would erase the answer list of ALL alternatives of a symbol - // due to the return decl replacement happening in Symbol::NT::eliminate_list_ass + // due to the return decl replacement happening in + // Symbol::NT::eliminate_list_ass } else { guard->els.push_back(new Statement::Fn_Call( Statement::Fn_Call::EMPTY, *ret_decl)); @@ -3084,34 +3093,33 @@ bool Alt::Base::choice_set() { void Alt::Base::get_nonterminals(std::list *nt_list) { } void Alt::Simple::get_nonterminals(std::list *nt_list) { - for (Fn_Arg::Base *arg : this->args) { - Fn_Arg::Alt *arg_alt = dynamic_cast(arg); - if (arg_alt) { - // recurse into arguments - arg_alt->alt->get_nonterminals(nt_list); - } - } + for (Fn_Arg::Base *arg : this->args) { + Fn_Arg::Alt *arg_alt = dynamic_cast(arg); + if (arg_alt) { + // recurse into arguments + arg_alt->alt->get_nonterminals(nt_list); + } + } } void Alt::Link::get_nonterminals(std::list *nt_list) { - // test if Link is a non-terminal - if (this->nt) { - if (this->nt->is(Symbol::NONTERMINAL)) { - nt_list->push_back(dynamic_cast(this->nt)); - } else if (this->nt->is(Symbol::TERMINAL)) { - - } - } else { - throw LogError("Link is something else"); - } + // test if Link is a non-terminal + if (this->nt) { + if (this->nt->is(Symbol::NONTERMINAL)) { + nt_list->push_back(dynamic_cast(this->nt)); + } else if (this->nt->is(Symbol::TERMINAL)) { + } + } else { + throw LogError("Link is something else"); + } } void Alt::Block::get_nonterminals(std::list *nt_list) { - for (Alt::Base *alt : this->alts) { - // recurse into alternatives - alt->get_nonterminals(nt_list); - } + for (Alt::Base *alt : this->alts) { + // recurse into alternatives + alt->get_nonterminals(nt_list); + } } void Alt::Multi::get_nonterminals(std::list *nt_list) { - throw LogError("Alt::Multi::get_nonterminals is not yet implemented!"); + throw LogError("Alt::Multi::get_nonterminals is not yet implemented!"); } // recursively flag this alternative as being part of an inside (=false=default) @@ -3119,90 +3127,95 @@ void Alt::Multi::get_nonterminals(std::list *nt_list) { void Alt::Base::set_partof_outside(bool is_outside) { } void Alt::Simple::set_partof_outside(bool is_outside) { - this->is_partof_outside = is_outside; - for (std::list::iterator i = args.begin(); i != args.end(); ++i) { - Fn_Arg::Alt *arg_alt = dynamic_cast(*i); - if (arg_alt) { - arg_alt->alt->set_partof_outside(is_outside); - } - } + this->is_partof_outside = is_outside; + for (std::list::iterator i = args.begin(); + i != args.end(); ++i) { + Fn_Arg::Alt *arg_alt = dynamic_cast(*i); + if (arg_alt) { + arg_alt->alt->set_partof_outside(is_outside); + } + } } void Alt::Link::set_partof_outside(bool is_outside) { - if (this->nt) { - if (this->nt->is(Symbol::NONTERMINAL)) { - // end recursion on non-terminal call - } else if (this->nt->is(Symbol::TERMINAL)) { - this->is_partof_outside = is_outside; - } - } else { - throw LogError("Link is something else"); - } + if (this->nt) { + if (this->nt->is(Symbol::NONTERMINAL)) { + // end recursion on non-terminal call + } else if (this->nt->is(Symbol::TERMINAL)) { + this->is_partof_outside = is_outside; + } + } else { + throw LogError("Link is something else"); + } } void Alt::Block::set_partof_outside(bool is_outside) { - throw LogError("Alt::Block::set_partof_outside is not yet implemented!"); + throw LogError("Alt::Block::set_partof_outside is not yet implemented!"); } void Alt::Multi::set_partof_outside(bool is_outside) { - throw LogError("Alt::Multi::set_partof_outside is not yet implemented!"); -} - - -bool Alt::Base::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences) { - return false; -} -bool Alt::Simple::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences) { - for (std::list::iterator it_arg = this->args.begin(); it_arg != this->args.end(); ++it_arg) { - Fn_Arg::Alt *arg_alt = dynamic_cast(*it_arg); - if (arg_alt) { - if (arg_alt->alt->replace_nonterminal(find, replace, skip_occurences)) { - return true; - } - } - } - return false; -} -bool Alt::Link::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences) { - if (this->nt) { - if (this->nt->is(Symbol::NONTERMINAL)) { - if (this->nt == find) { - if (skip_occurences[*(find->name)] == 0) { - // replace old NT (=find) with novel NT (=replace) - this->nt = replace; - this->name = replace->name; - this->is_partof_outside = replace->is_partof_outside; - return true; - } else { - skip_occurences[*(find->name)]--; - } - } - } else if (this->nt->is(Symbol::TERMINAL)) { - - } - } else { - throw LogError("Link is something else"); - } - return false; -} -bool Alt::Block::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences) { - for (Alt::Base *alt : this->alts) { - // recurse into alternatives - if (alt->replace_nonterminal(find, replace, skip_occurences)) { - return true; - } - } - return false; -} -bool Alt::Multi::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences) { - throw LogError("Alt::Multi::replace_nonterminal is not yet implemented!"); - return false; + throw LogError("Alt::Multi::set_partof_outside is not yet implemented!"); +} + + +bool Alt::Base::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, + hashtable &skip_occurences) { + return false; +} +bool Alt::Simple::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, + hashtable &skip_occurences) { + for (std::list::iterator it_arg = this->args.begin(); + it_arg != this->args.end(); ++it_arg) { + Fn_Arg::Alt *arg_alt = dynamic_cast(*it_arg); + if (arg_alt) { + if (arg_alt->alt->replace_nonterminal(find, replace, skip_occurences)) { + return true; + } + } + } + return false; +} +bool Alt::Link::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, + hashtable &skip_occurences) { + if (this->nt) { + if (this->nt->is(Symbol::NONTERMINAL)) { + if (this->nt == find) { + if (skip_occurences[*(find->name)] == 0) { + // replace old NT (=find) with novel NT (=replace) + this->nt = replace; + this->name = replace->name; + this->is_partof_outside = replace->is_partof_outside; + return true; + } else { + skip_occurences[*(find->name)]--; + } + } + } else if (this->nt->is(Symbol::TERMINAL)) { + } + } else { + throw LogError("Link is something else"); + } + return false; +} +bool Alt::Block::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, + hashtable &skip_occurences) { + for (Alt::Base *alt : this->alts) { + // recurse into alternatives + if (alt->replace_nonterminal(find, replace, skip_occurences)) { + return true; + } + } + return false; +} +bool Alt::Multi::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, + hashtable &skip_occurences) { + throw LogError("Alt::Multi::replace_nonterminal is not yet implemented!"); + return false; } -// the following functions produce graphViz code to represent the grammar // the following functions produce graphViz code to represent the grammar void to_dot_indices(std::vector indices, std::ostream &out) { out << ""; for (std::vector::const_iterator track = indices.begin(); track != indices.end(); ++track) { - assert(*track != NULL); + assert(*track != NULL); (*track)->put(out); if (std::next(track) != indices.end()) { out << "
"; @@ -3337,29 +3350,34 @@ unsigned int Alt::Multi::to_dot(unsigned int *nodeID, std::ostream &out) { lastID = childID; } out << "};\n"; - return thisID+1; // return not the cluster ID but the id of the first element + // return not the cluster ID but the id of the first element + return thisID+1; } // END functions produce graphViz code to represent the grammar -// traverses an alternative, collects all non-terminals and returns the one outside non-terminal if present, otherwise NULL +// traverses an alternative, collects all non-terminals and returns the one +// outside non-terminal if present, otherwise NULL Symbol::NT *get_outside_NT(Alt::Base *alt) { - std::list *nt_list = new std::list(); - alt->get_nonterminals(nt_list); - for (std::list::iterator i = nt_list->begin(); i != nt_list->end(); ++i) { - //std::cerr << *((*i)->name) << "\n"; - if ((*i)->is_partof_outside == false) { - // i-- because we need to make sure not to destroy the iterator - // we are using at this moment: - // https://www.techiedelight.com/remove-elements-list-iterating-cpp/ - nt_list->erase(i--); - } - } - assert(nt_list->size() <= 1); // by design of the outside NT injection, there can at most be ONE outside-nt on any rhs of a production! - if (nt_list->size() == 1) { - return nt_list->front(); - } - return NULL; + std::list *nt_list = new std::list(); + alt->get_nonterminals(nt_list); + for (std::list::iterator i = nt_list->begin(); + i != nt_list->end(); ++i) { + // std::cerr << *((*i)->name) << "\n"; + if ((*i)->is_partof_outside == false) { + // i-- because we need to make sure not to destroy the iterator + // we are using at this moment: + // https://www.techiedelight.com/remove-elements-list-iterating-cpp/ + nt_list->erase(i--); + } + } + // by design of the outside NT injection, there can at most be ONE + // outside-nt on any rhs of a production! + assert(nt_list->size() <= 1); + if (nt_list->size() == 1) { + return nt_list->front(); + } + return NULL; } Expr::Base *get_next_var_name(unsigned &k, size_t track) { @@ -3369,332 +3387,404 @@ Expr::Base *get_next_var_name(unsigned &k, size_t track) { Expr::Vacc *ivar = new Expr::Vacc(new std::string(o.str())); return ivar; } -Expr::Base *Alt::Simple::get_next_var_right2left(Expr::Base *left_index, Expr::Base *innermost_left_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_lefts) { +Expr::Base *Alt::Simple::get_next_var_right2left(Expr::Base *left_index, + Expr::Base *innermost_left_index, unsigned &k, size_t track, + Yield::Size ys_this, Yield::Size *ys_lefts) { if (ys_this.low() == ys_this.high()) { // constant yield size - if (ys_lefts->low() == 0) { - return innermost_left_index; - } else { - return left_index->plus(ys_this.low()); - } + if (ys_lefts->low() == 0) { + return innermost_left_index; + } else { + return left_index->plus(ys_this.low()); + } } if (ys_lefts->low() == ys_lefts->high()) { - return innermost_left_index->plus(ys_lefts->low()); + return innermost_left_index->plus(ys_lefts->low()); } // variable yield size Expr::Base *next_index = get_next_var_name(k, track); // add for loop - Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), next_index, left_index->plus(ys_this.low())); + Statement::Var_Decl *loopvariable = new Statement::Var_Decl( + new ::Type::Size(), next_index, left_index->plus(ys_this.low())); loopvariable->set_itr(true); - Expr::Base *cond = new Expr::Less_Eq(next_index, innermost_left_index->minus(ys_lefts->low())); + Expr::Base *cond = new Expr::Less_Eq(next_index, + innermost_left_index->minus(ys_lefts->low())); Statement::For *f = new Statement::For(loopvariable, cond); loops.push_back(f); return next_index; } -Expr::Base *Alt::Simple::get_next_var_left2right(Expr::Base *right_index, Expr::Base *innermost_right_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_rights) { +Expr::Base *Alt::Simple::get_next_var_left2right(Expr::Base *right_index, + Expr::Base *innermost_right_index, unsigned &k, size_t track, + Yield::Size ys_this, Yield::Size *ys_rights) { Expr::Base *next_index; // constant yield size if (ys_this.low() == ys_this.high()) { - if (ys_rights->low() == 0) { - // we might fall back to j, if yield between outside NT and current right hand side argument is 0 - next_index = innermost_right_index; - } else { - // normally, we just decrease the current index - next_index = right_index->minus(ys_this.low()); - } + if (ys_rights->low() == 0) { + // we might fall back to j, if yield between outside NT and current + // right hand side argument is 0 + next_index = innermost_right_index; + } else { + // normally, we just decrease the current index + next_index = right_index->minus(ys_this.low()); + } } else { - // constant yield size, on the left hand side towards outside non terminal - // e.g. ..., outside_bar, BASE, foo, ... - // where border between BASE and foo shall be j+1 instead of k_x - if (ys_rights->low() == ys_rights->high()) { - next_index = innermost_right_index->plus(ys_rights->low()); - } else { - // variable yield size - next_index = get_next_var_name(k, track); - - // add for loop - Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), next_index, innermost_right_index->plus(ys_rights->low())); - loopvariable->set_itr(true); - Expr::Base *cond = new Expr::Less_Eq(next_index, right_index->minus(ys_this.low())); - Statement::For *f = new Statement::For(loopvariable, cond); - loops.push_back(f); - } + // constant yield size, on the left hand side towards outside non terminal + // e.g. ..., outside_bar, BASE, foo, ... + // where border between BASE and foo shall be j+1 instead of k_x + if (ys_rights->low() == ys_rights->high()) { + next_index = innermost_right_index->plus(ys_rights->low()); + } else { + // variable yield size + next_index = get_next_var_name(k, track); + + // add for loop + Statement::Var_Decl *loopvariable = new Statement::Var_Decl( + new ::Type::Size(), next_index, + innermost_right_index->plus(ys_rights->low())); + loopvariable->set_itr(true); + Expr::Base *cond = new Expr::Less_Eq(next_index, + right_index->minus(ys_this.low())); + Statement::For *f = new Statement::For(loopvariable, cond); + loops.push_back(f); + } } return next_index; } -void Alt::Base::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { - this->left_indices[track] = left; - this->right_indices[track] = right; -} -void Alt::Simple::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { - // 1) identify outside non-terminal - for (std::list::const_iterator i = args.begin(); i != args.end(); ++i) { - Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); - if (ont) { - // e.g. outside_iloop - Alt::Link *link = dynamic_cast((*i)->alt_ref()); - if (link) { - if (link->nt == ont) { - // the link->nt could also be a terminal - Symbol::NT *check = dynamic_cast(link->nt); - assert(check); - // we have identified this fn_arg as the one being the outside non-terminal, - // 2) let's expand its indices - link->expand_outside_nt_indices(left, right, track); - // we are done - return; - } - } - // e.g. incl(outside_iloop) - Alt::Simple *algfct = dynamic_cast((*i)->alt_ref()); - if (algfct) { - algfct->expand_outside_nt_indices(left, right, track); - } - } - } -} -void Alt::Link::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { - Alt::Base::expand_outside_nt_indices(left, right, track); -} -void Alt::Block::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { - throw LogError("Alt::Block::expand_outside_nt_indices is not yet implemented!"); -} -void Alt::Multi::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { - throw LogError("Alt::Block::expand_outside_nt_indices is not yet implemented!"); -} -void Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { - Alt::Base::init_indices(left, right, k, track); -} - -void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { - // if this alternative is called from an outside non-terminal, but does not contain another outside non-terminal, e.g. hl(BASE, REGION, BASE) - // we can default to inside indices generation! - if (this->is_partof_outside == false) { - Alt::Simple::init_indices(left, right, k, track); - return; - } -// std::cerr << *this->name << ": Alt::Simple::init_indices_outside(" << *left << ", " << *right << ", "; -// if (center_left) { -// std::cerr << *center_left; -// } else { -// std::cerr << center_left; -// } -// std::cerr << ", "; -// if (center_right) { -// std::cerr << *center_right; -// } else { -// std::cerr << center_right; -// } -// std::cerr << ", " << is_right_of_outside_nt << ")\n"; - - std::list *args_left = new std::list(); - std::list *args_right = new std::list(); - Fn_Arg::Base *arg_outside = NULL; - Symbol::NT *nt_outside = NULL; - for (std::list::const_iterator i = args.begin(); i != args.end(); ++i) { - Fn_Arg::Const *constarg = dynamic_cast(*i); - if (constarg != NULL) { - // argument is a constant like 'A' in CHAR('A') - continue; - } - Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); - if (ont) { - Alt::Link *link = dynamic_cast((*i)->alt_ref()); - if (link) { - if (link->nt == ont) { - // the link->nt could also be a terminal - Symbol::NT *check = dynamic_cast(link->nt); - assert(check); - nt_outside = check; - } - } - arg_outside = *i; - continue; - } - if ((arg_outside == NULL) and (is_right_of_outside_nt == false)) { - args_left->push_back(*i); - } else { - args_right->push_back(*i); - } - } - - // yield sizes for sub-structure - std::pair *ys_sub = new std::pair(new Yield::Size(), new Yield::Size()); - if (arg_outside != NULL) { - ys_sub = arg_outside->alt_ref()->get_outside_accum_yieldsizes(track, is_right_of_outside_nt); - } - std::pair *ys_this = this->get_outside_accum_yieldsizes(track, is_right_of_outside_nt); - - // arguments left of outside NT - Expr::Base *left_index; - if (center_left) { - left_index = center_left; - } else { - if (ys_this->first->low() != ys_this->first->high()) { - left_index = get_next_var_name(k, track); - - // add for-loop - std::ostringstream o; - o << "t_" << track << "_left_most"; - Expr::Vacc *left_border = new Expr::Vacc(new std::string(o.str())); - Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), left_index, left_border); - loopvariable->set_itr(true); - Expr::Base *cond = new Expr::Less_Eq(left_index, left->minus(ys_this->first->low())); - Statement::For *f = new Statement::For(loopvariable, cond); - loops.push_back(f); - } else { - // no new index if there are no parsers or only parsers with const yield size left of outside nt - // e.g. incl(outside_foo) - // e.g. sadd(BASE, outside_foo) - left_index = left->minus(ys_this->first->low()); - } - } - Expr::Base *leftmost_index = left_index; - Expr::Base *right_index = left_index; - for (std::list::const_iterator i = args_left->begin(); i != args_left->end(); ++i) { - Yield::Size ys_arg = (*i)->multi_ys()(track); - Yield::Size *ys_right2center = new Yield::Size(); - *ys_right2center += *(ys_sub->first); - *ys_right2center += *(sum_ys_rights(&ys_arg, i, args_left->end(), track)); - if ((arg_outside == NULL) and (ys_arg.low() > 0)) { - // if this level does NOT hold the outside non-terminal, but args are left of the outside non-terminal - // propagate the rightmost index (must be the one from the outside non-terminal in higher levels) - // as the left index - left = right; - } - right_index = get_next_var_right2left(left_index, left, k, track, ys_arg, ys_right2center); - (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, leftmost_index, center_right); - left_index = right_index; - } - Expr::Base *lhs_right_index = right_index; - - // arguments right of outside NT - if (center_right) { - right_index = center_right; - } else { - if (ys_this->second->low() != ys_this->second->high()) { - right_index = get_next_var_name(k, track); - - // add for-loop - std::ostringstream o; - o << "t_" << track << "_right_most"; - Expr::Vacc *right_border = new Expr::Vacc(new std::string(o.str())); - Statement::Var_Decl *loopvariable = new Statement::Var_Decl(new ::Type::Size(), right_index, right->plus(ys_this->second->low())); - loopvariable->set_itr(true); - Expr::Base *cond = new Expr::Less_Eq(right_index, right_border); - Statement::For *f = new Statement::For(loopvariable, cond); - loops.push_back(f); - } else { - // no new index if there are no parsers or only parsers with const yield size right of outside nt - // e.g. incl(outside_foo) - // e.g. adds(outside_foo, BASE) - right_index = right->plus(ys_this->second->low()); - } - } - Expr::Base *rightmost_index = right_index; - left_index = NULL; - if (is_right_of_outside_nt) { - right = left; - } - for (std::list::const_reverse_iterator i = args_right->rbegin(); i != args_right->rend(); ++i) { - Yield::Size ys_arg = (*i)->multi_ys()(track); - Yield::Size *ys_left2center = new Yield::Size(); - *ys_left2center += *(ys_sub->second); - *ys_left2center += *(sum_ys_lefts(&ys_arg, i, args_right->rend(), track)); - left_index = get_next_var_left2right(right_index, right, k, track, ys_arg, ys_left2center); - (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, center_left, rightmost_index, true); - right_index = left_index; - } - - Alt::Base::init_indices_outside(leftmost_index, rightmost_index, k, track, center_left, center_right); - // store original left/right for e.g. - // correct initialization of IF guards; see function init_guards() - this->left_inside_indices.resize(this->left_indices.size()); - this->right_inside_indices.resize(this->right_indices.size()); - this->left_inside_indices[track] = left; - this->right_inside_indices[track] = right; - - // argument containing outside NT - if (arg_outside != NULL) { - arg_outside->alt_ref()->init_indices_outside(left, right, k, track, lhs_right_index, right_index); - this->expand_outside_nt_indices(leftmost_index, rightmost_index, track); - } -} -void Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { - Alt::Base::init_indices_outside(left, right, k, track, center_left, center_right); -} -void Alt::Block::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { +void Alt::Base::expand_outside_nt_indices(Expr::Base *left, + Expr::Base *right, size_t track) { + this->left_indices[track] = left; + this->right_indices[track] = right; +} +void Alt::Simple::expand_outside_nt_indices(Expr::Base *left, + Expr::Base *right, size_t track) { + // 1) identify outside non-terminal + for (std::list::const_iterator i = args.begin(); + i != args.end(); ++i) { + Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); + if (ont) { + // e.g. outside_iloop + Alt::Link *link = dynamic_cast((*i)->alt_ref()); + if (link) { + if (link->nt == ont) { + // the link->nt could also be a terminal + Symbol::NT *check = dynamic_cast(link->nt); + assert(check); + // we have identified this fn_arg as the one being the outside + // non-terminal, 2) let's expand its indices + link->expand_outside_nt_indices(left, right, track); + // we are done + return; + } + } + // e.g. incl(outside_iloop) + Alt::Simple *algfct = dynamic_cast((*i)->alt_ref()); + if (algfct) { + algfct->expand_outside_nt_indices(left, right, track); + } + } + } +} +void Alt::Link::expand_outside_nt_indices(Expr::Base *left, + Expr::Base *right, size_t track) { + Alt::Base::expand_outside_nt_indices(left, right, track); +} +void Alt::Block::expand_outside_nt_indices(Expr::Base *left, + Expr::Base *right, size_t track) { + throw LogError( + "Alt::Block::expand_outside_nt_indices is not yet implemented!"); +} +void Alt::Multi::expand_outside_nt_indices(Expr::Base *left, + Expr::Base *right, size_t track) { + throw LogError( + "Alt::Block::expand_outside_nt_indices is not yet implemented!"); +} +void Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, + unsigned int &k, size_t track, Expr::Base *center_left, + Expr::Base *center_right, bool is_right_of_outside_nt) { + Alt::Base::init_indices(left, right, k, track); +} + +void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, + unsigned int &k, size_t track, Expr::Base *center_left, + Expr::Base *center_right, bool is_right_of_outside_nt) { + // if this alternative is called from an outside non-terminal, but does + // not contain another outside non-terminal, e.g. hl(BASE, REGION, BASE) + // we can default to inside indices generation! + if (this->is_partof_outside == false) { + Alt::Simple::init_indices(left, right, k, track); + return; + } +// std::cerr << *this->name << ": Alt::Simple::init_indices_outside(" +// << *left << ", " << *right << ", "; +// if (center_left) { +// std::cerr << *center_left; +// } else { +// std::cerr << center_left; +// } +// std::cerr << ", "; +// if (center_right) { +// std::cerr << *center_right; +// } else { +// std::cerr << center_right; +// } +// std::cerr << ", " << is_right_of_outside_nt << ")\n"; + + std::list *args_left = new std::list(); + std::list *args_right = new std::list(); + Fn_Arg::Base *arg_outside = NULL; + Symbol::NT *nt_outside = NULL; + for (std::list::const_iterator i = args.begin(); + i != args.end(); ++i) { + Fn_Arg::Const *constarg = dynamic_cast(*i); + if (constarg != NULL) { + // argument is a constant like 'A' in CHAR('A') + continue; + } + Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); + if (ont) { + Alt::Link *link = dynamic_cast((*i)->alt_ref()); + if (link) { + if (link->nt == ont) { + // the link->nt could also be a terminal + Symbol::NT *check = dynamic_cast(link->nt); + assert(check); + nt_outside = check; + } + } + arg_outside = *i; + continue; + } + if ((arg_outside == NULL) && (is_right_of_outside_nt == false)) { + args_left->push_back(*i); + } else { + args_right->push_back(*i); + } + } + + // yield sizes for sub-structure + std::pair *ys_sub = + new std::pair( + new Yield::Size(), new Yield::Size()); + if (arg_outside != NULL) { + ys_sub = arg_outside->alt_ref()->get_outside_accum_yieldsizes(track, + is_right_of_outside_nt); + } + std::pair *ys_this = + this->get_outside_accum_yieldsizes(track, is_right_of_outside_nt); + + // arguments left of outside NT + Expr::Base *left_index; + if (center_left) { + left_index = center_left; + } else { + if (ys_this->first->low() != ys_this->first->high()) { + left_index = get_next_var_name(k, track); + + // add for-loop + std::ostringstream o; + o << "t_" << track << "_left_most"; + Expr::Vacc *left_border = new Expr::Vacc(new std::string(o.str())); + Statement::Var_Decl *loopvariable = new Statement::Var_Decl( + new ::Type::Size(), left_index, left_border); + loopvariable->set_itr(true); + Expr::Base *cond = new Expr::Less_Eq(left_index, left->minus( + ys_this->first->low())); + Statement::For *f = new Statement::For(loopvariable, cond); + loops.push_back(f); + } else { + // no new index if there are no parsers or only parsers with const + // yield size left of outside nt + // e.g. incl(outside_foo) + // e.g. sadd(BASE, outside_foo) + left_index = left->minus(ys_this->first->low()); + } + } + Expr::Base *leftmost_index = left_index; + Expr::Base *right_index = left_index; + for (std::list::const_iterator i = args_left->begin(); + i != args_left->end(); ++i) { + Yield::Size ys_arg = (*i)->multi_ys()(track); + Yield::Size *ys_right2center = new Yield::Size(); + *ys_right2center += *(ys_sub->first); + *ys_right2center += *(sum_ys_rights(&ys_arg, i, args_left->end(), track)); + if ((arg_outside == NULL) && (ys_arg.low() > 0)) { + // if this level does NOT hold the outside non-terminal, but args + // are left of the outside non-terminal propagate the rightmost index + // (must be the one from the outside non-terminal in higher levels) + // as the left index + left = right; + } + right_index = get_next_var_right2left(left_index, left, k, track, ys_arg, + ys_right2center); + (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, + leftmost_index, center_right); + left_index = right_index; + } + Expr::Base *lhs_right_index = right_index; + + // arguments right of outside NT + if (center_right) { + right_index = center_right; + } else { + if (ys_this->second->low() != ys_this->second->high()) { + right_index = get_next_var_name(k, track); + + // add for-loop + std::ostringstream o; + o << "t_" << track << "_right_most"; + Expr::Vacc *right_border = new Expr::Vacc(new std::string(o.str())); + Statement::Var_Decl *loopvariable = new Statement::Var_Decl( + new ::Type::Size(), right_index, right->plus(ys_this->second->low())); + loopvariable->set_itr(true); + Expr::Base *cond = new Expr::Less_Eq(right_index, right_border); + Statement::For *f = new Statement::For(loopvariable, cond); + loops.push_back(f); + } else { + // no new index if there are no parsers or only parsers with const + // yield size right of outside nt + // e.g. incl(outside_foo) + // e.g. adds(outside_foo, BASE) + right_index = right->plus(ys_this->second->low()); + } + } + Expr::Base *rightmost_index = right_index; + left_index = NULL; + if (is_right_of_outside_nt) { + right = left; + } + for (std::list::const_reverse_iterator + i = args_right->rbegin(); i != args_right->rend(); ++i) { + Yield::Size ys_arg = (*i)->multi_ys()(track); + Yield::Size *ys_left2center = new Yield::Size(); + *ys_left2center += *(ys_sub->second); + *ys_left2center += *(sum_ys_lefts(&ys_arg, i, args_right->rend(), track)); + left_index = get_next_var_left2right(right_index, right, k, track, + ys_arg, ys_left2center); + (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, + track, center_left, rightmost_index, true); + right_index = left_index; + } + + Alt::Base::init_indices_outside(leftmost_index, rightmost_index, k, track, + center_left, center_right); + // store original left/right for e.g. + // correct initialization of IF guards; see function init_guards() + this->left_inside_indices.resize(this->left_indices.size()); + this->right_inside_indices.resize(this->right_indices.size()); + this->left_inside_indices[track] = left; + this->right_inside_indices[track] = right; + + // argument containing outside NT + if (arg_outside != NULL) { + arg_outside->alt_ref()->init_indices_outside(left, right, k, track, + lhs_right_index, right_index); + this->expand_outside_nt_indices(leftmost_index, rightmost_index, track); + } +} +void Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, + unsigned int &k, size_t track, Expr::Base *center_left, + Expr::Base *center_right, bool is_right_of_outside_nt) { + Alt::Base::init_indices_outside(left, right, k, track, center_left, + center_right); +} +void Alt::Block::init_indices_outside(Expr::Base *left, Expr::Base *right, + unsigned int &k, size_t track, Expr::Base *center_left, + Expr::Base *center_right, bool is_right_of_outside_nt) { throw LogError("Alt::Block::init_indices_outside is not yet implemented!"); } -void Alt::Multi::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { +void Alt::Multi::init_indices_outside(Expr::Base *left, Expr::Base *right, + unsigned int &k, size_t track, Expr::Base *center_left, + Expr::Base *center_right, bool is_right_of_outside_nt) { throw LogError("Alt::Multi::init_indices_outside is not yet implemented!"); } -std::pair *Alt::Base::get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt) { - std::pair *res = new std::pair(new Yield::Size(), new Yield::Size()); +std::pair *Alt::Base::get_outside_accum_yieldsizes( + size_t track, bool is_right_of_outside_nt) { + std::pair *res = + new std::pair( + new Yield::Size(), new Yield::Size()); return res; } -std::pair *Alt::Simple::get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt) { - //first = left, second = right - std::pair *res = new std::pair(new Yield::Size(), new Yield::Size()); - - Fn_Arg::Base *arg_outside = NULL; - for (std::list::const_iterator i = args.begin(); i != args.end(); ++i) { - Fn_Arg::Const *constarg = dynamic_cast(*i); - if (constarg != NULL) { - // skip constant arguments like CHAR('A') - continue; - } - Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); - if (ont) { - arg_outside = *i; - std::pair *sub_ys = arg_outside->alt_ref()->get_outside_accum_yieldsizes(track, is_right_of_outside_nt); - *(res->first) += *(sub_ys->first); - *(res->second) += *(sub_ys->second); - continue; - } - if ((arg_outside == NULL) and (is_right_of_outside_nt == false)) { - *(res->first) += (*i)->multi_ys()(track); - } else { - // if the alternative links to a non-terminal AND the non-terminal has a sub-quadratic table, we do NOT add the yield size - if (arg_outside != NULL) { - if (arg_outside->alt_ref()->is(Alt::LINK)) { - Symbol::NT *linkedNT = dynamic_cast(dynamic_cast(arg_outside->alt_ref())->nt); - if (linkedNT) { - if (linkedNT->tables()[track].is_cyk_right()) { - continue; - } - } - } - } - *(res->second) += (*i)->multi_ys()(track); - } - } - - return res; -} -std::pair *Alt::Link::get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt) { +std::pair +*Alt::Simple::get_outside_accum_yieldsizes(size_t track, + bool is_right_of_outside_nt) { + // first = left, second = right + std::pair *res = + new std::pair( + new Yield::Size(), new Yield::Size()); + + Fn_Arg::Base *arg_outside = NULL; + for (std::list::const_iterator i = args.begin(); + i != args.end(); ++i) { + Fn_Arg::Const *constarg = dynamic_cast(*i); + if (constarg != NULL) { + // skip constant arguments like CHAR('A') + continue; + } + Symbol::NT *ont = get_outside_NT((*i)->alt_ref()); + if (ont) { + arg_outside = *i; + std::pair *sub_ys = + arg_outside->alt_ref()->get_outside_accum_yieldsizes( + track, is_right_of_outside_nt); + *(res->first) += *(sub_ys->first); + *(res->second) += *(sub_ys->second); + continue; + } + if ((arg_outside == NULL) && (is_right_of_outside_nt == false)) { + *(res->first) += (*i)->multi_ys()(track); + } else { + // if the alternative links to a non-terminal AND the non-terminal + // has a sub-quadratic table, we do NOT add the yield size + if (arg_outside != NULL) { + if (arg_outside->alt_ref()->is(Alt::LINK)) { + Symbol::NT *linkedNT = dynamic_cast( + dynamic_cast(arg_outside->alt_ref())->nt); + if (linkedNT) { + if (linkedNT->tables()[track].is_cyk_right()) { + continue; + } + } + } + } + *(res->second) += (*i)->multi_ys()(track); + } + } + + return res; +} +std::pair +*Alt::Link::get_outside_accum_yieldsizes(size_t track, + bool is_right_of_outside_nt) { return Alt::Base::get_outside_accum_yieldsizes(track, is_right_of_outside_nt); } -std::pair *Alt::Block::get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt) { - throw LogError("Alt::Block::get_outside_accum_yieldsizes is not yet implemented!"); +std::pair +*Alt::Block::get_outside_accum_yieldsizes(size_t track, + bool is_right_of_outside_nt) { + throw LogError( + "Alt::Block::get_outside_accum_yieldsizes is not yet implemented!"); return NULL; - } -std::pair *Alt::Multi::get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt) { - throw LogError("Alt::Block::get_outside_accum_yieldsizes is not yet implemented!"); +std::pair +*Alt::Multi::get_outside_accum_yieldsizes(size_t track, + bool is_right_of_outside_nt) { + throw LogError( + "Alt::Block::get_outside_accum_yieldsizes is not yet implemented!"); return NULL; } -Yield::Size *Alt::Simple::sum_ys_lefts(Yield::Size *y, std::list::const_reverse_iterator i, const std::list::const_reverse_iterator &end, size_t track) const { +Yield::Size *Alt::Simple::sum_ys_lefts( + Yield::Size *y, std::list::const_reverse_iterator i, + const std::list::const_reverse_iterator &end, size_t track) + const { Yield::Size *res = new Yield::Size(); // *res += *y; ++i; @@ -3703,7 +3793,10 @@ Yield::Size *Alt::Simple::sum_ys_lefts(Yield::Size *y, std::list: } return res; } -Yield::Size *Alt::Simple::sum_ys_rights(Yield::Size *y, std::list::const_iterator i, const std::list::const_iterator &end, size_t track) const { +Yield::Size *Alt::Simple::sum_ys_rights( + Yield::Size *y, std::list::const_iterator i, + const std::list::const_iterator &end, size_t track) + const { Yield::Size *res = new Yield::Size(); // *res += *y; ++i; diff --git a/src/alt.hh b/src/alt.hh index f0150fc26..d753c5425 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -28,6 +28,7 @@ #include #include #include +#include #include "grammar.hh" #include "runtime.hh" @@ -136,7 +137,8 @@ class Base { protected: std::vector left_indices; std::vector right_indices; - // for outside non-terminals: store original inner left/right indices for guards construction + // for outside non-terminals: store original inner left/right indices for + // guards construction std::vector left_inside_indices; std::vector right_inside_indices; @@ -222,15 +224,22 @@ class Base { virtual void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); - // recurses into alternatives, finds the single outside NT and updates only its left and right indices - // used after init_indices_outside initializes all other incides - // e.g. foo(k1_REGION_i, i_bar_j, j_REGION_k2) --> foo(k1_REGION_i, k1_bar_k2, j_REGION_k2) - // ^ ^ ^^ ^^ - virtual void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); - virtual std::pair *get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt=false); + // recurses into alternatives, finds the single outside NT and updates + // only its left and right indices used after init_indices_outside + // initializes all other incides + // e.g. foo(k1_REGION_i, i_bar_j, j_REGION_k2) --> foo(k1_REGION_i, + // ^ ^ + // k1_bar_k2, j_REGION_k2) + // ^^ ^^ + virtual void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, + size_t track); + virtual std::pair *get_outside_accum_yieldsizes( + size_t track, bool is_right_of_outside_nt = false); // sets indices for outside rules, respecting yield sizes virtual void init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt=false); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, + Expr::Base *center_left, Expr::Base *center_right, + bool is_right_of_outside_nt = false); virtual void init_ret_decl(unsigned int i, const std::string &prefix); @@ -323,27 +332,33 @@ class Base { bool choice_set(); - // traverses the alternative (=rhs of a production) and collects pointers to all referenced non-terminals - // e.g. ml_comps = cadd(incl(dangle), ml_comps1) should return [*dangle, *ml_comps1] + // traverses the alternative (=rhs of a production) and collects pointers + // to all referenced non-terminals e.g. + // ml_comps = cadd(incl(dangle), ml_comps1) + // should return [*dangle, *ml_comps1] virtual void get_nonterminals(std::list *nt_list); // flag alternative as being part of a generated outside non-terminal // such that code generation can discriminate between inside (default) // and outside. virtual void set_partof_outside(bool is_outside); - // traverses the alternative (=rhs of a production), iff non-terminal find is contained, replace with first occurence of non-terminal replace - virtual bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences); + // traverses the alternative (=rhs of a production), iff non-terminal find + // is contained, replace with first occurence of non-terminal replace + virtual bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, + hashtable &skip_occurences); virtual unsigned int to_dot(unsigned int *nodeID, std::ostream &out); // returns either the single outside non-terminal or NULL Symbol::NT *get_outside_nt(Alt::Base *alt); + protected: - // private flag to indicate if alternative is for inside (the default) or outside - // production rules. See set_partof_outside() + // private flag to indicate if alternative is for inside (the default) or + // outside production rules. See set_partof_outside() bool is_partof_outside; + public: bool get_is_partof_outside() { - return this->is_partof_outside; + return this->is_partof_outside; } }; @@ -460,12 +475,20 @@ class Simple : public Base { public: void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); - Expr::Base *get_next_var_right2left(Expr::Base *left_index, Expr::Base *innermost_left_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_lefts); - Expr::Base *get_next_var_left2right(Expr::Base *right_index, Expr::Base *innermost_right_index, unsigned &k, size_t track, Yield::Size ys_this, Yield::Size *ys_rights); - void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); - std::pair *get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt=false); + Expr::Base *get_next_var_right2left(Expr::Base *left_index, + Expr::Base *innermost_left_index, unsigned &k, size_t track, + Yield::Size ys_this, Yield::Size *ys_lefts); + Expr::Base *get_next_var_left2right(Expr::Base *right_index, + Expr::Base *innermost_right_index, unsigned &k, size_t track, + Yield::Size ys_this, Yield::Size *ys_rights); + void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, + size_t track); + std::pair *get_outside_accum_yieldsizes( + size_t track, bool is_right_of_outside_nt = false); void init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt=false); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, + Expr::Base *center_left, Expr::Base *center_right, + bool is_right_of_outside_nt = false); void put_indices(std::ostream &s); void reset(); @@ -494,7 +517,7 @@ class Simple : public Base { bool has_index_overlay); std::list *add_guards( std::list *stmts, - Statement::If *guards); + Statement::If *guards); void sum_rhs( Yield::Multi &y, std::list::const_iterator i, const std::list::const_iterator &end) const; @@ -502,8 +525,13 @@ class Simple : public Base { Yield::Size &y, std::list::const_iterator i, const std::list::const_iterator &end, size_t track) const; - Yield::Size *sum_ys_lefts(Yield::Size *y, std::list::const_reverse_iterator i, const std::list::const_reverse_iterator &end, size_t track) const; - Yield::Size *sum_ys_rights(Yield::Size *y, std::list::const_iterator i, const std::list::const_iterator &end, size_t track) const; + Yield::Size *sum_ys_lefts( + Yield::Size *y, std::list::const_reverse_iterator i, + const std::list::const_reverse_iterator &end, size_t track) + const; + Yield::Size *sum_ys_rights( + Yield::Size *y, std::list::const_iterator i, + const std::list::const_iterator &end, size_t track) const; public: @@ -539,7 +567,8 @@ class Simple : public Base { void get_nonterminals(std::list *nt_list); void set_partof_outside(bool is_outside); - bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences); + bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, + hashtable &skip_occurences); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); private: @@ -617,10 +646,14 @@ class Link : public Base { void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); - std::pair *get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt=false); - void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); + std::pair *get_outside_accum_yieldsizes( + size_t track, bool is_right_of_outside_nt = false); + void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, + size_t track); void init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt=false); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, + Expr::Base *center_left, Expr::Base *center_right, + bool is_right_of_outside_nt = false); // void init_ret_decl(unsigned int i); @@ -670,7 +703,8 @@ class Link : public Base { void get_nonterminals(std::list *nt_list); void set_partof_outside(bool is_outside); - bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences); + bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, + hashtable &skip_occurences); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); void init_outside_guards(); }; @@ -723,10 +757,14 @@ class Block : public Base { void traverse(Visitor &v); void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); - void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); - std::pair *get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt=false); + void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, + size_t track); + std::pair *get_outside_accum_yieldsizes( + size_t track, bool is_right_of_outside_nt = false); void init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt=false); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, + Expr::Base *center_left, Expr::Base *center_right, + bool is_right_of_outside_nt = false); void codegen(AST &ast); @@ -747,7 +785,8 @@ class Block : public Base { void get_nonterminals(std::list *nt_list); void set_partof_outside(bool is_outside); - bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences); + bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, + hashtable &skip_occurences); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); }; @@ -800,10 +839,14 @@ class Multi : public Base { void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); - void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track); - std::pair *get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt=false); + void expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, + size_t track); + std::pair *get_outside_accum_yieldsizes( + size_t track, bool is_right_of_outside_nt = false); void init_indices_outside( - Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt=false); + Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, + Expr::Base *center_left, Expr::Base *center_right, + bool is_right_of_outside_nt = false); void codegen(AST &ast); void print_dot_edge(std::ostream &out, Symbol::NT &nt); @@ -827,7 +870,8 @@ class Multi : public Base { void get_nonterminals(std::list *nt_list); void set_partof_outside(bool is_outside); - bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences); + bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, + hashtable &skip_occurences); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); }; diff --git a/src/cpp.cc b/src/cpp.cc index 386bd43b8..876c1a7cc 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -81,11 +81,11 @@ void Printer::Cpp::print(const Statement::For &stmt) { stream << *stmt.var_decl; stream << ' ' << *stmt.cond << "; "; if (!stmt.inc) { - if (stmt.decrement) { - stream << "--"; - } else { - stream << "++"; - } + if (stmt.decrement) { + stream << "--"; + } else { + stream << "++"; + } stream << *stmt.var_decl->name << ")"; } else { bool t = in_fn_head; diff --git a/src/gapc.cc b/src/gapc.cc index 0e2e3ac1e..83b12f7ba 100644 --- a/src/gapc.cc +++ b/src/gapc.cc @@ -104,10 +104,11 @@ static void parse_options(int argc, char **argv, Options *rec) { "uses the selected instance and creates a GAP program which creates " "specialized GAP programs that recognize a subset of candidates of the " "original grammar.") - ("outside_grammar", po::value< std::vector >(), + ("outside_grammar", po::value< std::vector >(), "generate an outside version of the grammar and report outside results " - "for the provided list of inside non-terminals. Will report ALL non-" - "terminals, if provided list says 'ALL'.") + "for an inside non-terminal. Provide multiple times for lists of " + "non-terminals or type \"ALL\" to report results for all non-" + "terminals.") ("verbose", "show suppressed warnings and messages") ("log-level,l", po::value(), "the log level, valid values are 0 (VERBOSE), 1 (INFO), 2 (NORMAL), 3 " @@ -226,7 +227,8 @@ static void parse_options(int argc, char **argv, Options *rec) { rec->specializeGrammar = true; } if (vm.count("outside_grammar")) - rec->outside_nt_list = vm["outside_grammar"].as< std::vector >(); + rec->outside_nt_list = vm["outside_grammar"] + .as< std::vector >(); if (vm.count("verbose")) rec->verbose_mode = true; if (vm.count("log-level")) { @@ -364,8 +366,9 @@ class Main { } // inject rules for outside grammar - grammar->inject_outside_nts(); - + if (opts.outside_nt_list.size() > 0) { + grammar->inject_outside_nts(opts.outside_nt_list); + } // configure the window and k-best mode driver.ast.set_window_mode(opts.window_mode); diff --git a/src/grammar.cc b/src/grammar.cc index e84d4f232..126e96d84 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -24,6 +24,8 @@ #include #include +#include +#include #include "grammar.hh" #include "log.hh" @@ -1000,97 +1002,150 @@ void Grammar::remove(Symbol::NT *x) { tabulated.erase(nt); } -void Grammar::inject_outside_nts() { - std::string OUTSIDE_NT_PREFIX = "outside_"; - - std::set outside_nts = std::set(); - - // 1) collect non-terminals used in rhs of productions together with their calling lhs non-terminal - // we thus exclude e.g. non-terminal foo in production "foo = BASE;" - for (hashtable::iterator i = NTs.begin(); i != NTs.end(); ++i) { - Symbol::NT *nt = dynamic_cast(i->second); - if (nt) { // skip terminals - for (std::list::iterator alt = nt->alts.begin(); alt != nt->alts.end(); ++alt) { - std::list *rhs_nts = new std::list(); - (*alt)->get_nonterminals(rhs_nts); - if (rhs_nts->size() > 0) { - // if rhs uses at least one non-terminal, both (= calling lhs non-terminal and all rhs non-terminal(s)) will be added as novel outside non-terminals - outside_nts.insert(nt); - outside_nts.insert(rhs_nts->begin(), rhs_nts->end()); - } - } - } - } - - // 2) create a novel outside non-terminal for each collected non-terminal in the inside grammar - // add both (new outside and user provided inside) non-terminal **names** as keys to the hash, - // pointing to the very same novel outside non-terminal. - hashtable outside_NTs; - if (outside_nts.size() == 0) { - // the grammar does not have any non-terminals on the rhs of productions. - // Thus, no new NTs have to be injected - return; - } - for (std::set::iterator nt = outside_nts.begin(); nt != outside_nts.end(); ++nt) { - // use inside NT as template ... - Symbol::NT *inside_nt = dynamic_cast(this->NTs.find(*(*nt)->name)->second); - // and clone for outside version - Symbol::NT *outside_nt = inside_nt->clone(inside_nt->track_pos()); - // remove all existing alternative production rules - outside_nt->alts.clear(); - // and correct NT name, to now point to outside version - outside_nt->name = new std::string(OUTSIDE_NT_PREFIX + (*(*nt)->name)); - - // flag this new non-terminal as being part of the outside rules - outside_nt->is_partof_outside = true; - - outside_NTs[OUTSIDE_NT_PREFIX + (*(*nt)->name)] = outside_nt; - outside_NTs[(*(*nt)->name)] = outside_nt; - } - - // 3) create copy of inside alternative, replace called non-terminal with calling outside version and append to called outside version - std::set *axiom_candidates = new std::set(); - for (hashtable::iterator i = NTs.begin(); i != NTs.end(); ++i) { - Symbol::NT *nt = dynamic_cast(i->second); - if (nt) { // skip terminals - for (std::list::iterator alt = nt->alts.begin(); alt != nt->alts.end(); ++alt) { - std::list *rhs_nts = new std::list(); - (*alt)->get_nonterminals(rhs_nts); - if (rhs_nts->size() > 0) { - hashtable skip_occurences; - for (std::list::iterator it_nt = rhs_nts->begin(); it_nt != rhs_nts->end(); ++it_nt) { - skip_occurences[*((*it_nt)->name)] = 0; - } - //unsigned int skip_occurences = 0; - for (std::list::iterator it_nt = rhs_nts->begin(); it_nt != rhs_nts->end(); ++it_nt) { - // create a copy (=clone) of the inside alternative - Alt::Base *outside_alt = (*alt)->clone(); - // replace one of the inside non-terminals with the outside version of the calling non-terminal - outside_alt->replace_nonterminal(*it_nt, dynamic_cast(outside_NTs.find(i->first)->second), skip_occurences); - // flag outside_alt as being part of an automatically generated outside production rule - outside_alt->set_partof_outside(true); - - outside_alt->m_ys_inside = outside_alt->multi_ys(); - // append copied+replaced alternative to the list of alternatives for the outside version of the called non-terminal - (dynamic_cast(outside_NTs.find(*((*it_nt)->name))->second))->alts.push_back(outside_alt); - skip_occurences[*((*it_nt)->name)]++; - } - } else { -// // e.g. struct = nil(LOC) has no non-terminal on the rhs, but should become an alternative for outside version of struct -// Alt::Base *outside_alt = (*alt)->clone(); -// // flag outside_alt as being part of an automatically generated outside production rule -// outside_alt->set_partof_outside(true); -// (dynamic_cast(outside_NTs.find(*(nt->name))->second))->alts.push_back(outside_alt); - - // if production rule does not have any non-terminals on the right hand side, the calling non-terminal - // is a candidate for being called from the axiom for outside - // e.g. hairpin = hl(BASE, REGION, BASE) --> outside_hairin get's added - axiom_candidates->insert(dynamic_cast(outside_NTs.find(*(nt->name))->second)); - } - } - } - } - Alt::Link *link = new Alt::Link(this->axiom_name, this->axiom_loc); +void Grammar::inject_outside_nts(std::vector outside_nt_list) { + std::string OUTSIDE_NT_PREFIX = "outside_"; + hashtable outside_NTs; + std::set outside_nts = std::set(); + + // 0) check if all user provided NTs for cmd flag are actually in the grammar + std::list *warn_missing_nts = new std::list(); + for (std::vector::iterator i = outside_nt_list.begin(); + i != outside_nt_list.end(); ++i) { + // we don't need to check of non-terminals are in inside grammar, if user + // requests ALL non-terminals + if ((*i).compare(std::string("ALL")) == 0) { + break; + } + if (this->NTs.find(*i) == this->NTs.end()) { + warn_missing_nts->push_back(*i); + } + } + if (warn_missing_nts->size() > 0) { + std::string msg = std::string( + "You requested outside grammar generation and reporting of results for " + "outside versions of non-terminal(s) "); + for (std::list::iterator i = warn_missing_nts->begin(); + i != warn_missing_nts->end(); ++i) { + msg += "'" + *i + "'"; + if (next(i) != warn_missing_nts->end()) { + msg += ", "; + } + } + msg += ", which do(es) NOT exist in your grammar!"; + throw LogError(msg); + } + + // 1) collect non-terminals used in rhs of productions together with their + // calling lhs non-terminal we thus exclude e.g. non-terminal foo in + // production "foo = BASE;" + for (hashtable::iterator i = NTs.begin(); + i != NTs.end(); ++i) { + Symbol::NT *nt = dynamic_cast(i->second); + if (nt) { // skip terminals + for (std::list::iterator alt = nt->alts.begin(); + alt != nt->alts.end(); ++alt) { + std::list *rhs_nts = new std::list(); + (*alt)->get_nonterminals(rhs_nts); + if (rhs_nts->size() > 0) { + // if rhs uses at least one non-terminal, both (= calling lhs + // non-terminal and all rhs non-terminal(s)) will be added as + // novel outside non-terminals + outside_nts.insert(nt); + outside_nts.insert(rhs_nts->begin(), rhs_nts->end()); + } + } + } + } + + // 2) create a novel outside non-terminal for each collected non-terminal + // in the inside grammar add both (new outside and user provided inside) + // non-terminal **names** as keys to the hash, pointing to the very same + // novel outside non-terminal. + if (outside_nts.size() == 0) { + // the grammar does not have any non-terminals on the rhs of productions. + // Thus, no new NTs have to be injected + return; + } + for (std::set::iterator nt = outside_nts.begin(); + nt != outside_nts.end(); ++nt) { + // use inside NT as template ... + Symbol::NT *inside_nt = dynamic_cast( + this->NTs.find(*(*nt)->name)->second); + // and clone for outside version + Symbol::NT *outside_nt = inside_nt->clone(inside_nt->track_pos()); + // remove all existing alternative production rules + outside_nt->alts.clear(); + // and correct NT name, to now point to outside version + outside_nt->name = new std::string(OUTSIDE_NT_PREFIX + (*(*nt)->name)); + + // flag this new non-terminal as being part of the outside rules + outside_nt->is_partof_outside = true; + + outside_NTs[OUTSIDE_NT_PREFIX + (*(*nt)->name)] = outside_nt; + outside_NTs[(*(*nt)->name)] = outside_nt; + } + + // 3) create copy of inside alternative, replace called non-terminal with + // calling outside version and append to called outside version + std::set *axiom_candidates = new std::set(); + for (hashtable::iterator i = NTs.begin(); + i != NTs.end(); ++i) { + Symbol::NT *nt = dynamic_cast(i->second); + if (nt) { // skip terminals + for (std::list::iterator alt = nt->alts.begin(); + alt != nt->alts.end(); ++alt) { + std::list *rhs_nts = new std::list(); + (*alt)->get_nonterminals(rhs_nts); + if (rhs_nts->size() > 0) { + hashtable skip_occurences; + for (std::list::iterator it_nt = rhs_nts->begin(); + it_nt != rhs_nts->end(); ++it_nt) { + skip_occurences[*((*it_nt)->name)] = 0; + } + // unsigned int skip_occurences = 0; + for (std::list::iterator it_nt = rhs_nts->begin(); + it_nt != rhs_nts->end(); ++it_nt) { + // create a copy (=clone) of the inside alternative + Alt::Base *outside_alt = (*alt)->clone(); + // replace one of the inside non-terminals with the outside + // version of the calling non-terminal + outside_alt->replace_nonterminal( + *it_nt, dynamic_cast( + outside_NTs.find(i->first)->second), skip_occurences); + // flag outside_alt as being part of an automatically generated + // outside production rule + outside_alt->set_partof_outside(true); + + outside_alt->m_ys_inside = outside_alt->multi_ys(); + // append copied+replaced alternative to the list of alternatives + // for the outside version of the called non-terminal + (dynamic_cast( + outside_NTs.find(*((*it_nt)->name))->second))->alts.push_back( + outside_alt); + skip_occurences[*((*it_nt)->name)]++; + } + } else { +// // e.g. struct = nil(LOC) has no non-terminal on the rhs, but + // should become an alternative for outside version of struct +// Alt::Base *outside_alt = (*alt)->clone(); +// // flag outside_alt as being part of an automatically generated + // outside production rule +// outside_alt->set_partof_outside(true); +// (dynamic_cast(outside_NTs.find(*(nt->name))->second)) + // ->alts.push_back(outside_alt); + + // if production rule does not have any non-terminals on the right + // hand side, the calling non-terminal is a candidate for being + // called from the axiom for outside + // e.g. hairpin = hl(BASE, REGION, BASE) --> outside_hairin + // get's added + axiom_candidates->insert(dynamic_cast( + outside_NTs.find(*(nt->name))->second)); + } + } + } + } + Alt::Link *link = new Alt::Link(this->axiom_name, this->axiom_loc); link->nt = dynamic_cast(this->axiom); Filter *f = new Filter(new std::string("complete_track"), Loc()); f->type = Filter::WITH; @@ -1098,68 +1153,77 @@ void Grammar::inject_outside_nts() { link->set_tracks(this->axiom->tracks(), this->axiom->track_pos()); link->init_multi_ys(); link->top_level = Bool(true); - dynamic_cast(outside_NTs.find(*this->axiom_name)->second)->alts.push_back(link); - - // 4) add novel outside NTs to grammar - for (std::pair nt : outside_NTs) { - if (this->NTs.find(nt.first) == this->NTs.end()) { // only for NTs that are not already part of the grammar - Symbol::NT *nt_nt = dynamic_cast(nt.second); - /* calling NT might never occur on rhs of productions, e.g. "start" in: - * grammar gra_nodangle uses sig_foldrna(axiom = start) { - * start = incl(struct) # h; - * struct = sadd(BASE, struct) | nil(LOC) # h; } - * the outside version of such non terminals shall point to the inside axiom - */ - if (nt_nt->alts.size() == 0) { - Alt::Link *link = new Alt::Link(this->axiom_name, this->axiom_loc); - link->nt = dynamic_cast(this->axiom); - nt_nt->alts.push_back(link); - } - this->NTs.insert(nt); - this->nt_list.push_back(nt_nt); - } - } - - // 5) inside production rules that have NO non-terminals on their right hand sides - // must be those that parse the final sub-words of the input. - // Therefore, they must be the smallest start-points for outside construction, - // i.e. the axiom should point to them. These non-terminals have been - // collected in step 3. - if (axiom_candidates->size() == 1) { - // if there is only one inside non-terminal to be accessed from the axiom - // we can directly define its outside version as the axiom - this->axiom_name = (*axiom_candidates->begin())->name; - this->init_axiom(); - } else { - // otherwise, we need to add another left hand side non-terminal - // which points to multiple non-terminals WITHOUT making a choice - std::string *nt_axiom_name = new std::string("outside_axioms"); - Symbol::NT *nt_axiom = new Symbol::NT(nt_axiom_name, Loc()); - // carry over tracks from original inside axiom - nt_axiom->set_tracks(this->axiom->tracks(), this->axiom->track_pos()); - nt_axiom->setup_multi_ys(); - - for (std::set::iterator i = axiom_candidates->begin(); i != axiom_candidates->end(); ++i) { - Alt::Link *link = new Alt::Link((*i)->name, Loc()); - link->nt = dynamic_cast(outside_NTs.find(*(*i)->name)->second); - link->set_tracks((*i)->tracks(), (*i)->track_pos()); - link->init_multi_ys(); - nt_axiom->alts.push_back(link); - } - // add new lhs non-terminal to grammar - this->NTs.insert(std::make_pair(*nt_axiom_name, dynamic_cast(nt_axiom))); - this->nt_list.push_back(nt_axiom); - - this->axiom_name = nt_axiom_name; - this->init_axiom(); - } - // re-run parts of "check_semantics" to properly initialize novel non- - // terminals and links to non-terminals, but explicitly do NOT - // re-run yield size analysis since it does not respect outside - // situations. - this->init_calls(); - this->init_in_out(); - this->init_table_dims(); + dynamic_cast(outside_NTs.find( + *this->axiom_name)->second)->alts.push_back(link); + + // 4) add novel outside NTs to grammar + for (std::pair nt : outside_NTs) { + // only for NTs that are not already part of the grammar + if (this->NTs.find(nt.first) == this->NTs.end()) { + Symbol::NT *nt_nt = dynamic_cast(nt.second); + /* calling NT might never occur on rhs of productions, e.g. "start" in: + * grammar gra_nodangle uses sig_foldrna(axiom = start) { + * start = incl(struct) # h; + * struct = sadd(BASE, struct) | nil(LOC) # h; } + * the outside version of such non terminals shall point to the inside + * axiom + */ + if (nt_nt->alts.size() == 0) { + Alt::Link *link = new Alt::Link(this->axiom_name, this->axiom_loc); + link->nt = dynamic_cast(this->axiom); + nt_nt->alts.push_back(link); + } + this->NTs.insert(nt); + this->nt_list.push_back(nt_nt); + } + } + + // 5) inside production rules that have NO non-terminals on their right hand + // sides must be those that parse the final sub-words of the input. + // Therefore, they must be the smallest start-points for outside + // construction, i.e. the axiom should point to them. These non-terminals + // have been collected in step 3. + if (axiom_candidates->size() == 1) { + // if there is only one inside non-terminal to be accessed from the axiom + // we can directly define its outside version as the axiom + this->axiom_name = (*axiom_candidates->begin())->name; + this->init_axiom(); + } else { + // otherwise, we need to add another left hand side non-terminal + // which points to multiple non-terminals WITHOUT making a choice + std::string *nt_axiom_name = new std::string("outside_axioms"); + Symbol::NT *nt_axiom = new Symbol::NT(nt_axiom_name, Loc()); + // carry over tracks from original inside axiom + nt_axiom->set_tracks(this->axiom->tracks(), this->axiom->track_pos()); + nt_axiom->setup_multi_ys(); + + for (std::set::iterator i = axiom_candidates->begin(); + i != axiom_candidates->end(); ++i) { + Alt::Link *link = new Alt::Link((*i)->name, Loc()); + link->nt = dynamic_cast( + outside_NTs.find(*(*i)->name)->second); + link->set_tracks((*i)->tracks(), (*i)->track_pos()); + link->init_multi_ys(); + nt_axiom->alts.push_back(link); + } + // add new lhs non-terminal to grammar + this->NTs.insert(std::make_pair( + *nt_axiom_name, dynamic_cast(nt_axiom))); + this->nt_list.push_back(nt_axiom); + + this->axiom_name = nt_axiom_name; + this->init_axiom(); + } + // re-run parts of "check_semantics" to properly initialize novel non- + // terminals and links to non-terminals, but explicitly do NOT + // re-run yield size analysis since it does not respect outside + // situations. + this->init_calls(); + this->init_in_out(); + this->init_table_dims(); + + Log::instance()->verboseMessage( + "Grammar has been modified into an outside version."); } unsigned int Grammar::to_dot(unsigned int *nodeID, std::ostream &out) { diff --git a/src/grammar.hh b/src/grammar.hh index abb114036..3ee251f05 100644 --- a/src/grammar.hh +++ b/src/grammar.hh @@ -208,7 +208,7 @@ class Grammar { // inject additional non-terminals to grammar, to automatically create // an outside version for user provided grammar - void inject_outside_nts(); + void inject_outside_nts(std::vector outside_nt_list); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); }; diff --git a/src/symbol.cc b/src/symbol.cc index 4ab60e857..95d047ac0 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -751,10 +751,12 @@ void Symbol::NT::init_indices(Expr::Vacc *left, Expr::Vacc *right, right_indices[track] = right; for (std::list::iterator i = alts.begin(); i != alts.end(); ++i) { - if (this->is_partof_outside) { + if (this->is_partof_outside) { if ((*i)->get_is_partof_outside() == false) { - // special treatment for a link to the original inside axiom, ensure that it has to consume the full input sequence(s) - (*i)->init_indices_outside(left->plus(right), right->minus(left), k, track, NULL, NULL, false); + // special treatment for a link to the original inside axiom, ensure + // that it has to consume the full input sequence(s) + (*i)->init_indices_outside( + left->plus(right), right->minus(left), k, track, NULL, NULL, false); dynamic_cast(*i)->init_outside_guards(); } else { (*i)->init_indices_outside(left, right, k, track, NULL, NULL, false); diff --git a/src/symbol.hh b/src/symbol.hh index f3ce9fa43..d9711a3f1 100644 --- a/src/symbol.hh +++ b/src/symbol.hh @@ -249,7 +249,8 @@ class Base { const Yield::Multi &right, Symbol::NT *nt); virtual bool multi_detect_loop(); - // flag to indicate if alternative is for inside (the default) or outside production rules. + // flag to indicate if alternative is for inside (the default) or outside + // production rules. bool is_partof_outside; // prints grammar rules as GraphViz commands virtual unsigned int to_dot(unsigned int *nodeID, std::ostream &out, From 07eec591cc45789d869c8d8b77f43a3855f43eed Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 12 Apr 2022 14:38:24 +0200 Subject: [PATCH 059/209] provide a version without checking user arguments --- src/grammar.cc | 4 ++++ src/grammar.hh | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/grammar.cc b/src/grammar.cc index 126e96d84..5b9a36e80 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1002,6 +1002,10 @@ void Grammar::remove(Symbol::NT *x) { tabulated.erase(nt); } +void Grammar::inject_outside_nts() { + std::vector outside_nt_list; + Grammar::inject_outside_nts(outside_nt_list); +} void Grammar::inject_outside_nts(std::vector outside_nt_list) { std::string OUTSIDE_NT_PREFIX = "outside_"; hashtable outside_NTs; diff --git a/src/grammar.hh b/src/grammar.hh index 3ee251f05..4479ca434 100644 --- a/src/grammar.hh +++ b/src/grammar.hh @@ -209,6 +209,8 @@ class Grammar { // inject additional non-terminals to grammar, to automatically create // an outside version for user provided grammar void inject_outside_nts(std::vector outside_nt_list); + // same as above, without checking user provided NT list + void inject_outside_nts(); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); }; From d3413518b3d0f2d113747e8ed1a50cf2a4641066 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 12 Apr 2022 14:41:11 +0200 Subject: [PATCH 060/209] no need for -- loops --- src/cpp.cc | 7 +------ src/statement.hh | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 876c1a7cc..3a03308f1 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -81,12 +81,7 @@ void Printer::Cpp::print(const Statement::For &stmt) { stream << *stmt.var_decl; stream << ' ' << *stmt.cond << "; "; if (!stmt.inc) { - if (stmt.decrement) { - stream << "--"; - } else { - stream << "++"; - } - stream << *stmt.var_decl->name << ")"; + stream << "++" << *stmt.var_decl->name << ")"; } else { bool t = in_fn_head; in_fn_head = true; diff --git a/src/statement.hh b/src/statement.hh index c928ce486..3e519d8ce 100644 --- a/src/statement.hh +++ b/src/statement.hh @@ -272,7 +272,6 @@ class For : public Block_Base { Var_Decl *var_decl; Expr::Base *cond; Statement::Base *inc; - bool decrement = false; For(Var_Decl *v, Expr::Base* e) : Block_Base(FOR), var_decl(v), cond(e), inc(NULL) { From 363e8b3b57d626161319003924ba67ba4d8afbd5 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 12 Apr 2022 14:43:17 +0200 Subject: [PATCH 061/209] cpplint --- rtlib/filter.hh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rtlib/filter.hh b/rtlib/filter.hh index 5b012b994..30731a7c3 100644 --- a/rtlib/filter.hh +++ b/rtlib/filter.hh @@ -128,8 +128,9 @@ inline bool samesize(const Basic_Sequence &s1, } template -inline bool complete_track(const Basic_Sequence &seq, T i, T j) { - return ((i == seq.n) and (j == seq.n)); +inline bool complete_track( + const Basic_Sequence &seq, T i, T j) { + return ((i == seq.n) && (j == seq.n)); } #endif // RTLIB_FILTER_HH_ From 3ca48e03ec40a83428deb1c270aa062a34a8cb31 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 12 Apr 2022 18:43:42 +0200 Subject: [PATCH 062/209] reporting function for outside computation --- src/cpp.cc | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/cpp.hh | 2 ++ src/gapc.cc | 1 + 3 files changed, 57 insertions(+) diff --git a/src/cpp.cc b/src/cpp.cc index 3a03308f1..b6597b5e7 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2050,6 +2050,60 @@ void Printer::Cpp::print_cyk_fn(const AST &ast) { stream << endl; } +void Printer::Cpp::print_insideoutside(Symbol::NT *nt, Table::Dim table_dimension) { + std::string args = std::string(""); + std::string args_print = std::string(""); + if (table_dimension == Table::QUADRATIC) { + args = std::string("i,j"); + args_print = std::string(" << i << \",\" << j << "); + } + if (table_dimension == Table::LINEAR) { + args = std::string("i"); + args_print = std::string(" << i << "); + } + stream << indent() << "std::cout << \"start answers " << *nt->name << "(\"" << args_print << "\"):\\n\";\n"; + stream << indent() << "print_result(std::cout, nt_" << *nt->name << "(" << args << "));\n"; + stream << indent() << "std::cout << \"//end answers " << *nt->name << "(\"" << args_print << "\")\\n\";\n"; +} +void Printer::Cpp::print_insideoutside_report_fn(std::vector outside_nt_list, const AST &ast) { + stream << indent() << "void report() {" << endl; + inc_indent(); + for (std::vector::iterator i = outside_nt_list.begin(); i != outside_nt_list.end(); ++i) { + Symbol::NT *nt_inside = dynamic_cast((*ast.grammar()->NTs.find(*i)).second); + Symbol::NT *nt_outside = dynamic_cast((*ast.grammar()->NTs.find(std::string("outside_") + *i)).second); + size_t track = 0; + if (nt_inside->tables()[track].type() == Table::QUADRATIC) { + stream << indent() << "for (unsigned int i = t_" << track << "_left_most; i <= t_" << track << "_right_most; ++i) {" << endl; + inc_indent(); + stream << indent() << "for (unsigned int j = i; j <= t_" << track << "_right_most; ++j) {" << endl; + inc_indent(); + + print_insideoutside(nt_inside, nt_inside->tables()[track].type()); + print_insideoutside(nt_outside, nt_inside->tables()[track].type()); + + dec_indent(); + stream << indent() << "}" << endl; + dec_indent(); + stream << indent() << "}" << endl; + } + if (nt_inside->tables()[track].type() == Table::LINEAR) { + stream << indent() << "for (unsigned int i = t_" << track << "_left_most; i <= t_" << track << "_right_most; ++i) {" << endl; + inc_indent(); + + print_insideoutside(nt_inside, nt_inside->tables()[track].type()); + print_insideoutside(nt_outside, nt_inside->tables()[track].type()); + + dec_indent(); + stream << indent() << "}" << endl; + } + if (nt_inside->tables()[track].type() == Table::CONSTANT) { + print_insideoutside(nt_inside, nt_inside->tables()[track].type()); + print_insideoutside(nt_outside, nt_inside->tables()[track].type()); + } + } + dec_indent(); + stream << indent() << "}" << endl << endl; +} void Printer::Cpp::print_run_fn(const AST &ast) { stream << indent() << *ast.grammar()->axiom->code()->return_type; diff --git a/src/cpp.hh b/src/cpp.hh index 582d96b85..1f116f055 100644 --- a/src/cpp.hh +++ b/src/cpp.hh @@ -96,8 +96,10 @@ class Cpp : public Base { public: void print_cyk_fn(const AST &ast); + void print_insideoutside_report_fn(std::vector outside_nt_list, const AST &ast); private: + void print_insideoutside(Symbol::NT *nt, Table::Dim table_dimension); void print_run_fn(const AST &ast); void print_stats_fn(const AST &ast); diff --git a/src/gapc.cc b/src/gapc.cc index 83b12f7ba..926e60bd5 100644 --- a/src/gapc.cc +++ b/src/gapc.cc @@ -630,6 +630,7 @@ class Main { hh.footer(driver.ast); hh.end_fwd_decls(); hh.header_footer(driver.ast); + hh.print_insideoutside_report_fn(opts.outside_nt_list, driver.ast); // Write out the C++ implementation file of the // compile-result. From 2d1894dba02ec851dbdc1eaf2778bd897fc65f57 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 12 Apr 2022 21:44:23 +0200 Subject: [PATCH 063/209] report DP matrices in outside mode --- rtlib/generic_main.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rtlib/generic_main.cc b/rtlib/generic_main.cc index 91f4a5e3d..9a55d6482 100644 --- a/rtlib/generic_main.cc +++ b/rtlib/generic_main.cc @@ -93,8 +93,12 @@ int main(int argc, char **argv) { gapc::add_event("end_computation"); +#ifndef OUTSIDE std::cout << "Answer: \n"; obj.print_result(std::cout, res); +#else + obj.report_insideoutside(std::cout); +#endif gapc::add_event("end_result_pp"); From 03066e0575857063309d0ccb60f8403833af7097 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 12 Apr 2022 21:44:42 +0200 Subject: [PATCH 064/209] commit outside axiom as being part of outside --- src/grammar.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/grammar.cc b/src/grammar.cc index 5b9a36e80..d2ad2c09e 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1200,6 +1200,7 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { // carry over tracks from original inside axiom nt_axiom->set_tracks(this->axiom->tracks(), this->axiom->track_pos()); nt_axiom->setup_multi_ys(); + nt_axiom->is_partof_outside = true; for (std::set::iterator i = axiom_candidates->begin(); i != axiom_candidates->end(); ++i) { From 9a950c1e80b35986d4f53e790ffdc2e70e25d44f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 12 Apr 2022 21:45:05 +0200 Subject: [PATCH 065/209] generate function to return DP matrix values --- src/cpp.cc | 62 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index b6597b5e7..700b39089 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -1682,7 +1682,10 @@ void Printer::Cpp::header(const AST &ast) { stream << "#define USE_GSL\n"; } if (ast.get_float_acc() > 0) { - stream << "#define FLOAT_ACC " << ast.get_float_acc() << "\n"; + stream << "#define FLOAT_ACC " << ast.get_float_acc() << "\n"; + } + if (ast.grammar()->axiom->is_partof_outside) { + stream << "#define OUTSIDE\n"; } includes(); print_subseq_typedef(ast); @@ -2061,44 +2064,71 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt, Table::Dim table_dimensio args = std::string("i"); args_print = std::string(" << i << "); } - stream << indent() << "std::cout << \"start answers " << *nt->name << "(\"" << args_print << "\"):\\n\";\n"; - stream << indent() << "print_result(std::cout, nt_" << *nt->name << "(" << args << "));\n"; - stream << indent() << "std::cout << \"//end answers " << *nt->name << "(\"" << args_print << "\")\\n\";\n"; + stream << indent() << "out << \"start answers " << *nt->name << "(\"" << args_print << "\"):\\n\";\n"; + std::list &l = nt->code_list(); + for (std::list::iterator i = l.begin(); i != l.end(); ++i) { + std::string res = "res_"; + if (nt->is_partof_outside) { + res += std::string("outside"); + } else { + res += std::string("inside"); + } + stream << indent() << *((*i)->return_type) << " " << res << " = nt_" << *nt->name << "(" << args << ");\n"; + stream << indent() << "print_result(std::cout, " << res << ");\n"; + } + stream << indent() << "out << \"//end answers " << *nt->name << "(\"" << args_print << "\")\\n\";\n"; } void Printer::Cpp::print_insideoutside_report_fn(std::vector outside_nt_list, const AST &ast) { - stream << indent() << "void report() {" << endl; + stream << indent() << "void report_insideoutside(std::ostream &out) {" << endl; inc_indent(); + std::list *reported_nts = new std::list(); for (std::vector::iterator i = outside_nt_list.begin(); i != outside_nt_list.end(); ++i) { - Symbol::NT *nt_inside = dynamic_cast((*ast.grammar()->NTs.find(*i)).second); - Symbol::NT *nt_outside = dynamic_cast((*ast.grammar()->NTs.find(std::string("outside_") + *i)).second); + if ((*i).compare(std::string("ALL")) == 0) { + reported_nts->clear(); + for (hashtable::iterator j = (*ast.grammar()).NTs.begin(); j != (*ast.grammar()).NTs.end(); ++j) { + Symbol::NT *inside_nt = dynamic_cast(j->second); + if (inside_nt) { + if (!inside_nt->is_partof_outside) { + reported_nts->push_back(inside_nt); + } + } + } + break; + } else { + reported_nts->push_back(dynamic_cast((*ast.grammar()->NTs.find(*i)).second)); + } + } + + for (std::list::iterator nt_inside = reported_nts->begin(); nt_inside != reported_nts->end(); ++nt_inside) { + Symbol::NT *nt_outside = dynamic_cast((*ast.grammar()->NTs.find(std::string("outside_") + *((*nt_inside)->name))).second); size_t track = 0; - if (nt_inside->tables()[track].type() == Table::QUADRATIC) { + if ((*nt_inside)->tables()[track].type() == Table::QUADRATIC) { stream << indent() << "for (unsigned int i = t_" << track << "_left_most; i <= t_" << track << "_right_most; ++i) {" << endl; inc_indent(); stream << indent() << "for (unsigned int j = i; j <= t_" << track << "_right_most; ++j) {" << endl; inc_indent(); - print_insideoutside(nt_inside, nt_inside->tables()[track].type()); - print_insideoutside(nt_outside, nt_inside->tables()[track].type()); + print_insideoutside(*nt_inside, (*nt_inside)->tables()[track].type()); + print_insideoutside(nt_outside, nt_outside->tables()[track].type()); dec_indent(); stream << indent() << "}" << endl; dec_indent(); stream << indent() << "}" << endl; } - if (nt_inside->tables()[track].type() == Table::LINEAR) { + if ((*nt_inside)->tables()[track].type() == Table::LINEAR) { stream << indent() << "for (unsigned int i = t_" << track << "_left_most; i <= t_" << track << "_right_most; ++i) {" << endl; inc_indent(); - print_insideoutside(nt_inside, nt_inside->tables()[track].type()); - print_insideoutside(nt_outside, nt_inside->tables()[track].type()); + print_insideoutside(*nt_inside, (*nt_inside)->tables()[track].type()); + print_insideoutside(nt_outside, nt_outside->tables()[track].type()); dec_indent(); stream << indent() << "}" << endl; } - if (nt_inside->tables()[track].type() == Table::CONSTANT) { - print_insideoutside(nt_inside, nt_inside->tables()[track].type()); - print_insideoutside(nt_outside, nt_inside->tables()[track].type()); + if ((*nt_inside)->tables()[track].type() == Table::CONSTANT) { + print_insideoutside(*nt_inside, (*nt_inside)->tables()[track].type()); + print_insideoutside(nt_outside, nt_outside->tables()[track].type()); } } dec_indent(); From ab2b3e450dd277dbd886d586c4ec15d76f3906db Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 12 Apr 2022 21:51:41 +0200 Subject: [PATCH 066/209] codestyle --- src/cpp.cc | 138 ++++++++++++++++++++++++++++++----------------------- src/cpp.hh | 3 +- 2 files changed, 80 insertions(+), 61 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 700b39089..5dd8d4d3b 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2053,7 +2053,8 @@ void Printer::Cpp::print_cyk_fn(const AST &ast) { stream << endl; } -void Printer::Cpp::print_insideoutside(Symbol::NT *nt, Table::Dim table_dimension) { +void Printer::Cpp::print_insideoutside( + Symbol::NT *nt, Table::Dim table_dimension) { std::string args = std::string(""); std::string args_print = std::string(""); if (table_dimension == Table::QUADRATIC) { @@ -2064,72 +2065,89 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt, Table::Dim table_dimensio args = std::string("i"); args_print = std::string(" << i << "); } - stream << indent() << "out << \"start answers " << *nt->name << "(\"" << args_print << "\"):\\n\";\n"; + stream << indent() << "out << \"start answers " << *nt->name + << "(\"" << args_print << "\"):\\n\";\n"; std::list &l = nt->code_list(); for (std::list::iterator i = l.begin(); i != l.end(); ++i) { - std::string res = "res_"; - if (nt->is_partof_outside) { - res += std::string("outside"); - } else { - res += std::string("inside"); - } - stream << indent() << *((*i)->return_type) << " " << res << " = nt_" << *nt->name << "(" << args << ");\n"; + std::string res = "res_"; + if (nt->is_partof_outside) { + res += std::string("outside"); + } else { + res += std::string("inside"); + } + stream << indent() << *((*i)->return_type) << " " << res << " = nt_" + << *nt->name << "(" << args << ");\n"; stream << indent() << "print_result(std::cout, " << res << ");\n"; } - stream << indent() << "out << \"//end answers " << *nt->name << "(\"" << args_print << "\")\\n\";\n"; + stream << indent() << "out << \"//end answers " << *nt->name << "(\"" + << args_print << "\")\\n\";\n"; } -void Printer::Cpp::print_insideoutside_report_fn(std::vector outside_nt_list, const AST &ast) { - stream << indent() << "void report_insideoutside(std::ostream &out) {" << endl; +void Printer::Cpp::print_insideoutside_report_fn( + std::vector outside_nt_list, const AST &ast) { + stream << indent() << "void report_insideoutside(std::ostream &out) {" + << endl; inc_indent(); std::list *reported_nts = new std::list(); - for (std::vector::iterator i = outside_nt_list.begin(); i != outside_nt_list.end(); ++i) { - if ((*i).compare(std::string("ALL")) == 0) { - reported_nts->clear(); - for (hashtable::iterator j = (*ast.grammar()).NTs.begin(); j != (*ast.grammar()).NTs.end(); ++j) { - Symbol::NT *inside_nt = dynamic_cast(j->second); - if (inside_nt) { - if (!inside_nt->is_partof_outside) { - reported_nts->push_back(inside_nt); - } - } - } - break; - } else { - reported_nts->push_back(dynamic_cast((*ast.grammar()->NTs.find(*i)).second)); - } - } - - for (std::list::iterator nt_inside = reported_nts->begin(); nt_inside != reported_nts->end(); ++nt_inside) { - Symbol::NT *nt_outside = dynamic_cast((*ast.grammar()->NTs.find(std::string("outside_") + *((*nt_inside)->name))).second); - size_t track = 0; - if ((*nt_inside)->tables()[track].type() == Table::QUADRATIC) { - stream << indent() << "for (unsigned int i = t_" << track << "_left_most; i <= t_" << track << "_right_most; ++i) {" << endl; - inc_indent(); - stream << indent() << "for (unsigned int j = i; j <= t_" << track << "_right_most; ++j) {" << endl; - inc_indent(); - - print_insideoutside(*nt_inside, (*nt_inside)->tables()[track].type()); - print_insideoutside(nt_outside, nt_outside->tables()[track].type()); - - dec_indent(); - stream << indent() << "}" << endl; - dec_indent(); - stream << indent() << "}" << endl; - } - if ((*nt_inside)->tables()[track].type() == Table::LINEAR) { - stream << indent() << "for (unsigned int i = t_" << track << "_left_most; i <= t_" << track << "_right_most; ++i) {" << endl; - inc_indent(); - - print_insideoutside(*nt_inside, (*nt_inside)->tables()[track].type()); - print_insideoutside(nt_outside, nt_outside->tables()[track].type()); - - dec_indent(); - stream << indent() << "}" << endl; - } - if ((*nt_inside)->tables()[track].type() == Table::CONSTANT) { - print_insideoutside(*nt_inside, (*nt_inside)->tables()[track].type()); - print_insideoutside(nt_outside, nt_outside->tables()[track].type()); - } + for (std::vector::iterator i = outside_nt_list.begin(); + i != outside_nt_list.end(); ++i) { + if ((*i).compare(std::string("ALL")) == 0) { + reported_nts->clear(); + for (hashtable::iterator + j = (*ast.grammar()).NTs.begin(); + j != (*ast.grammar()).NTs.end(); ++j) { + Symbol::NT *inside_nt = dynamic_cast(j->second); + if (inside_nt) { + if (!inside_nt->is_partof_outside) { + reported_nts->push_back(inside_nt); + } + } + } + break; + } else { + reported_nts->push_back( + dynamic_cast((*ast.grammar()->NTs.find(*i)).second)); + } + } + + for (std::list::iterator nt_inside = reported_nts->begin(); + nt_inside != reported_nts->end(); ++nt_inside) { + Symbol::NT *nt_outside = dynamic_cast( + (*ast.grammar()->NTs.find(std::string("outside_") + + *((*nt_inside)->name))).second); + size_t track = 0; + if ((*nt_inside)->tables()[track].type() == Table::QUADRATIC) { + stream << indent() << "for (unsigned int i = t_" << track + << "_left_most; i <= t_" << track << "_right_most; ++i) {" + << endl; + inc_indent(); + stream << indent() << "for (unsigned int j = i; j <= t_" << track + << "_right_most; ++j) {" << endl; + inc_indent(); + + print_insideoutside(*nt_inside, (*nt_inside)->tables()[track].type()); + print_insideoutside(nt_outside, nt_outside->tables()[track].type()); + + dec_indent(); + stream << indent() << "}" << endl; + dec_indent(); + stream << indent() << "}" << endl; + } + if ((*nt_inside)->tables()[track].type() == Table::LINEAR) { + stream << indent() << "for (unsigned int i = t_" << track + << "_left_most; i <= t_" << track << "_right_most; ++i) {" + << endl; + inc_indent(); + + print_insideoutside(*nt_inside, (*nt_inside)->tables()[track].type()); + print_insideoutside(nt_outside, nt_outside->tables()[track].type()); + + dec_indent(); + stream << indent() << "}" << endl; + } + if ((*nt_inside)->tables()[track].type() == Table::CONSTANT) { + print_insideoutside(*nt_inside, (*nt_inside)->tables()[track].type()); + print_insideoutside(nt_outside, nt_outside->tables()[track].type()); + } } dec_indent(); stream << indent() << "}" << endl << endl; diff --git a/src/cpp.hh b/src/cpp.hh index 1f116f055..1547a775f 100644 --- a/src/cpp.hh +++ b/src/cpp.hh @@ -96,7 +96,8 @@ class Cpp : public Base { public: void print_cyk_fn(const AST &ast); - void print_insideoutside_report_fn(std::vector outside_nt_list, const AST &ast); + void print_insideoutside_report_fn( + std::vector outside_nt_list, const AST &ast); private: void print_insideoutside(Symbol::NT *nt, Table::Dim table_dimension); From 88d909b1b57756266239b70f76f57a45d81eb144 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 12 Apr 2022 22:55:06 +0200 Subject: [PATCH 067/209] revert --- src/grammar.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/grammar.cc b/src/grammar.cc index d2ad2c09e..5b9a36e80 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1200,7 +1200,6 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { // carry over tracks from original inside axiom nt_axiom->set_tracks(this->axiom->tracks(), this->axiom->track_pos()); nt_axiom->setup_multi_ys(); - nt_axiom->is_partof_outside = true; for (std::set::iterator i = axiom_candidates->begin(); i != axiom_candidates->end(); ++i) { From 63c4a898b27b3a653e1b285c4eb0485ea4cba5ce Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 12 Apr 2022 22:55:22 +0200 Subject: [PATCH 068/209] iterate grammar NTs and look for outside rules to learn if program shall report outside results --- src/cpp.cc | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/cpp.cc b/src/cpp.cc index 5dd8d4d3b..6bd8e1727 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -1684,7 +1684,19 @@ void Printer::Cpp::header(const AST &ast) { if (ast.get_float_acc() > 0) { stream << "#define FLOAT_ACC " << ast.get_float_acc() << "\n"; } - if (ast.grammar()->axiom->is_partof_outside) { + bool contains_outside_rules = false; + for (hashtable::iterator + i = (*ast.grammar()).NTs.begin(); + i != (*ast.grammar()).NTs.end(); ++i) { + Symbol::NT *nt = dynamic_cast((*i).second); + if (nt) { + if (nt->is_partof_outside) { + contains_outside_rules = true; + break; + } + } + } + if (contains_outside_rules) { stream << "#define OUTSIDE\n"; } includes(); @@ -2097,6 +2109,9 @@ void Printer::Cpp::print_insideoutside_report_fn( j != (*ast.grammar()).NTs.end(); ++j) { Symbol::NT *inside_nt = dynamic_cast(j->second); if (inside_nt) { + if (inside_nt->name->compare(std::string("outside_axioms")) == 0) { + continue; + } if (!inside_nt->is_partof_outside) { reported_nts->push_back(inside_nt); } From 9f57a5dbd6ca3f1dfe2e5a2a0588429bdd2bd53b Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 12 Apr 2022 23:56:45 +0200 Subject: [PATCH 069/209] provide mechanism to ask grammar if it is outside --- src/cpp.cc | 14 +------------- src/gapc.cc | 4 +++- src/grammar.cc | 3 +++ src/grammar.hh | 7 +++++++ 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 6bd8e1727..af8abb224 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -1684,19 +1684,7 @@ void Printer::Cpp::header(const AST &ast) { if (ast.get_float_acc() > 0) { stream << "#define FLOAT_ACC " << ast.get_float_acc() << "\n"; } - bool contains_outside_rules = false; - for (hashtable::iterator - i = (*ast.grammar()).NTs.begin(); - i != (*ast.grammar()).NTs.end(); ++i) { - Symbol::NT *nt = dynamic_cast((*i).second); - if (nt) { - if (nt->is_partof_outside) { - contains_outside_rules = true; - break; - } - } - } - if (contains_outside_rules) { + if ((*ast.grammar()).is_outside()) { stream << "#define OUTSIDE\n"; } includes(); diff --git a/src/gapc.cc b/src/gapc.cc index 926e60bd5..508e52512 100644 --- a/src/gapc.cc +++ b/src/gapc.cc @@ -630,7 +630,9 @@ class Main { hh.footer(driver.ast); hh.end_fwd_decls(); hh.header_footer(driver.ast); - hh.print_insideoutside_report_fn(opts.outside_nt_list, driver.ast); + if (grammar->is_outside()) { + hh.print_insideoutside_report_fn(opts.outside_nt_list, driver.ast); + } // Write out the C++ implementation file of the // compile-result. diff --git a/src/grammar.cc b/src/grammar.cc index 5b9a36e80..612e8969a 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1226,6 +1226,9 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { this->init_in_out(); this->init_table_dims(); + // mark the modified grammar as an outside one + this->outside = true; + Log::instance()->verboseMessage( "Grammar has been modified into an outside version."); } diff --git a/src/grammar.hh b/src/grammar.hh index 4479ca434..3a26b812b 100644 --- a/src/grammar.hh +++ b/src/grammar.hh @@ -64,7 +64,14 @@ class Grammar { void renumber_nts(); void move_new_nts(); + // flag the grammar as being an outside version + // default is false, will be set to true by function inject_outside_nts + bool outside = false; + public: + bool is_outside() { + return this->outside; + } // The name of the grammar as defined in the grammar name of // the gap-source-code file. std::string *name; From ccb9622ca07297a8d8b0bd896631f18e340ea848 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 13 Apr 2022 08:49:29 +0200 Subject: [PATCH 070/209] added alg_pfunc + made grammar independent from fold-grammars repo --- testdata/grammar_outside/nodangle.gap | 63 ++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/testdata/grammar_outside/nodangle.gap b/testdata/grammar_outside/nodangle.gap index acdcbcc21..53831f409 100644 --- a/testdata/grammar_outside/nodangle.gap +++ b/testdata/grammar_outside/nodangle.gap @@ -63,6 +63,48 @@ algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { } } +algebra alg_pfunc implements sig_foldrna(alphabet = char, answer = double) { + double sadd(Subsequence lb, double x) { + return scale(1) * x * mk_pf(sbase_energy()); + } + double cadd(double x, double y) { + return x * y; + } + double drem(Subsequence lb, double x, Subsequence rb) { + return x * mk_pf(termau_energy(lb, rb)); + } + double sr(Subsequence lb, double x, Subsequence rb) { + return scale(2) * x * mk_pf(sr_energy(lb, rb)); + } + double hl(Subsequence lb, Subsequence r, Subsequence rb) { + return scale(2+r.j-r.i) * mk_pf(hl_energy(r)); + } + double bl(Subsequence lb, Subsequence lr, double x, Subsequence rb) { + return scale(2+lr.j-lr.i) * x * mk_pf(bl_energy(lr, rb)); + } + double br(Subsequence lb, double x, Subsequence rr, Subsequence rb) { + return scale(2+rr.j-rr.i) * x * mk_pf(br_energy(lb, rr)); + } + double il(Subsequence lb, Subsequence lr, double x, Subsequence rr, Subsequence rb) { + return scale(2+lr.j-lr.i+rr.j-rr.i) * x * mk_pf(il_energy(lr, rr)); + } + double ml(Subsequence lb, double x, Subsequence rb) { + return scale(2) * x * mk_pf(ml_energy() + ul_energy() + termau_energy(lb, rb)); + } + double incl(double x) { + return x * mk_pf(ul_energy()); + } + double addss(double x, Subsequence r) { + return scale(r.j-r.i) * x * mk_pf(ss_energy(r)); + } + double nil(Subsequence n) { + return 1; + } + choice [double] h([double] i) { + return list(sum(i)); + } +} + algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { string sadd(Subsequence lb,string e) { string res; @@ -157,7 +199,7 @@ algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) } grammar gra_nodangle uses sig_foldrna(axiom = struct) { - struct = sadd(BASE with unpaired, struct) | + struct = sadd(BASE, struct) | cadd(dangle, struct) | nil(LOC) # h; @@ -172,20 +214,21 @@ grammar gra_nodangle uses sig_foldrna(axiom = struct) { iloop | multiloop # h; - stack = sr(BASE, weak, BASE) with basepair # h; - hairpin = hl(BASE, REGION with minsize(3) with unpaired, BASE) with basepair # h; - leftB = bl(BASE, REGION with maxsize(30) with unpaired, strong, BASE) with basepair # h; - rightB = br(BASE, strong, REGION with maxsize(30) with unpaired, BASE) with basepair # h; - iloop = il(BASE, REGION with maxsize(30) with unpaired, strong, REGION with maxsize(30) with unpaired, BASE) with basepair # h; - multiloop = ml(BASE, ml_comps, BASE) with basepair # h; + stack = sr(BASE, weak, BASE) with basepairing # h; + hairpin = hl(BASE, REGION with minsize(3), BASE) with basepairing # h; + leftB = bl(BASE, REGION with maxsize(30), strong, BASE) with basepairing # h; + rightB = br(BASE, strong, REGION with maxsize(30), BASE) with basepairing # h; + iloop = il(BASE, REGION with maxsize(30), strong, REGION with maxsize(30), BASE) with basepairing # h; + multiloop = ml(BASE, ml_comps, BASE) with basepairing # h; - ml_comps = sadd(BASE with unpaired, ml_comps) | + ml_comps = sadd(BASE, ml_comps) | cadd(incl(dangle), ml_comps1) # h; - ml_comps1 = sadd(BASE with unpaired, ml_comps1) | + ml_comps1 = sadd(BASE, ml_comps1) | cadd(incl(dangle), ml_comps1) | incl(dangle) | - addss(incl(dangle), REGION with unpaired) # h; + addss(incl(dangle), REGION) # h; } instance mfe = gra_nodangle(alg_mfe); +instance pfunc = gra_nodangle(alg_pfunc); From e7faf36f8bac61ed1c727479d44ff006c0a4a8d9 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 13 Apr 2022 08:54:09 +0200 Subject: [PATCH 071/209] added count instance --- testdata/grammar_outside/nodangle.gap | 1 + 1 file changed, 1 insertion(+) diff --git a/testdata/grammar_outside/nodangle.gap b/testdata/grammar_outside/nodangle.gap index 53831f409..f83293d10 100644 --- a/testdata/grammar_outside/nodangle.gap +++ b/testdata/grammar_outside/nodangle.gap @@ -232,3 +232,4 @@ grammar gra_nodangle uses sig_foldrna(axiom = struct) { instance mfe = gra_nodangle(alg_mfe); instance pfunc = gra_nodangle(alg_pfunc); +instance count = gra_nodangle(alg_count); From 0f92ad0bfe7c53580d7e5a26d610a61b27d01081 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 13 Apr 2022 08:54:24 +0200 Subject: [PATCH 072/209] added first three outside tests --- testdata/regresstest/config | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 523b3083c..3d950faba 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -443,3 +443,9 @@ check_new_old_eq eraseanswers.gap unused ins_count_failing "AC" eraseanswer check_new_old_eq eraseanswers.gap unused ins_count_working "AC" eraseanswer check_new_old_eq eraseanswers_product.gap unused ins "CaaG" eraseanswer_one check_new_old_eq eraseanswers_product.gap unused ins "CaaaG" eraseanswer_four + +GRAMMAR=../../grammar_outside +GAPC="../../../gapc --outside_grammar weak --outside_grammar struct" +check_new_old_eq nodangle.gap unused pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside +check_new_old_eq nodangle.gap unused mfe "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside +check_new_old_eq nodangle.gap unused count "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside From 9ee2f750a70558a45afa622a36fd7b4a171257f3 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 13 Apr 2022 08:55:33 +0200 Subject: [PATCH 073/209] leave some comments --- testdata/regresstest/config | 1 + 1 file changed, 1 insertion(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 3d950faba..62ec18b64 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -444,6 +444,7 @@ check_new_old_eq eraseanswers.gap unused ins_count_working "AC" eraseanswer check_new_old_eq eraseanswers_product.gap unused ins "CaaG" eraseanswer_one check_new_old_eq eraseanswers_product.gap unused ins "CaaaG" eraseanswer_four +# results have been verified against "RNAfold -p -d0 --bppmThreshold=0" v2.4.17 and Stefan's manual python outside version of nodangle GRAMMAR=../../grammar_outside GAPC="../../../gapc --outside_grammar weak --outside_grammar struct" check_new_old_eq nodangle.gap unused pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside From 519d133b8125a978466a0e9548fb4010993ec689 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 13 Apr 2022 10:41:04 +0200 Subject: [PATCH 074/209] function implemented --- src/alt.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/alt.cc b/src/alt.cc index 6e74a9fde..7b3bf93dc 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3149,7 +3149,9 @@ void Alt::Link::set_partof_outside(bool is_outside) { } } void Alt::Block::set_partof_outside(bool is_outside) { - throw LogError("Alt::Block::set_partof_outside is not yet implemented!"); + for (std::list::iterator i = this->alts.begin(); i != this->alts.end(); ++i) { + (*i)->set_partof_outside(is_outside); + } } void Alt::Multi::set_partof_outside(bool is_outside) { throw LogError("Alt::Multi::set_partof_outside is not yet implemented!"); From 1363a03f42452de9cc161370680ab85266828ee4 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 13 Apr 2022 13:09:46 +0200 Subject: [PATCH 075/209] report multiple tracks --- src/cpp.cc | 90 +++++++++++++++++++++++++++++------------------------- src/cpp.hh | 2 +- 2 files changed, 49 insertions(+), 43 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index af8abb224..e39ad9754 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2054,19 +2054,25 @@ void Printer::Cpp::print_cyk_fn(const AST &ast) { } void Printer::Cpp::print_insideoutside( - Symbol::NT *nt, Table::Dim table_dimension) { - std::string args = std::string(""); - std::string args_print = std::string(""); - if (table_dimension == Table::QUADRATIC) { - args = std::string("i,j"); - args_print = std::string(" << i << \",\" << j << "); - } - if (table_dimension == Table::LINEAR) { - args = std::string("i"); - args_print = std::string(" << i << "); + Symbol::NT *nt) { + std::ostringstream args; + std::ostringstream args_print; + for (size_t track = 0; track < (*nt).tracks(); ++track) { + if ((*nt).tables()[track].type() == Table::QUADRATIC) { + args << "t_" << track << "_i,t_" << track << "_j"; + args_print << " << t_" << track << "_i << \",\" << t_" << track << "_j"; + } + if ((*nt).tables()[track].type() == Table::LINEAR) { + args << "t_" << track << "_i"; + args_print << " << t_" << track << "_i"; + } + if (track+1 < (*nt).tracks()) { + args << ","; + args_print << " << \",\""; + } } stream << indent() << "out << \"start answers " << *nt->name - << "(\"" << args_print << "\"):\\n\";\n"; + << "(\"" << args_print.str() << " << \"):\\n\";\n"; std::list &l = nt->code_list(); for (std::list::iterator i = l.begin(); i != l.end(); ++i) { std::string res = "res_"; @@ -2076,11 +2082,11 @@ void Printer::Cpp::print_insideoutside( res += std::string("inside"); } stream << indent() << *((*i)->return_type) << " " << res << " = nt_" - << *nt->name << "(" << args << ");\n"; + << *nt->name << "(" << args.str() << ");\n"; stream << indent() << "print_result(std::cout, " << res << ");\n"; } stream << indent() << "out << \"//end answers " << *nt->name << "(\"" - << args_print << "\")\\n\";\n"; + << args_print.str() << " << \")\\n\";\n"; } void Printer::Cpp::print_insideoutside_report_fn( std::vector outside_nt_list, const AST &ast) { @@ -2117,39 +2123,39 @@ void Printer::Cpp::print_insideoutside_report_fn( Symbol::NT *nt_outside = dynamic_cast( (*ast.grammar()->NTs.find(std::string("outside_") + *((*nt_inside)->name))).second); - size_t track = 0; - if ((*nt_inside)->tables()[track].type() == Table::QUADRATIC) { - stream << indent() << "for (unsigned int i = t_" << track - << "_left_most; i <= t_" << track << "_right_most; ++i) {" - << endl; - inc_indent(); - stream << indent() << "for (unsigned int j = i; j <= t_" << track - << "_right_most; ++j) {" << endl; - inc_indent(); - - print_insideoutside(*nt_inside, (*nt_inside)->tables()[track].type()); - print_insideoutside(nt_outside, nt_outside->tables()[track].type()); - dec_indent(); - stream << indent() << "}" << endl; - dec_indent(); - stream << indent() << "}" << endl; + for (size_t track = 0; track < (*nt_inside)->tracks(); ++track) { + if ((*nt_inside)->tables()[track].type() == Table::QUADRATIC) { + stream << indent() << "for (unsigned int t_" << track << "_i = t_" << track + << "_left_most; t_" << track << "_i <= t_" << track << "_right_most; ++t_" << track << "_i) {" + << endl; + inc_indent(); + stream << indent() << "for (unsigned int t_" << track << "_j = t_" << track << "_i; t_" << track << "_j <= t_" << track + << "_right_most; ++t_" << track << "_j) {" << endl; + inc_indent(); + } + if ((*nt_inside)->tables()[track].type() == Table::LINEAR) { + stream << indent() << "for (unsigned int t_" << track << "_i = t_" << track + << "_left_most; t_" << track << "_i <= t_" << track << "_right_most; ++t_" << track << "_i) {" + << endl; + inc_indent(); + } } - if ((*nt_inside)->tables()[track].type() == Table::LINEAR) { - stream << indent() << "for (unsigned int i = t_" << track - << "_left_most; i <= t_" << track << "_right_most; ++i) {" - << endl; - inc_indent(); - print_insideoutside(*nt_inside, (*nt_inside)->tables()[track].type()); - print_insideoutside(nt_outside, nt_outside->tables()[track].type()); + print_insideoutside(*nt_inside); + print_insideoutside(nt_outside); - dec_indent(); - stream << indent() << "}" << endl; - } - if ((*nt_inside)->tables()[track].type() == Table::CONSTANT) { - print_insideoutside(*nt_inside, (*nt_inside)->tables()[track].type()); - print_insideoutside(nt_outside, nt_outside->tables()[track].type()); + for (size_t track = 0; track < (*nt_inside)->tracks(); ++track) { + if ((*nt_inside)->tables()[track].type() == Table::QUADRATIC) { + dec_indent(); + stream << indent() << "}" << endl; + dec_indent(); + stream << indent() << "}" << endl; + } + if ((*nt_inside)->tables()[track].type() == Table::LINEAR) { + dec_indent(); + stream << indent() << "}" << endl; + } } } dec_indent(); diff --git a/src/cpp.hh b/src/cpp.hh index 1547a775f..8e490f521 100644 --- a/src/cpp.hh +++ b/src/cpp.hh @@ -100,7 +100,7 @@ class Cpp : public Base { std::vector outside_nt_list, const AST &ast); private: - void print_insideoutside(Symbol::NT *nt, Table::Dim table_dimension); + void print_insideoutside(Symbol::NT *nt); void print_run_fn(const AST &ast); void print_stats_fn(const AST &ast); From 3350471a9c9c3842d0c8230598a1c1978ed22da7 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 13 Apr 2022 13:11:36 +0200 Subject: [PATCH 076/209] apply complete_track filter to ALL tracks --- src/grammar.cc | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/grammar.cc b/src/grammar.cc index 612e8969a..10867f972 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1150,15 +1150,19 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { } } Alt::Link *link = new Alt::Link(this->axiom_name, this->axiom_loc); - link->nt = dynamic_cast(this->axiom); - Filter *f = new Filter(new std::string("complete_track"), Loc()); - f->type = Filter::WITH; - link->filters.push_back(f); - link->set_tracks(this->axiom->tracks(), this->axiom->track_pos()); - link->init_multi_ys(); - link->top_level = Bool(true); - dynamic_cast(outside_NTs.find( - *this->axiom_name)->second)->alts.push_back(link); + link->nt = dynamic_cast(this->axiom); + Filter *f = new Filter(new std::string("complete_track"), Loc()); + f->type = Filter::WITH; + std::list *comptracks = new std::list(); + for (unsigned int track = 0; track < this->axiom->tracks(); ++track) { + comptracks->push_back(f); + } + link->set_tracks(this->axiom->tracks(), this->axiom->track_pos()); + link->init_multi_ys(); + link->add_multitrack_filter(*comptracks, f->type, Loc()); + link->top_level = Bool(true); + dynamic_cast(outside_NTs.find( + *this->axiom_name)->second)->alts.push_back(link); // 4) add novel outside NTs to grammar for (std::pair nt : outside_NTs) { From e6d3d496e3e96eb313162e6f71b4fb64f9be9d3e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 13 Apr 2022 13:13:58 +0200 Subject: [PATCH 077/209] implemented multi --- src/alt.cc | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 7b3bf93dc..473ffafbe 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3120,7 +3120,9 @@ void Alt::Block::get_nonterminals(std::list *nt_list) { } } void Alt::Multi::get_nonterminals(std::list *nt_list) { - throw LogError("Alt::Multi::get_nonterminals is not yet implemented!"); + for (std::list::iterator i = list.begin(); i != list.end(); ++i) { + (*i)->get_nonterminals(nt_list); + } } // recursively flag this alternative as being part of an inside (=false=default) @@ -3150,11 +3152,13 @@ void Alt::Link::set_partof_outside(bool is_outside) { } void Alt::Block::set_partof_outside(bool is_outside) { for (std::list::iterator i = this->alts.begin(); i != this->alts.end(); ++i) { - (*i)->set_partof_outside(is_outside); + (*i)->set_partof_outside(is_outside); } } void Alt::Multi::set_partof_outside(bool is_outside) { - throw LogError("Alt::Multi::set_partof_outside is not yet implemented!"); + for (std::list::iterator i = list.begin(); i != list.end(); ++i) { + (*i)->set_partof_outside(is_outside); + } } @@ -3209,7 +3213,11 @@ bool Alt::Block::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, } bool Alt::Multi::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences) { - throw LogError("Alt::Multi::replace_nonterminal is not yet implemented!"); + for (std::list::iterator i = list.begin(); i != list.end(); ++i) { + if ((*i)->replace_nonterminal(find, replace, skip_occurences)) { + return true; + } + } return false; } @@ -3705,7 +3713,16 @@ void Alt::Block::init_indices_outside(Expr::Base *left, Expr::Base *right, void Alt::Multi::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { - throw LogError("Alt::Multi::init_indices_outside is not yet implemented!"); + + Alt::Base::init_indices_outside(left, right, k, track, center_left, center_right); + size_t j = 0; + assert(track < list.size()); + std::list::iterator i = list.begin(); + for (; j < track; ++i, ++j) {} + + // each component is in a single-track context, thus we here deal only with + // ONE list element, chosen by the track + (*i)->init_indices_outside(left, right, k, 0, center_left, center_right); } std::pair *Alt::Base::get_outside_accum_yieldsizes( From b167d5fb00254360625cf2dee742e5654528be3c Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 13 Apr 2022 14:28:55 +0200 Subject: [PATCH 078/209] multi_filter: dot print --- src/alt.cc | 59 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 473ffafbe..f5211a954 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3234,6 +3234,21 @@ void to_dot_indices(std::vector indices, std::ostream &out) { } out << ""; } +void to_dot_filter(Filter *filter, std::ostream &out) { + out << *filter->name; + for (std::list::const_iterator arg = filter->args.begin(); + arg != filter->args.end(); ++arg) { + if (arg == filter->args.begin()) { + out << "("; + } + (*arg)->put(out); + if (std::next(arg) != filter->args.end()) { + out << ", "; + } else { + out << ")"; + } + } +} unsigned int Alt::Base::to_dot(unsigned int *nodeID, std::ostream &out) { unsigned int thisID = (unsigned int)((*nodeID)++); out << "node_" << thisID << " [ label=<"; @@ -3294,20 +3309,42 @@ unsigned int Alt::Base::to_dot(unsigned int *nodeID, std::ostream &out) { for (std::list::const_iterator filter = this->filters.begin(); filter != this->filters.end(); ++filter) { unsigned int childID = (unsigned int)((*nodeID)++); - out << "node_" << childID << " [ label=\"" << *(*filter)->name; - for (std::list::const_iterator arg = (*filter)->args.begin(); - arg != (*filter)->args.end(); ++arg) { - if (arg == (*filter)->args.begin()) { - out << "("; - } - (*arg)->put(out); - if (std::next(arg) != (*filter)->args.end()) { - out << ", "; + out << "node_" << childID << " [ label=\""; + to_dot_filter(*filter, out); + out << "\" , fontcolor=\"magenta\" , shape=none ];\n"; + out << "node_" << thisID << " -> node_" << childID + << " [ arrowhead=none, color=\"magenta\" ];\n"; + } + // Multiple filter can be added to each track. + // Unfortunately, filters are stored in a list per track, containing + // lists of filter. However, we want to draw one node per set of + // filters, i.e. filters at same position on all tracks. Therefore, + // we need to know the max number of filters across tracks and + // then add one node per position each containing all tracks. + unsigned int max_multifilter_number = 0; + for (std::vector >::iterator track = this->multi_filter.begin(); + track != this->multi_filter.end(); ++track) { + if ((*track).size() > max_multifilter_number) { + max_multifilter_number = (*track).size(); + } + } + for (unsigned int filterpos = 0; filterpos < max_multifilter_number; ++filterpos) { + unsigned int childID = (unsigned int)((*nodeID)++); + out << "node_" << childID << " [ label=<
"; + for (std::vector >::iterator i = this->multi_filter.begin(); + i != this->multi_filter.end(); ++i) { + out << ""; } - out << "\" , fontcolor=\"magenta\" , shape=none ];\n"; + out << "
"; + std::list::const_iterator filter = (*i).begin(); + // advance to filter position + for (unsigned int i = 0; i < filterpos; ++i, ++filter) {} + if (filter != (*i).end()) { + to_dot_filter(*filter, out); } else { - out << ")"; + out << "-"; } + out << "
>, fontcolor=\"magenta\", shape=none ];\n"; out << "node_" << thisID << " -> node_" << childID << " [ arrowhead=none, color=\"magenta\" ];\n"; } From f8a2b7c7297ed353a63f57deb08c024fe4e35082 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 13 Apr 2022 14:44:50 +0200 Subject: [PATCH 079/209] codestyle --- src/alt.cc | 15 ++++++++++----- src/cpp.cc | 14 ++++++++------ src/grammar.cc | 2 +- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index f5211a954..941069fd0 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3151,7 +3151,8 @@ void Alt::Link::set_partof_outside(bool is_outside) { } } void Alt::Block::set_partof_outside(bool is_outside) { - for (std::list::iterator i = this->alts.begin(); i != this->alts.end(); ++i) { + for (std::list::iterator i = this->alts.begin(); + i != this->alts.end(); ++i) { (*i)->set_partof_outside(is_outside); } } @@ -3322,16 +3323,19 @@ unsigned int Alt::Base::to_dot(unsigned int *nodeID, std::ostream &out) { // we need to know the max number of filters across tracks and // then add one node per position each containing all tracks. unsigned int max_multifilter_number = 0; - for (std::vector >::iterator track = this->multi_filter.begin(); + for (std::vector >::iterator + track = this->multi_filter.begin(); track != this->multi_filter.end(); ++track) { if ((*track).size() > max_multifilter_number) { max_multifilter_number = (*track).size(); } } - for (unsigned int filterpos = 0; filterpos < max_multifilter_number; ++filterpos) { + for (unsigned int filterpos = 0; filterpos < max_multifilter_number; + ++filterpos) { unsigned int childID = (unsigned int)((*nodeID)++); out << "node_" << childID << " [ label=<"; - for (std::vector >::iterator i = this->multi_filter.begin(); + for (std::vector >::iterator + i = this->multi_filter.begin(); i != this->multi_filter.end(); ++i) { out << ""; From cd605d109a20ac61f06bed9a558802e4d5a600ba Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 17 May 2022 15:41:47 +0200 Subject: [PATCH 108/209] print NULL instead of crashing --- src/symbol.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/symbol.cc b/src/symbol.cc index af1aa0745..195413025 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -1658,7 +1658,11 @@ unsigned int Symbol::Base::to_dot(unsigned int *nodeID, std::ostream &out, if (true) { // if we want to also print out datatypes out << "
"; - this->datatype->put(out); + if (this->datatype == NULL) { + out << "NULL"; + } else { + this->datatype->put(out); + } out << ""; } out << ""; From b5b552e4de56459e036eea57f19ccf3af20372df Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 20 May 2022 16:00:48 +0200 Subject: [PATCH 109/209] cannot expect all inside NTs to have outside analogons --- src/cpp.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 5fe488c40..1e0d98664 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2261,9 +2261,16 @@ void Printer::Cpp::print_insideoutside_report_fn( for (std::list::iterator nt_inside = reported_nts->begin(); nt_inside != reported_nts->end(); ++nt_inside) { - Symbol::NT *nt_outside = dynamic_cast( - (*ast.grammar()->NTs.find(std::string("outside_") + - *((*nt_inside)->name))).second); + hashtable::iterator outside_ntpair = + ast.grammar()->NTs.find(std::string("outside_") + *((*nt_inside)->name)); + if (outside_ntpair == ast.grammar()->NTs.end()) { + // not all inside NTs will be translated into outside NTs, e.g. + // times = CHAR('*') won't have an outside counterpart. + continue; + } + + Symbol::NT *nt_outside = + dynamic_cast((*outside_ntpair).second); for (size_t track = 0; track < (*nt_inside)->tracks(); ++track) { if ((*nt_inside)->tables()[track].type() == Table::QUADRATIC) { From cddbbd35cc6f764996c36e7b549f8751366fc525 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 20 May 2022 16:04:55 +0200 Subject: [PATCH 110/209] escape < > chars when printing datatype + flag for terminal NTs --- src/alt.cc | 4 ++-- src/alt.hh | 1 + src/symbol.cc | 2 +- src/symbol.hh | 2 +- src/type/base.cc | 21 +++++++++++++++++++++ src/type/base.hh | 4 ++++ 6 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index f9a35921d..c05b85e74 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3240,12 +3240,12 @@ unsigned int Alt::Base::to_dot(unsigned int *nodeID, std::ostream &out) { out << ""; } +void to_dot_multiys(Yield::Multi m_ys, std::ostream &out) { + out << ""; + for (Yield::Multi::iterator ys = m_ys.begin(); ys != m_ys.end(); ++ys) { + out << ""; + } + out << ""; +} void to_dot_filternameargs(Filter *filter, std::ostream &out) { out << *filter->name; for (std::list::const_iterator arg = filter->args.begin(); @@ -3274,7 +3283,10 @@ unsigned int* Alt::Base::to_dot(unsigned int *nodeID, std::ostream &out, to_dot_indices(this->right_indices, out); } } - out << "
"; std::list::const_iterator filter = (*i).begin(); @@ -3751,7 +3755,8 @@ void Alt::Multi::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { - Alt::Base::init_indices_outside(left, right, k, track, center_left, center_right); + Alt::Base::init_indices_outside(left, right, k, track, center_left, + center_right); size_t j = 0; assert(track < list.size()); std::list::iterator i = list.begin(); diff --git a/src/cpp.cc b/src/cpp.cc index e39ad9754..1617e540c 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2126,17 +2126,19 @@ void Printer::Cpp::print_insideoutside_report_fn( for (size_t track = 0; track < (*nt_inside)->tracks(); ++track) { if ((*nt_inside)->tables()[track].type() == Table::QUADRATIC) { - stream << indent() << "for (unsigned int t_" << track << "_i = t_" << track - << "_left_most; t_" << track << "_i <= t_" << track << "_right_most; ++t_" << track << "_i) {" - << endl; + stream << indent() << "for (unsigned int t_" << track << "_i = t_" + << track << "_left_most; t_" << track << "_i <= t_" << track + << "_right_most; ++t_" << track << "_i) {" << endl; inc_indent(); - stream << indent() << "for (unsigned int t_" << track << "_j = t_" << track << "_i; t_" << track << "_j <= t_" << track + stream << indent() << "for (unsigned int t_" << track << "_j = t_" + << track << "_i; t_" << track << "_j <= t_" << track << "_right_most; ++t_" << track << "_j) {" << endl; inc_indent(); } if ((*nt_inside)->tables()[track].type() == Table::LINEAR) { - stream << indent() << "for (unsigned int t_" << track << "_i = t_" << track - << "_left_most; t_" << track << "_i <= t_" << track << "_right_most; ++t_" << track << "_i) {" + stream << indent() << "for (unsigned int t_" << track << "_i = t_" + << track << "_left_most; t_" << track << "_i <= t_" << track + << "_right_most; ++t_" << track << "_i) {" << endl; inc_indent(); } diff --git a/src/grammar.cc b/src/grammar.cc index 10867f972..42eb74ade 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1155,7 +1155,7 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { f->type = Filter::WITH; std::list *comptracks = new std::list(); for (unsigned int track = 0; track < this->axiom->tracks(); ++track) { - comptracks->push_back(f); + comptracks->push_back(f); } link->set_tracks(this->axiom->tracks(), this->axiom->track_pos()); link->init_multi_ys(); From 8b2d5df44ae86d791add2de476f6b8761e6dbff2 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 14 Apr 2022 14:32:16 +0200 Subject: [PATCH 080/209] init_indices_outside: fn_arg instead of alt --- src/alt.cc | 6 +++--- src/fn_arg.cc | 22 ++++++++++++++++++++++ src/fn_arg.hh | 13 +++++++++++-- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 941069fd0..f2b13bb68 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3675,7 +3675,7 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, } right_index = get_next_var_right2left(left_index, left, k, track, ys_arg, ys_right2center); - (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, track, + (*i)->init_indices_outside(left_index, right_index, k, track, leftmost_index, center_right); left_index = right_index; } @@ -3719,7 +3719,7 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, *ys_left2center += *(sum_ys_lefts(&ys_arg, i, args_right->rend(), track)); left_index = get_next_var_left2right(right_index, right, k, track, ys_arg, ys_left2center); - (*i)->alt_ref()->init_indices_outside(left_index, right_index, k, + (*i)->init_indices_outside(left_index, right_index, k, track, center_left, rightmost_index, true); right_index = left_index; } @@ -3735,7 +3735,7 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, // argument containing outside NT if (arg_outside != NULL) { - arg_outside->alt_ref()->init_indices_outside(left, right, k, track, + arg_outside->init_indices_outside(left, right, k, track, lhs_right_index, right_index); this->expand_outside_nt_indices(leftmost_index, rightmost_index, track); } diff --git a/src/fn_arg.cc b/src/fn_arg.cc index d46cfc542..9178075b6 100644 --- a/src/fn_arg.cc +++ b/src/fn_arg.cc @@ -312,6 +312,28 @@ void Fn_Arg::Const::init_indices(Expr::Base *left, Expr::Base *right, Base::init_indices(left, right, k, track); } +void Fn_Arg::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, + unsigned int &k, size_t track, Expr::Base *center_left, + Expr::Base *center_right, bool is_right_of_outside_nt) { + Fn_Arg::Base::init_indices(left, right, k, track); +} + +void Fn_Arg::Alt::init_indices_outside(Expr::Base *left, Expr::Base *right, + unsigned int &k, size_t track, Expr::Base *center_left, + Expr::Base *center_right, bool is_right_of_outside_nt) { + Base::init_indices_outside(left, right, k, track, center_left, + center_right, is_right_of_outside_nt); + alt->init_indices_outside(left, right, k, track, center_left, + center_right, is_right_of_outside_nt); +} + +void Fn_Arg::Const::init_indices_outside(Expr::Base *left, Expr::Base *right, + unsigned int &k, size_t track, Expr::Base *center_left, + Expr::Base *center_right, bool is_right_of_outside_nt) { + Base::init_indices_outside(left, right, k, track, center_left, + center_right, is_right_of_outside_nt); +} + void Fn_Arg::Base::init_ret_decl(unsigned int i, const std::string &prefix) { ret_decls_.clear(); var_decls_.clear(); diff --git a/src/fn_arg.hh b/src/fn_arg.hh index a34cebf42..76fb4210e 100644 --- a/src/fn_arg.hh +++ b/src/fn_arg.hh @@ -125,7 +125,10 @@ class Base { virtual void traverse(Visitor &v) = 0; virtual void init_indices(Expr::Base *left, Expr::Base *right, - unsigned int &k, size_t track); + unsigned int &k, size_t track); + virtual void init_indices_outside(Expr::Base *left, Expr::Base *right, + unsigned int &k, size_t track, Expr::Base *center_left, + Expr::Base *center_right, bool is_right_of_outside_nt = false); virtual void init_ret_decl(unsigned int i, const std::string &prefix); @@ -180,6 +183,9 @@ class Alt : public Base { void traverse(Visitor &v); void init_indices( Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track); + virtual void init_indices_outside(Expr::Base *left, Expr::Base *right, + unsigned int &k, size_t track, Expr::Base *center_left, + Expr::Base *center_right, bool is_right_of_outside_nt = false); void codegen(AST &ast); std::list &statements(); @@ -230,7 +236,10 @@ class Const : public Base { void traverse(Visitor &v); void init_indices(Expr::Base *left, Expr::Base *right, - unsigned int &k, size_t track); + unsigned int &k, size_t track); + virtual void init_indices_outside(Expr::Base *left, Expr::Base *right, + unsigned int &k, size_t track, Expr::Base *center_left, + Expr::Base *center_right, bool is_right_of_outside_nt = false); void codegen(AST &ast); std::list &statements(); From fb5cd37279e3bae7ebfc67de95d7219236766b5f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 14 Apr 2022 14:32:39 +0200 Subject: [PATCH 081/209] added an overlay filter --- testdata/gapc_filter/singlefold.hh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/testdata/gapc_filter/singlefold.hh b/testdata/gapc_filter/singlefold.hh index 12704b8ea..bc05817ab 100644 --- a/testdata/gapc_filter/singlefold.hh +++ b/testdata/gapc_filter/singlefold.hh @@ -9,4 +9,14 @@ inline bool basepair(const Basic_Sequence &seq, T i, T j) return basepairing(seq, i, j); } +//in the Vienna Package, iloop regions are restricted such that their _combined_ length, i.e. |left region| + |right region| cannot exceed 30 bases. +//our restrictions is usually more relaxed, because each region may have up to 30 bases individually, i.e. 60 bases for both regions in the worst case. +//with iloopSumMax we can enforce the Vienna behaviour +template +inline bool iloopSumMax(int size, const Basic_Sequence &seq, T lb_i, T lb_j, T lr_i, T lr_j, T x_i, T x_j, T rr_i, T rr_j, T rb_i, T rb_j) { + assert(lr_i < lr_j); + assert(rr_i < rr_j); + return ((lr_j-lr_i) + (rr_j-rr_i)) <= unsigned(size); +} + #endif From 0ae4df25594f2d06711de05b8b1614be0c348fff Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 14 Apr 2022 14:32:56 +0200 Subject: [PATCH 082/209] execution of new test --- testdata/regresstest/config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 62ec18b64..dc0206c61 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -450,3 +450,6 @@ GAPC="../../../gapc --outside_grammar weak --outside_grammar struct" check_new_old_eq nodangle.gap unused pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside check_new_old_eq nodangle.gap unused mfe "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside check_new_old_eq nodangle.gap unused count "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside + +# check that fn_arg (with alt) instead of only alt gets initialized +check_new_old_eq nodangle_withoverlay.gap unused ins_pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside_overlay From 69c4ced18863366e739440e4cfffde0e88eb852f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 14 Apr 2022 14:33:11 +0200 Subject: [PATCH 083/209] new testcase --- .../grammar_outside/nodangle_withoverlay.gap | 237 ++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 testdata/grammar_outside/nodangle_withoverlay.gap diff --git a/testdata/grammar_outside/nodangle_withoverlay.gap b/testdata/grammar_outside/nodangle_withoverlay.gap new file mode 100644 index 000000000..c38ad4539 --- /dev/null +++ b/testdata/grammar_outside/nodangle_withoverlay.gap @@ -0,0 +1,237 @@ +import rna +import "singlefold.hh" + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int sadd(Subsequence lb, int x) { + return x + sbase_energy(); + } + int cadd(int x, int y) { + return x + y; + } + int drem(Subsequence lb, int x, Subsequence rb) { + return x + termau_energy(lb, rb); + } + int sr(Subsequence lb, int x, Subsequence rb) { + return x + sr_energy(lb, rb); + } + int hl(Subsequence lb, Subsequence r, Subsequence rb) { + return hl_energy(r); + } + int bl(Subsequence lb, Subsequence lr, int x, Subsequence rb) { + return x + bl_energy(lr, rb); + } + int br(Subsequence lb, int x, Subsequence rr, Subsequence rb) { + return x + br_energy(lb, rr); + } + int il(Subsequence lb, Subsequence lr, int x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } + int ml(Subsequence lb, int x, Subsequence rb) { + return x + ml_energy() + ul_energy() + termau_energy(lb, rb); + } + int incl(int x) { + return x + ul_energy(); + } + int addss(int x, Subsequence r) { + return x + ss_energy(r); + } + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } +} + +algebra alg_pfunc implements sig_foldrna(alphabet = char, answer = double) { + double sadd(Subsequence lb, double x) { + return scale(1) * x * mk_pf(sbase_energy()); + } + double cadd(double x, double y) { + return x * y; + } + double drem(Subsequence lb, double x, Subsequence rb) { + return x * mk_pf(termau_energy(lb, rb)); + } + double sr(Subsequence lb, double x, Subsequence rb) { + return scale(2) * x * mk_pf(sr_energy(lb, rb)); + } + double hl(Subsequence lb, Subsequence r, Subsequence rb) { + return scale(2+r.j-r.i) * mk_pf(hl_energy(r)); + } + double bl(Subsequence lb, Subsequence lr, double x, Subsequence rb) { + return scale(2+lr.j-lr.i) * x * mk_pf(bl_energy(lr, rb)); + } + double br(Subsequence lb, double x, Subsequence rr, Subsequence rb) { + return scale(2+rr.j-rr.i) * x * mk_pf(br_energy(lb, rr)); + } + double il(Subsequence lb, Subsequence lr, double x, Subsequence rr, Subsequence rb) { + return scale(2+lr.j-lr.i+rr.j-rr.i) * x * mk_pf(il_energy(lr, rr)); + } + double ml(Subsequence lb, double x, Subsequence rb) { + return scale(2) * x * mk_pf(ml_energy() + ul_energy() + termau_energy(lb, rb)); + } + double incl(double x) { + return x * mk_pf(ul_energy()); + } + double addss(double x, Subsequence r) { + return scale(r.j-r.i) * x * mk_pf(ss_energy(r)); + } + double nil(Subsequence n) { + return 1; + } + choice [double] h([double] i) { + return list(sum(i)); + } +} + +algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { + string sadd(Subsequence lb,string e) { + string res; + append(res, '.'); + append(res, e); + return res; + } + + string cadd(string le,string re) { + string res; + append(res, le); + append(res, re); + return res; + } + + string nil(Subsequence loc) { + string r; + return r; + } + + string drem(Subsequence lloc, string e, Subsequence rloc) { + return e; + } + + string sr(Subsequence lb,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string hl(Subsequence lb,Subsequence region,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(region)); + append(res, ')'); + return res; + } + + + string bl(Subsequence lb,Subsequence lregion,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, ')'); + return res; + } + + string br(Subsequence lb,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string il(Subsequence lb,Subsequence lregion,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string ml(Subsequence lb, string e, Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string addss(string e,Subsequence rb) { + string res; + append(res, e); + append(res, '.', size(rb)); + return res; + } + + string incl(string e) { + return e; + } + + choice [string] h([string] i) { + return i; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = sadd(BASE, struct) | + cadd(dangle , struct) | + nil(LOC) # h; + + dangle = drem(LOC, strong, LOC) # h; + + strong = weak # h; + + weak = stack | + hairpin | + leftB | + rightB | + iloop | + multiloop # h; + + stack = sr(BASE, weak, BASE) with basepairing # h; + hairpin = hl(BASE, REGION with minsize(3), BASE) with basepairing # h; + leftB = bl(BASE, REGION with maxsize(30), strong, BASE) with basepairing # h; + rightB = br(BASE, strong, REGION with maxsize(30), BASE) with basepairing # h; + iloop = il(BASE, REGION with maxsize(30), strong, REGION with maxsize(30), BASE) with basepairing with_overlay iloopSumMax(30) # h; + multiloop = ml(BASE, ml_comps, BASE) with basepairing # h; + + ml_comps = sadd(BASE, ml_comps) | + cadd(incl(dangle), ml_comps1) # h; + + ml_comps1 = sadd(BASE, ml_comps1) | + cadd(incl(dangle), ml_comps1) | + incl(dangle) | + addss(incl(dangle), REGION) # h; +} + +instance ins_mfe = gra_nodangle(alg_mfe); +instance ins_pfunc = gra_nodangle(alg_pfunc); +instance ins_count = gra_nodangle(alg_count); +instance ins_enum = gra_nodangle(alg_enum); From bd2a06e25b9c2ab5f43b922af505e542acd832e0 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 18 Apr 2022 23:44:48 +0200 Subject: [PATCH 084/209] special indices for CYK outside tables --- src/cpp.cc | 29 ++++++++++++++++++++++------- src/cpp.hh | 2 +- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 1617e540c..2ce3bd2b7 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -1733,7 +1733,8 @@ void Printer::Cpp::print_openmp_cyk_nt_calls(const AST &ast) { assert(!(*i)->tables().empty()); if ((*i)->is_tabulated() && !(*i)->tables().front().is_cyk_right()) { o1 << indent() << "nt_tabulate_" << *(*i)->name << "(" - << multi_index_str((*i)->tables(), (*i)->multi_ys()) + << multi_index_str((*i)->tables(), (*i)->multi_ys(), + (*i)->is_partof_outside) << ");\n"; } } @@ -1748,7 +1749,8 @@ void Printer::Cpp::print_openmp_cyk_all_nt_calls(const AST &ast) { i != tord.end(); ++i) { if ((*i)->is_tabulated()) { o2 << indent() << "nt_tabulate_" << *(*i)->name << "(" - << multi_index_str((*i)->tables(), (*i)->multi_ys()) + << multi_index_str((*i)->tables(), (*i)->multi_ys(), + (*i)->is_partof_outside) << ");\n"; } } @@ -1860,7 +1862,8 @@ void Printer::Cpp::print_openmp_cyk(const AST &ast) { std::string Printer::Cpp::multi_index_str( - const std::vector &tables, const Yield::Multi &mys) { + const std::vector
&tables, const Yield::Multi &mys, + bool is_outside) { assert(tables.size() == mys.tracks()); std::ostringstream o; size_t track = 0; @@ -1868,10 +1871,18 @@ std::string Printer::Cpp::multi_index_str( for (std::vector
::const_iterator i = tables.begin(); i != tables.end(); ++i, ++j, ++track) { if (!(*i).delete_left_index()) { - o << ", t_" << track << "_i-1"; + if (is_outside == false) { + o << ", t_" << track << "_i-1"; + } else { + o << ", (t_" << track << "_j - t_" << track << "_i + 1)"; + } } if (!(*i).delete_right_index()) { - o << ", t_" << track << "_j"; + if (is_outside == false) { + o << ", t_" << track << "_j"; + } else { + o << ", (t_" << track << "_n - t_" << track << "_i + 1)"; + } } } std::string r(o.str()); @@ -1894,7 +1905,8 @@ void Printer::Cpp::multi_print_inner_cyk( } for (std::list::const_iterator i = l.begin(); i != l.end(); ++i) { - std::string index_str = multi_index_str((*i)->tables(), (*i)->multi_ys()); + std::string index_str = multi_index_str((*i)->tables(), (*i)->multi_ys(), + (*i)->is_partof_outside); stream << indent() << "nt_tabulate_" << *(*i)->name << '(' << index_str << ");" << endl; } @@ -2057,6 +2069,8 @@ void Printer::Cpp::print_insideoutside( Symbol::NT *nt) { std::ostringstream args; std::ostringstream args_print; + std::list &l = nt->code_list(); + for (size_t track = 0; track < (*nt).tracks(); ++track) { if ((*nt).tables()[track].type() == Table::QUADRATIC) { args << "t_" << track << "_i,t_" << track << "_j"; @@ -2073,7 +2087,6 @@ void Printer::Cpp::print_insideoutside( } stream << indent() << "out << \"start answers " << *nt->name << "(\"" << args_print.str() << " << \"):\\n\";\n"; - std::list &l = nt->code_list(); for (std::list::iterator i = l.begin(); i != l.end(); ++i) { std::string res = "res_"; if (nt->is_partof_outside) { @@ -2084,6 +2097,8 @@ void Printer::Cpp::print_insideoutside( stream << indent() << *((*i)->return_type) << " " << res << " = nt_" << *nt->name << "(" << args.str() << ");\n"; stream << indent() << "print_result(std::cout, " << res << ");\n"; + // only for first return statement + break; } stream << indent() << "out << \"//end answers " << *nt->name << "(\"" << args_print.str() << " << \")\\n\";\n"; diff --git a/src/cpp.hh b/src/cpp.hh index 8e490f521..425e3b9ca 100644 --- a/src/cpp.hh +++ b/src/cpp.hh @@ -82,7 +82,7 @@ class Cpp : public Base { void print_openmp_cyk(const AST &ast); std::string multi_index_str(const std::vector
&tables, - const Yield::Multi &mys); + const Yield::Multi &mys, bool is_outside = false); void multi_print_inner_cyk(const std::list &l, const std::list &tord, size_t track, size_t tracks, size_t track_pos, From bf63de6fd968101547f143deae3fa7bf5427b961 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 28 Apr 2022 20:58:56 +0200 Subject: [PATCH 085/209] cyk for outside --- src/cpp.cc | 177 ++++++++++++++++++++++++++++++++++++++--------------- src/cpp.hh | 33 +++++++++- 2 files changed, 160 insertions(+), 50 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 2ce3bd2b7..17672a143 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -1897,7 +1897,8 @@ std::string Printer::Cpp::multi_index_str( void Printer::Cpp::multi_print_inner_cyk( const std::list &l, const std::list &tord, - size_t track, size_t tracks, size_t track_pos, Type::Base *t) { + size_t track, size_t tracks, size_t track_pos, Type::Base *t, + bool for_outsideNTs) { assert(track < tracks); if (track+1 != tracks) { multi_print_cyk(tord, track+1, tracks, track_pos, t); @@ -1905,19 +1906,21 @@ void Printer::Cpp::multi_print_inner_cyk( } for (std::list::const_iterator i = l.begin(); i != l.end(); ++i) { - std::string index_str = multi_index_str((*i)->tables(), (*i)->multi_ys(), - (*i)->is_partof_outside); - stream << indent() << "nt_tabulate_" << *(*i)->name << '(' - << index_str << ");" << endl; + if ((*i)->is_partof_outside == for_outsideNTs) { + std::string index_str = multi_index_str((*i)->tables(), (*i)->multi_ys(), + (*i)->is_partof_outside); + stream << indent() << "nt_tabulate_" << *(*i)->name << '(' + << index_str << ");" << endl; + } } } void Printer::Cpp::multi_partition_nts( - const std::list &tord, std::list &all, - std::list &inner, std::list &left, - std::list &right, - size_t track, size_t tracks, size_t track_pos) { + const std::list &tord, std::list &all, + std::list &inner, std::list &left, + std::list &right, + size_t track, size_t tracks, size_t track_pos) { for (std::list::const_iterator i = tord.begin(); i != tord.end(); ++i) { if (!(*i)->is_tabulated()) { @@ -1946,65 +1949,141 @@ void Printer::Cpp::multi_partition_nts( } -void Printer::Cpp::multi_print_cyk( - const std::list &tord, size_t track, size_t tracks, - size_t track_pos, Type::Base *t) { - std::list all, inner, left, right; - multi_partition_nts(tord, all, inner, left, right, track, tracks, track_pos); - - size_t real_track = !track_pos ? track : track_pos; - - std::ostringstream sis, sjs, sns; - sis << "t_" << track << "_i"; - sjs << "t_" << track << "_j"; - sns << "t_" << real_track << "_n"; - std::string is(sis.str()), js(sjs.str()), ns(sns.str()); - - stream << indent() << *t << " t_" << track << "_n = t_" << real_track - << "_seq.size();" << endl << endl; - - if (!inner.empty()) { - stream << indent() << "for (" << *t << " " << js << " = 0; " << js << " < " - << ns << "; " << "++" << js << ") {" << endl; +void Printer::Cpp::multi_print_cyk_loops_quadratic( + const std::list &tord, + size_t track, + size_t tracks, + size_t track_pos, + Type::Base *t, + std::list *inner, + std::list *left, + std::string *is, + std::string *js, + std::string *ns, bool for_outsideNTs) { + if (!inner->empty()) { + stream << indent() << "for (" << *t << " " << *js << " = 0; " << *js + << " < " << *ns << "; " << "++" << *js << ") {" << endl; inc_indent(); - stream << indent() << "for (" << *t << " " << is << " = " << js << " + 1; " - << is << " > 1; " << is << "--) {" << endl; + stream << indent() << "for (" << *t << " " << *is << " = " << *js + << " + 1; " << *is << " > 1; " << *is << "--) {" << endl; inc_indent(); - multi_print_inner_cyk(inner, tord, track, tracks, track_pos, t); + multi_print_inner_cyk(*inner, tord, track, tracks, track_pos, t, + for_outsideNTs); dec_indent(); stream << indent() << "}" << endl << endl; - if (!left.empty()) { - stream << indent() << *t << " " << is << " = 1;" << endl; - multi_print_inner_cyk(left, tord, track, tracks, track_pos, t); + if (!left->empty()) { + stream << indent(); + if (!for_outsideNTs) { + stream << *t << " "; + } + stream << *is << " = 1;" << endl; + multi_print_inner_cyk(*left, tord, track, tracks, track_pos, t, + for_outsideNTs); } dec_indent(); stream << indent() << "}" << endl << endl; } - if (inner.empty() && !left.empty()) { - stream << indent() << "for (" << *t << " " << js << " = 0; " - << js << " < " << ns - << "; " << "++" << js << ") {" << endl; +} +void Printer::Cpp::multi_print_cyk_loops_linear( + const std::list &tord, + size_t track, + size_t tracks, + size_t track_pos, + Type::Base *t, + std::list *inner, + std::list *left, + std::list *right, + std::string *is, + std::string *js, + std::string *ns, bool for_outsideNTs + ) { + if (inner->empty() && !left->empty()) { + stream << indent() << "for (" << *t << " " << *js << " = 0; " + << *js << " < " << *ns + << "; " << "++" << *js << ") {" << endl; inc_indent(); - stream << indent() << *t << " " << is << " = 1;" << endl; - multi_print_inner_cyk(left, tord, track, tracks, track_pos, t); + stream << indent(); + if (!for_outsideNTs) { + stream << *t << " "; + } + stream << *is << " = 1;" << endl; + multi_print_inner_cyk(*left, tord, track, tracks, track_pos, t, + for_outsideNTs); dec_indent(); stream << indent() << "}" << endl << endl; } - if (!right.empty()) { - stream << indent() << *t << " " << js << " = " << ns << ";" << endl; - stream << indent() << "for (" << *t << " "<< is << " = " << js << " + 1; " - << is << " > 1; " << is << "--) {" << endl; + if (!right->empty()) { + stream << indent(); + if (!for_outsideNTs) { + stream << *t << " "; + } + stream << *js << " = " << *ns << ";" << endl; + stream << indent() << "for (" << *t << " "<< *is << " = " << *js + << " + 1; " << *is << " > 1; " << *is << "--) {" << endl; inc_indent(); - multi_print_inner_cyk(right, tord, track, tracks, track_pos, t); + multi_print_inner_cyk(*right, tord, track, tracks, track_pos, t, + for_outsideNTs); dec_indent(); stream << indent() << "}" << endl << endl; } +} +void Printer::Cpp::multi_print_cyk_loops_constant( + const std::list &tord, + size_t track, + size_t tracks, + size_t track_pos, + Type::Base *t, + std::list *all, + std::string *is, bool for_outsideNTs + ) { + if (!all->empty()) { + stream << indent(); + if (!for_outsideNTs) { + stream << *t << " "; + } + stream << *is << " = 1;" << endl; + multi_print_inner_cyk(*all, tord, track, tracks, track_pos, t, + for_outsideNTs); + } +} +void Printer::Cpp::multi_print_cyk( + const std::list &tord, size_t track, size_t tracks, + size_t track_pos, Type::Base *t) { + std::list all, inner, left, right; + multi_partition_nts(tord, all, inner, left, right, track, tracks, track_pos); + + size_t real_track = !track_pos ? track : track_pos; + + std::ostringstream sis, sjs, sns; + sis << "t_" << track << "_i"; + sjs << "t_" << track << "_j"; + sns << "t_" << real_track << "_n"; + std::string is(sis.str()), js(sjs.str()), ns(sns.str()); + + stream << indent() << *t << " t_" << track << "_n = t_" << real_track + << "_seq.size();" << endl << endl; - if (!all.empty()) { - stream << indent() << *t << " " << is << " = 1;" << endl; - multi_print_inner_cyk(all, tord, track, tracks, track_pos, t); + // inside NTs + multi_print_cyk_loops_quadratic( + tord, track, tracks, track_pos, t, &inner, &left, &is, &js, &ns, false); + multi_print_cyk_loops_linear( + tord, track, tracks, track_pos, t, &inner, &left, &right, &is, &js, &ns, + false); + multi_print_cyk_loops_constant( + tord, track, tracks, track_pos, t, &all, &is, false); + + // outside NTs (which access inside NT values) + if (this->ast->grammar()->is_outside()) { + stream << endl << endl; + multi_print_cyk_loops_linear( + tord, track, tracks, track_pos, t, &inner, &left, &right, &is, &js, &ns, + true); + multi_print_cyk_loops_constant( + tord, track, tracks, track_pos, t, &all, &is, true); + multi_print_cyk_loops_quadratic( + tord, track, tracks, track_pos, t, &inner, &left, &is, &js, &ns, true); } } diff --git a/src/cpp.hh b/src/cpp.hh index 425e3b9ca..de2f8bd10 100644 --- a/src/cpp.hh +++ b/src/cpp.hh @@ -86,11 +86,42 @@ class Cpp : public Base { void multi_print_inner_cyk(const std::list &l, const std::list &tord, size_t track, size_t tracks, size_t track_pos, - Type::Base *t); + Type::Base *t, bool for_outsideNTs = false); void multi_partition_nts(const std::list &tord, std::list &all, std::list &inner, std::list &left, std::list &right, size_t track, size_t tracks, size_t track_pos); + void multi_print_cyk_loops_quadratic( + const std::list &tord, + size_t track, + size_t tracks, + size_t track_pos, + Type::Base *t, + std::list *inner, + std::list *left, + std::string *is, + std::string *js, + std::string *ns, bool for_outsideNTs = false); + void multi_print_cyk_loops_linear( + const std::list &tord, + size_t track, + size_t tracks, + size_t track_pos, + Type::Base *t, + std::list *inner, + std::list *left, + std::list *right, + std::string *is, + std::string *js, + std::string *ns, bool for_outsideNTs = false); + void multi_print_cyk_loops_constant( + const std::list &tord, + size_t track, + size_t tracks, + size_t track_pos, + Type::Base *t, + std::list *all, + std::string *is, bool for_outsideNTs = false); void multi_print_cyk(const std::list &tord, size_t track, size_t tracks, size_t track_pos, Type::Base *t); From a0a0272d3294945fec207695e5dbfc374a3d3ced Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 28 Apr 2022 22:35:14 +0200 Subject: [PATCH 086/209] adding tests for CYK and openMP --- testdata/regresstest/config | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index dc0206c61..f7d30d917 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -453,3 +453,15 @@ check_new_old_eq nodangle.gap unused count "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outsi # check that fn_arg (with alt) instead of only alt gets initialized check_new_old_eq nodangle_withoverlay.gap unused ins_pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside_overlay + +# same as above, but with CYK code generation +GAPC="../../../gapc --outside_grammar weak --outside_grammar struct --cyk" +check_new_old_eq nodangle.gap unused pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside + +# same as above, but with CYK code generation AND openMP parallelization +GAPC="../../../gapc --outside_grammar weak --outside_grammar struct --cyk" +CPPFLAGS_EXTRA="$DEFAULT_CPPFLAGS_EXTRA -fopenmp" +LDLIBS_EXTRA="$DEFAULT_LDLIBS_EXTRA -fopenmp" +OMP_NUM_THREADS=2 +export OMP_NUM_THREADS +check_new_old_eq nodangle.gap unused pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside From 45328590a704d5329beedc0cb5cfe8a16ce2ac9e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 28 Apr 2022 22:38:19 +0200 Subject: [PATCH 087/209] openMP for outside NTs --- src/cpp.cc | 141 +++++++++++++++++++++++++++++++++++------------------ src/cpp.hh | 13 +++-- 2 files changed, 104 insertions(+), 50 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 17672a143..5fe488c40 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -1724,65 +1724,64 @@ void Printer::Cpp::header(const AST &ast) { } -void Printer::Cpp::print_openmp_cyk_nt_calls(const AST &ast) { +void Printer::Cpp::print_openmp_cyk_nt_calls(const AST &ast, + bool for_outsideNTs) { std::list &tord = ast.grammar()->topological_ord(); assert(tord.size() <= ast.grammar()->NTs.size()); std::ostringstream o1; for (std::list::iterator i = tord.begin(); i != tord.end(); ++i) { - assert(!(*i)->tables().empty()); - if ((*i)->is_tabulated() && !(*i)->tables().front().is_cyk_right()) { - o1 << indent() << "nt_tabulate_" << *(*i)->name << "(" - << multi_index_str((*i)->tables(), (*i)->multi_ys(), - (*i)->is_partof_outside) - << ");\n"; + if ((*i)->is_partof_outside == for_outsideNTs) { + assert(!(*i)->tables().empty()); + if ((*i)->is_tabulated() && !(*i)->tables().front().is_cyk_right()) { + o1 << indent() << "nt_tabulate_" << *(*i)->name << "(" + << multi_index_str((*i)->tables(), (*i)->multi_ys(), + (*i)->is_partof_outside) + << ");\n"; + } } } std::string nt_calls = o1.str(); stream << nt_calls; } -void Printer::Cpp::print_openmp_cyk_all_nt_calls(const AST &ast) { +void Printer::Cpp::print_openmp_cyk_all_nt_calls(const AST &ast, + bool for_outsideNTs) { std::list &tord = ast.grammar()->topological_ord(); std::ostringstream o2; for (std::list::iterator i = tord.begin(); i != tord.end(); ++i) { - if ((*i)->is_tabulated()) { - o2 << indent() << "nt_tabulate_" << *(*i)->name << "(" - << multi_index_str((*i)->tables(), (*i)->multi_ys(), - (*i)->is_partof_outside) - << ");\n"; + if ((*i)->is_partof_outside == for_outsideNTs) { + if ((*i)->is_tabulated()) { + o2 << indent() << "nt_tabulate_" << *(*i)->name << "(" + << multi_index_str((*i)->tables(), (*i)->multi_ys(), + (*i)->is_partof_outside) + << ");\n"; + } } } std::string all_nt_calls = o2.str(); stream << all_nt_calls; } -void Printer::Cpp::print_openmp_cyk_helpervars() { - stream << indent() << "unsigned int n = t_0_seq.size();" << endl; - stream << indent() << "unsigned int tile_size = 32;" << endl; - stream << "#ifdef TILE_SIZE" << endl; - stream << indent() << "tile_size = TILE_SIZE;" << endl; - stream << "#endif" << endl; - stream << indent() << "assert(tile_size);" << endl; - stream << indent() << "unsigned int max_tiles = n / tile_size;" << endl; - stream << indent() << "int max_tiles_n = max_tiles * tile_size;" << endl; +void Printer::Cpp::print_openmp_cyk_helpervars(bool for_outsideNTs) { + stream << indent() << "unsigned int "; + if (for_outsideNTs) { + stream << "t_0_"; + } + stream << "n = t_0_seq.size();" << endl; + if (!for_outsideNTs) { + stream << indent() << "unsigned int tile_size = 32;" << endl; + stream << "#ifdef TILE_SIZE" << endl; + stream << indent() << "tile_size = TILE_SIZE;" << endl; + stream << "#endif" << endl; + stream << indent() << "assert(tile_size);" << endl; + stream << indent() << "unsigned int max_tiles = n / tile_size;" << endl; + stream << indent() << "int max_tiles_n = max_tiles * tile_size;" << endl; + } } - -// FIXME adjust for multi-track (> 1 track) -void Printer::Cpp::print_openmp_cyk(const AST &ast) { - // FIXME abstract from unsigned int, int -> perhaps wait for OpenMP 3 - // since OpenMP < 3 doesn't allow unsigned int in workshared fors - - // paral_start - stream << indent() << "#pragma omp parallel" << endl; - stream << indent() << '{' << endl; - inc_indent(); - - // helper_vars - print_openmp_cyk_helpervars(); - - // diag +void Printer::Cpp::print_openmp_cyk_loops_diag(const AST &ast, + bool for_outsideNTs) { stream << indent() << "#pragma omp for" << endl; stream << indent() << "// OPENMP < 3 requires signed int here ..." << endl; stream << indent(); @@ -1795,15 +1794,16 @@ void Printer::Cpp::print_openmp_cyk(const AST &ast) { stream << indent() << "for (int t_0_i = t_0_j + 1; t_0_i > z; --t_0_i) {"; stream << endl; inc_indent(); - print_openmp_cyk_nt_calls(ast); + print_openmp_cyk_nt_calls(ast, for_outsideNTs); dec_indent(); stream << indent() << "}" << endl; dec_indent(); stream << indent() << "}" << endl; dec_indent(); stream << indent() << "}" << endl; - - // middle +} +void Printer::Cpp::print_openmp_cyk_loops_middle(const AST &ast, + bool for_outsideNTs) { stream << endl; stream << indent() << "for (int z = tile_size; z < max_tiles_n; z+=tile_size) {" @@ -1824,7 +1824,7 @@ void Printer::Cpp::print_openmp_cyk(const AST &ast) { << endl; inc_indent(); - print_openmp_cyk_nt_calls(ast); + print_openmp_cyk_nt_calls(ast, for_outsideNTs); // paral_end dec_indent(); @@ -1838,11 +1838,9 @@ void Printer::Cpp::print_openmp_cyk(const AST &ast) { dec_indent(); stream << indent() << "} // end parallel" << endl; stream << endl; - - // REPEAT: helper_vars - print_openmp_cyk_helpervars(); - - // single +} +void Printer::Cpp::print_openmp_cyk_loops_single(const AST &ast, + bool for_outsideNTs) { stream << indent() << "for (unsigned int t_0_j = max_tiles_n; t_0_j <= n; ++t_0_j) {" << endl; @@ -1851,7 +1849,7 @@ void Printer::Cpp::print_openmp_cyk(const AST &ast) { << "for (unsigned int t_0_i=t_0_j+1; t_0_i > 0; --t_0_i) {" << endl; inc_indent(); - print_openmp_cyk_all_nt_calls(ast); + print_openmp_cyk_all_nt_calls(ast, for_outsideNTs); // single_end dec_indent(); @@ -1859,6 +1857,55 @@ void Printer::Cpp::print_openmp_cyk(const AST &ast) { dec_indent(); stream << indent() << "}" << endl; } +// FIXME adjust for multi-track (> 1 track) +void Printer::Cpp::print_openmp_cyk(const AST &ast) { + // FIXME abstract from unsigned int, int -> perhaps wait for OpenMP 3 + // since OpenMP < 3 doesn't allow unsigned int in workshared fors + + // paral_start + stream << indent() << "#pragma omp parallel" << endl; + stream << indent() << '{' << endl; + inc_indent(); + + // helper_vars + print_openmp_cyk_helpervars(); + + // diag + print_openmp_cyk_loops_diag(ast); + + // middle + print_openmp_cyk_loops_middle(ast); + + // REPEAT: helper_vars + print_openmp_cyk_helpervars(); + + // single + print_openmp_cyk_loops_single(ast); + + // same as above but for outside non terminals + if (ast.grammar()->is_outside()) { + // paral_start + stream << endl << indent() << "// repeat for outside non-terminals" << endl; + stream << indent() << "#pragma omp parallel" << endl; + stream << indent() << '{' << endl; + inc_indent(); + + // helper_vars + print_openmp_cyk_helpervars(true); + + // diag + print_openmp_cyk_loops_diag(ast, true); + + // middle + print_openmp_cyk_loops_middle(ast, true); + + // REPEAT: helper_vars + print_openmp_cyk_helpervars(true); + + // single + print_openmp_cyk_loops_single(ast, true); + } +} std::string Printer::Cpp::multi_index_str( diff --git a/src/cpp.hh b/src/cpp.hh index de2f8bd10..824bc0b2b 100644 --- a/src/cpp.hh +++ b/src/cpp.hh @@ -76,9 +76,16 @@ class Cpp : public Base { void print_window_inc_fn(const AST &ast); - void print_openmp_cyk_nt_calls(const AST &ast); - void print_openmp_cyk_all_nt_calls(const AST &ast); - void print_openmp_cyk_helpervars(); + void print_openmp_cyk_nt_calls(const AST &ast, bool for_outsideNTs = false); + void print_openmp_cyk_all_nt_calls(const AST &ast, + bool for_outsideNTs = false); + void print_openmp_cyk_helpervars(bool for_outsideNTs = false); + void print_openmp_cyk_loops_diag(const AST &ast, + bool for_outsideNTs = false); + void print_openmp_cyk_loops_middle(const AST &ast, + bool for_outsideNTs = false); + void print_openmp_cyk_loops_single(const AST &ast, + bool for_outsideNTs = false); void print_openmp_cyk(const AST &ast); std::string multi_index_str(const std::vector
&tables, From fb725417afdd73e4270000b23bf75543fb5c6ba7 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 11 May 2022 16:44:01 +0200 Subject: [PATCH 088/209] two new functions to identify Alt::Block use and find parent elements using these Alt::Blocks --- src/alt.cc | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/alt.hh | 15 +++++++++++ 2 files changed, 89 insertions(+) diff --git a/src/alt.cc b/src/alt.cc index 410773c5f..dc4054fd9 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3923,3 +3923,77 @@ Yield::Size *Alt::Simple::sum_ys_rights( return res; } + +// find_block can traverse a (top-level) alternative and returns +// the first occurrence of a Alt::Block instance OR NULL if this +// alternative does not contain any Alt::Block instances. +Alt::Base *Alt::Base::find_block() { + throw LogError("Should not be called directly!"); + return NULL; +} +Alt::Base *Alt::Simple::find_block() { + for (std::list::iterator i = this->args.begin(); + i != this->args.end(); ++i) { + Alt::Block *block = dynamic_cast((*i)->alt_ref()); + if (block) { + return (*i)->alt_ref(); + } + } + return NULL; +} +Alt::Base *Alt::Link::find_block() { + // a Link is a leaf and thus cannot have a Block as a child + return NULL; +} +Alt::Base *Alt::Block::find_block() { + return this; +} +Alt::Base *Alt::Multi::find_block() { + for (std::list::iterator i = this->list.begin(); + i != this->list.end(); ++i) { + Alt::Block *block = dynamic_cast(*i); + if (block) { + return *i; + } + } + return NULL; +} + +Alt::Base *Alt::Base::find_block_parent(const Alt::Base &block) { + throw LogError("Should not be called directly!"); + return NULL; +} +Alt::Base *Alt::Simple::find_block_parent(const Alt::Base &block) { + for (std::list::iterator i = this->args.begin(); + i != this->args.end(); ++i) { + Alt::Block *child_block = dynamic_cast((*i)->alt_ref()); + if (child_block && (child_block == &block)) { + return this; + } + } + return NULL; +} +Alt::Base *Alt::Link::find_block_parent(const Alt::Base &block) { + // a Link is a leaf and thus cannot have a Block as a child + return NULL; +} +Alt::Base *Alt::Block::find_block_parent(const Alt::Base &block) { + for (std::list::iterator i = this->alts.begin(); + i != this->alts.end(); ++i) { + Alt::Block *child_block = dynamic_cast(*i); + if (child_block && (child_block == &block)) { + return this; + } + } + return NULL; +} +Alt::Base *Alt::Multi::find_block_parent(const Alt::Base &block) { + for (std::list::iterator i = this->list.begin(); + i != this->list.end(); ++i) { + Alt::Block *child_block = dynamic_cast(*i); + if (child_block && (child_block == &block)) { + return this; + } + } + return NULL; +} diff --git a/src/alt.hh b/src/alt.hh index f0ddf1484..d2c9d50fc 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -362,6 +362,13 @@ class Base { bool get_is_partof_outside() { return this->is_partof_outside; } + // traverses production rule and returns pointer to topmost Alt::Block IFF + // production rule contains an Alt::Block, otherwise NULL + virtual Alt::Base* find_block(); + // given a Alt::Block reference, returns the Alt::Base reference of the + // parent with child Alt::Block (necessary to change pointer when resolving + // blocks) + virtual Alt::Base *find_block_parent(const Alt::Base &block); }; @@ -572,6 +579,8 @@ class Simple : public Base { bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); + Alt::Base* find_block(); + Alt::Base *find_block_parent(const Alt::Base &block); private: std::list *insert_index_stmts( @@ -710,6 +719,8 @@ class Link : public Base { hashtable &skip_occurences); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); void init_outside_guards(); + Alt::Base* find_block(); + Alt::Base *find_block_parent(const Alt::Base &block); }; @@ -791,6 +802,8 @@ class Block : public Base { bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); + Alt::Base* find_block(); + Alt::Base *find_block_parent(const Alt::Base &block); }; @@ -876,6 +889,8 @@ class Multi : public Base { bool replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, hashtable &skip_occurences); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); + Alt::Base* find_block(); + Alt::Base *find_block_parent(const Alt::Base &block); }; } // namespace Alt From e0ee9262c423d574a632d2cde229fdc435485152 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 11 May 2022 16:44:39 +0200 Subject: [PATCH 089/209] more test cases for grammars with Alt::Block --- .../unblock_multialt_parentNT.gap | 39 +++++++++++++++++++ .../unblock_multialt_parentSimple.gap | 39 +++++++++++++++++++ .../unblock_onealt_parentNT.gap | 39 +++++++++++++++++++ .../unblock_onealt_parentSimple.gap | 39 +++++++++++++++++++ 4 files changed, 156 insertions(+) create mode 100644 testdata/grammar_outside/unblock_multialt_parentNT.gap create mode 100644 testdata/grammar_outside/unblock_multialt_parentSimple.gap create mode 100644 testdata/grammar_outside/unblock_onealt_parentNT.gap create mode 100644 testdata/grammar_outside/unblock_onealt_parentSimple.gap diff --git a/testdata/grammar_outside/unblock_multialt_parentNT.gap b/testdata/grammar_outside/unblock_multialt_parentNT.gap new file mode 100644 index 000000000..92f51d29f --- /dev/null +++ b/testdata/grammar_outside/unblock_multialt_parentNT.gap @@ -0,0 +1,39 @@ +import rna +import "singlefold.hh" + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = {sadd(BASE, struct) | sadd(BASE, hairpin) | weak} + | nil(LOC) + # h; + + dangle = drem(LOC, weak, LOC) # h; + + weak = {hairpin} # h; + + hairpin = hl(BASE, REGION with minsize(3), BASE) with basepairing # h; +} + + +instance ins_count = gra_nodangle(alg_count); diff --git a/testdata/grammar_outside/unblock_multialt_parentSimple.gap b/testdata/grammar_outside/unblock_multialt_parentSimple.gap new file mode 100644 index 000000000..3d57191c0 --- /dev/null +++ b/testdata/grammar_outside/unblock_multialt_parentSimple.gap @@ -0,0 +1,39 @@ +import rna +import "singlefold.hh" + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = sadd(BASE, {struct | weak | incl(dangle)}) + | nil(LOC) + # h; + + dangle = drem(LOC, weak, LOC) # h; + + weak = hairpin # h; + + hairpin = hl(BASE, REGION with minsize(3), BASE) with basepairing # h; +} + + +instance ins_count = gra_nodangle(alg_count); diff --git a/testdata/grammar_outside/unblock_onealt_parentNT.gap b/testdata/grammar_outside/unblock_onealt_parentNT.gap new file mode 100644 index 000000000..caff68a6b --- /dev/null +++ b/testdata/grammar_outside/unblock_onealt_parentNT.gap @@ -0,0 +1,39 @@ +import rna +import "singlefold.hh" + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = {sadd(BASE, struct)} + | nil(LOC) + # h; + + dangle = drem(LOC, weak, LOC) # h; + + weak = {hairpin} # h; + + hairpin = hl(BASE, REGION with minsize(3), BASE) with basepairing # h; +} + + +instance ins_count = gra_nodangle(alg_count); diff --git a/testdata/grammar_outside/unblock_onealt_parentSimple.gap b/testdata/grammar_outside/unblock_onealt_parentSimple.gap new file mode 100644 index 000000000..9711b86da --- /dev/null +++ b/testdata/grammar_outside/unblock_onealt_parentSimple.gap @@ -0,0 +1,39 @@ +import rna +import "singlefold.hh" + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = sadd(BASE, {struct}) + | nil(LOC) + # h; + + dangle = drem(LOC, weak, LOC) # h; + + weak = {hairpin} # h; + + hairpin = hl(BASE, REGION with minsize(3), BASE) with basepairing # h; +} + + +instance ins_count = gra_nodangle(alg_count); From 1b285082cf104c702df99d9da468f8c3f834d61c Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 11 May 2022 16:45:41 +0200 Subject: [PATCH 090/209] resolve blocks before outside generation --- Makefile | 4 +- src/grammar.cc | 107 +++++++++++++++++++++++ src/grammar.hh | 5 ++ testdata/modtest/multi_resolve_blocks.cc | 72 +++++++++++++++ 4 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 testdata/modtest/multi_resolve_blocks.cc diff --git a/Makefile b/Makefile index 34ed23e94..53d756a87 100644 --- a/Makefile +++ b/Makefile @@ -128,7 +128,7 @@ gapc: src/gapc.o $(MAIN_OBJ) src/version.o src/prefix.o stats \ test_rt_tab \ -multi_cyk multi_deps multi_indices multi_list_size multi_eliminate_lists_more multi_eliminate_lists multi_algebra multi_signature multi_rt_approx multi_rt_all multi_in_out multi_self_rec multi_calls multi_table_dim multi_max_filter multi_loops multi_ys multi_outside_indices parser: src/version.o src/prefix.o +multi_cyk multi_deps multi_indices multi_list_size multi_eliminate_lists_more multi_eliminate_lists multi_algebra multi_signature multi_rt_approx multi_rt_all multi_in_out multi_self_rec multi_calls multi_table_dim multi_max_filter multi_loops multi_ys multi_outside_indices multi_resolve_blocks parser: src/version.o src/prefix.o backtrack: src/version.o @@ -307,7 +307,7 @@ test-mod: modtests cd testdata/modtest &&\ $(SHELL) run.sh $(TRUTH_DIR)/Mod$(TRUTH_SUFFIX) $(MODTESTS) -test-mod_outside: multi_outside_indices +test-mod_outside: multi_outside_indices multi_resolve_blocks cd testdata/modtest &&\ $(SHELL) run_outside.sh $(TRUTH_DIR)/Mod$(TRUTH_SUFFIX) $? diff --git a/src/grammar.cc b/src/grammar.cc index 42eb74ade..183dbfe02 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1002,6 +1002,109 @@ void Grammar::remove(Symbol::NT *x) { tabulated.erase(nt); } +// a user provided grammar might contain production rules that use blocks +// e.g. NT = sadd(BASE, {hairpin | iloop}) +// this basically is a short hand notation for multiple alternatives, e.g. +// NT = sadd(BASE, hairpin) | sadd(BASE, iloop) +// For outside grammar generation, we first want to make short-hand notations +// explicit by applying this function to the selected grammar. +void Grammar::resolve_blocks() { + for (hashtable::iterator i = NTs.begin(); + i != NTs.end(); ++i) { + Symbol::NT *nt = dynamic_cast(i->second); + if (nt) { + for (std::list::iterator alt = nt->alts.begin(); + alt != nt->alts.end(); ++alt) { + // check if alternative contains any Alt::Block + Alt::Block *block = dynamic_cast((*alt)->find_block()); + while (block) { + // locate parent of found Alt::Block + Alt::Base *parent = (*alt)->find_block_parent(*block); + if (block->alts.size() == 1) { + // if a Alt::Block holds only one alternative, + // we can simply remove Alt::Block altogether + if (parent == NULL) { + // parent is the non-terminal + *alt = block->alts.front(); + } else { + Alt::Simple *p_simple = dynamic_cast(parent); + if (p_simple) { + // if parent is of type Alt::Simple + for (std::list::iterator j = + p_simple->args.begin(); + j != p_simple->args.end(); ++j) { + Fn_Arg::Alt *p_simple_fnarg = dynamic_cast(*j); + if (p_simple_fnarg && (p_simple_fnarg->alt == block)) { + p_simple_fnarg->alt = block->alts.front(); + break; + } + } + } + + // a Link is a leaf and thus cannot contain a block + // Alt::Link *p_link = dynamic_cast(parent); + + // parent block should have been removed first! + // Alt::Block *p_block = dynamic_cast(parent); + + // Alternative is not allowed in Multi-Track link. + // Alt::Multi *p_multi = dynamic_cast(parent); + } + } else { + // the Alt::Block holds multiple alternatives. + if (parent == NULL) { + nt->alts.push_back(block->alts.front()->clone()); + block->alts.pop_front(); + } else { + Alt::Simple *p_simple = dynamic_cast(parent); + if (p_simple) { + // if parent is of type Alt::Simple + // 1) clone the full existing alternative + Alt::Base *newalt = (*alt)->clone(); + + // 2) remove first block alternative from original object + Alt::Base *first_block_alt = block->alts.front(); + block->alts.pop_front(); + + // 3) remove Alt::Block and connect first block + // alternative to cloned object + Alt::Base *cloned_block = newalt->find_block(); + // a clone of the alt rule containing block should have this + // block as well! + assert(cloned_block); + Alt::Simple *cloned_p_simple = dynamic_cast( + newalt->find_block_parent(*cloned_block)); + // must exist and be Alt::Simple for above reasons + assert(cloned_p_simple); + for (std::list::iterator j = + cloned_p_simple->args.begin(); + j != cloned_p_simple->args.end(); ++j) { + Fn_Arg::Alt *cloned_p_simple_fnarg = + dynamic_cast(*j); + if (cloned_p_simple_fnarg && + (cloned_p_simple_fnarg->alt == cloned_block)) { + cloned_p_simple_fnarg->alt = first_block_alt; + break; + } + } + + nt->alts.push_back(newalt); + } + + // as with the single case above, Link, Block, + // Multi cannot be parents + } + } + + // for next iteration: check if production rules might still + // contain other blocks + block = dynamic_cast((*alt)->find_block()); + } + } + } + } +} + void Grammar::inject_outside_nts() { std::vector outside_nt_list; Grammar::inject_outside_nts(outside_nt_list); @@ -1039,6 +1142,10 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { throw LogError(msg); } + // before we inject outside rules, we need to make sure + // that grammar does not contain any Alt::Blocks + this->resolve_blocks(); + // 1) collect non-terminals used in rhs of productions together with their // calling lhs non-terminal we thus exclude e.g. non-terminal foo in // production "foo = BASE;" diff --git a/src/grammar.hh b/src/grammar.hh index 3a26b812b..718245e70 100644 --- a/src/grammar.hh +++ b/src/grammar.hh @@ -219,6 +219,11 @@ class Grammar { // same as above, without checking user provided NT list void inject_outside_nts(); unsigned int to_dot(unsigned int *nodeID, std::ostream &out); + + // blocks allow alternatives for sub-productions. This function shell free + // a grammar of Alt::Block by explicitly adding alternatives as top level + // alternative production rules for non-terminals. + void resolve_blocks(); }; diff --git a/testdata/modtest/multi_resolve_blocks.cc b/testdata/modtest/multi_resolve_blocks.cc new file mode 100644 index 000000000..1a6f14a4d --- /dev/null +++ b/testdata/modtest/multi_resolve_blocks.cc @@ -0,0 +1,72 @@ +#include "../../src/driver.hh" +#include "../../src/log.hh" + +#include +#include +#include +#include + +int main(int argc, char **argv) { + std::ostringstream o; + Log log; + log.set_debug(false); + log.set_ostream(o); + + Driver driver; + + if (argc != 2) { + std::cerr << "Call " << *argv << " *.gap-file\n"; + return 1; + } + // === front + // set the file name of the gap-source-code + driver.setFilename(argv[1]); + + // parses the input file and builds the AST + driver.parse(); + + // simply gets the selected grammar, which is either the + // grammar that occurred first in the source code or is the + // one that was named in the parameters on the command line + Grammar *grammar = driver.ast.grammar(); + // Now check the semantic, which does more than the function + // name suggests. The semantic check is embedded in the algorithm + // that links the grammar graph together, and computes yield-sizes. + // If the method returns false, there are some semantic errors, + // which leads to the end of the compilation process. + bool r = grammar->check_semantic(); + if (!r) { + throw LogError("Seen semantic errors."); + } + + // replace Alt::Block from grammar rules with explicit + // alternatives + grammar->resolve_blocks(); + + // set approx table design + grammar->approx_table_conf(); + + // find what type of input is read + // chars, sequence of ints etc. + driver.ast.derive_temp_alphabet(); + + r = driver.ast.check_signature(); + if (!r) { + throw LogError("Seen signature errors."); + } + + // apply this to identify standard functions like Min, Max, Exp etc. + driver.ast.derive_roles(); + + + // ------------- back ------------ + grammar->init_list_sizes(); + + grammar->init_indices(); + grammar->init_decls(); + // for cyk (ordering of NT for parsing, see page 101 of the thesis) + grammar->dep_analysis(); + + unsigned int nodeID = 1; + grammar->to_dot(&nodeID, std::cout); +} From dce496f9f48aa4565cdd100825c75f4f14c342ad Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 11 May 2022 17:36:01 +0200 Subject: [PATCH 091/209] cope with Fn_Arg != Alt --- src/alt.cc | 67 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index dc4054fd9..834eb2a24 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3605,7 +3605,8 @@ void Alt::Link::expand_outside_nt_indices(Expr::Base *left, void Alt::Block::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { throw LogError( - "Alt::Block::expand_outside_nt_indices is not yet implemented!"); + "Alt::Block::expand_outside_nt_indices blocks must be resolved prior " + "to outside generation, see function Grammar::resolve_blocks!"); } void Alt::Multi::expand_outside_nt_indices(Expr::Base *left, Expr::Base *right, size_t track) { @@ -3804,7 +3805,9 @@ void Alt::Link::init_indices_outside(Expr::Base *left, Expr::Base *right, void Alt::Block::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { - throw LogError("Alt::Block::init_indices_outside is not yet implemented!"); + throw LogError("Alt::Block::init_indices_outside blocks must be " + "resolved prior to outside generation, see function " + "Grammar::resolve_blocks!"); } void Alt::Multi::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, @@ -3886,7 +3889,8 @@ std::pair *Alt::Block::get_outside_accum_yieldsizes(size_t track, bool is_right_of_outside_nt) { throw LogError( - "Alt::Block::get_outside_accum_yieldsizes is not yet implemented!"); + "Alt::Block::get_outside_accum_yieldsizes blocks must be resolved prior " + "to outside generation, see function Grammar::resolve_blocks!"); return NULL; } std::pair @@ -3934,9 +3938,17 @@ Alt::Base *Alt::Base::find_block() { Alt::Base *Alt::Simple::find_block() { for (std::list::iterator i = this->args.begin(); i != this->args.end(); ++i) { - Alt::Block *block = dynamic_cast((*i)->alt_ref()); - if (block) { - return (*i)->alt_ref(); + Fn_Arg::Alt *fn_arg = dynamic_cast(*i); + if (fn_arg) { + Alt::Block *block = dynamic_cast((*i)->alt_ref()); + if (block) { + return (*i)->alt_ref(); + } else { + Alt::Base *sub = (*i)->alt_ref()->find_block(); + if (sub) { + return sub; + } + } } } return NULL; @@ -3954,6 +3966,11 @@ Alt::Base *Alt::Multi::find_block() { Alt::Block *block = dynamic_cast(*i); if (block) { return *i; + } else { + Alt::Base *sub = (*i)->find_block(); + if (sub) { + return sub; + } } } return NULL; @@ -3966,9 +3983,19 @@ Alt::Base *Alt::Base::find_block_parent(const Alt::Base &block) { Alt::Base *Alt::Simple::find_block_parent(const Alt::Base &block) { for (std::list::iterator i = this->args.begin(); i != this->args.end(); ++i) { - Alt::Block *child_block = dynamic_cast((*i)->alt_ref()); - if (child_block && (child_block == &block)) { - return this; + Fn_Arg::Alt *fn_arg = dynamic_cast(*i); + if (fn_arg) { + Alt::Block *child_block = dynamic_cast((*i)->alt_ref()); + if (child_block) { + if (child_block == &block) { + return this; + } + } else { + Alt::Base *sub = (*i)->alt_ref()->find_block_parent(block); + if (sub) { + return sub; + } + } } } return NULL; @@ -3981,8 +4008,15 @@ Alt::Base *Alt::Block::find_block_parent(const Alt::Base &block) { for (std::list::iterator i = this->alts.begin(); i != this->alts.end(); ++i) { Alt::Block *child_block = dynamic_cast(*i); - if (child_block && (child_block == &block)) { - return this; + if (child_block) { + if (child_block == &block) { + return this; + } + } else { + Alt::Base *sub = (*i)->find_block_parent(block); + if (sub) { + return sub; + } } } return NULL; @@ -3991,8 +4025,15 @@ Alt::Base *Alt::Multi::find_block_parent(const Alt::Base &block) { for (std::list::iterator i = this->list.begin(); i != this->list.end(); ++i) { Alt::Block *child_block = dynamic_cast(*i); - if (child_block && (child_block == &block)) { - return this; + if (child_block) { + if (child_block == &block) { + return this; + } + } else { + Alt::Base *sub = (*i)->find_block_parent(block); + if (sub) { + return sub; + } } } return NULL; From 69630b66e82e9664f430122e40353f62b8cab924 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 11 May 2022 22:34:09 +0200 Subject: [PATCH 092/209] adding test case grammar --- testdata/grammar_outside/unblock_alot.gap | 239 ++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 testdata/grammar_outside/unblock_alot.gap diff --git a/testdata/grammar_outside/unblock_alot.gap b/testdata/grammar_outside/unblock_alot.gap new file mode 100644 index 000000000..74fdb37b8 --- /dev/null +++ b/testdata/grammar_outside/unblock_alot.gap @@ -0,0 +1,239 @@ +import rna +import "singlefold.hh" + +input rna + +signature sig_foldrna(alphabet,answer) { + answer sadd(Subsequence,answer); //add one unpaired base + answer cadd(answer,answer); //adds one component, which has dangling bases from both sides, next component has a dangling base from left + answer nil(Subsequence); //empty structure + answer drem(Subsequence,answer,Subsequence); //no dangle, just the component + answer sr(Subsequence,answer,Subsequence); //elongate a stack by one base-pair + answer hl(Subsequence,Subsequence,Subsequence); //a hairpin loop with a closing base-pair + answer bl(Subsequence, Subsequence, answer, Subsequence); // a bulge loop to the left with a closing base-pair + answer br(Subsequence, answer, Subsequence, Subsequence); // a bulge loop to the right with a closing base-pair + answer il(Subsequence, Subsequence, answer, Subsequence, Subsequence); // an internal loop with a closing base-pair + answer ml(Subsequence,answer,Subsequence); // a multi-loop with a closing base-pair and no dangling bases + answer addss(answer,Subsequence); // append a region of unpaired bases + answer incl(answer); // add penalty for one more multi-loop component + choice [answer] h([answer]); +} + +algebra alg_count auto count; +algebra alg_enum auto enum; + +algebra alg_mfe implements sig_foldrna(alphabet = char, answer = int) { + int sadd(Subsequence lb, int x) { + return x + sbase_energy(); + } + int cadd(int x, int y) { + return x + y; + } + int drem(Subsequence lb, int x, Subsequence rb) { + return x + termau_energy(lb, rb); + } + int sr(Subsequence lb, int x, Subsequence rb) { + return x + sr_energy(lb, rb); + } + int hl(Subsequence lb, Subsequence r, Subsequence rb) { + return hl_energy(r); + } + int bl(Subsequence lb, Subsequence lr, int x, Subsequence rb) { + return x + bl_energy(lr, rb); + } + int br(Subsequence lb, int x, Subsequence rr, Subsequence rb) { + return x + br_energy(lb, rr); + } + int il(Subsequence lb, Subsequence lr, int x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } + int ml(Subsequence lb, int x, Subsequence rb) { + return x + ml_energy() + ul_energy() + termau_energy(lb, rb); + } + int incl(int x) { + return x + ul_energy(); + } + int addss(int x, Subsequence r) { + return x + ss_energy(r); + } + int nil(Subsequence n) { + return 0; + } + choice [int] h([int] i) { + return list(minimum(i)); + } +} + +algebra alg_pfunc implements sig_foldrna(alphabet = char, answer = double) { + double sadd(Subsequence lb, double x) { + return scale(1) * x * mk_pf(sbase_energy()); + } + double cadd(double x, double y) { + return x * y; + } + double drem(Subsequence lb, double x, Subsequence rb) { + return x * mk_pf(termau_energy(lb, rb)); + } + double sr(Subsequence lb, double x, Subsequence rb) { + return scale(2) * x * mk_pf(sr_energy(lb, rb)); + } + double hl(Subsequence lb, Subsequence r, Subsequence rb) { + return scale(2+r.j-r.i) * mk_pf(hl_energy(r)); + } + double bl(Subsequence lb, Subsequence lr, double x, Subsequence rb) { + return scale(2+lr.j-lr.i) * x * mk_pf(bl_energy(lr, rb)); + } + double br(Subsequence lb, double x, Subsequence rr, Subsequence rb) { + return scale(2+rr.j-rr.i) * x * mk_pf(br_energy(lb, rr)); + } + double il(Subsequence lb, Subsequence lr, double x, Subsequence rr, Subsequence rb) { + return scale(2+lr.j-lr.i+rr.j-rr.i) * x * mk_pf(il_energy(lr, rr)); + } + double ml(Subsequence lb, double x, Subsequence rb) { + return scale(2) * x * mk_pf(ml_energy() + ul_energy() + termau_energy(lb, rb)); + } + double incl(double x) { + return x * mk_pf(ul_energy()); + } + double addss(double x, Subsequence r) { + return scale(r.j-r.i) * x * mk_pf(ss_energy(r)); + } + double nil(Subsequence n) { + return 1; + } + choice [double] h([double] i) { + return list(sum(i)); + } +} + +algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { + string sadd(Subsequence lb,string e) { + string res; + append(res, '.'); + append(res, e); + return res; + } + + string cadd(string le,string re) { + string res; + append(res, le); + append(res, re); + return res; + } + + string nil(Subsequence loc) { + string r; + return r; + } + + string drem(Subsequence lloc, string e, Subsequence rloc) { + return e; + } + + string sr(Subsequence lb,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string hl(Subsequence lb,Subsequence region,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(region)); + append(res, ')'); + return res; + } + + + string bl(Subsequence lb,Subsequence lregion,string e,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, ')'); + return res; + } + + string br(Subsequence lb,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string il(Subsequence lb,Subsequence lregion,string e,Subsequence rregion,Subsequence rb) { + string res; + append(res, '('); + append(res, '.', size(lregion)); + append(res, e); + append(res, '.', size(rregion)); + append(res, ')'); + return res; + } + + string ml(Subsequence lb, string e, Subsequence rb) { + string res; + append(res, '('); + append(res, e); + append(res, ')'); + return res; + } + + string addss(string e,Subsequence rb) { + string res; + append(res, e); + append(res, '.', size(rb)); + return res; + } + + string incl(string e) { + return e; + } + + choice [string] h([string] i) { + return i; + } +} + +grammar gra_nodangle uses sig_foldrna(axiom = struct) { + struct = sadd(BASE, {{{struct}}}) | + cadd({dangle | sadd(BASE, {dangle | incl(hairpin) | incl(strong)})}, struct) | + nil(LOC) # h; + + dangle = drem(LOC, strong, LOC) # h; + + strong = weak # h; + + weak = stack | + hairpin | + leftB | + rightB | + iloop | + multiloop # h; + + stack = sr(BASE, weak, BASE) with basepairing # h; + hairpin = hl(BASE, REGION with minsize(3), BASE) with basepairing # h; + leftB = bl(BASE, REGION with maxsize(30), strong, BASE) with basepairing # h; + rightB = br(BASE, strong, REGION with maxsize(30), BASE) with basepairing # h; + iloop = il(BASE, REGION with maxsize(30), strong, REGION with maxsize(30), BASE) with basepairing with_overlay iloopSumMax(30) # h; + multiloop = ml(BASE, ml_comps, BASE) with basepairing # h; + + ml_comps = sadd(BASE, ml_comps) | + cadd(incl(dangle), ml_comps1) # h; + + ml_comps1 = sadd(BASE, ml_comps1) | + cadd(incl(dangle), ml_comps1) | + incl(dangle) | + addss(incl(dangle), REGION) # h; +} + + + +instance ins_mfe = gra_nodangle(alg_mfe); +instance ins_pfunc = gra_nodangle(alg_pfunc); +instance ins_count = gra_nodangle(alg_count); +instance ins_enum = gra_nodangle(alg_enum); From 638a14c1d0e6a30d70b94e1fb4ca96195bb8fe16 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 11 May 2022 23:01:02 +0200 Subject: [PATCH 093/209] moving resolve_blocks function from Grammar to Symbol::NT thus we can operate on cloned NTs for outside generation and leave inside rules untouched --- src/grammar.cc | 110 ++++--------------------------------------------- src/symbol.cc | 103 +++++++++++++++++++++++++++++++++++++++++++++ src/symbol.hh | 5 +++ 3 files changed, 115 insertions(+), 103 deletions(-) diff --git a/src/grammar.cc b/src/grammar.cc index 183dbfe02..b1bfb5f05 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1002,106 +1002,10 @@ void Grammar::remove(Symbol::NT *x) { tabulated.erase(nt); } -// a user provided grammar might contain production rules that use blocks -// e.g. NT = sadd(BASE, {hairpin | iloop}) -// this basically is a short hand notation for multiple alternatives, e.g. -// NT = sadd(BASE, hairpin) | sadd(BASE, iloop) -// For outside grammar generation, we first want to make short-hand notations -// explicit by applying this function to the selected grammar. void Grammar::resolve_blocks() { for (hashtable::iterator i = NTs.begin(); i != NTs.end(); ++i) { - Symbol::NT *nt = dynamic_cast(i->second); - if (nt) { - for (std::list::iterator alt = nt->alts.begin(); - alt != nt->alts.end(); ++alt) { - // check if alternative contains any Alt::Block - Alt::Block *block = dynamic_cast((*alt)->find_block()); - while (block) { - // locate parent of found Alt::Block - Alt::Base *parent = (*alt)->find_block_parent(*block); - if (block->alts.size() == 1) { - // if a Alt::Block holds only one alternative, - // we can simply remove Alt::Block altogether - if (parent == NULL) { - // parent is the non-terminal - *alt = block->alts.front(); - } else { - Alt::Simple *p_simple = dynamic_cast(parent); - if (p_simple) { - // if parent is of type Alt::Simple - for (std::list::iterator j = - p_simple->args.begin(); - j != p_simple->args.end(); ++j) { - Fn_Arg::Alt *p_simple_fnarg = dynamic_cast(*j); - if (p_simple_fnarg && (p_simple_fnarg->alt == block)) { - p_simple_fnarg->alt = block->alts.front(); - break; - } - } - } - - // a Link is a leaf and thus cannot contain a block - // Alt::Link *p_link = dynamic_cast(parent); - - // parent block should have been removed first! - // Alt::Block *p_block = dynamic_cast(parent); - - // Alternative is not allowed in Multi-Track link. - // Alt::Multi *p_multi = dynamic_cast(parent); - } - } else { - // the Alt::Block holds multiple alternatives. - if (parent == NULL) { - nt->alts.push_back(block->alts.front()->clone()); - block->alts.pop_front(); - } else { - Alt::Simple *p_simple = dynamic_cast(parent); - if (p_simple) { - // if parent is of type Alt::Simple - // 1) clone the full existing alternative - Alt::Base *newalt = (*alt)->clone(); - - // 2) remove first block alternative from original object - Alt::Base *first_block_alt = block->alts.front(); - block->alts.pop_front(); - - // 3) remove Alt::Block and connect first block - // alternative to cloned object - Alt::Base *cloned_block = newalt->find_block(); - // a clone of the alt rule containing block should have this - // block as well! - assert(cloned_block); - Alt::Simple *cloned_p_simple = dynamic_cast( - newalt->find_block_parent(*cloned_block)); - // must exist and be Alt::Simple for above reasons - assert(cloned_p_simple); - for (std::list::iterator j = - cloned_p_simple->args.begin(); - j != cloned_p_simple->args.end(); ++j) { - Fn_Arg::Alt *cloned_p_simple_fnarg = - dynamic_cast(*j); - if (cloned_p_simple_fnarg && - (cloned_p_simple_fnarg->alt == cloned_block)) { - cloned_p_simple_fnarg->alt = first_block_alt; - break; - } - } - - nt->alts.push_back(newalt); - } - - // as with the single case above, Link, Block, - // Multi cannot be parents - } - } - - // for next iteration: check if production rules might still - // contain other blocks - block = dynamic_cast((*alt)->find_block()); - } - } - } + (*i).second->resolve_blocks(); } } @@ -1142,10 +1046,6 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { throw LogError(msg); } - // before we inject outside rules, we need to make sure - // that grammar does not contain any Alt::Blocks - this->resolve_blocks(); - // 1) collect non-terminals used in rhs of productions together with their // calling lhs non-terminal we thus exclude e.g. non-terminal foo in // production "foo = BASE;" @@ -1203,8 +1103,12 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { i != NTs.end(); ++i) { Symbol::NT *nt = dynamic_cast(i->second); if (nt) { // skip terminals - for (std::list::iterator alt = nt->alts.begin(); - alt != nt->alts.end(); ++alt) { + // don't directly operate on given inside NTs, + // but first resolve Alt::Block use for outside rules + Symbol::NT *nt_resolved = nt->clone(nt->track_pos()); + nt_resolved->resolve_blocks(); + for (std::list::iterator alt = nt_resolved->alts.begin(); + alt != nt_resolved->alts.end(); ++alt) { std::list *rhs_nts = new std::list(); (*alt)->get_nonterminals(rhs_nts); if (rhs_nts->size() > 0) { diff --git a/src/symbol.cc b/src/symbol.cc index b716db6c7..75a5a7772 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -46,6 +46,7 @@ #include "type/backtrace.hh" #include "alt.hh" +#include "fn_arg.hh" Symbol::Base::Base(std::string *n, Type t, const Loc &l) @@ -1694,3 +1695,105 @@ unsigned int Symbol::NT::to_dot(unsigned int *nodeID, std::ostream &out, return thisID; } // END functions produce graphViz code to represent the grammar + +// a user provided grammar might contain production rules that use blocks +// e.g. NT = sadd(BASE, {hairpin | iloop}) +// this basically is a short hand notation for multiple alternatives, e.g. +// NT = sadd(BASE, hairpin) | sadd(BASE, iloop) +// For outside grammar generation, we first want to make short-hand notations +// explicit by applying this function to the selected grammar. +// This only applies for NTs, not for Base or Terminal +void Symbol::Base::resolve_blocks() { +} +void Symbol::Terminal::resolve_blocks() { +} +void Symbol::NT::resolve_blocks() { + for (std::list::iterator alt = this->alts.begin(); + alt != this->alts.end(); ++alt) { + // check if alternative contains any Alt::Block + Alt::Block *block = dynamic_cast((*alt)->find_block()); + while (block) { + // locate parent of found Alt::Block + Alt::Base *parent = (*alt)->find_block_parent(*block); + if (block->alts.size() == 1) { + // if a Alt::Block holds only one alternative, + // we can simply remove Alt::Block altogether + if (parent == NULL) { + // parent is the non-terminal + *alt = block->alts.front(); + } else { + Alt::Simple *p_simple = dynamic_cast(parent); + if (p_simple) { + // if parent is of type Alt::Simple + for (std::list::iterator j = + p_simple->args.begin(); + j != p_simple->args.end(); ++j) { + Fn_Arg::Alt *p_simple_fnarg = dynamic_cast(*j); + if (p_simple_fnarg && (p_simple_fnarg->alt == block)) { + p_simple_fnarg->alt = block->alts.front(); + break; + } + } + } + + // a Link is a leaf and thus cannot contain a block + // Alt::Link *p_link = dynamic_cast(parent); + + // parent block should have been removed first! + // Alt::Block *p_block = dynamic_cast(parent); + + // Alternative is not allowed in Multi-Track link. + // Alt::Multi *p_multi = dynamic_cast(parent); + } + } else { + // the Alt::Block holds multiple alternatives. + if (parent == NULL) { + this->alts.push_back(block->alts.front()->clone()); + block->alts.pop_front(); + } else { + Alt::Simple *p_simple = dynamic_cast(parent); + if (p_simple) { + // if parent is of type Alt::Simple + // 1) clone the full existing alternative + Alt::Base *newalt = (*alt)->clone(); + + // 2) remove first block alternative from original object + Alt::Base *first_block_alt = block->alts.front(); + block->alts.pop_front(); + + // 3) remove Alt::Block and connect first block + // alternative to cloned object + Alt::Base *cloned_block = newalt->find_block(); + // a clone of the alt rule containing block should have this + // block as well! + assert(cloned_block); + Alt::Simple *cloned_p_simple = dynamic_cast( + newalt->find_block_parent(*cloned_block)); + // must exist and be Alt::Simple for above reasons + assert(cloned_p_simple); + for (std::list::iterator j = + cloned_p_simple->args.begin(); + j != cloned_p_simple->args.end(); ++j) { + Fn_Arg::Alt *cloned_p_simple_fnarg = + dynamic_cast(*j); + if (cloned_p_simple_fnarg && + (cloned_p_simple_fnarg->alt == cloned_block)) { + cloned_p_simple_fnarg->alt = first_block_alt; + break; + } + } + + this->alts.push_back(newalt); + } + + // as with the single case above, Link, Block, + // Multi cannot be parents + } + } + + // for next iteration: check if production rules might still + // contain other blocks + block = dynamic_cast((*alt)->find_block()); + } + } +} diff --git a/src/symbol.hh b/src/symbol.hh index d9711a3f1..5afbbf6f1 100644 --- a/src/symbol.hh +++ b/src/symbol.hh @@ -255,6 +255,9 @@ class Base { // prints grammar rules as GraphViz commands virtual unsigned int to_dot(unsigned int *nodeID, std::ostream &out, bool is_rhs, Symbol::NT *axiom); + // only for NTs: check if alternatives contain Alt::Block short-cuts + // if so, remove them by making alternatives explicit + virtual void resolve_blocks(); }; @@ -314,6 +317,7 @@ class Terminal : public Base { bool isPredefinedTerminalParser(); unsigned int to_dot(unsigned int *nodeID, std::ostream &out, bool is_rhs, Symbol::NT *axiom); + void resolve_blocks(); }; @@ -519,6 +523,7 @@ class NT : public Base { } unsigned int to_dot(unsigned int *nodeID, std::ostream &out, bool is_rhs, Symbol::NT *axiom); + void resolve_blocks(); }; From a30205c3a3bb0ff84a9f834e8f2749402cff1670 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 12 May 2022 14:36:32 +0200 Subject: [PATCH 094/209] make sure bool value is_partof_outside is initialized to false! --- src/symbol.cc | 2 +- src/symbol.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/symbol.cc b/src/symbol.cc index 75a5a7772..94d2bc8ee 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -57,7 +57,7 @@ Symbol::Base::Base(std::string *n, Type t, const Loc &l) terminal_type(false), rt_computed(false), name(n), orig_name(n), location(l), tracks_(0), - track_pos_(0) { + track_pos_(0), is_partof_outside(false) { assert(name); } diff --git a/src/symbol.hh b/src/symbol.hh index 5afbbf6f1..972181d39 100644 --- a/src/symbol.hh +++ b/src/symbol.hh @@ -251,7 +251,7 @@ class Base { // flag to indicate if alternative is for inside (the default) or outside // production rules. - bool is_partof_outside; + bool is_partof_outside = false; // prints grammar rules as GraphViz commands virtual unsigned int to_dot(unsigned int *nodeID, std::ostream &out, bool is_rhs, Symbol::NT *axiom); From c9f7a107bf18747b8e5e16bac9647bd160ac8917 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 12 May 2022 16:57:34 +0200 Subject: [PATCH 095/209] don't throw errors but return status codes != 0 --- testdata/modtest/multi_resolve_blocks.cc | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/testdata/modtest/multi_resolve_blocks.cc b/testdata/modtest/multi_resolve_blocks.cc index 1a6f14a4d..44bff3bf1 100644 --- a/testdata/modtest/multi_resolve_blocks.cc +++ b/testdata/modtest/multi_resolve_blocks.cc @@ -7,6 +7,11 @@ #include int main(int argc, char **argv) { + if (argc != 2) { + std::cerr << "Call " << *argv << " *.gap-file\n"; + return 1; + } + std::ostringstream o; Log log; log.set_debug(false); @@ -14,16 +19,16 @@ int main(int argc, char **argv) { Driver driver; - if (argc != 2) { - std::cerr << "Call " << *argv << " *.gap-file\n"; - return 1; - } // === front // set the file name of the gap-source-code - driver.setFilename(argv[1]); + std::string filename(argv[1]); + driver.setFilename(filename); // parses the input file and builds the AST driver.parse(); + if (driver.is_failing()) { + return 4; + } // simply gets the selected grammar, which is either the // grammar that occurred first in the source code or is the @@ -36,7 +41,7 @@ int main(int argc, char **argv) { // which leads to the end of the compilation process. bool r = grammar->check_semantic(); if (!r) { - throw LogError("Seen semantic errors."); + return 2; } // replace Alt::Block from grammar rules with explicit @@ -52,7 +57,7 @@ int main(int argc, char **argv) { r = driver.ast.check_signature(); if (!r) { - throw LogError("Seen signature errors."); + return 3; } // apply this to identify standard functions like Min, Max, Exp etc. @@ -69,4 +74,6 @@ int main(int argc, char **argv) { unsigned int nodeID = 1; grammar->to_dot(&nodeID, std::cout); + + return 0; } From 9f40163a55900979d670d8d744c82745cea05869 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 12 May 2022 17:18:05 +0200 Subject: [PATCH 096/209] lint --- testdata/modtest/multi_resolve_blocks.cc | 25 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/testdata/modtest/multi_resolve_blocks.cc b/testdata/modtest/multi_resolve_blocks.cc index 44bff3bf1..525f0f6cc 100644 --- a/testdata/modtest/multi_resolve_blocks.cc +++ b/testdata/modtest/multi_resolve_blocks.cc @@ -1,15 +1,17 @@ -#include "../../src/driver.hh" -#include "../../src/log.hh" +// Copyright 2022 stefan.m.janssen@gmail.com #include #include #include #include +#include "../../src/driver.hh" +#include "../../src/log.hh" + int main(int argc, char **argv) { if (argc != 2) { - std::cerr << "Call " << *argv << " *.gap-file\n"; - return 1; + std::cerr << "Call " << *argv << " *.gap-file\n"; + return 1; } std::ostringstream o; @@ -47,7 +49,8 @@ int main(int argc, char **argv) { // replace Alt::Block from grammar rules with explicit // alternatives grammar->resolve_blocks(); - + + // set approx table design grammar->approx_table_conf(); @@ -55,9 +58,15 @@ int main(int argc, char **argv) { // chars, sequence of ints etc. driver.ast.derive_temp_alphabet(); - r = driver.ast.check_signature(); - if (!r) { - return 3; + log.set_debug(true); + log.set_ostream(std::cerr); + try { + r = driver.ast.check_signature(); + if (!r) { + return 3; + } + } catch (LogThreshException) { + return 9; } // apply this to identify standard functions like Min, Max, Exp etc. From 686a668c800e134de4dbf148f08e56f4c9850a61 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 13 May 2022 11:35:01 +0200 Subject: [PATCH 097/209] mind filters when resolving blocks --- src/alt.hh | 2 +- src/symbol.cc | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/alt.hh b/src/alt.hh index d2c9d50fc..baa6f88b1 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -298,7 +298,7 @@ class Base { } Yield::Multi m_ys_inside; - private: + public: std::vector > multi_filter; public: diff --git a/src/symbol.cc b/src/symbol.cc index 94d2bc8ee..46c66e5d3 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -1720,7 +1720,11 @@ void Symbol::NT::resolve_blocks() { // we can simply remove Alt::Block altogether if (parent == NULL) { // parent is the non-terminal - *alt = block->alts.front(); + Alt::Base *child = block->alts.front(); + child->top_level = block->top_level; + child->filters = block->filters; + child->multi_filter = block->multi_filter; + *alt = child; } else { Alt::Simple *p_simple = dynamic_cast(parent); if (p_simple) { @@ -1730,7 +1734,11 @@ void Symbol::NT::resolve_blocks() { j != p_simple->args.end(); ++j) { Fn_Arg::Alt *p_simple_fnarg = dynamic_cast(*j); if (p_simple_fnarg && (p_simple_fnarg->alt == block)) { - p_simple_fnarg->alt = block->alts.front(); + Alt::Base *child = block->alts.front(); + child->top_level = block->top_level; + child->filters = block->filters; + child->multi_filter = block->multi_filter; + p_simple_fnarg->alt = child; break; } } @@ -1748,7 +1756,11 @@ void Symbol::NT::resolve_blocks() { } else { // the Alt::Block holds multiple alternatives. if (parent == NULL) { - this->alts.push_back(block->alts.front()->clone()); + Alt::Base *child = block->alts.front()->clone(); + child->top_level = block->top_level; + child->filters = block->filters; + child->multi_filter = block->multi_filter; + this->alts.push_back(child); block->alts.pop_front(); } else { Alt::Simple *p_simple = dynamic_cast(parent); @@ -1759,6 +1771,9 @@ void Symbol::NT::resolve_blocks() { // 2) remove first block alternative from original object Alt::Base *first_block_alt = block->alts.front(); + first_block_alt->top_level = block->top_level; + first_block_alt->filters = block->filters; + first_block_alt->multi_filter = block->multi_filter; block->alts.pop_front(); // 3) remove Alt::Block and connect first block From 0ae32f7199fdf7e3972046bdbe1791394078f7f3 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 13 May 2022 12:01:23 +0200 Subject: [PATCH 098/209] extend filters instead of replacing --- src/symbol.cc | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/symbol.cc b/src/symbol.cc index 46c66e5d3..733e18761 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -1721,9 +1721,9 @@ void Symbol::NT::resolve_blocks() { if (parent == NULL) { // parent is the non-terminal Alt::Base *child = block->alts.front(); - child->top_level = block->top_level; - child->filters = block->filters; - child->multi_filter = block->multi_filter; + child->top_level = Bool(true); + child->filters.insert(child->filters.end(), block->filters.begin(), block->filters.end()); + child->multi_filter.insert(child->multi_filter.end(), block->multi_filter.begin(), block->multi_filter.end()); *alt = child; } else { Alt::Simple *p_simple = dynamic_cast(parent); @@ -1735,9 +1735,8 @@ void Symbol::NT::resolve_blocks() { Fn_Arg::Alt *p_simple_fnarg = dynamic_cast(*j); if (p_simple_fnarg && (p_simple_fnarg->alt == block)) { Alt::Base *child = block->alts.front(); - child->top_level = block->top_level; - child->filters = block->filters; - child->multi_filter = block->multi_filter; + child->filters.insert(child->filters.end(), block->filters.begin(), block->filters.end()); + child->multi_filter.insert(child->multi_filter.end(), block->multi_filter.begin(), block->multi_filter.end()); p_simple_fnarg->alt = child; break; } @@ -1757,9 +1756,9 @@ void Symbol::NT::resolve_blocks() { // the Alt::Block holds multiple alternatives. if (parent == NULL) { Alt::Base *child = block->alts.front()->clone(); - child->top_level = block->top_level; - child->filters = block->filters; - child->multi_filter = block->multi_filter; + child->top_level = Bool(true); + child->filters.insert(child->filters.end(), block->filters.begin(), block->filters.end()); + child->multi_filter.insert(child->multi_filter.end(), block->multi_filter.begin(), block->multi_filter.end()); this->alts.push_back(child); block->alts.pop_front(); } else { @@ -1771,9 +1770,8 @@ void Symbol::NT::resolve_blocks() { // 2) remove first block alternative from original object Alt::Base *first_block_alt = block->alts.front(); - first_block_alt->top_level = block->top_level; - first_block_alt->filters = block->filters; - first_block_alt->multi_filter = block->multi_filter; + first_block_alt->filters.insert(first_block_alt->filters.end(), block->filters.begin(), block->filters.end()); + first_block_alt->multi_filter.insert(first_block_alt->multi_filter.end(), block->multi_filter.begin(), block->multi_filter.end()); block->alts.pop_front(); // 3) remove Alt::Block and connect first block From 510c0b60290d389d948fef1190b1157f94b96d71 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 13 May 2022 15:05:07 +0200 Subject: [PATCH 099/209] reset iterator to first element to avoid undefined behaviour --- src/symbol.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/symbol.cc b/src/symbol.cc index 733e18761..2d77f72a1 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -1724,7 +1724,11 @@ void Symbol::NT::resolve_blocks() { child->top_level = Bool(true); child->filters.insert(child->filters.end(), block->filters.begin(), block->filters.end()); child->multi_filter.insert(child->multi_filter.end(), block->multi_filter.begin(), block->multi_filter.end()); - *alt = child; + // keep original ordering, i.e. delete block and push single = last alt to the end + this->alts.push_back(child); + this->alts.erase(alt); + // reset iterator to first element + alt = this->alts.begin(); } else { Alt::Simple *p_simple = dynamic_cast(parent); if (p_simple) { @@ -1796,7 +1800,7 @@ void Symbol::NT::resolve_blocks() { } } - this->alts.push_back(newalt); + this->alts.insert(alt, newalt); } // as with the single case above, Link, Block, From 3f2e673bc9bf5abe3a5ffaf83b7149a1a9f336eb Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 13 May 2022 15:09:49 +0200 Subject: [PATCH 100/209] don't use method that won't be encoded in gapc any longer --- testdata/modtest/multi_resolve_blocks.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/testdata/modtest/multi_resolve_blocks.cc b/testdata/modtest/multi_resolve_blocks.cc index 525f0f6cc..911544313 100644 --- a/testdata/modtest/multi_resolve_blocks.cc +++ b/testdata/modtest/multi_resolve_blocks.cc @@ -48,8 +48,10 @@ int main(int argc, char **argv) { // replace Alt::Block from grammar rules with explicit // alternatives - grammar->resolve_blocks(); - + for (hashtable::iterator i = grammar->NTs.begin(); + i != grammar->NTs.end(); ++i) { + (*i).second->resolve_blocks(); + } // set approx table design grammar->approx_table_conf(); @@ -58,8 +60,6 @@ int main(int argc, char **argv) { // chars, sequence of ints etc. driver.ast.derive_temp_alphabet(); - log.set_debug(true); - log.set_ostream(std::cerr); try { r = driver.ast.check_signature(); if (!r) { From 481a13b0af3439b258d5fb991ed9103e41eeeb08 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 13 May 2022 15:24:55 +0200 Subject: [PATCH 101/209] renaming to separate multi from novel outside tests --- testdata/modtest/{multi_outside_indices.cc => outside_indices.cc} | 0 .../{multi_resolve_blocks.cc => outside_resolve_blocks.cc} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename testdata/modtest/{multi_outside_indices.cc => outside_indices.cc} (100%) rename testdata/modtest/{multi_resolve_blocks.cc => outside_resolve_blocks.cc} (100%) diff --git a/testdata/modtest/multi_outside_indices.cc b/testdata/modtest/outside_indices.cc similarity index 100% rename from testdata/modtest/multi_outside_indices.cc rename to testdata/modtest/outside_indices.cc diff --git a/testdata/modtest/multi_resolve_blocks.cc b/testdata/modtest/outside_resolve_blocks.cc similarity index 100% rename from testdata/modtest/multi_resolve_blocks.cc rename to testdata/modtest/outside_resolve_blocks.cc From 3263432726a71a1f54d3e815ab548aae1fa26c57 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 13 May 2022 15:52:51 +0200 Subject: [PATCH 102/209] a more general split for outside VS multi tests --- Makefile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 53d756a87..52c49c966 100644 --- a/Makefile +++ b/Makefile @@ -128,7 +128,7 @@ gapc: src/gapc.o $(MAIN_OBJ) src/version.o src/prefix.o stats \ test_rt_tab \ -multi_cyk multi_deps multi_indices multi_list_size multi_eliminate_lists_more multi_eliminate_lists multi_algebra multi_signature multi_rt_approx multi_rt_all multi_in_out multi_self_rec multi_calls multi_table_dim multi_max_filter multi_loops multi_ys multi_outside_indices multi_resolve_blocks parser: src/version.o src/prefix.o +multi_cyk multi_deps multi_indices multi_list_size multi_eliminate_lists_more multi_eliminate_lists multi_algebra multi_signature multi_rt_approx multi_rt_all multi_in_out multi_self_rec multi_calls multi_table_dim multi_max_filter multi_loops multi_ys multi_print_grammar outside_resolve_blocks outside_indices parser: src/version.o src/prefix.o backtrack: src/version.o @@ -283,11 +283,13 @@ testdata/unittest/%: testdata/unittest/%.o # modtest -MODTESTS_CXX:= $(wildcard testdata/modtest/*.cc) +MODTESTS_CXX:= $(wildcard testdata/modtest/multi*.cc) +MODOUTSIDETESTS_CXX:= $(wildcard testdata/modtest/outside*.cc) MODTESTS:=$(sort $(subst testdata/modtest/,,$(MODTESTS_CXX:.cc=))) +MODOUTSIDETESTS:=$(sort $(subst testdata/modtest/,,$(MODOUTSIDETESTS_CXX:.cc=))) modtests: $(MODTESTS) - +outsidetests: $(MODOUTSIDETESTS) # paraltest @@ -307,9 +309,9 @@ test-mod: modtests cd testdata/modtest &&\ $(SHELL) run.sh $(TRUTH_DIR)/Mod$(TRUTH_SUFFIX) $(MODTESTS) -test-mod_outside: multi_outside_indices multi_resolve_blocks +test-mod_outside: outsidetests cd testdata/modtest &&\ - $(SHELL) run_outside.sh $(TRUTH_DIR)/Mod$(TRUTH_SUFFIX) $? + $(SHELL) run_outside.sh $(TRUTH_DIR)/Mod$(TRUTH_SUFFIX) $(MODOUTSIDETESTS) gen-mod: modtests cd testdata/modtest &&\ From d72131a511141829213349613c2a0373ad5977d6 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 13 May 2022 15:55:56 +0200 Subject: [PATCH 103/209] adding test for print-grammar --- testdata/modtest/multi_print_grammar.cc | 84 +++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 testdata/modtest/multi_print_grammar.cc diff --git a/testdata/modtest/multi_print_grammar.cc b/testdata/modtest/multi_print_grammar.cc new file mode 100644 index 000000000..28068f35a --- /dev/null +++ b/testdata/modtest/multi_print_grammar.cc @@ -0,0 +1,84 @@ +// Copyright 2022 stefan.m.janssen@gmail.com + +#include +#include +#include +#include + +#include "../../src/driver.hh" +#include "../../src/log.hh" + +int main(int argc, char **argv) { + if (argc != 2) { + std::cerr << "Call " << *argv << " *.gap-file\n"; + return 1; + } + + std::ostringstream o; + Log log; + log.set_debug(false); + log.set_ostream(o); + + Driver driver; + + // === front + // set the file name of the gap-source-code + std::string filename(argv[1]); + driver.setFilename(filename); + + // parses the input file and builds the AST + driver.parse(); + if (driver.is_failing()) { + return 4; + } + + // simply gets the selected grammar, which is either the + // grammar that occurred first in the source code or is the + // one that was named in the parameters on the command line + Grammar *grammar = driver.ast.grammar(); + // Now check the semantic, which does more than the function + // name suggests. The semantic check is embedded in the algorithm + // that links the grammar graph together, and computes yield-sizes. + // If the method returns false, there are some semantic errors, + // which leads to the end of the compilation process. + + // skip semantic check for testing + bool r = grammar->check_semantic(); + if (!r) { + return 2; + } + + // set approx table design + grammar->approx_table_conf(); + + // find what type of input is read + // chars, sequence of ints etc. + driver.ast.derive_temp_alphabet(); + + //log.set_ostream(std::cerr); + try { + r = driver.ast.check_signature(); + if (!r) { + return 3; + } + } catch (LogThreshException) { + return 9; + } + + // apply this to identify standard functions like Min, Max, Exp etc. + driver.ast.derive_roles(); + + + // ------------- back ------------ + grammar->init_list_sizes(); + + grammar->init_indices(); + grammar->init_decls(); + // for cyk (ordering of NT for parsing, see page 101 of the thesis) + grammar->dep_analysis(); + + unsigned int nodeID = 1; + grammar->to_dot(&nodeID, std::cout); + + return 0; +} From ce3d3e53f9f5f0d2592b920d4d3126dba14e6107 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 13 May 2022 16:00:10 +0200 Subject: [PATCH 104/209] cpplint --- src/symbol.cc | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/src/symbol.cc b/src/symbol.cc index 2d77f72a1..eb6225a49 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -1722,9 +1722,13 @@ void Symbol::NT::resolve_blocks() { // parent is the non-terminal Alt::Base *child = block->alts.front(); child->top_level = Bool(true); - child->filters.insert(child->filters.end(), block->filters.begin(), block->filters.end()); - child->multi_filter.insert(child->multi_filter.end(), block->multi_filter.begin(), block->multi_filter.end()); - // keep original ordering, i.e. delete block and push single = last alt to the end + child->filters.insert(child->filters.end(), block->filters.begin(), + block->filters.end()); + child->multi_filter.insert(child->multi_filter.end(), + block->multi_filter.begin(), + block->multi_filter.end()); + // keep original ordering, i.e. delete block and push + // single = last alt to the end this->alts.push_back(child); this->alts.erase(alt); // reset iterator to first element @@ -1739,8 +1743,12 @@ void Symbol::NT::resolve_blocks() { Fn_Arg::Alt *p_simple_fnarg = dynamic_cast(*j); if (p_simple_fnarg && (p_simple_fnarg->alt == block)) { Alt::Base *child = block->alts.front(); - child->filters.insert(child->filters.end(), block->filters.begin(), block->filters.end()); - child->multi_filter.insert(child->multi_filter.end(), block->multi_filter.begin(), block->multi_filter.end()); + child->filters.insert(child->filters.end(), + block->filters.begin(), + block->filters.end()); + child->multi_filter.insert(child->multi_filter.end(), + block->multi_filter.begin(), + block->multi_filter.end()); p_simple_fnarg->alt = child; break; } @@ -1761,8 +1769,12 @@ void Symbol::NT::resolve_blocks() { if (parent == NULL) { Alt::Base *child = block->alts.front()->clone(); child->top_level = Bool(true); - child->filters.insert(child->filters.end(), block->filters.begin(), block->filters.end()); - child->multi_filter.insert(child->multi_filter.end(), block->multi_filter.begin(), block->multi_filter.end()); + child->filters.insert(child->filters.end(), + block->filters.begin(), + block->filters.end()); + child->multi_filter.insert(child->multi_filter.end(), + block->multi_filter.begin(), + block->multi_filter.end()); this->alts.push_back(child); block->alts.pop_front(); } else { @@ -1774,8 +1786,13 @@ void Symbol::NT::resolve_blocks() { // 2) remove first block alternative from original object Alt::Base *first_block_alt = block->alts.front(); - first_block_alt->filters.insert(first_block_alt->filters.end(), block->filters.begin(), block->filters.end()); - first_block_alt->multi_filter.insert(first_block_alt->multi_filter.end(), block->multi_filter.begin(), block->multi_filter.end()); + first_block_alt->filters.insert(first_block_alt->filters.end(), + block->filters.begin(), + block->filters.end()); + first_block_alt->multi_filter.insert( + first_block_alt->multi_filter.end(), + block->multi_filter.begin(), + block->multi_filter.end()); block->alts.pop_front(); // 3) remove Alt::Block and connect first block From bfac2cb342ab40a5fbb06a44f92dcef532835abe Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 15 May 2022 22:39:48 +0200 Subject: [PATCH 105/209] except outside NTs --- src/grammar.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/grammar.cc b/src/grammar.cc index b1bfb5f05..8071a1424 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -655,7 +655,11 @@ void Grammar::put_table_conf(std::ostream &s) { hashtable::iterator a = tabulated.find( *i->second->name); if (a == tabulated.end()) { - assert(false); + // user might not have considered outside NTs to be tabulated in + // his/her manual annotated table-design + if (i->second->is_partof_outside == false) { + assert(false); + } } } } From 347bf121c029a496422fda1286973fe4f5eb20f6 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 16 May 2022 16:37:24 +0200 Subject: [PATCH 106/209] also print datatypes --- src/alt.cc | 13 +++++++++---- src/symbol.cc | 9 ++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 834eb2a24..cb10fcaa5 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3228,16 +3228,21 @@ unsigned int Alt::Base::to_dot(unsigned int *nodeID, std::ostream &out) { } } } - - out << ""; } if (link) { - out << ""; + out << ""; + out << ""; if (link && (link->is_explicit() == true)) { // indices have been given via index hack in source file: link->to_dot_overlayindices(out, false); diff --git a/src/symbol.cc b/src/symbol.cc index eb6225a49..af1aa0745 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -1654,7 +1654,14 @@ unsigned int Symbol::Base::to_dot(unsigned int *nodeID, std::ostream &out, unsigned int thisID = (unsigned int)((*nodeID)++); out << "node_" << thisID << " [ label=<
" << *link->name << "" << *link->name; } Alt::Block *block = dynamic_cast(this); if (block) { - out << "a blocka block"; + } + if (true) { + // if we want to also print out datatypes + out << "
"; + this->datatype->put(out); + out << ""; } + out << "
"; to_dot_indices(this->left_indices, out); - out << ""; + out << ""; to_dot_indices(this->right_indices, out); out << "
" << *this->name << "" << *this->name; + if (true) { + // if we want to also print out datatypes + out << "
"; + this->datatype->put(out); + out << ""; + } + out << "
>"; return thisID; From 615758d221c34ad7f5c8b71c1d14dc9536c52472 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 17 May 2022 15:41:08 +0200 Subject: [PATCH 107/209] print NULL instead of aborting of variable is not set --- src/alt.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index cb10fcaa5..f9a35921d 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3104,8 +3104,11 @@ void to_dot_indices(std::vector indices, std::ostream &out) { out << "		"; for (std::vector::const_iterator track = indices.begin(); track != indices.end(); ++track) { - assert(*track != NULL); - (*track)->put(out); + if (*track == NULL) { + out << "NULL"; + } else { + (*track)->put(out); + } if (std::next(track) != indices.end()) { out << "
"; } @@ -3239,7 +3242,11 @@ unsigned int Alt::Base::to_dot(unsigned int *nodeID, std::ostream &out) { if (true) { // if we want to also print out datatypes out << "
"; - this->datatype->put(out); + if (this->datatype == NULL) { + out << "NULL"; + } else { + this->datatype->put(out); + } out << ""; } out << "		a block"; } if (true) { - // if we want to also print out datatypes + // if we want to also print out datatypes out << "
"; if (this->datatype == NULL) { out << "NULL"; } else { - this->datatype->put(out); + this->datatype->to_dot(out); } out << ""; } diff --git a/src/alt.hh b/src/alt.hh index baa6f88b1..6649305c8 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -369,6 +369,7 @@ class Base { // parent with child Alt::Block (necessary to change pointer when resolving // blocks) virtual Alt::Base *find_block_parent(const Alt::Base &block); + bool inside_end = false; }; diff --git a/src/symbol.cc b/src/symbol.cc index 195413025..1698d2377 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -1661,7 +1661,7 @@ unsigned int Symbol::Base::to_dot(unsigned int *nodeID, std::ostream &out, if (this->datatype == NULL) { out << "NULL"; } else { - this->datatype->put(out); + this->datatype->to_dot(out); } out << ""; } diff --git a/src/symbol.hh b/src/symbol.hh index 972181d39..397ab5c2e 100644 --- a/src/symbol.hh +++ b/src/symbol.hh @@ -345,7 +345,7 @@ class NT : public Base { // rules. There is one pointer stored for each alternative. std::list alts; void set_alts(const std::list &a); - + bool inside_end = false; public: std::string *eval_fn; diff --git a/src/type/base.cc b/src/type/base.cc index e2593db36..8a9ee0810 100644 --- a/src/type/base.cc +++ b/src/type/base.cc @@ -68,3 +68,24 @@ Type::Base *Type::Base::component() { Type::Base *Type::Base::deref() { return this; } + +// replaces from with to in str +void replaceAll(std::string& str, const std::string& from, const std::string& to) { + if(from.empty()) + return; + size_t start_pos = 0; + while((start_pos = str.find(from, start_pos)) != std::string::npos) { + str.replace(start_pos, from.length(), to); + start_pos += to.length(); // In case 'to' contains 'from', like replacing 'x' with 'yx' + } +} + +// graphViz compatible text representation of datatype +void Type::Base::to_dot(std::ostream &out) { + std::ostringstream dtype_stream; + this->put(dtype_stream); + std::string dtype = dtype_stream.str(); + replaceAll(dtype, std::string("<"), std::string("<")); + replaceAll(dtype, std::string(">"), std::string(">")); + out << dtype; +} diff --git a/src/type/base.hh b/src/type/base.hh index 45bb9f84a..93e89afcc 100644 --- a/src/type/base.hh +++ b/src/type/base.hh @@ -25,6 +25,7 @@ #define SRC_TYPE_BASE_HH_ #include +#include #include "../loc.hh" #include "../printer_fwd.hh" @@ -101,6 +102,7 @@ class Base { virtual Base *deref(); virtual void print(Printer::Base &s) const = 0; + void to_dot(std::ostream &out); }; @@ -111,5 +113,7 @@ inline std::ostream & operator<< (std::ostream &s, const Base &p) { } // namespace Type +void replaceAll(std::string& str, const std::string& from, + const std::string& to); #endif // SRC_TYPE_BASE_HH_ From eb0ac67b591a34151d822016195e0495069e8b02 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 20 May 2022 16:06:09 +0200 Subject: [PATCH 111/209] skip outside NT generation for inside NTs that only point to terminals --- src/grammar.cc | 67 +++++++++++++++++++++++++++++++++++++++++++++++--- src/grammar.hh | 4 +++ 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/grammar.cc b/src/grammar.cc index 8071a1424..884e3675c 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1013,6 +1013,43 @@ void Grammar::resolve_blocks() { } } + +void Grammar::flag_inside_terminal_ends() { + // defines the visitor used a few lines later + // Traverses grammar and flags non-terminals that NEITHER link to other + // non-terminals NOR apply algebra functions, e.g. times = CHAR('*') + // These NTs cannot translated into outside versions. + struct Find_Inside_Ends : public Visitor { + void visit_end(Alt::Link &alt) { + alt.inside_end = alt.nt == NULL; + } + void visit_end(Alt::Simple &alt) { + for (std::list::iterator i = alt.args.begin(); + i != alt.args.end(); ++i) { + if (dynamic_cast(*i)) { + alt.inside_end = false; + return; + } + } + alt.inside_end = true; + } + void visit_end(Symbol::NT &nt) { + for (std::list::iterator i = nt.alts.begin(); + i != nt.alts.end(); ++i) { + if ((*i)->inside_end == false) { + nt.inside_end = false; + return; + } + } + nt.inside_end = true; + } + }; + + // create a visitor and traverse the + Find_Inside_Ends visitor; + traverse(visitor); +} + void Grammar::inject_outside_nts() { std::vector outside_nt_list; Grammar::inject_outside_nts(outside_nt_list); @@ -1050,13 +1087,17 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { throw LogError(msg); } + // traverse grammar and flag NTs that only use terminals on their rhs + this->flag_inside_terminal_ends(); + // 1) collect non-terminals used in rhs of productions together with their // calling lhs non-terminal we thus exclude e.g. non-terminal foo in // production "foo = BASE;" for (hashtable::iterator i = NTs.begin(); i != NTs.end(); ++i) { Symbol::NT *nt = dynamic_cast(i->second); - if (nt) { // skip terminals + if ((nt) && (!nt->inside_end)) { + // skip terminals and NTs that only point to terminals for (std::list::iterator alt = nt->alts.begin(); alt != nt->alts.end(); ++alt) { std::list *rhs_nts = new std::list(); @@ -1066,7 +1107,12 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { // non-terminal and all rhs non-terminal(s)) will be added as // novel outside non-terminals outside_nts.insert(nt); - outside_nts.insert(rhs_nts->begin(), rhs_nts->end()); + for (std::list::const_iterator rhs_nt = rhs_nts->begin(); + rhs_nt != rhs_nts->end(); ++rhs_nt) { + if (!(*rhs_nt)->inside_end) { + outside_nts.insert(*rhs_nt); + } + } } } } @@ -1086,6 +1132,14 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { // use inside NT as template ... Symbol::NT *inside_nt = dynamic_cast( this->NTs.find(*(*nt)->name)->second); + + // skip inside NTs that only point to terminals like + // times = CHAR('*') + if (inside_nt->inside_end) { + // inside terminal is NOT transformed into outside NT + continue; + } + // and clone for outside version Symbol::NT *outside_nt = inside_nt->clone(inside_nt->track_pos()); // remove all existing alternative production rules @@ -1106,7 +1160,8 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { for (hashtable::iterator i = NTs.begin(); i != NTs.end(); ++i) { Symbol::NT *nt = dynamic_cast(i->second); - if (nt) { // skip terminals + // skip terminals and NTs that end in terminals + if ((nt) && (!nt->inside_end)) { // don't directly operate on given inside NTs, // but first resolve Alt::Block use for outside rules Symbol::NT *nt_resolved = nt->clone(nt->track_pos()); @@ -1124,6 +1179,12 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { // unsigned int skip_occurences = 0; for (std::list::iterator it_nt = rhs_nts->begin(); it_nt != rhs_nts->end(); ++it_nt) { + // skip inside NTs that only point to terminals like + // times = CHAR('*') + if ((*it_nt)->inside_end) { + continue; + } + // create a copy (=clone) of the inside alternative Alt::Base *outside_alt = (*alt)->clone(); // replace one of the inside non-terminals with the outside diff --git a/src/grammar.hh b/src/grammar.hh index 718245e70..9b6d2741f 100644 --- a/src/grammar.hh +++ b/src/grammar.hh @@ -213,6 +213,10 @@ class Grammar { void multi_propagate_max_filter(); + // traverse grammar and flag those NTs (and production rules) + // that only use terminal parsers, because these shall not be + // translated into outside NTs, e.g. times = CHAR('*') + void flag_inside_terminal_ends(); // inject additional non-terminals to grammar, to automatically create // an outside version for user provided grammar void inject_outside_nts(std::vector outside_nt_list); From e43aa76ec5b338948c62f9e24a133de06581a82c Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 20 May 2022 16:41:08 +0200 Subject: [PATCH 112/209] exit != 0 if signature check fails --- testdata/modtest/outside_indices.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testdata/modtest/outside_indices.cc b/testdata/modtest/outside_indices.cc index d9e7cd0d7..e081bf8f7 100644 --- a/testdata/modtest/outside_indices.cc +++ b/testdata/modtest/outside_indices.cc @@ -51,7 +51,7 @@ int main(int argc, char **argv) { r = driver.ast.check_signature(); if (!r) { - throw LogError("Seen signature errors."); + return 4; } // apply this to identify standard functions like Min, Max, Exp etc. From 06b2ef56a1fc813a052a0abc5f2cf50a25fca93d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 20 May 2022 16:49:49 +0200 Subject: [PATCH 113/209] also test grammars from existing dir --- testdata/grammar_outside/adpf.gap | 1 + testdata/grammar_outside/adpf.new | 1 + testdata/grammar_outside/adpf_hl.gap | 1 + testdata/grammar_outside/adpf_index.gap | 1 + testdata/grammar_outside/adpf_multi.gap | 1 + testdata/grammar_outside/adpf_nonamb.gap | 1 + testdata/grammar_outside/elm_filter.gap | 1 + testdata/grammar_outside/g3pl.gap | 1 + testdata/grammar_outside/helene.gap | 1 + testdata/grammar_outside/loco3stem.gap | 1 + testdata/grammar_outside/pknotsRG.gap | 1 + testdata/grammar_outside/rnashapes2wip.gap | 1 + testdata/grammar_outside/rnashapesmfe.gap | 1 + testdata/grammar_outside/rnashapespf.gap | 1 + testdata/grammar_outside/single_block.gap | 1 + testdata/grammar_outside/stefan3.gap | 1 + testdata/grammar_outside/stefan4.gap | 1 + testdata/grammar_outside/tdm2hp.gap | 1 + testdata/grammar_outside/tdm2hporig.gap | 1 + 19 files changed, 19 insertions(+) create mode 120000 testdata/grammar_outside/adpf.gap create mode 120000 testdata/grammar_outside/adpf.new create mode 120000 testdata/grammar_outside/adpf_hl.gap create mode 120000 testdata/grammar_outside/adpf_index.gap create mode 120000 testdata/grammar_outside/adpf_multi.gap create mode 120000 testdata/grammar_outside/adpf_nonamb.gap create mode 120000 testdata/grammar_outside/elm_filter.gap create mode 120000 testdata/grammar_outside/g3pl.gap create mode 120000 testdata/grammar_outside/helene.gap create mode 120000 testdata/grammar_outside/loco3stem.gap create mode 120000 testdata/grammar_outside/pknotsRG.gap create mode 120000 testdata/grammar_outside/rnashapes2wip.gap create mode 120000 testdata/grammar_outside/rnashapesmfe.gap create mode 120000 testdata/grammar_outside/rnashapespf.gap create mode 120000 testdata/grammar_outside/single_block.gap create mode 120000 testdata/grammar_outside/stefan3.gap create mode 120000 testdata/grammar_outside/stefan4.gap create mode 120000 testdata/grammar_outside/tdm2hp.gap create mode 120000 testdata/grammar_outside/tdm2hporig.gap diff --git a/testdata/grammar_outside/adpf.gap b/testdata/grammar_outside/adpf.gap new file mode 120000 index 000000000..72d0590a9 --- /dev/null +++ b/testdata/grammar_outside/adpf.gap @@ -0,0 +1 @@ +../grammar/adpf.gap \ No newline at end of file diff --git a/testdata/grammar_outside/adpf.new b/testdata/grammar_outside/adpf.new new file mode 120000 index 000000000..20e9e812a --- /dev/null +++ b/testdata/grammar_outside/adpf.new @@ -0,0 +1 @@ +../grammar/adpf.new \ No newline at end of file diff --git a/testdata/grammar_outside/adpf_hl.gap b/testdata/grammar_outside/adpf_hl.gap new file mode 120000 index 000000000..90e228caf --- /dev/null +++ b/testdata/grammar_outside/adpf_hl.gap @@ -0,0 +1 @@ +../grammar/adpf_hl.gap \ No newline at end of file diff --git a/testdata/grammar_outside/adpf_index.gap b/testdata/grammar_outside/adpf_index.gap new file mode 120000 index 000000000..fc3c81715 --- /dev/null +++ b/testdata/grammar_outside/adpf_index.gap @@ -0,0 +1 @@ +../grammar/adpf_index.gap \ No newline at end of file diff --git a/testdata/grammar_outside/adpf_multi.gap b/testdata/grammar_outside/adpf_multi.gap new file mode 120000 index 000000000..1f4d881ea --- /dev/null +++ b/testdata/grammar_outside/adpf_multi.gap @@ -0,0 +1 @@ +../grammar/adpf_multi.gap \ No newline at end of file diff --git a/testdata/grammar_outside/adpf_nonamb.gap b/testdata/grammar_outside/adpf_nonamb.gap new file mode 120000 index 000000000..9aea05e15 --- /dev/null +++ b/testdata/grammar_outside/adpf_nonamb.gap @@ -0,0 +1 @@ +../grammar/adpf_nonamb.gap \ No newline at end of file diff --git a/testdata/grammar_outside/elm_filter.gap b/testdata/grammar_outside/elm_filter.gap new file mode 120000 index 000000000..468e184e9 --- /dev/null +++ b/testdata/grammar_outside/elm_filter.gap @@ -0,0 +1 @@ +../grammar/elm_filter.gap \ No newline at end of file diff --git a/testdata/grammar_outside/g3pl.gap b/testdata/grammar_outside/g3pl.gap new file mode 120000 index 000000000..f8dda6785 --- /dev/null +++ b/testdata/grammar_outside/g3pl.gap @@ -0,0 +1 @@ +../grammar/g3pl.gap \ No newline at end of file diff --git a/testdata/grammar_outside/helene.gap b/testdata/grammar_outside/helene.gap new file mode 120000 index 000000000..69eca1c62 --- /dev/null +++ b/testdata/grammar_outside/helene.gap @@ -0,0 +1 @@ +../grammar/helene.gap \ No newline at end of file diff --git a/testdata/grammar_outside/loco3stem.gap b/testdata/grammar_outside/loco3stem.gap new file mode 120000 index 000000000..405777244 --- /dev/null +++ b/testdata/grammar_outside/loco3stem.gap @@ -0,0 +1 @@ +../grammar/loco3stem.gap \ No newline at end of file diff --git a/testdata/grammar_outside/pknotsRG.gap b/testdata/grammar_outside/pknotsRG.gap new file mode 120000 index 000000000..f0c8dfb7c --- /dev/null +++ b/testdata/grammar_outside/pknotsRG.gap @@ -0,0 +1 @@ +../grammar/pknotsRG.gap \ No newline at end of file diff --git a/testdata/grammar_outside/rnashapes2wip.gap b/testdata/grammar_outside/rnashapes2wip.gap new file mode 120000 index 000000000..70c8653ce --- /dev/null +++ b/testdata/grammar_outside/rnashapes2wip.gap @@ -0,0 +1 @@ +../grammar/rnashapes2wip.gap \ No newline at end of file diff --git a/testdata/grammar_outside/rnashapesmfe.gap b/testdata/grammar_outside/rnashapesmfe.gap new file mode 120000 index 000000000..e9c947e7e --- /dev/null +++ b/testdata/grammar_outside/rnashapesmfe.gap @@ -0,0 +1 @@ +../grammar/rnashapesmfe.gap \ No newline at end of file diff --git a/testdata/grammar_outside/rnashapespf.gap b/testdata/grammar_outside/rnashapespf.gap new file mode 120000 index 000000000..98b81292a --- /dev/null +++ b/testdata/grammar_outside/rnashapespf.gap @@ -0,0 +1 @@ +../grammar/rnashapespf.gap \ No newline at end of file diff --git a/testdata/grammar_outside/single_block.gap b/testdata/grammar_outside/single_block.gap new file mode 120000 index 000000000..3df269ae9 --- /dev/null +++ b/testdata/grammar_outside/single_block.gap @@ -0,0 +1 @@ +../grammar/single_block.gap \ No newline at end of file diff --git a/testdata/grammar_outside/stefan3.gap b/testdata/grammar_outside/stefan3.gap new file mode 120000 index 000000000..049192085 --- /dev/null +++ b/testdata/grammar_outside/stefan3.gap @@ -0,0 +1 @@ +../grammar/stefan3.gap \ No newline at end of file diff --git a/testdata/grammar_outside/stefan4.gap b/testdata/grammar_outside/stefan4.gap new file mode 120000 index 000000000..2944f1b55 --- /dev/null +++ b/testdata/grammar_outside/stefan4.gap @@ -0,0 +1 @@ +../grammar/stefan4.gap \ No newline at end of file diff --git a/testdata/grammar_outside/tdm2hp.gap b/testdata/grammar_outside/tdm2hp.gap new file mode 120000 index 000000000..90dd67c24 --- /dev/null +++ b/testdata/grammar_outside/tdm2hp.gap @@ -0,0 +1 @@ +../grammar/tdm2hp.gap \ No newline at end of file diff --git a/testdata/grammar_outside/tdm2hporig.gap b/testdata/grammar_outside/tdm2hporig.gap new file mode 120000 index 000000000..a61558bde --- /dev/null +++ b/testdata/grammar_outside/tdm2hporig.gap @@ -0,0 +1 @@ +../grammar/tdm2hporig.gap \ No newline at end of file From 39971b92117163db6a5a603acf393b41d7c4facd Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 20 May 2022 16:50:12 +0200 Subject: [PATCH 114/209] also print datatype of choice function --- src/symbol.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/symbol.cc b/src/symbol.cc index 1698d2377..6f36eb42d 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -1697,8 +1697,18 @@ unsigned int Symbol::NT::to_dot(unsigned int *nodeID, std::ostream &out, } if (this->eval_fn != NULL) { unsigned int choiceID = (unsigned int)((*nodeID)++); - out << "node_" << choiceID << " [ label=" << *this->eval_fn - << ", fontcolor=\"purple\" , shape=none ];\n"; + out << "node_" << choiceID << " [ label=<" << *this->eval_fn; + if (true) { + // if we want to also print out datatypes + out << "
"; + if (this->eval_decl == NULL) { + out << "NULL"; + } else { + this->eval_decl->return_type->to_dot(out); + } + out << ""; + } + out << ">, fontcolor=\"purple\" , shape=none ];\n"; out << "node_" << thisID << " -> node_" << choiceID << " [ arrowhead=none, color=\"purple\" ];\n"; } From f0986b93141cda2edbdfa492943a71fd22c1d37b Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 20 May 2022 16:57:24 +0200 Subject: [PATCH 115/209] codestyle --- src/type/base.cc | 11 +++++++---- src/type/base.hh | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/type/base.cc b/src/type/base.cc index 8a9ee0810..fe27c9fa3 100644 --- a/src/type/base.cc +++ b/src/type/base.cc @@ -24,6 +24,7 @@ #include "base.hh" #include +#include Type::Base::~Base() {} @@ -70,13 +71,15 @@ Type::Base *Type::Base::deref() { } // replaces from with to in str -void replaceAll(std::string& str, const std::string& from, const std::string& to) { - if(from.empty()) +void replaceAll(std::string& str, const std::string& from, + const std::string& to) { + if (from.empty()) return; size_t start_pos = 0; - while((start_pos = str.find(from, start_pos)) != std::string::npos) { + while ((start_pos = str.find(from, start_pos)) != std::string::npos) { str.replace(start_pos, from.length(), to); - start_pos += to.length(); // In case 'to' contains 'from', like replacing 'x' with 'yx' + // In case 'to' contains 'from', like replacing 'x' with 'yx' + start_pos += to.length(); } } diff --git a/src/type/base.hh b/src/type/base.hh index 93e89afcc..4045a992d 100644 --- a/src/type/base.hh +++ b/src/type/base.hh @@ -26,6 +26,7 @@ #include #include +#include #include "../loc.hh" #include "../printer_fwd.hh" From eb6a3111c4e65eca8110fd0c4d8d17441a621017 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 23 May 2022 12:11:40 +0200 Subject: [PATCH 116/209] some extra magic for openMP in OSX --- testdata/regresstest/config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index f7d30d917..b105dd0e4 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -462,6 +462,10 @@ check_new_old_eq nodangle.gap unused pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outsi GAPC="../../../gapc --outside_grammar weak --outside_grammar struct --cyk" CPPFLAGS_EXTRA="$DEFAULT_CPPFLAGS_EXTRA -fopenmp" LDLIBS_EXTRA="$DEFAULT_LDLIBS_EXTRA -fopenmp" +if [ $(uname) = "Darwin" ]; then + CPPFLAGS_EXTRA="$DEFAULT_CPPFLAGS_EXTRA -Xpreprocessor -fopenmp -lomp " + LDLIBS_EXTRA="$DEFAULT_LDLIBS_EXTRA -Xpreprocessor -fopenmp -lomp -I\"$(brew --prefix libomp)/include\" -L\"$(brew --prefix libomp)/lib\"" +fi OMP_NUM_THREADS=2 export OMP_NUM_THREADS check_new_old_eq nodangle.gap unused pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside From e7ceb65138e5b8add805b35974fe944ed7826815 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 30 May 2022 11:27:45 +0200 Subject: [PATCH 117/209] don't init NULL guards --- src/alt.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index c05b85e74..c14772b5a 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -1967,8 +1967,9 @@ void Alt::Simple::codegen(AST &ast) { push_back_ret_decl(); std::list *stmts = &statements; - - init_guards(); + if (guards) { + init_guards(); + } // if (guards) { // stmts->push_back(guards); // if (datatype->simple()->is(::Type::LIST)) { From 13c8da4d21d1de6b60a975abde261e5ecf5d0ff9 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 30 May 2022 12:38:25 +0200 Subject: [PATCH 118/209] implement case for < 2 dim tables --- src/alt.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index c14772b5a..c933a5d43 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -1554,6 +1554,8 @@ void Alt::Simple::init_guards() { if (lname && (lname->name()->compare(*leftmost->name()) == 0)) { // the rare case where guards check themselves // happens e.g. if table has only one dimension + l.push_back(new Expr::Greater_Eq( + lname->minus(leftmost), new Expr::Const(0))); } else { l.push_back(new Expr::Greater_Eq((*it_left_inner_indices), leftmost->plus(ys->first->low()))); @@ -1568,6 +1570,8 @@ void Alt::Simple::init_guards() { if (rname && (rname->name()->compare(*rightmost->name()) == 0)) { // the rare case where guards check themselves // happens e.g. if table has only one dimension + l.push_back(new Expr::Less_Eq( + rname->minus(rightmost), new Expr::Const(0))); } else { l.push_back(new Expr::Less_Eq( (*it_right_inner_indices)->plus(ys->second->low()), rightmost)); @@ -1967,9 +1971,7 @@ void Alt::Simple::codegen(AST &ast) { push_back_ret_decl(); std::list *stmts = &statements; - if (guards) { - init_guards(); - } + init_guards(); // if (guards) { // stmts->push_back(guards); // if (datatype->simple()->is(::Type::LIST)) { From a17c53fd9fa821674a6f81aab1062cf29e5c5dc9 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 30 May 2022 14:23:34 +0200 Subject: [PATCH 119/209] use NT name to avoid name clashes for multiple constant NTs --- src/cpp.cc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 1e0d98664..8ff685a55 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2214,12 +2214,7 @@ void Printer::Cpp::print_insideoutside( stream << indent() << "out << \"start answers " << *nt->name << "(\"" << args_print.str() << " << \"):\\n\";\n"; for (std::list::iterator i = l.begin(); i != l.end(); ++i) { - std::string res = "res_"; - if (nt->is_partof_outside) { - res += std::string("outside"); - } else { - res += std::string("inside"); - } + std::string res = "res_" + *nt->name; stream << indent() << *((*i)->return_type) << " " << res << " = nt_" << *nt->name << "(" << args.str() << ");\n"; stream << indent() << "print_result(std::cout, " << res << ");\n"; From 15a84aaf136c0664f681ceb47b1ceb210ba0e303 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 1 Jun 2022 09:51:56 +0200 Subject: [PATCH 120/209] consider table dimensions for outside report + in/out not necc. identical --- src/cpp.cc | 79 +++++++++++++++++++++++++++++++++--------------------- src/cpp.hh | 2 +- 2 files changed, 49 insertions(+), 32 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 8ff685a55..ca76b0af6 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2192,20 +2192,20 @@ void Printer::Cpp::print_cyk_fn(const AST &ast) { } void Printer::Cpp::print_insideoutside( - Symbol::NT *nt) { + Symbol::NT *nt, unsigned int dim) { std::ostringstream args; std::ostringstream args_print; std::list &l = nt->code_list(); for (size_t track = 0; track < (*nt).tracks(); ++track) { - if ((*nt).tables()[track].type() == Table::QUADRATIC) { - args << "t_" << track << "_i,t_" << track << "_j"; - args_print << " << t_" << track << "_i << \",\" << t_" << track << "_j"; - } - if ((*nt).tables()[track].type() == Table::LINEAR) { + if (dim >= 1) { args << "t_" << track << "_i"; args_print << " << t_" << track << "_i"; } + if (dim >= 2) { + args << ",t_" << track << "_j"; + args_print << " << \",\" << t_" << track << "_j"; + } if (track+1 < (*nt).tracks()) { args << ","; args_print << " << \",\""; @@ -2268,38 +2268,55 @@ void Printer::Cpp::print_insideoutside_report_fn( dynamic_cast((*outside_ntpair).second); for (size_t track = 0; track < (*nt_inside)->tracks(); ++track) { - if ((*nt_inside)->tables()[track].type() == Table::QUADRATIC) { - stream << indent() << "for (unsigned int t_" << track << "_i = t_" - << track << "_left_most; t_" << track << "_i <= t_" << track - << "_right_most; ++t_" << track << "_i) {" << endl; - inc_indent(); - stream << indent() << "for (unsigned int t_" << track << "_j = t_" - << track << "_i; t_" << track << "_j <= t_" << track - << "_right_most; ++t_" << track << "_j) {" << endl; - inc_indent(); + unsigned int dim_inside = 2; + if ((*nt_inside)->tables()[track].delete_left_index()) { + dim_inside--; } - if ((*nt_inside)->tables()[track].type() == Table::LINEAR) { - stream << indent() << "for (unsigned int t_" << track << "_i = t_" - << track << "_left_most; t_" << track << "_i <= t_" << track - << "_right_most; ++t_" << track << "_i) {" - << endl; - inc_indent(); + if ((*nt_inside)->tables()[track].delete_right_index()) { + dim_inside--; } - } - print_insideoutside(*nt_inside); - print_insideoutside(nt_outside); + unsigned int dim_outside = 2; + if (nt_outside->tables()[track].delete_left_index()) { + dim_outside--; + } + if (nt_outside->tables()[track].delete_right_index()) { + dim_outside--; + } - for (size_t track = 0; track < (*nt_inside)->tracks(); ++track) { - if ((*nt_inside)->tables()[track].type() == Table::QUADRATIC) { - dec_indent(); - stream << indent() << "}" << endl; - dec_indent(); - stream << indent() << "}" << endl; + if (dim_inside == 0) { + print_insideoutside(*nt_inside, dim_inside); + } + if (dim_outside == 0) { + print_insideoutside(nt_outside, dim_outside); } - if ((*nt_inside)->tables()[track].type() == Table::LINEAR) { + if ((dim_inside > 0) || (dim_outside > 0)) { + stream << indent() << "for (unsigned int t_" << track << "_i = t_" + << track << "_left_most; t_" << track << "_i <= t_" << track + << "_right_most; ++t_" << track << "_i) {" << endl; + inc_indent(); + if (dim_inside == 1) { + print_insideoutside(*nt_inside, dim_inside); + } + if (dim_outside == 1) { + print_insideoutside(nt_outside, dim_outside); + } + if ((dim_inside == 2) || (dim_outside == 2)) { + stream << indent() << "for (unsigned int t_" << track << "_j = t_" + << track << "_i; t_" << track << "_j <= t_" << track + << "_right_most; ++t_" << track << "_j) {" << endl; + inc_indent(); + if (dim_inside == 2) { + print_insideoutside(*nt_inside, dim_inside); + } + if (dim_outside == 2) { + print_insideoutside(nt_outside, dim_outside); + } dec_indent(); stream << indent() << "}" << endl; + } + dec_indent(); + stream << indent() << "}" << endl; } } } diff --git a/src/cpp.hh b/src/cpp.hh index 824bc0b2b..86eae0eeb 100644 --- a/src/cpp.hh +++ b/src/cpp.hh @@ -138,7 +138,7 @@ class Cpp : public Base { std::vector outside_nt_list, const AST &ast); private: - void print_insideoutside(Symbol::NT *nt); + void print_insideoutside(Symbol::NT *nt, unsigned int dim); void print_run_fn(const AST &ast); void print_stats_fn(const AST &ast); From 122bcd71f0f8b05f13b73b18bd87a8c221107382 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 1 Jun 2022 09:54:55 +0200 Subject: [PATCH 121/209] also run actions for this branch --- .github/workflows/outside.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/outside.yml b/.github/workflows/outside.yml index 930433082..739a0134b 100644 --- a/.github/workflows/outside.yml +++ b/.github/workflows/outside.yml @@ -2,9 +2,9 @@ name: test outside injection on: push: - branches: [ master ] + branches: [ master, outside_looporder ] pull_request: - branches: [ master ] + branches: [ master, outside_looporder ] workflow_dispatch: inputs: logLevel: From 82fc8c2a64a00dec2f6ade6a0cf4a3668a5d95dc Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 1 Jun 2022 09:56:28 +0200 Subject: [PATCH 122/209] limit to PRs --- .github/workflows/outside.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/outside.yml b/.github/workflows/outside.yml index 739a0134b..168daa139 100644 --- a/.github/workflows/outside.yml +++ b/.github/workflows/outside.yml @@ -2,7 +2,7 @@ name: test outside injection on: push: - branches: [ master, outside_looporder ] + branches: [ master ] pull_request: branches: [ master, outside_looporder ] workflow_dispatch: From f4b8048f01119c0ab56a46594c709127218dc841 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 1 Jun 2022 09:58:53 +0200 Subject: [PATCH 123/209] same here --- .github/workflows/c-cpp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml index 2d821fbed..030d71811 100644 --- a/.github/workflows/c-cpp.yml +++ b/.github/workflows/c-cpp.yml @@ -4,7 +4,7 @@ on: push: branches: [ master ] pull_request: - branches: [ master ] + branches: [ master, outside_looporder ] workflow_dispatch: inputs: logLevel: From af2d55d3dab7684301193ac8e350e3896f8a4f96 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 2 Jun 2022 13:04:41 +0200 Subject: [PATCH 124/209] adding HMM regress tests --- testdata/grammar_outside/hmm_sonneregen.gap | 250 ++++++++++++++++++++ testdata/regresstest/config | 8 + 2 files changed, 258 insertions(+) create mode 100644 testdata/grammar_outside/hmm_sonneregen.gap diff --git a/testdata/grammar_outside/hmm_sonneregen.gap b/testdata/grammar_outside/hmm_sonneregen.gap new file mode 100644 index 000000000..db382d303 --- /dev/null +++ b/testdata/grammar_outside/hmm_sonneregen.gap @@ -0,0 +1,250 @@ +type Rope = extern + +signature sig_weather(alphabet, answer) { + answer transition_start_hoch(float, answer, answer); + answer transition_start_tief(float, answer, answer); + answer transition_hoch_tief(float, answer, answer); + answer transition_hoch_hoch(float, answer, answer); + answer transition_tief_tief(float, answer, answer); + answer transition_tief_hoch(float, answer, answer); + + answer emission_hoch_sonne(float, alphabet); + answer emission_hoch_regen(float, alphabet); + answer emission_tief_sonne(float, alphabet); + answer emission_tief_regen(float, alphabet); + answer nil(void); + + choice [answer] h([answer]); +} + +algebra alg_enum auto enum; +algebra alg_count auto count; + +algebra alg_viterbi implements sig_weather(alphabet=char, answer=float) { + float transition_start_hoch(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_start_tief(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_hoch_tief(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_hoch_hoch(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_tief_tief(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_tief_hoch(float transition, float emission, float x) { + return transition * emission * x; + } + + float emission_hoch_sonne(float emission, char a) { + return emission; + } + float emission_hoch_regen(float emission, char a) { + return emission; + } + float emission_tief_sonne(float emission, char a) { + return emission; + } + float emission_tief_regen(float emission, char a) { + return emission; + } + + float nil(void) { + return 1.0; + } + choice [float] h([float] candidates) { + return list(maximum(candidates)); + } +} +algebra alg_fwd extends alg_viterbi { + choice [float] h([float] candidates) { + return list(sum(candidates)); + } +} + +algebra alg_states implements sig_weather(alphabet=char, answer=Rope) { + Rope transition_start_hoch(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'H'); + append(res, x); + return res; + } + Rope transition_start_tief(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'T'); + append(res, x); + return res; + } + Rope transition_hoch_tief(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'T'); + append(res, x); + return res; + } + Rope transition_hoch_hoch(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'H'); + append(res, x); + return res; + } + Rope transition_tief_tief(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'T'); + append(res, x); + return res; + } + Rope transition_tief_hoch(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'H'); + append(res, x); + return res; + } + + Rope emission_hoch_sonne(float emission, char a) { + Rope res; + return res; + } + Rope emission_hoch_regen(float emission, char a) { + Rope res; + return res; + } + Rope emission_tief_sonne(float emission, char a) { + Rope res; + return res; + } + Rope emission_tief_regen(float emission, char a) { + Rope res; + return res; + } + + Rope nil(void) { + Rope res; + return res; + } + choice [Rope] h([Rope] candidates) { + return unique(candidates); + } +} + +algebra alg_mult implements sig_weather(alphabet=char, answer=Rope) { + Rope transition_start_hoch(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_start_tief(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_hoch_tief(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_hoch_hoch(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_tief_tief(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_tief_hoch(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + + Rope emission_hoch_sonne(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope emission_hoch_regen(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope emission_tief_sonne(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope emission_tief_regen(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + + Rope nil(void) { + Rope res; + append(res, 1.0); + return res; + } + choice [Rope] h([Rope] candidates) { + return unique(candidates); + } +} + + +grammar gra_weather uses sig_weather(axiom=start) { + start = transition_start_hoch(CONST_FLOAT(0.5), hoch_emission, hoch) + | transition_start_tief(CONST_FLOAT(0.5), tief_emission, tief) + | nil(EMPTY) + # h; + + hoch = transition_hoch_tief(CONST_FLOAT(0.3), tief_emission, tief) + | transition_hoch_hoch(CONST_FLOAT(0.7), hoch_emission, hoch) + | nil(EMPTY) + # h; + + tief = transition_tief_tief(CONST_FLOAT(0.6), tief_emission, tief) + | transition_tief_hoch(CONST_FLOAT(0.4), hoch_emission, hoch) + | nil(EMPTY) + # h; + + hoch_emission = emission_hoch_sonne(CONST_FLOAT(0.8), CHAR('S')) + | emission_hoch_regen(CONST_FLOAT(0.2), CHAR('R')) + # h; + + tief_emission = emission_tief_sonne(CONST_FLOAT(0.1), CHAR('S')) + | emission_tief_regen(CONST_FLOAT(0.9), CHAR('R')) + # h; +} + +instance enum = gra_weather(alg_enum); +instance viterbistatesmult = gra_weather(alg_viterbi * alg_states * alg_mult); +instance fwd = gra_weather(alg_fwd); +instance multviterbistates = gra_weather(alg_mult * alg_viterbi * alg_states); +instance count = gra_weather(alg_count); diff --git a/testdata/regresstest/config b/testdata/regresstest/config index b105dd0e4..61c406fae 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -469,3 +469,11 @@ fi OMP_NUM_THREADS=2 export OMP_NUM_THREADS check_new_old_eq nodangle.gap unused pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside + + +GAPC="../../../gapc --outside_grammar ALL" +CPPFLAGS_EXTRA="" +LDLIBS_EXTRA="" +check_new_old_eq hmm_sonneregen.gap unused count "SS" outside +check_new_old_eq hmm_sonneregen.gap unused fwd "SS" outside + From 0b110a4c13c695072f52b259b27a21313f1814a1 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 10 Jun 2022 11:13:07 +0200 Subject: [PATCH 125/209] change names of HMM non-terminals --- testdata/grammar_outside/hmm_sonneregen.gap | 47 +++++++++++---------- testdata/regresstest/config | 1 + 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/testdata/grammar_outside/hmm_sonneregen.gap b/testdata/grammar_outside/hmm_sonneregen.gap index db382d303..8821c9f8d 100644 --- a/testdata/grammar_outside/hmm_sonneregen.gap +++ b/testdata/grammar_outside/hmm_sonneregen.gap @@ -218,29 +218,30 @@ algebra alg_mult implements sig_weather(alphabet=char, answer=Rope) { } -grammar gra_weather uses sig_weather(axiom=start) { - start = transition_start_hoch(CONST_FLOAT(0.5), hoch_emission, hoch) - | transition_start_tief(CONST_FLOAT(0.5), tief_emission, tief) - | nil(EMPTY) - # h; - - hoch = transition_hoch_tief(CONST_FLOAT(0.3), tief_emission, tief) - | transition_hoch_hoch(CONST_FLOAT(0.7), hoch_emission, hoch) - | nil(EMPTY) - # h; - - tief = transition_tief_tief(CONST_FLOAT(0.6), tief_emission, tief) - | transition_tief_hoch(CONST_FLOAT(0.4), hoch_emission, hoch) - | nil(EMPTY) - # h; - - hoch_emission = emission_hoch_sonne(CONST_FLOAT(0.8), CHAR('S')) - | emission_hoch_regen(CONST_FLOAT(0.2), CHAR('R')) - # h; - - tief_emission = emission_tief_sonne(CONST_FLOAT(0.1), CHAR('S')) - | emission_tief_regen(CONST_FLOAT(0.9), CHAR('R')) - # h; +grammar gra_weather uses sig_weather(axiom=state_start) { + state_start = transition_start_hoch(CONST_FLOAT(0.5), emit_hoch, state_hoch) + | transition_start_tief(CONST_FLOAT(0.5), emit_tief, state_tief) + | nil(EMPTY) + # h; + + state_hoch = transition_hoch_tief(CONST_FLOAT(0.3), emit_tief, state_tief) + | transition_hoch_hoch(CONST_FLOAT(0.7), emit_hoch, state_hoch) + | nil(EMPTY) + # h; + + state_tief = transition_tief_tief(CONST_FLOAT(0.6), emit_tief, state_tief) + | transition_tief_hoch(CONST_FLOAT(0.4), emit_hoch, state_hoch) + | nil(EMPTY) + # h; + + + emit_hoch = emission_hoch_sonne(CONST_FLOAT(0.8), CHAR('S')) + | emission_hoch_regen(CONST_FLOAT(0.2), CHAR('R')) + # h; + + emit_tief = emission_tief_sonne(CONST_FLOAT(0.1), CHAR('S')) + | emission_tief_regen(CONST_FLOAT(0.9), CHAR('R')) + # h; } instance enum = gra_weather(alg_enum); diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 61c406fae..8610f96db 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -476,4 +476,5 @@ CPPFLAGS_EXTRA="" LDLIBS_EXTRA="" check_new_old_eq hmm_sonneregen.gap unused count "SS" outside check_new_old_eq hmm_sonneregen.gap unused fwd "SS" outside +check_new_old_eq hmm_sonneregen.gap unused fwd "SSRR" outside_ssrr From fa52b3fa0fc8c805daa48e34ff2ff3370016e567 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 10 Jun 2022 17:33:23 +0200 Subject: [PATCH 126/209] raise warning if inside grammar's yield size min is > 0 --- src/grammar.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/grammar.cc b/src/grammar.cc index 884e3675c..6ae1eb957 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1087,6 +1087,23 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { throw LogError(msg); } + // 0a) semantic check: grammar must be able to parse the empty input in + // order to provide recursion base for outside candidates + Yield::Multi ys = this->axiom->multi_ys(); + for (std::vector::iterator i = ys.begin(); i != ys.end(); ++i) { + if ((*i).low() > 0) { + std::ostringstream msg; + msg << "Minimal yield size of your grammar is "; + (*i).low().put(msg); + msg << ", i.e. it cannot parse the empty input\nstring ''." + << " For outside grammar generation, this means you are lacking a\n" + << "recursion basis which will result in empty results for\n" + << "outside candidates!"; + Log::instance()->warning(msg.str()); + break; + } + } + // traverse grammar and flag NTs that only use terminals on their rhs this->flag_inside_terminal_ends(); From 07d2ccc86cf8207760b77153301a94e19ebc27ef Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Jun 2022 08:53:12 +0200 Subject: [PATCH 127/209] adding alignments as 2 track example --- testdata/grammar_outside/alignments.gap | 112 ++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 testdata/grammar_outside/alignments.gap diff --git a/testdata/grammar_outside/alignments.gap b/testdata/grammar_outside/alignments.gap new file mode 100644 index 000000000..415dd038f --- /dev/null +++ b/testdata/grammar_outside/alignments.gap @@ -0,0 +1,112 @@ +input + +signature sig_alignments(alphabet, answer) { + answer Ins(, , answer); + answer Del(, , answer); + answer Ers(, , answer); + answer Sto(); + + answer Region(, answer, ); + answer Region_Pr(, answer, ); + answer Region_Pr_Pr(, answer, ); + + answer Insx(, , answer); + answer Delx(, , answer); + + choice [answer] h([answer]); +} + +algebra alg_enum auto enum; +algebra alg_count auto count; + +algebra alg_similarity implements sig_alignments(alphabet=char, answer=int) { + int Ins(, , int x) { + return x -3; + } + int Del(, , int x) { + return x -2; + } + int Ers(, , int x) { + if (a == b) { + return x +1; + } else { + return x -1; + } + } + int Sto() { + return 0; + } + + int Region(, int x, ) { + return x; + } + int Region_Pr(, int x, ) { + return x; + } + int Region_Pr_Pr(, int x, ) { + return x; + } + + // this is slightly different form http://rna.informatik.uni-freiburg.de/Teaching/index.jsp?toolName=Gotoh# + // as there Ins + Insx is computed for first blank, we here score Ins for first blank and Insx for all following ones + int Insx(, , int x) { + return x -1; + } + int Delx(, , int x) { + return x -1; + } + + choice [int] h([int] candidates) { + return list(maximum(candidates)); + } +} + + +algebra alg_countmanual implements sig_alignments(alphabet=char, answer=int) { + int Ins(, , int x) { + return x; + } + int Del(, , int x) { + return x; + } + int Ers(, , int x) { + return x; + } + int Sto() { + return 1; + } + + int Region(, int x, ) { + return x; + } + int Region_Pr(, int x, ) { + return x; + } + int Region_Pr_Pr(, int x, ) { + return x; + } + + int Insx(, , int x) { + return x; + } + int Delx(, , int x) { + return x; + } + + choice [int] h([int] candidates) { + return list(sum(candidates)); + } +} + + +// pair-wise global alignment +grammar gra_needlemanwunsch uses sig_alignments(axiom=A) { + A = Ins(, , A) + | Del(, , A) + | Ers(, , A) + | Sto() + # h; +} + + +instance count = gra_needlemanwunsch(alg_count); From de42ec993586e67ec3362e6fec391dbdf03821f5 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Jun 2022 08:54:21 +0200 Subject: [PATCH 128/209] adding test --- testdata/regresstest/config | 1 + 1 file changed, 1 insertion(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 8610f96db..95f92eb81 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -478,3 +478,4 @@ check_new_old_eq hmm_sonneregen.gap unused count "SS" outside check_new_old_eq hmm_sonneregen.gap unused fwd "SS" outside check_new_old_eq hmm_sonneregen.gap unused fwd "SSRR" outside_ssrr +check_new_old_eq alignments.gap unused count "AAAA" outside_2track "BBB" From 6696b6174929153852dea689d2f36d9265846b6a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Jun 2022 08:54:32 +0200 Subject: [PATCH 129/209] using Rope --- testdata/grammar_outside/alignments.gap | 1 + 1 file changed, 1 insertion(+) diff --git a/testdata/grammar_outside/alignments.gap b/testdata/grammar_outside/alignments.gap index 415dd038f..675de892e 100644 --- a/testdata/grammar_outside/alignments.gap +++ b/testdata/grammar_outside/alignments.gap @@ -1,4 +1,5 @@ input +type Rope = extern signature sig_alignments(alphabet, answer) { answer Ins(, , answer); From 96324e793ec74cbcd351971f716db269028f6efb Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Jun 2022 14:05:15 +0200 Subject: [PATCH 130/209] separat loops for inside and outside NTs --- src/cpp.cc | 124 ++++++++---------- src/cpp.hh | 2 +- testdata/grammar_outside/tmhmm_debug.gap | 159 +++++++++++++++++++++++ 3 files changed, 212 insertions(+), 73 deletions(-) create mode 100644 testdata/grammar_outside/tmhmm_debug.gap diff --git a/src/cpp.cc b/src/cpp.cc index ca76b0af6..d69e32627 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2191,38 +2191,69 @@ void Printer::Cpp::print_cyk_fn(const AST &ast) { stream << endl; } -void Printer::Cpp::print_insideoutside( - Symbol::NT *nt, unsigned int dim) { - std::ostringstream args; - std::ostringstream args_print; - std::list &l = nt->code_list(); - - for (size_t track = 0; track < (*nt).tracks(); ++track) { +void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { + // aggregated level (=dim + tracks) of nested loops + unsigned int nesting = 0; + std::vector *args = new std::vector(); + // opening for loops + for (size_t track = 0; track < nt->tracks(); ++track) { + unsigned int dim = 2; + if (nt->tables()[track].delete_left_index()) { + dim--; + } + if (nt->tables()[track].delete_right_index()) { + dim--; + } if (dim >= 1) { - args << "t_" << track << "_i"; - args_print << " << t_" << track << "_i"; + stream << indent() << "for (unsigned int t_" << track << "_i = t_" + << track << "_left_most; t_" << track << "_i <= t_" << track + << "_right_most; ++t_" << track << "_i) {" << endl; + inc_indent(); + args->push_back("t_" + std::to_string(track) + "_i"); } if (dim >= 2) { - args << ",t_" << track << "_j"; - args_print << " << \",\" << t_" << track << "_j"; + stream << indent() << "for (unsigned int t_" << track << "_j = t_" + << track << "_i; t_" << track << "_j <= t_" << track + << "_right_most; ++t_" << track << "_j) {" << endl; + inc_indent(); + args->push_back("t_" + std::to_string(track) + "_j"); } - if (track+1 < (*nt).tracks()) { - args << ","; - args_print << " << \",\""; + nesting += dim; + } + + // loop body + std::list &l = nt->code_list(); + std::stringstream list_args; + std::stringstream list_args_print; + bool first = true; + for (std::vector::const_iterator a = args->begin(); + a != args->end(); ++a) { + if (!first) { + list_args << ", "; + list_args_print << " << \",\""; } + first = false; + list_args << *a; + list_args_print << " << " << *a; } stream << indent() << "out << \"start answers " << *nt->name - << "(\"" << args_print.str() << " << \"):\\n\";\n"; + << "(\"" << list_args_print.str() << " << \"):\\n\";\n"; for (std::list::iterator i = l.begin(); i != l.end(); ++i) { - std::string res = "res_" + *nt->name; + std::string res = "res_" + *(nt->name); stream << indent() << *((*i)->return_type) << " " << res << " = nt_" - << *nt->name << "(" << args.str() << ");\n"; - stream << indent() << "print_result(std::cout, " << res << ");\n"; + << *nt->name << "(" << list_args.str() << ");\n" + << indent() << "print_result(std::cout, " << res << ");\n"; // only for first return statement break; } stream << indent() << "out << \"//end answers " << *nt->name << "(\"" - << args_print.str() << " << \")\\n\";\n"; + << list_args_print.str() << " << \")\\n\";\n"; + + // close loops + for (unsigned int d = 0; d < nesting; ++d) { + dec_indent(); + stream << indent() << "}" << endl; + } } void Printer::Cpp::print_insideoutside_report_fn( std::vector outside_nt_list, const AST &ast) { @@ -2264,61 +2295,10 @@ void Printer::Cpp::print_insideoutside_report_fn( continue; } + print_insideoutside(*nt_inside); Symbol::NT *nt_outside = dynamic_cast((*outside_ntpair).second); - - for (size_t track = 0; track < (*nt_inside)->tracks(); ++track) { - unsigned int dim_inside = 2; - if ((*nt_inside)->tables()[track].delete_left_index()) { - dim_inside--; - } - if ((*nt_inside)->tables()[track].delete_right_index()) { - dim_inside--; - } - - unsigned int dim_outside = 2; - if (nt_outside->tables()[track].delete_left_index()) { - dim_outside--; - } - if (nt_outside->tables()[track].delete_right_index()) { - dim_outside--; - } - - if (dim_inside == 0) { - print_insideoutside(*nt_inside, dim_inside); - } - if (dim_outside == 0) { - print_insideoutside(nt_outside, dim_outside); - } - if ((dim_inside > 0) || (dim_outside > 0)) { - stream << indent() << "for (unsigned int t_" << track << "_i = t_" - << track << "_left_most; t_" << track << "_i <= t_" << track - << "_right_most; ++t_" << track << "_i) {" << endl; - inc_indent(); - if (dim_inside == 1) { - print_insideoutside(*nt_inside, dim_inside); - } - if (dim_outside == 1) { - print_insideoutside(nt_outside, dim_outside); - } - if ((dim_inside == 2) || (dim_outside == 2)) { - stream << indent() << "for (unsigned int t_" << track << "_j = t_" - << track << "_i; t_" << track << "_j <= t_" << track - << "_right_most; ++t_" << track << "_j) {" << endl; - inc_indent(); - if (dim_inside == 2) { - print_insideoutside(*nt_inside, dim_inside); - } - if (dim_outside == 2) { - print_insideoutside(nt_outside, dim_outside); - } - dec_indent(); - stream << indent() << "}" << endl; - } - dec_indent(); - stream << indent() << "}" << endl; - } - } + print_insideoutside(nt_outside); } dec_indent(); stream << indent() << "}" << endl << endl; diff --git a/src/cpp.hh b/src/cpp.hh index 86eae0eeb..824bc0b2b 100644 --- a/src/cpp.hh +++ b/src/cpp.hh @@ -138,7 +138,7 @@ class Cpp : public Base { std::vector outside_nt_list, const AST &ast); private: - void print_insideoutside(Symbol::NT *nt, unsigned int dim); + void print_insideoutside(Symbol::NT *nt); void print_run_fn(const AST &ast); void print_stats_fn(const AST &ast); diff --git a/testdata/grammar_outside/tmhmm_debug.gap b/testdata/grammar_outside/tmhmm_debug.gap new file mode 100644 index 000000000..c3490f46e --- /dev/null +++ b/testdata/grammar_outside/tmhmm_debug.gap @@ -0,0 +1,159 @@ +type Rope = extern + +signature sig_tmhmm(alphabet, answer) { + answer silent_step(answer); + answer step(char, answer, answer); + answer nil(void); + answer trans(float, answer); + answer only(float, alphabet); + choice [answer] h([answer]); +} + +algebra alg_enum auto enum; + +algebra alg_count auto count; + +algebra alg_viterbi implements sig_tmhmm(alphabet=char, answer=float) { + float silent_step(float x) { + return x; + } + float step(char label, float e, float t) { + return e*t; + } + float nil(void) { + return 1; + } + float trans(float prob, float x) { + return prob * x; + } + float only(float prob, char emission) { + return prob; + } + choice [float] h([float] candidates) { + return list(maximum(candidates)); + } +} + +algebra alg_viterbi_bit extends alg_viterbi { + float step(char label, float e, float t) { + return log(1.0/e) + t; + } + float nil(void) { + return 0.0; + } + float trans(float prob, float x) { + return log(1.0/prob) + x; + } + choice [float] h([float] candidates) { + return list(minimum(candidates)); + } +} + +algebra alg_label implements sig_tmhmm(alphabet=char, answer=Rope) { + Rope silent_step(Rope x) { + return x; + } + Rope step(char label, Rope e, Rope t) { + Rope res; + append(res, label); + append(res, t); + return res; + } + Rope nil(void) { + Rope res; + return res; + } + Rope trans(float prob, Rope x) { + return x; + } + Rope only(float prob, char emission) { + Rope res; + return res; + } + choice [Rope] h([Rope] candidates) { + return unique(candidates); + } +} + +grammar gra_tmhmm uses sig_tmhmm(axiom = state_begin) { + state_begin = silent_step(transitions_begin) # h; + transitions_begin = + trans(CONST_FLOAT(0.549251), state_in10) | + trans(CONST_FLOAT(0.207469), state_outglob10) | + trans(CONST_FLOAT(0.24328), state_out10) + # h; + state_in10 = step(CONST_CHAR('i'), emissions_in10, { transitions_in10 | nil(EMPTY) }) # h; + emissions_in10 = + only(CONST_FLOAT(0.0713632), CHAR('A')) | + only(CONST_FLOAT(0.0336358), CHAR('Y')) + # h; + transitions_in10 = + trans(CONST_FLOAT(0.995851), state_in11) | + trans(CONST_FLOAT(0.00111283), state_in29) | + trans(CONST_FLOAT(0.00303586), state_ohelixi1) + # h; + state_in11 = step(CONST_CHAR('i'), emissions_in10, { transitions_in11 | nil(EMPTY) }) # h; + transitions_in11 = + trans(CONST_FLOAT(0.976066), nil(EMPTY)) | + trans(CONST_FLOAT(0.0239339), nil(EMPTY)) | + trans(CONST_FLOAT(1.08323e-09), state_in29) + # h; + state_in29 = step(CONST_CHAR('i'), emissions_in10, { transitions_in29 | nil(EMPTY) }) # h; + transitions_in29 = + trans(CONST_FLOAT(1.0), state_ohelixi1) + # h; + state_ohelixi1 = step(CONST_CHAR('M'), emissions_ohelixi1, transitions_ohelixi1) # h; + emissions_ohelixi1 = + only(CONST_FLOAT(0.0890049), CHAR('A')) | + only(CONST_FLOAT(0.0661077), CHAR('Y')) + # h; + transitions_ohelixi1 = + trans(CONST_FLOAT(1.0), nil(EMPTY)) + # h; + state_outglob10 = step(CONST_CHAR('O'), emissions_outglob10, { transitions_outglob10 | nil(EMPTY) }) # h; + emissions_outglob10 = + only(CONST_FLOAT(0.0693743), CHAR('A')) | + only(CONST_FLOAT(0.0275), CHAR('Y')) + # h; + transitions_outglob10 = + trans(CONST_FLOAT(1.0), state_outglob11) | + trans(CONST_FLOAT(0.0), state_outglob29) | + trans(CONST_FLOAT(0.0), state_ihelixo1) + # h; + state_outglob11 = step(CONST_CHAR('O'), emissions_outglob10, { transitions_outglob11 | nil(EMPTY) }) # h; + transitions_outglob11 = + trans(CONST_FLOAT(1.0), nil(EMPTY)) | + trans(CONST_FLOAT(0.0), nil(EMPTY)) | + trans(CONST_FLOAT(0.0), state_outglob29) + # h; + state_outglob29 = step(CONST_CHAR('O'), emissions_outglob10, { transitions_outglob29 | nil(EMPTY) }) # h; + transitions_outglob29 = + trans(CONST_FLOAT(1.0), state_ihelixo1) + # h; + state_ihelixo1 = step(CONST_CHAR('M'), emissions_out10, transitions_ihelixo1) # h; + transitions_ihelixo1 = + trans(CONST_FLOAT(1.0), nil(EMPTY)) + # h; + state_out10 = step(CONST_CHAR('o'), emissions_out10, { transitions_out10 | nil(EMPTY) }) # h; + emissions_out10 = + only(CONST_FLOAT(0.0690346), CHAR('A')) | + only(CONST_FLOAT(0.0346461), CHAR('Y')) + # h; + transitions_out10 = + trans(CONST_FLOAT(1.0), state_out11) | + trans(CONST_FLOAT(3.54571e-14), state_out29) | + trans(CONST_FLOAT(0.0), state_ihelixo1) + # h; + state_out11 = step(CONST_CHAR('o'), emissions_out10, { transitions_out11 | nil(EMPTY) }) # h; + transitions_out11 = + trans(CONST_FLOAT(0.910217), nil(EMPTY)) | + trans(CONST_FLOAT(0.0495866), nil(EMPTY)) | + trans(CONST_FLOAT(0.0401965), state_out29) + # h; + state_out29 = step(CONST_CHAR('o'), emissions_out10, { transitions_out29 | nil(EMPTY) }) # h; + transitions_out29 = + trans(CONST_FLOAT(1.0), state_ihelixo1) + # h; +} + +instance dummy = gra_tmhmm(alg_enum); From b50fdaa83a2d04fd7827fe7d7a1e5781453820ab Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Jun 2022 14:31:05 +0200 Subject: [PATCH 131/209] new functions for test suit to run two-track inputs --- testdata/regresstest/run.sh | 29 +++++++++++++++++++++++++++++ testdata/tool.sh | 5 +++++ 2 files changed, 34 insertions(+) diff --git a/testdata/regresstest/run.sh b/testdata/regresstest/run.sh index 595c7267c..6ddf1e0e3 100644 --- a/testdata/regresstest/run.sh +++ b/testdata/regresstest/run.sh @@ -107,6 +107,35 @@ check_new_old_eq() rm -f string.o } +check_new_old_eq_twotrack() +{ + if [[ `echo $1$3$5 | grep $FILTER` != $1$3$5 ]]; then + return + fi + + # work around 1 sec timestamp filesystems ... WTF?!? + sleep 1 + + echo +------------------------------------------------------------------------------+ + failed=0 + temp=$failed + + cpp_base=${1%%.*} + build_cpp $GRAMMAR/$1 $cpp_base $3 + run_cpp_twotrack $cpp_base $3 $4 $5 $6 + cmp_new_old_output $cpp_base $REF $3 $5 $6 + + if [ $temp != $failed ]; then + echo --++--FAIL--++-- + err_count=$((err_count+1)) + else + echo OK + succ_count=$((succ_count+1)) + fi + echo +------------------------------------------------------------------------------+ + rm -f string.o +} + run_check_feature() { out=$1.$2.$3.$4 diff --git a/testdata/tool.sh b/testdata/tool.sh index cf5e4f617..63e26c5d0 100644 --- a/testdata/tool.sh +++ b/testdata/tool.sh @@ -27,6 +27,11 @@ run_cpp() log2 $1.$2.$4.out $1.$2.$4.err ./$1 $RUN_CPP_FLAGS $3 } +run_cpp_twotrack() +{ + log2 $1.$2.$4.out $1.$2.$4.err ./$1 $RUN_CPP_FLAGS $3 $5 +} + run_haskell() { log2 $1.$2.$4.hs.out $1.$2.$4.hs.err ./$1 $2 $3 From 7e8136bc2a7aa938c5ec1bc460c9c5f08b2ff202 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Jun 2022 14:31:35 +0200 Subject: [PATCH 132/209] new test for two track --- testdata/regresstest/config | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 95f92eb81..0135d35ed 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -478,4 +478,8 @@ check_new_old_eq hmm_sonneregen.gap unused count "SS" outside check_new_old_eq hmm_sonneregen.gap unused fwd "SS" outside check_new_old_eq hmm_sonneregen.gap unused fwd "SSRR" outside_ssrr -check_new_old_eq alignments.gap unused count "AAAA" outside_2track "BBB" +# example in which inside and outside NTs have different dimensions +check_new_old_eq tmhmm_debug.gap unused count "AA" outside_report + +# example with multiple tracks for outside report function +check_new_old_eq_twotrack alignments.gap unused count "AAAA" outside_2track "BBB" From 5a9ff865749bd6271187d55666e49eab421fefab Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Jun 2022 14:31:53 +0200 Subject: [PATCH 133/209] additional instance for testing --- testdata/grammar_outside/tmhmm_debug.gap | 1 + 1 file changed, 1 insertion(+) diff --git a/testdata/grammar_outside/tmhmm_debug.gap b/testdata/grammar_outside/tmhmm_debug.gap index c3490f46e..ace396b17 100644 --- a/testdata/grammar_outside/tmhmm_debug.gap +++ b/testdata/grammar_outside/tmhmm_debug.gap @@ -157,3 +157,4 @@ grammar gra_tmhmm uses sig_tmhmm(axiom = state_begin) { } instance dummy = gra_tmhmm(alg_enum); +instance count = gra_tmhmm(alg_count); From 5c194d839d533cf1bb917c8c35a98bc7d0b57a46 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Jun 2022 14:41:09 +0200 Subject: [PATCH 134/209] adding failing instance as in issue https://github.com/jlab/gapc/issues/138 --- testdata/grammar_outside/alignments.gap | 1 + 1 file changed, 1 insertion(+) diff --git a/testdata/grammar_outside/alignments.gap b/testdata/grammar_outside/alignments.gap index 675de892e..070c5fc40 100644 --- a/testdata/grammar_outside/alignments.gap +++ b/testdata/grammar_outside/alignments.gap @@ -111,3 +111,4 @@ grammar gra_needlemanwunsch uses sig_alignments(axiom=A) { instance count = gra_needlemanwunsch(alg_count); +instance sim_enum = gra_needlemanwunsch(alg_similarity * alg_enum); From 98e565c3f936504584e10743ee86253a4c199b43 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Jun 2022 14:41:29 +0200 Subject: [PATCH 135/209] reproduce https://github.com/jlab/gapc/issues/138 --- testdata/regresstest/config | 1 + 1 file changed, 1 insertion(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 0135d35ed..97a7f4c8b 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -483,3 +483,4 @@ check_new_old_eq tmhmm_debug.gap unused count "AA" outside_report # example with multiple tracks for outside report function check_new_old_eq_twotrack alignments.gap unused count "AAAA" outside_2track "BBB" +check_new_old_eq_twotrack alignments.gap unused sim_enum "AA" outside_undefvar "B" From cce2a4eb896d86c92cab37b5e5952670cddb740d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Jun 2022 15:55:46 +0200 Subject: [PATCH 136/209] also mind muliti-filters not only filters --- src/symbol.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/symbol.cc b/src/symbol.cc index 6f36eb42d..320c42428 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -944,7 +944,7 @@ void Symbol::NT::set_ret_decl_rhs(Code::Mode mode) { // if the link to another non-terminal is decorated by a filter, // add an "is_not_empty" guard around the addition of potentially // empty list of solutions. Can only be empty due to filtering. - if ((*i)->filters.size() > 0) { + if (((*i)->filters.size() > 0) || ((*i)->multi_filter.size() > 0)) { cond->then.push_back(fn); post_alt_stmts.push_back(cond); // avoid declarations like "ret_1 = answers" if potentially From 5d48b544f2a04be84829f2205c2d629c69334ca7 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 5 Jul 2022 09:30:13 +0200 Subject: [PATCH 137/209] adding plot level to to_dot function call --- testdata/modtest/outside_indices.cc | 2 +- testdata/modtest/outside_resolve_blocks.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testdata/modtest/outside_indices.cc b/testdata/modtest/outside_indices.cc index e081bf8f7..c2715b9db 100644 --- a/testdata/modtest/outside_indices.cc +++ b/testdata/modtest/outside_indices.cc @@ -67,5 +67,5 @@ int main(int argc, char **argv) { grammar->dep_analysis(); unsigned int nodeID = 1; - grammar->to_dot(&nodeID, std::cout); + grammar->to_dot(&nodeID, std::cout, 99); } diff --git a/testdata/modtest/outside_resolve_blocks.cc b/testdata/modtest/outside_resolve_blocks.cc index 911544313..6268b2728 100644 --- a/testdata/modtest/outside_resolve_blocks.cc +++ b/testdata/modtest/outside_resolve_blocks.cc @@ -82,7 +82,7 @@ int main(int argc, char **argv) { grammar->dep_analysis(); unsigned int nodeID = 1; - grammar->to_dot(&nodeID, std::cout); + grammar->to_dot(&nodeID, std::cout, 99); return 0; } From 1e8ce38d1ebe7fb4826d224e309a2762de7d494e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 5 Jul 2022 09:30:33 +0200 Subject: [PATCH 138/209] removing duplicate test --- testdata/modtest/multi_print_grammar.cc | 84 ------------------------- 1 file changed, 84 deletions(-) delete mode 100644 testdata/modtest/multi_print_grammar.cc diff --git a/testdata/modtest/multi_print_grammar.cc b/testdata/modtest/multi_print_grammar.cc deleted file mode 100644 index 28068f35a..000000000 --- a/testdata/modtest/multi_print_grammar.cc +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2022 stefan.m.janssen@gmail.com - -#include -#include -#include -#include - -#include "../../src/driver.hh" -#include "../../src/log.hh" - -int main(int argc, char **argv) { - if (argc != 2) { - std::cerr << "Call " << *argv << " *.gap-file\n"; - return 1; - } - - std::ostringstream o; - Log log; - log.set_debug(false); - log.set_ostream(o); - - Driver driver; - - // === front - // set the file name of the gap-source-code - std::string filename(argv[1]); - driver.setFilename(filename); - - // parses the input file and builds the AST - driver.parse(); - if (driver.is_failing()) { - return 4; - } - - // simply gets the selected grammar, which is either the - // grammar that occurred first in the source code or is the - // one that was named in the parameters on the command line - Grammar *grammar = driver.ast.grammar(); - // Now check the semantic, which does more than the function - // name suggests. The semantic check is embedded in the algorithm - // that links the grammar graph together, and computes yield-sizes. - // If the method returns false, there are some semantic errors, - // which leads to the end of the compilation process. - - // skip semantic check for testing - bool r = grammar->check_semantic(); - if (!r) { - return 2; - } - - // set approx table design - grammar->approx_table_conf(); - - // find what type of input is read - // chars, sequence of ints etc. - driver.ast.derive_temp_alphabet(); - - //log.set_ostream(std::cerr); - try { - r = driver.ast.check_signature(); - if (!r) { - return 3; - } - } catch (LogThreshException) { - return 9; - } - - // apply this to identify standard functions like Min, Max, Exp etc. - driver.ast.derive_roles(); - - - // ------------- back ------------ - grammar->init_list_sizes(); - - grammar->init_indices(); - grammar->init_decls(); - // for cyk (ordering of NT for parsing, see page 101 of the thesis) - grammar->dep_analysis(); - - unsigned int nodeID = 1; - grammar->to_dot(&nodeID, std::cout); - - return 0; -} From 77d3d49fde810843b9f451b524c0dc55143f3cc9 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 29 Sep 2022 21:01:05 +0200 Subject: [PATCH 139/209] yield size must be subtracted (not added) when left of outside_NT --- src/alt.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alt.cc b/src/alt.cc index 9eba4ef52..6689c637c 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3563,7 +3563,7 @@ Expr::Base *Alt::Simple::get_next_var_right2left(Expr::Base *left_index, } if (ys_lefts->low() == ys_lefts->high()) { - return innermost_left_index->plus(ys_lefts->low()); + return innermost_left_index->minus(ys_lefts->low()); } // variable yield size From 90081b23de8062c2f3497279d55d0a076d6972a0 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 29 Sep 2022 21:05:14 +0200 Subject: [PATCH 140/209] I don't think a) that this terminated recursion nor b) should further arguments NOT marked --- src/alt.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 6689c637c..f57fd7f4b 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3438,11 +3438,7 @@ void Alt::Simple::set_partof_outside(bool is_outside) { } void Alt::Link::set_partof_outside(bool is_outside) { if (this->nt) { - if (this->nt->is(Symbol::NONTERMINAL)) { - // end recursion on non-terminal call - } else if (this->nt->is(Symbol::TERMINAL)) { - this->is_partof_outside = is_outside; - } + this->is_partof_outside = is_outside; } else { throw LogError("Link is something else"); } From f63b1d05a5178c77fb67ac43885c45a2e46478b1 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 29 Sep 2022 23:14:39 +0200 Subject: [PATCH 141/209] properly initialize indices for parameterized terminals --- src/alt.cc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/alt.cc b/src/alt.cc index f57fd7f4b..cc70426c9 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3674,6 +3674,21 @@ void Alt::Base::init_indices_outside(Expr::Base *left, Expr::Base *right, void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, unsigned int &k, size_t track, Expr::Base *center_left, Expr::Base *center_right, bool is_right_of_outside_nt) { + + if (this->is_terminal_) { + // a parameterized Terminal like ROPE("manfred") is of type Alt::Simple + // and will therefore be called here, while ROPE without a parameter + // would be of type Alt::Link + Alt::Base::init_indices_outside(left, right, k, track, center_left, + center_right, is_right_of_outside_nt); + // store original left/right for e.g. + // correct initialization of IF guards; see function init_guards() + this->left_inside_indices.resize(this->left_indices.size()); + this->right_inside_indices.resize(this->right_indices.size()); + this->left_inside_indices[track] = left; + this->right_inside_indices[track] = right; + return; + } // if this alternative is called from an outside non-terminal, but does // not contain another outside non-terminal, e.g. hl(BASE, REGION, BASE) // we can default to inside indices generation! From 16b05d23421025245d92721d3de720620d96c734 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 29 Sep 2022 23:31:56 +0200 Subject: [PATCH 142/209] tab to whitespace --- src/alt.cc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index cc70426c9..196a9deef 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3676,18 +3676,18 @@ void Alt::Simple::init_indices_outside(Expr::Base *left, Expr::Base *right, Expr::Base *center_right, bool is_right_of_outside_nt) { if (this->is_terminal_) { - // a parameterized Terminal like ROPE("manfred") is of type Alt::Simple - // and will therefore be called here, while ROPE without a parameter - // would be of type Alt::Link - Alt::Base::init_indices_outside(left, right, k, track, center_left, - center_right, is_right_of_outside_nt); - // store original left/right for e.g. - // correct initialization of IF guards; see function init_guards() - this->left_inside_indices.resize(this->left_indices.size()); - this->right_inside_indices.resize(this->right_indices.size()); - this->left_inside_indices[track] = left; - this->right_inside_indices[track] = right; - return; + // a parameterized Terminal like ROPE("manfred") is of type Alt::Simple + // and will therefore be called here, while ROPE without a parameter + // would be of type Alt::Link + Alt::Base::init_indices_outside(left, right, k, track, center_left, + center_right, is_right_of_outside_nt); + // store original left/right for e.g. + // correct initialization of IF guards; see function init_guards() + this->left_inside_indices.resize(this->left_indices.size()); + this->right_inside_indices.resize(this->right_indices.size()); + this->left_inside_indices[track] = left; + this->right_inside_indices[track] = right; + return; } // if this alternative is called from an outside non-terminal, but does // not contain another outside non-terminal, e.g. hl(BASE, REGION, BASE) From ca3710b9b73601eb4d53e14769effe8d5d07050a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 29 Sep 2022 23:39:46 +0200 Subject: [PATCH 143/209] add test with new grammar elmamun.gap --- testdata/grammar_outside/elmamun.gap | 203 +++++++++++++++++++++++++++ testdata/regresstest/config | 3 + 2 files changed, 206 insertions(+) create mode 100644 testdata/grammar_outside/elmamun.gap diff --git a/testdata/grammar_outside/elmamun.gap b/testdata/grammar_outside/elmamun.gap new file mode 100644 index 000000000..8bc7858b8 --- /dev/null +++ b/testdata/grammar_outside/elmamun.gap @@ -0,0 +1,203 @@ +/* +# Calif El Mamun's Caravan + + +Back in the old days, around the year 800, [Calif El Mamun of Bagdad](https://en.wikipedia.org/wiki/Al-Ma%27mun) planned to take his two sons on their first [hadj to Mekka](https://en.wikipedia.org/wiki/Hajj) ... +El Mamun called the camel dealer to negotiate about the required resources. + +After a long day of bargaining, the bill was written in the sand. + +(1 Calif, 2 sons, 3 baggage camels for each one, costing 4 tubes of oil, plus one riding camel for each one, costing 5 tubes of oil) +Computation was rather slow in those days ... + +There came the evening prayers, and there came the wind ... + +A mystery! Allah's hand had erased the parentheses, but left untouched the rest of the formula. + +The dealer was eager to redraw the parentheses: + +He even claimed that now the formula showed beautiful symmetry, pleasing the eye of God. +El Mamun was not good at figures, but he knew everything about camel dealers. +He felt suspicious. + +El Mamun called for the mathematician. [Al Chwarizmi](https://en.wikipedia.org/wiki/Muhammad_ibn_Musa_al-Khwarizmi), famous already in those days, studied the formula carefully, before he spoke. + +"Some possible answers are + + * `(1 + 2) * (3 * (4 + 5)) = 81` + * `1 + ((2 * (3 * 4)) + 5) = 30` + * `(1 + 2) * ((3 * 4) + 5) = 51` + +which are all equal in the eyes of God." +Apparently, Al Chwarizmi was a wise as well as a cautious man. + +El Mamun's contemplated this answer over night, and in the next morning, he ruled as follows: + + * The dealer should be buried in the sand, next to his formula `(1 + 2) * 3 * (4 + 5)`. + * Someone had to take care of the dealer's camels, and the Calif volunteered to collect them in his stables. + * Al Chwarizmi was awarded a research grant (51 tubes of oil) to study the optimal placement of parentheses, both from a buyer's or from a seller's perspective (depending on which side of the counter the Calif might find himself). + * Until this problem was solved, there should hold the provisional rule: "`*` takes priority over `+`", wherever parentheses were lacking in some formula. + +Some results from this episode can still be observed today! + + + * El Mamun became very, very rich, and his name gave rise to the word "mammon" in many modern languages. + * Studying hard, Al Chwarizmi became the father of algorithmics. He did not solve the problem, because [Dynamic Programming](https://en.wikipedia.org/wiki/Dynamic_programming) was only developed by [Richard Bellman](https://en.wikipedia.org/wiki/Richard_E._Bellman) in the 1950s. + * As a consequence of the question being unsettled, today `*` still takes priority over `+`. + +*/ + +type Rope = extern + +signature sig_elmamun(alphabet, answer) { + answer number(int); + answer add(answer, alphabet, answer); + answer mult(answer, alphabet, answer); + answer minus(answer, alphabet, answer); + answer heinz(answer, Rope, answer); + answer nil(void); + choice [answer] h([answer]); +} + +algebra alg_enum auto enum; +algebra alg_count auto count; + +algebra alg_pretty implements sig_elmamun(alphabet=char, answer=Rope) { + Rope number(int value) { + Rope res; + append(res, value); + return res; + } + Rope add(Rope left, char opSymbol, Rope right) { + Rope res; + append(res, '('); + append(res, left); + append(res, opSymbol); + append(res, right); + append(res, ')'); + return res; + } + Rope heinz(Rope left, Rope opSymbol, Rope right) { + Rope res; + append(res, '('); + append(res, left); + append(res, opSymbol); + append(res, right); + append(res, ')'); + return res; + } + Rope mult(Rope left, char opSymbol, Rope right) { + Rope res; + append(res, '('); + append(res, left); + append(res, opSymbol); + append(res, right); + append(res, ')'); + return res; + } + Rope minus(Rope left, char opSymbol, Rope right) { + Rope res; + append(res, '('); + append(res, left); + append(res, opSymbol); + append(res, right); + append(res, ')'); + return res; + } + Rope nil(void) { + Rope res; + return res; + } + choice [Rope] h([Rope] candidates) { + return candidates; + } +} + +algebra alg_buyer implements sig_elmamun(alphabet=char, answer=int) { + int number(int value) { + return value; + } + int add(int left, char opSymbol, int right) { + return left + right; + } + int heinz(int left, Rope opSymbol, int right) { + return left + right; + } + int mult(int left, char opSymbol, int right) { + return left * right; + } + int minus(int left, char opSymbol, int right) { + return left - right; + } + int nil(void) { + return 0; + } + choice [int] h([int] candidates) { + return list(minimum(candidates)); + } +} +algebra alg_seller extends alg_buyer { + choice [int] h([int] candidates) { + return list(maximum(candidates)); + } +} + +algebra alg_time implements sig_elmamun(alphabet=char, answer=int) { + int number(int value) { + return 0; + } + int add(int left, char opSymbol, int right) { + if (left > right) { + return left + 2; + } else { + return right + 2; + } + } + int heinz(int left, Rope opSymbol, int right) { + if (left > right) { + return left + 2; + } else { + return right + 2; + } + } + int mult(int left, char opSymbol, int right) { + if (left > right) { + return left + 5; + } else { + return right + 5; + } + } + int minus(int left, char opSymbol, int right) { + if (left > right) { + return left + 3; + } else { + return right + 3; + } + } + int nil(void) { + return 0; + } + choice [int] h([int] candidates) { + return list(minimum(candidates)); + } +} + +grammar gra_elmamun uses sig_elmamun(axiom = formula) { + formula = number(INT) + | add(formula, CHAR, formula) + | mult(formula, CHAR('*'), formula) + | heinz(formula, ROPE("manfred"), formula) + | nil(EMPTY) + # h; +} + +instance pp = gra_elmamun(alg_pretty); +instance enum = gra_elmamun(alg_enum); +instance count = gra_elmamun(alg_count); +instance sellerpp = gra_elmamun(alg_seller * alg_pretty); +instance buyerpp = gra_elmamun(alg_buyer * alg_pretty); +instance ppbuyer = gra_elmamun(alg_pretty * alg_buyer); +instance timepp = gra_elmamun(alg_time * alg_pretty); + +// example input: +// "1+2*3*4+5" diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 97a7f4c8b..ff6164588 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -484,3 +484,6 @@ check_new_old_eq tmhmm_debug.gap unused count "AA" outside_report # example with multiple tracks for outside report function check_new_old_eq_twotrack alignments.gap unused count "AAAA" outside_2track "BBB" check_new_old_eq_twotrack alignments.gap unused sim_enum "AA" outside_undefvar "B" + +# example with parameterized terminals e.g. ROPE("manfred") +check_new_old_eq elmamun.gap unused count "1+2*3" outside_elm From eb583fd604f9fff2ebb040f06f140aa44a70e560 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 7 Dec 2022 16:25:59 +0100 Subject: [PATCH 144/209] explicitely set variable to NULL --- src/alt.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/alt.cc b/src/alt.cc index 196a9deef..3ad006b46 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -1606,6 +1606,9 @@ void Alt::Simple::init_guards() { guards_inside = new Statement::If(cond); ret_decl_empty_block(guards_inside); + } else { + // don't leave this variable in an undefined state + guards_inside = NULL; } } From 7498daac7cb207f00b92f0effda2fa745439c8cf Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 7 Dec 2022 16:40:16 +0100 Subject: [PATCH 145/209] adding elmamun version for derivatve computation, used in other PRs --- .../grammar_outside/elmamun_derivatives.gap | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 testdata/grammar_outside/elmamun_derivatives.gap diff --git a/testdata/grammar_outside/elmamun_derivatives.gap b/testdata/grammar_outside/elmamun_derivatives.gap new file mode 100644 index 000000000..38ebc7ce8 --- /dev/null +++ b/testdata/grammar_outside/elmamun_derivatives.gap @@ -0,0 +1,73 @@ +type Rope = extern + +signature sig_elmamun(alphabet, answer) { + answer number(int); + answer add(answer, alphabet, answer); + answer mult(answer, alphabet, answer); + answer minus(answer, alphabet, answer); + answer nil(void); + choice [answer] h([answer]); +} + +algebra alg_enum auto enum; +algebra alg_count auto count; + +algebra alg_score implements sig_elmamun(alphabet=char, answer=float) { + float number(int value) { + return 1.0; + } + float add(float left, char opSymbol, float right) { + return left * right * exp(2); + } + float heinz(float left, Rope opSymbol, float right) { + return left * right; + } + float mult(float left, char opSymbol, float right) { + return left * right * exp(3); + } + float minus(float left, char opSymbol, float right) { + return left * right; + } + float nil(void) { + return 1.0; + } + choice [float] h([float] candidates) { + return list(sum(candidates)); + } +} + +algebra alg_hessians implements sig_elmamun(alphabet=char, answer=float) { + float number(int value) { + return 0.0; + } + float add(float left, char opSymbol, float right) { + return left + right + 2.0; + } + float heinz(float left, Rope opSymbol, float right) { + return left + right; + } + float mult(float left, char opSymbol, float right) { + return left + right + 3.0; + } + float minus(float left, char opSymbol, float right) { + return left + right; + } + float nil(void) { + return 0.0; + } + choice [float] h([float] candidates) { + return list(sum(candidates)); + } +} + + +grammar gra_elmamun uses sig_elmamun(axiom = formula) { + formula = number(INT) + | add(formula, CHAR('+'), formula) + | mult(formula, CHAR('*'), formula) + | nil(EMPTY) + # h; +} + +instance firstD = gra_elmamun(alg_score); +instance bothD = gra_elmamun(alg_score * alg_hessians); From adfcd863cf076352fd0704783f90d3c035c7a462 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 16 Jan 2023 09:28:18 +0100 Subject: [PATCH 146/209] adding test grammar --- .../hmm_sonneregen_properEnd.gap | 526 ++++++++++++++++++ 1 file changed, 526 insertions(+) create mode 100644 testdata/grammar_outside/hmm_sonneregen_properEnd.gap diff --git a/testdata/grammar_outside/hmm_sonneregen_properEnd.gap b/testdata/grammar_outside/hmm_sonneregen_properEnd.gap new file mode 100644 index 000000000..bda500272 --- /dev/null +++ b/testdata/grammar_outside/hmm_sonneregen_properEnd.gap @@ -0,0 +1,526 @@ +import negexpsum + +type Rope = extern + +signature sig_weather(alphabet, answer) { + answer transition_start_hoch(float, answer, answer); + answer transition_start_tief(float, answer, answer); + answer transition_start_ende(float, answer); + answer transition_hoch_tief(float, answer, answer); + answer transition_hoch_hoch(float, answer, answer); + answer transition_hoch_ende(float, answer); + answer transition_tief_tief(float, answer, answer); + answer transition_tief_hoch(float, answer, answer); + answer transition_tief_ende(float, answer); + + answer emission_hoch_sonne(float, alphabet); + answer emission_hoch_regen(float, alphabet); + answer emission_tief_sonne(float, alphabet); + answer emission_tief_regen(float, alphabet); + answer nil(void); + + choice [answer] h([answer]); +} + +algebra alg_enum auto enum; +algebra alg_count auto count; + +algebra alg_viterbi implements sig_weather(alphabet=char, answer=float) { + float transition_start_hoch(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_start_tief(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_start_ende(float transition, float x) { + return transition * x; + } + float transition_hoch_tief(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_hoch_hoch(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_hoch_ende(float transition, float x) { + return transition * x; + } + float transition_tief_tief(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_tief_hoch(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_tief_ende(float transition, float x) { + return transition * x; + } + + float emission_hoch_sonne(float emission, char a) { + return emission; + } + float emission_hoch_regen(float emission, char a) { + return emission; + } + float emission_tief_sonne(float emission, char a) { + return emission; + } + float emission_tief_regen(float emission, char a) { + return emission; + } + float nil(void) { + return 1.0; + } + + choice [float] h([float] candidates) { + return list(maximum(candidates)); + } +} + +algebra alg_experiment implements sig_weather(alphabet=char, answer=float) { + float transition_start_hoch(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_start_tief(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_start_ende(float transition, float x) { + return transition * x; + } + float transition_hoch_tief(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_hoch_hoch(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_hoch_ende(float transition, float x) { + return transition * x; + } + float transition_tief_tief(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_tief_hoch(float transition, float emission, float x) { + return transition * emission * x; + } + float transition_tief_ende(float transition, float x) { + return transition * x; + } + + float emission_hoch_sonne(float emission, char a) { + return 1.0; + } + float emission_hoch_regen(float emission, char a) { + return 1.0; + } + float emission_tief_sonne(float emission, char a) { + return 1.0; + } + float emission_tief_regen(float emission, char a) { + return 1.0; + } + float nil(void) { + return 1.0; + } + + choice [float] h([float] candidates) { + return list(sum(candidates)); + } +} + +algebra alg_fwd extends alg_viterbi { + float normalize_derivative(float q, float pfunc) { + return q / pfunc; + } + choice [float] h([float] candidates) { + return list(sum(candidates)); + } +} + +algebra alg_hessians implements sig_weather(alphabet=char, answer=float) { + float transition_start_hoch(float transition, float emission, float x) { + return 0.48 * emission * x; + } + float transition_start_tief(float transition, float emission, float x) { + return 0.51 * emission * x; + } + float transition_start_ende(float transition, float x) { + return 0.01 * x; + } + float transition_hoch_tief(float transition, float emission, float x) { + return 0.20 * emission * x; + } + float transition_hoch_hoch(float transition, float emission, float x) { + return 0.70 * emission * x; + } + float transition_hoch_ende(float transition, float x) { + return 0.10 * x; + } + float transition_tief_tief(float transition, float emission, float x) { + return 0.50 * emission * x; + } + float transition_tief_hoch(float transition, float emission, float x) { + return 0.40 * emission * x; + } + float transition_tief_ende(float transition, float x) { + return 0.10 * x; + } + + float emission_hoch_sonne(float emission, char a) { + return 0.6; + } + float emission_hoch_regen(float emission, char a) { + return 0.4; + } + float emission_tief_sonne(float emission, char a) { + return 0.5; + } + float emission_tief_regen(float emission, char a) { + return 0.5; + } + float nil(void) { + return 1.0; + } + + choice [float] h([float] candidates) { + return list(sum(candidates)); + } +} + + +algebra alg_fwd_log implements sig_weather(alphabet=char, answer=float) { + float transition_start_hoch(float transition, float emission, float x) { + return log(transition) + emission + x; + } + float transition_start_tief(float transition, float emission, float x) { + return log(transition) + emission + x; + } + float transition_start_ende(float transition, float x) { + return log(transition) + x; + } + float transition_hoch_tief(float transition, float emission, float x) { + return log(transition) + emission + x; + } + float transition_hoch_hoch(float transition, float emission, float x) { + return log(transition) + emission + x; + } + float transition_hoch_ende(float transition, float x) { + return log(transition) + x; + } + float transition_tief_tief(float transition, float emission, float x) { + return log(transition) + emission + x; + } + float transition_tief_hoch(float transition, float emission, float x) { + return log(transition) + emission + x; + } + float transition_tief_ende(float transition, float x) { + return log(transition) + x; + } + + float emission_hoch_sonne(float emission, char a) { + return log(emission); + } + float emission_hoch_regen(float emission, char a) { + return log(emission); + } + float emission_tief_sonne(float emission, char a) { + return log(emission); + } + float emission_tief_regen(float emission, char a) { + return log(emission); + } + float nil(void) { + return log(1.0); + } + + float normalize_derivative(float q, float pfunc) { + return exp(q - pfunc); + } + + choice [float] h([float] candidates) { + return list(expsum(candidates)); + } +} + +algebra alg_fwd_neglog implements sig_weather(alphabet=char, answer=float) { + float transition_start_hoch(float transition, float emission, float x) { + return log(1.0/transition) + emission + x; + } + float transition_start_tief(float transition, float emission, float x) { + return log(1.0/transition) + emission + x; + } + float transition_start_ende(float transition, float x) { + return log(1.0/transition) + x; + } + float transition_hoch_tief(float transition, float emission, float x) { + return log(1.0/transition) + emission + x; + } + float transition_hoch_hoch(float transition, float emission, float x) { + return log(1.0/transition) + emission + x; + } + float transition_hoch_ende(float transition, float x) { + return log(1.0/transition) + x; + } + float transition_tief_tief(float transition, float emission, float x) { + return log(1.0/transition) + emission + x; + } + float transition_tief_hoch(float transition, float emission, float x) { + return log(1.0/transition) + emission + x; + } + float transition_tief_ende(float transition, float x) { + return log(1.0/transition) + x; + } + + float emission_hoch_sonne(float emission, char a) { + return log(1.0/emission); + } + float emission_hoch_regen(float emission, char a) { + return log(1.0/emission); + } + float emission_tief_sonne(float emission, char a) { + return log(1.0/emission); + } + float emission_tief_regen(float emission, char a) { + return log(1.0/emission); + } + float nil(void) { + return log(1.0/1.0); + } + + float normalize_derivative(float q, float pfunc) { + return exp(pfunc - q); + } + + synoptic choice [float] h([float] candidates) { + return list(negexpsum(candidates)); + } +} + + +algebra alg_states implements sig_weather(alphabet=char, answer=Rope) { + Rope transition_start_hoch(float transition, Rope emission, Rope x) { + Rope res; + append(res, '^'); + append(res, 'H'); + append(res, x); + return res; + } + Rope transition_start_tief(float transition, Rope emission, Rope x) { + Rope res; + append(res, '^'); + append(res, 'T'); + append(res, x); + return res; + } + Rope transition_start_ende(float transition, Rope x) { + Rope res; + append(res, '^'); + append(res, x); + return res; + } + Rope transition_hoch_tief(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'T'); + append(res, x); + return res; + } + Rope transition_hoch_hoch(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'H'); + append(res, x); + return res; + } + Rope transition_hoch_ende(float transition, Rope x) { + Rope res; + append(res, x); + return res; + } + Rope transition_tief_tief(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'T'); + append(res, x); + return res; + } + Rope transition_tief_hoch(float transition, Rope emission, Rope x) { + Rope res; + append(res, 'H'); + append(res, x); + return res; + } + Rope transition_tief_ende(float transition, Rope x) { + Rope res; + append(res, x); + return res; + } + + Rope emission_hoch_sonne(float emission, char a) { + Rope res; + return res; + } + Rope emission_hoch_regen(float emission, char a) { + Rope res; + return res; + } + Rope emission_tief_sonne(float emission, char a) { + Rope res; + return res; + } + Rope emission_tief_regen(float emission, char a) { + Rope res; + return res; + } + Rope nil(void) { + Rope res; + append(res, '$'); + return res; + } + + choice [Rope] h([Rope] candidates) { + return unique(candidates); + } +} + +algebra alg_mult implements sig_weather(alphabet=char, answer=Rope) { + Rope transition_start_hoch(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_start_tief(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_start_ende(float transition, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_hoch_tief(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_hoch_hoch(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_hoch_ende(float transition, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_tief_tief(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_tief_hoch(float transition, Rope emission, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, emission); + append(res, " * ", 3); + append(res, x); + return res; + } + Rope transition_tief_ende(float transition, Rope x) { + Rope res; + append(res, transition); + append(res, " * ", 3); + append(res, x); + return res; + } + + Rope emission_hoch_sonne(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope emission_hoch_regen(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope emission_tief_sonne(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope emission_tief_regen(float emission, char a) { + Rope res; + append(res, emission); + return res; + } + Rope nil(void) { + Rope res; + append(res, 1.0); + return res; + } + + choice [Rope] h([Rope] candidates) { + return unique(candidates); + } +} + + +grammar gra_weather uses sig_weather(axiom=start) { + start = transition_start_hoch(CONST_FLOAT(0.49), hoch_emission, hoch) + | transition_start_tief(CONST_FLOAT(0.49), tief_emission, tief) + | transition_start_ende(CONST_FLOAT(0.02), ende) + # h; + + hoch = transition_hoch_tief(CONST_FLOAT(0.29), tief_emission, tief) + | transition_hoch_hoch(CONST_FLOAT(0.69), hoch_emission, hoch) + | transition_hoch_ende(CONST_FLOAT(0.02), ende) + # h; + + tief = transition_tief_tief(CONST_FLOAT(0.59), tief_emission, tief) + | transition_tief_hoch(CONST_FLOAT(0.39), hoch_emission, hoch) + | transition_tief_ende(CONST_FLOAT(0.02), ende) + # h; + + ende = nil(EMPTY) + # h; + + hoch_emission = emission_hoch_sonne(CONST_FLOAT(0.8), CHAR('S')) + | emission_hoch_regen(CONST_FLOAT(0.2), CHAR('R')) + # h; + + tief_emission = emission_tief_sonne(CONST_FLOAT(0.1), CHAR('S')) + | emission_tief_regen(CONST_FLOAT(0.9), CHAR('R')) + # h; +} + +instance enum = gra_weather(alg_enum); +instance viterbistatesmult = gra_weather(alg_viterbi * alg_states * alg_mult); +instance fwd = gra_weather(alg_fwd); +instance fwd_log = gra_weather(alg_fwd_log); +instance fwd_neglog = gra_weather(alg_fwd_neglog); +instance multviterbistates = gra_weather(alg_mult * alg_viterbi * alg_states); + +instance bothD = gra_weather(alg_fwd * alg_hessians); +instance bothD_log = gra_weather(alg_fwd_log * alg_hessians); +instance bothD_neglog = gra_weather(alg_fwd_neglog * alg_hessians); From cfc6106e1354f44401173f2e6b92326c83f82488 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 31 Jan 2023 16:55:25 +0100 Subject: [PATCH 147/209] prepare further graphviz printing + add "completetrack" as single track filter when possible --- src/alt.cc | 14 +++++++++++++- src/alt.hh | 3 +++ src/grammar.cc | 6 +++++- src/symbol.cc | 5 ++++- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 30958cb3f..e1138c4c8 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -2620,6 +2620,7 @@ void Alt::Simple::init_multi_ys() { input, i.e. NOT for CONST_xxx terminal parser, that inject values for later use in algebras. */ + // TODO(sjanssen) isn't there a better method to check for constants? if (name->rfind("CONST_", 0) != 0) { Yield::Poly max_terminal_arg_yield = Yield::Poly(0); for (std::list::iterator i = args.begin(); @@ -3121,6 +3122,14 @@ void to_dot_indices(std::vector indices, std::ostream &out) { } out << "
min=" << ys->low(); + out << ", max=" << ys->high() << "
>, color=\""; + out << ""; + // TODO(sjanssen): update Truth before activating yield size plotting + // to_dot_multiys(m_ys, out); + out << ">, color=\""; if (simple) { if (simple->is_terminal()) { out << "blue"; diff --git a/src/alt.hh b/src/alt.hh index badeacba1..48957f169 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -912,4 +912,7 @@ class Multi : public Base { // used as a helper for to_dot functions void to_dot_indices(std::vector indices, std::ostream &out); +// adds another line to indicate yield sizes of grammar components +void to_dot_multiys(Yield::Multi m_ys, std::ostream &out); + #endif // SRC_ALT_HH_ diff --git a/src/grammar.cc b/src/grammar.cc index 4b37fd3ed..33df8d27a 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1254,7 +1254,11 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { } link->set_tracks(this->axiom->tracks(), this->axiom->track_pos()); link->init_multi_ys(); - link->add_multitrack_filter(*comptracks, f->type, Loc()); + if (link->nt->tracks() == 1) { + link->filters.push_back(f); + } else { + link->add_multitrack_filter(*comptracks, f->type, Loc()); + } link->top_level = Bool(true); dynamic_cast(outside_NTs.find( *this->axiom_name)->second)->alts.push_back(link); diff --git a/src/symbol.cc b/src/symbol.cc index a3e8e8ed4..94055b1dc 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -1668,7 +1668,10 @@ unsigned int Symbol::Base::to_dot(unsigned int *nodeID, std::ostream &out, } out << ""; to_dot_indices(this->right_indices, out); - out << ">"; + out << ""; + // TODO(sjanssen): wait for truth update + // to_dot_multiys(this->m_ys, out); + out << ">"; } else { out << "" << *this->name << ">"; } From 11ab05c83603af67a0d64a2cf74262c8884ff9c2 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 9 Feb 2023 16:03:21 +0100 Subject: [PATCH 148/209] limit to level 3 --- testdata/modtest/outside_indices.cc | 2 +- testdata/modtest/outside_resolve_blocks.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testdata/modtest/outside_indices.cc b/testdata/modtest/outside_indices.cc index c2715b9db..351c9cd52 100644 --- a/testdata/modtest/outside_indices.cc +++ b/testdata/modtest/outside_indices.cc @@ -67,5 +67,5 @@ int main(int argc, char **argv) { grammar->dep_analysis(); unsigned int nodeID = 1; - grammar->to_dot(&nodeID, std::cout, 99); + grammar->to_dot(&nodeID, std::cout, 3); } diff --git a/testdata/modtest/outside_resolve_blocks.cc b/testdata/modtest/outside_resolve_blocks.cc index 6268b2728..f00bc5e38 100644 --- a/testdata/modtest/outside_resolve_blocks.cc +++ b/testdata/modtest/outside_resolve_blocks.cc @@ -82,7 +82,7 @@ int main(int argc, char **argv) { grammar->dep_analysis(); unsigned int nodeID = 1; - grammar->to_dot(&nodeID, std::cout, 99); + grammar->to_dot(&nodeID, std::cout, 3); return 0; } From 255a7abd6955d6677e9d5d492b71f4ea5f6d3678 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 10 Feb 2023 20:14:26 +0100 Subject: [PATCH 149/209] redefine filter logic, such that complete input sequence is required --- rtlib/filter.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtlib/filter.hh b/rtlib/filter.hh index 30731a7c3..21af2e1d5 100644 --- a/rtlib/filter.hh +++ b/rtlib/filter.hh @@ -130,7 +130,7 @@ inline bool samesize(const Basic_Sequence &s1, template inline bool complete_track( const Basic_Sequence &seq, T i, T j) { - return ((i == seq.n) && (j == seq.n)); + return ((i == 0) && (j == seq.n)); } #endif // RTLIB_FILTER_HH_ From 11b0e09af73e7d108c7b5a68871a18f0be57ea49 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 10 Feb 2023 20:18:18 +0100 Subject: [PATCH 150/209] special handling of transitioning outside towards inside grammar parts, i.e. require (0,n) but call (0,0) --- src/alt.cc | 46 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 2c80f4822..7cb9d486d 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -2146,13 +2146,52 @@ void Alt::Link::add_args(Expr::Fn_Call *fn) { assert(left_indices.size() == tables.size()); std::vector::iterator k = right_indices.begin(); std::vector::iterator j = left_indices.begin(); + std::vector::iterator odl = x->left_indices_outsidedummy.begin(); + std::vector::iterator odr = + x->right_indices_outsidedummy.begin(); + std::vector::iterator ontl = x->left_indices.begin(); + std::vector::iterator ontr = x->right_indices.begin(); for (std::vector::const_iterator i = tables.begin(); - i != tables.end(); ++i, ++j, ++k) { + i != tables.end(); ++i, ++j, ++k, ++odl, ++odr) { + if (this->is_outside_inside_transition) { + /* we encounter the one and only situation for outside grammars + * where we transit from outside (full input sequence) into inside + * and need to make sure we parse the empty word. + */ + if (((*i).delete_left_index() && (!(*i).delete_right_index()))) { + fn->add_arg(*ontl); + fn->add_arg(*ontl); + } else if ((!(*i).delete_left_index() && ((*i).delete_right_index()))) { + fn->add_arg(*ontr); + fn->add_arg(*ontr); + } else { + fn->add_arg(new Expr::Const(0)); + fn->add_arg(new Expr::Const(0)); + } + // ignore the "normal" logic below + continue; + } if (!(*i).delete_left_index()) { fn->add_arg(*j); + } else { + if (x->is_inside_axiom) { + if (this->is_partof_outside) { + fn->add_arg(*ontl); + } else { + fn->add_arg(*odl); + } + } } if (!(*i).delete_right_index()) { fn->add_arg(*k); + } else { + if (x->is_inside_axiom) { + if (this->is_partof_outside) { + fn->add_arg(*ontr); + } else { + fn->add_arg(*odr); + } + } } } @@ -3539,6 +3578,7 @@ bool Alt::Link::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, if (skip_occurences[*(find->name)] == 0) { // replace old NT (=find) with novel NT (=replace) this->nt = replace; + this->m_ys = replace->multi_ys(); this->name = replace->name; this->is_partof_outside = replace->is_partof_outside; return true; @@ -3609,7 +3649,7 @@ Expr::Base *Alt::Simple::get_next_var_right2left(Expr::Base *left_index, Yield::Size ys_this, Yield::Size *ys_lefts) { if (ys_this.low() == ys_this.high()) { // constant yield size - if (ys_lefts->low() == 0) { + if ((ys_lefts->low() == 0) && (ys_lefts->high() == 0)) { return innermost_left_index; } else { return left_index->plus(ys_this.low()); @@ -3641,7 +3681,7 @@ Expr::Base *Alt::Simple::get_next_var_left2right(Expr::Base *right_index, // constant yield size if (ys_this.low() == ys_this.high()) { - if (ys_rights->low() == 0) { + if ((ys_rights->low() == 0) && (ys_rights->high() == 0)) { // we might fall back to j, if yield between outside NT and current // right hand side argument is 0 next_index = innermost_right_index; From 9ba94dedefac926d9c5facc18ac12c1ab6c6dbd7 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 10 Feb 2023 20:19:00 +0100 Subject: [PATCH 151/209] add a new "flag" to indicate the one and only transition from outside to inside parts of grammar --- src/alt.hh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/alt.hh b/src/alt.hh index 72a635843..4fb2b1944 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -732,6 +732,12 @@ class Link : public Base { void init_outside_guards(); Alt::Base* find_block(); Alt::Base *find_block_parent(const Alt::Base &block); + + /* flags the one and only situation where outside grammar + * transitions into inside rules, i.e. user defined axiom. + * we need to ensure that according NT call asks for the + * empy word. */ + bool is_outside_inside_transition = false; }; From 30ae228f0b668a58dc157235dc591ed439246dcb Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 10 Feb 2023 20:20:38 +0100 Subject: [PATCH 152/209] add dummy variables for inside axiom if called from outside context, to ensure (0,n) consumption --- src/fn_def.cc | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/fn_def.cc b/src/fn_def.cc index 780f248ec..efd005840 100644 --- a/src/fn_def.cc +++ b/src/fn_def.cc @@ -317,13 +317,31 @@ void Fn_Def::add_para(Symbol::NT &nt) { std::vector::iterator j = left.begin(); std::vector::iterator k = right.begin(); + std::vector::iterator odl = nt.left_indices_outsidedummy.begin(); + std::vector::iterator odr = + nt.right_indices_outsidedummy.begin(); for (std::vector
::const_iterator i = tables.begin(); - i != tables.end(); ++i, ++j, ++k) { + i != tables.end(); ++i, ++j, ++k, ++odl, ++odr) { // only add the - if (!(*i).delete_left_index()) + /* do NOT reduce indices, if this NT was the inside axiom and is + * used in an outside context. We need them to ensure that + * complete input sequence is parsed for outside and empty + * word for the inside part of the grammar, as this NT marks + * the transition between both grammar parts */ + if (!(*i).delete_left_index()) { add_para(t, (*j)->vacc()->name()); - if (!(*i).delete_right_index()) + } else { + if (nt.is_inside_axiom) { + add_para(t, (*odl)->vacc()->name()); + } + } + if (!(*i).delete_right_index()) { add_para(t, (*k)->vacc()->name()); + } else { + if (nt.is_inside_axiom) { + add_para(t, (*odr)->vacc()->name()); + } + } } set_paras(nt.ntargs()); From 0420b3f368d4440d88c5c1ef6a6ed670ef070e86 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 10 Feb 2023 20:45:41 +0100 Subject: [PATCH 153/209] special treatment is handled elsewhere --- src/symbol.cc | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/symbol.cc b/src/symbol.cc index febe4e65f..96543ea4d 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -753,15 +753,7 @@ void Symbol::NT::init_indices(Expr::Vacc *left, Expr::Vacc *right, for (std::list::iterator i = alts.begin(); i != alts.end(); ++i) { if (this->is_partof_outside) { - if ((*i)->get_is_partof_outside() == false) { - // special treatment for a link to the original inside axiom, ensure - // that it has to consume the full input sequence(s) - (*i)->init_indices_outside( - left->plus(right), right->minus(left), k, track, NULL, NULL, false); - dynamic_cast(*i)->init_outside_guards(); - } else { - (*i)->init_indices_outside(left, right, k, track, NULL, NULL, false); - } + (*i)->init_indices_outside(left, right, k, track, NULL, NULL, false); } else { (*i)->init_indices(left, right, k, track); } From 5b505180bd43cecdbabf89e5f59e33820a9daea6 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 10 Feb 2023 20:45:56 +0100 Subject: [PATCH 154/209] add three new attributes to handle edge case of reduced table dims for inside axiom --- src/symbol.hh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/symbol.hh b/src/symbol.hh index d0d5617a2..cbb7c8195 100644 --- a/src/symbol.hh +++ b/src/symbol.hh @@ -126,6 +126,15 @@ class Base { Loc location; std::vector left_indices; std::vector right_indices; + /* the user defined inside axiom must be called from outside grammar parts + * such that the outside parts consumes the full input sequence(s) (0,n). + * But the inside NT call must parse the remainder, i.e. the empty word + * If however, table dimension of this axiom was reduced, we cannot pass + * prober boundaries. For such cases, we generate dummy variables that + * do not extend table dimensions, but can ensure that (0,0) and (n, 0) are + * passed. */ + std::vector left_indices_outsidedummy; + std::vector right_indices_outsidedummy; bool is(Type t) const { return type == t; } @@ -525,6 +534,12 @@ class NT : public Base { unsigned int to_dot(unsigned int *nodeID, std::ostream &out, bool is_rhs, Symbol::NT *axiom, int plot_grammar); void resolve_blocks(); + + // in outside grammar generation context: flags the one non-terminal + // that was the axiom for the inside part of the grammar, i.e. user + // defined axiom. Axiom will be re-set to an outside version during + // Grammar::inject_outside_nts() + bool is_inside_axiom = false; }; From 4c1abc0f99b114fabbc1965747a081fbcf08ed9d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 10 Feb 2023 20:47:26 +0100 Subject: [PATCH 155/209] flag edge cases: 1) inside axiom 2) call of inside axiom with full input from outside rule --- src/grammar.cc | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/grammar.cc b/src/grammar.cc index 33df8d27a..ffd011751 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -868,11 +868,16 @@ void Grammar::init_indices() { traverse(cl); for (size_t track = 0; track < axiom->tracks(); ++track) { // variable access for all possible boundaries - std::ostringstream i, j, lm, rm; + std::ostringstream i, j, lm, rm, outside_l_dummy, outside_r_dummy; i << "t_" << track << "_i"; j << "t_" << track << "_j"; lm << "t_" << track << "_left_most"; rm << "t_" << track << "_right_most"; + /* helper variables for outside/inside transition + * which must be parameterized, but table dimension might be + * reduced to const or linear */ + outside_l_dummy << "t_" << track << "_outsidedummy_left"; + outside_r_dummy << "t_" << track << "_outsidedummy_right"; Expr::Vacc *left = new Expr::Vacc(new std::string(i.str())); Expr::Vacc *right = new Expr::Vacc(new std::string(j.str())); Expr::Vacc *left_most = new Expr::Vacc(new std::string(lm.str())); @@ -886,7 +891,17 @@ void Grammar::init_indices() { const Table &table = (*i)->tables()[idx]; Expr::Vacc *l = table.delete_left_index() ? left_most : left; Expr::Vacc *r = table.delete_right_index() ? right_most : right; - + if ((*i)->is_inside_axiom) { + // always false for pure inside grammars + if (table.delete_left_index()) { + (*i)->left_indices_outsidedummy.push_back( + new Expr::Vacc(new std::string(outside_l_dummy.str()))); + } + if (table.delete_right_index()) { + (*i)->right_indices_outsidedummy.push_back( + new Expr::Vacc(new std::string(outside_r_dummy.str()))); + } + } if ((*i)->tracks() == 1 && (*i)->track_pos() != track) { continue; } @@ -1173,6 +1188,9 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { outside_NTs[(*(*nt)->name)] = outside_nt; } + // flag inside axiom + this->axiom->is_inside_axiom = true; + // 3) create copy of inside alternative, replace called non-terminal with // calling outside version and append to called outside version std::set *axiom_candidates = new std::set(); @@ -1254,6 +1272,7 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { } link->set_tracks(this->axiom->tracks(), this->axiom->track_pos()); link->init_multi_ys(); + link->is_outside_inside_transition = true; if (link->nt->tracks() == 1) { link->filters.push_back(f); } else { From 1ef586c389693a46cc6ee16445b4a6df1bb89b39 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 10 Feb 2023 20:51:16 +0100 Subject: [PATCH 156/209] insert dummy variables for report_outside function and CYK calls --- src/cpp.cc | 43 ++++++++++++++++++++++++++++++++++++------- src/cpp.hh | 3 ++- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 6eed19f1f..5d0acf115 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -1743,7 +1743,8 @@ void Printer::Cpp::print_openmp_cyk_nt_calls(const AST &ast, if ((*i)->is_tabulated() && !(*i)->tables().front().is_cyk_right()) { o1 << indent() << "nt_tabulate_" << *(*i)->name << "(" << multi_index_str((*i)->tables(), (*i)->multi_ys(), - (*i)->is_partof_outside) + (*i)->is_partof_outside, + (*i)->is_inside_axiom) << ");\n"; } } @@ -1762,7 +1763,8 @@ void Printer::Cpp::print_openmp_cyk_all_nt_calls(const AST &ast, if ((*i)->is_tabulated()) { o2 << indent() << "nt_tabulate_" << *(*i)->name << "(" << multi_index_str((*i)->tables(), (*i)->multi_ys(), - (*i)->is_partof_outside) + (*i)->is_partof_outside, + (*i)->is_inside_axiom) << ");\n"; } } @@ -1917,7 +1919,7 @@ void Printer::Cpp::print_openmp_cyk(const AST &ast) { std::string Printer::Cpp::multi_index_str( const std::vector
&tables, const Yield::Multi &mys, - bool is_outside) { + bool is_outside, bool is_inside_axiom) { assert(tables.size() == mys.tracks()); std::ostringstream o; size_t track = 0; @@ -1930,6 +1932,10 @@ std::string Printer::Cpp::multi_index_str( } else { o << ", (t_" << track << "_j - t_" << track << "_i + 1)"; } + } else { + if (is_inside_axiom) { + o << ", t_" << track << "_left_most"; + } } if (!(*i).delete_right_index()) { if (is_outside == false) { @@ -1937,6 +1943,10 @@ std::string Printer::Cpp::multi_index_str( } else { o << ", (t_" << track << "_n - t_" << track << "_i + 1)"; } + } else { + if (is_inside_axiom) { + o << ", t_" << track << "_right_most"; + } } } std::string r(o.str()); @@ -1962,7 +1972,8 @@ void Printer::Cpp::multi_print_inner_cyk( i != l.end(); ++i) { if ((*i)->is_partof_outside == for_outsideNTs) { std::string index_str = multi_index_str((*i)->tables(), (*i)->multi_ys(), - (*i)->is_partof_outside); + (*i)->is_partof_outside, + (*i)->is_inside_axiom); stream << indent() << "nt_tabulate_" << *(*i)->name << '(' << index_str << ");" << endl; } @@ -2202,6 +2213,7 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { // aggregated level (=dim + tracks) of nested loops unsigned int nesting = 0; std::vector *args = new std::vector(); + std::vector *args_call = new std::vector(); // opening for loops for (size_t track = 0; track < nt->tracks(); ++track) { unsigned int dim = 2; @@ -2217,6 +2229,7 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { << "_right_most; ++t_" << track << "_i) {" << endl; inc_indent(); args->push_back("t_" + std::to_string(track) + "_i"); + args_call->push_back("t_" + std::to_string(track) + "_i"); } if (dim >= 2) { stream << indent() << "for (unsigned int t_" << track << "_j = t_" @@ -2224,25 +2237,41 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { << "_right_most; ++t_" << track << "_j) {" << endl; inc_indent(); args->push_back("t_" + std::to_string(track) + "_j"); + args_call->push_back("t_" + std::to_string(track) + "_j"); + } + if (nt->is_inside_axiom) { + if (dim == 0) { + args_call->push_back("t_" + std::to_string(track) + "_left_most"); + } + if (dim == 1) { + args_call->push_back("t_" + std::to_string(track) + "_right_most"); + } } nesting += dim; } // loop body std::list &l = nt->code_list(); - std::stringstream list_args; std::stringstream list_args_print; bool first = true; for (std::vector::const_iterator a = args->begin(); a != args->end(); ++a) { if (!first) { - list_args << ", "; list_args_print << " << \",\""; } first = false; - list_args << *a; list_args_print << " << " << *a; } + std::stringstream list_args; + first = true; + for (std::vector::const_iterator a = args_call->begin(); + a != args_call->end(); ++a) { + if (!first) { + list_args << ", "; + } + first = false; + list_args << *a; + } stream << indent() << "out << \"start answers " << *nt->name << "(\"" << list_args_print.str() << " << \"):\\n\";\n"; for (std::list::iterator i = l.begin(); i != l.end(); ++i) { diff --git a/src/cpp.hh b/src/cpp.hh index 824bc0b2b..56e0c997a 100644 --- a/src/cpp.hh +++ b/src/cpp.hh @@ -89,7 +89,8 @@ class Cpp : public Base { void print_openmp_cyk(const AST &ast); std::string multi_index_str(const std::vector
&tables, - const Yield::Multi &mys, bool is_outside = false); + const Yield::Multi &mys, bool is_outside = false, + bool is_inside_axiom = false); void multi_print_inner_cyk(const std::list &l, const std::list &tord, size_t track, size_t tracks, size_t track_pos, From a91878ae4e6e53459d52188cdff7a04baad56055 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 10 Feb 2023 22:20:21 +0100 Subject: [PATCH 157/209] don't change multi yield size when replacing NT --- src/alt.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/alt.cc b/src/alt.cc index 7cb9d486d..3593c4592 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3578,7 +3578,6 @@ bool Alt::Link::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, if (skip_occurences[*(find->name)] == 0) { // replace old NT (=find) with novel NT (=replace) this->nt = replace; - this->m_ys = replace->multi_ys(); this->name = replace->name; this->is_partof_outside = replace->is_partof_outside; return true; From 94201a5a541aab2aaec9dadce4287fdddd736048 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 10 Feb 2023 22:29:18 +0100 Subject: [PATCH 158/209] forgot to also advance the nt left/right indices --- src/alt.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alt.cc b/src/alt.cc index 3593c4592..12989f8aa 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -2152,7 +2152,7 @@ void Alt::Link::add_args(Expr::Fn_Call *fn) { std::vector::iterator ontl = x->left_indices.begin(); std::vector::iterator ontr = x->right_indices.begin(); for (std::vector
::const_iterator i = tables.begin(); - i != tables.end(); ++i, ++j, ++k, ++odl, ++odr) { + i != tables.end(); ++i, ++j, ++k, ++odl, ++odr, ++ontl, ++ontr) { if (this->is_outside_inside_transition) { /* we encounter the one and only situation for outside grammars * where we transit from outside (full input sequence) into inside From aa664e510caa22da8d6c2a2d8625ff744c91fd36 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 10 Feb 2023 22:37:34 +0100 Subject: [PATCH 159/209] adding another test --- testdata/regresstest/config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 362d2bab4..96b70ec55 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -497,3 +497,6 @@ check_new_old_eq_twotrack alignments.gap unused sim_enum "AA" outside_undefvar " # example with parameterized terminals e.g. ROPE("manfred") check_new_old_eq elmamun.gap unused count "1+2*3" outside_elm + +# check correct indices for multiple CONST_FLOAT args for outside generation; caused by incorrectly set yield size when replacing NTs +check_new_old_eq nonzero_constsize.gap unused count "G" outside_constissue \ No newline at end of file From 4c606ea3e6bf071eef5902fa87afe4070e4dbeb9 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 10 Feb 2023 22:38:31 +0100 Subject: [PATCH 160/209] add test case of grammar with multiple const terminal parsers in a row --- .../grammar_outside/nonzero_constsize.gap | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 testdata/grammar_outside/nonzero_constsize.gap diff --git a/testdata/grammar_outside/nonzero_constsize.gap b/testdata/grammar_outside/nonzero_constsize.gap new file mode 100644 index 000000000..40724e4a3 --- /dev/null +++ b/testdata/grammar_outside/nonzero_constsize.gap @@ -0,0 +1,50 @@ +signature sig_cm(alphabet, answer) { + answer silent_transition(float, answer; int); + answer left_transition(float, alphabet, answer; int); + answer right_transition(float, answer, alphabet; int); + answer pair_transition(float, alphabet, answer, alphabet; int); + answer bifurcation_transition(float, float, answer, answer; int); + answer nil(float, void; int); + choice [answer] h([answer]); +} + + +algebra alg_enum auto enum; + +algebra alg_count auto count; + +grammar gra_cm uses sig_cm(axiom = state_B_3) { + state_B_3 = bifurcation_transition(CONST_FLOAT(0.000000), CONST_FLOAT(0.000000), state_S_4, state_S_12; 3) + # h; + + state_S_4 = silent_transition(CONST_FLOAT(-9.761000), state_ML_6; 4) + | silent_transition(CONST_FLOAT(-9.808000), state_D_8; 4) + # h; + + state_ML_6 = left_transition(CONST_FLOAT(-1.000000), CHAR, state_E_11; 6) + # h; + + state_D_8 = silent_transition(CONST_FLOAT(-1.000000), state_E_11; 8) + # h; + + state_E_11 = nil(CONST_FLOAT(0.000000), EMPTY; 11) + # h; + + + state_S_12 = silent_transition(CONST_FLOAT(-10.445000), state_ML_15; 12) + | silent_transition(CONST_FLOAT(-11.549000), state_D_17; 12) + # h; + + state_ML_15 = left_transition(CONST_FLOAT(-1.000000), CHAR, state_E_20; 15) + # h; + + state_D_17 = silent_transition(CONST_FLOAT(-1.000000), state_E_20; 17) + # h; + + state_E_20 = nil(CONST_FLOAT(0.000000), EMPTY; 20) + # h; + +} + +instance count = gra_cm(alg_count); + \ No newline at end of file From 8d4effef8646b6c111aded15ee8c05f5bdc7fbcb Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sat, 11 Feb 2023 21:28:37 +0100 Subject: [PATCH 161/209] adding test grammar --- .../grammar_outside/nonzero_constsize.gap | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 testdata/grammar_outside/nonzero_constsize.gap diff --git a/testdata/grammar_outside/nonzero_constsize.gap b/testdata/grammar_outside/nonzero_constsize.gap new file mode 100644 index 000000000..40724e4a3 --- /dev/null +++ b/testdata/grammar_outside/nonzero_constsize.gap @@ -0,0 +1,50 @@ +signature sig_cm(alphabet, answer) { + answer silent_transition(float, answer; int); + answer left_transition(float, alphabet, answer; int); + answer right_transition(float, answer, alphabet; int); + answer pair_transition(float, alphabet, answer, alphabet; int); + answer bifurcation_transition(float, float, answer, answer; int); + answer nil(float, void; int); + choice [answer] h([answer]); +} + + +algebra alg_enum auto enum; + +algebra alg_count auto count; + +grammar gra_cm uses sig_cm(axiom = state_B_3) { + state_B_3 = bifurcation_transition(CONST_FLOAT(0.000000), CONST_FLOAT(0.000000), state_S_4, state_S_12; 3) + # h; + + state_S_4 = silent_transition(CONST_FLOAT(-9.761000), state_ML_6; 4) + | silent_transition(CONST_FLOAT(-9.808000), state_D_8; 4) + # h; + + state_ML_6 = left_transition(CONST_FLOAT(-1.000000), CHAR, state_E_11; 6) + # h; + + state_D_8 = silent_transition(CONST_FLOAT(-1.000000), state_E_11; 8) + # h; + + state_E_11 = nil(CONST_FLOAT(0.000000), EMPTY; 11) + # h; + + + state_S_12 = silent_transition(CONST_FLOAT(-10.445000), state_ML_15; 12) + | silent_transition(CONST_FLOAT(-11.549000), state_D_17; 12) + # h; + + state_ML_15 = left_transition(CONST_FLOAT(-1.000000), CHAR, state_E_20; 15) + # h; + + state_D_17 = silent_transition(CONST_FLOAT(-1.000000), state_E_20; 17) + # h; + + state_E_20 = nil(CONST_FLOAT(0.000000), EMPTY; 20) + # h; + +} + +instance count = gra_cm(alg_count); + \ No newline at end of file From 5a6cea134ba534450b0f330f2b4cd00022379013 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sat, 11 Feb 2023 22:06:16 +0100 Subject: [PATCH 162/209] adding full TMHMM model for testing --- testdata/gapc_filter/ext_hmm.hh | 102 ++++ testdata/grammar_outside/tmhmm.gap | 951 +++++++++++++++++++++++++++++ 2 files changed, 1053 insertions(+) create mode 100644 testdata/gapc_filter/ext_hmm.hh create mode 100644 testdata/grammar_outside/tmhmm.gap diff --git a/testdata/gapc_filter/ext_hmm.hh b/testdata/gapc_filter/ext_hmm.hh new file mode 100644 index 000000000..f9afeb5a1 --- /dev/null +++ b/testdata/gapc_filter/ext_hmm.hh @@ -0,0 +1,102 @@ +#ifndef EXT_HMM_HH +#define EXT_HMM_HH + +template +inline +T negexpsum(T t) { + return t; +} + +template +inline +typename std::iterator_traits::value_type negexpsum(Itr begin, Itr end) { + typename std::iterator_traits::value_type n; + if (begin == end) { + empty(n); + return n; + } + assert(!isEmpty(*begin)); + n = exp(-1.0 * *begin); + ++begin; + for (; begin != end; ++begin) { + assert(!isEmpty(*begin)); + n += exp(-1.0 * *begin); + } + assert((n > 0 && "Your algebra produces (partial) candidates with negative " + "score, which cannot be logarithmized. Avoid h=negexpsum or " + "ensure all positive values!")); + return -1.0 * log(n); +} + +template +inline +typename std::iterator_traits::value_type +negexpsum(std::pair &p) { + return negexpsum(p.first, p.second); +} + +const static double THRESHOLD = 0.000000001; +inline List_Ref > filterLowProbLabels(List_Ref > candidates_orig) { + //return candidates_orig; + // do nothing if there is just one candidate at all + if (candidates_orig.ref().size() <= 1) { + return candidates_orig; + } + + List_Ref > candidates; + // obtain most probable candidate up to here + double bestValue = (*candidates_orig.ref().begin()).second; + for (List_Ref >::iterator i = candidates_orig.ref().begin(); i != candidates_orig.ref().end(); ++i) { + if (bestValue > (*i).second) { + bestValue = (*i).second; + } + } + + // only keep candidates that are not smaller than the best candidate times THRESHOLD + // a value that should be changable by the user instead of hard coding it here + for (List_Ref >::iterator i = candidates_orig.ref().begin(); i != candidates_orig.ref().end(); ++i) { + if ((*i).second <= (bestValue + log(1/THRESHOLD))) { + candidates.ref().push_back(*i); + } + } + return candidates; +} + +//template +//struct filterLowProbLabels2 { +// double bestValue; +// filterLowProbLabels2() : bestValue(0.0) { +// } +// void update(const T &src) { +// if (src.second > bestValue) { +// bestValue = src.second; +// } +// } +// bool ok(const T &x) const { +// return x.second <= lowProbabilityFilter(); +//// //sum = sum + x.second; +//// //return true; +//// double thresh = pow(22.56 * lowProbabilityFilter(), (x.first.size()-1)); +//// //double thresh = 0.1 * sum; +////// if (sum > 0) { +//// std::cerr << sum << ", " << x.second << "\t" << x.first << "\n"; +////// } +////// if (x.second != 1.0) { +////// std::cerr << (thresh > x.second) << "\t" << thresh << "\t" << x.second << "\n"; +////// } +//// //std::cerr << (pow(0.01, x.first.size())) << " " << x.second << "\n"; // "thresh: " << x.second << ", sum: " << sum << "\n"; +//// //return (sum > 0.01) || (count <= 1); +//// return true; //x.second > 0.3; +//// return (x.second > thresh) || (x.second == 1.0); +//// //return x.second > 0.5; +// } +//}; +// +//template +//inline double getPfuncValue(std::pair x) { +// return sum_elems(x.second.pf); +//} + + + +#endif diff --git a/testdata/grammar_outside/tmhmm.gap b/testdata/grammar_outside/tmhmm.gap new file mode 100644 index 000000000..655cfdb64 --- /dev/null +++ b/testdata/grammar_outside/tmhmm.gap @@ -0,0 +1,951 @@ +import "ext_hmm.hh" +type Rope = extern + +signature sig_tmhmm(alphabet, answer) { + answer silent_transition(float, answer); + answer transition(char, float, answer, answer); + answer nil(void); + answer emission(float, alphabet); + choice [answer] h([answer]); +} + +algebra alg_enum auto enum; + +algebra alg_count auto count; + +algebra alg_viterbi implements sig_tmhmm(alphabet=char, answer=float) { + float silent_transition(float prob, float t) { + return prob * t; + } + float transition(char label, float prob, float e, float t) { + return prob * e * t; + } + float nil(void) { + return 1.0; + } + float emission(float prob, char emission) { + return prob; + } + choice [float] h([float] candidates) { + return list(maximum(candidates)); + } +} + +algebra alg_fwd extends alg_viterbi { + choice [float] h([float] candidates) { + return list(sum(candidates)); + } +} + +algebra alg_fwd_scaled extends alg_viterbi { + float emission(float prob, char emission) { + /* 43.38 is a scaling factor against numeric instability, + * as candidate probabilities tend to become very small. + * The value is 1 / median of all emission probabilities + * in the TMHMM2 model; but in principle can be any value > 1. + */ + return 22.56 * prob; + } + float normalize_derivative(float q, float pfunc) { + return q / pfunc; + } + choice [float] h([float] candidates) { + return list(sum(candidates)); + } +} + +algebra alg_viterbi_bit extends alg_viterbi { + float silent_transition(float prob, float t) { + return log(1.0/prob) + t; + } + float transition(char label, float prob, float e, float t) { + return log(1.0/prob) + e + t; + } + float nil(void) { + return 0.0; + } + float emission(float prob, char emission) { + return log(1.0/prob); + } + choice [float] h([float] candidates) { + return list(minimum(candidates)); + } +} + +algebra alg_fwd_bit extends alg_viterbi_bit { + float normalize_derivative(float q, float pfunc) { + return exp(pfunc - q); + } + choice [float] h([float] candidates) { + return list(negexpsum(candidates)); + } +} + +algebra alg_label implements sig_tmhmm(alphabet=char, answer=Rope) { + Rope silent_transition(float prob, Rope x) { + return x; + } + Rope transition(char label, float prob, Rope e, Rope t) { + Rope res; + append(res, label); + append(res, t); + return res; + } + Rope nil(void) { + Rope res; + return res; + } + Rope emission(float prob, char emission) { + Rope res; + return res; + } + choice [Rope] h([Rope] candidates) { + return unique(candidates); + } +} + +grammar gra_tmhmm uses sig_tmhmm(axiom = state_begin) { + state_begin = + silent_transition(CONST_FLOAT(0.549251), state_in10) | + silent_transition(CONST_FLOAT(0.207469), state_outglob10) | + silent_transition(CONST_FLOAT(0.24328), state_out10) | + state_end + # h; + + state_in10 = + transition(CONST_CHAR('i'), CONST_FLOAT(0.995851), emit_in10, state_in11) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.00111283), emit_in10, state_in29) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.00303586), emit_in10, state_ohelixi1) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + emit_in10 = + emission(CONST_FLOAT(0.0713632), CHAR('A')) | + emission(CONST_FLOAT(0.0188491), CHAR('C')) | + emission(CONST_FLOAT(0.043014), CHAR('D')) | + emission(CONST_FLOAT(0.0471663), CHAR('E')) | + emission(CONST_FLOAT(0.0298789), CHAR('F')) | + emission(CONST_FLOAT(0.0564853), CHAR('G')) | + emission(CONST_FLOAT(0.0233906), CHAR('H')) | + emission(CONST_FLOAT(0.038904), CHAR('I')) | + emission(CONST_FLOAT(0.0894525), CHAR('K')) | + emission(CONST_FLOAT(0.0551519), CHAR('L')) | + emission(CONST_FLOAT(0.0427067), CHAR('M')) | + emission(CONST_FLOAT(0.0544149), CHAR('N')) | + emission(CONST_FLOAT(0.0370019), CHAR('P')) | + emission(CONST_FLOAT(0.0524006), CHAR('Q')) | + emission(CONST_FLOAT(0.114758), CHAR('R')) | + emission(CONST_FLOAT(0.0661936), CHAR('S')) | + emission(CONST_FLOAT(0.0689907), CHAR('T')) | + emission(CONST_FLOAT(0.0416332), CHAR('V')) | + emission(CONST_FLOAT(0.0146085), CHAR('W')) | + emission(CONST_FLOAT(0.0336358), CHAR('Y')) + # h; + + state_in11 = + transition(CONST_CHAR('i'), CONST_FLOAT(0.976066), emit_in10, state_in12) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.0239339), emit_in10, state_in28) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.08323e-09), emit_in10, state_in29) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in12 = + transition(CONST_CHAR('i'), CONST_FLOAT(0.895077), emit_in10, state_in13) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.104922), emit_in10, state_in27) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.76891e-06), emit_in10, state_in28) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in13 = + transition(CONST_CHAR('i'), CONST_FLOAT(0.979527), emit_in10, state_in14) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.0204673), emit_in10, state_in26) | + transition(CONST_CHAR('i'), CONST_FLOAT(5.81809e-06), emit_in10, state_in27) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in14 = + transition(CONST_CHAR('i'), CONST_FLOAT(0.993341), emit_in10, state_in15) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.00660812), emit_in10, state_in25) | + transition(CONST_CHAR('i'), CONST_FLOAT(5.08664e-05), emit_in10, state_in26) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in15 = + transition(CONST_CHAR('i'), CONST_FLOAT(0.738969), emit_in10, state_in16) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.159734), emit_in10, state_in24) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.101297), emit_in10, state_in25) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in16 = + transition(CONST_CHAR('i'), CONST_FLOAT(0.999938), emit_in10, state_in17) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.4427e-06), emit_in10, state_in23) | + transition(CONST_CHAR('i'), CONST_FLOAT(6.0424e-05), emit_in10, state_in24) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in17 = + transition(CONST_CHAR('i'), CONST_FLOAT(0.973203), emit_in10, state_in18) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.0168132), emit_in10, state_in22) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.00998417), emit_in10, state_in23) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in18 = + transition(CONST_CHAR('i'), CONST_FLOAT(0.977498), emit_in10, state_in19) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.0217216), emit_in10, state_in21) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.000780768), emit_in10, state_in22) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in19 = + transition(CONST_CHAR('i'), CONST_FLOAT(0.16223), emit_in10, state_in20) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.10126), emit_in10, state_in21) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.73651), emit_in10, state_inglob1) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in20 = + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_in21) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in21 = + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_in22) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in22 = + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_in23) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in23 = + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_in24) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in24 = + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_in25) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in25 = + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_in26) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in26 = + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_in27) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in27 = + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_in28) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in28 = + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_in29) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_in29 = + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_ohelixi1) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_in10, state_end) + # h; + + state_inglob1 = + transition(CONST_CHAR('i'), CONST_FLOAT(0.0132918), emit_inglob1, state_in20) | + transition(CONST_CHAR('i'), CONST_FLOAT(0.986708), emit_inglob1, state_inglob1) | + transition(CONST_CHAR('i'), CONST_FLOAT(1.0), emit_inglob1, state_end) + # h; + emit_inglob1 = + emission(CONST_FLOAT(0.0773341), CHAR('A')) | + emission(CONST_FLOAT(0.0212026), CHAR('C')) | + emission(CONST_FLOAT(0.0556231), CHAR('D')) | + emission(CONST_FLOAT(0.0789783), CHAR('E')) | + emission(CONST_FLOAT(0.0291466), CHAR('F')) | + emission(CONST_FLOAT(0.0821038), CHAR('G')) | + emission(CONST_FLOAT(0.02529), CHAR('H')) | + emission(CONST_FLOAT(0.0392883), CHAR('I')) | + emission(CONST_FLOAT(0.0466567), CHAR('K')) | + emission(CONST_FLOAT(0.0718204), CHAR('L')) | + emission(CONST_FLOAT(0.0191835), CHAR('M')) | + emission(CONST_FLOAT(0.0490524), CHAR('N')) | + emission(CONST_FLOAT(0.0671432), CHAR('P')) | + emission(CONST_FLOAT(0.0472671), CHAR('Q')) | + emission(CONST_FLOAT(0.0492684), CHAR('R')) | + emission(CONST_FLOAT(0.0852997), CHAR('S')) | + emission(CONST_FLOAT(0.0610192), CHAR('T')) | + emission(CONST_FLOAT(0.0528717), CHAR('V')) | + emission(CONST_FLOAT(0.0166592), CHAR('W')) | + emission(CONST_FLOAT(0.0247916), CHAR('Y')) + # h; + + state_outglob10 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob11) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + emit_outglob10 = + emission(CONST_FLOAT(0.0693743), CHAR('A')) | + emission(CONST_FLOAT(0.0149605), CHAR('C')) | + emission(CONST_FLOAT(0.0406956), CHAR('D')) | + emission(CONST_FLOAT(0.0538397), CHAR('E')) | + emission(CONST_FLOAT(0.0531778), CHAR('F')) | + emission(CONST_FLOAT(0.0792746), CHAR('G')) | + emission(CONST_FLOAT(0.0221055), CHAR('H')) | + emission(CONST_FLOAT(0.0440866), CHAR('I')) | + emission(CONST_FLOAT(0.0565779), CHAR('K')) | + emission(CONST_FLOAT(0.0988165), CHAR('L')) | + emission(CONST_FLOAT(0.0432829), CHAR('M')) | + emission(CONST_FLOAT(0.0414346), CHAR('N')) | + emission(CONST_FLOAT(0.0615562), CHAR('P')) | + emission(CONST_FLOAT(0.0412212), CHAR('Q')) | + emission(CONST_FLOAT(0.0677628), CHAR('R')) | + emission(CONST_FLOAT(0.0732544), CHAR('S')) | + emission(CONST_FLOAT(0.0524824), CHAR('T')) | + emission(CONST_FLOAT(0.0445653), CHAR('V')) | + emission(CONST_FLOAT(0.0140309), CHAR('W')) | + emission(CONST_FLOAT(0.0275), CHAR('Y')) + # h; + + state_outglob11 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob12) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob12 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob13) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob13 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob14) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob14 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob15) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob15 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob16) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob16 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob17) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob17 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob18) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob18 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob19) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob19 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglobLong) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglobLong = + transition(CONST_CHAR('O'), CONST_FLOAT(0.999093), emit_inglob1, state_outglobLong) | + transition(CONST_CHAR('O'), CONST_FLOAT(0.000906913), emit_inglob1, state_outglob20) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_inglob1, state_end) + # h; + + state_outglob20 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob21) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob21 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob22) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob22 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob23) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob23 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob24) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob24 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob25) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob25 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob26) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob26 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob27) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob27 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob28) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob28 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_outglob29) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_outglob29 = + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_ihelixo1) | + transition(CONST_CHAR('O'), CONST_FLOAT(1.0), emit_outglob10, state_end) + # h; + + state_out10 = + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_out11) | + transition(CONST_CHAR('o'), CONST_FLOAT(3.54571e-14), emit_out10, state_out29) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + emit_out10 = + emission(CONST_FLOAT(0.0690346), CHAR('A')) | + emission(CONST_FLOAT(0.0129673), CHAR('C')) | + emission(CONST_FLOAT(0.0627756), CHAR('D')) | + emission(CONST_FLOAT(0.0541851), CHAR('E')) | + emission(CONST_FLOAT(0.0425402), CHAR('F')) | + emission(CONST_FLOAT(0.0835626), CHAR('G')) | + emission(CONST_FLOAT(0.0271581), CHAR('H')) | + emission(CONST_FLOAT(0.0292546), CHAR('I')) | + emission(CONST_FLOAT(0.0530401), CHAR('K')) | + emission(CONST_FLOAT(0.0822093), CHAR('L')) | + emission(CONST_FLOAT(0.0334625), CHAR('M')) | + emission(CONST_FLOAT(0.0506017), CHAR('N')) | + emission(CONST_FLOAT(0.0693889), CHAR('P')) | + emission(CONST_FLOAT(0.0389539), CHAR('Q')) | + emission(CONST_FLOAT(0.0432109), CHAR('R')) | + emission(CONST_FLOAT(0.0863749), CHAR('S')) | + emission(CONST_FLOAT(0.0587278), CHAR('T')) | + emission(CONST_FLOAT(0.0480586), CHAR('V')) | + emission(CONST_FLOAT(0.0198473), CHAR('W')) | + emission(CONST_FLOAT(0.0346461), CHAR('Y')) + # h; + + state_out11 = + transition(CONST_CHAR('o'), CONST_FLOAT(0.910217), emit_out10, state_out12) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.0495866), emit_out10, state_out28) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.0401965), emit_out10, state_out29) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out12 = + transition(CONST_CHAR('o'), CONST_FLOAT(0.984498), emit_out10, state_out13) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.00440955), emit_out10, state_out27) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.0110921), emit_out10, state_out28) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out13 = + transition(CONST_CHAR('o'), CONST_FLOAT(0.997286), emit_out10, state_out14) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.00258189), emit_out10, state_out26) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.000132519), emit_out10, state_out27) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out14 = + transition(CONST_CHAR('o'), CONST_FLOAT(0.812906), emit_out10, state_out15) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.0130127), emit_out10, state_out25) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.174082), emit_out10, state_out26) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out15 = + transition(CONST_CHAR('o'), CONST_FLOAT(0.951951), emit_out10, state_out16) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.0400992), emit_out10, state_out24) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.00795001), emit_out10, state_out25) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out16 = + transition(CONST_CHAR('o'), CONST_FLOAT(0.660205), emit_out10, state_out17) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.33979), emit_out10, state_out23) | + transition(CONST_CHAR('o'), CONST_FLOAT(4.76867e-06), emit_out10, state_out24) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out17 = + transition(CONST_CHAR('o'), CONST_FLOAT(0.992733), emit_out10, state_out18) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.0072618), emit_out10, state_out22) | + transition(CONST_CHAR('o'), CONST_FLOAT(5.34599e-06), emit_out10, state_out23) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out18 = + transition(CONST_CHAR('o'), CONST_FLOAT(0.971165), emit_out10, state_out19) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.0119931), emit_out10, state_out21) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.0168416), emit_out10, state_out22) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out19 = + transition(CONST_CHAR('o'), CONST_FLOAT(0.770716), emit_out10, state_outglobShort) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.00133523), emit_out10, state_out20) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.227948), emit_out10, state_out21) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_outglobShort = + transition(CONST_CHAR('o'), CONST_FLOAT(0.960334), emit_inglob1, state_outglobShort) | + transition(CONST_CHAR('o'), CONST_FLOAT(0.0396664), emit_inglob1, state_out20) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_inglob1, state_end) + # h; + + state_out20 = + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_out21) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out21 = + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_out22) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out22 = + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_out23) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out23 = + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_out24) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out24 = + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_out25) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out25 = + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_out26) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out26 = + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_out27) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out27 = + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_out28) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out28 = + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_out29) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_out29 = + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_ihelixo1) | + transition(CONST_CHAR('o'), CONST_FLOAT(1.0), emit_out10, state_end) + # h; + + state_ohelixi1 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixi1, state_ohelixi2) + # h; + emit_ohelixi1 = + emission(CONST_FLOAT(0.0890049), CHAR('A')) | + emission(CONST_FLOAT(0.0170933), CHAR('C')) | + emission(CONST_FLOAT(0.00159548), CHAR('D')) | + emission(CONST_FLOAT(0.00489647), CHAR('E')) | + emission(CONST_FLOAT(0.108081), CHAR('F')) | + emission(CONST_FLOAT(0.0692068), CHAR('G')) | + emission(CONST_FLOAT(0.00923517), CHAR('H')) | + emission(CONST_FLOAT(0.113471), CHAR('I')) | + emission(CONST_FLOAT(0.00466208), CHAR('K')) | + emission(CONST_FLOAT(0.19417), CHAR('L')) | + emission(CONST_FLOAT(0.0239901), CHAR('M')) | + emission(CONST_FLOAT(0.0233656), CHAR('N')) | + emission(CONST_FLOAT(0.0145168), CHAR('P')) | + emission(CONST_FLOAT(0.00487025), CHAR('Q')) | + emission(CONST_FLOAT(0.0127643), CHAR('R')) | + emission(CONST_FLOAT(0.0455139), CHAR('S')) | + emission(CONST_FLOAT(0.0292949), CHAR('T')) | + emission(CONST_FLOAT(0.128208), CHAR('V')) | + emission(CONST_FLOAT(0.0399529), CHAR('W')) | + emission(CONST_FLOAT(0.0661077), CHAR('Y')) + # h; + + state_ohelixi2 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixi1, state_ohelixi3) + # h; + + state_ohelixi3 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixi1, state_ohelixi4) + # h; + + state_ohelixi4 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixi1, state_ohelixi5) + # h; + + state_ohelixi5 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixi1, state_ohelixi6) + # h; + + state_ohelixi6 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelixi7) + # h; + + state_ohelixi7 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelixm) + # h; + + state_ohelixm = + transition(CONST_CHAR('M'), CONST_FLOAT(0.000534023), emit_ohelixm, state_ohelix2) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.000567583), emit_ohelixm, state_ohelix3) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00064457), emit_ohelixm, state_ohelix4) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00159544), emit_ohelixm, state_ohelix5) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.000669433), emit_ohelixm, state_ohelix6) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00161008), emit_ohelixm, state_ohelix7) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.000762887), emit_ohelixm, state_ohelix8) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.000789084), emit_ohelixm, state_ohelix9) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.000868204), emit_ohelixm, state_ohelix10) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00519996), emit_ohelixm, state_ohelix11) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00774891), emit_ohelixm, state_ohelix12) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.0108947), emit_ohelixm, state_ohelix13) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.697722), emit_ohelixm, state_ohelix14) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.0444777), emit_ohelixm, state_ohelix15) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.0328707), emit_ohelixm, state_ohelix16) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.111827), emit_ohelixm, state_ohelix17) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.0324248), emit_ohelixm, state_ohelix18) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.0448694), emit_ohelixm, state_ohelix19) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00212234), emit_ohelixm, state_ohelix20) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00107586), emit_ohelixm, state_ohelix21) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00072618), emit_ohelixm, state_ohelixo1) + # h; + emit_ohelixm = + emission(CONST_FLOAT(0.110896), CHAR('A')) | + emission(CONST_FLOAT(0.0254931), CHAR('C')) | + emission(CONST_FLOAT(0.00235724), CHAR('D')) | + emission(CONST_FLOAT(0.00383965), CHAR('E')) | + emission(CONST_FLOAT(0.0942003), CHAR('F')) | + emission(CONST_FLOAT(0.0818095), CHAR('G')) | + emission(CONST_FLOAT(0.00408389), CHAR('H')) | + emission(CONST_FLOAT(0.144377), CHAR('I')) | + emission(CONST_FLOAT(0.00236432), CHAR('K')) | + emission(CONST_FLOAT(0.182902), CHAR('L')) | + emission(CONST_FLOAT(0.0385107), CHAR('M')) | + emission(CONST_FLOAT(0.00978815), CHAR('N')) | + emission(CONST_FLOAT(0.0201094), CHAR('P')) | + emission(CONST_FLOAT(0.00437833), CHAR('Q')) | + emission(CONST_FLOAT(0.00115335), CHAR('R')) | + emission(CONST_FLOAT(0.0421756), CHAR('S')) | + emission(CONST_FLOAT(0.0514071), CHAR('T')) | + emission(CONST_FLOAT(0.132167), CHAR('V')) | + emission(CONST_FLOAT(0.0158643), CHAR('W')) | + emission(CONST_FLOAT(0.0321232), CHAR('Y')) + # h; + + state_ohelix2 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix3) + # h; + + state_ohelix3 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix4) + # h; + + state_ohelix4 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix5) + # h; + + state_ohelix5 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix6) + # h; + + state_ohelix6 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix7) + # h; + + state_ohelix7 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix8) + # h; + + state_ohelix8 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix9) + # h; + + state_ohelix9 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix10) + # h; + + state_ohelix10 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix11) + # h; + + state_ohelix11 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix12) + # h; + + state_ohelix12 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix13) + # h; + + state_ohelix13 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix14) + # h; + + state_ohelix14 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix15) + # h; + + state_ohelix15 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix16) + # h; + + state_ohelix16 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix17) + # h; + + state_ohelix17 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix18) + # h; + + state_ohelix18 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix19) + # h; + + state_ohelix19 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix20) + # h; + + state_ohelix20 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelix21) + # h; + + state_ohelix21 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelixo1) + # h; + + state_ohelixo1 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelixo2) + # h; + + state_ohelixo2 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ohelixo3) + # h; + + state_ohelixo3 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixo7, state_ohelixo4) + # h; + + state_ohelixo4 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixo7, state_ohelixo5) + # h; + + state_ohelixo5 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixo7, state_ohelixo6) + # h; + + state_ohelixo6 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixo7, state_ohelixo7) + # h; + + state_ohelixo7 = + transition(CONST_CHAR('M'), CONST_FLOAT(0.0757404), emit_ohelixo7, state_outglob10) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.92426), emit_ohelixo7, state_out10) + # h; + emit_ohelixo7 = + emission(CONST_FLOAT(0.110353), CHAR('A')) | + emission(CONST_FLOAT(0.00498206), CHAR('C')) | + emission(CONST_FLOAT(0.00469973), CHAR('D')) | + emission(CONST_FLOAT(0.00649427), CHAR('E')) | + emission(CONST_FLOAT(0.0973043), CHAR('F')) | + emission(CONST_FLOAT(0.0737631), CHAR('G')) | + emission(CONST_FLOAT(0.0119931), CHAR('H')) | + emission(CONST_FLOAT(0.12167), CHAR('I')) | + emission(CONST_FLOAT(0.00180854), CHAR('K')) | + emission(CONST_FLOAT(0.157124), CHAR('L')) | + emission(CONST_FLOAT(0.044721), CHAR('M')) | + emission(CONST_FLOAT(0.0107496), CHAR('N')) | + emission(CONST_FLOAT(0.0283821), CHAR('P')) | + emission(CONST_FLOAT(0.0143416), CHAR('Q')) | + emission(CONST_FLOAT(0.00857182), CHAR('R')) | + emission(CONST_FLOAT(0.0402204), CHAR('S')) | + emission(CONST_FLOAT(0.0501039), CHAR('T')) | + emission(CONST_FLOAT(0.107462), CHAR('V')) | + emission(CONST_FLOAT(0.0501891), CHAR('W')) | + emission(CONST_FLOAT(0.0550665), CHAR('Y')) + # h; + + state_ihelixo1 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixo7, state_ihelixo2) + # h; + + state_ihelixo2 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixo7, state_ihelixo3) + # h; + + state_ihelixo3 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixo7, state_ihelixo4) + # h; + + state_ihelixo4 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixo7, state_ihelixo5) + # h; + + state_ihelixo5 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixo7, state_ihelixo6) + # h; + + state_ihelixo6 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelixo7) + # h; + + state_ihelixo7 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelixm) + # h; + + state_ihelixm = + transition(CONST_CHAR('M'), CONST_FLOAT(0.000534023), emit_ohelixm, state_ihelix2) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.000567583), emit_ohelixm, state_ihelix3) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00064457), emit_ohelixm, state_ihelix4) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00159544), emit_ohelixm, state_ihelix5) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.000669433), emit_ohelixm, state_ihelix6) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00161008), emit_ohelixm, state_ihelix7) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.000762887), emit_ohelixm, state_ihelix8) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.000789084), emit_ohelixm, state_ihelix9) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.000868204), emit_ohelixm, state_ihelix10) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00519996), emit_ohelixm, state_ihelix11) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00774891), emit_ohelixm, state_ihelix12) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.0108947), emit_ohelixm, state_ihelix13) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.697722), emit_ohelixm, state_ihelix14) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.0444777), emit_ohelixm, state_ihelix15) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.0328707), emit_ohelixm, state_ihelix16) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.111827), emit_ohelixm, state_ihelix17) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.0324248), emit_ohelixm, state_ihelix18) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.0448694), emit_ohelixm, state_ihelix19) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00212234), emit_ohelixm, state_ihelix20) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00107586), emit_ohelixm, state_ihelix21) | + transition(CONST_CHAR('M'), CONST_FLOAT(0.00072618), emit_ohelixm, state_ihelixi1) + # h; + + state_ihelix2 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix3) + # h; + + state_ihelix3 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix4) + # h; + + state_ihelix4 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix5) + # h; + + state_ihelix5 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix6) + # h; + + state_ihelix6 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix7) + # h; + + state_ihelix7 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix8) + # h; + + state_ihelix8 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix9) + # h; + + state_ihelix9 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix10) + # h; + + state_ihelix10 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix11) + # h; + + state_ihelix11 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix12) + # h; + + state_ihelix12 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix13) + # h; + + state_ihelix13 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix14) + # h; + + state_ihelix14 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix15) + # h; + + state_ihelix15 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix16) + # h; + + state_ihelix16 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix17) + # h; + + state_ihelix17 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix18) + # h; + + state_ihelix18 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix19) + # h; + + state_ihelix19 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix20) + # h; + + state_ihelix20 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelix21) + # h; + + state_ihelix21 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelixi1) + # h; + + state_ihelixi1 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelixi2) + # h; + + state_ihelixi2 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixm, state_ihelixi3) + # h; + + state_ihelixi3 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixi1, state_ihelixi4) + # h; + + state_ihelixi4 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixi1, state_ihelixi5) + # h; + + state_ihelixi5 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixi1, state_ihelixi6) + # h; + + state_ihelixi6 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixi1, state_ihelixi7) + # h; + + state_ihelixi7 = + transition(CONST_CHAR('M'), CONST_FLOAT(1.0), emit_ohelixi1, state_in10) + # h; + + state_end = nil(EMPTY) # h; +} + +instance dummy = gra_tmhmm(alg_enum); +instance fwd_scaled = gra_tmhmm(alg_fwd_scaled); \ No newline at end of file From 46e6c29baab47f7483d86f691404655d61cbf3b2 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sat, 11 Feb 2023 23:26:26 +0100 Subject: [PATCH 163/209] move declarations to front for debugging + inject left/right most depending on MISSING dimensions --- src/cpp.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 5d0acf115..5e6d28b04 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2210,6 +2210,10 @@ void Printer::Cpp::print_cyk_fn(const AST &ast) { } void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { + std::stringstream list_args_print; + std::stringstream list_args; + std::list &l = nt->code_list(); + // aggregated level (=dim + tracks) of nested loops unsigned int nesting = 0; std::vector *args = new std::vector(); @@ -2240,10 +2244,10 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { args_call->push_back("t_" + std::to_string(track) + "_j"); } if (nt->is_inside_axiom) { - if (dim == 0) { + if (dim <= 1) { args_call->push_back("t_" + std::to_string(track) + "_left_most"); } - if (dim == 1) { + if (dim < 2) { args_call->push_back("t_" + std::to_string(track) + "_right_most"); } } @@ -2251,8 +2255,6 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { } // loop body - std::list &l = nt->code_list(); - std::stringstream list_args_print; bool first = true; for (std::vector::const_iterator a = args->begin(); a != args->end(); ++a) { @@ -2262,7 +2264,6 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { first = false; list_args_print << " << " << *a; } - std::stringstream list_args; first = true; for (std::vector::const_iterator a = args_call->begin(); a != args_call->end(); ++a) { From 43e29d66a2e7aaa41ee8d5b60875b7857236c8b0 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sat, 11 Feb 2023 23:27:14 +0100 Subject: [PATCH 164/209] add executing of two more test cases --- testdata/regresstest/config | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 96b70ec55..cedc508ac 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -499,4 +499,12 @@ check_new_old_eq_twotrack alignments.gap unused sim_enum "AA" outside_undefvar " check_new_old_eq elmamun.gap unused count "1+2*3" outside_elm # check correct indices for multiple CONST_FLOAT args for outside generation; caused by incorrectly set yield size when replacing NTs -check_new_old_eq nonzero_constsize.gap unused count "G" outside_constissue \ No newline at end of file +check_new_old_eq nonzero_constsize.gap unused count "G" outside_constissue + +# check TMHMM outside with real sequence which previously revealed < 1.0 prob sums +CPPFLAGS_EXTRA=$DEFAULT_CPPFLAGS_EXTRA +check_new_old_eq tmhmm.gap unused fwd_scaled "MENLNMDLLYMAAAVMMGLAAIGAAIGIGILGGKFLEGAARQPDLIPLLRTQFFIVMGLVDAIPMIAVGLGLYVMFAVA" outside_tmhmm +CPPFLAGS_EXTRA="" + +# test if outside to inside transitions works also for constant non-terminal tables, i.e. those missing t_0_i t_0_j parameters which should be replaced via dummies in this one and only situation +check_new_old_eq constAxiom.gap unused enum "G" outside_out2in From 479ca8a156ad58913b5e6b02db41f3d7ffda0c8b Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 13 Feb 2023 13:48:29 +0100 Subject: [PATCH 165/209] adding further test cases --- testdata/gapc_filter/RFmini.hh | 1 + testdata/grammar_outside/RFmini.gap | 155 ++++++++++++++++++++++++ testdata/grammar_outside/constAxiom.gap | 35 ++++++ 3 files changed, 191 insertions(+) create mode 120000 testdata/gapc_filter/RFmini.hh create mode 100644 testdata/grammar_outside/RFmini.gap create mode 100644 testdata/grammar_outside/constAxiom.gap diff --git a/testdata/gapc_filter/RFmini.hh b/testdata/gapc_filter/RFmini.hh new file mode 120000 index 000000000..349324b2b --- /dev/null +++ b/testdata/gapc_filter/RFmini.hh @@ -0,0 +1 @@ +../ADP_collection/CM/RFmini.hh \ No newline at end of file diff --git a/testdata/grammar_outside/RFmini.gap b/testdata/grammar_outside/RFmini.gap new file mode 100644 index 000000000..240064caa --- /dev/null +++ b/testdata/grammar_outside/RFmini.gap @@ -0,0 +1,155 @@ + +import "RFmini.hh" + +signature sig_cm(alphabet, answer) { + answer silent_transition(float, answer); + answer left_transition(float, alphabet, answer; int); + answer right_transition(float, answer, alphabet; int); + answer pair_transition(float, alphabet, answer, alphabet; int); + answer bifurcation_transition(float, float, answer, answer; int); + answer nil(float, void; int); + choice [answer] h([answer]); +} + + +algebra alg_enum auto enum; + +algebra alg_count auto count; + +algebra alg_cyk implements sig_cm(alphabet=char, answer=float) { + float silent_transition(float tsc, float x) { + return tsc + x; + } + float left_transition(float tsc, alphabet a, float x; int pos) { + return tsc + getEmission(pos, a) + x; + } + float right_transition(float tsc, float x, alphabet a; int pos) { + return tsc + getEmission(pos, a) + x; + } + float pair_transition(float tsc, alphabet a, float x, alphabet b; int pos) { + return tsc + getPairEmission(pos, a, b) + x; + } + float bifurcation_transition(float tsc_left, float tsc_right, float left, float right; int pos) { + return tsc_left + tsc_right + left + right; + } + float nil(float tsc, void; int pos) { + return tsc; + } + choice [float] h([float] candidates) { + return list(maximum(candidates)); + } +} + +algebra alg_inside extends alg_cyk { + choice [float] h([float] candidates) { + return list(bitsum(candidates)); + } +} + +grammar gra_cm uses sig_cm(axiom = state_S_0) { + // node [ ROOT 0 ] + state_S_0 = silent_transition(CONST_FLOAT(-9.486000), state_IL_1) + | silent_transition(CONST_FLOAT(-19.955000), state_IR_2) + | silent_transition(CONST_FLOAT(-0.002000), state_B_3) + # h; + + state_IL_1 = left_transition(CONST_FLOAT(-0.610000), CHAR, state_IL_1; 1) + | left_transition(CONST_FLOAT(-4.491000), CHAR, state_IR_2; 1) + | left_transition(CONST_FLOAT(-1.736000), CHAR, state_B_3; 1) + # h; + + state_IR_2 = right_transition(CONST_FLOAT(-1.823000), state_IR_2, CHAR; 2) + | right_transition(CONST_FLOAT(-0.479000), state_B_3, CHAR; 2) + # h; + + // node [ BIF 1 ] + state_B_3 = bifurcation_transition(CONST_FLOAT(0.000000), CONST_FLOAT(0.000000), state_S_4, state_S_12; 3) + # h; + + // node [ BEGL 2 ] + state_S_4 = silent_transition(CONST_FLOAT(-0.006000), state_MP_5) + | silent_transition(CONST_FLOAT(-9.761000), state_ML_6) + | silent_transition(CONST_FLOAT(-9.168000), state_MR_7) + | silent_transition(CONST_FLOAT(-9.808000), state_D_8) + # h; + + // node [ MATP 3 ] + state_MP_5 = pair_transition(CONST_FLOAT(-9.486000), CHAR, state_IL_9, CHAR; 5) + | pair_transition(CONST_FLOAT(-0.002000), CHAR, state_E_11, CHAR; 5) + # h; + + state_ML_6 = left_transition(CONST_FLOAT(-1.000000), CHAR, state_IL_9; 6) + | left_transition(CONST_FLOAT(-1.000000), CHAR, state_E_11; 6) + # h; + + state_MR_7 = right_transition(CONST_FLOAT(-1.000000), state_IL_9, CHAR; 7) + | right_transition(CONST_FLOAT(-1.000000), state_E_11, CHAR; 7) + # h; + + state_D_8 = silent_transition(CONST_FLOAT(-1.000000), state_IL_9) + | silent_transition(CONST_FLOAT(-1.000000), state_E_11) + # h; + + state_IL_9 = left_transition(CONST_FLOAT(-0.544000), CHAR, state_IL_9; 9) + | left_transition(CONST_FLOAT(-1.670000), CHAR, state_E_11; 9) + # h; + + state_IR_10 = right_transition(CONST_FLOAT(-1.823000), state_IR_10, CHAR; 10) + | right_transition(CONST_FLOAT(-0.479000), state_E_11, CHAR; 10) + # h; + + // node [ END 4 ] + state_E_11 = nil(CONST_FLOAT(0.000000), EMPTY; 11) + # h; + + // node [ BEGR 5 ] + state_S_12 = silent_transition(CONST_FLOAT(-10.630000), state_IL_13) + | silent_transition(CONST_FLOAT(-0.003000), state_MP_14) + | silent_transition(CONST_FLOAT(-10.445000), state_ML_15) + | silent_transition(CONST_FLOAT(-10.657000), state_MR_16) + | silent_transition(CONST_FLOAT(-11.549000), state_D_17) + # h; + + state_IL_13 = left_transition(CONST_FLOAT(-2.408000), CHAR, state_IL_13; 13) + | left_transition(CONST_FLOAT(-0.496000), CHAR, state_MP_14; 13) + | left_transition(CONST_FLOAT(-4.087000), CHAR, state_ML_15; 13) + | left_transition(CONST_FLOAT(-5.920000), CHAR, state_MR_16; 13) + | left_transition(CONST_FLOAT(-5.193000), CHAR, state_D_17; 13) + # h; + + // node [ MATP 6 ] + state_MP_14 = pair_transition(CONST_FLOAT(-9.486000), CHAR, state_IL_18, CHAR; 14) + | pair_transition(CONST_FLOAT(-0.002000), CHAR, state_E_20, CHAR; 14) + # h; + + state_ML_15 = left_transition(CONST_FLOAT(-1.000000), CHAR, state_IL_18; 15) + | left_transition(CONST_FLOAT(-1.000000), CHAR, state_E_20; 15) + # h; + + state_MR_16 = right_transition(CONST_FLOAT(-1.000000), state_IL_18, CHAR; 16) + | right_transition(CONST_FLOAT(-1.000000), state_E_20, CHAR; 16) + # h; + + state_D_17 = silent_transition(CONST_FLOAT(-1.000000), state_IL_18) + | silent_transition(CONST_FLOAT(-1.000000), state_E_20) + # h; + + state_IL_18 = left_transition(CONST_FLOAT(-0.544000), CHAR, state_IL_18; 18) + | left_transition(CONST_FLOAT(-1.670000), CHAR, state_E_20; 18) + # h; + + state_IR_19 = right_transition(CONST_FLOAT(-1.823000), state_IR_19, CHAR; 19) + | right_transition(CONST_FLOAT(-0.479000), state_E_20, CHAR; 19) + # h; + + // node [ END 7 ] + state_E_20 = nil(CONST_FLOAT(0.000000), EMPTY; 20) + # h; + +} + +instance count = gra_cm(alg_count); +instance enum = gra_cm(alg_enum); +instance enumcyk = gra_cm(alg_enum * alg_cyk); +instance cykenum = gra_cm(alg_cyk * alg_enum); + \ No newline at end of file diff --git a/testdata/grammar_outside/constAxiom.gap b/testdata/grammar_outside/constAxiom.gap new file mode 100644 index 000000000..f5ddcdbf3 --- /dev/null +++ b/testdata/grammar_outside/constAxiom.gap @@ -0,0 +1,35 @@ +signature sig_test(alphabet, answer) { + answer silent(char, answer); + answer consume_left(char, alphabet, answer); + answer consume_right(char, answer, alphabet); + answer dummy(answer); + answer nil(char, void); + choice [answer] h([answer]); +} + +algebra alg_enum auto enum; + +algebra alg_count auto count; + +grammar gra_test uses sig_test(axiom = Sm1) { + // tabulated {S9} + Sm1 = dummy(S0) + # h; + + S0 = silent(CONST_CHAR('A'), S9) + | silent(CONST_CHAR('B'), S1) + # h; + + S9 = nil(CONST_CHAR('Z'), EMPTY) + # h; + + S1 = consume_left(CONST_CHAR('C'), CHAR, S1) + | consume_right(CONST_CHAR('D'), S1, CHAR) + | silent(CONST_CHAR('E'), S9) + # h; +} + +instance count = gra_test(alg_count); +instance enum = gra_test(alg_enum); + + \ No newline at end of file From 2c8abfed1f729205cf06281a0dffca486509c90f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 13 Feb 2023 13:49:20 +0100 Subject: [PATCH 166/209] include external header files --- testdata/regresstest/config | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index cedc508ac..62d3ac265 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -501,10 +501,13 @@ check_new_old_eq elmamun.gap unused count "1+2*3" outside_elm # check correct indices for multiple CONST_FLOAT args for outside generation; caused by incorrectly set yield size when replacing NTs check_new_old_eq nonzero_constsize.gap unused count "G" outside_constissue +# test if outside to inside transitions works also for constant non-terminal tables, i.e. those missing t_0_i t_0_j parameters which should be replaced via dummies in this one and only situation +check_new_old_eq constAxiom.gap unused enum "G" outside_out2in + # check TMHMM outside with real sequence which previously revealed < 1.0 prob sums CPPFLAGS_EXTRA=$DEFAULT_CPPFLAGS_EXTRA check_new_old_eq tmhmm.gap unused fwd_scaled "MENLNMDLLYMAAAVMMGLAAIGAAIGIGILGGKFLEGAARQPDLIPLLRTQFFIVMGLVDAIPMIAVGLGLYVMFAVA" outside_tmhmm -CPPFLAGS_EXTRA="" -# test if outside to inside transitions works also for constant non-terminal tables, i.e. those missing t_0_i t_0_j parameters which should be replaced via dummies in this one and only situation -check_new_old_eq constAxiom.gap unused enum "G" outside_out2in +check_new_old_eq RFmini.gap unused enum "G" outside_cm +check_new_old_eq RFmini.gap unused count "G" outside_cm +CPPFLAGS_EXTRA="" From 64b23c5f262b074acf0ae71effaefbea2bf4f45a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 13 Feb 2023 17:21:37 +0100 Subject: [PATCH 167/209] add a function to test if table dimension is constant --- src/table.cc | 3 +++ src/table.hh | 1 + 2 files changed, 4 insertions(+) diff --git a/src/table.cc b/src/table.cc index bb92d9722..49b1a5d30 100644 --- a/src/table.cc +++ b/src/table.cc @@ -70,6 +70,9 @@ bool Table::delete_right_index() const { return (dim == CONSTANT || (dim == LINEAR && sticky_ == RIGHT)) && right_rest_.high() == 0; } +bool Table::is_const_table() const { + return delete_left_index() && delete_right_index(); +} void Table::set_sticky(Sticky s) { if (sticky_ != NO_INDEX && sticky_ != s) { diff --git a/src/table.hh b/src/table.hh index e2a8f174e..80de62982 100644 --- a/src/table.hh +++ b/src/table.hh @@ -95,6 +95,7 @@ class Table { } bool delete_left_index() const; bool delete_right_index() const; + bool is_const_table() const; bool is_cyk_const() const { return dim == CONSTANT && From e1179b0c2839b4a4a7eb2b5cbea3d17cf9f71238 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 13 Feb 2023 17:22:15 +0100 Subject: [PATCH 168/209] remove dummy vars again --- src/symbol.hh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/symbol.hh b/src/symbol.hh index cbb7c8195..d8ccc9f02 100644 --- a/src/symbol.hh +++ b/src/symbol.hh @@ -126,15 +126,6 @@ class Base { Loc location; std::vector left_indices; std::vector right_indices; - /* the user defined inside axiom must be called from outside grammar parts - * such that the outside parts consumes the full input sequence(s) (0,n). - * But the inside NT call must parse the remainder, i.e. the empty word - * If however, table dimension of this axiom was reduced, we cannot pass - * prober boundaries. For such cases, we generate dummy variables that - * do not extend table dimensions, but can ensure that (0,0) and (n, 0) are - * passed. */ - std::vector left_indices_outsidedummy; - std::vector right_indices_outsidedummy; bool is(Type t) const { return type == t; } From 49b4af6a4cb71a9cdfef1f2a17338023271c4d0c Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 13 Feb 2023 17:22:33 +0100 Subject: [PATCH 169/209] restore special treatment of outside->inside transition + propagate outside grammar generation information --- src/symbol.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/symbol.cc b/src/symbol.cc index 96543ea4d..2dcef2e06 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -753,7 +753,15 @@ void Symbol::NT::init_indices(Expr::Vacc *left, Expr::Vacc *right, for (std::list::iterator i = alts.begin(); i != alts.end(); ++i) { if (this->is_partof_outside) { - (*i)->init_indices_outside(left, right, k, track, NULL, NULL, false); + if ((*i)->get_is_partof_outside() == false) { + // special treatment for a link to the original inside axiom, ensure + // that it has to consume the full input sequence(s) + (*i)->init_indices_outside( + left->plus(right), right->minus(left), k, track, NULL, NULL, false); + dynamic_cast(*i)->init_outside_guards(); + } else { + (*i)->init_indices_outside(left, right, k, track, NULL, NULL, false); + } } else { (*i)->init_indices(left, right, k, track); } @@ -1228,7 +1236,7 @@ void Symbol::NT::codegen(AST &ast) { } else { f = new Fn_Def(dt, new std::string("nt_" + *name)); } - f->add_para(*this); + f->add_para(*this, ast.grammar()->is_outside()); subopt_header(ast, score_code, f, stmts); From ce55b16eb1576da47450405936a6890948c52c0a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 13 Feb 2023 17:23:11 +0100 Subject: [PATCH 170/209] revert dummy var index injection --- src/grammar.cc | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/grammar.cc b/src/grammar.cc index ffd011751..c88bd5e21 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -868,16 +868,11 @@ void Grammar::init_indices() { traverse(cl); for (size_t track = 0; track < axiom->tracks(); ++track) { // variable access for all possible boundaries - std::ostringstream i, j, lm, rm, outside_l_dummy, outside_r_dummy; + std::ostringstream i, j, lm, rm; i << "t_" << track << "_i"; j << "t_" << track << "_j"; lm << "t_" << track << "_left_most"; rm << "t_" << track << "_right_most"; - /* helper variables for outside/inside transition - * which must be parameterized, but table dimension might be - * reduced to const or linear */ - outside_l_dummy << "t_" << track << "_outsidedummy_left"; - outside_r_dummy << "t_" << track << "_outsidedummy_right"; Expr::Vacc *left = new Expr::Vacc(new std::string(i.str())); Expr::Vacc *right = new Expr::Vacc(new std::string(j.str())); Expr::Vacc *left_most = new Expr::Vacc(new std::string(lm.str())); @@ -891,17 +886,6 @@ void Grammar::init_indices() { const Table &table = (*i)->tables()[idx]; Expr::Vacc *l = table.delete_left_index() ? left_most : left; Expr::Vacc *r = table.delete_right_index() ? right_most : right; - if ((*i)->is_inside_axiom) { - // always false for pure inside grammars - if (table.delete_left_index()) { - (*i)->left_indices_outsidedummy.push_back( - new Expr::Vacc(new std::string(outside_l_dummy.str()))); - } - if (table.delete_right_index()) { - (*i)->right_indices_outsidedummy.push_back( - new Expr::Vacc(new std::string(outside_r_dummy.str()))); - } - } if ((*i)->tracks() == 1 && (*i)->track_pos() != track) { continue; } From 2e5578f24bb3ff5f9c17bf1e45e8d0a60bcce942 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 13 Feb 2023 17:24:28 +0100 Subject: [PATCH 171/209] pass _x_most when table dim is constant if outside grammar generation was requested --- src/fn_def.cc | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/fn_def.cc b/src/fn_def.cc index efd005840..51896ac74 100644 --- a/src/fn_def.cc +++ b/src/fn_def.cc @@ -308,7 +308,7 @@ void Fn_Def::add_paras(const std::list &l) { #include "var_acc.hh" // add a nonterminal -void Fn_Def::add_para(Symbol::NT &nt) { +void Fn_Def::add_para(Symbol::NT &nt, bool for_outside_grammar) { Type::Base *t = new Type::Size(); const std::vector
&tables = nt.tables(); @@ -317,29 +317,27 @@ void Fn_Def::add_para(Symbol::NT &nt) { std::vector::iterator j = left.begin(); std::vector::iterator k = right.begin(); - std::vector::iterator odl = nt.left_indices_outsidedummy.begin(); - std::vector::iterator odr = - nt.right_indices_outsidedummy.begin(); for (std::vector
::const_iterator i = tables.begin(); - i != tables.end(); ++i, ++j, ++k, ++odl, ++odr) { + i != tables.end(); ++i, ++j, ++k) { // only add the - /* do NOT reduce indices, if this NT was the inside axiom and is - * used in an outside context. We need them to ensure that - * complete input sequence is parsed for outside and empty - * word for the inside part of the grammar, as this NT marks - * the transition between both grammar parts */ if (!(*i).delete_left_index()) { add_para(t, (*j)->vacc()->name()); - } else { - if (nt.is_inside_axiom) { - add_para(t, (*odl)->vacc()->name()); - } } if (!(*i).delete_right_index()) { add_para(t, (*k)->vacc()->name()); - } else { - if (nt.is_inside_axiom) { - add_para(t, (*odr)->vacc()->name()); + } + if (for_outside_grammar) { + /* A non-terminal table dimension might have been optimized to be + * constant. Thus, the NT can only be called implicitly with the + * full input sequence. That is fine for pure inside grammars. However, + * in an outside context we at least have to be able to call ALL NTs + * either with full input sequence (as before) OR with the empty input. + * We therefore call the NT in those cases with global left_most / + * right_most values, which will be shadowed by 0, 0 in their call + */ + if ((*i).is_const_table()) { + add_para(t, (*j)->vacc()->name()); + add_para(t, (*k)->vacc()->name()); } } } From 3815b133e266c2ce733ac513d8e71d546528fee1 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 13 Feb 2023 17:25:06 +0100 Subject: [PATCH 172/209] enable to pass outside grammar generation request --- src/fn_def.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fn_def.hh b/src/fn_def.hh index 3f2f37ae6..691d6531b 100644 --- a/src/fn_def.hh +++ b/src/fn_def.hh @@ -135,7 +135,7 @@ class Fn_Def : public Fn_Decl { void add_para(Type::Base *type, std::string *n); void add_paras(const std::list &l); - void add_para(Symbol::NT &nt); + void add_para(Symbol::NT &nt, bool for_outside_grammar = false); void annotate_terminal_arguments(Fn_Decl &d); From c21f08d93d582ceb26eaa068deca82a417b281cd Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 13 Feb 2023 17:27:34 +0100 Subject: [PATCH 173/209] revert --- src/cpp.hh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cpp.hh b/src/cpp.hh index 56e0c997a..824bc0b2b 100644 --- a/src/cpp.hh +++ b/src/cpp.hh @@ -89,8 +89,7 @@ class Cpp : public Base { void print_openmp_cyk(const AST &ast); std::string multi_index_str(const std::vector
&tables, - const Yield::Multi &mys, bool is_outside = false, - bool is_inside_axiom = false); + const Yield::Multi &mys, bool is_outside = false); void multi_print_inner_cyk(const std::list &l, const std::list &tord, size_t track, size_t tracks, size_t track_pos, From 4b3f5c83a0939268526a77d2bc0d98c7840168a6 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 13 Feb 2023 17:29:02 +0100 Subject: [PATCH 174/209] a simpler way to pass args when calling nt for reporting --- src/cpp.cc | 84 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 5e6d28b04..e0137071e 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -1743,8 +1743,7 @@ void Printer::Cpp::print_openmp_cyk_nt_calls(const AST &ast, if ((*i)->is_tabulated() && !(*i)->tables().front().is_cyk_right()) { o1 << indent() << "nt_tabulate_" << *(*i)->name << "(" << multi_index_str((*i)->tables(), (*i)->multi_ys(), - (*i)->is_partof_outside, - (*i)->is_inside_axiom) + (*i)->is_partof_outside) << ");\n"; } } @@ -1763,8 +1762,7 @@ void Printer::Cpp::print_openmp_cyk_all_nt_calls(const AST &ast, if ((*i)->is_tabulated()) { o2 << indent() << "nt_tabulate_" << *(*i)->name << "(" << multi_index_str((*i)->tables(), (*i)->multi_ys(), - (*i)->is_partof_outside, - (*i)->is_inside_axiom) + (*i)->is_partof_outside) << ");\n"; } } @@ -1919,7 +1917,7 @@ void Printer::Cpp::print_openmp_cyk(const AST &ast) { std::string Printer::Cpp::multi_index_str( const std::vector
&tables, const Yield::Multi &mys, - bool is_outside, bool is_inside_axiom) { + bool is_outside) { assert(tables.size() == mys.tracks()); std::ostringstream o; size_t track = 0; @@ -1932,10 +1930,6 @@ std::string Printer::Cpp::multi_index_str( } else { o << ", (t_" << track << "_j - t_" << track << "_i + 1)"; } - } else { - if (is_inside_axiom) { - o << ", t_" << track << "_left_most"; - } } if (!(*i).delete_right_index()) { if (is_outside == false) { @@ -1943,10 +1937,6 @@ std::string Printer::Cpp::multi_index_str( } else { o << ", (t_" << track << "_n - t_" << track << "_i + 1)"; } - } else { - if (is_inside_axiom) { - o << ", t_" << track << "_right_most"; - } } } std::string r(o.str()); @@ -1972,8 +1962,7 @@ void Printer::Cpp::multi_print_inner_cyk( i != l.end(); ++i) { if ((*i)->is_partof_outside == for_outsideNTs) { std::string index_str = multi_index_str((*i)->tables(), (*i)->multi_ys(), - (*i)->is_partof_outside, - (*i)->is_inside_axiom); + (*i)->is_partof_outside); stream << indent() << "nt_tabulate_" << *(*i)->name << '(' << index_str << ");" << endl; } @@ -2210,16 +2199,19 @@ void Printer::Cpp::print_cyk_fn(const AST &ast) { } void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { - std::stringstream list_args_print; std::stringstream list_args; + std::stringstream list_args_print; std::list &l = nt->code_list(); // aggregated level (=dim + tracks) of nested loops unsigned int nesting = 0; std::vector *args = new std::vector(); - std::vector *args_call = new std::vector(); + std::vector *args_ntcall = new std::vector(); + std::vector::iterator lidx = nt->left_indices.begin(); + std::vector::iterator ridx = nt->right_indices.begin(); + // opening for loops - for (size_t track = 0; track < nt->tracks(); ++track) { + for (size_t track = 0; track < nt->tracks(); ++track, ++lidx, ++ridx) { unsigned int dim = 2; if (nt->tables()[track].delete_left_index()) { dim--; @@ -2233,7 +2225,7 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { << "_right_most; ++t_" << track << "_i) {" << endl; inc_indent(); args->push_back("t_" + std::to_string(track) + "_i"); - args_call->push_back("t_" + std::to_string(track) + "_i"); + args_ntcall->push_back("t_" + std::to_string(track) + "_i"); } if (dim >= 2) { stream << indent() << "for (unsigned int t_" << track << "_j = t_" @@ -2241,14 +2233,27 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { << "_right_most; ++t_" << track << "_j) {" << endl; inc_indent(); args->push_back("t_" + std::to_string(track) + "_j"); - args_call->push_back("t_" + std::to_string(track) + "_j"); - } - if (nt->is_inside_axiom) { - if (dim <= 1) { - args_call->push_back("t_" + std::to_string(track) + "_left_most"); - } - if (dim < 2) { - args_call->push_back("t_" + std::to_string(track) + "_right_most"); + args_ntcall->push_back("t_" + std::to_string(track) + "_j"); + } + if (this->ast->grammar()->is_outside()) { + if (nt->tables()[track].is_const_table()) { + if (!nt->is_partof_outside) { + // TODO(sjanssen): can't we put Expr::Base args to a list + // and print this list later on? + // call const inside NT with complete input sequence + std::stringstream hl; + (*lidx)->put(hl); + args_ntcall->push_back(hl.str()); + std::stringstream hr; + (*ridx)->put(hr); + args_ntcall->push_back(hr.str()); + } else { + // call const outside NT with empty input sequence + std::stringstream hl; + (*ridx)->minus(*lidx)->put(hl); + args_ntcall->push_back(hl.str()); + args_ntcall->push_back(hl.str()); + } } } nesting += dim; @@ -2265,8 +2270,8 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { list_args_print << " << " << *a; } first = true; - for (std::vector::const_iterator a = args_call->begin(); - a != args_call->end(); ++a) { + for (std::vector::const_iterator a = args_ntcall->begin(); + a != args_ntcall->end(); ++a) { if (!first) { list_args << ", "; } @@ -2342,6 +2347,8 @@ void Printer::Cpp::print_insideoutside_report_fn( } void Printer::Cpp::print_run_fn(const AST &ast) { + // TODO(sjanssen) can we redesign this function and use Expr::Base + // fn_arg functions?! stream << indent() << *ast.grammar()->axiom->code()->return_type; stream << " run() {" << endl; inc_indent(); @@ -2350,8 +2357,12 @@ void Printer::Cpp::print_run_fn(const AST &ast) { bool first = true; size_t track = 0; const std::vector
&tables = ast.grammar()->axiom->tables(); + std::vector::iterator lidx = + ast.grammar()->axiom->left_indices.begin(); + std::vector::iterator ridx = + ast.grammar()->axiom->right_indices.begin(); for (std::vector
::const_iterator i = tables.begin(); - i != tables.end(); ++i, ++track) { + i != tables.end(); ++i, ++track, ++lidx, ++ridx) { Table t = *i; if (!t.delete_left_index()) { if (!first) { @@ -2367,6 +2378,19 @@ void Printer::Cpp::print_run_fn(const AST &ast) { first = false; stream << "t_" << track << "_right_most"; } + if (ast.grammar()->is_outside()) { + if (t.is_const_table()) { + if (!first) { + stream << ", "; + } + std::stringstream hl; + (*lidx)->put(hl); + stream << hl.str() << ", "; + std::stringstream hr; + (*ridx)->put(hr); + stream << hr.str(); + } + } } stream << ");" << endl; From 9ee546dfcbf1b5927824efed57abc7cf320be408 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 13 Feb 2023 17:30:30 +0100 Subject: [PATCH 175/209] just pass _x_most to const NTs in outside generation --- src/alt.cc | 62 ++++++++++++++++++++---------------------------------- src/alt.hh | 2 +- 2 files changed, 24 insertions(+), 40 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 12989f8aa..e8bd5afad 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -2123,7 +2123,7 @@ void Alt::Simple::codegen(AST &ast) { } -void Alt::Link::add_args(Expr::Fn_Call *fn) { +void Alt::Link::add_args(Expr::Fn_Call *fn, bool for_outside_grammar) { if (is_explicit()) { fn->add(indices); fn->add(ntparas); @@ -2146,50 +2146,34 @@ void Alt::Link::add_args(Expr::Fn_Call *fn) { assert(left_indices.size() == tables.size()); std::vector::iterator k = right_indices.begin(); std::vector::iterator j = left_indices.begin(); - std::vector::iterator odl = x->left_indices_outsidedummy.begin(); - std::vector::iterator odr = - x->right_indices_outsidedummy.begin(); - std::vector::iterator ontl = x->left_indices.begin(); - std::vector::iterator ontr = x->right_indices.begin(); for (std::vector
::const_iterator i = tables.begin(); - i != tables.end(); ++i, ++j, ++k, ++odl, ++odr, ++ontl, ++ontr) { - if (this->is_outside_inside_transition) { - /* we encounter the one and only situation for outside grammars - * where we transit from outside (full input sequence) into inside - * and need to make sure we parse the empty word. - */ - if (((*i).delete_left_index() && (!(*i).delete_right_index()))) { - fn->add_arg(*ontl); - fn->add_arg(*ontl); - } else if ((!(*i).delete_left_index() && ((*i).delete_right_index()))) { - fn->add_arg(*ontr); - fn->add_arg(*ontr); - } else { - fn->add_arg(new Expr::Const(0)); - fn->add_arg(new Expr::Const(0)); - } - // ignore the "normal" logic below - continue; - } + i != tables.end(); ++i, ++j, ++k) { if (!(*i).delete_left_index()) { fn->add_arg(*j); - } else { - if (x->is_inside_axiom) { - if (this->is_partof_outside) { - fn->add_arg(*ontl); - } else { - fn->add_arg(*odl); - } - } } if (!(*i).delete_right_index()) { fn->add_arg(*k); - } else { - if (x->is_inside_axiom) { - if (this->is_partof_outside) { - fn->add_arg(*ontr); + } + if (for_outside_grammar) { + /* A non-terminal table dimension might have been optimized to be + * constant. Thus, the NT can only be called implicitly with the + * full input sequence. That is fine for pure inside grammars. However, + * in an outside context we at least have to be able to call ALL NTs + * either with full input sequence (as before) OR with the empty input. + * We therefore call the NT in those cases with global left_most / + * right_most values, which will be shadowed by 0, 0 in their call + */ + if ((*i).is_const_table()) { + if (this->is_outside_inside_transition) { + /* we encounter the one and only situation for outside grammars + * where we transit from outside (full input sequence) into inside + * and need to make sure we parse the empty word. + */ + fn->add_arg((*k)->minus(*j)); + fn->add_arg((*k)->minus(*j)); } else { - fn->add_arg(*odr); + fn->add_arg(*j); + fn->add_arg(*k); } } } @@ -2222,7 +2206,7 @@ void Alt::Link::codegen(AST &ast) { add_seqs(fn, ast); } - add_args(fn); + add_args(fn, ast.grammar()->is_outside()); if (nt->is(Symbol::NONTERMINAL) && ast.code_mode() == Code::Mode::SUBOPT) { fn->exprs.push_back(new Expr::Vacc(new std::string("global_score"))); diff --git a/src/alt.hh b/src/alt.hh index 4fb2b1944..3340b190a 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -698,7 +698,7 @@ class Link : public Base { void multi_init_calls(const Runtime::Poly &p, size_t base_tracks); private: - void add_args(Expr::Fn_Call *fn); + void add_args(Expr::Fn_Call *fn, bool for_outside_grammar = false); private: std::list indices; From af35e88c122d62091d727c571186b952ad928432 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 13 Feb 2023 17:30:56 +0100 Subject: [PATCH 176/209] revert --- rtlib/filter.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtlib/filter.hh b/rtlib/filter.hh index 21af2e1d5..30731a7c3 100644 --- a/rtlib/filter.hh +++ b/rtlib/filter.hh @@ -130,7 +130,7 @@ inline bool samesize(const Basic_Sequence &s1, template inline bool complete_track( const Basic_Sequence &seq, T i, T j) { - return ((i == 0) && (j == seq.n)); + return ((i == seq.n) && (j == seq.n)); } #endif // RTLIB_FILTER_HH_ From 7c84e6e1912950292c932cd0d6d6d3cd688b73d1 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Feb 2023 12:45:54 +0100 Subject: [PATCH 177/209] fix yield size computation for outside grammars --- src/alt.cc | 1 + src/grammar.cc | 26 +++++++++++++++++++++++--- src/symbol.hh | 7 +++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 2c80f4822..f86084495 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -3539,6 +3539,7 @@ bool Alt::Link::replace_nonterminal(Symbol::NT *find, Symbol::NT *replace, if (skip_occurences[*(find->name)] == 0) { // replace old NT (=find) with novel NT (=replace) this->nt = replace; + this->m_ys = replace->multi_ys(); this->name = replace->name; this->is_partof_outside = replace->is_partof_outside; return true; diff --git a/src/grammar.cc b/src/grammar.cc index 33df8d27a..023654e3a 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1325,9 +1325,29 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { // terminals and links to non-terminals, but explicitly do NOT // re-run yield size analysis since it does not respect outside // situations. - this->init_calls(); - this->init_in_out(); - this->init_table_dims(); + //~ this->init_calls(); + //~ this->init_in_out(); + //~ this->init_table_dims(); + /* NT-table dimension optimization (all start quadratic, but depending on + * yield size, some NT-tables can be reduced to linear or even constant + * "tables") must be re-considered, since original inside NTs will now + * be called from outside NTs and former constant tables (e.g. consuming + * the full input) might now be within a variable infix size and thus + * must become quadratic again. + * We therefore here reset the flag of all NTs to enable re-computation + * of optimal table dimensions ... within the outside context. + */ + for (hashtable::iterator i = NTs.begin(); + i != NTs.end(); ++i) { + if ((*i).second->is(Symbol::NONTERMINAL)) { + dynamic_cast((*i).second)->reset_table_dim(); + } + } + /* re-run "check_semantics" to properly initialize novel non- + * terminals, links to non-terminals, update yield size analysis and + * update table dimensions for NTs + */ + this->check_semantic(); // mark the modified grammar as an outside one this->outside = true; diff --git a/src/symbol.hh b/src/symbol.hh index d0d5617a2..e0d6c8dcf 100644 --- a/src/symbol.hh +++ b/src/symbol.hh @@ -348,6 +348,13 @@ class NT : public Base { void set_alts(const std::list &a); bool inside_end = false; + // This flags the table dimension computation as invalid. + // Used for outside grammar generation, as table dimensions + // have to be re-computed after outside NTs were injected + void reset_table_dim() { + this->tab_dim_ready = false; + } + public: std::string *eval_fn; Fn_Decl *eval_decl; From eee7010298df1f737f54ad6769a07523a1213626 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Feb 2023 22:48:54 +0100 Subject: [PATCH 178/209] use empty word when transitioning from outside to inside rhs outside NTs always have yield size (0,n) --- src/alt.cc | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 937389b32..e7d290c19 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -2149,34 +2149,18 @@ void Alt::Link::add_args(Expr::Fn_Call *fn, bool for_outside_grammar) { for (std::vector
::const_iterator i = tables.begin(); i != tables.end(); ++i, ++j, ++k) { if (!(*i).delete_left_index()) { - fn->add_arg(*j); + if (for_outside_grammar && this->is_outside_inside_transition) { + /* if this is the transition from outside into inside grammar + * we have to call inside with empty input sequence, i.e. + * (right, right) index. */ + fn->add_arg(*k); + } else { + fn->add_arg(*j); + } } if (!(*i).delete_right_index()) { fn->add_arg(*k); } - if (for_outside_grammar) { - /* A non-terminal table dimension might have been optimized to be - * constant. Thus, the NT can only be called implicitly with the - * full input sequence. That is fine for pure inside grammars. However, - * in an outside context we at least have to be able to call ALL NTs - * either with full input sequence (as before) OR with the empty input. - * We therefore call the NT in those cases with global left_most / - * right_most values, which will be shadowed by 0, 0 in their call - */ - if ((*i).is_const_table()) { - if (this->is_outside_inside_transition) { - /* we encounter the one and only situation for outside grammars - * where we transit from outside (full input sequence) into inside - * and need to make sure we parse the empty word. - */ - fn->add_arg((*k)->minus(*j)); - fn->add_arg((*k)->minus(*j)); - } else { - fn->add_arg(*j); - fn->add_arg(*k); - } - } - } } fn->add(ntparas); @@ -2679,7 +2663,16 @@ void Alt::Simple::init_multi_ys() { void Alt::Link::init_multi_ys() { - m_ys = nt->multi_ys(); + // TODO(sjanssen): would it make sense to run an outside specific yield + // size analysis for more thight guards?? + if (nt->is_partof_outside) { + m_ys.set_tracks(tracks_); + for (Yield::Multi::iterator i = m_ys.begin(); i != m_ys.end(); ++i) { + *i = Yield::Size(0, Yield::UP); + } + } else { + m_ys = nt->multi_ys(); + } Base::init_multi_ys(); } From ab32fdccb616c186a3f2cfb95df0decf29063583 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Feb 2023 22:51:28 +0100 Subject: [PATCH 179/209] make a copy of original inside table dimensions --- src/grammar.cc | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/grammar.cc b/src/grammar.cc index 030d32275..eb94636db 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -1160,6 +1160,14 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { // and clone for outside version Symbol::NT *outside_nt = inside_nt->clone(inside_nt->track_pos()); + + /* create copies of the current inside table dimensions of the NT + * when reporting inside/outside results, we can thus skip a few + * cells of NTs that are now linear or const but will become + * quadratic due to outside context, e.g. struct of nodangle */ + outside_nt->table_dims_inside = inside_nt->tables(); + inside_nt->table_dims_inside = inside_nt->tables(); + // remove all existing alternative production rules outside_nt->alts.clear(); // and correct NT name, to now point to outside version @@ -1324,13 +1332,6 @@ void Grammar::inject_outside_nts(std::vector outside_nt_list) { this->axiom_name = nt_axiom_name; this->init_axiom(); } - // re-run parts of "check_semantics" to properly initialize novel non- - // terminals and links to non-terminals, but explicitly do NOT - // re-run yield size analysis since it does not respect outside - // situations. - //~ this->init_calls(); - //~ this->init_in_out(); - //~ this->init_table_dims(); /* NT-table dimension optimization (all start quadratic, but depending on * yield size, some NT-tables can be reduced to linear or even constant * "tables") must be re-considered, since original inside NTs will now From 253f17ce4143f91c00ceb158947f307a546579aa Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Feb 2023 22:51:54 +0100 Subject: [PATCH 180/209] add a field as a copy of inside table dimensions --- src/symbol.hh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/symbol.hh b/src/symbol.hh index 54a5180d1..717648e5b 100644 --- a/src/symbol.hh +++ b/src/symbol.hh @@ -397,6 +397,9 @@ class NT : public Base { private: std::vector
table_dims; + public: + std::vector
table_dims_inside; + public: const std::vector
&tables() const { return table_dims; From 6ba91dc62c2642648e57d76d925533d1e6e7bcb5 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Feb 2023 22:52:23 +0100 Subject: [PATCH 181/209] hack reporting such that only cells of original inside tables get reported, although more could be computed --- src/cpp.cc | 55 ++++++++++++++++++++++-------------------------------- 1 file changed, 22 insertions(+), 33 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index e0137071e..df6641a17 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2201,22 +2201,19 @@ void Printer::Cpp::print_cyk_fn(const AST &ast) { void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { std::stringstream list_args; std::stringstream list_args_print; - std::list &l = nt->code_list(); // aggregated level (=dim + tracks) of nested loops unsigned int nesting = 0; std::vector *args = new std::vector(); - std::vector *args_ntcall = new std::vector(); - std::vector::iterator lidx = nt->left_indices.begin(); - std::vector::iterator ridx = nt->right_indices.begin(); - // opening for loops - for (size_t track = 0; track < nt->tracks(); ++track, ++lidx, ++ridx) { + for (size_t track = 0; track < nt->tracks(); ++track) { unsigned int dim = 2; - if (nt->tables()[track].delete_left_index()) { + //if (nt->tables()[track].delete_left_index()) { + if (nt->table_dims_inside[track].delete_left_index()) { dim--; } - if (nt->tables()[track].delete_right_index()) { + //if (nt->tables()[track].delete_right_index()) { + if (nt->table_dims_inside[track].delete_right_index()) { dim--; } if (dim >= 1) { @@ -2225,7 +2222,6 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { << "_right_most; ++t_" << track << "_i) {" << endl; inc_indent(); args->push_back("t_" + std::to_string(track) + "_i"); - args_ntcall->push_back("t_" + std::to_string(track) + "_i"); } if (dim >= 2) { stream << indent() << "for (unsigned int t_" << track << "_j = t_" @@ -2233,33 +2229,26 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { << "_right_most; ++t_" << track << "_j) {" << endl; inc_indent(); args->push_back("t_" + std::to_string(track) + "_j"); - args_ntcall->push_back("t_" + std::to_string(track) + "_j"); - } - if (this->ast->grammar()->is_outside()) { - if (nt->tables()[track].is_const_table()) { - if (!nt->is_partof_outside) { - // TODO(sjanssen): can't we put Expr::Base args to a list - // and print this list later on? - // call const inside NT with complete input sequence - std::stringstream hl; - (*lidx)->put(hl); - args_ntcall->push_back(hl.str()); - std::stringstream hr; - (*ridx)->put(hr); - args_ntcall->push_back(hr.str()); - } else { - // call const outside NT with empty input sequence - std::stringstream hl; - (*ridx)->minus(*lidx)->put(hl); - args_ntcall->push_back(hl.str()); - args_ntcall->push_back(hl.str()); - } - } } nesting += dim; } + std::vector *args_call = new std::vector(); + for (size_t track = 0; track < nt->tracks(); ++track) { + if (nt->table_dims_inside[track].delete_left_index()) { + args_call->push_back("t_" + std::to_string(track) + "_left_most"); + } else { + args_call->push_back("t_" + std::to_string(track) + "_i"); + } + if (nt->table_dims_inside[track].delete_right_index()) { + args_call->push_back("t_" + std::to_string(track) + "_right_most"); + } else { + args_call->push_back("t_" + std::to_string(track) + "_j"); + } + } + // loop body + std::list &l = nt->code_list(); bool first = true; for (std::vector::const_iterator a = args->begin(); a != args->end(); ++a) { @@ -2270,8 +2259,8 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { list_args_print << " << " << *a; } first = true; - for (std::vector::const_iterator a = args_ntcall->begin(); - a != args_ntcall->end(); ++a) { + for (std::vector::const_iterator a = args_call->begin(); + a != args_call->end(); ++a) { if (!first) { list_args << ", "; } From 006b6633f5112fa5b40a2896dc4edcf78a3be86f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 16 Feb 2023 22:54:32 +0100 Subject: [PATCH 182/209] provide right arg twice, if in transit outside -> inside and table dimension is CONST --- src/alt.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index e7d290c19..7e980ba42 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -2148,16 +2148,16 @@ void Alt::Link::add_args(Expr::Fn_Call *fn, bool for_outside_grammar) { std::vector::iterator j = left_indices.begin(); for (std::vector
::const_iterator i = tables.begin(); i != tables.end(); ++i, ++j, ++k) { - if (!(*i).delete_left_index()) { - if (for_outside_grammar && this->is_outside_inside_transition) { - /* if this is the transition from outside into inside grammar - * we have to call inside with empty input sequence, i.e. - * (right, right) index. */ - fn->add_arg(*k); - } else { + if (for_outside_grammar && this->is_outside_inside_transition) { + if ((*i).is_const_table()) { fn->add_arg(*j); + fn->add_arg(*j); + continue; } } + if (!(*i).delete_left_index()) { + fn->add_arg(*j); + } if (!(*i).delete_right_index()) { fn->add_arg(*k); } From 974e192093c13276e3025f51147976cb62d000e8 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 16 Feb 2023 22:57:10 +0100 Subject: [PATCH 183/209] fix report indices and for loops --- src/cpp.cc | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index df6641a17..2cfdaa563 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2201,18 +2201,19 @@ void Printer::Cpp::print_cyk_fn(const AST &ast) { void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { std::stringstream list_args; std::stringstream list_args_print; + std::list &l = nt->code_list(); + std::vector *args = new std::vector(); + std::vector *args_call = new std::vector(); // aggregated level (=dim + tracks) of nested loops unsigned int nesting = 0; - std::vector *args = new std::vector(); + // opening for loops for (size_t track = 0; track < nt->tracks(); ++track) { unsigned int dim = 2; - //if (nt->tables()[track].delete_left_index()) { if (nt->table_dims_inside[track].delete_left_index()) { dim--; } - //if (nt->tables()[track].delete_right_index()) { if (nt->table_dims_inside[track].delete_right_index()) { dim--; } @@ -2233,22 +2234,26 @@ void Printer::Cpp::print_insideoutside(Symbol::NT *nt) { nesting += dim; } - std::vector *args_call = new std::vector(); for (size_t track = 0; track < nt->tracks(); ++track) { - if (nt->table_dims_inside[track].delete_left_index()) { - args_call->push_back("t_" + std::to_string(track) + "_left_most"); - } else { - args_call->push_back("t_" + std::to_string(track) + "_i"); - } - if (nt->table_dims_inside[track].delete_right_index()) { - args_call->push_back("t_" + std::to_string(track) + "_right_most"); - } else { - args_call->push_back("t_" + std::to_string(track) + "_j"); - } + std::string nextLoopVar = "i"; + if (nt->table_dims_inside[track].delete_left_index()) { + if (!nt->tables()[track].delete_left_index()) { + args_call->push_back("t_" + std::to_string(track) + "_left_most"); + } + } else { + args_call->push_back("t_" + std::to_string(track) + "_" + nextLoopVar); + nextLoopVar = "j"; + } + if (nt->table_dims_inside[track].delete_right_index()) { + if (!nt->tables()[track].delete_right_index()) { + args_call->push_back("t_" + std::to_string(track) + "_right_most"); + } + } else { + args_call->push_back("t_" + std::to_string(track) + "_" + nextLoopVar); + } } // loop body - std::list &l = nt->code_list(); bool first = true; for (std::vector::const_iterator a = args->begin(); a != args->end(); ++a) { From 4a07768aa19c882481e71cf1ef0ffab3ce4dba3c Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 16 Feb 2023 23:25:39 +0100 Subject: [PATCH 184/209] adding enum instance --- testdata/grammar_outside/nodangle.gap | 1 + 1 file changed, 1 insertion(+) diff --git a/testdata/grammar_outside/nodangle.gap b/testdata/grammar_outside/nodangle.gap index f83293d10..c1fa7c6a0 100644 --- a/testdata/grammar_outside/nodangle.gap +++ b/testdata/grammar_outside/nodangle.gap @@ -233,3 +233,4 @@ grammar gra_nodangle uses sig_foldrna(axiom = struct) { instance mfe = gra_nodangle(alg_mfe); instance pfunc = gra_nodangle(alg_pfunc); instance count = gra_nodangle(alg_count); +instance enum = gra_nodangle(alg_enum); From 64f72aaf48536a3c41ea05e91d483c17012edf64 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 16 Feb 2023 23:25:53 +0100 Subject: [PATCH 185/209] adding enum instance --- testdata/grammar_outside/nonzero_constsize.gap | 1 + 1 file changed, 1 insertion(+) diff --git a/testdata/grammar_outside/nonzero_constsize.gap b/testdata/grammar_outside/nonzero_constsize.gap index 40724e4a3..e02eb62db 100644 --- a/testdata/grammar_outside/nonzero_constsize.gap +++ b/testdata/grammar_outside/nonzero_constsize.gap @@ -47,4 +47,5 @@ grammar gra_cm uses sig_cm(axiom = state_B_3) { } instance count = gra_cm(alg_count); +instance enum = gra_cm(alg_enum); \ No newline at end of file From eb09c6bc66276c9425c923f471c9c68efc65405f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 17 Feb 2023 10:08:42 +0100 Subject: [PATCH 186/209] fix and comment CYK loop order --- src/cpp.cc | 82 ++++++++++++++++++++++++++++++++++-------------------- src/cpp.hh | 9 ++++++ 2 files changed, 61 insertions(+), 30 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 2cfdaa563..756d03434 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -1924,14 +1924,14 @@ std::string Printer::Cpp::multi_index_str( Yield::Multi::const_iterator j = mys.begin(); for (std::vector
::const_iterator i = tables.begin(); i != tables.end(); ++i, ++j, ++track) { - if (!(*i).delete_left_index()) { + if (!(*i).delete_left_index() || (*i).is_const_table()) { if (is_outside == false) { o << ", t_" << track << "_i-1"; } else { o << ", (t_" << track << "_j - t_" << track << "_i + 1)"; } } - if (!(*i).delete_right_index()) { + if (!(*i).delete_right_index() || (*i).is_const_table()) { if (is_outside == false) { o << ", t_" << track << "_j"; } else { @@ -2015,9 +2015,13 @@ void Printer::Cpp::multi_print_cyk_loops_quadratic( std::string *js, std::string *ns, bool for_outsideNTs) { if (!inner->empty()) { - stream << indent() << "for (" << *t << " " << *js << " = 0; " << *js + stream << indent() << "for (" << *t << " " << *js << " = 0; " << *js << " < " << *ns << "; " << "++" << *js << ") {" << endl; inc_indent(); + if (for_outsideNTs) { + multi_print_cyk_loops_quadratic_inner(tord, track, tracks, track_pos, t, left, is, for_outsideNTs); + } + stream << indent() << "// A: quadratic loops" << endl; stream << indent() << "for (" << *t << " " << *is << " = " << *js << " + 1; " << *is << " > 1; " << *is << "--) {" << endl; inc_indent(); @@ -2027,19 +2031,34 @@ void Printer::Cpp::multi_print_cyk_loops_quadratic( dec_indent(); stream << indent() << "}" << endl << endl; - if (!left->empty()) { - stream << indent(); - if (!for_outsideNTs) { - stream << *t << " "; - } - stream << *is << " = 1;" << endl; - multi_print_inner_cyk(*left, tord, track, tracks, track_pos, t, - for_outsideNTs); + if (!for_outsideNTs) { + multi_print_cyk_loops_quadratic_inner(tord, track, tracks, track_pos, t, left, is, for_outsideNTs); } dec_indent(); stream << indent() << "}" << endl << endl; } } +void Printer::Cpp::multi_print_cyk_loops_quadratic_inner( + const std::list &tord, + size_t track, + size_t tracks, + size_t track_pos, + Type::Base *t, + std::list *left, + std::string *is, + bool for_outsideNTs + ) { + if (!left->empty()) { + stream << indent() << "// B: inner quadratic loops" << endl; + stream << indent(); + if (!for_outsideNTs) { + stream << *t << " "; + } + stream << *is << " = 1;" << endl; + multi_print_inner_cyk(*left, tord, track, tracks, track_pos, t, + for_outsideNTs); + } +} void Printer::Cpp::multi_print_cyk_loops_linear( const std::list &tord, size_t track, @@ -2053,6 +2072,8 @@ void Printer::Cpp::multi_print_cyk_loops_linear( std::string *js, std::string *ns, bool for_outsideNTs ) { + stream << indent() << "// C: linear loops" << endl; + if (inner->empty() && !left->empty()) { stream << indent() << "for (" << *t << " " << *js << " = 0; " << *js << " < " << *ns @@ -2092,6 +2113,7 @@ void Printer::Cpp::multi_print_cyk_loops_constant( std::list *all, std::string *is, bool for_outsideNTs ) { + stream << indent() << "// D: constant loops" << endl; if (!all->empty()) { stream << indent(); if (!for_outsideNTs) { @@ -2131,13 +2153,13 @@ void Printer::Cpp::multi_print_cyk( // outside NTs (which access inside NT values) if (this->ast->grammar()->is_outside()) { stream << endl << endl; + multi_print_cyk_loops_quadratic( + tord, track, tracks, track_pos, t, &inner, &left, &is, &js, &ns, true); multi_print_cyk_loops_linear( - tord, track, tracks, track_pos, t, &inner, &left, &right, &is, &js, &ns, - true); + tord, track, tracks, track_pos, t, &inner, &left, &right, &is, &js, &ns, + true); multi_print_cyk_loops_constant( tord, track, tracks, track_pos, t, &all, &is, true); - multi_print_cyk_loops_quadratic( - tord, track, tracks, track_pos, t, &inner, &left, &is, &js, &ns, true); } } @@ -2358,33 +2380,33 @@ void Printer::Cpp::print_run_fn(const AST &ast) { for (std::vector
::const_iterator i = tables.begin(); i != tables.end(); ++i, ++track, ++lidx, ++ridx) { Table t = *i; - if (!t.delete_left_index()) { + if (!t.delete_left_index() || t.is_const_table()) { if (!first) { stream << ", "; } first = false; stream << "t_" << track << "_left_most"; } - if (!t.delete_right_index()) { + if (!t.delete_right_index() || t.is_const_table()) { if (!first) { stream << ", "; } first = false; stream << "t_" << track << "_right_most"; } - if (ast.grammar()->is_outside()) { - if (t.is_const_table()) { - if (!first) { - stream << ", "; - } - std::stringstream hl; - (*lidx)->put(hl); - stream << hl.str() << ", "; - std::stringstream hr; - (*ridx)->put(hr); - stream << hr.str(); - } - } +// if (ast.grammar()->is_outside()) { +// if (t.is_const_table()) { +// if (!first) { +// stream << ", "; +// } +// std::stringstream hl; +// (*lidx)->put(hl); +// stream << hl.str() << ", "; +// std::stringstream hr; +// (*ridx)->put(hr); +// stream << hr.str(); +// } +// } } stream << ");" << endl; diff --git a/src/cpp.hh b/src/cpp.hh index 824bc0b2b..f24716fb2 100644 --- a/src/cpp.hh +++ b/src/cpp.hh @@ -109,6 +109,15 @@ class Cpp : public Base { std::string *is, std::string *js, std::string *ns, bool for_outsideNTs = false); + void multi_print_cyk_loops_quadratic_inner( + const std::list &tord, + size_t track, + size_t tracks, + size_t track_pos, + Type::Base *t, + std::list *left, + std::string *is, + bool for_outsideNTs = false); void multi_print_cyk_loops_linear( const std::list &tord, size_t track, From 9775e9b03c022464f99897cb7aaacbbaecdae979 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 17 Feb 2023 13:01:06 +0100 Subject: [PATCH 187/209] edge case no longer exists --- src/cpp.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 756d03434..342f98fc6 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2380,14 +2380,14 @@ void Printer::Cpp::print_run_fn(const AST &ast) { for (std::vector
::const_iterator i = tables.begin(); i != tables.end(); ++i, ++track, ++lidx, ++ridx) { Table t = *i; - if (!t.delete_left_index() || t.is_const_table()) { + if (!t.delete_left_index()) { if (!first) { stream << ", "; } first = false; stream << "t_" << track << "_left_most"; } - if (!t.delete_right_index() || t.is_const_table()) { + if (!t.delete_right_index()) { if (!first) { stream << ", "; } From 90eb9cb873dae02baccfba8ca19431156055a8c0 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 17 Feb 2023 13:02:02 +0100 Subject: [PATCH 188/209] revert to previous version --- src/fn_def.cc | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/src/fn_def.cc b/src/fn_def.cc index 51896ac74..5e7ce3f28 100644 --- a/src/fn_def.cc +++ b/src/fn_def.cc @@ -320,26 +320,8 @@ void Fn_Def::add_para(Symbol::NT &nt, bool for_outside_grammar) { for (std::vector
::const_iterator i = tables.begin(); i != tables.end(); ++i, ++j, ++k) { // only add the - if (!(*i).delete_left_index()) { - add_para(t, (*j)->vacc()->name()); - } - if (!(*i).delete_right_index()) { - add_para(t, (*k)->vacc()->name()); - } - if (for_outside_grammar) { - /* A non-terminal table dimension might have been optimized to be - * constant. Thus, the NT can only be called implicitly with the - * full input sequence. That is fine for pure inside grammars. However, - * in an outside context we at least have to be able to call ALL NTs - * either with full input sequence (as before) OR with the empty input. - * We therefore call the NT in those cases with global left_most / - * right_most values, which will be shadowed by 0, 0 in their call - */ - if ((*i).is_const_table()) { - add_para(t, (*j)->vacc()->name()); - add_para(t, (*k)->vacc()->name()); - } - } + if (!(*i).delete_left_index()) add_para(t, (*j)->vacc()->name()); + if (!(*i).delete_right_index()) add_para(t, (*k)->vacc()->name()); } set_paras(nt.ntargs()); From 7294f150037651afc6f474bceaf54f0e685a805c Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 17 Feb 2023 13:08:37 +0100 Subject: [PATCH 189/209] remove leftovers --- src/cpp.cc | 19 +------------------ src/fn_def.cc | 2 +- src/fn_def.hh | 2 +- src/grammar.cc | 1 + src/symbol.cc | 2 +- 5 files changed, 5 insertions(+), 21 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 342f98fc6..daf3fefc7 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2373,12 +2373,8 @@ void Printer::Cpp::print_run_fn(const AST &ast) { bool first = true; size_t track = 0; const std::vector
&tables = ast.grammar()->axiom->tables(); - std::vector::iterator lidx = - ast.grammar()->axiom->left_indices.begin(); - std::vector::iterator ridx = - ast.grammar()->axiom->right_indices.begin(); for (std::vector
::const_iterator i = tables.begin(); - i != tables.end(); ++i, ++track, ++lidx, ++ridx) { + i != tables.end(); ++i, ++track) { Table t = *i; if (!t.delete_left_index()) { if (!first) { @@ -2394,19 +2390,6 @@ void Printer::Cpp::print_run_fn(const AST &ast) { first = false; stream << "t_" << track << "_right_most"; } -// if (ast.grammar()->is_outside()) { -// if (t.is_const_table()) { -// if (!first) { -// stream << ", "; -// } -// std::stringstream hl; -// (*lidx)->put(hl); -// stream << hl.str() << ", "; -// std::stringstream hr; -// (*ridx)->put(hr); -// stream << hr.str(); -// } -// } } stream << ");" << endl; diff --git a/src/fn_def.cc b/src/fn_def.cc index 5e7ce3f28..5ef70e5d8 100644 --- a/src/fn_def.cc +++ b/src/fn_def.cc @@ -308,7 +308,7 @@ void Fn_Def::add_paras(const std::list &l) { #include "var_acc.hh" // add a nonterminal -void Fn_Def::add_para(Symbol::NT &nt, bool for_outside_grammar) { +void Fn_Def::add_para(Symbol::NT &nt) { Type::Base *t = new Type::Size(); const std::vector
&tables = nt.tables(); diff --git a/src/fn_def.hh b/src/fn_def.hh index 691d6531b..3f2f37ae6 100644 --- a/src/fn_def.hh +++ b/src/fn_def.hh @@ -135,7 +135,7 @@ class Fn_Def : public Fn_Decl { void add_para(Type::Base *type, std::string *n); void add_paras(const std::list &l); - void add_para(Symbol::NT &nt, bool for_outside_grammar = false); + void add_para(Symbol::NT &nt); void annotate_terminal_arguments(Fn_Decl &d); diff --git a/src/grammar.cc b/src/grammar.cc index eb94636db..2a17b8999 100644 --- a/src/grammar.cc +++ b/src/grammar.cc @@ -886,6 +886,7 @@ void Grammar::init_indices() { const Table &table = (*i)->tables()[idx]; Expr::Vacc *l = table.delete_left_index() ? left_most : left; Expr::Vacc *r = table.delete_right_index() ? right_most : right; + if ((*i)->tracks() == 1 && (*i)->track_pos() != track) { continue; } diff --git a/src/symbol.cc b/src/symbol.cc index 2dcef2e06..febe4e65f 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -1236,7 +1236,7 @@ void Symbol::NT::codegen(AST &ast) { } else { f = new Fn_Def(dt, new std::string("nt_" + *name)); } - f->add_para(*this, ast.grammar()->is_outside()); + f->add_para(*this); subopt_header(ast, score_code, f, stmts); From 12bca4026fe1280b0c580671e2c0b3577e106303 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 17 Feb 2023 13:12:23 +0100 Subject: [PATCH 190/209] linting --- src/alt.cc | 2 +- src/cpp.cc | 29 +++++++++++++++-------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 7e980ba42..e6cabf015 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -2666,7 +2666,7 @@ void Alt::Link::init_multi_ys() { // TODO(sjanssen): would it make sense to run an outside specific yield // size analysis for more thight guards?? if (nt->is_partof_outside) { - m_ys.set_tracks(tracks_); + m_ys.set_tracks(tracks_); for (Yield::Multi::iterator i = m_ys.begin(); i != m_ys.end(); ++i) { *i = Yield::Size(0, Yield::UP); } diff --git a/src/cpp.cc b/src/cpp.cc index daf3fefc7..244de4754 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2015,11 +2015,12 @@ void Printer::Cpp::multi_print_cyk_loops_quadratic( std::string *js, std::string *ns, bool for_outsideNTs) { if (!inner->empty()) { - stream << indent() << "for (" << *t << " " << *js << " = 0; " << *js + stream << indent() << "for (" << *t << " " << *js << " = 0; " << *js << " < " << *ns << "; " << "++" << *js << ") {" << endl; inc_indent(); if (for_outsideNTs) { - multi_print_cyk_loops_quadratic_inner(tord, track, tracks, track_pos, t, left, is, for_outsideNTs); + multi_print_cyk_loops_quadratic_inner( + tord, track, tracks, track_pos, t, left, is, for_outsideNTs); } stream << indent() << "// A: quadratic loops" << endl; stream << indent() << "for (" << *t << " " << *is << " = " << *js @@ -2032,22 +2033,22 @@ void Printer::Cpp::multi_print_cyk_loops_quadratic( stream << indent() << "}" << endl << endl; if (!for_outsideNTs) { - multi_print_cyk_loops_quadratic_inner(tord, track, tracks, track_pos, t, left, is, for_outsideNTs); + multi_print_cyk_loops_quadratic_inner( + tord, track, tracks, track_pos, t, left, is, for_outsideNTs); } dec_indent(); stream << indent() << "}" << endl << endl; } } void Printer::Cpp::multi_print_cyk_loops_quadratic_inner( - const std::list &tord, - size_t track, - size_t tracks, - size_t track_pos, - Type::Base *t, - std::list *left, - std::string *is, - bool for_outsideNTs - ) { + const std::list &tord, + size_t track, + size_t tracks, + size_t track_pos, + Type::Base *t, + std::list *left, + std::string *is, + bool for_outsideNTs) { if (!left->empty()) { stream << indent() << "// B: inner quadratic loops" << endl; stream << indent(); @@ -2156,8 +2157,8 @@ void Printer::Cpp::multi_print_cyk( multi_print_cyk_loops_quadratic( tord, track, tracks, track_pos, t, &inner, &left, &is, &js, &ns, true); multi_print_cyk_loops_linear( - tord, track, tracks, track_pos, t, &inner, &left, &right, &is, &js, &ns, - true); + tord, track, tracks, track_pos, t, &inner, &left, &right, &is, &js + &ns, true); multi_print_cyk_loops_constant( tord, track, tracks, track_pos, t, &all, &is, true); } From 6c7ff36c3786ab19722b2c6427bd9af03a86644a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 17 Feb 2023 13:16:24 +0100 Subject: [PATCH 191/209] single , was missing --- src/cpp.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp.cc b/src/cpp.cc index 244de4754..4c16d1017 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2157,7 +2157,7 @@ void Printer::Cpp::multi_print_cyk( multi_print_cyk_loops_quadratic( tord, track, tracks, track_pos, t, &inner, &left, &is, &js, &ns, true); multi_print_cyk_loops_linear( - tord, track, tracks, track_pos, t, &inner, &left, &right, &is, &js + tord, track, tracks, track_pos, t, &inner, &left, &right, &is, &js, &ns, true); multi_print_cyk_loops_constant( tord, track, tracks, track_pos, t, &all, &is, true); From 0843e266313f842c083b35f3fcac49c36fdf3fb1 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 17 Feb 2023 13:50:42 +0100 Subject: [PATCH 192/209] leave const tables untouched --- src/cpp.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 4c16d1017..2db4ebea0 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -1924,14 +1924,14 @@ std::string Printer::Cpp::multi_index_str( Yield::Multi::const_iterator j = mys.begin(); for (std::vector
::const_iterator i = tables.begin(); i != tables.end(); ++i, ++j, ++track) { - if (!(*i).delete_left_index() || (*i).is_const_table()) { + if (!(*i).delete_left_index()) { if (is_outside == false) { o << ", t_" << track << "_i-1"; } else { o << ", (t_" << track << "_j - t_" << track << "_i + 1)"; } } - if (!(*i).delete_right_index() || (*i).is_const_table()) { + if (!(*i).delete_right_index()) { if (is_outside == false) { o << ", t_" << track << "_j"; } else { From 809a7adb35ae49403623eb6d30dfe376a32aa654 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 17 Feb 2023 17:16:38 +0100 Subject: [PATCH 193/209] avoid name collision --- testdata/gapc_filter/RFmini.hh | 1 - testdata/gapc_filter/RFmini_data.hh | 83 +++++++++++++++++++++++++++++ testdata/grammar_outside/RFmini.gap | 4 +- 3 files changed, 85 insertions(+), 3 deletions(-) delete mode 120000 testdata/gapc_filter/RFmini.hh create mode 100644 testdata/gapc_filter/RFmini_data.hh diff --git a/testdata/gapc_filter/RFmini.hh b/testdata/gapc_filter/RFmini.hh deleted file mode 120000 index 349324b2b..000000000 --- a/testdata/gapc_filter/RFmini.hh +++ /dev/null @@ -1 +0,0 @@ -../ADP_collection/CM/RFmini.hh \ No newline at end of file diff --git a/testdata/gapc_filter/RFmini_data.hh b/testdata/gapc_filter/RFmini_data.hh new file mode 100644 index 000000000..750f6f362 --- /dev/null +++ b/testdata/gapc_filter/RFmini_data.hh @@ -0,0 +1,83 @@ + +#ifndef CMPROBS_HH +#define CMPROBS_HH + +inline int charToIndex(char a) { + if ((a == 'A') || (a == 'a')) return 0; + if ((a == 'C') || (a == 'c')) return 1; + if ((a == 'G') || (a == 'g')) return 2; + if ((a == 'U') || (a == 'u')) return 3; + return 99; +} +inline int charToIndex(char a, char b) { + if (((a == 'A') || (a == 'a')) && ((b == 'A') || (b == 'a'))) return 0; + if (((a == 'A') || (a == 'a')) && ((b == 'C') || (b == 'c'))) return 1; + if (((a == 'A') || (a == 'a')) && ((b == 'G') || (b == 'g'))) return 2; + if (((a == 'A') || (a == 'a')) && ((b == 'U') || (b == 'u'))) return 3; + if (((a == 'C') || (a == 'c')) && ((b == 'A') || (b == 'a'))) return 4; + if (((a == 'C') || (a == 'c')) && ((b == 'C') || (b == 'c'))) return 5; + if (((a == 'C') || (a == 'c')) && ((b == 'G') || (b == 'g'))) return 6; + if (((a == 'C') || (a == 'c')) && ((b == 'U') || (b == 'u'))) return 7; + if (((a == 'G') || (a == 'g')) && ((b == 'A') || (b == 'a'))) return 8; + if (((a == 'G') || (a == 'g')) && ((b == 'C') || (b == 'c'))) return 9; + if (((a == 'G') || (a == 'g')) && ((b == 'G') || (b == 'g'))) return 10; + if (((a == 'G') || (a == 'g')) && ((b == 'U') || (b == 'u'))) return 11; + if (((a == 'U') || (a == 'u')) && ((b == 'A') || (b == 'a'))) return 12; + if (((a == 'U') || (a == 'u')) && ((b == 'C') || (b == 'c'))) return 13; + if (((a == 'U') || (a == 'u')) && ((b == 'G') || (b == 'g'))) return 14; + if (((a == 'U') || (a == 'u')) && ((b == 'U') || (b == 'u'))) return 15; + return 99; +} +static const float emissions[21][4] = { + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, +}; +inline float getEmission(int pos, char a) { + return emissions[pos][charToIndex(a)]; +} +static const float pair_emissions[21][16] = { + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {1.981, -8.618, 2.903, -4.869, -6.316, -8.108, 2.094, -7.200, -7.986, -4.905, -8.133, -6.722, -3.042, -8.451, -4.018, -6.800}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {1.986, -6.270, -7.785, -2.585, -9.584, -10.804, -5.093, -7.806, 2.908, 2.062, -8.009, -3.561, -5.179, -9.057, -6.956, -6.389}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, +}; +inline float getPairEmission(int pos, char a, char b) { + return pair_emissions[pos][charToIndex(a, b)]; +} +#endif diff --git a/testdata/grammar_outside/RFmini.gap b/testdata/grammar_outside/RFmini.gap index 240064caa..5999c6b3f 100644 --- a/testdata/grammar_outside/RFmini.gap +++ b/testdata/grammar_outside/RFmini.gap @@ -1,5 +1,5 @@ -import "RFmini.hh" +import "RFmini_data.hh" signature sig_cm(alphabet, answer) { answer silent_transition(float, answer); @@ -152,4 +152,4 @@ instance count = gra_cm(alg_count); instance enum = gra_cm(alg_enum); instance enumcyk = gra_cm(alg_enum * alg_cyk); instance cykenum = gra_cm(alg_cyk * alg_enum); - \ No newline at end of file +instance inside = gra_cm(alg_inside); From 15b7c5709165249d540ec491499a5c6b187f64d3 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 17 Feb 2023 17:17:24 +0100 Subject: [PATCH 194/209] add first CM prob test, intensively validated against cmalign --- testdata/regresstest/config | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 62d3ac265..331f59ada 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -510,4 +510,11 @@ check_new_old_eq tmhmm.gap unused fwd_scaled "MENLNMDLLYMAAAVMMGLAAIGAAIGIGILGGK check_new_old_eq RFmini.gap unused enum "G" outside_cm check_new_old_eq RFmini.gap unused count "G" outside_cm + +# Results can be converted into probabilities via nt_inside(i,j) + nt_outside(i,j) - axiom(0,n) - axiom(0,0) +# where axiom(0,0) is the pfunc value of the empty word. This words because we are in log2-space. +# Converted to probabilities: 2^nt_inside(i,j) * 2^nt_outside(i,j) / 2^axiom(0,n) / 2^axiom(0,0) +# The important aspect is that we have to also normalize by axiom(0,0) since the inside result of +# the empty word "" != 0 <-- log2-space but will be integrated in each and every outside candidate +check_new_old_eq RFmini.gap unused inside "G" outside_cm CPPFLAGS_EXTRA="" From fe2aa9c8bf57189240e87ad8540e5214e85c8a53 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 17 Feb 2023 21:46:53 +0100 Subject: [PATCH 195/209] adding larger tRNA CM --- testdata/gapc_filter/RF00005_data.hh | 495 ++++++++ testdata/grammar_outside/RF00005.gap | 1550 ++++++++++++++++++++++++++ testdata/regresstest/config | 5 + 3 files changed, 2050 insertions(+) create mode 100644 testdata/gapc_filter/RF00005_data.hh create mode 100644 testdata/grammar_outside/RF00005.gap diff --git a/testdata/gapc_filter/RF00005_data.hh b/testdata/gapc_filter/RF00005_data.hh new file mode 100644 index 000000000..280bc6449 --- /dev/null +++ b/testdata/gapc_filter/RF00005_data.hh @@ -0,0 +1,495 @@ + +#ifndef CMPROBS_HH +#define CMPROBS_HH + +inline int charToIndex(char a) { + if ((a == 'A') || (a == 'a')) return 0; + if ((a == 'C') || (a == 'c')) return 1; + if ((a == 'G') || (a == 'g')) return 2; + if ((a == 'U') || (a == 'u')) return 3; + return 99; +} +inline int charToIndex(char a, char b) { + if (((a == 'A') || (a == 'a')) && ((b == 'A') || (b == 'a'))) return 0; + if (((a == 'A') || (a == 'a')) && ((b == 'C') || (b == 'c'))) return 1; + if (((a == 'A') || (a == 'a')) && ((b == 'G') || (b == 'g'))) return 2; + if (((a == 'A') || (a == 'a')) && ((b == 'U') || (b == 'u'))) return 3; + if (((a == 'C') || (a == 'c')) && ((b == 'A') || (b == 'a'))) return 4; + if (((a == 'C') || (a == 'c')) && ((b == 'C') || (b == 'c'))) return 5; + if (((a == 'C') || (a == 'c')) && ((b == 'G') || (b == 'g'))) return 6; + if (((a == 'C') || (a == 'c')) && ((b == 'U') || (b == 'u'))) return 7; + if (((a == 'G') || (a == 'g')) && ((b == 'A') || (b == 'a'))) return 8; + if (((a == 'G') || (a == 'g')) && ((b == 'C') || (b == 'c'))) return 9; + if (((a == 'G') || (a == 'g')) && ((b == 'G') || (b == 'g'))) return 10; + if (((a == 'G') || (a == 'g')) && ((b == 'U') || (b == 'u'))) return 11; + if (((a == 'U') || (a == 'u')) && ((b == 'A') || (b == 'a'))) return 12; + if (((a == 'U') || (a == 'u')) && ((b == 'C') || (b == 'c'))) return 13; + if (((a == 'U') || (a == 'u')) && ((b == 'G') || (b == 'g'))) return 14; + if (((a == 'U') || (a == 'u')) && ((b == 'U') || (b == 'u'))) return 15; + return 99; +} +static const float emissions[227][4] = { + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.967, -1.762, 0.127, -0.603}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {-0.245, -0.619, -0.992, 1.001}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.232, -0.364, -0.344, 0.334}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {-1.820, -4.082, -2.351, 1.791}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {1.129, -2.503, 0.522, -2.312}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.797, -0.772, -1.416, 0.380}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.271, -2.060, 0.792, -0.282}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.603, -2.320, 0.645, -0.480}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {-1.828, 0.976, -2.145, 0.609}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {-0.043, 0.441, -0.742, 0.103}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {-1.717, -2.581, -3.687, 1.787}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {-1.901, -2.867, -2.450, 1.771}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {-0.776, 1.547, -3.413, -1.324}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.711, -2.398, 0.934, -1.928}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {1.824, -2.680, -3.033, -2.468}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.973, -1.119, -0.690, -0.063}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {-2.268, -1.024, -4.138, 1.698}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.232, -0.043, -0.423, 0.149}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {1.792, -4.422, -2.541, -1.645}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.666, -2.273, 0.835, -1.244}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {-0.563, -0.298, -2.598, 1.229}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {-1.088, -2.493, 1.635, -2.025}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {-1.735, -2.296, 1.633, -1.345}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {-1.587, -0.860, -2.982, 1.580}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {1.688, -3.294, -1.536, -1.599}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.683, -2.722, 0.793, -0.970}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {-0.163, 0.616, -0.875, 0.042}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.368, -0.385, -0.191, 0.094}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.225, -0.363, -0.352, 0.346}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {0.0, 0.0, 0.0, 0.0}, + {0.368, -0.385, -0.191, 0.094}, + {0.251, -0.082, -0.397, 0.145}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, + {-2.829, 1.248, -4.245, 0.518}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {-6.408, -3.367, -7.171, 1.958}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {-1.730, -0.660, 0.415, 0.793}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.264, -0.063, -0.290, 0.034}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.507, -0.440, -0.526, 0.199}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {1.656, -5.193, -0.360, -4.593}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {1.517, -0.862, -2.536, -1.270}, + {0.0, 0.0, 0.0, 0.0}, + {0.000, 0.000, 0.000, 0.000}, + {0.000, 0.000, 0.000, 0.000}, +}; +inline float getEmission(int pos, char a) { + return emissions[pos][charToIndex(a)]; +} +static const float pair_emissions[227][16] = { + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-5.364, -3.089, -4.327, 1.380, -3.808, -4.601, 0.302, -4.695, -6.319, 3.165, -5.456, 0.495, 0.320, -5.774, -4.107, -3.653}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-6.007, -4.160, -5.951, 1.497, -4.299, -5.407, 2.066, -5.173, -6.133, 2.581, -6.530, -0.771, 0.996, -5.787, -2.677, -4.763}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-3.248, -3.450, -3.395, 2.144, -3.133, -4.398, 1.760, -3.669, -3.855, 1.775, -3.934, -0.650, 1.490, -3.635, -1.280, -2.611}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-2.221, -3.413, -3.535, 1.722, -2.977, -3.650, 1.427, -3.771, -3.398, 2.245, -4.398, -0.248, 1.432, -3.897, -0.505, -2.806}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-2.472, -2.910, -3.298, 2.334, -2.865, -3.884, 1.561, -3.745, -3.505, 1.638, -3.961, -0.809, 1.348, -3.685, -0.565, -2.502}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-3.394, -3.984, -3.723, 1.568, -2.885, -4.444, 1.480, -4.093, -4.044, 1.659, -4.615, -0.375, 2.136, -4.127, -0.039, -1.657}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-6.166, -3.166, -5.989, 2.553, -7.067, -6.589, -0.358, -6.661, -5.268, 2.491, -6.669, -0.828, 1.504, -5.848, -4.163, -4.220}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-3.854, -4.558, -5.947, 2.137, -4.877, -6.553, 0.809, -6.612, -6.367, 2.775, -6.628, 0.743, -0.271, -3.669, -2.651, -4.070}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-4.666, -4.173, -4.824, 1.843, -4.040, -5.672, 2.065, -4.967, -4.849, 1.244, -5.407, -0.370, 1.918, -4.134, -0.764, -1.473}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-5.903, -3.408, -5.802, 2.181, -6.466, -6.449, 0.850, -5.574, -5.510, 2.656, -5.702, -0.845, 1.069, -5.715, -1.692, -2.625}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-6.161, -2.671, -5.979, 1.682, -4.238, -6.575, -1.024, -5.274, -3.362, 3.395, -6.664, -1.585, -0.233, -5.834, -2.993, -3.844}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-6.091, -2.555, -5.909, 0.014, -2.422, -4.254, -1.966, -4.716, -6.348, 3.744, -5.330, -1.897, -1.507, -3.823, -3.341, -4.875}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-6.151, -4.431, -3.700, 0.635, -7.080, -6.572, 0.534, -4.868, -4.084, 3.314, -4.822, 0.708, -0.271, -5.831, -3.270, -2.447}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-6.442, -5.026, -5.615, -0.442, -5.322, -7.289, 3.134, -5.973, -6.306, 0.420, -6.820, -3.345, 2.133, -4.092, -1.261, -4.187}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-2.956, -3.236, -5.498, 0.763, -2.438, -6.346, 1.852, -5.433, -4.783, 1.324, -6.017, -3.202, 2.763, -5.656, -0.526, -4.430}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.671, -2.885, -1.628, 0.325, -2.963, -3.779, 1.747, -3.851, 1.255, -0.689, -3.286, -0.364, 1.696, -3.043, 0.401, -0.995}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-2.237, -2.053, -3.155, 1.253, -2.279, -3.398, 2.076, -1.914, -3.153, 0.023, -4.318, 0.413, 2.214, -2.674, -0.614, -1.686}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-5.608, -5.549, -5.706, 1.290, -5.005, -3.722, 2.496, -3.844, -5.660, 0.382, -6.200, -1.388, 2.386, -5.869, -0.958, -2.332}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-4.525, -2.838, -5.648, 2.002, -4.830, -4.261, 0.835, -6.105, -5.916, 2.561, -6.293, -2.010, 1.817, -5.606, -3.378, -3.507}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-5.176, -3.076, -5.982, 1.354, -6.059, -6.578, 1.193, -5.535, -6.417, 3.221, -6.665, -0.241, -1.060, -5.837, -2.639, -3.364}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {-2.699, -2.468, -3.094, 2.445, -3.539, -4.541, 1.514, -4.041, -2.655, 1.418, -3.793, -1.041, 1.370, -3.433, -0.480, -1.862}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, +}; +inline float getPairEmission(int pos, char a, char b) { + return pair_emissions[pos][charToIndex(a, b)]; +} +#endif diff --git a/testdata/grammar_outside/RF00005.gap b/testdata/grammar_outside/RF00005.gap new file mode 100644 index 000000000..cb9015c34 --- /dev/null +++ b/testdata/grammar_outside/RF00005.gap @@ -0,0 +1,1550 @@ +import "RF00005_data.hh" + +signature sig_cm(alphabet, answer) { + answer silent_transition(float, answer; int); + answer left_transition(float, alphabet, answer; int); + answer right_transition(float, answer, alphabet; int); + answer pair_transition(float, alphabet, answer, alphabet; int); + answer bifurcation_transition(float, float, answer, answer; int); + answer nil(float, void; int); + choice [answer] h([answer]); +} + + +algebra alg_enum auto enum; + +algebra alg_count auto count; + +algebra alg_cyk implements sig_cm(alphabet=char, answer=float) { + float silent_transition(float tsc, float x; int pos) { + return tsc + x; + } + float left_transition(float tsc, alphabet a, float x; int pos) { + return tsc + getEmission(pos, a) + x; + } + float right_transition(float tsc, float x, alphabet a; int pos) { + return tsc + getEmission(pos, a) + x; + } + float pair_transition(float tsc, alphabet a, float x, alphabet b; int pos) { + return tsc + getPairEmission(pos, a, b) + x; + } + float bifurcation_transition(float tsc_left, float tsc_right, float left, float right; int pos) { + return tsc_left + tsc_right + left + right; + } + float nil(float tsc, void; int pos) { + return tsc; + } + choice [float] h([float] candidates) { + return list(maximum(candidates)); + } +} + +algebra alg_inside extends alg_cyk { + choice [float] h([float] candidates) { + return list(bitsum(candidates)); + } +} + +grammar gra_cm uses sig_cm(axiom = state_S_0) { + // node [ ROOT 0 ] + state_S_0 = silent_transition(CONST_FLOAT(-11.033000), state_IL_1; 0) + | silent_transition(CONST_FLOAT(-12.278000), state_IR_2; 0) + | silent_transition(CONST_FLOAT(-0.061000), state_MR_3; 0) + | silent_transition(CONST_FLOAT(-4.629000), state_D_4; 0) + # h; + + state_IL_1 = left_transition(CONST_FLOAT(-2.817000), CHAR, state_IL_1; 1) + | left_transition(CONST_FLOAT(-4.319000), CHAR, state_IR_2; 1) + | left_transition(CONST_FLOAT(-0.613000), CHAR, state_MR_3; 1) + | left_transition(CONST_FLOAT(-2.698000), CHAR, state_D_4; 1) + # h; + + state_IR_2 = right_transition(CONST_FLOAT(-1.925000), state_IR_2, CHAR; 2) + | right_transition(CONST_FLOAT(-0.554000), state_MR_3, CHAR; 2) + | right_transition(CONST_FLOAT(-4.164000), state_D_4, CHAR; 2) + # h; + + // node [ MATR 1 ] + state_MR_3 = right_transition(CONST_FLOAT(-11.036000), state_IR_5, CHAR; 3) + | right_transition(CONST_FLOAT(-0.003000), state_MP_6, CHAR; 3) + | right_transition(CONST_FLOAT(-10.852000), state_ML_7, CHAR; 3) + | right_transition(CONST_FLOAT(-11.064000), state_MR_8, CHAR; 3) + | right_transition(CONST_FLOAT(-11.956000), state_D_9, CHAR; 3) + # h; + + state_D_4 = silent_transition(CONST_FLOAT(-1.727000), state_IR_5; 4) + | silent_transition(CONST_FLOAT(-1.310000), state_MP_6; 4) + | silent_transition(CONST_FLOAT(-4.986000), state_ML_7; 4) + | silent_transition(CONST_FLOAT(-2.211000), state_MR_8; 4) + | silent_transition(CONST_FLOAT(-4.411000), state_D_9; 4) + # h; + + state_IR_5 = right_transition(CONST_FLOAT(-3.146000), state_IR_5, CHAR; 5) + | right_transition(CONST_FLOAT(-0.277000), state_MP_6, CHAR; 5) + | right_transition(CONST_FLOAT(-6.657000), state_ML_7, CHAR; 5) + | right_transition(CONST_FLOAT(-4.825000), state_MR_8, CHAR; 5) + | right_transition(CONST_FLOAT(-5.931000), state_D_9, CHAR; 5) + # h; + + // node [ MATP 2 ] + state_MP_6 = pair_transition(CONST_FLOAT(-12.104000), CHAR, state_IL_10, CHAR; 6) + | pair_transition(CONST_FLOAT(-12.043000), CHAR, state_IR_11, CHAR; 6) + | pair_transition(CONST_FLOAT(-0.003000), CHAR, state_MP_12, CHAR; 6) + | pair_transition(CONST_FLOAT(-10.819000), CHAR, state_ML_13, CHAR; 6) + | pair_transition(CONST_FLOAT(-11.099000), CHAR, state_MR_14, CHAR; 6) + | pair_transition(CONST_FLOAT(-11.494000), CHAR, state_D_15, CHAR; 6) + # h; + + state_ML_7 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_10; 7) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_11; 7) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_12; 7) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_13; 7) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_14; 7) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_15; 7) + # h; + + state_MR_8 = right_transition(CONST_FLOAT(-7.909000), state_IL_10, CHAR; 8) + | right_transition(CONST_FLOAT(-6.638000), state_IR_11, CHAR; 8) + | right_transition(CONST_FLOAT(-0.637000), state_MP_12, CHAR; 8) + | right_transition(CONST_FLOAT(-6.616000), state_ML_13, CHAR; 8) + | right_transition(CONST_FLOAT(-1.750000), state_MR_14, CHAR; 8) + | right_transition(CONST_FLOAT(-4.829000), state_D_15, CHAR; 8) + # h; + + state_D_9 = silent_transition(CONST_FLOAT(-9.049000), state_IL_10; 9) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_11; 9) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_12; 9) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_13; 9) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_14; 9) + | silent_transition(CONST_FLOAT(-0.319000), state_D_15; 9) + # h; + + state_IL_10 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_10; 10) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_11; 10) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_12; 10) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_13; 10) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_14; 10) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_15; 10) + # h; + + state_IR_11 = right_transition(CONST_FLOAT(-2.408000), state_IR_11, CHAR; 11) + | right_transition(CONST_FLOAT(-0.496000), state_MP_12, CHAR; 11) + | right_transition(CONST_FLOAT(-5.920000), state_ML_13, CHAR; 11) + | right_transition(CONST_FLOAT(-4.087000), state_MR_14, CHAR; 11) + | right_transition(CONST_FLOAT(-5.193000), state_D_15, CHAR; 11) + # h; + + // node [ MATP 3 ] + state_MP_12 = pair_transition(CONST_FLOAT(-12.117000), CHAR, state_IL_16, CHAR; 12) + | pair_transition(CONST_FLOAT(-12.057000), CHAR, state_IR_17, CHAR; 12) + | pair_transition(CONST_FLOAT(-0.003000), CHAR, state_MP_18, CHAR; 12) + | pair_transition(CONST_FLOAT(-10.833000), CHAR, state_ML_19, CHAR; 12) + | pair_transition(CONST_FLOAT(-11.113000), CHAR, state_MR_20, CHAR; 12) + | pair_transition(CONST_FLOAT(-11.508000), CHAR, state_D_21, CHAR; 12) + # h; + + state_ML_13 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_16; 13) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_17; 13) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_18; 13) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_19; 13) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_20; 13) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_21; 13) + # h; + + state_MR_14 = right_transition(CONST_FLOAT(-6.988000), state_IL_16, CHAR; 14) + | right_transition(CONST_FLOAT(-5.717000), state_IR_17, CHAR; 14) + | right_transition(CONST_FLOAT(-1.625000), state_MP_18, CHAR; 14) + | right_transition(CONST_FLOAT(-5.695000), state_ML_19, CHAR; 14) + | right_transition(CONST_FLOAT(-0.829000), state_MR_20, CHAR; 14) + | right_transition(CONST_FLOAT(-3.908000), state_D_21, CHAR; 14) + # h; + + state_D_15 = silent_transition(CONST_FLOAT(-9.049000), state_IL_16; 15) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_17; 15) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_18; 15) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_19; 15) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_20; 15) + | silent_transition(CONST_FLOAT(-0.319000), state_D_21; 15) + # h; + + state_IL_16 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_16; 16) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_17; 16) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_18; 16) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_19; 16) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_20; 16) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_21; 16) + # h; + + state_IR_17 = right_transition(CONST_FLOAT(-2.408000), state_IR_17, CHAR; 17) + | right_transition(CONST_FLOAT(-0.496000), state_MP_18, CHAR; 17) + | right_transition(CONST_FLOAT(-5.920000), state_ML_19, CHAR; 17) + | right_transition(CONST_FLOAT(-4.087000), state_MR_20, CHAR; 17) + | right_transition(CONST_FLOAT(-5.193000), state_D_21, CHAR; 17) + # h; + + // node [ MATP 4 ] + state_MP_18 = pair_transition(CONST_FLOAT(-12.117000), CHAR, state_IL_22, CHAR; 18) + | pair_transition(CONST_FLOAT(-12.057000), CHAR, state_IR_23, CHAR; 18) + | pair_transition(CONST_FLOAT(-0.003000), CHAR, state_MP_24, CHAR; 18) + | pair_transition(CONST_FLOAT(-10.833000), CHAR, state_ML_25, CHAR; 18) + | pair_transition(CONST_FLOAT(-11.113000), CHAR, state_MR_26, CHAR; 18) + | pair_transition(CONST_FLOAT(-11.508000), CHAR, state_D_27, CHAR; 18) + # h; + + state_ML_19 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_22; 19) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_23; 19) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_24; 19) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_25; 19) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_26; 19) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_27; 19) + # h; + + state_MR_20 = right_transition(CONST_FLOAT(-6.988000), state_IL_22, CHAR; 20) + | right_transition(CONST_FLOAT(-5.717000), state_IR_23, CHAR; 20) + | right_transition(CONST_FLOAT(-1.625000), state_MP_24, CHAR; 20) + | right_transition(CONST_FLOAT(-5.695000), state_ML_25, CHAR; 20) + | right_transition(CONST_FLOAT(-0.829000), state_MR_26, CHAR; 20) + | right_transition(CONST_FLOAT(-3.908000), state_D_27, CHAR; 20) + # h; + + state_D_21 = silent_transition(CONST_FLOAT(-9.049000), state_IL_22; 21) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_23; 21) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_24; 21) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_25; 21) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_26; 21) + | silent_transition(CONST_FLOAT(-0.319000), state_D_27; 21) + # h; + + state_IL_22 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_22; 22) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_23; 22) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_24; 22) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_25; 22) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_26; 22) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_27; 22) + # h; + + state_IR_23 = right_transition(CONST_FLOAT(-2.408000), state_IR_23, CHAR; 23) + | right_transition(CONST_FLOAT(-0.496000), state_MP_24, CHAR; 23) + | right_transition(CONST_FLOAT(-5.920000), state_ML_25, CHAR; 23) + | right_transition(CONST_FLOAT(-4.087000), state_MR_26, CHAR; 23) + | right_transition(CONST_FLOAT(-5.193000), state_D_27, CHAR; 23) + # h; + + // node [ MATP 5 ] + state_MP_24 = pair_transition(CONST_FLOAT(-12.117000), CHAR, state_IL_28, CHAR; 24) + | pair_transition(CONST_FLOAT(-12.057000), CHAR, state_IR_29, CHAR; 24) + | pair_transition(CONST_FLOAT(-0.003000), CHAR, state_MP_30, CHAR; 24) + | pair_transition(CONST_FLOAT(-10.833000), CHAR, state_ML_31, CHAR; 24) + | pair_transition(CONST_FLOAT(-11.113000), CHAR, state_MR_32, CHAR; 24) + | pair_transition(CONST_FLOAT(-11.508000), CHAR, state_D_33, CHAR; 24) + # h; + + state_ML_25 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_28; 25) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_29; 25) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_30; 25) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_31; 25) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_32; 25) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_33; 25) + # h; + + state_MR_26 = right_transition(CONST_FLOAT(-6.988000), state_IL_28, CHAR; 26) + | right_transition(CONST_FLOAT(-5.717000), state_IR_29, CHAR; 26) + | right_transition(CONST_FLOAT(-1.625000), state_MP_30, CHAR; 26) + | right_transition(CONST_FLOAT(-5.695000), state_ML_31, CHAR; 26) + | right_transition(CONST_FLOAT(-0.829000), state_MR_32, CHAR; 26) + | right_transition(CONST_FLOAT(-3.908000), state_D_33, CHAR; 26) + # h; + + state_D_27 = silent_transition(CONST_FLOAT(-9.049000), state_IL_28; 27) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_29; 27) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_30; 27) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_31; 27) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_32; 27) + | silent_transition(CONST_FLOAT(-0.319000), state_D_33; 27) + # h; + + state_IL_28 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_28; 28) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_29; 28) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_30; 28) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_31; 28) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_32; 28) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_33; 28) + # h; + + state_IR_29 = right_transition(CONST_FLOAT(-2.408000), state_IR_29, CHAR; 29) + | right_transition(CONST_FLOAT(-0.496000), state_MP_30, CHAR; 29) + | right_transition(CONST_FLOAT(-5.920000), state_ML_31, CHAR; 29) + | right_transition(CONST_FLOAT(-4.087000), state_MR_32, CHAR; 29) + | right_transition(CONST_FLOAT(-5.193000), state_D_33, CHAR; 29) + # h; + + // node [ MATP 6 ] + state_MP_30 = pair_transition(CONST_FLOAT(-12.117000), CHAR, state_IL_34, CHAR; 30) + | pair_transition(CONST_FLOAT(-12.057000), CHAR, state_IR_35, CHAR; 30) + | pair_transition(CONST_FLOAT(-0.003000), CHAR, state_MP_36, CHAR; 30) + | pair_transition(CONST_FLOAT(-10.833000), CHAR, state_ML_37, CHAR; 30) + | pair_transition(CONST_FLOAT(-11.113000), CHAR, state_MR_38, CHAR; 30) + | pair_transition(CONST_FLOAT(-11.508000), CHAR, state_D_39, CHAR; 30) + # h; + + state_ML_31 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_34; 31) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_35; 31) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_36; 31) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_37; 31) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_38; 31) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_39; 31) + # h; + + state_MR_32 = right_transition(CONST_FLOAT(-6.988000), state_IL_34, CHAR; 32) + | right_transition(CONST_FLOAT(-5.717000), state_IR_35, CHAR; 32) + | right_transition(CONST_FLOAT(-1.625000), state_MP_36, CHAR; 32) + | right_transition(CONST_FLOAT(-5.695000), state_ML_37, CHAR; 32) + | right_transition(CONST_FLOAT(-0.829000), state_MR_38, CHAR; 32) + | right_transition(CONST_FLOAT(-3.908000), state_D_39, CHAR; 32) + # h; + + state_D_33 = silent_transition(CONST_FLOAT(-9.049000), state_IL_34; 33) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_35; 33) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_36; 33) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_37; 33) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_38; 33) + | silent_transition(CONST_FLOAT(-0.319000), state_D_39; 33) + # h; + + state_IL_34 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_34; 34) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_35; 34) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_36; 34) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_37; 34) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_38; 34) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_39; 34) + # h; + + state_IR_35 = right_transition(CONST_FLOAT(-2.408000), state_IR_35, CHAR; 35) + | right_transition(CONST_FLOAT(-0.496000), state_MP_36, CHAR; 35) + | right_transition(CONST_FLOAT(-5.920000), state_ML_37, CHAR; 35) + | right_transition(CONST_FLOAT(-4.087000), state_MR_38, CHAR; 35) + | right_transition(CONST_FLOAT(-5.193000), state_D_39, CHAR; 35) + # h; + + // node [ MATP 7 ] + state_MP_36 = pair_transition(CONST_FLOAT(-12.117000), CHAR, state_IL_40, CHAR; 36) + | pair_transition(CONST_FLOAT(-12.057000), CHAR, state_IR_41, CHAR; 36) + | pair_transition(CONST_FLOAT(-0.004000), CHAR, state_MP_42, CHAR; 36) + | pair_transition(CONST_FLOAT(-9.656000), CHAR, state_ML_43, CHAR; 36) + | pair_transition(CONST_FLOAT(-11.113000), CHAR, state_MR_44, CHAR; 36) + | pair_transition(CONST_FLOAT(-11.508000), CHAR, state_D_45, CHAR; 36) + # h; + + state_ML_37 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_40; 37) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_41; 37) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_42; 37) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_43; 37) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_44; 37) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_45; 37) + # h; + + state_MR_38 = right_transition(CONST_FLOAT(-6.988000), state_IL_40, CHAR; 38) + | right_transition(CONST_FLOAT(-5.717000), state_IR_41, CHAR; 38) + | right_transition(CONST_FLOAT(-1.625000), state_MP_42, CHAR; 38) + | right_transition(CONST_FLOAT(-5.695000), state_ML_43, CHAR; 38) + | right_transition(CONST_FLOAT(-0.829000), state_MR_44, CHAR; 38) + | right_transition(CONST_FLOAT(-3.908000), state_D_45, CHAR; 38) + # h; + + state_D_39 = silent_transition(CONST_FLOAT(-9.049000), state_IL_40; 39) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_41; 39) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_42; 39) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_43; 39) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_44; 39) + | silent_transition(CONST_FLOAT(-0.319000), state_D_45; 39) + # h; + + state_IL_40 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_40; 40) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_41; 40) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_42; 40) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_43; 40) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_44; 40) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_45; 40) + # h; + + state_IR_41 = right_transition(CONST_FLOAT(-2.408000), state_IR_41, CHAR; 41) + | right_transition(CONST_FLOAT(-0.496000), state_MP_42, CHAR; 41) + | right_transition(CONST_FLOAT(-5.920000), state_ML_43, CHAR; 41) + | right_transition(CONST_FLOAT(-4.087000), state_MR_44, CHAR; 41) + | right_transition(CONST_FLOAT(-5.193000), state_D_45, CHAR; 41) + # h; + + // node [ MATP 8 ] + state_MP_42 = pair_transition(CONST_FLOAT(-11.232000), CHAR, state_IL_46, CHAR; 42) + | pair_transition(CONST_FLOAT(-6.672000), CHAR, state_IR_47, CHAR; 42) + | pair_transition(CONST_FLOAT(-0.016000), CHAR, state_ML_48, CHAR; 42) + | pair_transition(CONST_FLOAT(-9.853000), CHAR, state_D_49, CHAR; 42) + # h; + + state_ML_43 = left_transition(CONST_FLOAT(-3.836000), CHAR, state_IL_46; 43) + | left_transition(CONST_FLOAT(-4.018000), CHAR, state_IR_47; 43) + | left_transition(CONST_FLOAT(-0.475000), CHAR, state_ML_48; 43) + | left_transition(CONST_FLOAT(-2.747000), CHAR, state_D_49; 43) + # h; + + state_MR_44 = right_transition(CONST_FLOAT(-4.809000), state_IL_46, CHAR; 44) + | right_transition(CONST_FLOAT(-3.838000), state_IR_47, CHAR; 44) + | right_transition(CONST_FLOAT(-1.706000), state_ML_48, CHAR; 44) + | right_transition(CONST_FLOAT(-0.766000), state_D_49, CHAR; 44) + # h; + + state_D_45 = silent_transition(CONST_FLOAT(-4.568000), state_IL_46; 45) + | silent_transition(CONST_FLOAT(-4.250000), state_IR_47; 45) + | silent_transition(CONST_FLOAT(-2.265000), state_ML_48; 45) + | silent_transition(CONST_FLOAT(-0.520000), state_D_49; 45) + # h; + + state_IL_46 = left_transition(CONST_FLOAT(-1.686000), CHAR, state_IL_46; 46) + | left_transition(CONST_FLOAT(-2.369000), CHAR, state_IR_47; 46) + | left_transition(CONST_FLOAT(-1.117000), CHAR, state_ML_48; 46) + | left_transition(CONST_FLOAT(-4.855000), CHAR, state_D_49; 46) + # h; + + state_IR_47 = right_transition(CONST_FLOAT(-1.924000), state_IR_47, CHAR; 47) + | right_transition(CONST_FLOAT(-0.523000), state_ML_48, CHAR; 47) + | right_transition(CONST_FLOAT(-4.624000), state_D_49, CHAR; 47) + # h; + + // node [ MATL 9 ] + state_ML_48 = left_transition(CONST_FLOAT(-5.352000), CHAR, state_IL_50; 48) + | left_transition(CONST_FLOAT(-0.048000), CHAR, state_ML_51; 48) + | left_transition(CONST_FLOAT(-6.940000), CHAR, state_D_52; 48) + # h; + + state_D_49 = silent_transition(CONST_FLOAT(-6.174000), state_IL_50; 49) + | silent_transition(CONST_FLOAT(-1.687000), state_ML_51; 49) + | silent_transition(CONST_FLOAT(-0.566000), state_D_52; 49) + # h; + + state_IL_50 = left_transition(CONST_FLOAT(-2.459000), CHAR, state_IL_50; 50) + | left_transition(CONST_FLOAT(-0.340000), CHAR, state_ML_51; 50) + | left_transition(CONST_FLOAT(-5.159000), CHAR, state_D_52; 50) + # h; + + // node [ MATL 10 ] + state_ML_51 = left_transition(CONST_FLOAT(-5.143000), CHAR, state_IL_53; 51) + | left_transition(CONST_FLOAT(-0.041000), CHAR, state_B_54; 51) + # h; + + state_D_52 = silent_transition(CONST_FLOAT(-8.552000), state_IL_53; 52) + | silent_transition(CONST_FLOAT(-0.004000), state_B_54; 52) + # h; + + state_IL_53 = left_transition(CONST_FLOAT(-3.425000), CHAR, state_IL_53; 53) + | left_transition(CONST_FLOAT(-0.141000), CHAR, state_B_54; 53) + # h; + + // node [ BIF 11 ] + state_B_54 = bifurcation_transition(CONST_FLOAT(0.000000), CONST_FLOAT(0.000000), state_S_121, state_S_55; 54) + # h; + + // node [ BEGR 42 ] + state_S_55 = silent_transition(CONST_FLOAT(-12.148000), state_IL_56; 55) + | silent_transition(CONST_FLOAT(-0.001000), state_ML_57; 55) + | silent_transition(CONST_FLOAT(-10.802000), state_D_58; 55) + # h; + + state_IL_56 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_56; 56) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_57; 56) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_58; 56) + # h; + + // node [ MATL 43 ] + state_ML_57 = left_transition(CONST_FLOAT(-12.148000), CHAR, state_IL_59; 57) + | left_transition(CONST_FLOAT(-0.425000), CHAR, state_ML_60; 57) + | left_transition(CONST_FLOAT(-1.972000), CHAR, state_D_61; 57) + # h; + + state_D_58 = silent_transition(CONST_FLOAT(-6.174000), state_IL_59; 58) + | silent_transition(CONST_FLOAT(-1.687000), state_ML_60; 58) + | silent_transition(CONST_FLOAT(-0.566000), state_D_61; 58) + # h; + + state_IL_59 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_59; 59) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_60; 59) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_61; 59) + # h; + + // node [ MATL 44 ] + state_ML_60 = left_transition(CONST_FLOAT(-4.403000), CHAR, state_IL_62; 60) + | left_transition(CONST_FLOAT(-0.079000), CHAR, state_ML_63; 60) + | left_transition(CONST_FLOAT(-7.327000), CHAR, state_D_64; 60) + # h; + + state_D_61 = silent_transition(CONST_FLOAT(-0.051000), state_IL_62; 61) + | silent_transition(CONST_FLOAT(-5.676000), state_ML_63; 61) + | silent_transition(CONST_FLOAT(-6.016000), state_D_64; 61) + # h; + + state_IL_62 = left_transition(CONST_FLOAT(-0.113000), CHAR, state_IL_62; 62) + | left_transition(CONST_FLOAT(-4.469000), CHAR, state_ML_63; 62) + | left_transition(CONST_FLOAT(-5.038000), CHAR, state_D_64; 62) + # h; + + // node [ MATL 45 ] + state_ML_63 = left_transition(CONST_FLOAT(-1.234000), CHAR, state_IL_65; 63) + | left_transition(CONST_FLOAT(-0.803000), CHAR, state_ML_66; 63) + | left_transition(CONST_FLOAT(-9.120000), CHAR, state_D_67; 63) + # h; + + state_D_64 = silent_transition(CONST_FLOAT(-6.563000), state_IL_65; 64) + | silent_transition(CONST_FLOAT(-0.061000), state_ML_66; 64) + | silent_transition(CONST_FLOAT(-5.013000), state_D_67; 64) + # h; + + state_IL_65 = left_transition(CONST_FLOAT(-3.838000), CHAR, state_IL_65; 65) + | left_transition(CONST_FLOAT(-0.110000), CHAR, state_ML_66; 65) + | left_transition(CONST_FLOAT(-8.281000), CHAR, state_D_67; 65) + # h; + + // node [ MATL 46 ] + state_ML_66 = left_transition(CONST_FLOAT(-11.090000), CHAR, state_IL_68; 66) + | left_transition(CONST_FLOAT(-0.002000), CHAR, state_MP_69; 66) + | left_transition(CONST_FLOAT(-10.906000), CHAR, state_ML_70; 66) + | left_transition(CONST_FLOAT(-11.118000), CHAR, state_MR_71; 66) + | left_transition(CONST_FLOAT(-12.010000), CHAR, state_D_72; 66) + # h; + + state_D_67 = silent_transition(CONST_FLOAT(-5.092000), state_IL_68; 67) + | silent_transition(CONST_FLOAT(-0.712000), state_MP_69; 67) + | silent_transition(CONST_FLOAT(-4.353000), state_ML_70; 67) + | silent_transition(CONST_FLOAT(-2.728000), state_MR_71; 67) + | silent_transition(CONST_FLOAT(-2.640000), state_D_72; 67) + # h; + + state_IL_68 = left_transition(CONST_FLOAT(-2.408000), CHAR, state_IL_68; 68) + | left_transition(CONST_FLOAT(-0.496000), CHAR, state_MP_69; 68) + | left_transition(CONST_FLOAT(-4.087000), CHAR, state_ML_70; 68) + | left_transition(CONST_FLOAT(-5.920000), CHAR, state_MR_71; 68) + | left_transition(CONST_FLOAT(-5.193000), CHAR, state_D_72; 68) + # h; + + // node [ MATP 47 ] + state_MP_69 = pair_transition(CONST_FLOAT(-12.117000), CHAR, state_IL_73, CHAR; 69) + | pair_transition(CONST_FLOAT(-6.010000), CHAR, state_IR_74, CHAR; 69) + | pair_transition(CONST_FLOAT(-0.025000), CHAR, state_MP_75, CHAR; 69) + | pair_transition(CONST_FLOAT(-10.833000), CHAR, state_ML_76, CHAR; 69) + | pair_transition(CONST_FLOAT(-11.113000), CHAR, state_MR_77, CHAR; 69) + | pair_transition(CONST_FLOAT(-11.508000), CHAR, state_D_78, CHAR; 69) + # h; + + state_ML_70 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_73; 70) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_74; 70) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_75; 70) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_76; 70) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_77; 70) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_78; 70) + # h; + + state_MR_71 = right_transition(CONST_FLOAT(-6.988000), state_IL_73, CHAR; 71) + | right_transition(CONST_FLOAT(-5.717000), state_IR_74, CHAR; 71) + | right_transition(CONST_FLOAT(-1.625000), state_MP_75, CHAR; 71) + | right_transition(CONST_FLOAT(-5.695000), state_ML_76, CHAR; 71) + | right_transition(CONST_FLOAT(-0.829000), state_MR_77, CHAR; 71) + | right_transition(CONST_FLOAT(-3.908000), state_D_78, CHAR; 71) + # h; + + state_D_72 = silent_transition(CONST_FLOAT(-9.049000), state_IL_73; 72) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_74; 72) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_75; 72) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_76; 72) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_77; 72) + | silent_transition(CONST_FLOAT(-0.319000), state_D_78; 72) + # h; + + state_IL_73 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_73; 73) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_74; 73) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_75; 73) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_76; 73) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_77; 73) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_78; 73) + # h; + + state_IR_74 = right_transition(CONST_FLOAT(-3.202000), state_IR_74, CHAR; 74) + | right_transition(CONST_FLOAT(-0.265000), state_MP_75, CHAR; 74) + | right_transition(CONST_FLOAT(-6.713000), state_ML_76, CHAR; 74) + | right_transition(CONST_FLOAT(-4.881000), state_MR_77, CHAR; 74) + | right_transition(CONST_FLOAT(-5.987000), state_D_78, CHAR; 74) + # h; + + // node [ MATP 48 ] + state_MP_75 = pair_transition(CONST_FLOAT(-12.117000), CHAR, state_IL_79, CHAR; 75) + | pair_transition(CONST_FLOAT(-12.057000), CHAR, state_IR_80, CHAR; 75) + | pair_transition(CONST_FLOAT(-0.003000), CHAR, state_MP_81, CHAR; 75) + | pair_transition(CONST_FLOAT(-10.833000), CHAR, state_ML_82, CHAR; 75) + | pair_transition(CONST_FLOAT(-11.113000), CHAR, state_MR_83, CHAR; 75) + | pair_transition(CONST_FLOAT(-11.508000), CHAR, state_D_84, CHAR; 75) + # h; + + state_ML_76 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_79; 76) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_80; 76) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_81; 76) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_82; 76) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_83; 76) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_84; 76) + # h; + + state_MR_77 = right_transition(CONST_FLOAT(-6.988000), state_IL_79, CHAR; 77) + | right_transition(CONST_FLOAT(-5.717000), state_IR_80, CHAR; 77) + | right_transition(CONST_FLOAT(-1.625000), state_MP_81, CHAR; 77) + | right_transition(CONST_FLOAT(-5.695000), state_ML_82, CHAR; 77) + | right_transition(CONST_FLOAT(-0.829000), state_MR_83, CHAR; 77) + | right_transition(CONST_FLOAT(-3.908000), state_D_84, CHAR; 77) + # h; + + state_D_78 = silent_transition(CONST_FLOAT(-9.049000), state_IL_79; 78) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_80; 78) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_81; 78) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_82; 78) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_83; 78) + | silent_transition(CONST_FLOAT(-0.319000), state_D_84; 78) + # h; + + state_IL_79 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_79; 79) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_80; 79) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_81; 79) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_82; 79) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_83; 79) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_84; 79) + # h; + + state_IR_80 = right_transition(CONST_FLOAT(-2.408000), state_IR_80, CHAR; 80) + | right_transition(CONST_FLOAT(-0.496000), state_MP_81, CHAR; 80) + | right_transition(CONST_FLOAT(-5.920000), state_ML_82, CHAR; 80) + | right_transition(CONST_FLOAT(-4.087000), state_MR_83, CHAR; 80) + | right_transition(CONST_FLOAT(-5.193000), state_D_84, CHAR; 80) + # h; + + // node [ MATP 49 ] + state_MP_81 = pair_transition(CONST_FLOAT(-6.146000), CHAR, state_IL_85, CHAR; 81) + | pair_transition(CONST_FLOAT(-12.057000), CHAR, state_IR_86, CHAR; 81) + | pair_transition(CONST_FLOAT(-0.026000), CHAR, state_MP_87, CHAR; 81) + | pair_transition(CONST_FLOAT(-10.833000), CHAR, state_ML_88, CHAR; 81) + | pair_transition(CONST_FLOAT(-11.113000), CHAR, state_MR_89, CHAR; 81) + | pair_transition(CONST_FLOAT(-8.649000), CHAR, state_D_90, CHAR; 81) + # h; + + state_ML_82 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_85; 82) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_86; 82) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_87; 82) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_88; 82) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_89; 82) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_90; 82) + # h; + + state_MR_83 = right_transition(CONST_FLOAT(-6.988000), state_IL_85, CHAR; 83) + | right_transition(CONST_FLOAT(-5.717000), state_IR_86, CHAR; 83) + | right_transition(CONST_FLOAT(-1.625000), state_MP_87, CHAR; 83) + | right_transition(CONST_FLOAT(-5.695000), state_ML_88, CHAR; 83) + | right_transition(CONST_FLOAT(-0.829000), state_MR_89, CHAR; 83) + | right_transition(CONST_FLOAT(-3.908000), state_D_90, CHAR; 83) + # h; + + state_D_84 = silent_transition(CONST_FLOAT(-9.049000), state_IL_85; 84) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_86; 84) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_87; 84) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_88; 84) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_89; 84) + | silent_transition(CONST_FLOAT(-0.319000), state_D_90; 84) + # h; + + state_IL_85 = left_transition(CONST_FLOAT(-3.649000), CHAR, state_IL_85; 85) + | left_transition(CONST_FLOAT(-3.912000), CHAR, state_IR_86; 85) + | left_transition(CONST_FLOAT(-0.313000), CHAR, state_MP_87; 85) + | left_transition(CONST_FLOAT(-5.567000), CHAR, state_ML_88; 85) + | left_transition(CONST_FLOAT(-6.343000), CHAR, state_MR_89; 85) + | left_transition(CONST_FLOAT(-6.004000), CHAR, state_D_90; 85) + # h; + + state_IR_86 = right_transition(CONST_FLOAT(-2.408000), state_IR_86, CHAR; 86) + | right_transition(CONST_FLOAT(-0.496000), state_MP_87, CHAR; 86) + | right_transition(CONST_FLOAT(-5.920000), state_ML_88, CHAR; 86) + | right_transition(CONST_FLOAT(-4.087000), state_MR_89, CHAR; 86) + | right_transition(CONST_FLOAT(-5.193000), state_D_90, CHAR; 86) + # h; + + // node [ MATP 50 ] + state_MP_87 = pair_transition(CONST_FLOAT(-12.114000), CHAR, state_IL_91, CHAR; 87) + | pair_transition(CONST_FLOAT(-12.054000), CHAR, state_IR_92, CHAR; 87) + | pair_transition(CONST_FLOAT(-0.078000), CHAR, state_MP_93, CHAR; 87) + | pair_transition(CONST_FLOAT(-8.063000), CHAR, state_ML_94, CHAR; 87) + | pair_transition(CONST_FLOAT(-11.110000), CHAR, state_MR_95, CHAR; 87) + | pair_transition(CONST_FLOAT(-4.379000), CHAR, state_D_96, CHAR; 87) + # h; + + state_ML_88 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_91; 88) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_92; 88) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_93; 88) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_94; 88) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_95; 88) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_96; 88) + # h; + + state_MR_89 = right_transition(CONST_FLOAT(-6.988000), state_IL_91, CHAR; 89) + | right_transition(CONST_FLOAT(-5.717000), state_IR_92, CHAR; 89) + | right_transition(CONST_FLOAT(-1.625000), state_MP_93, CHAR; 89) + | right_transition(CONST_FLOAT(-5.695000), state_ML_94, CHAR; 89) + | right_transition(CONST_FLOAT(-0.829000), state_MR_95, CHAR; 89) + | right_transition(CONST_FLOAT(-3.908000), state_D_96, CHAR; 89) + # h; + + state_D_90 = silent_transition(CONST_FLOAT(-9.420000), state_IL_91; 90) + | silent_transition(CONST_FLOAT(-8.118000), state_IR_92; 90) + | silent_transition(CONST_FLOAT(-3.915000), state_MP_93; 90) + | silent_transition(CONST_FLOAT(-4.597000), state_ML_94; 90) + | silent_transition(CONST_FLOAT(-4.615000), state_MR_95; 90) + | silent_transition(CONST_FLOAT(-0.240000), state_D_96; 90) + # h; + + state_IL_91 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_91; 91) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_92; 91) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_93; 91) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_94; 91) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_95; 91) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_96; 91) + # h; + + state_IR_92 = right_transition(CONST_FLOAT(-2.408000), state_IR_92, CHAR; 92) + | right_transition(CONST_FLOAT(-0.496000), state_MP_93, CHAR; 92) + | right_transition(CONST_FLOAT(-5.920000), state_ML_94, CHAR; 92) + | right_transition(CONST_FLOAT(-4.087000), state_MR_95, CHAR; 92) + | right_transition(CONST_FLOAT(-5.193000), state_D_96, CHAR; 92) + # h; + + // node [ MATP 51 ] + state_MP_93 = pair_transition(CONST_FLOAT(-11.148000), CHAR, state_IL_97, CHAR; 93) + | pair_transition(CONST_FLOAT(-11.355000), CHAR, state_IR_98, CHAR; 93) + | pair_transition(CONST_FLOAT(-0.030000), CHAR, state_ML_99, CHAR; 93) + | pair_transition(CONST_FLOAT(-5.670000), CHAR, state_D_100, CHAR; 93) + # h; + + state_ML_94 = left_transition(CONST_FLOAT(-4.084000), CHAR, state_IL_97; 94) + | left_transition(CONST_FLOAT(-4.267000), CHAR, state_IR_98; 94) + | left_transition(CONST_FLOAT(-0.389000), CHAR, state_ML_99; 94) + | left_transition(CONST_FLOAT(-2.996000), CHAR, state_D_100; 94) + # h; + + state_MR_95 = right_transition(CONST_FLOAT(-4.809000), state_IL_97, CHAR; 95) + | right_transition(CONST_FLOAT(-3.838000), state_IR_98, CHAR; 95) + | right_transition(CONST_FLOAT(-1.706000), state_ML_99, CHAR; 95) + | right_transition(CONST_FLOAT(-0.766000), state_D_100, CHAR; 95) + # h; + + state_D_96 = silent_transition(CONST_FLOAT(-7.445000), state_IL_97; 96) + | silent_transition(CONST_FLOAT(-7.126000), state_IR_98; 96) + | silent_transition(CONST_FLOAT(-0.812000), state_ML_99; 96) + | silent_transition(CONST_FLOAT(-1.260000), state_D_100; 96) + # h; + + state_IL_97 = left_transition(CONST_FLOAT(-1.686000), CHAR, state_IL_97; 97) + | left_transition(CONST_FLOAT(-2.369000), CHAR, state_IR_98; 97) + | left_transition(CONST_FLOAT(-1.117000), CHAR, state_ML_99; 97) + | left_transition(CONST_FLOAT(-4.855000), CHAR, state_D_100; 97) + # h; + + state_IR_98 = right_transition(CONST_FLOAT(-1.442000), state_IR_98, CHAR; 98) + | right_transition(CONST_FLOAT(-0.798000), state_ML_99, CHAR; 98) + | right_transition(CONST_FLOAT(-4.142000), state_D_100, CHAR; 98) + # h; + + // node [ MATL 52 ] + state_ML_99 = left_transition(CONST_FLOAT(-6.272000), CHAR, state_IL_101; 99) + | left_transition(CONST_FLOAT(-0.020000), CHAR, state_ML_102; 99) + | left_transition(CONST_FLOAT(-10.747000), CHAR, state_D_103; 99) + # h; + + state_D_100 = silent_transition(CONST_FLOAT(-9.039000), state_IL_101; 100) + | silent_transition(CONST_FLOAT(-0.168000), state_ML_102; 100) + | silent_transition(CONST_FLOAT(-3.210000), state_D_103; 100) + # h; + + state_IL_101 = left_transition(CONST_FLOAT(-2.042000), CHAR, state_IL_101; 101) + | left_transition(CONST_FLOAT(-0.474000), CHAR, state_ML_102; 101) + | left_transition(CONST_FLOAT(-4.742000), CHAR, state_D_103; 101) + # h; + + // node [ MATL 53 ] + state_ML_102 = left_transition(CONST_FLOAT(-12.147000), CHAR, state_IL_104; 102) + | left_transition(CONST_FLOAT(-0.002000), CHAR, state_ML_105; 102) + | left_transition(CONST_FLOAT(-9.386000), CHAR, state_D_106; 102) + # h; + + state_D_103 = silent_transition(CONST_FLOAT(-6.327000), state_IL_104; 103) + | silent_transition(CONST_FLOAT(-1.396000), state_ML_105; 103) + | silent_transition(CONST_FLOAT(-0.719000), state_D_106; 103) + # h; + + state_IL_104 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_104; 104) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_105; 104) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_106; 104) + # h; + + // node [ MATL 54 ] + state_ML_105 = left_transition(CONST_FLOAT(-12.146000), CHAR, state_IL_107; 105) + | left_transition(CONST_FLOAT(-0.002000), CHAR, state_ML_108; 105) + | left_transition(CONST_FLOAT(-9.678000), CHAR, state_D_109; 105) + # h; + + state_D_106 = silent_transition(CONST_FLOAT(-6.384000), state_IL_107; 106) + | silent_transition(CONST_FLOAT(-1.897000), state_ML_108; 106) + | silent_transition(CONST_FLOAT(-0.475000), state_D_109; 106) + # h; + + state_IL_107 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_107; 107) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_108; 107) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_109; 107) + # h; + + // node [ MATL 55 ] + state_ML_108 = left_transition(CONST_FLOAT(-5.589000), CHAR, state_IL_110; 108) + | left_transition(CONST_FLOAT(-0.037000), CHAR, state_ML_111; 108) + | left_transition(CONST_FLOAT(-7.884000), CHAR, state_D_112; 108) + # h; + + state_D_109 = silent_transition(CONST_FLOAT(-6.516000), state_IL_110; 109) + | silent_transition(CONST_FLOAT(-1.133000), state_ML_111; 109) + | silent_transition(CONST_FLOAT(-0.908000), state_D_112; 109) + # h; + + state_IL_110 = left_transition(CONST_FLOAT(-2.342000), CHAR, state_IL_110; 110) + | left_transition(CONST_FLOAT(-0.373000), CHAR, state_ML_111; 110) + | left_transition(CONST_FLOAT(-5.042000), CHAR, state_D_112; 110) + # h; + + // node [ MATL 56 ] + state_ML_111 = left_transition(CONST_FLOAT(-12.142000), CHAR, state_IL_113; 111) + | left_transition(CONST_FLOAT(-0.043000), CHAR, state_ML_114; 111) + | left_transition(CONST_FLOAT(-5.107000), CHAR, state_D_115; 111) + # h; + + state_D_112 = silent_transition(CONST_FLOAT(-6.866000), state_IL_113; 112) + | silent_transition(CONST_FLOAT(-2.379000), state_ML_114; 112) + | silent_transition(CONST_FLOAT(-0.323000), state_D_115; 112) + # h; + + state_IL_113 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_113; 113) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_114; 113) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_115; 113) + # h; + + // node [ MATL 57 ] + state_ML_114 = left_transition(CONST_FLOAT(-4.739000), CHAR, state_IL_116; 114) + | left_transition(CONST_FLOAT(-0.155000), CHAR, state_ML_117; 114) + | left_transition(CONST_FLOAT(-3.959000), CHAR, state_D_118; 114) + # h; + + state_D_115 = silent_transition(CONST_FLOAT(-3.742000), state_IL_116; 115) + | silent_transition(CONST_FLOAT(-3.655000), state_ML_117; 115) + | silent_transition(CONST_FLOAT(-0.241000), state_D_118; 115) + # h; + + state_IL_116 = left_transition(CONST_FLOAT(-1.931000), CHAR, state_IL_116; 116) + | left_transition(CONST_FLOAT(-0.475000), CHAR, state_ML_117; 116) + | left_transition(CONST_FLOAT(-5.762000), CHAR, state_D_118; 116) + # h; + + // node [ MATL 58 ] + state_ML_117 = left_transition(CONST_FLOAT(0.000000), CHAR, state_E_120; 117) + # h; + + state_D_118 = silent_transition(CONST_FLOAT(0.000000), state_E_120; 118) + # h; + + state_IL_119 = left_transition(CONST_FLOAT(-1.823000), CHAR, state_IL_119; 119) + | left_transition(CONST_FLOAT(-0.479000), CHAR, state_E_120; 119) + # h; + + // node [ END 59 ] + state_E_120 = nil(CONST_FLOAT(0.000000), EMPTY; 120) + # h; + + // node [ BEGL 12 ] + state_S_121 = silent_transition(CONST_FLOAT(0.000000), state_B_122; 121) + # h; + + // node [ BIF 13 ] + state_B_122 = bifurcation_transition(CONST_FLOAT(0.000000), CONST_FLOAT(0.000000), state_S_123, state_S_170; 122) + # h; + + // node [ BEGL 14 ] + state_S_123 = silent_transition(CONST_FLOAT(-0.004000), state_MP_124; 123) + | silent_transition(CONST_FLOAT(-10.203000), state_ML_125; 123) + | silent_transition(CONST_FLOAT(-9.611000), state_MR_126; 123) + | silent_transition(CONST_FLOAT(-10.251000), state_D_127; 123) + # h; + + // node [ MATP 15 ] + state_MP_124 = pair_transition(CONST_FLOAT(-12.117000), CHAR, state_IL_128, CHAR; 124) + | pair_transition(CONST_FLOAT(-12.057000), CHAR, state_IR_129, CHAR; 124) + | pair_transition(CONST_FLOAT(-0.003000), CHAR, state_MP_130, CHAR; 124) + | pair_transition(CONST_FLOAT(-10.833000), CHAR, state_ML_131, CHAR; 124) + | pair_transition(CONST_FLOAT(-11.113000), CHAR, state_MR_132, CHAR; 124) + | pair_transition(CONST_FLOAT(-11.508000), CHAR, state_D_133, CHAR; 124) + # h; + + state_ML_125 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_128; 125) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_129; 125) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_130; 125) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_131; 125) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_132; 125) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_133; 125) + # h; + + state_MR_126 = right_transition(CONST_FLOAT(-6.988000), state_IL_128, CHAR; 126) + | right_transition(CONST_FLOAT(-5.717000), state_IR_129, CHAR; 126) + | right_transition(CONST_FLOAT(-1.625000), state_MP_130, CHAR; 126) + | right_transition(CONST_FLOAT(-5.695000), state_ML_131, CHAR; 126) + | right_transition(CONST_FLOAT(-0.829000), state_MR_132, CHAR; 126) + | right_transition(CONST_FLOAT(-3.908000), state_D_133, CHAR; 126) + # h; + + state_D_127 = silent_transition(CONST_FLOAT(-9.049000), state_IL_128; 127) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_129; 127) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_130; 127) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_131; 127) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_132; 127) + | silent_transition(CONST_FLOAT(-0.319000), state_D_133; 127) + # h; + + state_IL_128 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_128; 128) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_129; 128) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_130; 128) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_131; 128) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_132; 128) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_133; 128) + # h; + + state_IR_129 = right_transition(CONST_FLOAT(-2.408000), state_IR_129, CHAR; 129) + | right_transition(CONST_FLOAT(-0.496000), state_MP_130, CHAR; 129) + | right_transition(CONST_FLOAT(-5.920000), state_ML_131, CHAR; 129) + | right_transition(CONST_FLOAT(-4.087000), state_MR_132, CHAR; 129) + | right_transition(CONST_FLOAT(-5.193000), state_D_133, CHAR; 129) + # h; + + // node [ MATP 16 ] + state_MP_130 = pair_transition(CONST_FLOAT(-12.117000), CHAR, state_IL_134, CHAR; 130) + | pair_transition(CONST_FLOAT(-12.057000), CHAR, state_IR_135, CHAR; 130) + | pair_transition(CONST_FLOAT(-0.004000), CHAR, state_MP_136, CHAR; 130) + | pair_transition(CONST_FLOAT(-9.619000), CHAR, state_ML_137, CHAR; 130) + | pair_transition(CONST_FLOAT(-11.113000), CHAR, state_MR_138, CHAR; 130) + | pair_transition(CONST_FLOAT(-11.508000), CHAR, state_D_139, CHAR; 130) + # h; + + state_ML_131 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_134; 131) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_135; 131) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_136; 131) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_137; 131) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_138; 131) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_139; 131) + # h; + + state_MR_132 = right_transition(CONST_FLOAT(-6.988000), state_IL_134, CHAR; 132) + | right_transition(CONST_FLOAT(-5.717000), state_IR_135, CHAR; 132) + | right_transition(CONST_FLOAT(-1.625000), state_MP_136, CHAR; 132) + | right_transition(CONST_FLOAT(-5.695000), state_ML_137, CHAR; 132) + | right_transition(CONST_FLOAT(-0.829000), state_MR_138, CHAR; 132) + | right_transition(CONST_FLOAT(-3.908000), state_D_139, CHAR; 132) + # h; + + state_D_133 = silent_transition(CONST_FLOAT(-9.049000), state_IL_134; 133) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_135; 133) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_136; 133) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_137; 133) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_138; 133) + | silent_transition(CONST_FLOAT(-0.319000), state_D_139; 133) + # h; + + state_IL_134 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_134; 134) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_135; 134) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_136; 134) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_137; 134) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_138; 134) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_139; 134) + # h; + + state_IR_135 = right_transition(CONST_FLOAT(-2.408000), state_IR_135, CHAR; 135) + | right_transition(CONST_FLOAT(-0.496000), state_MP_136, CHAR; 135) + | right_transition(CONST_FLOAT(-5.920000), state_ML_137, CHAR; 135) + | right_transition(CONST_FLOAT(-4.087000), state_MR_138, CHAR; 135) + | right_transition(CONST_FLOAT(-5.193000), state_D_139, CHAR; 135) + # h; + + // node [ MATP 17 ] + state_MP_136 = pair_transition(CONST_FLOAT(-12.116000), CHAR, state_IL_140, CHAR; 136) + | pair_transition(CONST_FLOAT(-12.056000), CHAR, state_IR_141, CHAR; 136) + | pair_transition(CONST_FLOAT(-0.003000), CHAR, state_MP_142, CHAR; 136) + | pair_transition(CONST_FLOAT(-10.832000), CHAR, state_ML_143, CHAR; 136) + | pair_transition(CONST_FLOAT(-11.112000), CHAR, state_MR_144, CHAR; 136) + | pair_transition(CONST_FLOAT(-11.507000), CHAR, state_D_145, CHAR; 136) + # h; + + state_ML_137 = left_transition(CONST_FLOAT(-6.358000), CHAR, state_IL_140; 137) + | left_transition(CONST_FLOAT(-6.704000), CHAR, state_IR_141; 137) + | left_transition(CONST_FLOAT(-1.164000), CHAR, state_MP_142; 137) + | left_transition(CONST_FLOAT(-1.113000), CHAR, state_ML_143; 137) + | left_transition(CONST_FLOAT(-6.554000), CHAR, state_MR_144; 137) + | left_transition(CONST_FLOAT(-4.083000), CHAR, state_D_145; 137) + # h; + + state_MR_138 = right_transition(CONST_FLOAT(-6.988000), state_IL_140, CHAR; 138) + | right_transition(CONST_FLOAT(-5.717000), state_IR_141, CHAR; 138) + | right_transition(CONST_FLOAT(-1.625000), state_MP_142, CHAR; 138) + | right_transition(CONST_FLOAT(-5.695000), state_ML_143, CHAR; 138) + | right_transition(CONST_FLOAT(-0.829000), state_MR_144, CHAR; 138) + | right_transition(CONST_FLOAT(-3.908000), state_D_145, CHAR; 138) + # h; + + state_D_139 = silent_transition(CONST_FLOAT(-9.049000), state_IL_140; 139) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_141; 139) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_142; 139) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_143; 139) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_144; 139) + | silent_transition(CONST_FLOAT(-0.319000), state_D_145; 139) + # h; + + state_IL_140 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_140; 140) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_141; 140) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_142; 140) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_143; 140) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_144; 140) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_145; 140) + # h; + + state_IR_141 = right_transition(CONST_FLOAT(-2.408000), state_IR_141, CHAR; 141) + | right_transition(CONST_FLOAT(-0.496000), state_MP_142, CHAR; 141) + | right_transition(CONST_FLOAT(-5.920000), state_ML_143, CHAR; 141) + | right_transition(CONST_FLOAT(-4.087000), state_MR_144, CHAR; 141) + | right_transition(CONST_FLOAT(-5.193000), state_D_145, CHAR; 141) + # h; + + // node [ MATP 18 ] + state_MP_142 = pair_transition(CONST_FLOAT(-11.233000), CHAR, state_IL_146, CHAR; 142) + | pair_transition(CONST_FLOAT(-11.440000), CHAR, state_IR_147, CHAR; 142) + | pair_transition(CONST_FLOAT(-0.005000), CHAR, state_ML_148, CHAR; 142) + | pair_transition(CONST_FLOAT(-8.416000), CHAR, state_D_149, CHAR; 142) + # h; + + state_ML_143 = left_transition(CONST_FLOAT(-3.758000), CHAR, state_IL_146; 143) + | left_transition(CONST_FLOAT(-3.940000), CHAR, state_IR_147; 143) + | left_transition(CONST_FLOAT(-0.507000), CHAR, state_ML_148; 143) + | left_transition(CONST_FLOAT(-2.670000), CHAR, state_D_149; 143) + # h; + + state_MR_144 = right_transition(CONST_FLOAT(-4.809000), state_IL_146, CHAR; 144) + | right_transition(CONST_FLOAT(-3.838000), state_IR_147, CHAR; 144) + | right_transition(CONST_FLOAT(-1.706000), state_ML_148, CHAR; 144) + | right_transition(CONST_FLOAT(-0.766000), state_D_149, CHAR; 144) + # h; + + state_D_145 = silent_transition(CONST_FLOAT(-4.568000), state_IL_146; 145) + | silent_transition(CONST_FLOAT(-4.250000), state_IR_147; 145) + | silent_transition(CONST_FLOAT(-2.265000), state_ML_148; 145) + | silent_transition(CONST_FLOAT(-0.520000), state_D_149; 145) + # h; + + state_IL_146 = left_transition(CONST_FLOAT(-1.686000), CHAR, state_IL_146; 146) + | left_transition(CONST_FLOAT(-2.369000), CHAR, state_IR_147; 146) + | left_transition(CONST_FLOAT(-1.117000), CHAR, state_ML_148; 146) + | left_transition(CONST_FLOAT(-4.855000), CHAR, state_D_149; 146) + # h; + + state_IR_147 = right_transition(CONST_FLOAT(-1.442000), state_IR_147, CHAR; 147) + | right_transition(CONST_FLOAT(-0.798000), state_ML_148, CHAR; 147) + | right_transition(CONST_FLOAT(-4.142000), state_D_149, CHAR; 147) + # h; + + // node [ MATL 19 ] + state_ML_148 = left_transition(CONST_FLOAT(-12.145000), CHAR, state_IL_150; 148) + | left_transition(CONST_FLOAT(-0.014000), CHAR, state_ML_151; 148) + | left_transition(CONST_FLOAT(-6.756000), CHAR, state_D_152; 148) + # h; + + state_D_149 = silent_transition(CONST_FLOAT(-6.563000), state_IL_150; 149) + | silent_transition(CONST_FLOAT(-2.076000), state_ML_151; 149) + | silent_transition(CONST_FLOAT(-0.411000), state_D_152; 149) + # h; + + state_IL_150 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_150; 150) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_151; 150) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_152; 150) + # h; + + // node [ MATL 20 ] + state_ML_151 = left_transition(CONST_FLOAT(-12.132000), CHAR, state_IL_153; 151) + | left_transition(CONST_FLOAT(-0.253000), CHAR, state_ML_154; 151) + | left_transition(CONST_FLOAT(-2.636000), CHAR, state_D_155; 151) + # h; + + state_D_152 = silent_transition(CONST_FLOAT(-7.642000), state_IL_153; 152) + | silent_transition(CONST_FLOAT(-3.155000), state_ML_154; 152) + | silent_transition(CONST_FLOAT(-0.180000), state_D_155; 152) + # h; + + state_IL_153 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_153; 153) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_154; 153) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_155; 153) + # h; + + // node [ MATL 21 ] + state_ML_154 = left_transition(CONST_FLOAT(-1.470000), CHAR, state_IL_156; 154) + | left_transition(CONST_FLOAT(-0.709000), CHAR, state_ML_157; 154) + | left_transition(CONST_FLOAT(-5.191000), CHAR, state_D_158; 154) + # h; + + state_D_155 = silent_transition(CONST_FLOAT(-11.053000), state_IL_156; 155) + | silent_transition(CONST_FLOAT(-0.280000), state_ML_157; 155) + | silent_transition(CONST_FLOAT(-2.506000), state_D_158; 155) + # h; + + state_IL_156 = left_transition(CONST_FLOAT(-1.948000), CHAR, state_IL_156; 156) + | left_transition(CONST_FLOAT(-0.495000), CHAR, state_ML_157; 156) + | left_transition(CONST_FLOAT(-5.006000), CHAR, state_D_158; 156) + # h; + + // node [ MATL 22 ] + state_ML_157 = left_transition(CONST_FLOAT(-12.057000), CHAR, state_IL_159; 157) + | left_transition(CONST_FLOAT(-0.001000), CHAR, state_ML_160; 157) + | left_transition(CONST_FLOAT(-10.711000), CHAR, state_D_161; 157) + # h; + + state_D_158 = silent_transition(CONST_FLOAT(-9.663000), state_IL_159; 158) + | silent_transition(CONST_FLOAT(-5.176000), state_ML_160; 158) + | silent_transition(CONST_FLOAT(-0.042000), state_D_161; 158) + # h; + + state_IL_159 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_159; 159) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_160; 159) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_161; 159) + # h; + + // node [ MATL 23 ] + state_ML_160 = left_transition(CONST_FLOAT(-12.057000), CHAR, state_IL_162; 160) + | left_transition(CONST_FLOAT(-0.235000), CHAR, state_ML_163; 160) + | left_transition(CONST_FLOAT(-2.736000), CHAR, state_D_164; 160) + # h; + + state_D_161 = silent_transition(CONST_FLOAT(-9.663000), state_IL_162; 161) + | silent_transition(CONST_FLOAT(-5.176000), state_ML_163; 161) + | silent_transition(CONST_FLOAT(-0.042000), state_D_164; 161) + # h; + + state_IL_162 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_162; 162) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_163; 162) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_164; 162) + # h; + + // node [ MATL 24 ] + state_ML_163 = left_transition(CONST_FLOAT(-0.840000), CHAR, state_IL_165; 163) + | left_transition(CONST_FLOAT(-1.186000), CHAR, state_ML_166; 163) + | left_transition(CONST_FLOAT(-9.087000), CHAR, state_D_167; 163) + # h; + + state_D_164 = silent_transition(CONST_FLOAT(-8.061000), state_IL_165; 164) + | silent_transition(CONST_FLOAT(-0.048000), state_ML_166; 164) + | silent_transition(CONST_FLOAT(-5.093000), state_D_167; 164) + # h; + + state_IL_165 = left_transition(CONST_FLOAT(-1.693000), CHAR, state_IL_165; 165) + | left_transition(CONST_FLOAT(-0.554000), CHAR, state_ML_166; 165) + | left_transition(CONST_FLOAT(-6.680000), CHAR, state_D_167; 165) + # h; + + // node [ MATL 25 ] + state_ML_166 = left_transition(CONST_FLOAT(0.000000), CHAR, state_E_169; 166) + # h; + + state_D_167 = silent_transition(CONST_FLOAT(0.000000), state_E_169; 167) + # h; + + state_IL_168 = left_transition(CONST_FLOAT(-1.823000), CHAR, state_IL_168; 168) + | left_transition(CONST_FLOAT(-0.479000), CHAR, state_E_169; 168) + # h; + + // node [ END 26 ] + state_E_169 = nil(CONST_FLOAT(0.000000), EMPTY; 169) + # h; + + // node [ BEGR 27 ] + state_S_170 = silent_transition(CONST_FLOAT(-12.148000), state_IL_171; 170) + | silent_transition(CONST_FLOAT(-0.015000), state_ML_172; 170) + | silent_transition(CONST_FLOAT(-6.633000), state_D_173; 170) + # h; + + state_IL_171 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_171; 171) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_172; 171) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_173; 171) + # h; + + // node [ MATL 28 ] + state_ML_172 = left_transition(CONST_FLOAT(-5.513000), CHAR, state_IL_174; 172) + | left_transition(CONST_FLOAT(-0.041000), CHAR, state_MP_175; 172) + | left_transition(CONST_FLOAT(-7.499000), CHAR, state_ML_176; 172) + | left_transition(CONST_FLOAT(-11.106000), CHAR, state_MR_177; 172) + | left_transition(CONST_FLOAT(-11.997000), CHAR, state_D_178; 172) + # h; + + state_D_173 = silent_transition(CONST_FLOAT(-5.885000), state_IL_174; 173) + | silent_transition(CONST_FLOAT(-0.367000), state_MP_175; 173) + | silent_transition(CONST_FLOAT(-5.147000), state_ML_176; 173) + | silent_transition(CONST_FLOAT(-3.521000), state_MR_177; 173) + | silent_transition(CONST_FLOAT(-3.434000), state_D_178; 173) + # h; + + state_IL_174 = left_transition(CONST_FLOAT(-3.373000), CHAR, state_IL_174; 174) + | left_transition(CONST_FLOAT(-0.233000), CHAR, state_MP_175; 174) + | left_transition(CONST_FLOAT(-5.052000), CHAR, state_ML_176; 174) + | left_transition(CONST_FLOAT(-6.884000), CHAR, state_MR_177; 174) + | left_transition(CONST_FLOAT(-6.158000), CHAR, state_D_178; 174) + # h; + + // node [ MATP 29 ] + state_MP_175 = pair_transition(CONST_FLOAT(-6.814000), CHAR, state_IL_179, CHAR; 175) + | pair_transition(CONST_FLOAT(-6.792000), CHAR, state_IR_180, CHAR; 175) + | pair_transition(CONST_FLOAT(-0.028000), CHAR, state_MP_181, CHAR; 175) + | pair_transition(CONST_FLOAT(-10.827000), CHAR, state_ML_182, CHAR; 175) + | pair_transition(CONST_FLOAT(-11.106000), CHAR, state_MR_183, CHAR; 175) + | pair_transition(CONST_FLOAT(-11.501000), CHAR, state_D_184, CHAR; 175) + # h; + + state_ML_176 = left_transition(CONST_FLOAT(-6.831000), CHAR, state_IL_179; 176) + | left_transition(CONST_FLOAT(-7.177000), CHAR, state_IR_180; 176) + | left_transition(CONST_FLOAT(-0.735000), CHAR, state_MP_181; 176) + | left_transition(CONST_FLOAT(-1.586000), CHAR, state_ML_182; 176) + | left_transition(CONST_FLOAT(-7.027000), CHAR, state_MR_183; 176) + | left_transition(CONST_FLOAT(-4.556000), CHAR, state_D_184; 176) + # h; + + state_MR_177 = right_transition(CONST_FLOAT(-6.988000), state_IL_179, CHAR; 177) + | right_transition(CONST_FLOAT(-5.717000), state_IR_180, CHAR; 177) + | right_transition(CONST_FLOAT(-1.625000), state_MP_181, CHAR; 177) + | right_transition(CONST_FLOAT(-5.695000), state_ML_182, CHAR; 177) + | right_transition(CONST_FLOAT(-0.829000), state_MR_183, CHAR; 177) + | right_transition(CONST_FLOAT(-3.908000), state_D_184, CHAR; 177) + # h; + + state_D_178 = silent_transition(CONST_FLOAT(-9.049000), state_IL_179; 178) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_180; 178) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_181; 178) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_182; 178) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_183; 178) + | silent_transition(CONST_FLOAT(-0.319000), state_D_184; 178) + # h; + + state_IL_179 = left_transition(CONST_FLOAT(-3.329000), CHAR, state_IL_179; 179) + | left_transition(CONST_FLOAT(-3.592000), CHAR, state_IR_180; 179) + | left_transition(CONST_FLOAT(-0.403000), CHAR, state_MP_181; 179) + | left_transition(CONST_FLOAT(-5.247000), CHAR, state_ML_182; 179) + | left_transition(CONST_FLOAT(-6.023000), CHAR, state_MR_183; 179) + | left_transition(CONST_FLOAT(-5.684000), CHAR, state_D_184; 179) + # h; + + state_IR_180 = right_transition(CONST_FLOAT(-2.914000), state_IR_180, CHAR; 180) + | right_transition(CONST_FLOAT(-0.331000), state_MP_181, CHAR; 180) + | right_transition(CONST_FLOAT(-6.425000), state_ML_182, CHAR; 180) + | right_transition(CONST_FLOAT(-4.593000), state_MR_183, CHAR; 180) + | right_transition(CONST_FLOAT(-5.699000), state_D_184, CHAR; 180) + # h; + + // node [ MATP 30 ] + state_MP_181 = pair_transition(CONST_FLOAT(-12.117000), CHAR, state_IL_185, CHAR; 181) + | pair_transition(CONST_FLOAT(-12.057000), CHAR, state_IR_186, CHAR; 181) + | pair_transition(CONST_FLOAT(-0.003000), CHAR, state_MP_187, CHAR; 181) + | pair_transition(CONST_FLOAT(-10.833000), CHAR, state_ML_188, CHAR; 181) + | pair_transition(CONST_FLOAT(-11.113000), CHAR, state_MR_189, CHAR; 181) + | pair_transition(CONST_FLOAT(-11.508000), CHAR, state_D_190, CHAR; 181) + # h; + + state_ML_182 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_185; 182) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_186; 182) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_187; 182) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_188; 182) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_189; 182) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_190; 182) + # h; + + state_MR_183 = right_transition(CONST_FLOAT(-6.988000), state_IL_185, CHAR; 183) + | right_transition(CONST_FLOAT(-5.717000), state_IR_186, CHAR; 183) + | right_transition(CONST_FLOAT(-1.625000), state_MP_187, CHAR; 183) + | right_transition(CONST_FLOAT(-5.695000), state_ML_188, CHAR; 183) + | right_transition(CONST_FLOAT(-0.829000), state_MR_189, CHAR; 183) + | right_transition(CONST_FLOAT(-3.908000), state_D_190, CHAR; 183) + # h; + + state_D_184 = silent_transition(CONST_FLOAT(-9.049000), state_IL_185; 184) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_186; 184) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_187; 184) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_188; 184) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_189; 184) + | silent_transition(CONST_FLOAT(-0.319000), state_D_190; 184) + # h; + + state_IL_185 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_185; 185) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_186; 185) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_187; 185) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_188; 185) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_189; 185) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_190; 185) + # h; + + state_IR_186 = right_transition(CONST_FLOAT(-2.408000), state_IR_186, CHAR; 186) + | right_transition(CONST_FLOAT(-0.496000), state_MP_187, CHAR; 186) + | right_transition(CONST_FLOAT(-5.920000), state_ML_188, CHAR; 186) + | right_transition(CONST_FLOAT(-4.087000), state_MR_189, CHAR; 186) + | right_transition(CONST_FLOAT(-5.193000), state_D_190, CHAR; 186) + # h; + + // node [ MATP 31 ] + state_MP_187 = pair_transition(CONST_FLOAT(-12.117000), CHAR, state_IL_191, CHAR; 187) + | pair_transition(CONST_FLOAT(-12.057000), CHAR, state_IR_192, CHAR; 187) + | pair_transition(CONST_FLOAT(-0.004000), CHAR, state_MP_193, CHAR; 187) + | pair_transition(CONST_FLOAT(-10.833000), CHAR, state_ML_194, CHAR; 187) + | pair_transition(CONST_FLOAT(-9.718000), CHAR, state_MR_195, CHAR; 187) + | pair_transition(CONST_FLOAT(-11.508000), CHAR, state_D_196, CHAR; 187) + # h; + + state_ML_188 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_191; 188) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_192; 188) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_193; 188) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_194; 188) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_195; 188) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_196; 188) + # h; + + state_MR_189 = right_transition(CONST_FLOAT(-6.988000), state_IL_191, CHAR; 189) + | right_transition(CONST_FLOAT(-5.717000), state_IR_192, CHAR; 189) + | right_transition(CONST_FLOAT(-1.625000), state_MP_193, CHAR; 189) + | right_transition(CONST_FLOAT(-5.695000), state_ML_194, CHAR; 189) + | right_transition(CONST_FLOAT(-0.829000), state_MR_195, CHAR; 189) + | right_transition(CONST_FLOAT(-3.908000), state_D_196, CHAR; 189) + # h; + + state_D_190 = silent_transition(CONST_FLOAT(-9.049000), state_IL_191; 190) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_192; 190) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_193; 190) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_194; 190) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_195; 190) + | silent_transition(CONST_FLOAT(-0.319000), state_D_196; 190) + # h; + + state_IL_191 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_191; 191) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_192; 191) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_193; 191) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_194; 191) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_195; 191) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_196; 191) + # h; + + state_IR_192 = right_transition(CONST_FLOAT(-2.408000), state_IR_192, CHAR; 192) + | right_transition(CONST_FLOAT(-0.496000), state_MP_193, CHAR; 192) + | right_transition(CONST_FLOAT(-5.920000), state_ML_194, CHAR; 192) + | right_transition(CONST_FLOAT(-4.087000), state_MR_195, CHAR; 192) + | right_transition(CONST_FLOAT(-5.193000), state_D_196, CHAR; 192) + # h; + + // node [ MATP 32 ] + state_MP_193 = pair_transition(CONST_FLOAT(-12.116000), CHAR, state_IL_197, CHAR; 193) + | pair_transition(CONST_FLOAT(-12.056000), CHAR, state_IR_198, CHAR; 193) + | pair_transition(CONST_FLOAT(-0.003000), CHAR, state_MP_199, CHAR; 193) + | pair_transition(CONST_FLOAT(-10.832000), CHAR, state_ML_200, CHAR; 193) + | pair_transition(CONST_FLOAT(-9.881000), CHAR, state_MR_201, CHAR; 193) + | pair_transition(CONST_FLOAT(-11.507000), CHAR, state_D_202, CHAR; 193) + # h; + + state_ML_194 = left_transition(CONST_FLOAT(-6.250000), CHAR, state_IL_197; 194) + | left_transition(CONST_FLOAT(-6.596000), CHAR, state_IR_198; 194) + | left_transition(CONST_FLOAT(-1.310000), CHAR, state_MP_199; 194) + | left_transition(CONST_FLOAT(-1.005000), CHAR, state_ML_200; 194) + | left_transition(CONST_FLOAT(-6.446000), CHAR, state_MR_201; 194) + | left_transition(CONST_FLOAT(-3.975000), CHAR, state_D_202; 194) + # h; + + state_MR_195 = right_transition(CONST_FLOAT(-7.083000), state_IL_197, CHAR; 195) + | right_transition(CONST_FLOAT(-5.812000), state_IR_198, CHAR; 195) + | right_transition(CONST_FLOAT(-1.444000), state_MP_199, CHAR; 195) + | right_transition(CONST_FLOAT(-5.790000), state_ML_200, CHAR; 195) + | right_transition(CONST_FLOAT(-0.924000), state_MR_201, CHAR; 195) + | right_transition(CONST_FLOAT(-4.004000), state_D_202, CHAR; 195) + # h; + + state_D_196 = silent_transition(CONST_FLOAT(-9.049000), state_IL_197; 196) + | silent_transition(CONST_FLOAT(-7.747000), state_IR_198; 196) + | silent_transition(CONST_FLOAT(-3.544000), state_MP_199; 196) + | silent_transition(CONST_FLOAT(-4.226000), state_ML_200; 196) + | silent_transition(CONST_FLOAT(-4.244000), state_MR_201; 196) + | silent_transition(CONST_FLOAT(-0.319000), state_D_202; 196) + # h; + + state_IL_197 = left_transition(CONST_FLOAT(-2.579000), CHAR, state_IL_197; 197) + | left_transition(CONST_FLOAT(-2.842000), CHAR, state_IR_198; 197) + | left_transition(CONST_FLOAT(-0.760000), CHAR, state_MP_199; 197) + | left_transition(CONST_FLOAT(-4.497000), CHAR, state_ML_200; 197) + | left_transition(CONST_FLOAT(-5.274000), CHAR, state_MR_201; 197) + | left_transition(CONST_FLOAT(-4.934000), CHAR, state_D_202; 197) + # h; + + state_IR_198 = right_transition(CONST_FLOAT(-2.408000), state_IR_198, CHAR; 198) + | right_transition(CONST_FLOAT(-0.496000), state_MP_199, CHAR; 198) + | right_transition(CONST_FLOAT(-5.920000), state_ML_200, CHAR; 198) + | right_transition(CONST_FLOAT(-4.087000), state_MR_201, CHAR; 198) + | right_transition(CONST_FLOAT(-5.193000), state_D_202, CHAR; 198) + # h; + + // node [ MATP 33 ] + state_MP_199 = pair_transition(CONST_FLOAT(-11.232000), CHAR, state_IL_203, CHAR; 199) + | pair_transition(CONST_FLOAT(-11.439000), CHAR, state_IR_204, CHAR; 199) + | pair_transition(CONST_FLOAT(-0.003000), CHAR, state_ML_205, CHAR; 199) + | pair_transition(CONST_FLOAT(-9.853000), CHAR, state_D_206, CHAR; 199) + # h; + + state_ML_200 = left_transition(CONST_FLOAT(-3.758000), CHAR, state_IL_203; 200) + | left_transition(CONST_FLOAT(-3.940000), CHAR, state_IR_204; 200) + | left_transition(CONST_FLOAT(-0.507000), CHAR, state_ML_205; 200) + | left_transition(CONST_FLOAT(-2.670000), CHAR, state_D_206; 200) + # h; + + state_MR_201 = right_transition(CONST_FLOAT(-4.901000), state_IL_203, CHAR; 201) + | right_transition(CONST_FLOAT(-3.930000), state_IR_204, CHAR; 201) + | right_transition(CONST_FLOAT(-1.519000), state_ML_205, CHAR; 201) + | right_transition(CONST_FLOAT(-0.858000), state_D_206, CHAR; 201) + # h; + + state_D_202 = silent_transition(CONST_FLOAT(-4.568000), state_IL_203; 202) + | silent_transition(CONST_FLOAT(-4.250000), state_IR_204; 202) + | silent_transition(CONST_FLOAT(-2.265000), state_ML_205; 202) + | silent_transition(CONST_FLOAT(-0.520000), state_D_206; 202) + # h; + + state_IL_203 = left_transition(CONST_FLOAT(-1.686000), CHAR, state_IL_203; 203) + | left_transition(CONST_FLOAT(-2.369000), CHAR, state_IR_204; 203) + | left_transition(CONST_FLOAT(-1.117000), CHAR, state_ML_205; 203) + | left_transition(CONST_FLOAT(-4.855000), CHAR, state_D_206; 203) + # h; + + state_IR_204 = right_transition(CONST_FLOAT(-1.442000), state_IR_204, CHAR; 204) + | right_transition(CONST_FLOAT(-0.798000), state_ML_205, CHAR; 204) + | right_transition(CONST_FLOAT(-4.142000), state_D_206, CHAR; 204) + # h; + + // node [ MATL 34 ] + state_ML_205 = left_transition(CONST_FLOAT(-6.418000), CHAR, state_IL_207; 205) + | left_transition(CONST_FLOAT(-0.018000), CHAR, state_ML_208; 205) + | left_transition(CONST_FLOAT(-10.802000), CHAR, state_D_209; 205) + # h; + + state_D_206 = silent_transition(CONST_FLOAT(-6.174000), state_IL_207; 206) + | silent_transition(CONST_FLOAT(-1.687000), state_ML_208; 206) + | silent_transition(CONST_FLOAT(-0.566000), state_D_209; 206) + # h; + + state_IL_207 = left_transition(CONST_FLOAT(-2.011000), CHAR, state_IL_207; 207) + | left_transition(CONST_FLOAT(-0.486000), CHAR, state_ML_208; 207) + | left_transition(CONST_FLOAT(-4.712000), CHAR, state_D_209; 207) + # h; + + // node [ MATL 35 ] + state_ML_208 = left_transition(CONST_FLOAT(-12.148000), CHAR, state_IL_210; 208) + | left_transition(CONST_FLOAT(-0.001000), CHAR, state_ML_211; 208) + | left_transition(CONST_FLOAT(-10.802000), CHAR, state_D_212; 208) + # h; + + state_D_209 = silent_transition(CONST_FLOAT(-6.174000), state_IL_210; 209) + | silent_transition(CONST_FLOAT(-1.687000), state_ML_211; 209) + | silent_transition(CONST_FLOAT(-0.566000), state_D_212; 209) + # h; + + state_IL_210 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_210; 210) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_211; 210) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_212; 210) + # h; + + // node [ MATL 36 ] + state_ML_211 = left_transition(CONST_FLOAT(-12.148000), CHAR, state_IL_213; 211) + | left_transition(CONST_FLOAT(-0.001000), CHAR, state_ML_214; 211) + | left_transition(CONST_FLOAT(-10.802000), CHAR, state_D_215; 211) + # h; + + state_D_212 = silent_transition(CONST_FLOAT(-6.174000), state_IL_213; 212) + | silent_transition(CONST_FLOAT(-1.687000), state_ML_214; 212) + | silent_transition(CONST_FLOAT(-0.566000), state_D_215; 212) + # h; + + state_IL_213 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_213; 213) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_214; 213) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_215; 213) + # h; + + // node [ MATL 37 ] + state_ML_214 = left_transition(CONST_FLOAT(-12.148000), CHAR, state_IL_216; 214) + | left_transition(CONST_FLOAT(-0.001000), CHAR, state_ML_217; 214) + | left_transition(CONST_FLOAT(-10.802000), CHAR, state_D_218; 214) + # h; + + state_D_215 = silent_transition(CONST_FLOAT(-6.174000), state_IL_216; 215) + | silent_transition(CONST_FLOAT(-1.687000), state_ML_217; 215) + | silent_transition(CONST_FLOAT(-0.566000), state_D_218; 215) + # h; + + state_IL_216 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_216; 216) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_217; 216) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_218; 216) + # h; + + // node [ MATL 38 ] + state_ML_217 = left_transition(CONST_FLOAT(-12.148000), CHAR, state_IL_219; 217) + | left_transition(CONST_FLOAT(-0.001000), CHAR, state_ML_220; 217) + | left_transition(CONST_FLOAT(-10.802000), CHAR, state_D_221; 217) + # h; + + state_D_218 = silent_transition(CONST_FLOAT(-6.174000), state_IL_219; 218) + | silent_transition(CONST_FLOAT(-1.687000), state_ML_220; 218) + | silent_transition(CONST_FLOAT(-0.566000), state_D_221; 218) + # h; + + state_IL_219 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_219; 219) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_220; 219) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_221; 219) + # h; + + // node [ MATL 39 ] + state_ML_220 = left_transition(CONST_FLOAT(-12.148000), CHAR, state_IL_222; 220) + | left_transition(CONST_FLOAT(-0.001000), CHAR, state_ML_223; 220) + | left_transition(CONST_FLOAT(-10.802000), CHAR, state_D_224; 220) + # h; + + state_D_221 = silent_transition(CONST_FLOAT(-6.174000), state_IL_222; 221) + | silent_transition(CONST_FLOAT(-1.687000), state_ML_223; 221) + | silent_transition(CONST_FLOAT(-0.566000), state_D_224; 221) + # h; + + state_IL_222 = left_transition(CONST_FLOAT(-1.442000), CHAR, state_IL_222; 222) + | left_transition(CONST_FLOAT(-0.798000), CHAR, state_ML_223; 222) + | left_transition(CONST_FLOAT(-4.142000), CHAR, state_D_224; 222) + # h; + + // node [ MATL 40 ] + state_ML_223 = left_transition(CONST_FLOAT(0.000000), CHAR, state_E_226; 223) + # h; + + state_D_224 = silent_transition(CONST_FLOAT(0.000000), state_E_226; 224) + # h; + + state_IL_225 = left_transition(CONST_FLOAT(-1.823000), CHAR, state_IL_225; 225) + | left_transition(CONST_FLOAT(-0.479000), CHAR, state_E_226; 225) + # h; + + // node [ END 41 ] + state_E_226 = nil(CONST_FLOAT(0.000000), EMPTY; 226) + # h; + +} + +instance count = gra_cm(alg_count); +instance enumcyk = gra_cm(alg_enum * alg_cyk); +instance cykenum = gra_cm(alg_cyk * alg_enum); +instance inside = gra_cm(alg_inside); + \ No newline at end of file diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 331f59ada..4adc5f329 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -517,4 +517,9 @@ check_new_old_eq RFmini.gap unused count "G" outside_cm # The important aspect is that we have to also normalize by axiom(0,0) since the inside result of # the empty word "" != 0 <-- log2-space but will be integrated in each and every outside candidate check_new_old_eq RFmini.gap unused inside "G" outside_cm + +# more extensive example as before, also checked against cm_OutsideAlign of Infernal +GAPC="../../../gapc --outside_grammar state_MR_3 --outside_grammar state_MP_6 --outside_grammar state_S_0 --outside_grammar state_E_120" +check_new_old_eq RF00005.gap unused inside "GACGGUAU" outside_cm5 + CPPFLAGS_EXTRA="" From 11792d961f421099fcba802655e3916cc6f05427 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 17 Feb 2023 22:05:17 +0100 Subject: [PATCH 196/209] revert layout --- src/fn_def.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/fn_def.cc b/src/fn_def.cc index 5ef70e5d8..780f248ec 100644 --- a/src/fn_def.cc +++ b/src/fn_def.cc @@ -320,8 +320,10 @@ void Fn_Def::add_para(Symbol::NT &nt) { for (std::vector
::const_iterator i = tables.begin(); i != tables.end(); ++i, ++j, ++k) { // only add the - if (!(*i).delete_left_index()) add_para(t, (*j)->vacc()->name()); - if (!(*i).delete_right_index()) add_para(t, (*k)->vacc()->name()); + if (!(*i).delete_left_index()) + add_para(t, (*j)->vacc()->name()); + if (!(*i).delete_right_index()) + add_para(t, (*k)->vacc()->name()); } set_paras(nt.ntargs()); From 2dff6c64881f3b8139b35c90483adef5509a518e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sat, 18 Feb 2023 21:55:40 +0100 Subject: [PATCH 197/209] adding acm test --- testdata/regresstest/config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 4adc5f329..9fa6675d4 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -522,4 +522,7 @@ check_new_old_eq RFmini.gap unused inside "G" outside_cm GAPC="../../../gapc --outside_grammar state_MR_3 --outside_grammar state_MP_6 --outside_grammar state_S_0 --outside_grammar state_E_120" check_new_old_eq RF00005.gap unused inside "GACGGUAU" outside_cm5 +GAPC="../../../gapc --outside_grammar a_1 --outside_grammar a_47 --outside_grammar a_82" +check_new_old_eq acmsearch_RF00005.gap unused inside "GACGGUAU" outside_acm + CPPFLAGS_EXTRA="" From 3c192cb7d3afbfbdb4cd7a0032eaf2c3b7a8f71c Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sat, 18 Feb 2023 23:36:38 +0100 Subject: [PATCH 198/209] surpress comments if no computation occurs --- src/cpp.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 2db4ebea0..7777a1763 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2073,9 +2073,8 @@ void Printer::Cpp::multi_print_cyk_loops_linear( std::string *js, std::string *ns, bool for_outsideNTs ) { - stream << indent() << "// C: linear loops" << endl; - if (inner->empty() && !left->empty()) { + stream << indent() << "// C: linear loops" << endl; stream << indent() << "for (" << *t << " " << *js << " = 0; " << *js << " < " << *ns << "; " << "++" << *js << ") {" << endl; @@ -2091,6 +2090,7 @@ void Printer::Cpp::multi_print_cyk_loops_linear( stream << indent() << "}" << endl << endl; } if (!right->empty()) { + stream << indent() << "// C: linear loops" << endl; stream << indent(); if (!for_outsideNTs) { stream << *t << " "; @@ -2114,8 +2114,8 @@ void Printer::Cpp::multi_print_cyk_loops_constant( std::list *all, std::string *is, bool for_outsideNTs ) { - stream << indent() << "// D: constant loops" << endl; if (!all->empty()) { + stream << indent() << "// D: constant loops" << endl; stream << indent(); if (!for_outsideNTs) { stream << *t << " "; From 5e970dd25af8c8e80a74cf886e7ad0214f70d932 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Sun, 19 Feb 2023 00:09:47 +0100 Subject: [PATCH 199/209] deactivate acm test for now, as I have not yet checked results --- testdata/regresstest/config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 9fa6675d4..c58f66469 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -523,6 +523,7 @@ GAPC="../../../gapc --outside_grammar state_MR_3 --outside_grammar state_MP_6 -- check_new_old_eq RF00005.gap unused inside "GACGGUAU" outside_cm5 GAPC="../../../gapc --outside_grammar a_1 --outside_grammar a_47 --outside_grammar a_82" -check_new_old_eq acmsearch_RF00005.gap unused inside "GACGGUAU" outside_acm +# not yet explored +#check_new_old_eq acmsearch_RF00005.gap unused inside "GACGGUAU" outside_acm CPPFLAGS_EXTRA="" From 2308ddeabc51c558b1e162452999178dc6973406 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 15 Mar 2023 17:55:41 +0100 Subject: [PATCH 200/209] somehow last part of file was missing --- src/symbol.cc | 135 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/src/symbol.cc b/src/symbol.cc index a99362a98..2a947f05c 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -1650,3 +1650,138 @@ bool Symbol::Terminal::isPredefinedTerminalParser() { return this->predefinedTerminalParser; } +// a user provided grammar might contain production rules that use blocks +// e.g. NT = sadd(BASE, {hairpin | iloop}) +// this basically is a short hand notation for multiple alternatives, e.g. +// NT = sadd(BASE, hairpin) | sadd(BASE, iloop) +// For outside grammar generation, we first want to make short-hand notations +// explicit by applying this function to the selected grammar. +// This only applies for NTs, not for Base or Terminal +void Symbol::Base::resolve_blocks() { +} +void Symbol::Terminal::resolve_blocks() { +} +void Symbol::NT::resolve_blocks() { + for (std::list::iterator alt = this->alts.begin(); + alt != this->alts.end(); ++alt) { + // check if alternative contains any Alt::Block + Alt::Block *block = dynamic_cast((*alt)->find_block()); + while (block) { + // locate parent of found Alt::Block + Alt::Base *parent = (*alt)->find_block_parent(*block); + if (block->alts.size() == 1) { + // if a Alt::Block holds only one alternative, + // we can simply remove Alt::Block altogether + if (parent == NULL) { + // parent is the non-terminal + Alt::Base *child = block->alts.front(); + child->top_level = Bool(true); + child->filters.insert(child->filters.end(), block->filters.begin(), + block->filters.end()); + child->multi_filter.insert(child->multi_filter.end(), + block->multi_filter.begin(), + block->multi_filter.end()); + // keep original ordering, i.e. delete block and push + // single = last alt to the end + this->alts.push_back(child); + this->alts.erase(alt); + // reset iterator to first element + alt = this->alts.begin(); + } else { + Alt::Simple *p_simple = dynamic_cast(parent); + if (p_simple) { + // if parent is of type Alt::Simple + for (std::list::iterator j = + p_simple->args.begin(); + j != p_simple->args.end(); ++j) { + Fn_Arg::Alt *p_simple_fnarg = dynamic_cast(*j); + if (p_simple_fnarg && (p_simple_fnarg->alt == block)) { + Alt::Base *child = block->alts.front(); + child->filters.insert(child->filters.end(), + block->filters.begin(), + block->filters.end()); + child->multi_filter.insert(child->multi_filter.end(), + block->multi_filter.begin(), + block->multi_filter.end()); + p_simple_fnarg->alt = child; + break; + } + } + } + + // a Link is a leaf and thus cannot contain a block + // Alt::Link *p_link = dynamic_cast(parent); + + // parent block should have been removed first! + // Alt::Block *p_block = dynamic_cast(parent); + + // Alternative is not allowed in Multi-Track link. + // Alt::Multi *p_multi = dynamic_cast(parent); + } + } else { + // the Alt::Block holds multiple alternatives. + if (parent == NULL) { + Alt::Base *child = block->alts.front()->clone(); + child->top_level = Bool(true); + child->filters.insert(child->filters.end(), + block->filters.begin(), + block->filters.end()); + child->multi_filter.insert(child->multi_filter.end(), + block->multi_filter.begin(), + block->multi_filter.end()); + this->alts.push_back(child); + block->alts.pop_front(); + } else { + Alt::Simple *p_simple = dynamic_cast(parent); + if (p_simple) { + // if parent is of type Alt::Simple + // 1) clone the full existing alternative + Alt::Base *newalt = (*alt)->clone(); + + // 2) remove first block alternative from original object + Alt::Base *first_block_alt = block->alts.front(); + first_block_alt->filters.insert(first_block_alt->filters.end(), + block->filters.begin(), + block->filters.end()); + first_block_alt->multi_filter.insert( + first_block_alt->multi_filter.end(), + block->multi_filter.begin(), + block->multi_filter.end()); + block->alts.pop_front(); + + // 3) remove Alt::Block and connect first block + // alternative to cloned object + Alt::Base *cloned_block = newalt->find_block(); + // a clone of the alt rule containing block should have this + // block as well! + assert(cloned_block); + Alt::Simple *cloned_p_simple = dynamic_cast( + newalt->find_block_parent(*cloned_block)); + // must exist and be Alt::Simple for above reasons + assert(cloned_p_simple); + for (std::list::iterator j = + cloned_p_simple->args.begin(); + j != cloned_p_simple->args.end(); ++j) { + Fn_Arg::Alt *cloned_p_simple_fnarg = + dynamic_cast(*j); + if (cloned_p_simple_fnarg && + (cloned_p_simple_fnarg->alt == cloned_block)) { + cloned_p_simple_fnarg->alt = first_block_alt; + break; + } + } + + this->alts.insert(alt, newalt); + } + + // as with the single case above, Link, Block, + // Multi cannot be parents + } + } + + // for next iteration: check if production rules might still + // contain other blocks + block = dynamic_cast((*alt)->find_block()); + } + } +} From 8de6a67afe374dc5f7cb71344fbc6f9311b1c637 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 11 Apr 2023 16:30:17 +0200 Subject: [PATCH 201/209] merged wrong functions --- src/cpp.cc | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 17c7c4610..67482ffd8 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2043,6 +2043,7 @@ void Printer::Cpp::print_openmp_cyk_loops_diag(const AST &ast, } else { stream << "for (int z = 0; z < max_tiles_n; z+=tile_size) {"; } + stream << endl; inc_indent(); if (checkpoint) { stream << indent() << "mutex.lock_shared();" << endl; @@ -2709,17 +2710,7 @@ void Printer::Cpp::print_run_fn(const AST &ast) { stream << " run() {" << endl; inc_indent(); - if (ast.checkpoint && !ast.checkpoint->is_buddy) { - stream << indent() << "std::atomic_bool cancel_token;" << endl; - stream << indent() << "archive_periodically(cancel_token, "; - stream << "checkpoint_interval" << ");" << endl; - stream << indent() << *ast.grammar()->axiom->code()->return_type; - stream << " ans = "; - } else { - stream << indent() << "return "; - } - - stream << "nt_" << *ast.grammar()->axiom_name << '('; + stream << indent() << "return nt_" << *ast.grammar()->axiom_name << '('; bool first = true; size_t track = 0; @@ -2745,17 +2736,6 @@ void Printer::Cpp::print_run_fn(const AST &ast) { stream << ");" << endl; - if (ast.checkpoint && !ast.checkpoint->is_buddy) { - stream << indent() << "cancel_token.store(false); " - "// stop periodic checkpointing" << endl; - stream << indent() << "if (!keep_archives) {" << endl; - inc_indent(); - stream << indent() << "remove_tables();" << endl; - stream << indent() << "remove_log_file();" << endl; - dec_indent(); - stream << indent() << "}" << endl; - stream << indent() << "return ans;" << endl; - } dec_indent(); stream << indent() << '}' << endl << endl; } From 0937da7d3a3b12a72a45764e5091451ce6fbf209 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 12 Apr 2023 10:41:09 +0200 Subject: [PATCH 202/209] extend checkpointing for outside contexts, here CYK first computes (old) inside non-terminals and then (new) outside versions. --- src/checkpoint.hh | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/checkpoint.hh b/src/checkpoint.hh index c0ba33c86..6cb09eef0 100644 --- a/src/checkpoint.hh +++ b/src/checkpoint.hh @@ -983,7 +983,8 @@ SUPPORTED_EXTERNAL_TYPES = {"Rope", "answer_pknot_mfe", "pktype", dec_indent(); dec_indent(); } - void archive_cyk_indices(Printer::Base &stream, size_t n_tracks) { + void archive_cyk_indices(Printer::Base &stream, size_t n_tracks, + bool for_outside) { inc_indent(); stream << indent() << "void archive_cyk_indices() {" << endl; inc_indent(); @@ -1004,6 +1005,17 @@ SUPPORTED_EXTERNAL_TYPES = {"Rope", "answer_pknot_mfe", "pktype", stream << indent() << "array_out << " << "(t_" << i << "_j == 0 ? 0 : t_" << i << "_j - 1);" << endl; } + if (for_outside) { + for (size_t i = 0; i < n_tracks; i++) { + // archive the indices of the prior iteration to ensure + // that the whole iteration has been completed + stream << indent() << "array_out << outside_t_" << i << "_i + 1;" + << endl; + stream << indent() << "array_out << " + << "(outside_t_" << i << "_j == 0 ? 0 : outside_t_" << i + << "_j - 1);" << endl; + } + } stream << indent() << "#ifdef _OPENMP" << endl; stream << indent() << " array_out << outer_loop_1_idx;" << endl; stream << indent() << " array_out << outer_loop_2_idx;" << endl; @@ -1042,7 +1054,8 @@ SUPPORTED_EXTERNAL_TYPES = {"Rope", "answer_pknot_mfe", "pktype", dec_indent(); } - void load_cyk_indices(Printer::Base &stream, size_t n_tracks) { + void load_cyk_indices(Printer::Base &stream, size_t n_tracks, + bool for_outside) { inc_indent(); stream << indent() << "void load_cyk_indices() {" << endl; inc_indent(); @@ -1060,6 +1073,12 @@ SUPPORTED_EXTERNAL_TYPES = {"Rope", "answer_pknot_mfe", "pktype", stream << indent() << "array_in >> t_" << i << "_i;" << endl; stream << indent() << "array_in >> t_" << i << "_j;" << endl; } + if (for_outside) { + for (size_t i = 0; i < n_tracks; i++) { + stream << indent() << "array_in >> outside_t_" << i << "_i;" << endl; + stream << indent() << "array_in >> outside_t_" << i << "_j;" << endl; + } + } stream << indent() << "#ifdef _OPENMP" << endl; stream << indent() << " array_in >> outer_loop_1_idx;" << endl; stream << indent() << " array_in >> outer_loop_2_idx;" << endl; From 1e0d0b856b4869fb9627cc114c9bf765cdd84e62 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 12 Apr 2023 10:42:04 +0200 Subject: [PATCH 203/209] add checkpointing argument --- src/cpp.hh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/cpp.hh b/src/cpp.hh index 9cce74b50..027769f2a 100644 --- a/src/cpp.hh +++ b/src/cpp.hh @@ -92,7 +92,8 @@ class Cpp : public Base { const Yield::Multi &mys, bool is_outside = false); void multi_print_inner_cyk(const std::list &l, const std::list &tord, size_t track, size_t tracks, - size_t track_pos, Type::Base *t, bool checkpoint, + size_t track_pos, Type::Base *t, + bool checkpoint, bool no_mutex_def, bool for_outsideNTs = false); void multi_partition_nts(const std::list &tord, std::list &all, std::list &inner, @@ -108,7 +109,9 @@ class Cpp : public Base { std::list *left, std::string *is, std::string *js, - std::string *ns, bool for_outsideNTs = false); + std::string *ns, + bool checkpoint, + bool for_outsideNTs = false); void multi_print_cyk_loops_quadratic_inner( const std::list &tord, size_t track, @@ -117,6 +120,7 @@ class Cpp : public Base { Type::Base *t, std::list *left, std::string *is, + bool checkpoint, bool for_outsideNTs = false); void multi_print_cyk_loops_linear( const std::list &tord, @@ -129,7 +133,9 @@ class Cpp : public Base { std::list *right, std::string *is, std::string *js, - std::string *ns, bool for_outsideNTs = false); + std::string *ns, + bool checkpoint, + bool for_outsideNTs = false); void multi_print_cyk_loops_constant( const std::list &tord, size_t track, @@ -137,7 +143,9 @@ class Cpp : public Base { size_t track_pos, Type::Base *t, std::list *all, - std::string *is, bool for_outsideNTs = false); + std::string *is, + bool checkpoint, + bool for_outsideNTs = false); void multi_print_cyk(const std::list &tord, size_t track, size_t tracks, size_t track_pos, Type::Base *t); From 00fc7b407f5bb77974e470e76545a5c9596d0315 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 12 Apr 2023 10:42:30 +0200 Subject: [PATCH 204/209] merge CYK checkpointing & outside --- src/cpp.cc | 292 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 218 insertions(+), 74 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 67482ffd8..3f583167e 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -1552,6 +1552,12 @@ void Printer::Cpp::print_seq_init(const AST &ast) { stream << indent() << "t_" << i << "_i = 0;" << endl; stream << indent() << "t_" << i << "_j = 0;" << endl; } + if (ast.grammar()->is_outside()) { + for (size_t i = 0; i < ast.grammar()->axiom->tracks(); i++) { + stream << indent() << "outside_t_" << i << "_i = 0;" << endl; + stream << indent() << "outside_t_" << i << "_j = 0;" << endl; + } + } stream << indent() << "#ifdef _OPENMP" << endl; inc_indent(); stream << indent() << "outer_loop_1_idx = 0;" << endl; @@ -1936,6 +1942,12 @@ void Printer::Cpp::header(const AST &ast) { stream << indent() << *type << " t_" << t << "_i;" << endl; stream << indent() << *type << " t_" << t << "_j;" << endl; } + if (ast.grammar()->is_outside()) { + for (size_t t = 0; t < ast.grammar()->axiom->tracks(); t++) { + stream << indent() << *type << " outside_t_" << t << "_i;" << endl; + stream << indent() << *type << " outside_t_" << t << "_j;" << endl; + } + } stream << "#ifdef _OPENMP" << endl; stream << indent() << "int outer_loop_1_idx, outer_loop_2_idx, " << "inner_loop_2_idx;" << endl; @@ -2269,13 +2281,13 @@ void Printer::Cpp::multi_print_inner_cyk( const std::list &l, const std::list &tord, size_t track, size_t tracks, size_t track_pos, - Type::Base *t, bool checkpoint, bool for_outsideNTs) { + Type::Base *t, bool checkpoint, bool no_mutex_def, bool for_outsideNTs) { assert(track < tracks); if (track+1 != tracks) { multi_print_cyk(tord, track+1, tracks, track_pos, t); return; } - if (checkpoint) { + if (checkpoint && !no_mutex_def) { stream << indent() << "std::lock_guard lock(mutex);" << endl; } for (std::list::const_iterator i = l.begin(); @@ -2322,29 +2334,22 @@ void Printer::Cpp::multi_partition_nts( } } - -void Printer::Cpp::multi_print_cyk( - const std::list &tord, size_t track, size_t tracks, - size_t track_pos, Type::Base *t) { - std::list all, inner, left, right; - multi_partition_nts(tord, all, inner, left, right, track, tracks, track_pos); - - size_t real_track = !track_pos ? track : track_pos; - - std::ostringstream sis, sjs, sns; - sis << "t_" << track << "_i"; - sjs << "t_" << track << "_j"; - sns << "t_" << real_track << "_n"; - std::string is(sis.str()), js(sjs.str()), ns(sns.str()); - - stream << indent() << *t << " t_" << track << "_n = t_" << real_track - << "_seq.size();" << endl << endl; - - bool checkpoint = ast->checkpoint && ast->checkpoint->cyk; - - // SMJ 2023-02-18: I've added the x: xxx loops comments, but - // am not 100% sure if they are correct in multitrack contexts - if (!inner.empty()) { +void Printer::Cpp::multi_print_cyk_loops_quadratic( + const std::list &tord, + size_t track, + size_t tracks, + size_t track_pos, + Type::Base *t, + std::list *inner, + std::list *left, + std::string *is, + std::string *js, + std::string *ns, bool checkpoint, bool for_outsideNTs) { + std::string oprefix = ""; + if (for_outsideNTs) { + oprefix = "outside_"; + } + if (!inner->empty()) { if (checkpoint) { /* in checkpointing mode, the loop indices @@ -2362,18 +2367,20 @@ void Printer::Cpp::multi_print_cyk( so the loop variable will be set to whatever value is usually would be set to */ - stream << indent() << "for (" << js << " = (" << js << "_loaded++) " - << "? 0 : " << js << "; " - << js << " < " << ns << "; " << "++" << js << ") {" + stream << indent() << "for (" << *js << " = (" << oprefix << *js << "_loaded++) " + << "? 0 : " << oprefix << *js << "; " + << *js << " < " << *ns << "; " << "++" << *js << ") {" << endl; - inc_indent(); } else { - stream << indent() << "for (" << *t << " " << js << " = 0; " - << js << " < " << ns << "; " << "++" << js << ") {" - << endl; - inc_indent(); + stream << indent() << "for (" << *t << " " << *js << " = 0; " << *js + << " < " << *ns << "; " << "++" << *js << ") {" << endl; + } + inc_indent(); + if (for_outsideNTs) { + multi_print_cyk_loops_quadratic_inner( + tord, track, tracks, track_pos, t, left, is, + checkpoint, for_outsideNTs); } - stream << indent() << "// A: quadratic loops" << endl; if (checkpoint) { /* @@ -2384,74 +2391,198 @@ void Printer::Cpp::multi_print_cyk( for t_0_i in the first inner loop pass; afterwards it will regularly be set to t_0_j + 1; */ - stream << indent() << "for (" << is << " = (" << is << "_loaded++) " - << "? " << js << " + 1 : " << is << "; " - << is << " > 1; " << is << "--) {" << endl; - inc_indent(); + stream << indent() << "for (" << *is << " = (" << oprefix << *is << "_loaded++) " + << "? " << *js << " + 1 : " << oprefix << *is << "; " + << *is << " > 1; " << *is << "--) {" << endl; } else { - stream << indent() << "for (" << *t << " " << is << " = " - << js << " + 1; " << is << " > 1; " << is << "--) {" << endl; - inc_indent(); + stream << indent() << "for (" << *t << " " << *is << " = " << *js + << " + 1; " << *is << " > 1; " << *is << "--) {" << endl; } + inc_indent(); - multi_print_inner_cyk(inner, tord, track, tracks, track_pos, t, checkpoint); + multi_print_inner_cyk(*inner, tord, track, tracks, track_pos, t, + checkpoint, false, for_outsideNTs); dec_indent(); stream << indent() << "}" << endl << endl; - if (!left.empty()) { - stream << indent() << "// B: inner quadratic loops" << endl; - stream << indent() << *t << " " << is << " = 1;" << endl; - multi_print_inner_cyk(left, tord, track, tracks, - track_pos, t, checkpoint); + if (!for_outsideNTs) { + multi_print_cyk_loops_quadratic_inner( + tord, track, tracks, track_pos, t, left, is, + checkpoint, for_outsideNTs); } dec_indent(); stream << indent() << "}" << endl << endl; } - if (inner.empty() && !left.empty()) { +} +void Printer::Cpp::multi_print_cyk_loops_quadratic_inner( + const std::list &tord, + size_t track, + size_t tracks, + size_t track_pos, + Type::Base *t, + std::list *left, + std::string *is, + bool checkpoint, + bool for_outsideNTs) { + if (!left->empty()) { + stream << indent() << "// B: inner quadratic loops" << endl; + stream << indent(); + if (!for_outsideNTs) { + stream << *t << " "; + } + stream << *is << " = 1;" << endl; + multi_print_inner_cyk(*left, tord, track, tracks, track_pos, t, + checkpoint, for_outsideNTs, for_outsideNTs); + } +} +void Printer::Cpp::multi_print_cyk_loops_linear( + const std::list &tord, + size_t track, + size_t tracks, + size_t track_pos, + Type::Base *t, + std::list *inner, + std::list *left, + std::list *right, + std::string *is, + std::string *js, + std::string *ns, + bool checkpoint, + bool for_outsideNTs + ) { + std::string oprefix = ""; + if (for_outsideNTs) { + oprefix = "outside_"; + } + if (inner->empty() && !left->empty()) { stream << indent() << "// C: linear loops" << endl; if (checkpoint) { - stream << indent() << "for (" << js << " = (" << js << "_loaded++) " - << "? 0 : " << js << "; " << js << " < " << ns - << "; " << "++" << js << ") {" << endl; - inc_indent(); + stream << indent() << "for (" << *js << " = (" << oprefix << *js << "_loaded++) " + << "? 0 : " << oprefix << *js << "; " << *js << " < " << *ns + << "; " << "++" << *js << ") {" << endl; } else { - stream << indent() << "for (" << *t << " " << js << " = 0; " - << js << " < " << ns - << "; " << "++" << js << ") {" << endl; - inc_indent(); + stream << indent() << "for (" << *t << " " << *js << " = 0; " + << *js << " < " << *ns + << "; " << "++" << *js << ") {" << endl; } - stream << indent() << *t << " " << is << " = 1;" << endl; - multi_print_inner_cyk(left, tord, track, tracks, track_pos, t, checkpoint); + inc_indent(); + stream << indent(); + if (!for_outsideNTs) { + stream << *t << " "; + } + stream << *is << " = 1;" << endl; + multi_print_inner_cyk(*left, tord, track, tracks, track_pos, t, + checkpoint, false, for_outsideNTs); dec_indent(); stream << indent() << "}" << endl << endl; } - if (!right.empty()) { + if (!right->empty()) { stream << indent() << "// C: linear loops" << endl; - stream << indent() << *t << " " << js << " = " << ns << ";" << endl; + stream << indent(); + if (!for_outsideNTs) { + stream << *t << " "; + } + stream << *js << " = " << *ns << ";" << endl; if (checkpoint) { /* in checkpointing mode, the entire quadratic loop (A) will be skipped if the loaded value for t_0_j == t_0_n, so this is the loop that would get executed next */ - stream << indent() << "for (" << is << " = (" << is << "_loaded++) " - << "? " << js << " + 1 : " << is << "; " - << is << " > 1; " << is << "--) {" << endl; - inc_indent(); + stream << indent() << "for (" << *is << " = (" << oprefix << *is << "_loaded++) " + << "? " << *js << " + 1 : " << oprefix << *is << "; " + << *is << " > 1; " << *is << "--) {" << endl; } else { - stream << indent() << "for (" << *t << " "<< is << " = " << js << " + 1; " - << is << " > 1; " << is << "--) {" << endl; - inc_indent(); + stream << indent() << "for (" << *t << " "<< *is << " = " << *js + << " + 1; " << *is << " > 1; " << *is << "--) {" << endl; } - multi_print_inner_cyk(right, tord, track, tracks, track_pos, t, checkpoint); + inc_indent(); + multi_print_inner_cyk(*right, tord, track, tracks, track_pos, t, + checkpoint, false, for_outsideNTs); dec_indent(); stream << indent() << "}" << endl << endl; } - - if (!all.empty()) { +} +void Printer::Cpp::multi_print_cyk_loops_constant( + const std::list &tord, + size_t track, + size_t tracks, + size_t track_pos, + Type::Base *t, + std::list *all, + std::string *is, + bool checkpoint, + bool for_outsideNTs + ) { + if (!all->empty()) { stream << indent() << "// D: constant loops" << endl; - stream << indent() << *t << " " << is << " = 1;" << endl; - multi_print_inner_cyk(all, tord, track, tracks, track_pos, t, checkpoint); + stream << indent(); + if (!for_outsideNTs) { + stream << *t << " "; + } + stream << *is << " = 1;" << endl; + /* when constructing commands to compute constant results, there is no + * enclosing loop context, i.e. mutex definition will occure in function + * context. Since we repeat these commands for outside contexts, the + * mutex would be defined twice. We avoid this here. */ + bool no_mutex_def = for_outsideNTs; + multi_print_inner_cyk(*all, tord, track, tracks, track_pos, t, + checkpoint, no_mutex_def, for_outsideNTs); + } +} + +void Printer::Cpp::multi_print_cyk( + const std::list &tord, size_t track, size_t tracks, + size_t track_pos, Type::Base *t) { + std::list all, inner, left, right; + multi_partition_nts(tord, all, inner, left, right, track, tracks, track_pos); + + size_t real_track = !track_pos ? track : track_pos; + + std::ostringstream sis, sjs, sns; + sis << "t_" << track << "_i"; + sjs << "t_" << track << "_j"; + sns << "t_" << real_track << "_n"; + std::string is(sis.str()), js(sjs.str()), ns(sns.str()); + + stream << indent() << *t << " t_" << track << "_n = t_" << real_track + << "_seq.size();" << endl << endl; + + bool checkpoint = ast->checkpoint && ast->checkpoint->cyk; + + // inside NTs + multi_print_cyk_loops_quadratic( + tord, track, tracks, track_pos, t, &inner, &left, &is, &js, &ns, + checkpoint, false); + multi_print_cyk_loops_linear( + tord, track, tracks, track_pos, t, &inner, &left, &right, &is, &js, &ns, + checkpoint, false); + multi_print_cyk_loops_constant( + tord, track, tracks, track_pos, t, &all, &is, + checkpoint, false); + if (checkpoint) { + stream << indent() << "mutex.unlock();" << endl; + } + + // outside NTs (which access inside NT values) + if (this->ast->grammar()->is_outside()) { + stream << endl << endl; + stream << indent() << "// now compute for outside non-terminals" << endl; + multi_print_cyk_loops_quadratic( + tord, track, tracks, track_pos, t, &inner, &left, &is, &js, &ns, + checkpoint, true); + multi_print_cyk_loops_linear( + tord, track, tracks, track_pos, t, &inner, &left, &right, &is, &js, &ns, + checkpoint, true); + if (checkpoint) { + stream << indent() << "mutex.lock();" << endl; + } + multi_print_cyk_loops_constant( + tord, track, tracks, track_pos, t, &all, &is, + checkpoint, true); + if (checkpoint) { + stream << indent() << "mutex.unlock();" << endl; + } } } @@ -2498,6 +2629,17 @@ void Printer::Cpp::print_cyk_fn(const AST &ast) { << "_j_loaded = !load_checkpoint ||" << " !t_" << track_i << "_j;" << endl; } + if (ast.grammar()->is_outside()) { + for (size_t track_i = 0; + track_i < ast.grammar()->axiom->tracks(); ++track_i) { + stream << indent() << "int outside_t_" << track_i + << "_i_loaded = !load_checkpoint ||" + << " !outside_t_" << track_i << "_i;" << endl; + stream << indent() << "int outside_t_" << track_i + << "_j_loaded = !load_checkpoint ||" + << " !outside_t_" << track_i << "_j;" << endl; + } + } stream << "#ifdef _OPENMP" << endl; stream << indent() << "unsigned int tile_size = 32;" << endl; stream << "#ifdef TILE_SIZE" << endl; @@ -2772,9 +2914,11 @@ void Printer::Cpp::header_footer(const AST &ast) { nt_tables &tabulated = ast.grammar()->tabulated; if (ast.checkpoint->cyk) { ast.checkpoint->archive_cyk_indices(stream, - ast.grammar()->axiom->tracks()); + ast.grammar()->axiom->tracks(), + ast.grammar()->is_outside()); ast.checkpoint->load_cyk_indices(stream, - ast.grammar()->axiom->tracks()); + ast.grammar()->axiom->tracks(), + ast.grammar()->is_outside()); ast.checkpoint->parse_checkpoint_log(stream, true); } ast.checkpoint->archive_periodically(stream, tabulated); From dc39ebc2af2826338ab04f4201e89b55bf67b10f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 12 Apr 2023 10:43:40 +0200 Subject: [PATCH 205/209] add a test with CYK and checkpoint --- testdata/regresstest/config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index c0a3c0369..9c11faec5 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -496,6 +496,10 @@ check_new_old_eq nodangle_withoverlay.gap unused ins_pfunc "CCaCCaaaGGaCCaaaGGaC GAPC="../../../gapc --outside_grammar weak --outside_grammar struct --cyk" check_new_old_eq nodangle.gap unused pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside +# same as above, but with activated checkpointing +GAPC="../../../gapc --outside_grammar weak --outside_grammar struct --cyk --checkpoint" +check_new_old_eq nodangle.gap unused pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside + # same as above, but with CYK code generation AND openMP parallelization GAPC="../../../gapc --outside_grammar weak --outside_grammar struct --cyk" CPPFLAGS_EXTRA="$DEFAULT_CPPFLAGS_EXTRA -fopenmp" From 072f6b114fa6811e6771fecd80c01399c0bb31b7 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 12 Apr 2023 10:45:32 +0200 Subject: [PATCH 206/209] linting --- src/cpp.cc | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 3f583167e..00749f2bd 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2352,22 +2352,23 @@ void Printer::Cpp::multi_print_cyk_loops_quadratic( if (!inner->empty()) { if (checkpoint) { /* - in checkpointing mode, the loop indices - (e.g. t_0_i and t_0_j in single-track mode; - names can differ in multi-track mode) - are members of the out class instead of local loop variables - so they can be archived/loaded; - if they are loaded, the loop variable (e.g. t_0_j) starts - from whatever value was loaded from the checkpoint, - which is handled through a marker - for every loop variable (e.g. "t_0_j_loaded"); - initially, all of these marker values are set to 0, which will - set each loop variable to whatever value was loaded from the checkpoint; - after that, the respective marker value will be set to 1, - so the loop variable will be set to whatever - value is usually would be set to + in checkpointing mode, the loop indices + (e.g. t_0_i and t_0_j in single-track mode; + names can differ in multi-track mode) + are members of the out class instead of local loop variables + so they can be archived/loaded; + if they are loaded, the loop variable (e.g. t_0_j) starts + from whatever value was loaded from the checkpoint, + which is handled through a marker + for every loop variable (e.g. "t_0_j_loaded"); + initially, all of these marker values are set to 0, which will + set each loop variable to whatever value was loaded from the checkpoint; + after that, the respective marker value will be set to 1, + so the loop variable will be set to whatever + value is usually would be set to */ - stream << indent() << "for (" << *js << " = (" << oprefix << *js << "_loaded++) " + stream << indent() << "for (" << *js << " = (" << oprefix << *js + << "_loaded++) " << "? 0 : " << oprefix << *js << "; " << *js << " < " << *ns << "; " << "++" << *js << ") {" << endl; @@ -2391,7 +2392,8 @@ void Printer::Cpp::multi_print_cyk_loops_quadratic( for t_0_i in the first inner loop pass; afterwards it will regularly be set to t_0_j + 1; */ - stream << indent() << "for (" << *is << " = (" << oprefix << *is << "_loaded++) " + stream << indent() << "for (" << *is << " = (" << oprefix << *is + << "_loaded++) " << "? " << *js << " + 1 : " << oprefix << *is << "; " << *is << " > 1; " << *is << "--) {" << endl; } else { @@ -2457,7 +2459,8 @@ void Printer::Cpp::multi_print_cyk_loops_linear( if (inner->empty() && !left->empty()) { stream << indent() << "// C: linear loops" << endl; if (checkpoint) { - stream << indent() << "for (" << *js << " = (" << oprefix << *js << "_loaded++) " + stream << indent() << "for (" << *js << " = (" << oprefix << *js + << "_loaded++) " << "? 0 : " << oprefix << *js << "; " << *js << " < " << *ns << "; " << "++" << *js << ") {" << endl; } else { @@ -2489,7 +2492,8 @@ void Printer::Cpp::multi_print_cyk_loops_linear( if the loaded value for t_0_j == t_0_n, so this is the loop that would get executed next */ - stream << indent() << "for (" << *is << " = (" << oprefix << *is << "_loaded++) " + stream << indent() << "for (" << *is << " = (" << oprefix << *is + << "_loaded++) " << "? " << *js << " + 1 : " << oprefix << *is << "; " << *is << " > 1; " << *is << "--) {" << endl; } else { From 94f6e4876d7825e615f1a7540da1461746cb1df3 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 12 Apr 2023 11:09:58 +0200 Subject: [PATCH 207/209] only compute tile size once (but not again for outside NTs) + change var name y_loaded into y --- src/cpp.cc | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 00749f2bd..630ccf297 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2021,15 +2021,17 @@ void Printer::Cpp::print_openmp_cyk_helpervars(bool for_outsideNTs) { stream << "t_0_"; } stream << "n = t_0_seq.size();" << endl; - if (!ast->checkpoint) { - stream << indent() << "unsigned int tile_size = 32;" << endl; - stream << "#ifdef TILE_SIZE" << endl; - stream << indent() << "tile_size = TILE_SIZE;" << endl; - stream << "#endif" << endl; + if (!for_outsideNTs) { + if (!ast->checkpoint) { + stream << indent() << "unsigned int tile_size = 32;" << endl; + stream << "#ifdef TILE_SIZE" << endl; + stream << indent() << "tile_size = TILE_SIZE;" << endl; + stream << "#endif" << endl; + } + stream << indent() << "assert(tile_size);" << endl; + stream << indent() << "unsigned int max_tiles = n / tile_size;" << endl; + stream << indent() << "int max_tiles_n = max_tiles * tile_size;" << endl; } - stream << indent() << "assert(tile_size);" << endl; - stream << indent() << "unsigned int max_tiles = n / tile_size;" << endl; - stream << indent() << "int max_tiles_n = max_tiles * tile_size;" << endl; } void Printer::Cpp::print_openmp_cyk_loops_diag(const AST &ast, bool checkpoint, @@ -2101,7 +2103,7 @@ void Printer::Cpp::print_openmp_cyk_loops_middle(const AST &ast, << "z : inner_loop_2_idx_start;" << " y < max_tiles_n; y+=tile_size) {" << endl; inc_indent(); - stream << indent() << "++y_loaded;" << endl; + stream << indent() << "++y;" << endl; stream << indent() << "mutex.lock_shared();" << endl; } else { stream << indent() From 065ba7276b627a828d2590af88e66ce0b7361665 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 12 Apr 2023 11:12:42 +0200 Subject: [PATCH 208/209] adding CYK openmp checkpoint test --- testdata/regresstest/config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 9c11faec5..4615c263f 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -512,6 +512,9 @@ OMP_NUM_THREADS=2 export OMP_NUM_THREADS check_new_old_eq nodangle.gap unused pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside +# same as above, but with activated checkpointing +GAPC="../../../gapc --outside_grammar weak --outside_grammar struct --cyk --checkpoint" +check_new_old_eq nodangle.gap unused pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" outside GAPC="../../../gapc --outside_grammar ALL" CPPFLAGS_EXTRA="" From caa0e76c887277aa1afb61020e4406d92464ecd2 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 12 Apr 2023 13:47:43 +0200 Subject: [PATCH 209/209] only manually unlock mutex when outside generation was requested --- src/cpp.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 630ccf297..38afa9766 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2139,7 +2139,9 @@ void Printer::Cpp::print_openmp_cyk_loops_middle(const AST &ast, inc_indent(); stream << indent() << "inner_loop_2_idx += tile_size;" << endl; stream << indent() << "outer_loop_2_idx = z;" << endl; - stream << indent() << "mutex.unlock_shared();" << endl; + if (for_outsideNTs) { + stream << indent() << "mutex.unlock_shared();" << endl; + } dec_indent(); stream << indent() << "}" << endl; } @@ -2182,7 +2184,7 @@ void Printer::Cpp::print_openmp_cyk_loops_single(const AST &ast, stream << indent() << "mutex.lock_shared();" << endl; } print_openmp_cyk_all_nt_calls(ast, for_outsideNTs); - if (ast.checkpoint && ast.checkpoint->cyk) { + if (ast.checkpoint && ast.checkpoint->cyk && for_outsideNTs) { stream << indent() << "mutex.unlock_shared();" << endl; } @@ -2566,7 +2568,7 @@ void Printer::Cpp::multi_print_cyk( multi_print_cyk_loops_constant( tord, track, tracks, track_pos, t, &all, &is, checkpoint, false); - if (checkpoint) { + if (checkpoint && this->ast->grammar()->is_outside()) { stream << indent() << "mutex.unlock();" << endl; }