From e0759c43205bd56f66075f5353dee3fe01f502ea Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 10:15:29 +0100 Subject: [PATCH 01/45] adding additional member q, since D2 needs to store both vdot and q for candidates (note that naming schema is messed up!) adding soft_max_hessian operator --- rtlib/traces.hh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/rtlib/traces.hh b/rtlib/traces.hh index b333c3c51..4448f130f 100644 --- a/rtlib/traces.hh +++ b/rtlib/traces.hh @@ -79,6 +79,7 @@ bool is_same_index(std::vector a, std::vector b) { class candidate { private: double value; + double q; std::vector sub_components; public: @@ -94,6 +95,10 @@ class candidate { this->value = value; } + void set_q(double q) { + this->q = q; + } + void add_sub_component(std::string otherNT, std::vector *indices) { sub_components.push_back({otherNT, *indices, NULL}); @@ -107,6 +112,10 @@ class candidate { return value; } + double get_q() const { + return q; + } + std::vector get_normalized_candidate(double eval) const { std::vector res; for (std::vector::const_iterator part = this->sub_components.begin(); @@ -116,6 +125,17 @@ class candidate { } return res; } + + std::vector get_soft_max_hessian_candidate(double eval) const { + std::vector res; + for (std::vector::const_iterator part = this->sub_components.begin(); + part != this->sub_components.end(); ++part) { + //std::cerr << this->get_q() - (eval * this->get_value()) << "\n"; + res.push_back({std::get<0>(*part), std::get<1>(*part), + this->get_q() - (eval * this->get_value())}); + } + return res; + } // void empty() { // empty(this->value); // } @@ -137,6 +157,19 @@ std::vector normalize_traces(std::vector *tabulated, return res; } +inline +std::vector soft_max_hessian_product(std::vector *tabulated, + const std::vector &candidates, + double eval) { + std::vector, double > > res; + for (std::vector::const_iterator i = candidates.begin(); + i != candidates.end(); ++i) { + std::vector comp = (*i).get_soft_max_hessian_candidate(eval); + res.insert(res.end(), comp.begin(), comp.end()); + } + return res; +} + inline double get_trace_weights(const std::vector &traces, const std::string &to_nt, From 3d0b2b378f0a18d2c5f3e6cf79b750504fe49a25 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 11:57:29 +0100 Subject: [PATCH 02/45] added algebra alg_hessian for computation of second derivatives --- testdata/grammar_outside/alignments.gap | 55 ++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/testdata/grammar_outside/alignments.gap b/testdata/grammar_outside/alignments.gap index e806adac4..7b3aeced1 100644 --- a/testdata/grammar_outside/alignments.gap +++ b/testdata/grammar_outside/alignments.gap @@ -37,7 +37,7 @@ algebra alg_similarity implements sig_alignments(alphabet=char, answer=int) { int Sto() { return 0; } - + int Region(, int x, ) { return x; } @@ -47,7 +47,7 @@ algebra alg_similarity implements sig_alignments(alphabet=char, answer=int) { int Region_Pr_Pr(, int x, ) { return x; } - + // this is slightly different form http://rna.informatik.uni-freiburg.de/Teaching/index.jsp?toolName=Gotoh# // as there Ins + Insx is computed for first blank, we here score Ins for first blank and Insx for all following ones int Insx(, , int x) { @@ -62,6 +62,49 @@ algebra alg_similarity implements sig_alignments(alphabet=char, answer=int) { } } +algebra alg_hessian implements sig_alignments(alphabet=char, answer=float) { + float Ins(, , float x) { + return x + -2.0; + } + float Del(, , float x) { + return x + -2.0; + } + float Ers(, , float x) { + if (a == b) { + return x +2.0; + } else { + return x +1.0; + } + } + float Sto() { + return 0.0; + } + + float Region(, float x, ) { + return x; + } + float Region_Pr(, float x, ) { + return x; + } + float Region_Pr_Pr(, float x, ) { + return x; + } + + // this is slightly different form http://rna.informatik.uni-freiburg.de/Teaching/index.jsp?toolName=Gotoh# + // as there Ins + Insx is computed for first blank, we here score Ins for first blank and Insx for all following ones + float Insx(, , float x) { + return x + -1.0; + } + float Delx(, , float x) { + return x + -1.0; + } + + choice [float] h([float] candidates) { + return list(sum(candidates)); + } +} + + algebra alg_score implements sig_alignments(alphabet=char, answer=float) { float Ins(, , float x) { return x * exp(-2.0); @@ -104,6 +147,12 @@ algebra alg_score implements sig_alignments(alphabet=char, answer=float) { } } +algebra alg_jacobian extends alg_score { + choice [float] h([float] candidates) { + return list(expsum(candidates)); + } +} + algebra alg_countmanual implements sig_alignments(alphabet=char, answer=int) { int Ins(, , int x) { @@ -171,3 +220,5 @@ instance count = gra_needlemanwunsch(alg_count); instance sim_enum = gra_needlemanwunsch(alg_similarity * alg_enum); instance firstD = gra_needlemanwunsch(alg_score); instance firstD_gotoh = gra_gotoh(alg_score); + +instance bothD = gra_needlemanwunsch(alg_jacobian * alg_hessian); \ No newline at end of file From 36ddf3a32ba4161f460aa5ebe2e3106376c2658a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 11:59:17 +0100 Subject: [PATCH 03/45] depending on the number of derivative (first or second) normalize classically or via the hessian product operator --- src/tablegen.cc | 15 +++++++++------ src/tablegen.hh | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/tablegen.cc b/src/tablegen.cc index 3e32ac4fc..edd8e646e 100644 --- a/src/tablegen.cc +++ b/src/tablegen.cc @@ -348,7 +348,7 @@ void Tablegen::offset(size_t track_pos, itr f, const itr &e) { #include "symbol.hh" Statement::Table_Decl *Tablegen::create(Symbol::NT &nt, - std::string *name, bool cyk, bool for_derivatives) { + std::string *name, bool cyk, int forDerivative) { cyk_ = cyk; std::list ors; nt.gen_ys_guards(ors); @@ -369,7 +369,7 @@ Statement::Table_Decl *Tablegen::create(Symbol::NT &nt, offset(nt.track_pos(), nt.tables().begin(), nt.tables().end()); Fn_Def *fn_tab = gen_tab(); - Fn_Def *fn_set_traces = gen_set_traces(); + Fn_Def *fn_set_traces = gen_set_traces(forDerivative); ret_zero = new Statement::Return(new Expr::Vacc(new std::string("zero"))); offset(nt.track_pos(), nt.tables().begin(), nt.tables().end()); @@ -381,7 +381,7 @@ Statement::Table_Decl *Tablegen::create(Symbol::NT &nt, Statement::Table_Decl *td = new Statement::Table_Decl(nt, dtype, name, cyk, fn_is_tab, fn_tab, fn_set_traces, fn_get_traces, fn_get_tab, fn_size, ns); - td->for_derivatives = for_derivatives; + td->for_derivatives = forDerivative > 0; td->set_fn_untab(fn_untab); return td; } @@ -485,7 +485,7 @@ Fn_Def *Tablegen::gen_tab() { return f; } -Fn_Def *Tablegen::gen_set_traces() { +Fn_Def *Tablegen::gen_set_traces(int forDerivative) { Fn_Def *f = new Fn_Def(new Type::RealVoid(), new std::string("set_traces")); f->add_paras(paras); f->add_para(new ::Type::External(new std::string("NTtraces")), @@ -513,8 +513,11 @@ Fn_Def *Tablegen::gen_set_traces() { a->add_arg(new Expr::Less(off, new Expr::Fn_Call(new std::string("size")))); c.push_back(a); - Expr::Fn_Call *rhs_norm = new Expr::Fn_Call(new std::string( - "normalize_traces")); + std::string *fn_norm_name = new std::string("normalize_traces"); + if (forDerivative == 2) { + fn_norm_name = new std::string("soft_max_hessian_product"); + } + Expr::Fn_Call *rhs_norm = new Expr::Fn_Call(fn_norm_name); rhs_norm->add_arg(new Var_Acc::Array(new Var_Acc::Plain( new std::string("&traces")), off)); rhs_norm->add_arg(new std::string("candidates")); diff --git a/src/tablegen.hh b/src/tablegen.hh index 02a7689bf..90a8d429a 100644 --- a/src/tablegen.hh +++ b/src/tablegen.hh @@ -84,7 +84,7 @@ class Tablegen { Fn_Def *gen_is_tab(); Fn_Def *gen_untab(); Fn_Def *gen_tab(); - Fn_Def *gen_set_traces(); + Fn_Def *gen_set_traces(int forDerivative = 1); Fn_Def *gen_get_traces(); Fn_Def *gen_get_tab(); Fn_Def *gen_size(); @@ -97,7 +97,7 @@ class Tablegen { void offset(size_t track_pos, itr first, const itr &end); Statement::Table_Decl *create(Symbol::NT &nt, - std::string *name, bool cyk, bool for_derivatives); + std::string *name, bool cyk, int forDerivative); }; From a35eedcf60ba22c486519e29105a9ca77039e6b6 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 12:02:19 +0100 Subject: [PATCH 04/45] replace bool "inject_derivatives" with int "ast.current_derivative" --- src/symbol.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/symbol.cc b/src/symbol.cc index df0f1c547..8e2a8236c 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -1023,7 +1023,7 @@ void Symbol::NT::init_ret_stmts(Code::Mode mode, AST &ast) { tabfn->add_arg(ret); ret_stmts.push_back(tabfn); - if (ast.inject_derivatives && !this->is_partof_outside + if ((ast.current_derivative > 0) && !this->is_partof_outside && *this->name != OUTSIDE_AXIOMS) { Statement::Fn_Call *tracefn = new Statement::Fn_Call("set_traces"); tracefn->add(*table_decl); @@ -1090,8 +1090,10 @@ void Symbol::NT::init_table_decl(const AST &ast) { Tablegen tg; tg.set_window_mode(ast.window_mode); - table_decl = tg.create(*this, t, ast.code_mode() == Code::Mode::CYK, - ast.inject_derivatives && !this->is_partof_outside); + table_decl = tg.create( + *this, t, ast.code_mode() == Code::Mode::CYK, + this->is_partof_outside ? 0 : ast.current_derivative + ); } #include @@ -1268,7 +1270,7 @@ void Symbol::NT::codegen(AST &ast) { } stmts.push_back(ret_decl); - if (ast.inject_derivatives) { + if (ast.current_derivative > 0) { if (!this->is_partof_outside) { stmts.push_back(new Statement::Var_Decl(new ::Type::External( new std::string("NTtraces")), "candidates")); From 958b3874a3dc7c308fbb7231cdb4864e4cbc5b46 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 12:07:24 +0100 Subject: [PATCH 05/45] iterate through requested derivatives, i.e. set filenames correctly and re-execute "back()" let multiple functions additionally know if user requested derivative generation --- src/gapc.cc | 65 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/src/gapc.cc b/src/gapc.cc index 64dbe2762..183054093 100644 --- a/src/gapc.cc +++ b/src/gapc.cc @@ -368,7 +368,8 @@ class Main { // lets the AST know if code for derivative computation has // to be injected - driver.ast.inject_derivatives = opts.derivative > 0; + driver.ast.requested_derivative = opts.derivative; + driver.ast.current_derivative = opts.derivative > 0 ? 1 : 0; if (driver.is_failing()) { throw LogError("Seen parse errors."); @@ -624,7 +625,8 @@ class Main { // dot-file for the grammar. This is handy if gapc modifies the original // grammar from the source file. // activate with command line argument --plot-grammar - if (opts.plot_grammar > 0) { + // (if derivatives are requested: only plot in first iteration) + if (opts.plot_grammar > 0 && driver.ast.current_derivative <= 1) { unsigned int nodeID = 1; grammar->to_dot(&nodeID, opts.plotgrammar_stream(), opts.plot_grammar); Log::instance()->normalMessage( @@ -645,16 +647,19 @@ class Main { // also writes some lines to the header file. Printer::Cpp hh(driver.ast, opts.h_stream()); hh.set_argv(argv, argc); - hh.class_name = opts.class_name; + hh.set_class_name(opts.class_name, driver.ast.current_derivative); hh.header(driver.ast); hh.begin_fwd_decls(); driver.ast.print_code(hh); + /* TODO(sjanssen): is there a nice way to add this code generating + statement only to the header file? */ + opts.h_stream() << " private:"; instance->print_code(hh); hh.footer(driver.ast); hh.end_fwd_decls(); hh.header_footer(driver.ast); if (grammar->is_outside()) { - if (driver.ast.inject_derivatives) { + if (driver.ast.current_derivative > 0) { hh.print_run_derivative_fn(driver.ast); } else { hh.print_insideoutside_report_fn(opts.outside_nt_list, driver.ast); @@ -665,7 +670,7 @@ class Main { // compile-result. Printer::Cpp cc(driver.ast, opts.stream()); cc.set_argv(argv, argc); - cc.class_name = opts.class_name; + cc.set_class_name(opts.class_name, driver.ast.current_derivative); cc.set_files(opts.in_file, opts.out_file); cc.prelude(opts, driver.ast); cc.imports(driver.ast); @@ -711,8 +716,8 @@ class Main { */ void finish() { Printer::Cpp hh(driver.ast, opts.h_stream()); - hh.class_name = opts.class_name; - hh.typedefs(code_); + hh.set_class_name(opts.class_name, driver.ast.current_derivative); + hh.typedefs(code_, driver.ast.current_derivative); } @@ -721,10 +726,10 @@ class Main { * to compile the result of this gapc compiler. * Precondition: the AST must have been created and configured. */ - void makefile() { + void makefile(const AST &ast) { Printer::Cpp mm(driver.ast, opts.m_stream()); mm.set_argv(argv, argc); - mm.makefile(opts); + mm.makefile(opts, ast); } public: @@ -741,7 +746,7 @@ class Main { * This is the entry point where the software starts. */ void runKernal() { - makefile(); + makefile(driver.ast); conv_classified_product(&opts); @@ -767,6 +772,46 @@ class Main { driver.ast.set_code_mode(mode); back(r.second, r.first); + } else if (opts.derivative > 1) { + // split algebra product "left * right" into two instances for first and second derivative + std::pair bothD = driver.ast.split_instance_for_derivatives(opts.instance); + + // store user provided file name pattern for .hh and .cc + std::string orig_header_file = opts.header_file; + std::string orig_out_file = opts.out_file; + + driver.ast.current_derivative = 1; + + // prepend "_derivative1" to generated .hh and .cc file + opts.header_file = basename(orig_header_file) + "_derivative" + + std::to_string(driver.ast.current_derivative) + ".hh"; + opts.out_file = basename(orig_out_file) + "_derivative" + + std::to_string(driver.ast.current_derivative) + ".cc"; + + // start generating code for first derivative + back(bothD.first); + finish(); + // finish and close *.hh and *.cc for first derivative + delete opts.h_stream_; + opts.h_stream_ = NULL; + delete opts.out; + opts.out = NULL; + + + // start generating code for second derivative + driver.ast.current_derivative = 2; + + // prepend "_derivative2" to generated .hh and .cc file + opts.header_file = basename(orig_header_file) + "_derivative" + + std::to_string(driver.ast.current_derivative) + ".hh"; + opts.out_file = basename(orig_out_file) + "_derivative" + + std::to_string(driver.ast.current_derivative) + ".cc"; + + back(bothD.second); + + // revert opts filenames + opts.header_file = orig_header_file; + opts.out_file = orig_out_file; } else { back(); } From c3f548c3bbe41c22634ee76e969e21e330fc98b5 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 12:08:50 +0100 Subject: [PATCH 06/45] replace bool "inject_derivatives" with int "ast.current_derivative" --- src/codegen.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codegen.cc b/src/codegen.cc index c1128a0cc..4604fadbd 100644 --- a/src/codegen.cc +++ b/src/codegen.cc @@ -27,7 +27,7 @@ #include "fn_def.hh" Code::Gen::Gen(AST &ast) { - if (ast.inject_derivatives) { + if (ast.current_derivative > 0) { Symbol::NT *inside_axiom = dynamic_cast( ast.grammar()->NTs[*ast.grammar()->axiom_name_inside]); assert(inside_axiom); From d56cb8733e175e3e92420f76160d9e2802d444c6 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 12:12:06 +0100 Subject: [PATCH 07/45] AST keeps track of the current derivative and the last derivative requested by the user added function to split the algebra product "first * second" into two independet instances for derivative generation, i.e. "back()" will be executed twice, resulting in multiple .hh and .cc files --- src/ast.cc | 30 ++++++++++++++++++++++++++++++ src/ast.hh | 15 ++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/ast.cc b/src/ast.cc index 5e7bdad4e..f0b0b3f6f 100644 --- a/src/ast.cc +++ b/src/ast.cc @@ -774,6 +774,36 @@ std::pair AST::split_classified(const std::string &n) { return std::make_pair(score, i); } +std::pair AST::split_instance_for_derivatives(const std::string &n) { + Instance *i = instance(n); + if (!i) { + throw LogError("Instance does not exist."); + } + + if (!i->product->is(Product::TIMES)) { + throw LogError("Algebra product is not of type times, i.e. '*'!"); + } + if (!i->product->left()->is(Product::SINGLE)) { + throw LogError("Left algebra is no single algebra, but an algebra product!"); + } + if (!i->product->right()->is(Product::SINGLE)) { + throw LogError("Right algebra is no single algebra, but an algebra product!"); + } + + Instance *inst_firstD = new Instance(new std::string("first derivative"), i->product->left(), grammar()); + if (!inst_firstD->product->algebra()->is_compatible(Mode::SYNOPTIC)) { + throw LogError("Left algebra is not synoptic, e.g. choice function is not sum."); + } + check_instances(inst_firstD); + Instance *inst_secondD = new Instance(new std::string("second derivative"), i->product->right(), grammar()); + if (!inst_secondD->product->algebra()->is_compatible(Mode::SYNOPTIC)) { + throw LogError("Right algebra is not synoptic, e.g. choice function is not sum."); + } + check_instances(inst_secondD); + + return std::make_pair(inst_firstD, inst_secondD); +} + #include "unused_visitor.hh" diff --git a/src/ast.hh b/src/ast.hh index d0a8ca230..70e4ab44a 100644 --- a/src/ast.hh +++ b/src/ast.hh @@ -261,6 +261,8 @@ class AST { public: Instance *split_instance_for_backtrack(std::string &n); std::pair split_classified(const std::string &n); + std::pair split_instance_for_derivatives( + const std::string &n); void backtrack_gen(Backtrack_Base &bt); void warn_unused_fns(Instance &i); @@ -295,7 +297,18 @@ class AST { return backtrack_product; } - bool inject_derivatives = false; + // tracks which derivative is currently being generated (first or second) + unsigned int current_derivative = 0; + + /* the derivative requested by the user. + * 0 = default, i.e. no derivative generation + * 1 = generate code to compute first derivative (Jacobians), + * e.g. base pair probabilities, forward-backward, ... + * 2 = generate code to compute second derivative (Hessians), + * e.g. for machine learning, which also required generation + * of first derivatives + */ + unsigned int requested_derivative = 0; }; #endif // SRC_AST_HH_ From 3456d2fb4c17e8635417b34244f0f38994ae62a5 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 12:18:58 +0100 Subject: [PATCH 08/45] made return type of "inject_derivative_body" more general different code generation for first and second derivative in "inject_derivative_body" and "init_derivative_recording" added new function "derivative_collect_traces" replace bool "inject_derivatives" with int "ast.current_derivative" "push_back_ret_decl" also declares variables "edgeweight" to record these in addition to vdot --- src/alt.cc | 140 ++++++++++++++++++++++++++++++++++++++++++++++++----- src/alt.hh | 7 ++- 2 files changed, 132 insertions(+), 15 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index b293dc6d2..67e66b7d0 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -47,6 +47,7 @@ #include "instance.hh" #include "statement/fn_call.hh" +#include "statement/table_decl.hh" Alt::Base::Base(Type t, const Loc &l) : @@ -1424,7 +1425,7 @@ void Alt::Base::add_seqs(Expr::Fn_Call *fn_call, const AST &ast) const { } // TODO(sjanssen): rename outside_fn_arg to reflect actual datatype -Expr::Fn_Call *Alt::Base::inject_derivative_body(AST &ast, +Expr::Base *Alt::Base::inject_derivative_body(AST &ast, Symbol::NT &calling_nt, Alt::Base *outside_fn_arg, Expr::Base *outside_arg) { assert(outside_fn_arg); @@ -1463,12 +1464,96 @@ Expr::Fn_Call *Alt::Base::inject_derivative_body(AST &ast, mkidx->add_arg(new Expr::Const(static_cast(mkidx->exprs.size())), true); fn_call->exprs.push_back(mkidx); - // result of outside non terminal, e.g. a_0 - fn_call->exprs.push_back(outside_arg); + if (ast.current_derivative == 1) { + // result of outside non terminal, e.g. a_0 + fn_call->exprs.push_back(outside_arg); + } else if (ast.current_derivative == 2) { + // results of adjoint backward + Expr::Fn_Call *fn_e2 = dynamic_cast(outside_fn_arg->ret_decl->rhs); + + // results of backward + Expr::Fn_Call *fn_e1 = new Expr::Fn_Call(new std::string( + "derivative" + std::to_string(ast.current_derivative-1) + "->" + + *(fn_e2->name)) + ); + fn_e1->add(fn_e2->exprs); + + Expr::Fn_Call *fn_q1 = new Expr::Fn_Call(new std::string(*(fn_call->name))); + fn_q1->add_arg(new std::string( + "derivative" + std::to_string(ast.current_derivative-1) + "->" + + (*outside_alt->name).substr(sizeof(OUTSIDE_NT_PREFIX)-1, (*outside_alt->name).length()) + "_table" + )); + fn_q1->is_obj = Bool(true); + fn_q1->exprs.insert(fn_q1->exprs.end(), std::next(fn_call->exprs.begin()), fn_call->exprs.end()); + fn_q1->add_arg(outside_fn_arg->ret_decl->name); + + fn_call->exprs.push_back(fn_e1); + return new Expr::Plus(fn_call, fn_q1); + } return fn_call; } + +// iterates through the arguments of an alternative and adds one statement per NT called to obtain derivative edge weight +std::list *Alt::Simple::derivative_collect_traces(AST &ast, Symbol::NT &calling_nt) { + std::list *stmts = new std::list(); + + for (std::list::iterator i = args.begin(); i != args.end(); ++i) { + if ((*i)->is(Fn_Arg::CONST)) { + continue; + } + Fn_Arg::Alt *fn_alt = dynamic_cast(*i); + if (fn_alt) { + Alt::Link *alt_link = dynamic_cast((*fn_alt).alt_ref()); + if (alt_link) { + Symbol::NT *alt_nt = dynamic_cast(alt_link->nt); + if (alt_nt) { + Expr::Fn_Call *fn_call = new Expr::Fn_Call(new std::string("get_traces")); + + // access lower derivative table + fn_call->add_arg(new std::string( + "derivative" + + std::to_string(ast.current_derivative-1) + + "->" + alt_nt->table_decl->name())); + fn_call->is_obj = Bool(true); + + // index of the calling non-terminal + Fn_Def *x = new Fn_Def(); + x->add_para(calling_nt); + for (std::list::const_iterator i = x->paras.begin(); + i != x->paras.end(); ++i) { + Para_Decl::Simple *s = dynamic_cast(*i); + if (s) { + fn_call->add_arg((*s).name()); + } + } + + // add name of non-terminal that requests traces + // together with make_index, this information is used to sub-set + // stored traces to those that actually lead to this DP cell + fn_call->exprs.push_back(new Expr::Const(*alt_nt->name)); + + // calling make_index + Expr::Fn_Call *mkidx = new Expr::Fn_Call(new std::string("*make_index")); + alt_link->add_args(mkidx); + mkidx->add_arg(new Expr::Const(static_cast(mkidx->exprs.size())), true); + fn_call->exprs.push_back(mkidx); + + fn_call->exprs.push_back(new Expr::Const(1.0)); + + + Statement::Var_Decl *q = new Statement::Var_Decl(decl->return_type, new std::string("edgeweight_" + *ret_decl->name)); + q->rhs = fn_call; + Statement::Var_Assign *stmt_ass = new Statement::Var_Assign(*q, new Expr::Times(new Expr::Vacc(*q), fn_call)); + stmts->push_back(stmt_ass); + } + } + } + } + return stmts; +} + void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { body_stmts.clear(); @@ -1504,7 +1589,7 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { fn_call->exprs.push_back(c->ret_decls().front()->rhs); continue; } - if (this->get_is_partof_outside() && ast.inject_derivatives) { + if (this->get_is_partof_outside() && (ast.current_derivative > 0)) { Fn_Arg::Alt *fn_alt = dynamic_cast(*i); if (fn_alt) { Alt::Link *alt_link = dynamic_cast((*fn_alt).alt_ref()); @@ -1537,11 +1622,18 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { decl->return_type, new std::string("ans")); pre_decl.clear(); pre_decl.push_back(vdecl); - if (ast.inject_derivatives && this->get_is_partof_outside() && + if ((ast.current_derivative >= 1) && this->get_is_partof_outside() && outside_fn_arg) { vdecl->rhs = inject_derivative_body(ast, calling_nt, outside_fn_arg->alt_ref(), outside_arg); + } else if ((ast.current_derivative == 2) && !this->get_is_partof_outside() && !outside_fn_arg) { + // obtain edge weight q + std::list *stmts_qs = derivative_collect_traces(ast, calling_nt); + stmts->insert(stmts->end(), stmts_qs->begin(), stmts_qs->end()); + + // multiply combined q with nt result + vdecl->rhs = new Expr::Times(new Expr::Vacc(new std::string("edgeweight_" + *ret_decl->name)), fn_call); } else { vdecl->rhs = fn_call; } @@ -1569,11 +1661,17 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { Statement::Var_Assign *ass = new Statement::Var_Assign(*ret_decl); pre_decl.clear(); pre_decl.push_back(ret_decl); - if (ast.inject_derivatives && calling_nt.is_partof_outside && + if ((ast.current_derivative >= 1) && calling_nt.is_partof_outside && outside_fn_arg) { ass->rhs = inject_derivative_body(ast, calling_nt, outside_fn_arg->alt_ref(), outside_arg); + } else if ((ast.current_derivative == 2) && !calling_nt.is_partof_outside && !outside_fn_arg) { + // obtain edge weight q + std::list *stmts_qs = derivative_collect_traces(ast, calling_nt); + stmts->insert(stmts->end(), stmts_qs->begin(), stmts_qs->end()); + // multiply combined q with nt result + ass->rhs = new Expr::Times(new Expr::Vacc(new std::string("edgeweight_" + *ret_decl->name)), fn_call); } else { ass->rhs = fn_call; } @@ -1593,7 +1691,7 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { void Alt::Base::init_derivative_recording( AST &ast, std::string *result_name) { - if (ast.inject_derivatives) { + if (ast.current_derivative > 0) { if (!this->is_partof_outside) { // test if this alternative uses sub-solutions from other non-terminals std::list *stmts_record = \ @@ -1653,7 +1751,17 @@ void Alt::Base::init_derivative_recording( x->add_arg(new std::string("cand")); x->is_obj = Bool(true); assert(result_name); - x->add_arg(result_name); + if (ast.current_derivative == 1) { + x->add_arg(result_name); + } else { + x->add_arg(new std::string("edgeweight_" + *result_name)); + + Statement::Fn_Call *y = new Statement::Fn_Call("set_q"); + y->add_arg(new std::string("cand")); + y->is_obj = Bool(true); + y->add_arg(result_name); + stmts_record->push_front(y); + } stmts_record->push_front(x); Statement::Var_Decl *candidate = new Statement::Var_Decl( @@ -1777,8 +1885,13 @@ void Alt::Simple::init_guards() { } -void Alt::Base::push_back_ret_decl() { +void Alt::Base::push_back_ret_decl(unsigned int current_derivative) { statements.push_back(ret_decl); + if (top_level && current_derivative > 1 && !is_partof_outside) { // && ! + Statement::Var_Decl *decl_edgeweight = new Statement::Var_Decl(ret_decl->type, new std::string("edgeweight_" + *ret_decl->name)); + decl_edgeweight->rhs = new Expr::Const(1.0); + statements.push_back(decl_edgeweight); + } } @@ -2136,7 +2249,7 @@ void Alt::Simple::codegen(AST &ast, Symbol::NT &calling_nt) { } statements.clear(); - push_back_ret_decl(); + push_back_ret_decl(ast.current_derivative); std::list *stmts = &statements; init_guards(); @@ -2330,7 +2443,7 @@ void Alt::Link::codegen(AST &ast, Symbol::NT &calling_nt) { // std::cout << "link " << *name << std::endl; statements.clear(); - push_back_ret_decl(); + push_back_ret_decl(ast.current_derivative); std::string *s = NULL; if (nt->is(Symbol::TERMINAL)) { s = name; @@ -2376,7 +2489,8 @@ void Alt::Link::codegen(AST &ast, Symbol::NT &calling_nt) { statements.push_back(filter_guards); filter_guards->then.push_back(v); } else { - if (nt->is(Symbol::NONTERMINAL) && this->top_level && ast.inject_derivatives + if (nt->is(Symbol::NONTERMINAL) && this->top_level + && (ast.current_derivative > 0) && calling_nt.is_partof_outside) { ret_decl->rhs = inject_derivative_body(ast, calling_nt, this, fn); } else { @@ -2407,7 +2521,7 @@ void Alt::Link::codegen(AST &ast, Symbol::NT &calling_nt) { void Alt::Block::codegen(AST &ast, Symbol::NT &calling_nt) { // std::cout << "-----------------Block " << std::endl; statements.clear(); - push_back_ret_decl(); + push_back_ret_decl(ast.current_derivative); Statement::Fn_Call *fn = new Statement::Fn_Call(Statement::Fn_Call::EMPTY); fn->add_arg(*ret_decl); statements.push_back(fn); diff --git a/src/alt.hh b/src/alt.hh index f67a1e737..7a20f58a3 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -245,7 +245,7 @@ class Base { protected: Statement::If *filter_guards; - void push_back_ret_decl(); + void push_back_ret_decl(unsigned int current_derivative); Expr::Base *suchthat_code(Statement::Var_Decl &decl) const; @@ -376,7 +376,7 @@ class Base { bool inside_end = false; void init_derivative_recording(AST &ast, std::string *result_name); - Expr::Fn_Call *inject_derivative_body(AST &ast, Symbol::NT &calling_nt, + Expr::Base *inject_derivative_body(AST &ast, Symbol::NT &calling_nt, Alt::Base *outside_fn_arg, Expr::Base *outside_arg); }; @@ -592,6 +592,9 @@ class Simple : public Base { Alt::Base* find_block(); Alt::Base *find_block_parent(const Alt::Base &block); + // generate code to obtain edge weights (q) for each rhs non-terminal + std::list *derivative_collect_traces(AST &ast, Symbol::NT &calling_nt); + private: std::list *insert_index_stmts( std::list *stmts); From 993ca2dced9b6c97e6e17a5a9ccaf3fe4228536b Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 12:21:46 +0100 Subject: [PATCH 09/45] let functions "typedefs" and "makefile" additionally know if user requested derivative generation made "class_name" private to enforce use of convenience functions "set_class_name" and "get_class_name_lower_derivative" to prepend "_derivativeX" if necessary --- src/cpp.cc | 95 +++++++++++++++++++++++++++++++++++++++++------------- src/cpp.hh | 18 +++++++++-- 2 files changed, 88 insertions(+), 25 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 9e5b04f33..43f031af1 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -1586,7 +1586,11 @@ void Printer::Cpp::print_most_init(const AST &ast) { void Printer::Cpp::print_init_fn(const AST &ast) { stream << indent() << "void init("; - stream << "const gapc::Opts &opts)" << " {" << endl; + stream << "const gapc::Opts &opts"; + for (unsigned int i = 1; i < ast.current_derivative; ++i) { + stream << ", " << get_class_name_lower_derivative(ast.current_derivative, i) << " *derivative" << std::to_string(i); + } + stream << ") {" << endl; inc_indent(); stream << indent() << "const std::vector >" @@ -1608,6 +1612,13 @@ void Printer::Cpp::print_init_fn(const AST &ast) { } } + if (ast.requested_derivative > 0) { + stream << endl; + } + for (unsigned int i = 1; i < ast.current_derivative; ++i) { + stream << indent() << "this->derivative" << std::to_string(i) << " = derivative" << std::to_string(i) << ";" << endl; + } + dec_indent(); stream << indent() << '}' << endl << endl; } @@ -1698,7 +1709,7 @@ void Printer::Cpp::header(const AST &ast) { } if ((*ast.grammar()).is_outside()) { stream << "#define OUTSIDE\n"; - if (ast.inject_derivatives) { + if (ast.current_derivative > 0) { stream << "#define DERIVATIVES\n"; } } @@ -1707,7 +1718,9 @@ void Printer::Cpp::header(const AST &ast) { print_type_defs(ast); } imports(ast); + print_hash_decls(ast); + stream << indent() << "class " << class_name << " {" << endl; stream << indent() << " public:" << endl; inc_indent(); @@ -1724,6 +1737,11 @@ void Printer::Cpp::header(const AST &ast) { stream << indent() << "unsigned winc;" << endl; } + /* create pointer to lower derivative results */ + for (unsigned int i = 1; i < ast.current_derivative; ++i) { + stream << indent() << get_class_name_lower_derivative(ast.current_derivative, i) << " *derivative" << i << ";" << endl; + } + stream << endl; print_zero_decls(*ast.grammar()); @@ -1734,7 +1752,15 @@ void Printer::Cpp::header(const AST &ast) { print_init_fn(ast); print_window_inc_fn(ast); dec_indent(); - stream << indent() << " private:" << endl; + stream << indent(); + if ((ast.current_derivative > 0) && (ast.current_derivative < ast.requested_derivative)) { + // let higher derivatives access lower DP results, e.g. second needs first + // however, last derivative can stay private + stream << " public:"; + } else { + stream << " private:"; + } + stream << endl; inc_indent(); } @@ -2320,7 +2346,8 @@ void Printer::Cpp::print_insideoutside_report_fn( } void Printer::Cpp::print_derivative(Symbol::NT *nt) { - stream << indent() << "std::cout << \"first derivatives for non-terminal \\\"" + stream << indent() << "std::cout << \"" << ast->current_derivative + << ". derivatives for non-terminal \\\"" << (*nt->name).substr(sizeof(OUTSIDE_NT_PREFIX)-1, (*nt->name).length()) << "\\\":\\n\";" << endl; @@ -2425,7 +2452,7 @@ void Printer::Cpp::print_run_derivative_fn(const AST &ast) { void Printer::Cpp::print_run_fn(const AST &ast) { Symbol::NT *axiom = ast.grammar()->axiom; - if (ast.inject_derivatives) { + if (ast.current_derivative > 0) { axiom = dynamic_cast( ast.grammar()->NTs[*ast.grammar()->axiom_name_inside]); } @@ -2801,16 +2828,18 @@ void Printer::Cpp::close_class() { } -void Printer::Cpp::typedefs(Code::Gen &code) { - stream << "#ifndef NO_GAPC_TYPEDEFS" << endl; - stream << indent() << "namespace gapc {" << endl; - inc_indent(); - stream << indent() << "typedef " << class_name << " class_name;" << endl; - stream << indent() << "typedef " << *code.return_type() - << " return_type;" << endl; - dec_indent(); - stream << indent() << '}' << endl; - stream << "#endif" << endl; +void Printer::Cpp::typedefs(Code::Gen &code, unsigned int current_derivative) { + if (current_derivative <= 1) { + stream << "#ifndef NO_GAPC_TYPEDEFS" << endl; + stream << indent() << "namespace gapc {" << endl; + inc_indent(); + stream << indent() << "typedef " << class_name << " class_name;" << endl; + stream << indent() << "typedef " << *code.return_type() + << " return_type;" << endl; + dec_indent(); + stream << indent() << '}' << endl; + stream << "#endif" << endl; + } stream << endl; stream << "#endif" << endl; } @@ -2852,7 +2881,7 @@ static const char deps[] = #include "prefix.hh" -void Printer::Cpp::makefile(const Options &opts) { +void Printer::Cpp::makefile(const Options &opts, const AST &ast) { stream << endl << make_comments(id_string, "#") << endl << endl; // stream << "SED = sed\n"; @@ -2875,7 +2904,14 @@ void Printer::Cpp::makefile(const Options &opts) { << "endif" << endl << endl; std::string base = opts.class_name; // basename(opts.out_file); - std::string out_file = remove_dir(opts.out_file); + std::string out_file = ""; + if (ast.requested_derivative > 0) { + for (unsigned int i = 1; i < ast.requested_derivative; ++i) { + out_file += basename(remove_dir(opts.out_file)) + "_derivative" + std::to_string(i) + ".cc "; + } + } else { + out_file = remove_dir(opts.out_file); + } std::string header_file = remove_dir(opts.header_file); stream << "CXXFILES = " << base << "_main.cc " << out_file << endl << endl; @@ -2894,7 +2930,15 @@ void Printer::Cpp::makefile(const Options &opts) { stream << endl << endl << base << "_main.cc : $(RTLIB)/generic_main.cc " << out_file << endl - << "\techo '#include \"" << header_file << "\"' > $@" << endl + << "\techo '#include "; + if (ast.requested_derivative > 0) { + for (unsigned int i = 1; i < ast.requested_derivative; ++i) { + stream << "\"" << basename(remove_dir(opts.out_file)) << "_derivative" << std::to_string(i) << ".hh\""; + } + } else { + stream << "\"" << header_file << "\""; + } + stream << "' > $@" << endl << "\tcat $(RTLIB)/generic_main.cc >> " << base << "_main.cc" << endl << endl; stream << deps << endl; @@ -2913,7 +2957,7 @@ void Printer::Cpp::imports(const AST &ast) { return; } - if (ast.inject_derivatives) { + if (ast.current_derivative > 0) { stream << "#include \"rtlib/traces.hh\"" << endl; } @@ -2966,9 +3010,16 @@ void Printer::Cpp::imports(const AST &ast) { } } stream << endl; - stream << "#include \"rtlib/generic_opts.hh\"\n"; - stream << "#include \"rtlib/pareto_dom_sort.hh\"\n"; - stream << "#include \"rtlib/pareto_yukish_ref.hh\"\n\n"; + stream << "#include \"rtlib/generic_opts.hh\"" << endl; + stream << "#include \"rtlib/pareto_dom_sort.hh\"" << endl; + stream << "#include \"rtlib/pareto_yukish_ref.hh\"" << endl; + + /* include code of lower derivatives */ + for (unsigned int i = 1; i < ast.current_derivative; ++i) { + stream << indent() << "#include \"" << get_class_name_lower_derivative(ast.current_derivative, i) << ".hh\"" << endl; + } + + stream << endl; } diff --git a/src/cpp.hh b/src/cpp.hh index 67f90708e..e5811fb78 100644 --- a/src/cpp.hh +++ b/src/cpp.hh @@ -158,9 +158,21 @@ class Cpp : public Base { void print_marker_init(const AST &ast); void print_marker_clear(const AST &ast); + std::string class_name; + public: + void set_class_name(std::string class_name, unsigned int current_derivative=0) { + this->class_name = class_name; + if (current_derivative > 0) { + this->class_name = this->class_name + "_derivative" + std::to_string(current_derivative); + } + } + std::string get_class_name_lower_derivative(unsigned int current_derivative, unsigned int derivative) { + assert(current_derivative > 0); + assert(derivative < 10); + return class_name.substr(0, class_name.size()-1) + std::to_string(derivative); + } bool in_class; - std::string class_name; Cpp() : Base(), ast(0), pure_list_type(false), in_fn_head(false), pointer_as_itr(false), @@ -244,14 +256,14 @@ class Cpp : public Base { void header_footer(const AST &ast); void footer(const AST &ast); void close_class(); - void typedefs(Code::Gen &code); + void typedefs(Code::Gen &code, unsigned int current_derivative); void prelude(const Options &opts, const AST &ast); void imports(const AST &ast); void global_constants(const AST &ast); - void makefile(const Options &opts); + void makefile(const Options &opts, const AST &ast); private: bool print_axiom_args(const AST &ast); From 023da7bc449e4df46744022f4ce9e108809f719d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 12:57:25 +0100 Subject: [PATCH 10/45] remove comment --- rtlib/traces.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/rtlib/traces.hh b/rtlib/traces.hh index 4448f130f..bec119861 100644 --- a/rtlib/traces.hh +++ b/rtlib/traces.hh @@ -130,7 +130,6 @@ class candidate { std::vector res; for (std::vector::const_iterator part = this->sub_components.begin(); part != this->sub_components.end(); ++part) { - //std::cerr << this->get_q() - (eval * this->get_value()) << "\n"; res.push_back({std::get<0>(*part), std::get<1>(*part), this->get_q() - (eval * this->get_value())}); } From 7c4abd6e0b2ef062e1585a6a87393d85628dbbe4 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 13:03:04 +0100 Subject: [PATCH 11/45] linting --- src/alt.cc | 159 ++++++++++++++++++++++++++++++----------------------- src/alt.hh | 3 +- 2 files changed, 92 insertions(+), 70 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 67e66b7d0..2d2355c57 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -1469,87 +1469,99 @@ Expr::Base *Alt::Base::inject_derivative_body(AST &ast, fn_call->exprs.push_back(outside_arg); } else if (ast.current_derivative == 2) { // results of adjoint backward - Expr::Fn_Call *fn_e2 = dynamic_cast(outside_fn_arg->ret_decl->rhs); + Expr::Fn_Call *fn_e2 = dynamic_cast( + outside_fn_arg->ret_decl->rhs); // results of backward Expr::Fn_Call *fn_e1 = new Expr::Fn_Call(new std::string( "derivative" + std::to_string(ast.current_derivative-1) + "->" + - *(fn_e2->name)) - ); + *(fn_e2->name))); fn_e1->add(fn_e2->exprs); Expr::Fn_Call *fn_q1 = new Expr::Fn_Call(new std::string(*(fn_call->name))); fn_q1->add_arg(new std::string( "derivative" + std::to_string(ast.current_derivative-1) + "->" + - (*outside_alt->name).substr(sizeof(OUTSIDE_NT_PREFIX)-1, (*outside_alt->name).length()) + "_table" - )); + (*outside_alt->name).substr( + sizeof(OUTSIDE_NT_PREFIX)-1, + (*outside_alt->name).length()) + "_table")); fn_q1->is_obj = Bool(true); - fn_q1->exprs.insert(fn_q1->exprs.end(), std::next(fn_call->exprs.begin()), fn_call->exprs.end()); + fn_q1->exprs.insert( + fn_q1->exprs.end(), + std::next(fn_call->exprs.begin()), + fn_call->exprs.end()); fn_q1->add_arg(outside_fn_arg->ret_decl->name); - fn_call->exprs.push_back(fn_e1); - return new Expr::Plus(fn_call, fn_q1); + fn_call->exprs.push_back(fn_e1); + return new Expr::Plus(fn_call, fn_q1); } return fn_call; } -// iterates through the arguments of an alternative and adds one statement per NT called to obtain derivative edge weight -std::list *Alt::Simple::derivative_collect_traces(AST &ast, Symbol::NT &calling_nt) { +/* iterates through the arguments of an alternative and adds one statement per + NT called to obtain derivative edge weight */ +std::list *Alt::Simple::derivative_collect_traces( + AST &ast, Symbol::NT &calling_nt) { std::list *stmts = new std::list(); - for (std::list::iterator i = args.begin(); i != args.end(); ++i) { - if ((*i)->is(Fn_Arg::CONST)) { - continue; - } - Fn_Arg::Alt *fn_alt = dynamic_cast(*i); - if (fn_alt) { - Alt::Link *alt_link = dynamic_cast((*fn_alt).alt_ref()); - if (alt_link) { - Symbol::NT *alt_nt = dynamic_cast(alt_link->nt); - if (alt_nt) { - Expr::Fn_Call *fn_call = new Expr::Fn_Call(new std::string("get_traces")); - - // access lower derivative table - fn_call->add_arg(new std::string( - "derivative" + - std::to_string(ast.current_derivative-1) + - "->" + alt_nt->table_decl->name())); - fn_call->is_obj = Bool(true); - - // index of the calling non-terminal - Fn_Def *x = new Fn_Def(); - x->add_para(calling_nt); - for (std::list::const_iterator i = x->paras.begin(); - i != x->paras.end(); ++i) { - Para_Decl::Simple *s = dynamic_cast(*i); - if (s) { - fn_call->add_arg((*s).name()); - } - } - - // add name of non-terminal that requests traces - // together with make_index, this information is used to sub-set - // stored traces to those that actually lead to this DP cell - fn_call->exprs.push_back(new Expr::Const(*alt_nt->name)); - - // calling make_index - Expr::Fn_Call *mkidx = new Expr::Fn_Call(new std::string("*make_index")); - alt_link->add_args(mkidx); - mkidx->add_arg(new Expr::Const(static_cast(mkidx->exprs.size())), true); - fn_call->exprs.push_back(mkidx); - - fn_call->exprs.push_back(new Expr::Const(1.0)); - - - Statement::Var_Decl *q = new Statement::Var_Decl(decl->return_type, new std::string("edgeweight_" + *ret_decl->name)); - q->rhs = fn_call; - Statement::Var_Assign *stmt_ass = new Statement::Var_Assign(*q, new Expr::Times(new Expr::Vacc(*q), fn_call)); - stmts->push_back(stmt_ass); - } - } - } + for (std::list::iterator i = args.begin(); + i != args.end(); ++i) { + if ((*i)->is(Fn_Arg::CONST)) { + continue; + } + Fn_Arg::Alt *fn_alt = dynamic_cast(*i); + if (fn_alt) { + Alt::Link *alt_link = dynamic_cast((*fn_alt).alt_ref()); + if (alt_link) { + Symbol::NT *alt_nt = dynamic_cast(alt_link->nt); + if (alt_nt) { + Expr::Fn_Call *fn_call = new Expr::Fn_Call( + new std::string("get_traces")); + + // access lower derivative table + fn_call->add_arg(new std::string( + "derivative" + + std::to_string(ast.current_derivative-1) + + "->" + alt_nt->table_decl->name())); + fn_call->is_obj = Bool(true); + + // index of the calling non-terminal + Fn_Def *x = new Fn_Def(); + x->add_para(calling_nt); + for (std::list::const_iterator i = x->paras.begin(); + i != x->paras.end(); ++i) { + Para_Decl::Simple *s = dynamic_cast(*i); + if (s) { + fn_call->add_arg((*s).name()); + } + } + + // add name of non-terminal that requests traces + // together with make_index, this information is used to sub-set + // stored traces to those that actually lead to this DP cell + fn_call->exprs.push_back(new Expr::Const(*alt_nt->name)); + + // calling make_index + Expr::Fn_Call *mkidx = new Expr::Fn_Call( + new std::string("*make_index")); + alt_link->add_args(mkidx); + mkidx->add_arg(new Expr::Const(static_cast(mkidx->exprs.size())), + true); + fn_call->exprs.push_back(mkidx); + + fn_call->exprs.push_back(new Expr::Const(1.0)); + + Statement::Var_Decl *q = new Statement::Var_Decl( + decl->return_type, + new std::string("edgeweight_" + *ret_decl->name)); + q->rhs = fn_call; + Statement::Var_Assign *stmt_ass = new Statement::Var_Assign( + *q, new Expr::Times(new Expr::Vacc(*q), fn_call)); + stmts->push_back(stmt_ass); + } + } + } } return stmts; } @@ -1627,13 +1639,17 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { vdecl->rhs = inject_derivative_body(ast, calling_nt, outside_fn_arg->alt_ref(), outside_arg); - } else if ((ast.current_derivative == 2) && !this->get_is_partof_outside() && !outside_fn_arg) { + } else if ((ast.current_derivative == 2) && + !this->get_is_partof_outside() && + !outside_fn_arg) { // obtain edge weight q - std::list *stmts_qs = derivative_collect_traces(ast, calling_nt); + std::list *stmts_qs = derivative_collect_traces( + ast, calling_nt); stmts->insert(stmts->end(), stmts_qs->begin(), stmts_qs->end()); // multiply combined q with nt result - vdecl->rhs = new Expr::Times(new Expr::Vacc(new std::string("edgeweight_" + *ret_decl->name)), fn_call); + vdecl->rhs = new Expr::Times(new Expr::Vacc(new std::string( + "edgeweight_" + *ret_decl->name)), fn_call); } else { vdecl->rhs = fn_call; } @@ -1666,12 +1682,16 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { ass->rhs = inject_derivative_body(ast, calling_nt, outside_fn_arg->alt_ref(), outside_arg); - } else if ((ast.current_derivative == 2) && !calling_nt.is_partof_outside && !outside_fn_arg) { + } else if ((ast.current_derivative == 2) && + !calling_nt.is_partof_outside && + !outside_fn_arg) { // obtain edge weight q - std::list *stmts_qs = derivative_collect_traces(ast, calling_nt); + std::list *stmts_qs = derivative_collect_traces( + ast, calling_nt); stmts->insert(stmts->end(), stmts_qs->begin(), stmts_qs->end()); // multiply combined q with nt result - ass->rhs = new Expr::Times(new Expr::Vacc(new std::string("edgeweight_" + *ret_decl->name)), fn_call); + ass->rhs = new Expr::Times(new Expr::Vacc(new std::string( + "edgeweight_" + *ret_decl->name)), fn_call); } else { ass->rhs = fn_call; } @@ -1887,8 +1907,9 @@ void Alt::Simple::init_guards() { void Alt::Base::push_back_ret_decl(unsigned int current_derivative) { statements.push_back(ret_decl); - if (top_level && current_derivative > 1 && !is_partof_outside) { // && ! - Statement::Var_Decl *decl_edgeweight = new Statement::Var_Decl(ret_decl->type, new std::string("edgeweight_" + *ret_decl->name)); + if (top_level && current_derivative > 1 && !is_partof_outside) { + Statement::Var_Decl *decl_edgeweight = new Statement::Var_Decl( + ret_decl->type, new std::string("edgeweight_" + *ret_decl->name)); decl_edgeweight->rhs = new Expr::Const(1.0); statements.push_back(decl_edgeweight); } diff --git a/src/alt.hh b/src/alt.hh index 7a20f58a3..1960825bf 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -593,7 +593,8 @@ class Simple : public Base { Alt::Base *find_block_parent(const Alt::Base &block); // generate code to obtain edge weights (q) for each rhs non-terminal - std::list *derivative_collect_traces(AST &ast, Symbol::NT &calling_nt); + std::list *derivative_collect_traces( + AST &ast, Symbol::NT &calling_nt); private: std::list *insert_index_stmts( From 180f73dc461bee002fc08ee8030b475195a87637 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 13:04:17 +0100 Subject: [PATCH 12/45] linting --- src/ast.cc | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/ast.cc b/src/ast.cc index f0b0b3f6f..f2f3bc6e6 100644 --- a/src/ast.cc +++ b/src/ast.cc @@ -774,7 +774,8 @@ std::pair AST::split_classified(const std::string &n) { return std::make_pair(score, i); } -std::pair AST::split_instance_for_derivatives(const std::string &n) { +std::pair AST::split_instance_for_derivatives( + const std::string &n) { Instance *i = instance(n); if (!i) { throw LogError("Instance does not exist."); @@ -784,20 +785,26 @@ std::pair AST::split_instance_for_derivatives(const std::s throw LogError("Algebra product is not of type times, i.e. '*'!"); } if (!i->product->left()->is(Product::SINGLE)) { - throw LogError("Left algebra is no single algebra, but an algebra product!"); + throw LogError( + "Left algebra is no single algebra, but an algebra product!"); } if (!i->product->right()->is(Product::SINGLE)) { - throw LogError("Right algebra is no single algebra, but an algebra product!"); + throw LogError( + "Right algebra is no single algebra, but an algebra product!"); } - Instance *inst_firstD = new Instance(new std::string("first derivative"), i->product->left(), grammar()); + Instance *inst_firstD = new Instance( + new std::string("first derivative"), i->product->left(), grammar()); if (!inst_firstD->product->algebra()->is_compatible(Mode::SYNOPTIC)) { - throw LogError("Left algebra is not synoptic, e.g. choice function is not sum."); + throw LogError( + "Left algebra is not synoptic, e.g. choice function is not sum."); } check_instances(inst_firstD); - Instance *inst_secondD = new Instance(new std::string("second derivative"), i->product->right(), grammar()); + Instance *inst_secondD = new Instance( + new std::string("second derivative"), i->product->right(), grammar()); if (!inst_secondD->product->algebra()->is_compatible(Mode::SYNOPTIC)) { - throw LogError("Right algebra is not synoptic, e.g. choice function is not sum."); + throw LogError( + "Right algebra is not synoptic, e.g. choice function is not sum."); } check_instances(inst_secondD); From bf8ba8f1a67fe29a84a1f13ae3667a6597c750d7 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 13:07:44 +0100 Subject: [PATCH 13/45] linting --- src/cpp.cc | 41 +++++++++++++++++++++++++---------------- src/cpp.hh | 12 ++++++++---- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 43f031af1..eb07ad196 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -1588,7 +1588,8 @@ void Printer::Cpp::print_init_fn(const AST &ast) { stream << indent() << "void init("; stream << "const gapc::Opts &opts"; for (unsigned int i = 1; i < ast.current_derivative; ++i) { - stream << ", " << get_class_name_lower_derivative(ast.current_derivative, i) << " *derivative" << std::to_string(i); + stream << ", " << get_class_name_lower_derivative(ast.current_derivative, i) + << " *derivative" << std::to_string(i); } stream << ") {" << endl; @@ -1613,10 +1614,11 @@ void Printer::Cpp::print_init_fn(const AST &ast) { } if (ast.requested_derivative > 0) { - stream << endl; + stream << endl; } for (unsigned int i = 1; i < ast.current_derivative; ++i) { - stream << indent() << "this->derivative" << std::to_string(i) << " = derivative" << std::to_string(i) << ";" << endl; + stream << indent() << "this->derivative" << std::to_string(i) + << " = derivative" << std::to_string(i) << ";" << endl; } dec_indent(); @@ -1739,7 +1741,9 @@ void Printer::Cpp::header(const AST &ast) { /* create pointer to lower derivative results */ for (unsigned int i = 1; i < ast.current_derivative; ++i) { - stream << indent() << get_class_name_lower_derivative(ast.current_derivative, i) << " *derivative" << i << ";" << endl; + stream << indent() + << get_class_name_lower_derivative(ast.current_derivative, i) + << " *derivative" << i << ";" << endl; } stream << endl; @@ -1753,10 +1757,11 @@ void Printer::Cpp::header(const AST &ast) { print_window_inc_fn(ast); dec_indent(); stream << indent(); - if ((ast.current_derivative > 0) && (ast.current_derivative < ast.requested_derivative)) { - // let higher derivatives access lower DP results, e.g. second needs first - // however, last derivative can stay private - stream << " public:"; + if ((ast.current_derivative > 0) && + (ast.current_derivative < ast.requested_derivative)) { + // let higher derivatives access lower DP results, e.g. second needs first + // however, last derivative can stay private + stream << " public:"; } else { stream << " private:"; } @@ -2347,7 +2352,7 @@ void Printer::Cpp::print_insideoutside_report_fn( void Printer::Cpp::print_derivative(Symbol::NT *nt) { stream << indent() << "std::cout << \"" << ast->current_derivative - << ". derivatives for non-terminal \\\"" + << ". derivatives for non-terminal \\\"" << (*nt->name).substr(sizeof(OUTSIDE_NT_PREFIX)-1, (*nt->name).length()) << "\\\":\\n\";" << endl; @@ -2906,9 +2911,10 @@ void Printer::Cpp::makefile(const Options &opts, const AST &ast) { std::string base = opts.class_name; // basename(opts.out_file); std::string out_file = ""; if (ast.requested_derivative > 0) { - for (unsigned int i = 1; i < ast.requested_derivative; ++i) { - out_file += basename(remove_dir(opts.out_file)) + "_derivative" + std::to_string(i) + ".cc "; - } + for (unsigned int i = 1; i < ast.requested_derivative; ++i) { + out_file += basename(remove_dir(opts.out_file)) + \ + "_derivative" + std::to_string(i) + ".cc "; + } } else { out_file = remove_dir(opts.out_file); } @@ -2932,9 +2938,10 @@ void Printer::Cpp::makefile(const Options &opts, const AST &ast) { << base << "_main.cc : $(RTLIB)/generic_main.cc " << out_file << endl << "\techo '#include "; if (ast.requested_derivative > 0) { - for (unsigned int i = 1; i < ast.requested_derivative; ++i) { - stream << "\"" << basename(remove_dir(opts.out_file)) << "_derivative" << std::to_string(i) << ".hh\""; - } + for (unsigned int i = 1; i < ast.requested_derivative; ++i) { + stream << "\"" << basename(remove_dir(opts.out_file)) << "_derivative" + << std::to_string(i) << ".hh\""; + } } else { stream << "\"" << header_file << "\""; } @@ -3016,7 +3023,9 @@ void Printer::Cpp::imports(const AST &ast) { /* include code of lower derivatives */ for (unsigned int i = 1; i < ast.current_derivative; ++i) { - stream << indent() << "#include \"" << get_class_name_lower_derivative(ast.current_derivative, i) << ".hh\"" << endl; + stream << indent() << "#include \"" + << get_class_name_lower_derivative(ast.current_derivative, i) + << ".hh\"" << endl; } stream << endl; diff --git a/src/cpp.hh b/src/cpp.hh index e5811fb78..b585f88e3 100644 --- a/src/cpp.hh +++ b/src/cpp.hh @@ -161,16 +161,20 @@ class Cpp : public Base { std::string class_name; public: - void set_class_name(std::string class_name, unsigned int current_derivative=0) { + void set_class_name(std::string class_name, + unsigned int current_derivative = 0) { this->class_name = class_name; if (current_derivative > 0) { - this->class_name = this->class_name + "_derivative" + std::to_string(current_derivative); + this->class_name = this->class_name + "_derivative" + \ + std::to_string(current_derivative); } } - std::string get_class_name_lower_derivative(unsigned int current_derivative, unsigned int derivative) { + std::string get_class_name_lower_derivative( + unsigned int current_derivative, unsigned int derivative) { assert(current_derivative > 0); assert(derivative < 10); - return class_name.substr(0, class_name.size()-1) + std::to_string(derivative); + return class_name.substr(0, class_name.size()-1) + \ + std::to_string(derivative); } bool in_class; Cpp() From a05896e3c5e8025cab4e0679a467f9f223eac733 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 13:08:49 +0100 Subject: [PATCH 14/45] linting --- src/gapc.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/gapc.cc b/src/gapc.cc index 183054093..ded2f08d8 100644 --- a/src/gapc.cc +++ b/src/gapc.cc @@ -773,8 +773,10 @@ class Main { back(r.second, r.first); } else if (opts.derivative > 1) { - // split algebra product "left * right" into two instances for first and second derivative - std::pair bothD = driver.ast.split_instance_for_derivatives(opts.instance); + // split algebra product "left * right" into two instances for first + // and second derivative + std::pair bothD = + driver.ast.split_instance_for_derivatives(opts.instance); // store user provided file name pattern for .hh and .cc std::string orig_header_file = opts.header_file; @@ -802,10 +804,10 @@ class Main { driver.ast.current_derivative = 2; // prepend "_derivative2" to generated .hh and .cc file - opts.header_file = basename(orig_header_file) + "_derivative" + - std::to_string(driver.ast.current_derivative) + ".hh"; - opts.out_file = basename(orig_out_file) + "_derivative" + - std::to_string(driver.ast.current_derivative) + ".cc"; + opts.header_file = basename(orig_header_file) + "_derivative" + + std::to_string(driver.ast.current_derivative) + ".hh"; + opts.out_file = basename(orig_out_file) + "_derivative" + + std::to_string(driver.ast.current_derivative) + ".cc"; back(bothD.second); From 88dc258bf2ab6d64171444d34810944755bf617e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 13:09:28 +0100 Subject: [PATCH 15/45] linting --- src/symbol.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/symbol.cc b/src/symbol.cc index 8e2a8236c..3df18a864 100644 --- a/src/symbol.cc +++ b/src/symbol.cc @@ -1092,8 +1092,7 @@ void Symbol::NT::init_table_decl(const AST &ast) { tg.set_window_mode(ast.window_mode); table_decl = tg.create( *this, t, ast.code_mode() == Code::Mode::CYK, - this->is_partof_outside ? 0 : ast.current_derivative - ); + this->is_partof_outside ? 0 : ast.current_derivative); } #include From 24d0384c44a2c18a1922ca52523a5c52090a9c7d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 13:10:01 +0100 Subject: [PATCH 16/45] linting --- src/tablegen.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tablegen.cc b/src/tablegen.cc index edd8e646e..7ecd473c9 100644 --- a/src/tablegen.cc +++ b/src/tablegen.cc @@ -515,7 +515,7 @@ Fn_Def *Tablegen::gen_set_traces(int forDerivative) { std::string *fn_norm_name = new std::string("normalize_traces"); if (forDerivative == 2) { - fn_norm_name = new std::string("soft_max_hessian_product"); + fn_norm_name = new std::string("soft_max_hessian_product"); } Expr::Fn_Call *rhs_norm = new Expr::Fn_Call(fn_norm_name); rhs_norm->add_arg(new Var_Acc::Array(new Var_Acc::Plain( From b9718123f66cacdc7c0b9fe954014feb750a596f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 13:53:34 +0100 Subject: [PATCH 17/45] also extend out_main.cc --- rtlib/generic_main.cc | 12 ++++++++++++ src/cpp.cc | 24 +++++++++++++++--------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/rtlib/generic_main.cc b/rtlib/generic_main.cc index 0d33744a9..234f74a94 100644 --- a/rtlib/generic_main.cc +++ b/rtlib/generic_main.cc @@ -46,6 +46,9 @@ int main(int argc, char **argv) { std::exit(1); } gapc::class_name obj; +#ifdef SECOND_DERIVATIVE + out_derivative2 obj_D2; +#endif try { obj.init(opts); @@ -116,6 +119,15 @@ int main(int argc, char **argv) { gapc::add_event("end"); #endif +#ifdef SECOND_DERIVATIVE + obj_D2.init(opts, &obj); + gapc::add_event("start second derivative"); + obj_D2.run(); + gapc::add_event("end_computation of second derivative"); + obj_D2.report_derivative(std::cout); + gapc::add_event("end_result of second derivative"); +#endif + #ifdef STATS obj.print_stats(std::cerr); #endif diff --git a/src/cpp.cc b/src/cpp.cc index eb07ad196..c8411281b 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -1711,8 +1711,10 @@ void Printer::Cpp::header(const AST &ast) { } if ((*ast.grammar()).is_outside()) { stream << "#define OUTSIDE\n"; - if (ast.current_derivative > 0) { + if (ast.current_derivative == 1) { stream << "#define DERIVATIVES\n"; + } else if (ast.current_derivative == 2) { + stream << "#define SECOND_DERIVATIVE\n"; } } includes(); @@ -2834,17 +2836,21 @@ void Printer::Cpp::close_class() { void Printer::Cpp::typedefs(Code::Gen &code, unsigned int current_derivative) { + stream << "#ifndef NO_GAPC_TYPEDEFS" << endl; + stream << indent() << "namespace gapc {" << endl; + inc_indent(); + stream << indent() << "typedef " << class_name << " class_name"; + if (current_derivative >= 2) { + stream << "_D2"; + } + stream << ";" << endl; if (current_derivative <= 1) { - stream << "#ifndef NO_GAPC_TYPEDEFS" << endl; - stream << indent() << "namespace gapc {" << endl; - inc_indent(); - stream << indent() << "typedef " << class_name << " class_name;" << endl; stream << indent() << "typedef " << *code.return_type() << " return_type;" << endl; - dec_indent(); - stream << indent() << '}' << endl; - stream << "#endif" << endl; } + dec_indent(); + stream << indent() << '}' << endl; + stream << "#endif" << endl; stream << endl; stream << "#endif" << endl; } @@ -2911,7 +2917,7 @@ void Printer::Cpp::makefile(const Options &opts, const AST &ast) { std::string base = opts.class_name; // basename(opts.out_file); std::string out_file = ""; if (ast.requested_derivative > 0) { - for (unsigned int i = 1; i < ast.requested_derivative; ++i) { + for (unsigned int i = 1; i <= ast.requested_derivative; ++i) { out_file += basename(remove_dir(opts.out_file)) + \ "_derivative" + std::to_string(i) + ".cc "; } From 2fdb6c371777f93ef7566d8d08b75e54751bc50f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 14:27:41 +0100 Subject: [PATCH 18/45] include derivative header --- src/cpp.cc | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index c8411281b..2293f373b 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2941,18 +2941,22 @@ void Printer::Cpp::makefile(const Options &opts, const AST &ast) { } stream << endl << endl - << base << "_main.cc : $(RTLIB)/generic_main.cc " << out_file << endl - << "\techo '#include "; + << base << "_main.cc : $(RTLIB)/generic_main.cc " << out_file << endl; if (ast.requested_derivative > 0) { - for (unsigned int i = 1; i < ast.requested_derivative; ++i) { - stream << "\"" << basename(remove_dir(opts.out_file)) << "_derivative" - << std::to_string(i) << ".hh\""; + for (unsigned int i = 1; i <= ast.requested_derivative; ++i) { + stream << "\techo '#include \"" << basename(remove_dir(opts.out_file)) + << "_derivative" << std::to_string(i) << ".hh\"' >"; + if (i > 1) { + stream << ">"; + } + stream << " $@" << endl; } } else { - stream << "\"" << header_file << "\""; + stream << "\techo '#include " << "\"" << header_file << "\"" + << "' > $@" << endl; } - stream << "' > $@" << endl - << "\tcat $(RTLIB)/generic_main.cc >> " << base << "_main.cc" << endl + + stream << "\tcat $(RTLIB)/generic_main.cc >> " << base << "_main.cc" << endl << endl; stream << deps << endl; stream << ".PHONY: clean" << endl << "clean:" << endl From edfc2e983532d0a8c7f20ead7a523e88e6d22c72 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 2 Dec 2022 14:31:12 +0100 Subject: [PATCH 19/45] use out name indepentend class name --- rtlib/generic_main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtlib/generic_main.cc b/rtlib/generic_main.cc index 234f74a94..d428575d4 100644 --- a/rtlib/generic_main.cc +++ b/rtlib/generic_main.cc @@ -47,7 +47,7 @@ int main(int argc, char **argv) { } gapc::class_name obj; #ifdef SECOND_DERIVATIVE - out_derivative2 obj_D2; + gapc::class_name_D2 obj_D2; #endif try { From 55518163f3286fd37940f2eec341656c5e07d419 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 5 Dec 2022 15:14:16 +0100 Subject: [PATCH 20/45] normal filenames for first derivative --- src/cpp.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cpp.cc b/src/cpp.cc index 2293f373b..a21cbdfad 100644 --- a/src/cpp.cc +++ b/src/cpp.cc @@ -2916,7 +2916,7 @@ void Printer::Cpp::makefile(const Options &opts, const AST &ast) { std::string base = opts.class_name; // basename(opts.out_file); std::string out_file = ""; - if (ast.requested_derivative > 0) { + if (ast.requested_derivative > 1) { for (unsigned int i = 1; i <= ast.requested_derivative; ++i) { out_file += basename(remove_dir(opts.out_file)) + \ "_derivative" + std::to_string(i) + ".cc "; @@ -2942,7 +2942,7 @@ void Printer::Cpp::makefile(const Options &opts, const AST &ast) { stream << endl << endl << base << "_main.cc : $(RTLIB)/generic_main.cc " << out_file << endl; - if (ast.requested_derivative > 0) { + if (ast.requested_derivative > 1) { for (unsigned int i = 1; i <= ast.requested_derivative; ++i) { stream << "\techo '#include \"" << basename(remove_dir(opts.out_file)) << "_derivative" << std::to_string(i) << ".hh\"' >"; From d0d6bd93b43ef4c02db019a87d8a7804ea27101f Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 5 Dec 2022 15:47:01 +0100 Subject: [PATCH 21/45] use sum instead of expsum (since exp is computed in alg fns) --- testdata/grammar_outside/alignments.gap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testdata/grammar_outside/alignments.gap b/testdata/grammar_outside/alignments.gap index 7b3aeced1..1965a1bea 100644 --- a/testdata/grammar_outside/alignments.gap +++ b/testdata/grammar_outside/alignments.gap @@ -149,7 +149,7 @@ algebra alg_score implements sig_alignments(alphabet=char, answer=float) { algebra alg_jacobian extends alg_score { choice [float] h([float] candidates) { - return list(expsum(candidates)); + return list(sum(candidates)); } } From 69565fb6ea0677df8fff9b459a52d599876be0ff Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Mon, 5 Dec 2022 15:57:16 +0100 Subject: [PATCH 22/45] activate tests for NW second derivative --- testdata/regresstest/config | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 46faa52ed..52891f200 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -496,3 +496,9 @@ check_new_old_eq_twotrack alignments.gap unused firstD "frzeitei" nwjamiederiv " check_new_old_eq_twotrack alignments.gap unused firstD_gotoh "freizunt" gotohderiv "frnt" check_new_old_eq nodangle.gap unused pfunc "GCaaaGC" nodanglederiv check_new_old_eq nodangle.gap unused pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" nodanglederivlong + +# tests for second derivative computation +GRAMMAR=../../grammar_outside +GAPC="../../../gapc --derivative 2" +check_new_old_eq_twotrack alignments.gap unused bothD "aaaa" secondderiv "bbbb" # validated against jupyter python version +check_new_old_eq_twotrack alignments.gap unused bothD "frzeitei" nwjamie2deriv "zeit" # validated against jupyter python version From 10217d5a79473c2af918efe1969237b9deb4eff7 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Tue, 6 Dec 2022 14:28:00 +0100 Subject: [PATCH 23/45] pass info if symbol is part of outside to return declaration, such that unused variable does NOT get generated --- src/alt.cc | 12 +++++++----- src/alt.hh | 3 ++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 2d2355c57..bcd61ac80 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -1905,9 +1905,11 @@ void Alt::Simple::init_guards() { } -void Alt::Base::push_back_ret_decl(unsigned int current_derivative) { +void Alt::Base::push_back_ret_decl(unsigned int current_derivative, + bool outside_generation) { statements.push_back(ret_decl); - if (top_level && current_derivative > 1 && !is_partof_outside) { + if (top_level && current_derivative > 1 + && !is_partof_outside && !outside_generation) { Statement::Var_Decl *decl_edgeweight = new Statement::Var_Decl( ret_decl->type, new std::string("edgeweight_" + *ret_decl->name)); decl_edgeweight->rhs = new Expr::Const(1.0); @@ -2270,7 +2272,7 @@ void Alt::Simple::codegen(AST &ast, Symbol::NT &calling_nt) { } statements.clear(); - push_back_ret_decl(ast.current_derivative); + push_back_ret_decl(ast.current_derivative, calling_nt.is_partof_outside); std::list *stmts = &statements; init_guards(); @@ -2464,7 +2466,7 @@ void Alt::Link::codegen(AST &ast, Symbol::NT &calling_nt) { // std::cout << "link " << *name << std::endl; statements.clear(); - push_back_ret_decl(ast.current_derivative); + push_back_ret_decl(ast.current_derivative, calling_nt.is_partof_outside); std::string *s = NULL; if (nt->is(Symbol::TERMINAL)) { s = name; @@ -2542,7 +2544,7 @@ void Alt::Link::codegen(AST &ast, Symbol::NT &calling_nt) { void Alt::Block::codegen(AST &ast, Symbol::NT &calling_nt) { // std::cout << "-----------------Block " << std::endl; statements.clear(); - push_back_ret_decl(ast.current_derivative); + push_back_ret_decl(ast.current_derivative, calling_nt.is_partof_outside); Statement::Fn_Call *fn = new Statement::Fn_Call(Statement::Fn_Call::EMPTY); fn->add_arg(*ret_decl); statements.push_back(fn); diff --git a/src/alt.hh b/src/alt.hh index 1960825bf..02e73628a 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -245,7 +245,8 @@ class Base { protected: Statement::If *filter_guards; - void push_back_ret_decl(unsigned int current_derivative); + void push_back_ret_decl(unsigned int current_derivative, + bool outside_generation); Expr::Base *suchthat_code(Statement::Var_Decl &decl) const; From 020e565587f4d7277f71968e0486aaa89af36aad Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 7 Dec 2022 11:53:40 +0100 Subject: [PATCH 24/45] additional instances defined for testing --- testdata/grammar_outside/alignments.gap | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testdata/grammar_outside/alignments.gap b/testdata/grammar_outside/alignments.gap index 1965a1bea..381317b23 100644 --- a/testdata/grammar_outside/alignments.gap +++ b/testdata/grammar_outside/alignments.gap @@ -217,8 +217,11 @@ grammar gra_needlemanwunsch uses sig_alignments(axiom=A) { } instance count = gra_needlemanwunsch(alg_count); +instance count_gotoh = gra_gotoh(alg_count); +instance enum_gotoh = gra_gotoh(alg_enum); instance sim_enum = gra_needlemanwunsch(alg_similarity * alg_enum); instance firstD = gra_needlemanwunsch(alg_score); instance firstD_gotoh = gra_gotoh(alg_score); -instance bothD = gra_needlemanwunsch(alg_jacobian * alg_hessian); \ No newline at end of file +instance bothD = gra_needlemanwunsch(alg_jacobian * alg_hessian); +instance bothD_gotoh = gra_gotoh(alg_jacobian * alg_hessian); \ No newline at end of file From 09a7b0525654c1d656b1f54596002647ccc50873 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 7 Dec 2022 11:53:56 +0100 Subject: [PATCH 25/45] definition of algebra for second derivative --- .../grammar_outside/elmamun_derivatives.gap | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/testdata/grammar_outside/elmamun_derivatives.gap b/testdata/grammar_outside/elmamun_derivatives.gap index fd28fbb6a..e3a19bbf2 100644 --- a/testdata/grammar_outside/elmamun_derivatives.gap +++ b/testdata/grammar_outside/elmamun_derivatives.gap @@ -36,6 +36,30 @@ algebra alg_score implements sig_elmamun(alphabet=char, answer=float) { } } +algebra alg_hessians implements sig_elmamun(alphabet=char, answer=float) { + float number(int value) { + return 0.0; + } + float add(float left, char opSymbol, float right) { + return left + right + 2.0; + } + float heinz(float left, Rope opSymbol, float right) { + return left + right; + } + float mult(float left, char opSymbol, float right) { + return left + right + 3.0; + } + float minus(float left, char opSymbol, float right) { + return left + right; + } + float nil(void) { + return 0.0; + } + choice [float] h([float] candidates) { + return list(sum(candidates)); + } +} + grammar gra_elmamun uses sig_elmamun(axiom = formula) { formula = number(INT) @@ -46,3 +70,4 @@ grammar gra_elmamun uses sig_elmamun(axiom = formula) { } instance firstD = gra_elmamun(alg_score); +instance bothD = gra_elmamun(alg_score * alg_hessians); From 7c103024d9a85c3288d941fdcd1f000f088186e4 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 7 Dec 2022 11:55:33 +0100 Subject: [PATCH 26/45] stop if edgeweight is empty --- src/alt.cc | 84 +++++++++++++++++++++++++++++++++++++++++++----------- src/alt.hh | 2 ++ 2 files changed, 70 insertions(+), 16 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index bcd61ac80..2755c5329 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -1471,11 +1471,17 @@ Expr::Base *Alt::Base::inject_derivative_body(AST &ast, // results of adjoint backward Expr::Fn_Call *fn_e2 = dynamic_cast( outside_fn_arg->ret_decl->rhs); + /* if rhs is a direct link to another NT, outside_fn_arg->ret_decl->rhs + * is empty and we need to resort to outside_arg */ + if (!fn_e2) { + fn_e2 = dynamic_cast(outside_arg); + } // results of backward Expr::Fn_Call *fn_e1 = new Expr::Fn_Call(new std::string( "derivative" + std::to_string(ast.current_derivative-1) + "->" + *(fn_e2->name))); + //"stefan")); fn_e1->add(fn_e2->exprs); Expr::Fn_Call *fn_q1 = new Expr::Fn_Call(new std::string(*(fn_call->name))); @@ -1489,7 +1495,12 @@ Expr::Base *Alt::Base::inject_derivative_body(AST &ast, fn_q1->exprs.end(), std::next(fn_call->exprs.begin()), fn_call->exprs.end()); - fn_q1->add_arg(outside_fn_arg->ret_decl->name); + // don't access ret_X if it gets defined in the very same statement + if (outside_arg->is(Expr::VACC)) { + fn_q1->add_arg(outside_fn_arg->ret_decl->name); + } else { + fn_q1->add_arg(new Expr::Const(0.0)); + } fn_call->exprs.push_back(fn_e1); return new Expr::Plus(fn_call, fn_q1); @@ -1554,7 +1565,7 @@ std::list *Alt::Simple::derivative_collect_traces( Statement::Var_Decl *q = new Statement::Var_Decl( decl->return_type, - new std::string("edgeweight_" + *ret_decl->name)); + this->edgeweight_decl->name); q->rhs = fn_call; Statement::Var_Assign *stmt_ass = new Statement::Var_Assign( *q, new Expr::Times(new Expr::Vacc(*q), fn_call)); @@ -1648,8 +1659,19 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { stmts->insert(stmts->end(), stmts_qs->begin(), stmts_qs->end()); // multiply combined q with nt result - vdecl->rhs = new Expr::Times(new Expr::Vacc(new std::string( - "edgeweight_" + *ret_decl->name)), fn_call); + vdecl->rhs = new Expr::Times(new Expr::Vacc(this->edgeweight_decl->name), fn_call); +// if (ast.current_derivative == 2) { +// Expr::Fn_Call *e = new Expr::Fn_Call(Expr::Fn_Call::NOT_EMPTY); +// e->add_arg(vdecl->name); +// Statement::If *cond = new Statement::If(e); +// cond->then.push_back(vdecl); +// Statement::Fn_Call *erase = +// new Statement::Fn_Call(Statement::Fn_Call::ERASE); +// erase->add_arg(vdecl->name); +// cond->els.push_back(erase); +// +// //vdecl = cond; +// } } else { vdecl->rhs = fn_call; } @@ -1659,18 +1681,31 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { fn->add_arg(*vdecl); stmts->push_back(vdecl); init_derivative_recording(ast, vdecl->name); + std::list *stmts_cmp_push = new std::list(); Expr::Base *suchthat = suchthat_code(*vdecl); if (suchthat) { Statement::If *c = new Statement::If(suchthat); c->then.push_back(fn); c->then.insert(c->then.end(), this->derivative_statements.begin(), this->derivative_statements.end()); - stmts->push_back(c); + stmts_cmp_push->push_back(c); } else { - stmts->push_back(fn); - stmts->insert(stmts->end(), this->derivative_statements.begin(), - this->derivative_statements.end()); + stmts_cmp_push->push_back(fn); + stmts_cmp_push->insert(stmts_cmp_push->end(), + this->derivative_statements.begin(), + this->derivative_statements.end()); } + if ((ast.current_derivative == 2) && !this->get_is_partof_outside()) { + Expr::Fn_Call *e = new Expr::Fn_Call(Expr::Fn_Call::NOT_EMPTY); + e->add_arg(vdecl->name); + Statement::If *cond_edge_empty = new Statement::If(e); + cond_edge_empty->then.insert(cond_edge_empty->then.begin(), + stmts_cmp_push->begin(), stmts_cmp_push->end()); + stmts_cmp_push->clear(); + stmts_cmp_push->push_back(cond_edge_empty); + } + stmts->insert(stmts->end(), + stmts_cmp_push->begin(), stmts_cmp_push->end()); // clear this list, as it has just been added to the statements this->derivative_statements.clear(); } else { @@ -1690,22 +1725,39 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { ast, calling_nt); stmts->insert(stmts->end(), stmts_qs->begin(), stmts_qs->end()); // multiply combined q with nt result - ass->rhs = new Expr::Times(new Expr::Vacc(new std::string( - "edgeweight_" + *ret_decl->name)), fn_call); + ass->rhs = new Expr::Times(new Expr::Vacc(this->edgeweight_decl->name), fn_call); } else { ass->rhs = fn_call; } // derviative statements will later be added in Symbol::NT::codegen init_derivative_recording(ast, ret_decl->name); - stmts->push_back(ass); + // helper list of statements, to allow wrapping with IF condition + // in case of second derivative generation to check if edge weight + // is empty + std::list *stmts_cmp_push = new std::list(); + stmts_cmp_push->push_back(ass); Expr::Base *suchthat = suchthat_code(*ret_decl); if (suchthat) { Statement::If *c = new Statement::If(suchthat); Statement::Fn_Call *e = new Statement::Fn_Call(Statement::Fn_Call::EMPTY); e->add_arg(*ret_decl); c->els.push_back(e); - stmts->push_back(c); + stmts_cmp_push->push_back(c); + } + if ((ast.current_derivative == 2) && !this->get_is_partof_outside()) { + Expr::Fn_Call *e = new Expr::Fn_Call(Expr::Fn_Call::NOT_EMPTY); + e->add_arg(this->edgeweight_decl->name); + Statement::If *cond_edge_empty = new Statement::If(e); + cond_edge_empty->then.insert(cond_edge_empty->then.begin(), + stmts_cmp_push->begin(), stmts_cmp_push->end()); + Statement::Fn_Call *erase = new Statement::Fn_Call(Statement::Fn_Call::EMPTY); + erase->add_arg(ret_decl->name); + cond_edge_empty->els.push_back(erase); + stmts_cmp_push->clear(); + stmts_cmp_push->push_back(cond_edge_empty); } + stmts->insert(stmts->end(), + stmts_cmp_push->begin(), stmts_cmp_push->end()); } } @@ -1774,7 +1826,7 @@ void Alt::Base::init_derivative_recording( if (ast.current_derivative == 1) { x->add_arg(result_name); } else { - x->add_arg(new std::string("edgeweight_" + *result_name)); + x->add_arg(this->edgeweight_decl->name); Statement::Fn_Call *y = new Statement::Fn_Call("set_q"); y->add_arg(new std::string("cand")); @@ -1910,10 +1962,10 @@ void Alt::Base::push_back_ret_decl(unsigned int current_derivative, statements.push_back(ret_decl); if (top_level && current_derivative > 1 && !is_partof_outside && !outside_generation) { - Statement::Var_Decl *decl_edgeweight = new Statement::Var_Decl( + this->edgeweight_decl = new Statement::Var_Decl( ret_decl->type, new std::string("edgeweight_" + *ret_decl->name)); - decl_edgeweight->rhs = new Expr::Const(1.0); - statements.push_back(decl_edgeweight); + this->edgeweight_decl->rhs = new Expr::Const(1.0); + statements.push_back(this->edgeweight_decl); } } diff --git a/src/alt.hh b/src/alt.hh index 02e73628a..f1dad9275 100644 --- a/src/alt.hh +++ b/src/alt.hh @@ -144,6 +144,8 @@ class Base { public: Statement::Var_Decl *ret_decl; + // for derivative computation: stores the edge weight of lower derivatives: q + Statement::Var_Decl *edgeweight_decl; inline bool is(Type t) { return type == t; From 3e23ccc2f5d9efc344087ada61b0269ffbb07b1e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 7 Dec 2022 11:56:32 +0100 Subject: [PATCH 27/45] test for gotoh grammar --- testdata/regresstest/config | 1 + 1 file changed, 1 insertion(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 52891f200..7b5657aea 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -502,3 +502,4 @@ GRAMMAR=../../grammar_outside GAPC="../../../gapc --derivative 2" check_new_old_eq_twotrack alignments.gap unused bothD "aaaa" secondderiv "bbbb" # validated against jupyter python version check_new_old_eq_twotrack alignments.gap unused bothD "frzeitei" nwjamie2deriv "zeit" # validated against jupyter python version +check_new_old_eq_twotrack alignments.gap unused bothD_gotoh "frzeitei" gotoh2deriv "zeit" From 19d9632ec457a1692a8b7d7d1a355a5244f6627a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 7 Dec 2022 17:13:19 +0100 Subject: [PATCH 28/45] remove duplicate algebra --- .../grammar_outside/elmamun_derivatives.gap | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/testdata/grammar_outside/elmamun_derivatives.gap b/testdata/grammar_outside/elmamun_derivatives.gap index f96200261..10c302750 100644 --- a/testdata/grammar_outside/elmamun_derivatives.gap +++ b/testdata/grammar_outside/elmamun_derivatives.gap @@ -60,31 +60,6 @@ algebra alg_hessians implements sig_elmamun(alphabet=char, answer=float) { } } -algebra alg_hessians implements sig_elmamun(alphabet=char, answer=float) { - float number(int value) { - return 0.0; - } - float add(float left, char opSymbol, float right) { - return left + right + 2.0; - } - float heinz(float left, Rope opSymbol, float right) { - return left + right; - } - float mult(float left, char opSymbol, float right) { - return left + right + 3.0; - } - float minus(float left, char opSymbol, float right) { - return left + right; - } - float nil(void) { - return 0.0; - } - choice [float] h([float] candidates) { - return list(sum(candidates)); - } -} - - grammar gra_elmamun uses sig_elmamun(axiom = formula) { formula = number(INT) | add(formula, CHAR('+'), formula) From 8fc9e907171eda7fea10a63c80416a8e79f2eb57 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 7 Dec 2022 17:13:35 +0100 Subject: [PATCH 29/45] always initialize edgeweight declarations --- src/alt.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index e0c38d5ec..3c9291777 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -57,7 +57,7 @@ Alt::Base::Base(Type t, const Loc &l) : productive(false), datatype(NULL), eliminated(false), terminal_type(false), location(l), - ret_decl(NULL), filter_guards(NULL), + ret_decl(NULL), edgeweight_decl(NULL), filter_guards(NULL), choice_fn_type_(Expr::Fn_Call::NONE), tracks_(0), track_pos_(0), is_partof_outside(false) { } @@ -1278,6 +1278,8 @@ void Alt::Base::init_ret_decl(unsigned int i, const std::string &prefix) { std::ostringstream o; o << prefix << "ret_" << i; ret_decl = new Statement::Var_Decl(datatype, new std::string(o.str())); + edgeweight_decl = new Statement::Var_Decl(datatype, + new std::string("edgeweight_" + *ret_decl->name)); } @@ -1965,8 +1967,6 @@ void Alt::Base::push_back_ret_decl(unsigned int current_derivative, statements.push_back(ret_decl); if (top_level && current_derivative > 1 && !is_partof_outside && !outside_generation) { - this->edgeweight_decl = new Statement::Var_Decl( - ret_decl->type, new std::string("edgeweight_" + *ret_decl->name)); this->edgeweight_decl->rhs = new Expr::Const(1.0); statements.push_back(this->edgeweight_decl); } From ca5ae7587552abce9cf62ab37d8b985eeba818be Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 7 Dec 2022 17:20:35 +0100 Subject: [PATCH 30/45] linting --- src/alt.cc | 48 +++++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 3c9291777..fe4d49901 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -1483,7 +1483,6 @@ Expr::Base *Alt::Base::inject_derivative_body(AST &ast, Expr::Fn_Call *fn_e1 = new Expr::Fn_Call(new std::string( "derivative" + std::to_string(ast.current_derivative-1) + "->" + *(fn_e2->name))); - //"stefan")); fn_e1->add(fn_e2->exprs); Expr::Fn_Call *fn_q1 = new Expr::Fn_Call(new std::string(*(fn_call->name))); @@ -1661,18 +1660,19 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { stmts->insert(stmts->end(), stmts_qs->begin(), stmts_qs->end()); // multiply combined q with nt result - vdecl->rhs = new Expr::Times(new Expr::Vacc(this->edgeweight_decl->name), fn_call); + vdecl->rhs = new Expr::Times(new Expr::Vacc(this->edgeweight_decl->name), + fn_call); // if (ast.current_derivative == 2) { -// Expr::Fn_Call *e = new Expr::Fn_Call(Expr::Fn_Call::NOT_EMPTY); -// e->add_arg(vdecl->name); -// Statement::If *cond = new Statement::If(e); -// cond->then.push_back(vdecl); -// Statement::Fn_Call *erase = -// new Statement::Fn_Call(Statement::Fn_Call::ERASE); -// erase->add_arg(vdecl->name); -// cond->els.push_back(erase); +// Expr::Fn_Call *e = new Expr::Fn_Call(Expr::Fn_Call::NOT_EMPTY); +// e->add_arg(vdecl->name); +// Statement::If *cond = new Statement::If(e); +// cond->then.push_back(vdecl); +// Statement::Fn_Call *erase = +// new Statement::Fn_Call(Statement::Fn_Call::ERASE); +// erase->add_arg(vdecl->name); +// cond->els.push_back(erase); // -// //vdecl = cond; +// //vdecl = cond; // } } else { vdecl->rhs = fn_call; @@ -1683,7 +1683,8 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { fn->add_arg(*vdecl); stmts->push_back(vdecl); init_derivative_recording(ast, vdecl->name); - std::list *stmts_cmp_push = new std::list(); + std::list *stmts_cmp_push = + new std::list(); Expr::Base *suchthat = suchthat_code(*vdecl); if (suchthat) { Statement::If *c = new Statement::If(suchthat); @@ -1694,7 +1695,7 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { } else { stmts_cmp_push->push_back(fn); stmts_cmp_push->insert(stmts_cmp_push->end(), - this->derivative_statements.begin(), + this->derivative_statements.begin(), this->derivative_statements.end()); } if ((ast.current_derivative == 2) && !this->get_is_partof_outside()) { @@ -1702,12 +1703,13 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { e->add_arg(vdecl->name); Statement::If *cond_edge_empty = new Statement::If(e); cond_edge_empty->then.insert(cond_edge_empty->then.begin(), - stmts_cmp_push->begin(), stmts_cmp_push->end()); + stmts_cmp_push->begin(), + stmts_cmp_push->end()); stmts_cmp_push->clear(); stmts_cmp_push->push_back(cond_edge_empty); } stmts->insert(stmts->end(), - stmts_cmp_push->begin(), stmts_cmp_push->end()); + stmts_cmp_push->begin(), stmts_cmp_push->end()); // clear this list, as it has just been added to the statements this->derivative_statements.clear(); } else { @@ -1727,7 +1729,8 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { ast, calling_nt); stmts->insert(stmts->end(), stmts_qs->begin(), stmts_qs->end()); // multiply combined q with nt result - ass->rhs = new Expr::Times(new Expr::Vacc(this->edgeweight_decl->name), fn_call); + ass->rhs = new Expr::Times(new Expr::Vacc(this->edgeweight_decl->name), + fn_call); } else { ass->rhs = fn_call; } @@ -1736,7 +1739,8 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { // helper list of statements, to allow wrapping with IF condition // in case of second derivative generation to check if edge weight // is empty - std::list *stmts_cmp_push = new std::list(); + std::list *stmts_cmp_push = + new std::list(); stmts_cmp_push->push_back(ass); Expr::Base *suchthat = suchthat_code(*ret_decl); if (suchthat) { @@ -1751,15 +1755,17 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { e->add_arg(this->edgeweight_decl->name); Statement::If *cond_edge_empty = new Statement::If(e); cond_edge_empty->then.insert(cond_edge_empty->then.begin(), - stmts_cmp_push->begin(), stmts_cmp_push->end()); - Statement::Fn_Call *erase = new Statement::Fn_Call(Statement::Fn_Call::EMPTY); + stmts_cmp_push->begin(), + stmts_cmp_push->end()); + Statement::Fn_Call *erase = new Statement::Fn_Call( + Statement::Fn_Call::EMPTY); erase->add_arg(ret_decl->name); cond_edge_empty->els.push_back(erase); stmts_cmp_push->clear(); stmts_cmp_push->push_back(cond_edge_empty); } stmts->insert(stmts->end(), - stmts_cmp_push->begin(), stmts_cmp_push->end()); + stmts_cmp_push->begin(), stmts_cmp_push->end()); } } @@ -1967,7 +1973,7 @@ void Alt::Base::push_back_ret_decl(unsigned int current_derivative, statements.push_back(ret_decl); if (top_level && current_derivative > 1 && !is_partof_outside && !outside_generation) { - this->edgeweight_decl->rhs = new Expr::Const(1.0); + this->edgeweight_decl->rhs = new Expr::Const(1.0); statements.push_back(this->edgeweight_decl); } } From d7dd8a05e9efec4efb35c671d55c416ab268285d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 8 Dec 2022 13:56:23 +0100 Subject: [PATCH 31/45] don't check edgeweight for terminals --- src/alt.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index fe4d49901..d5861d411 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -1723,7 +1723,7 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { outside_arg); } else if ((ast.current_derivative == 2) && !calling_nt.is_partof_outside && - !outside_fn_arg) { + !outside_fn_arg && !is_terminal()) { // obtain edge weight q std::list *stmts_qs = derivative_collect_traces( ast, calling_nt); @@ -1745,12 +1745,14 @@ void Alt::Simple::init_body(AST &ast, Symbol::NT &calling_nt) { Expr::Base *suchthat = suchthat_code(*ret_decl); if (suchthat) { Statement::If *c = new Statement::If(suchthat); - Statement::Fn_Call *e = new Statement::Fn_Call(Statement::Fn_Call::EMPTY); + Statement::Fn_Call *e = new Statement::Fn_Call( + Statement::Fn_Call::EMPTY); e->add_arg(*ret_decl); c->els.push_back(e); stmts_cmp_push->push_back(c); } - if ((ast.current_derivative == 2) && !this->get_is_partof_outside()) { + if ((ast.current_derivative == 2) && !this->get_is_partof_outside() && + !is_terminal()) { Expr::Fn_Call *e = new Expr::Fn_Call(Expr::Fn_Call::NOT_EMPTY); e->add_arg(this->edgeweight_decl->name); Statement::If *cond_edge_empty = new Statement::If(e); From 0ab185bf189dd2bf8e09f5c7660cc5e006b9091e Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Thu, 8 Dec 2022 19:12:54 +0100 Subject: [PATCH 32/45] mixed up caller and called NT --- src/alt.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alt.cc b/src/alt.cc index d5861d411..1ec13cd9b 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -1535,7 +1535,7 @@ std::list *Alt::Simple::derivative_collect_traces( fn_call->add_arg(new std::string( "derivative" + std::to_string(ast.current_derivative-1) + - "->" + alt_nt->table_decl->name())); + "->" + calling_nt.table_decl->name())); fn_call->is_obj = Bool(true); // index of the calling non-terminal From cda02a3ea15b8c19c0689bf3390f1f789b7e9a4d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 9 Dec 2022 09:34:25 +0100 Subject: [PATCH 33/45] also activate outside tests --- .github/workflows/outside.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/outside.yml b/.github/workflows/outside.yml index 168daa139..947ed9526 100644 --- a/.github/workflows/outside.yml +++ b/.github/workflows/outside.yml @@ -4,7 +4,7 @@ on: push: branches: [ master ] pull_request: - branches: [ master, outside_looporder ] + branches: [ master, outside_looporder, first_derivative ] workflow_dispatch: inputs: logLevel: From e6257629a2c2331e6f45e5deedb630b8fbacb976 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 9 Dec 2022 09:58:55 +0100 Subject: [PATCH 34/45] propagate top level edgeweight name to lower levels, e.g. cadd(incl(dangle), ml_comps1) --- src/alt.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/alt.cc b/src/alt.cc index 1ec13cd9b..785b4ee9d 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -2043,6 +2043,9 @@ std::list *Alt::Simple::reorder_args_cg( Symbol::NT &calling_nt) { for (std::list::iterator i = args.begin(); i != args.end(); ++i) { + if (this->top_level) { + (*i)->alt_ref()->edgeweight_decl = this->edgeweight_decl; + } (*i)->codegen(ast, calling_nt); } return add_arg_code(ast, x); From c167c2304aab680a6e12a09c8e7adbd54d58917a Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 9 Dec 2022 10:00:26 +0100 Subject: [PATCH 35/45] added a first rudimentary algebra for Hessians --- testdata/grammar_outside/nodangle.gap | 44 +++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/testdata/grammar_outside/nodangle.gap b/testdata/grammar_outside/nodangle.gap index f83293d10..b458b832a 100644 --- a/testdata/grammar_outside/nodangle.gap +++ b/testdata/grammar_outside/nodangle.gap @@ -105,6 +105,49 @@ algebra alg_pfunc implements sig_foldrna(alphabet = char, answer = double) { } } +// similar to alg_mfe, but datatype changed from int to double and h is sum +algebra alg_hessians implements sig_foldrna(alphabet = char, answer = double) { + double sadd(Subsequence lb, double x) { + return x + sbase_energy(); + } + double cadd(double x, double y) { + return x + y; + } + double drem(Subsequence lb, double x, Subsequence rb) { + return x + termau_energy(lb, rb); + } + double sr(Subsequence lb, double x, Subsequence rb) { + return x + sr_energy(lb, rb); + } + double hl(Subsequence lb, Subsequence r, Subsequence rb) { + return hl_energy(r); + } + double bl(Subsequence lb, Subsequence lr, double x, Subsequence rb) { + return x + bl_energy(lr, rb); + } + double br(Subsequence lb, double x, Subsequence rr, Subsequence rb) { + return x + br_energy(lb, rr); + } + double il(Subsequence lb, Subsequence lr, double x, Subsequence rr, Subsequence rb) { + return x + il_energy(lr, rr); + } + double ml(Subsequence lb, double x, Subsequence rb) { + return x + ml_energy() + ul_energy() + termau_energy(lb, rb); + } + double incl(double x) { + return x + ul_energy(); + } + double addss(double x, Subsequence r) { + return x + ss_energy(r); + } + double nil(Subsequence n) { + return 0.0; + } + choice [double] h([double] i) { + return list(sum(i)); + } +} + algebra alg_dotBracket implements sig_foldrna(alphabet = char, answer = string) { string sadd(Subsequence lb,string e) { string res; @@ -233,3 +276,4 @@ grammar gra_nodangle uses sig_foldrna(axiom = struct) { instance mfe = gra_nodangle(alg_mfe); instance pfunc = gra_nodangle(alg_pfunc); instance count = gra_nodangle(alg_count); +instance bothD = gra_nodangle(alg_pfunc * alg_hessians); From 398494dd5a22dffe805475471bdb513279585882 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 9 Dec 2022 10:30:08 +0100 Subject: [PATCH 36/45] adding two new tests --- testdata/regresstest/config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index 7b5657aea..2607a1073 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -503,3 +503,5 @@ GAPC="../../../gapc --derivative 2" check_new_old_eq_twotrack alignments.gap unused bothD "aaaa" secondderiv "bbbb" # validated against jupyter python version check_new_old_eq_twotrack alignments.gap unused bothD "frzeitei" nwjamie2deriv "zeit" # validated against jupyter python version check_new_old_eq_twotrack alignments.gap unused bothD_gotoh "frzeitei" gotoh2deriv "zeit" +check_new_old_eq elmamun_derivatives.gap unused bothD "1+2*3*4+5" secondderiv +check_new_old_eq nodangle.gap unused bothD "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" secondderiv From c8d48115a3b7cd2a5923f68bc4880d1195c5d31d Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 9 Dec 2022 15:56:06 +0100 Subject: [PATCH 37/45] can't get alt, if Fn_Arg is not of type Alt --- src/alt.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alt.cc b/src/alt.cc index e26fb0217..10385ed61 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -2048,7 +2048,7 @@ std::list *Alt::Simple::reorder_args_cg( Symbol::NT &calling_nt) { for (std::list::iterator i = args.begin(); i != args.end(); ++i) { - if (this->top_level) { + if (this->top_level && (*i)->is(Fn_Arg::ALT)) { (*i)->alt_ref()->edgeweight_decl = this->edgeweight_decl; } (*i)->codegen(ast, calling_nt); From 7e97d8c065cd4b8256eea4a484bd68ba78d99933 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 11 Jan 2023 10:33:19 +0100 Subject: [PATCH 38/45] also test PRs against this branch --- .github/workflows/outside.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/outside.yml b/.github/workflows/outside.yml index 947ed9526..adf3e4c75 100644 --- a/.github/workflows/outside.yml +++ b/.github/workflows/outside.yml @@ -4,7 +4,7 @@ on: push: branches: [ master ] pull_request: - branches: [ master, outside_looporder, first_derivative ] + branches: [ master, outside_looporder, first_derivative, second_derivative ] workflow_dispatch: inputs: logLevel: From b1e8f54fa75038a12f7f3a4755e91e417a334431 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Wed, 11 Jan 2023 11:11:42 +0100 Subject: [PATCH 39/45] also run tests for PRs against second_derivative --- .github/workflows/c-cpp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml index e28901b48..e49fe21f9 100644 --- a/.github/workflows/c-cpp.yml +++ b/.github/workflows/c-cpp.yml @@ -4,7 +4,7 @@ on: push: branches: [ master ] pull_request: - branches: [ master, outside_looporder, first_derivative ] + branches: [ master, outside_looporder, first_derivative, second_derivative ] workflow_dispatch: inputs: logLevel: From 3076b1681e2bd8525c3c925db66051db31732e99 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 13 Jan 2023 09:03:39 +0100 Subject: [PATCH 40/45] only check for first (but not second) derivative --- src/gapc.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gapc.cc b/src/gapc.cc index ed9ee18df..f36487174 100644 --- a/src/gapc.cc +++ b/src/gapc.cc @@ -550,8 +550,8 @@ class Main { } } - if (opts.derivative > 0) { - // if user requests derivative computation, check that user also + if (driver.ast.current_derivative == 1) { + // if user requests first derivative computation, check that user also // provided a normalization function for forward computation instance->product->algebra()->check_derivative(); } From 82c90fcda162026a04a4945f89dc7d0e20667508 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 13 Jan 2023 09:04:04 +0100 Subject: [PATCH 41/45] don't pass normalization function for second derivative computation --- src/tablegen.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tablegen.cc b/src/tablegen.cc index 2ced255e3..60c5b77db 100644 --- a/src/tablegen.cc +++ b/src/tablegen.cc @@ -522,8 +522,10 @@ Fn_Def *Tablegen::gen_set_traces(int forDerivative) { new std::string("&traces")), off)); rhs_norm->add_arg(new std::string("candidates")); rhs_norm->add_arg(new std::string("e")); - rhs_norm->add_arg(new std::string("&" + *(new std::string( - FN_NAME_DERIVATIVE_NORMALIZER)))); + if (forDerivative == 1) { + rhs_norm->add_arg(new std::string("&" + *(new std::string( + FN_NAME_DERIVATIVE_NORMALIZER)))); + } Statement::Var_Assign *fn_norm = new Statement::Var_Assign( new Var_Acc::Array(new Var_Acc::Plain(new std::string("traces")), off), From 010069bfd3017df388a1751e1dca097d9d712223 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 13 Jan 2023 09:14:02 +0100 Subject: [PATCH 42/45] adding first (wrong?) version of hessian computation --- .../hmm_sonneregen_properEnd.gap | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/testdata/grammar_outside/hmm_sonneregen_properEnd.gap b/testdata/grammar_outside/hmm_sonneregen_properEnd.gap index b0e2e4f43..a6ad65042 100644 --- a/testdata/grammar_outside/hmm_sonneregen_properEnd.gap +++ b/testdata/grammar_outside/hmm_sonneregen_properEnd.gap @@ -134,6 +134,57 @@ algebra alg_fwd extends alg_viterbi { } } +algebra alg_hessians implements sig_weather(alphabet=char, answer=float) { + float transition_start_hoch(float transition, float emission, float x) { + return 0.48 * emission * x; + } + float transition_start_tief(float transition, float emission, float x) { + return 0.51 * emission * x; + } + float transition_start_ende(float transition, float x) { + return 0.01 * x; + } + float transition_hoch_tief(float transition, float emission, float x) { + return 0.20 * emission * x; + } + float transition_hoch_hoch(float transition, float emission, float x) { + return 0.70 * emission * x; + } + float transition_hoch_ende(float transition, float x) { + return 0.10 * x; + } + float transition_tief_tief(float transition, float emission, float x) { + return 0.50 * emission * x; + } + float transition_tief_hoch(float transition, float emission, float x) { + return 0.40 * emission * x; + } + float transition_tief_ende(float transition, float x) { + return 0.10 * x; + } + + float emission_hoch_sonne(float emission, char a) { + return 0.6; + } + float emission_hoch_regen(float emission, char a) { + return 0.4; + } + float emission_tief_sonne(float emission, char a) { + return 0.5; + } + float emission_tief_regen(float emission, char a) { + return 0.5; + } + float nil(void) { + return 1.0; + } + + choice [float] h([float] candidates) { + return list(sum(candidates)); + } +} + + algebra alg_fwd_log implements sig_weather(alphabet=char, answer=float) { float transition_start_hoch(float transition, float emission, float x) { return log(transition) + emission + x; @@ -469,3 +520,5 @@ instance fwd = gra_weather(alg_fwd); instance fwd_log = gra_weather(alg_fwd_log); instance fwd_neglog = gra_weather(alg_fwd_neglog); instance multviterbistates = gra_weather(alg_mult * alg_viterbi * alg_states); + +instance bothD = gra_weather(alg_fwd * alg_hessians); From 5a0d231b26a3d631c1e02a85b675ef308afd3989 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 13 Jan 2023 18:00:44 +0100 Subject: [PATCH 43/45] recursion base for D1 = 1 but for D2 = 0 --- src/alt.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/alt.cc b/src/alt.cc index 6167b41d8..18c7fb7e4 100644 --- a/src/alt.cc +++ b/src/alt.cc @@ -2597,9 +2597,12 @@ void Alt::Link::codegen(AST &ast, Symbol::NT &calling_nt) { /* In case of first derivatives, we want to normalize all result to * probabilities. Therefore, the recursion base of the outside pass must be * set to 1.0, instead of using the result of the initial inside pass, - * e.g. axiom with complete input substring */ + * e.g. axiom with complete input substring. + * It's similar for second derivatives, but since they are no probabilities + * recursion base must be 0.0 not 1.0. + * */ Expr::Base *fn_or_const = fn; - if ((ast.current_derivative == 1) && + if ((ast.current_derivative >= 1) && (*calling_nt.orig_name == *ast.grammar()->axiom_name_inside)) { unsigned int lacking_complete_tracks = calling_nt.tracks(); for (std::vector >::const_iterator track = \ @@ -2612,7 +2615,11 @@ void Alt::Link::codegen(AST &ast, Symbol::NT &calling_nt) { } } if (lacking_complete_tracks == 0) { - fn_or_const = new Expr::Const(1.0); + if (ast.current_derivative == 1) { + fn_or_const = new Expr::Const(1.0); + } else if (ast.current_derivative == 2) { + fn_or_const = new Expr::Const(0.0); + } } } From 9a5e0027ad2109f4f9bbb31931a8acd65e7c18c1 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 13 Jan 2023 18:07:22 +0100 Subject: [PATCH 44/45] defining new instances for 2D computation with different scoring normalizations --- testdata/grammar_outside/hmm_sonneregen_properEnd.gap | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testdata/grammar_outside/hmm_sonneregen_properEnd.gap b/testdata/grammar_outside/hmm_sonneregen_properEnd.gap index a6ad65042..bda500272 100644 --- a/testdata/grammar_outside/hmm_sonneregen_properEnd.gap +++ b/testdata/grammar_outside/hmm_sonneregen_properEnd.gap @@ -522,3 +522,5 @@ instance fwd_neglog = gra_weather(alg_fwd_neglog); instance multviterbistates = gra_weather(alg_mult * alg_viterbi * alg_states); instance bothD = gra_weather(alg_fwd * alg_hessians); +instance bothD_log = gra_weather(alg_fwd_log * alg_hessians); +instance bothD_neglog = gra_weather(alg_fwd_neglog * alg_hessians); From 1d38e3d234600654593821c5996c895919f54806 Mon Sep 17 00:00:00 2001 From: Stefan Janssen Date: Fri, 13 Jan 2023 18:08:09 +0100 Subject: [PATCH 45/45] three new tests that should all result in the same numbers --- testdata/regresstest/config | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/testdata/regresstest/config b/testdata/regresstest/config index edbf09a96..b298d3026 100644 --- a/testdata/regresstest/config +++ b/testdata/regresstest/config @@ -500,7 +500,6 @@ check_new_old_eq nodangle.gap unused pfunc "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" nodan # results of the below HMM where extensively validated by Stefan via Excel and python implementations CPPFLAGS_EXTRA="$DEFAULT_CPPFLAGS_EXTRA" # for negexpsum function, defined as external *.hh file # tests for special "normalize_derivative" algebra function -check_new_old_eq hmm_sonneregen_properEnd.gap unused fwd "SSRR" sonneregen1deriv check_new_old_eq hmm_sonneregen_properEnd.gap unused fwd "SSRR" sonneregen1deriv # probs - sum check_new_old_eq hmm_sonneregen_properEnd.gap unused fwd_log "SSRR" sonneregen1deriv_log # log - expsum check_new_old_eq hmm_sonneregen_properEnd.gap unused fwd_neglog "SSRR" sonneregen1deriv_neglog # neglog - negexpsum @@ -513,3 +512,8 @@ check_new_old_eq_twotrack alignments.gap unused bothD "frzeitei" nwjamie2deriv " check_new_old_eq_twotrack alignments.gap unused bothD_gotoh "frzeitei" gotoh2deriv "zeit" check_new_old_eq elmamun_derivatives.gap unused bothD "1+2*3*4+5" secondderiv check_new_old_eq nodangle.gap unused bothD "CCaCCaaaGGaCCaaaGGaCCaaaGGaGG" secondderiv + +# despite different score normalizations for jacobians=1D, hessians=2D are the same +check_new_old_eq hmm_sonneregen_properEnd.gap unused bothD "SSRR" sonneregen2deriv_normal # probs - sum +check_new_old_eq hmm_sonneregen_properEnd.gap unused bothD_log "SSRR" sonneregen2deriv_log # probs - sum +check_new_old_eq hmm_sonneregen_properEnd.gap unused bothD_neglog "SSRR" sonneregen2deriv_neglog # probs - sum