Use clucore

eidoscommon/build.sbt

@@ -9,12 +9,11 @@ resolvers ++= Seq(
 
 libraryDependencies ++= {
   // Versions were last checked 2021 Mar 12.
-  val procVer = "8.3.2"
+  val procVer = "8.4.3-SNAPSHOT"
 
   Seq(
     "org.clulab"                 %% "processors-corenlp"       % procVer,       // up to 8.3.0
     "org.clulab"                 %% "processors-main"          % procVer,       // up to 8.3.0
-    "org.clulab"                 %% "processors-odin"          % procVer,       // up to 8.3.0
     // local logging
     "ch.qos.logback"              % "logback-classic"          % "1.0.10",      // up to 1.2.
     "com.typesafe.scala-logging" %% "scala-logging"            % "3.7.2",       // up to 3.9.2

eidoscommon/src/main/scala/org/clulab/wm/eidoscommon/DocumentFilter.scala

@@ -47,7 +47,7 @@ class FilterByLength(processor: Processor, cutoff: Int = 200) extends DocumentFi
     val sanitizedText = sanitizeText(doc)
     val kept = doc.sentences.filter(s => s.words.length < cutoff)
     val skipped = doc.sentences.length - kept.length
-    val newDoc = Document(doc.id, kept, doc.coreferenceChains, doc.discourseTree, sanitizedText)
+    val newDoc = Document(doc.id, kept, doc.coreferenceChains, sanitizedText)
     val newerDoc = // This is a hack for lack of copy constructor for CoreNLPDocument
       if (doc.isInstanceOf[CoreNLPDocument])
         ShallowNLPProcessor.cluDocToCoreDoc(newDoc, keepText = true)

eidoscommon/src/main/scala/org/clulab/wm/eidoscommon/EidosProcessor.scala

@@ -1,8 +1,9 @@
 package org.clulab.wm.eidoscommon
 
+import org.clulab.dynet.Utils
+
 import java.text.Normalizer
 import java.util.regex.Pattern
-
 import org.clulab.processors.Document
 import org.clulab.processors.Processor
 import org.clulab.processors.Sentence
@@ -11,9 +12,11 @@ import org.clulab.processors.clu.SpanishCluProcessor
 import org.clulab.processors.clu.tokenizer.RawToken
 import org.clulab.processors.clu.tokenizer.SentenceSplitter
 import org.clulab.processors.clu.tokenizer.Tokenizer
+import org.clulab.processors.clucore.CluCoreProcessor
 import org.clulab.processors.fastnlp.FastNLPProcessorWithSemanticRoles
 import org.clulab.utils.ScienceUtils
 import org.clulab.wm.eidoscommon.utils.Logging
+import org.clulab.dynet.Utils.initializeDyNet
 
 import scala.collection.mutable.ArrayBuffer
 
@@ -42,8 +45,9 @@ trait LanguageSpecific {
   def getTagSet: TagSet
 }
 
-class EidosEnglishProcessor(val language: String, cutoff: Int) extends FastNLPProcessorWithSemanticRoles
+class EidosEnglishProcessor(val language: String, cutoff: Int) extends CluCoreProcessor
     with EidosProcessor {
+  Utils.initializeDyNet()
   lazy val eidosTokenizer: EidosTokenizer = new EidosTokenizer(localTokenizer, cutoff)
   override lazy val tokenizer: Tokenizer = eidosTokenizer
   val tagSet = new EnglishTagSet()
@@ -114,29 +118,6 @@ class EidosPortugueseProcessor(val language: String, cutoff: Int) extends Portug
   def getTagSet: TagSet = tagSet
 }
 
-class EidosCluProcessor(val language: String, cutoff: Int) extends FastNLPProcessorWithSemanticRoles
-  with EidosProcessor {
-  lazy val eidosTokenizer: EidosTokenizer = new EidosTokenizer(localTokenizer, cutoff)
-  override lazy val tokenizer: Tokenizer = eidosTokenizer
-  val tagSet = new EnglishTagSet()
-
-  def getTokenizer: EidosTokenizer = eidosTokenizer
-
-  // TODO: This should be checked with each update of processors.
-  def extractDocument(text: String): Document = {
-    // This mkDocument will now be subject to all of the EidosProcessor changes.
-    val document = mkDocument(text, keepText = false)
-
-    if (document.sentences.nonEmpty) {
-      tagPartsOfSpeech(document)
-      lemmatize(document)
-      recognizeNamedEntities(document)
-    }
-    document
-  }
-
-  def getTagSet: TagSet = tagSet
-}
 
 class ParagraphSplitter {
   // The idea here is to make sure that a paragraph ends with a complete sentence.
@@ -355,7 +336,6 @@ object EidosProcessor extends Logging {
       new EidosEnglishProcessor(language, cutoff)
     case Language.SPANISH => new EidosSpanishProcessor(language, cutoff)
     case Language.PORTUGUESE => new EidosPortugueseProcessor(language, cutoff)
-    case Language.CLU => new EidosCluProcessor(language, cutoff)
   }
 
   // Turn off warnings from this class.

eidoscommon/src/main/scala/org/clulab/wm/eidoscommon/Language.scala

@@ -5,5 +5,4 @@ object Language {
   val ENGLISH = "english"
   val SPANISH = "spanish"
   val PORTUGUESE = "portuguese"
-  val CLU = "clu"
 }

src/main/resources/eidos.conf

@@ -20,13 +20,13 @@ filtering {
 }
 
 apps {
-  inputDirectory = "/Users/bsharp/data/WM/doc_sample_2020-09-05/jsonld_flat/"
-  outputDirectory = "/Users/bsharp/data/WM/doc_sample_2020-09-05/regrounded_compositional_ranked/"
-  exportAs = ["reground"] // "incdec"
+  inputDirectory = "./debug_clucore/"
+  outputDirectory = "./debug_clucore/"
+//  exportAs = ["reground"] // "incdec"
 
   //inputDirectory = "/Users/bsharp/data/WM/doc_sample_2020-09-05/regrounded_compositional/"
   //outputDirectory = "/Users/bsharp/data/WM/doc_sample_2020-09-05/compositional_debug/"
-  //exportAs = ["debugGrounding"] // "incdec"
+  exportAs = ["debugGrounding"] // "incdec"
 
   outputFile = "../incdec/Doc500.tsv"
   inputFileExtension = ".txt"
@@ -54,7 +54,7 @@ apps {
 
 ontologies {
   // W2V
-  useGrounding = true
+  useGrounding = false
   // wordToVecPath = ${EidosSystem.path}/w2v/vectors
   //wordToVecPath = ${EidosSystem.path}/w2v/glove.840B.300d // Local resource
   //wordToVecPath = /org/clulab/glove/glove.840B.300d // Remote resource
@@ -98,7 +98,7 @@ geonorm {
 }
 
 timenorm {
-  useNeuralParser = false
+  useNeuralParser = true
   timeRegexPath = ${EidosSystem.path}/context/timenorm-regexes.txt
 }
 

src/main/resources/englishActionsExpander.conf

@@ -20,6 +20,7 @@
     "^nmod_as",
     "^nmod_because",
     "^nmod_due_to",
+    "^nmod_due"
     "^nmod_except",
     "^nmod_given",
     "^nmod_since",
@@ -54,6 +55,7 @@
     "nmod_between",
     "nmod_beyond",
     "nmod_compared_to",
+    "nmod_compared",
     "nmod_compared_with",
     "nmod_concerning",
     "nmod_despite",
@@ -107,7 +109,8 @@
     "nmod_with_regard_to",
     "nmod_with_respect_to",
     "nmod_without",
-    "nmod_worsen"
+    "nmod_worsen",
+    "nsubj:xsubj"
   ]
   invalidIncoming = []
   validOutgoing = [

src/main/resources/englishConceptExpander.conf

@@ -17,6 +17,7 @@
     "^nmod_as",
     "^nmod_because",
     "^nmod_due_to",
+    "^nmod_due",
     "^nmod_except",
     "^nmod_given",
     "^nmod_since",
@@ -52,6 +53,7 @@
     "nmod_beyond",
     "nmod_compared_to",
     "nmod_compared_with",
+    "nmod_compared",
     "nmod_concerning",
     "nmod_despite",
     "nmod_due",
@@ -104,7 +106,9 @@
     "nmod_with_regard_to",
     "nmod_with_respect_to",
     "nmod_without",
-    "nmod_worsen"
+    "nmod_worsen",
+//    "xcomp",
+    "nsubj:xsubj"
   ]
   invalidIncoming = []
   validOutgoing = [

src/main/resources/org/clulab/wm/eidos/english/grammars/causal.yml

@@ -23,7 +23,7 @@ rules:
     pattern: |
       trigger = [lemma="due" & tag=/JJ/]
       cause: Entity = nmod_to (${ conjunctions })?
-      effect: Entity = <advmod|<amod <nmod_due_to? nsubj
+      effect: Entity = <advmod|<amod </nmod_(due_to|due|to)/? nsubj
 
   #-----------------------------------------------------------------------------------------
   # These rules are based on a parse which has 'due' with an incoming 'case' relation:
@@ -39,11 +39,11 @@ rules:
     pattern: |
       trigger = [lemma="due" & tag=/JJ/]
       cause: Entity = <case /${preps}|${noun_modifiers}/{,2}
-      effect: Entity = (<case <nmod_due_to? (?! [outgoing=dobj]) /${agents}/ /${ preps }/{,2})
+      effect: Entity = (<case </nmod_(due_to|due|to)/? (?! [outgoing=dobj]) /${agents}/ /${ preps }/{,2})
       |
-      (<case <nmod_due_to (?! [incoming=nmod_than]))
+      (<case </nmod_(due_to|due|to)/ (?! [incoming=nmod_than]))
       |
-      (<case <nmod_due_to <nmod_than >'nsubj:xsubj')
+      (<case </nmod_(due_to|due|to)/ <nmod_than >'nsubj:xsubj')
 
   # Used when the effect text has an object which is the true effect (i.e., in example this is "livestock assets")
   - name: dueTo-caseSyntax-dobj_effect-${addlabel}
@@ -54,7 +54,7 @@ rules:
     pattern: |
       trigger = [lemma="due" & tag=/JJ/]
       cause: Entity = <case /${preps}/{,2}
-      effect: Entity = <case <nmod_due_to? /${objects}/
+      effect: Entity = <case </nmod_(due_to|due|to)/? /${objects}/
 
   - name: dueToSyntax4-${addlabel}
     priority: ${rulepriority}
@@ -65,7 +65,7 @@ rules:
       trigger = [lemma="due" & tag=/JJ/]
       cause: Entity = <case /^(${conjunctions})/? /${preps}/{,2} /${conjunctions}/{,2}
       #cause: Entity = <case conj_and /${preps}/{,2} /${conjunctions}/{,2}
-      effect: Entity = (<case </nmod_(due_to|to)/? </^advcl/? /${agents}/ /${ preps }/{,2}) | (<case <nmod_due_to <nmod_in /${agents}/)
+      effect: Entity = (<case </nmod_(due_to|due|to)/? </^advcl/? /${agents}/ /${ preps }/{,2}) | (<case </nmod_(due_to|due|to)/ <nmod_in /${agents}/)
 
     #-----------------------------------------------------------------------------------------
     # These rules are based on a parse which has 'due' with an outgoing 'nmod_to' relation:
@@ -79,7 +79,7 @@ rules:
     pattern: |
       trigger = [lemma="due" & tag=/JJ/]
       cause: Entity = nmod_to /^acl/ dobj? (${ conjunctions })?
-      effect: Entity = <advmod|<amod <nmod_due_to? nsubj
+      effect: Entity = <advmod|<amod </nmod_(due_to|due|to)/? nsubj
 
   - name: dueToSyntax5-${addlabel}
     priority: ${rulepriority}
@@ -123,6 +123,16 @@ rules:
       cause: Entity = nmod_by
       effect: Entity = (>/${agents}/|</acl/)
 
+  - name: constrainSyntax3-${addlabel}
+    priority: ${rulepriority}
+    label: ${label}
+    action: ${ action }
+    example: "X made JJ by Y"
+    pattern: |
+      trigger = [lemma="constrain" & tag=/^V/]
+      cause: Entity = nmod_by /^conj/?
+      effect: Entity = >/${agents}/
+
   - name: leadToSyntax1-${addlabel}
     priority: ${rulepriority}
     label: ${label}

src/main/resources/org/clulab/wm/eidos/english/grammars/locationAttachment.yml

@@ -10,3 +10,12 @@ rules:
     pattern: |
       location: Location
       entity: Entity = </^nmod/ >/^conj/? (>/^dobj/ | >/^nsubj/)? >/^nmod/*
+
+  - name: locationattachment2
+    priority: ${ rulepriority }
+    example: "rainfall over the Ethiopia highlands"
+    label: ${ label }
+    action: ${ action }
+    pattern: |
+      location: Location
+      entity: Entity = >/compound/? >/nmod_over/

src/main/resources/org/clulab/wm/eidos/english/grammars/temporalAttachment.yml

@@ -22,3 +22,13 @@ rules:
       time: Time
       entity: Entity = </^nmod/ >/^nsubj/
 
+
+#  - name: timeattachment3
+#    priority: ${ rulepriority }
+#    example: "Assistance is critical to save lives over the coming year."
+#    label: ${ label }
+#    action: ${ action }
+#    pattern: |
+#      time: Time
+#      entity: Entity = </^nmod/ </^xcomp/
+

src/main/resources/org/clulab/wm/eidos/english/grammars/vars.yml

@@ -1,5 +1,5 @@
 #agents: "nsubj|'nsubj:xsubj'|'nsubjpass:xsubj'|nsubjpass|csubj|csubjpass|<acl|nmod_along_with"  #Comment:  nsubjpass for cause should not be there in an ideal world; but it does show up in practice
-agents: "nsubj|'nsubj:xsubj'|'nsubjpass:xsubj'|csubj|csubjpass|nmod_along_with"
+agents: "nsubj|'nsubj:xsubj'|'nsubjpass:xsubj'|csubj|csubjpass|nmod_along_with|nmod_along"
 
 adverbial_clause: "advcl"
 

src/main/scala/org/clulab/wm/eidos/attachments/NegationHandler.scala

@@ -23,7 +23,6 @@ class NegationHandler(val language: String) {
     language match {
       case Language.ENGLISH => mentions.map(detectNegationEnglish)
       case Language.PORTUGUESE => mentions.map(detectNegationPortuguese)
-      case Language.CLU => mentions.map(detectNegationEnglish)
       case _ => throw new RuntimeException(s"Unsupported language: $language")
     }
   }

src/main/scala/org/clulab/wm/eidos/exporters/DebugGroundingExporter.scala

@@ -3,7 +3,7 @@ package org.clulab.wm.eidos.exporters
 import org.clulab.wm.eidos.EidosSystem
 import org.clulab.wm.eidos.document.AnnotatedDocument
 import org.clulab.wm.eidos.groundings.PredicateGrounding
-import org.clulab.wm.eidoscommon.{EidosCluProcessor, EidosProcessor}
+import org.clulab.wm.eidoscommon.EidosEnglishProcessor
 import org.clulab.wm.eidoscommon.utils.Closer._
 import org.clulab.wm.eidoscommon.utils.FileUtils
 
@@ -26,7 +26,7 @@ class DebugGroundingExporter(filename: String, reader: EidosSystem, reground: Bo
 
       val doc = annotatedDocument.document
       for (i <- doc.sentences.indices) {
-        val clusent = reader.components.procOpt.get.asInstanceOf[EidosCluProcessor].annotate(doc.sentences(i)
+        val clusent = reader.components.procOpt.get.asInstanceOf[EidosEnglishProcessor].annotate(doc.sentences(i)
           .getSentenceText).sentences.head
         pw.println("********************************************\n")
         pw.println(s"Sentence $i: ${clusent.getSentenceText}.\n")

src/test/scala/org/clulab/wm/eidos/text/english/cag/TestCagP1.scala

@@ -15,7 +15,7 @@ class TestCagP1 extends EnglishTest {
     val economy = NodeSpec("economy", Dec("collapsing"))
     val cerealProduction = NodeSpec("cereal production", Dec("low"), Quant("low"))
     val rainfall = NodeSpec("poor rainfall in southeastern areas", Dec("poor"), Quant("poor"))
-    val copingCapacities = NodeSpec("coping capacities", Dec("exhaustion"), TimEx("several years"))
+    val copingCapacities = NodeSpec("of coping capacities", Dec("exhaustion"), TimEx("several years"))
 
     behavior of "p1s1"
 

src/test/scala/org/clulab/wm/eidos/text/english/cag/TestCagP2.scala

@@ -41,7 +41,7 @@ class TestCagP2 extends EnglishTest {
     val marketDisruption = NodeSpec("market disruption", Dec("disruption")) //newNodeSpec("market disruption")
     val economic = NodeSpec("economic downturn", Dec("downturn"))
     val cropFailure = NodeSpec("localized crop failures", Dec("failures"))
-    val foodPrices = NodeSpec("record high food prices", Inc("high"), Quant("high", "record"))
+    val foodPrices = NodeSpec("record high food prices", Inc("high"), Quant("high", "record"))// Prop("prices)) //TODO: implement Prop
     val hunger = NodeSpec("hunger", Inc("spread"))
 
     behavior of "p2s2"

src/test/scala/org/clulab/wm/eidos/text/english/cag/TestCagP3.scala

@@ -36,7 +36,7 @@ class TestCagP3 extends EnglishTest {
     val economic = NodeSpec("economic collapse", Dec("collapse"))
     val conflict = NodeSpec("conflict")
     val production = NodeSpec("agricultural production", Dec("reduced"))
-    val insecurity = NodeSpec("2017, food insecurity in Unity, Jonglei and parts of Greater Equatoria and Greater Bahr el Ghazal remained critical", Quant("critical"), TimEx("2017"), GeoLoc("Jonglei"))
+    val insecurity = NodeSpec("2017, food insecurity in Unity, Jonglei and parts of Greater Equatoria and Greater Bahr el Ghazal", Quant("critical"), TimEx("2017"), GeoLoc("Jonglei"))
 
     behavior of "p3s2"
 

src/test/scala/org/clulab/wm/eidos/text/english/cag/TestExtraText.scala

@@ -23,9 +23,9 @@ class TestExtraText extends EnglishTest {
 
     val oil = NodeSpec("international price of oil", Inc("rise", "sharp"), TimEx("2007"))
     val reserves = NodeSpec("foreign reserves", Dec("drain", "significant"))
-    val rights = NodeSpec("curtailment of the rights enshrined in the constitution", Dec("curtailment"))
+    val rights = NodeSpec("curtailment of the rights", Dec("curtailment"))
     val investment = NodeSpec("investment opportunities", Inc("promote"))
-    val poverty = NodeSpec("alleviate poverty", Pos("alleviate"))
+    val poverty = NodeSpec("poverty", Pos("alleviate"))
     val trade = NodeSpec("intra-African trade", Inc("boost"))
     val tariffs = NodeSpec("tariffs", Dec("brought down"))
 

src/test/scala/org/clulab/wm/eidos/text/english/eval6/TestDoc3.scala

@@ -19,7 +19,7 @@ class TestDoc3 extends EnglishTest {
     val rainfall = NodeSpec("Rainfall", Quant("average"), Inc("above average"))
     val cropActivity = NodeSpec("cropping activities")
     val rainfall2 = NodeSpec("heavier than normal rainfall", Quant("heavier than normal"), Inc("heavier than normal"))
-    val floodRisk = NodeSpec("risk of flooding", Inc("increasing"))
+    val floodRisk = NodeSpec("risk of flooding in many of the flood prone areas", Inc("increasing"))
 
     behavior of "TestDoc3 Paragraph 1"
 
@@ -53,7 +53,7 @@ class TestDoc3 extends EnglishTest {
     val rainfall = NodeSpec("rainfall", Inc("heavy"), Quant("well above average"), Quant("persistently heavy"), TimEx("the past month"))
     val agrConditions = NodeSpec("favorable agricultural conditions", Quant("favorable"), Pos("favorable"))
     val flooding = NodeSpec("potential for flooding")
-    val rainfall2 = NodeSpec("above-average rainfall", Quant("Average to above-average"), Inc("above-average"))
+    val rainfall2 = NodeSpec("above-average rainfall", Quant("Average to above-average"), Inc("above-average"), GeoLoc("Kenya"), GeoLoc("Eritrea"), GeoLoc("South Sudan"), GeoLoc("Uganda"))
     val rainfall3 = NodeSpec("rainfall", Quant("below average"), GeoLoc("Afar"))
 
     behavior of "TestDoc3 Paragraph 2"
@@ -307,11 +307,11 @@ class TestDoc3 extends EnglishTest {
     behavior of "TestDoc3 Paragraph 9"
     val cropCond = NodeSpec("Cropping conditions", Pos("favorable"), Quant("favorable"))
     val rainfall = NodeSpec("good performance of seasonal rainfall", Quant("good"), Pos("good"))
-    val rainfall2 = NodeSpec("persistently well above-average rainfall over the western Ethiopia highlands", Inc("above-average","persistently", "well"), Quant("above-average", "persistently", "well"), GeoLoc("Ethiopia"), TimEx("the coming weeks"))
-    val flood = NodeSpec("flooding", TimEx("the coming weeks"))
+    val rainfall2 = NodeSpec("persistently well above-average rainfall over the western Ethiopia highlands", Inc("above-average", "well"), Quant("above-average", "persistently", "well"), GeoLoc("Ethiopia"), TimEx("coming weeks"))
+    val flood = NodeSpec("flooding", TimEx("coming weeks"))
     val rainfall3 = NodeSpec("continued rains")
     val worm = NodeSpec("impact of Fall Armyworm", Dec("reduce"), TimEx("Fall"))
-    val rainfall4 = NodeSpec("Rainfall", Quant("moderate to heavy"), TimEx("the coming weeks"))
+    val rainfall4 = NodeSpec("Rainfall", Quant("moderate to heavy"), TimEx("coming weeks"))
     val flood2 = NodeSpec("potential for flooding")
     val rainfallDeficit = NodeSpec("rainfall deficits", Dec("deficits"), Dec("erase"))
 
@@ -542,7 +542,7 @@ class TestDoc3 extends EnglishTest {
     val rainfallForecasts = NodeSpec("short-and long-term rainfall forecasts", Quant("favorable"), Pos("favorable"), Dec("short-and"))
     val agriculturalAreas = NodeSpec("agricultural areas of the western and central highlands", Pos("favorable"))
 
-    val production = NodeSpec("production of most crops", Quant("likely"), Dec("limited"), Quant("most"))
+    val production = NodeSpec("production of most crops", Quant ("most"))//, Quant("likely"), Dec("limited"), Quant("most"))
     val insecurity = NodeSpec("insecurity")
     val availability = NodeSpec("lack of availability and/or access to farm inputs", Dec("lack"))
     //val longFarmInput = NodeSpec("access to farm inputs due to ongoing conflict", Dec("lack"), Quant("ongoing"))