build.gradle

buildscript {
    repositories {
        mavenLocal()
        mavenCentral()
        maven { url = "https://plugins.gradle.org/m2/" }
        flatDir {
            dirs 'localLibs'
        }
    }
    dependencies {
        classpath 'gradle.plugin.org.kt3k.gradle.plugin:coveralls-gradle-plugin:2.12.2'
        classpath 'com.adarshr:gradle-test-logger-plugin:4.0.0'
        classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2'
    }
}

plugins {
    id 'com.gradleup.shadow' version '8.3.10'
    id 'distribution'
    id 'application'
}

apply plugin: 'jacoco'

jacoco {
    toolVersion = '0.8.14'
}

apply plugin: 'java-library'
apply plugin: 'base'
apply plugin: 'com.github.kt3k.coveralls'

def getGitRevision() {
    def gitRevision = "unknown"
    try {
        def result = providers.exec {
            workingDir = rootProject.rootDir
            commandLine 'git', 'describe', '--tags', '--always', '--first-parent'
        }
        gitRevision = result.standardOutput.asText.get().trim()
    } catch (Exception e) {
        println "Could not get Git revision: ${e}"
    }
    return gitRevision
}

project.ext.gitRevision = getGitRevision()

tasks.register('collectGitRevision') {
    project.ext.gitRevision = getGitRevision()
    doLast {
        println "Git revision: ${project.ext.gitRevision}"
    }
}

group = "org.grobid.datastet"
version = '0.9.0'

description = """datastet"""

import javax.inject.Inject
import org.apache.tools.ant.taskdefs.condition.Os
import org.gradle.process.ExecOperations

java {
    toolchain {
        languageVersion = JavaLanguageVersion.of(21)
    }
}

tasks.withType(JavaCompile).configureEach {
    options.encoding = 'UTF-8'
}

def conf = new org.yaml.snakeyaml.Yaml().load(new File("resources/config/config.yml").newInputStream())
def grobidHome = conf.grobidHome.replace("\$", "").replace('{', "").replace("GROBID_HOME:- ", "").replace("}", "")
if (grobidHome.startsWith("../")) {
    grobidHome = "${rootProject.rootDir}/${grobidHome}"
}

repositories {
    mavenLocal()
    mavenCentral()
    maven {
        url = new File(rootProject.rootDir, "localLibs").toURI()
    }
}

ext {
    // treating them separately, these jars will be flattened into grobid-core.jar on installing,
    // to avoid missing dependencies from the projects that include grobid-core (see 'jar' taskin grobid-core)
    localLibs = []
}

dependencies {

    implementation group: 'net.arnx', name: 'jsonic', version: '1.3.10'

    //Logging
    implementation 'org.slf4j:slf4j-api:1.7.36'

    //Apache commons
    implementation group: 'commons-pool', name: 'commons-pool', version: '1.6'
    implementation group: 'commons-io', name: 'commons-io', version: '2.21.0'
    implementation group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.14'
    implementation group: 'org.apache.httpcomponents', name: 'httpmime', version: '4.5.14'
    implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.20.0'
    implementation group: 'org.apache.commons', name: 'commons-collections4', version: '4.5.0'
    implementation group: 'org.apache.commons', name: 'commons-csv', version: '1.14.1'

    //Dropwizard
    implementation 'ru.vyarus:dropwizard-guicey:7.3.1'

    implementation 'io.dropwizard:dropwizard-bom:4.0.17'
    implementation 'io.dropwizard:dropwizard-core:4.0.17'
    implementation 'io.dropwizard:dropwizard-assets:4.0.17'
    implementation 'io.dropwizard:dropwizard-testing:4.0.17'
    implementation 'io.dropwizard:dropwizard-forms:4.0.17'
    implementation 'io.dropwizard:dropwizard-client:4.0.17'
    implementation 'io.dropwizard:dropwizard-auth:4.0.17'
    implementation 'io.dropwizard.metrics:metrics-core:4.2.38'
    implementation 'io.dropwizard.metrics:metrics-servlets:4.2.38'

    implementation group: 'com.google.guava', name: 'guava', version: '33.5.0-jre'

    //Parsing xml/json
    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.21.1'
    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.21.1'
    // jackson-annotations is pulled in transitively by jackson-databind
    implementation group: 'xom', name: 'xom', version: '1.3.9'
    implementation group: 'javax.xml.bind', name: 'jaxb-api', version: '2.3.0'
    implementation group: 'com.sun.xml.bind', name: 'jaxb-core', version: '2.3.0'
    implementation group: 'com.sun.xml.bind', name: 'jaxb-impl', version: '2.3.0'

    implementation group: 'javax.activation', name: 'activation', version: '1.1.1'

    implementation 'black.ninia:jep:4.3.1'
    // for sentence segmentation only
    implementation 'org.apache.opennlp:opennlp-tools:1.9.4'
    implementation "joda-time:joda-time:2.14.0"
    implementation "org.apache.lucene:lucene-analyzers-common:4.5.1"
    implementation group: 'org.jruby', name: 'jruby-complete', version: '9.4.12.1'
    implementation "org.apache.pdfbox:pdfbox:2.0.35"

    implementation group: 'org.semanticweb.yars', name: 'nxparser-parsers', version: '2.3.3'

    //Grobid
    implementation(group: 'org.grobid', name: 'grobid-core', version: '0.9.0') {
        exclude group: 'org.slf4j', module: "slf4j-log4j12"
    }
    api(group: 'org.grobid', name: 'grobid-service', version: '0.9.0') {
        exclude group: 'org.slf4j', module: "slf4j-log4j12"
    }
    implementation(group: 'org.grobid', name: 'grobid-trainer', version: '0.9.0') {
        exclude group: 'org.slf4j', module: "slf4j-log4j12"
    }

    //Tests
    testImplementation(platform('org.junit:junit-bom:5.10.2'))
    testRuntimeOnly("org.junit.platform:junit-platform-launcher") {
        because("Only needed to run tests in a version of IntelliJ IDEA that bundles older versions")
    }
    testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine")
    testImplementation('org.junit.jupiter:junit-jupiter')
    testRuntimeOnly("org.junit.vintage:junit-vintage-engine") {
        because 'allows JUnit 3 and JUnit 4 tests to run'
    }
    testRuntimeOnly("org.junit.platform:junit-platform-launcher") {
        because 'allows tests to run from IDEs that bundle older version of launcher'
    }
    testImplementation 'org.easymock:easymock:5.2.0'
    testImplementation 'org.hamcrest:hamcrest-all:1.3'
    testImplementation 'org.hamcrest:hamcrest-library:2.2'
    testImplementation 'org.powermock:powermock-module-junit4:2.0.9'
    testImplementation 'org.powermock:powermock-api-easymock:2.0.9'
}

configurations.all {
    resolutionStrategy {
        force 'xml-apis:xml-apis:1.4.01'
    }

    exclude group: 'org.slf4j', module: "slf4j-log4j12"
}

def getJavaLibraryPath = {
    def jepLocalLibraries = ""
    if (Os.isFamily(Os.FAMILY_MAC)) {
        if (Os.OS_ARCH.equals("aarch64")) {
            jepLocalLibraries = "${file("${grobidHome}/lib/mac_arm-64").absolutePath}"
        } else {
            jepLocalLibraries = "${file("${grobidHome}/lib/mac-64").absolutePath}"
        }
    } else if (Os.isFamily(Os.FAMILY_UNIX)) {
        jepLocalLibraries = "${grobidHome}/lib/lin-64/jep:${grobidHome}/lib/lin-64"
    } else {
        throw new RuntimeException("Unsupported platform!")
    }

    def javaLibraryPath = "${System.getProperty('java.library.path')}:${jepLocalLibraries}"

    def envPrefix = ""
    if (System.env.CONDA_PREFIX) {
        envPrefix = "${System.env.CONDA_PREFIX}"
    } else if (System.env.VIRTUAL_ENV) {
        envPrefix = "${System.env.VIRTUAL_ENV}"
    }

    if (envPrefix) {
        def envLibs = "${envPrefix}/lib"
        def pythonDirectory = file(envLibs).listFiles({ it.toString().contains("/lib/python") } as FileFilter)?.first()
        def pythonVersion = (pythonDirectory =~ /python([0-9]\.[0-9]+)/)[0][1]

        javaLibraryPath += ":${envLibs}:${envLibs}/python${pythonVersion}/site-packages/jep"
    }
    return javaLibraryPath
}

test {
    testLogging.showStandardStreams = true
    exclude '**/**IntegrationTest**'

    if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
        jvmArgs "--add-opens", "java.base/java.util.stream=ALL-UNNAMED",
            "--add-opens", "java.base/java.io=ALL-UNNAMED",
            "--add-opens", "java.xml/jdk.xml.internal=ALL-UNNAMED",
            "--add-opens", "java.base/java.lang=ALL-UNNAMED",
            "--add-opens", "java.base/java.util=ALL-UNNAMED"
    }
    systemProperty "java.library.path", "${System.getProperty('java.library.path')}:" + getJavaLibraryPath()
}

// return the default value if the property has not been specified in command line
ext.getArg = { propName, defaultVal ->
    return project.hasProperty(propName) ? project.getProperty(propName) : defaultVal;
}

import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar

apply plugin: 'java'
apply plugin: 'maven-publish'

publishing {
    publications {
        mavenJava(MavenPublication) {
            from components.java
        }
    }
    repositories {
        mavenLocal()
    }
}

task install(dependsOn: publishToMavenLocal)

task mainJar(type: ShadowJar) {
    zip64 = true
    from sourceSets.main.output

    from {
        project.configurations.runtimeClasspath.collect {
            it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : []
        }
    }

    exclude("logback.xml")

    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}

shadowJar {
    archiveClassifier = 'onejar'
    mergeServiceFiles()
    zip64 = true
    manifest {
        attributes 'Main-Class': 'org.grobid.core.main.batch.DatastetMain'
    }

    from {
        project.configurations.runtimeClasspath.collect {
            it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : []
        }
    }

    exclude("logback.xml")

    configurations = [project.configurations.runtimeClasspath]

    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}

jar {
    dependsOn mainJar
    enabled = true
}

distZip.enabled = true
distTar.enabled = false
shadowDistZip.enabled = false
shadowDistTar.enabled = false

artifacts {
    archives jar
    archives shadowJar
}

processResources {
    filesMatching(["version.txt", "revision.txt"]) {
        expand(
            project_version: project.property('version') ?: "unknown",
            project_revision: rootProject.ext.gitRevision
        )
    }
}

task copyModels(type: Copy) {
    from "${rootDir}/resources/models"
    include "**/*.wapiti"
    include "**/config.json"
    include "**/transformer-config.json"
    include "**/transformer-tokenizer/**"
    include "**/model_weights.hdf5"
    include "**/preprocessor.json"
    exclude "**/features-engineering/**"
    exclude "**/result-logs/**"
    into "${grobidHome}/models/"

    doLast {
        print "Copy models under grobid-home: ${grobidHome}"
    }
}

// Gradle 8+ removed the dynamic `exec{}` shortcut inside task actions, so we
// inject ExecOperations via a helper interface and call it from doLast.
interface InjectedExecOps {
    @Inject ExecOperations getExecOps()
}

task downloadModelsGit(dependsOn: copyModels) {
    def targetDir = file("${grobidHome}/models/datastet")
    def injected = objects.newInstance(InjectedExecOps)
    doLast {
        targetDir.parentFile.mkdirs()
        if (targetDir.exists()) {
            injected.execOps.exec {
                workingDir = targetDir
                commandLine 'git', 'pull'
            }
        } else {
            injected.execOps.exec {
                commandLine 'git', 'clone',
                    'https://huggingface.co/sciencialab/datastet-models',
                    targetDir.absolutePath
            }
        }
    }
}

task installModels(dependsOn: [copyModels, downloadModelsGit]){
    doLast {
        def sourceDir = file("${grobidHome}/models/datastet")
        def destDir = file("${grobidHome}/models/")

        Arrays.asList(
            "context_bert",
            "context_creation_bert",
            "context_shared_bert",
            "context_used_bert",
            "datasets-BERT_CRF",
            "dataseer-binary_bert",
            "dataseer-first_bert",
            "dataseer-reuse_bert"
        ).each { String subDirName ->
            var subDir = new File(sourceDir, subDirName)
            if (subDir.isDirectory()) {
                copy {
                    from subDir
                    into new File(destDir, subDirName)
                }
            }
        }

        delete files("${sourceDir}")
    }
}

wrapper {
    gradleVersion = "9.0.0"
}

//////////
// service
apply plugin: 'application'
application {
    mainClass = 'org.grobid.service.DatastetApplication'

    run {
        if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
            jvmArgs "--add-opens", "java.base/java.lang=ALL-UNNAMED"
        }
        workingDir = rootProject.rootDir
        systemProperty "java.library.path", "${System.getProperty('java.library.path')}:" + getJavaLibraryPath()

        args = ['server', 'resources/config/config.yml']
    }
}

jacocoTestReport {
    reports {
        xml.required = true // coveralls plugin depends on xml format report
        html.required = true
    }
    dependsOn test // tests are required to run before generating the report
}

coveralls {
    jacocoReportPath = 'build/reports/jacoco/test/jacocoTestReport.xml'
}