diff --git a/buildSrc/src/main/groovy/gor.java-common.gradle b/buildSrc/src/main/groovy/gor.java-common.gradle index e616da98..b326b5d7 100644 --- a/buildSrc/src/main/groovy/gor.java-common.gradle +++ b/buildSrc/src/main/groovy/gor.java-common.gradle @@ -178,7 +178,7 @@ tasks.withType(Test) { props.load(new FileInputStream("${project.rootDir}/tests/config/unittest.sysproperties")) systemProperties = props - jvmArgs = Arrays.asList("--add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.invoke=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/sun.nio.fs=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED".split(" ")) + jvmArgs = Arrays.asList("--enable-native-access=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.invoke=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/sun.nio.fs=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED".split(" ")) // Set logback config file. systemProperty 'logback.configurationFile', new File("${project.rootDir}/tests/config/logback-test.xml").absolutePath diff --git a/drivers/src/main/java/org/gorpipe/s3/driver/S3SourceProvider.java b/drivers/src/main/java/org/gorpipe/s3/driver/S3SourceProvider.java index 18c5be10..86649ba9 100644 --- a/drivers/src/main/java/org/gorpipe/s3/driver/S3SourceProvider.java +++ b/drivers/src/main/java/org/gorpipe/s3/driver/S3SourceProvider.java @@ -136,6 +136,7 @@ private S3Client createSyncCrtClient(Credentials cred) { AwsCrtHttpClient.Builder httpClientBuilder = AwsCrtHttpClient.builder() .connectionTimeout(s3Config.connectionTimeout()) // Default was 2s .maxConcurrency(s3Config.connectionPoolSize()) + //.readBufferSizeInBytes(16*1024*1024L) .tcpKeepAliveConfiguration(b -> b .keepAliveInterval(Duration.ofMillis(s3Config.socketTimeout().toMillis()/2)) .keepAliveTimeout(s3Config.connectionTimeout())) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2733ed5d..0b55a3bd 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/model/src/main/java/org/gorpipe/gor/model/DriverBackedFileReader.java b/model/src/main/java/org/gorpipe/gor/model/DriverBackedFileReader.java index abe85e7f..2dbb26bc 100644 --- a/model/src/main/java/org/gorpipe/gor/model/DriverBackedFileReader.java +++ b/model/src/main/java/org/gorpipe/gor/model/DriverBackedFileReader.java @@ -31,6 +31,7 @@ import org.gorpipe.gor.driver.GorDriverFactory; import org.gorpipe.gor.driver.PluggableGorDriver; import org.gorpipe.gor.driver.SourceProvider; +import org.gorpipe.gor.driver.adapters.PositionAwareInputStream; import org.gorpipe.gor.driver.adapters.StreamSourceRacFile; import org.gorpipe.gor.driver.meta.DataType; import org.gorpipe.gor.driver.meta.SourceReference; @@ -43,6 +44,7 @@ import org.gorpipe.gor.table.dictionary.gor.GorDictionaryTable; import org.gorpipe.gor.table.util.PathUtils; import org.gorpipe.gor.util.DataUtil; +import org.gorpipe.gor.util.NCGZIPInputStream; import org.gorpipe.gor.util.StringUtil; import org.gorpipe.util.standalone.GorStandalone; import org.slf4j.Logger; @@ -65,6 +67,8 @@ public class DriverBackedFileReader extends FileReader { private static final String DEFAULT_COMMON_ROOT = "./"; + final static int GZIP_BUFFER_SIZE = Integer.parseInt(System.getProperty("gor.gzip.buffer.size", "2046")); + private final boolean DEPENDENTS = System.getProperty("gor.filereader.dependents", "true").equalsIgnoreCase("true"); private final String securityContext; @@ -378,7 +382,7 @@ private static class SourceReader extends BufferedReader { SourceReader(DataSource source) throws IOException { super(DataUtil.isGZip(source.getName()) ? - new BufferedReader(new InputStreamReader(new GZIPInputStream(((StreamSource) source).open()/*, 32*1024*/))) : + new BufferedReader(new InputStreamReader(new GZIPInputStream(new NCGZIPInputStream(((StreamSource) source).open()), GZIP_BUFFER_SIZE))) : new InputStreamReader(((StreamSource) source).open())); this.source = source; } diff --git a/model/src/main/java/org/gorpipe/gor/model/VcfGzGenomicIterator.java b/model/src/main/java/org/gorpipe/gor/model/VcfGzGenomicIterator.java index 2ca47853..1f835e48 100755 --- a/model/src/main/java/org/gorpipe/gor/model/VcfGzGenomicIterator.java +++ b/model/src/main/java/org/gorpipe/gor/model/VcfGzGenomicIterator.java @@ -46,6 +46,8 @@ public class VcfGzGenomicIterator extends GenomicIteratorBase { final static int VCF_COLUMN_RENAME_COUNT = 10; final static String VCF_COLUMN_RENAME_NAME = "VALUES"; + final static int GZIP_BUFFER_SIZE = Integer.parseInt(System.getProperty("gor.gzip.buffer.size", "2046")); + public BufferedReader reader; private StreamSource streamSource; final ChromoLookup lookup; // chromosome name lookup service @@ -62,7 +64,7 @@ public enum ChrNameSystem { public VcfGzGenomicIterator(ChromoLookup lookup, String file, StreamSource streamsource, boolean compressed) throws IOException { - this(lookup, file, new BufferedReader(new InputStreamReader(compressed ? new GZIPInputStream(new NCGZIPInputStream(new PositionAwareInputStream(streamsource.open()))) : streamsource.open()))); + this(lookup, file, new BufferedReader(new InputStreamReader(compressed ? new GZIPInputStream(new NCGZIPInputStream(new PositionAwareInputStream(streamsource.open())), GZIP_BUFFER_SIZE) : streamsource.open()))); this.streamSource = streamsource; } diff --git a/model/src/main/java/org/gorpipe/gor/util/NCGZIPInputStream.java b/model/src/main/java/org/gorpipe/gor/util/NCGZIPInputStream.java index dd3017aa..527bbe02 100644 --- a/model/src/main/java/org/gorpipe/gor/util/NCGZIPInputStream.java +++ b/model/src/main/java/org/gorpipe/gor/util/NCGZIPInputStream.java @@ -52,6 +52,8 @@ public NCGZIPInputStream(InputStream in) { * http://bugs.java.com/bugdatabase/view_bug.do?bug_id=7036144 *
* Reference TT: 0034867351 + * + * 2025-06-20: This is fixed in Java 23 b15 */ @Override public int available() throws IOException {