diff --git a/pom.xml b/pom.xml index 7fd8096..1c5c4c0 100644 --- a/pom.xml +++ b/pom.xml @@ -56,7 +56,11 @@ - + + ome + formats-api + 6.10.0 + com.bc.zarr jzarr @@ -67,36 +71,17 @@ org.mockito mockito-inline 3.7.7 + test org.testng testng - 6.8 + 6.10 + test - - software.amazon.awssdk - s3 - 2.13.8 - - - software.amazon.awssdk - netty-nio-client - 2.13.8 - - - software.amazon.awssdk - sts - 2.13.8 - - - software.amazon.awssdk - bom - 2.11.10 - pom - xalan @@ -111,16 +96,7 @@ 2.7.2 runtime - - commons-io - commons-io - 2.8.0 - - - ome - formats-api - 6.7.0 - + @@ -134,7 +110,7 @@ ${project.basedir}/src - ${project.basedir}/src/test + ${project.basedir}/test org.codehaus.mojo diff --git a/src/loci/formats/S3FileSystemStore.java b/src/loci/formats/S3FileSystemStore.java deleted file mode 100644 index a9a7efd..0000000 --- a/src/loci/formats/S3FileSystemStore.java +++ /dev/null @@ -1,263 +0,0 @@ -package loci.formats; - -/*- - * #%L - * Implementation of Bio-Formats readers for the next-generation file formats - * %% - * Copyright (C) 2020 - 2022 Open Microscopy Environment - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ - -import com.bc.zarr.ZarrConstants; -import com.bc.zarr.ZarrUtils; -import com.bc.zarr.storage.Store; - -import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.ResponseInputStream; -import software.amazon.awssdk.core.sync.ResponseTransformer; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.S3Configuration; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.ListObjectsRequest; -import software.amazon.awssdk.services.s3.model.S3Object; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.TreeSet; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class S3FileSystemStore implements Store { - - private Path root; - S3Client client; - protected static final Logger LOGGER = - LoggerFactory.getLogger(S3FileSystemStore.class); - - public S3FileSystemStore(String path, FileSystem fileSystem) { - if (fileSystem == null) { - root = Paths.get(path); - } else { - root = fileSystem.getPath(path); - } - setupClient(); - } - - public void updateRoot(String path) { - root = Paths.get(path); - } - - private void setupClient() { - String[] pathSplit = root.toString().split(File.separator); - String endpoint = "https://" + pathSplit[1] + File.separator; - URI endpoint_uri; - try { - endpoint_uri = new URI(endpoint); - final S3Configuration config = S3Configuration.builder() - .pathStyleAccessEnabled(true) - .build(); - AwsCredentials credentials = AnonymousCredentialsProvider.create().resolveCredentials(); - client = S3Client.builder() - .endpointOverride(endpoint_uri) - .serviceConfiguration(config) - .region(Region.EU_WEST_1) // Ignored but required by the client - .credentialsProvider(StaticCredentialsProvider.create(credentials)).build(); - - } catch (URISyntaxException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (Exception e) { - // TODO Auto-generated catch block - //e.printStackTrace(); - } - - } - - public void close() { - if (client != null) { - client.close(); - } - } - - public S3FileSystemStore(Path rootPath) { - root = rootPath; - setupClient(); - } - - @Override - public InputStream getInputStream(String key) throws IOException { - String[] pathSplit = root.toString().split(File.separator); - String bucketName = pathSplit[2]; - String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()) + File.separator + key; - - try { - GetObjectRequest getRequest = GetObjectRequest.builder().bucket(bucketName).key(key2).build(); - ResponseInputStream responseStream = client.getObject(getRequest, ResponseTransformer.toInputStream()); - return responseStream; - } catch (Exception e) { - // TODO Auto-generated catch block - LOGGER.info( "Unable to locate or access key: " + key2); - e.printStackTrace(); - } - - return null; - } - - @Override - public OutputStream getOutputStream(String key) throws IOException { - final Path filePath = root.resolve(key); - final Path dir = filePath.getParent(); - Files.createDirectories(dir); - return Files.newOutputStream(filePath); - } - - @Override - public void delete(String key) throws IOException { - final Path toBeDeleted = root.resolve(key); - if (Files.isDirectory(toBeDeleted)) { - ZarrUtils.deleteDirectoryTreeRecursively(toBeDeleted); - } - if (Files.exists(toBeDeleted)){ - Files.delete(toBeDeleted); - } - if (Files.exists(toBeDeleted)|| Files.isDirectory(toBeDeleted)) { - throw new IOException("Unable to initialize " + toBeDeleted.toAbsolutePath().toString()); - } - } - - @Override - public TreeSet getArrayKeys() throws IOException { - return getKeysFor(ZarrConstants.FILENAME_DOT_ZARRAY); - } - - @Override - public TreeSet getGroupKeys() throws IOException { - return getKeysFor(ZarrConstants.FILENAME_DOT_ZGROUP); - } - - /** - * Copied from {@com.bc.zarr.storage.FileSystemStorage#getKeysEndingWith(String). - * - * @param suffix - * @return - * @throws IOException - */ - public TreeSet getKeysEndingWith(String suffix) throws IOException { - return (TreeSet)Files.walk(this.root).filter((path) -> { - return path.toString().endsWith(suffix); - }).map((path) -> { - return this.root.relativize(path).toString(); - }).collect(Collectors.toCollection(TreeSet::new)); - } - - /** - * Copied from {@com.bc.zarr.storage.FileSystemStorage#getRelativeLeafKeys(String). - * - * @param key - * @return - * @throws IOException - */ - public Stream getRelativeLeafKeys(String key) throws IOException { - Path walkingRoot = this.root.resolve(key); - return Files.walk(walkingRoot).filter((path) -> { - return !Files.isDirectory(path, new LinkOption[0]); - }).map((path) -> { - return walkingRoot.relativize(path).toString(); - }).map(ZarrUtils::normalizeStoragePath).filter((s) -> { - return s.trim().length() > 0; - }); - } - - private TreeSet getKeysFor(String suffix) throws IOException { - TreeSet keys = new TreeSet(); - - String[] pathSplit = root.toString().split(File.separator); - - String bucketName = pathSplit[2]; - String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()); - - List list = client.listObjects(ListObjectsRequest.builder() - .bucket(bucketName) - .prefix(key2) - .build()).contents(); - int n = list.size(); - - for (int i = 0; i < n; i++) { - S3Object object = list.get(i); - String k = object.key(); - if (k.contains(suffix)) { - String key = k.substring(k.indexOf(key2) + key2.length() + 1, k.indexOf(suffix)); - if (!key.isEmpty()) { - keys.add(key.substring(0, key.length()-1)); - } - } - } - - return keys; - } - - public ArrayList getFiles() throws IOException { - ArrayList keys = new ArrayList(); - - String[] pathSplit = root.toString().split(File.separator); - String bucketName = pathSplit[2]; - String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()); - - List list = client.listObjects(ListObjectsRequest.builder() - .bucket(bucketName) - .prefix(key2) - .build()).contents(); - int n = list.size(); - - for (int i = 0; i < n; i++) { - S3Object object = list.get(i); - String k = object.key(); - String key = k.substring(k.indexOf(key2) + key2.length() + 1, k.length()); - if (!key.isEmpty()) { - keys.add(key.substring(0, key.length()-1)); - } - } - return keys; - } - - -} diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 53cf4a3..453f219 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -32,18 +32,19 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.Hashtable; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; -import org.apache.commons.io.FileUtils; import org.w3c.dom.Document; import org.xml.sax.SAXException; @@ -62,7 +63,6 @@ import loci.formats.FormatReader; import loci.formats.FormatTools; import loci.formats.MetadataTools; -import loci.formats.S3FileSystemStore; import loci.formats.meta.MetadataStore; import loci.formats.ome.OMEXMLMetadata; import loci.formats.services.JZarrServiceImpl; @@ -832,22 +832,13 @@ public String[] getUsedFiles(boolean noPixels) { FormatTools.assertId(currentId, true, 1); String zarrRootPath = currentId.substring(0, currentId.indexOf(".zarr") + 5); ArrayList usedFiles = new ArrayList(); - if (!zarrRootPath.toLowerCase().contains("s3:")) { - File folder = new File(zarrRootPath); - Collection libs = FileUtils.listFiles(folder, null, true); - for (File file : libs) { - if (!file.isDirectory()) { - usedFiles.add(file.getAbsolutePath()); - } - } - } - else { - try { - usedFiles.addAll(new S3FileSystemStore(Paths.get(zarrRootPath)).getFiles()); - } catch (IOException e) { - e.printStackTrace(); - } + try (Stream paths = Files.walk(Paths.get(zarrRootPath))) { + paths.filter(Files::isRegularFile) + .forEach(path -> usedFiles.add(path.toFile().getAbsolutePath())); + } catch (IOException e) { + e.printStackTrace(); } + String[] fileArr = new String[usedFiles.size()]; fileArr = usedFiles.toArray(fileArr); return fileArr; diff --git a/src/loci/formats/services/JZarrServiceImpl.java b/src/loci/formats/services/JZarrServiceImpl.java index bf35bcc..dce0186 100644 --- a/src/loci/formats/services/JZarrServiceImpl.java +++ b/src/loci/formats/services/JZarrServiceImpl.java @@ -41,6 +41,9 @@ import java.util.Map; import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.bc.zarr.ArrayParams; import com.bc.zarr.Compressor; import com.bc.zarr.CompressorFactory; @@ -51,7 +54,6 @@ import loci.common.services.AbstractService; import loci.formats.FormatException; import loci.formats.FormatTools; -import loci.formats.S3FileSystemStore; import loci.formats.meta.IPyramidStore; import loci.formats.meta.MetadataRetrieve; import ucar.ma2.InvalidRangeException; @@ -59,11 +61,11 @@ public class JZarrServiceImpl extends AbstractService implements ZarrService { // -- Constants -- + private static final Logger LOGGER = LoggerFactory.getLogger(JZarrServiceImpl.class); public static final String NO_ZARR_MSG = "JZARR is required to read Zarr files."; // -- Fields -- ZarrArray zarrArray; - S3FileSystemStore s3fs; String currentId; Compressor zlibComp = CompressorFactory.create("zlib", "level", 8); // 8 = compression level .. valid values 0 .. 9 Compressor bloscComp = CompressorFactory.create("blosc", "cname", "lz4hc", "clevel", 7); @@ -75,7 +77,7 @@ public class JZarrServiceImpl extends AbstractService public JZarrServiceImpl(String root) { checkClassDependency(com.bc.zarr.ZarrArray.class); if (root != null && root.toLowerCase().contains("s3:")) { - s3fs = new S3FileSystemStore(Paths.get(root)); + LOGGER.warn("S3 access currently not supported"); } } @@ -87,8 +89,7 @@ public void open(String file) throws IOException, FormatException { zarrArray = ZarrArray.open(file); } else { - s3fs.updateRoot(file); - zarrArray = ZarrArray.open(s3fs); + LOGGER.warn("S3 access currently not supported"); } } @@ -103,8 +104,8 @@ public Map getGroupAttr(String path) throws IOException, FormatE group = ZarrGroup.open(path); } else { - s3fs.updateRoot(path); - group = ZarrGroup.open(s3fs); + LOGGER.warn("S3 access currently not supported"); + return null; } return group.getAttributes(); } @@ -115,8 +116,8 @@ public Map getArrayAttr(String path) throws IOException, FormatE array = ZarrArray.open(path); } else { - s3fs.updateRoot(path); - array = ZarrArray.open(s3fs); + LOGGER.warn("S3 access currently not supported"); + return null; } return array.getAttributes(); } @@ -127,8 +128,8 @@ public Set getGroupKeys(String path) throws IOException, FormatException group = ZarrGroup.open(path); } else { - s3fs.updateRoot(path); - group = ZarrGroup.open(s3fs); + LOGGER.warn("S3 access currently not supported"); + return null; } return group.getGroupKeys(); } @@ -139,8 +140,8 @@ public Set getArrayKeys(String path) throws IOException, FormatException group = ZarrGroup.open(path); } else { - s3fs.updateRoot(path); - group = ZarrGroup.open(s3fs); + LOGGER.warn("S3 access currently not supported"); + return null; } return group.getArrayKeys(); } @@ -246,9 +247,6 @@ public boolean isLittleEndian() { public void close() throws IOException { zarrArray = null; currentId = null; - if (s3fs != null) { - s3fs.close(); - } } @Override diff --git a/src/test/loci/formats/utests/JZarrServiceImplTest.java b/test/loci/formats/utests/JZarrServiceImplTest.java similarity index 100% rename from src/test/loci/formats/utests/JZarrServiceImplTest.java rename to test/loci/formats/utests/JZarrServiceImplTest.java diff --git a/src/test/loci/formats/utests/ZarrReaderMock.java b/test/loci/formats/utests/ZarrReaderMock.java similarity index 100% rename from src/test/loci/formats/utests/ZarrReaderMock.java rename to test/loci/formats/utests/ZarrReaderMock.java diff --git a/src/test/loci/formats/utests/ZarrReaderTest.java b/test/loci/formats/utests/ZarrReaderTest.java similarity index 100% rename from src/test/loci/formats/utests/ZarrReaderTest.java rename to test/loci/formats/utests/ZarrReaderTest.java