From 627d8fc54c0cd2b134e3eb5ee9524b05ad7cdd90 Mon Sep 17 00:00:00 2001 From: songpan Date: Thu, 4 Apr 2019 07:49:36 -0700 Subject: [PATCH] Add byte array implementation to profiler. Startblock: cl/241546007 is submitted PiperOrigin-RevId: 241929368 --- .../explainer/PatchExplainer.java | 17 +- .../explainer/PatchExplainerTest.java | 19 +- .../DefaultDeflateCompressionDiviner.java | 80 ++++---- .../DeltaFriendlyOldBlobSizeLimiter.java | 4 +- .../generator/DeltaGenerator.java | 27 ++- .../generator/FileByFileDeltaGenerator.java | 21 ++- .../generator/MinimalZipArchive.java | 50 +++-- .../generator/MinimalZipParser.java | 15 +- .../generator/PreDiffExecutor.java | 47 ++--- .../generator/PreDiffPlanEntryModifier.java | 4 +- .../generator/PreDiffPlanner.java | 28 ++- .../generator/TotalRecompressionLimiter.java | 4 +- .../generator/bsdiff/BsDiff.java | 35 ++-- .../bsdiff/BsDiffDeltaGenerator.java | 10 +- .../generator/bsdiff/BsDiffMatcher.java | 100 +++++----- .../generator/bsdiff/BsDiffPatchWriter.java | 177 ++++++++---------- .../generator/bsdiff/BsUtil.java | 24 +-- .../generator/bsdiff/DivSuffixSorter.java | 11 +- .../generator/bsdiff/SuffixSorter.java | 5 +- .../wrapper/BsDiffNativePatchWriter.java | 34 +++- .../similarity/Crc32SimilarityFinder.java | 7 +- .../similarity/SimilarityFinder.java | 19 +- .../DefaultDeflateCompressionDivinerTest.java | 23 ++- .../DeltaFriendlyOldBlobSizeLimiterTest.java | 44 +++-- .../generator/MinimalZipParserTest.java | 6 +- .../generator/PreDiffExecutorTest.java | 51 +++-- .../generator/PreDiffPlannerTest.java | 75 +++++--- .../TotalRecompressionLimiterTest.java | 6 +- .../generator/bsdiff/BsDiffTest.java | 125 +++++++------ .../generator/bsdiff/BsDiffTestData.java | 13 +- .../generator/bsdiff/BsUtilTest.java | 5 +- .../bsdiff/SuffixSorterTestBase.java | 28 +-- .../shared/DeltaFriendlyFile.java | 78 ++++++-- .../shared/RandomAccessFileInputStream.java | 19 +- .../shared/bytesource/ByteSource.java | 15 +- .../shared/bytesource/ByteStreams.java | 55 ++++++ .../RandomAccessFileInputStreamTest.java | 12 +- .../shared/bytesource/ByteStreamsTest.java | 65 +++++++ 38 files changed, 837 insertions(+), 521 deletions(-) create mode 100644 shared/src/main/java/com/google/archivepatcher/shared/bytesource/ByteStreams.java create mode 100644 shared/src/test/java/com/google/archivepatcher/shared/bytesource/ByteStreamsTest.java diff --git a/explainer/src/main/java/com/google/archivepatcher/explainer/PatchExplainer.java b/explainer/src/main/java/com/google/archivepatcher/explainer/PatchExplainer.java index 754a4b88..7b49c297 100644 --- a/explainer/src/main/java/com/google/archivepatcher/explainer/PatchExplainer.java +++ b/explainer/src/main/java/com/google/archivepatcher/explainer/PatchExplainer.java @@ -29,6 +29,7 @@ import com.google.archivepatcher.shared.DeflateUncompressor; import com.google.archivepatcher.shared.RandomAccessFileInputStream; import com.google.archivepatcher.shared.Uncompressor; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; @@ -119,12 +120,16 @@ public List explainPatch( } Uncompressor uncompressor = new DeflateUncompressor(); - PreDiffExecutor executor = - new PreDiffExecutor.Builder() - .readingOriginalFiles(oldFile, newFile) - .addPreDiffPlanEntryModifiers(Arrays.asList(preDiffPlanEntryModifiers)) - .build(); - PreDiffPlan plan = executor.prepareForDiffing(); + PreDiffPlan plan; + try (ByteSource oldBlob = ByteSource.fromFile(oldFile); + ByteSource newBlob = ByteSource.fromFile(newFile)) { + PreDiffExecutor executor = + new PreDiffExecutor.Builder() + .readingOriginalFiles(oldBlob, newBlob) + .addPreDiffPlanEntryModifiers(Arrays.asList(preDiffPlanEntryModifiers)) + .build(); + plan = executor.prepareForDiffing(); + } try (TempFileHolder oldTemp = new TempFileHolder(); TempFileHolder newTemp = new TempFileHolder(); TempFileHolder deltaTemp = new TempFileHolder()) { diff --git a/explainer/src/test/java/com/google/archivepatcher/explainer/PatchExplainerTest.java b/explainer/src/test/java/com/google/archivepatcher/explainer/PatchExplainerTest.java index 3b913044..a2ed9f34 100644 --- a/explainer/src/test/java/com/google/archivepatcher/explainer/PatchExplainerTest.java +++ b/explainer/src/test/java/com/google/archivepatcher/explainer/PatchExplainerTest.java @@ -25,10 +25,10 @@ import com.google.archivepatcher.shared.Compressor; import com.google.archivepatcher.shared.UnitTestZipArchive; import com.google.archivepatcher.shared.UnitTestZipEntry; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -96,7 +96,7 @@ public void compress(InputStream uncompressedIn, OutputStream compressedOut) * A "delta generator" that always outputs the same exact string regardless of the inputs and * asserts that the input is exactly as expected. */ - private static class FakeDeltaGenerator implements DeltaGenerator { + private static class FakeDeltaGenerator extends DeltaGenerator { static final String OUTPUT = "fakedeltagenerator output"; private final byte[] expectedOld; private final byte[] expectedNew; @@ -107,17 +107,18 @@ public FakeDeltaGenerator(byte[] expectedOld, byte[] expectedNew) { } @Override - public void generateDelta(File oldBlob, File newBlob, OutputStream deltaOut) + public void generateDelta(ByteSource oldBlob, ByteSource newBlob, OutputStream deltaOut) throws IOException { - assertFileEquals(oldBlob, expectedOld); - assertFileEquals(newBlob, expectedNew); + assertByteSourceEquals(oldBlob, expectedOld); + assertByteSourceEquals(newBlob, expectedNew); deltaOut.write(OUTPUT.getBytes("US-ASCII")); } - private final void assertFileEquals(File file, byte[] expected) throws IOException { - byte[] actual = new byte[(int) file.length()]; - try (FileInputStream fileIn = new FileInputStream(file); - DataInputStream dataIn = new DataInputStream(fileIn)) { + private final void assertByteSourceEquals(ByteSource byteSource, byte[] expected) + throws IOException { + byte[] actual = new byte[(int) byteSource.length()]; + try (InputStream in = byteSource.openStream(); + DataInputStream dataIn = new DataInputStream(in)) { dataIn.readFully(actual); } assertThat(actual).isEqualTo(expected); diff --git a/generator/src/main/java/com/google/archivepatcher/generator/DefaultDeflateCompressionDiviner.java b/generator/src/main/java/com/google/archivepatcher/generator/DefaultDeflateCompressionDiviner.java index d613998c..3f5d265f 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/DefaultDeflateCompressionDiviner.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/DefaultDeflateCompressionDiviner.java @@ -14,12 +14,14 @@ package com.google.archivepatcher.generator; +import static com.google.archivepatcher.shared.bytesource.ByteStreams.readFully; + import com.google.archivepatcher.shared.DefaultDeflateCompatibilityWindow; import com.google.archivepatcher.shared.JreDeflateParameters; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.RandomAccessFile; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -78,7 +80,26 @@ public DivinationResult( /** * Load the specified archive and attempt to divine deflate parameters for all entries within. * - * @param archiveFile the archive file to work on + * @param archiveBlob the archive blob to work on + * @return a list of results for each entry in the archive, in file order (not central directory + * order). There is exactly one result per entry, regardless of whether or not that entry is + * compressed. Callers can filter results by checking {@link + * MinimalZipEntry#getCompressionMethod()} to see if the result is or is not compressed, and + * by checking whether a non-null {@link JreDeflateParameters} was obtained. + * @throws IOException if unable to read or parse the file + * @see DivinationResult + */ + public static List divineDeflateParameters(File archiveBlob) + throws IOException { + try (ByteSource archiveData = ByteSource.fromFile(archiveBlob)) { + return divineDeflateParameters(archiveData); + } + } + + /** + * Load the specified archive and attempt to divine deflate parameters for all entries within. + * + * @param archiveBlob the archive blob to work on * @return a list of results for each entry in the archive, in file order (not central directory * order). There is exactly one result per entry, regardless of whether or not that entry is * compressed. Callers can filter results by checking {@link @@ -87,10 +108,9 @@ public DivinationResult( * @throws IOException if unable to read or parse the file * @see DivinationResult */ - public static List divineDeflateParameters(File archiveFile) + public static List divineDeflateParameters(ByteSource archiveBlob) throws IOException { - RandomAccessFile randomAccessArchiveFile = new RandomAccessFile(archiveFile, "r"); - List zipEntries = MinimalZipArchive.listEntries(archiveFile); + List zipEntries = MinimalZipArchive.listEntries(archiveBlob); List results = new ArrayList<>(zipEntries.size()); for (MinimalZipEntry minimalZipEntry : zipEntries) { JreDeflateParameters divinedParameters = null; @@ -99,29 +119,27 @@ public static List divineDeflateParameters(File archiveFile) if (minimalZipEntry.getCompressedSize() < (100 * 1024)) { try { byte[] compressedBytes = new byte[(int) minimalZipEntry.getCompressedSize()]; - randomAccessArchiveFile.seek(minimalZipEntry.getFileOffsetOfCompressedData()); - randomAccessArchiveFile.readFully(compressedBytes); - divinedParameters = - divineDeflateParameters(new ByteArrayInputStreamFactory(compressedBytes)); + try (InputStream in = + archiveBlob + .slice(minimalZipEntry.getFileOffsetOfCompressedData(), compressedBytes.length) + .openStream()) { + readFully(in, compressedBytes); + } + divinedParameters = divineDeflateParametersForEntry(ByteSource.wrap(compressedBytes)); } catch (Exception ignore) { divinedParameters = null; } } else { - divinedParameters = - divineDeflateParameters( - new RandomAccessFileInputStreamFactory( - archiveFile, - minimalZipEntry.getFileOffsetOfCompressedData(), - minimalZipEntry.getCompressedSize())); + try (ByteSource slice = + archiveBlob.slice( + minimalZipEntry.getFileOffsetOfCompressedData(), + minimalZipEntry.getCompressedSize())) { + divinedParameters = divineDeflateParametersForEntry(slice); + } } } results.add(new DivinationResult(minimalZipEntry, divinedParameters)); } - try { - randomAccessArchiveFile.close(); - } catch (Exception ignore) { - // Ignore - } return results; } @@ -129,8 +147,8 @@ public static List divineDeflateParameters(File archiveFile) * Determines the original {@link JreDeflateParameters} that were used to compress a given piece * of deflated delivery. * - * @param compressedDataInputStreamFactory a {@link MultiViewInputStreamFactory} that can provide - * multiple independent {@link InputStream} instances for the compressed delivery. + * @param entry a {@link MultiViewInputStreamFactory} that can provide multiple independent {@link + * InputStream} instances for the compressed delivery. * @return the parameters that can be used to replicate the compressed delivery in the {@link * DefaultDeflateCompatibilityWindow}, if any; otherwise null. Note that * null is also returned in the case of corrupt zip delivery since, by @@ -138,8 +156,8 @@ public static List divineDeflateParameters(File archiveFile) * @throws IOException if there is a problem reading the delivery, i.e. if the file contents are * changed while reading */ - public static JreDeflateParameters divineDeflateParameters( - MultiViewInputStreamFactory compressedDataInputStreamFactory) throws IOException { + public static JreDeflateParameters divineDeflateParametersForEntry(ByteSource entry) + throws IOException { byte[] copyBuffer = new byte[32 * 1024]; // Iterate over all relevant combinations of nowrap, strategy and level. for (boolean nowrap : new boolean[] {true, false}) { @@ -154,7 +172,7 @@ public static JreDeflateParameters divineDeflateParameters( inflater.reset(); deflater.reset(); try { - if (matches(inflater, deflater, compressedDataInputStreamFactory, copyBuffer)) { + if (matches(inflater, deflater, entry, copyBuffer)) { end(inflater, deflater); return JreDeflateParameters.of(level, strategy, nowrap); } @@ -214,6 +232,7 @@ private static void end(Inflater inflater, Deflater deflater) { * * @param inflater the inflater for uncompressing the stream * @param deflater the deflater for recompressing the output of the inflater + * @param compressedData {@link ByteSource} containing the compressed data. * @param copyBuffer buffer to use for copying bytes between the inflater and the deflater * @return true if the specified deflater reproduces the bytes in compressedDataIn, otherwise * false @@ -221,18 +240,13 @@ private static void end(Inflater inflater, Deflater deflater) { * there is a problem parsing compressedDataIn */ private static boolean matches( - Inflater inflater, - Deflater deflater, - MultiViewInputStreamFactory compressedDataInputStreamFactory, - byte[] copyBuffer) + Inflater inflater, Deflater deflater, ByteSource compressedData, byte[] copyBuffer) throws IOException { try (MatchingOutputStream matcher = - new MatchingOutputStream( - compressedDataInputStreamFactory.newStream(), copyBuffer.length); + new MatchingOutputStream(compressedData.openStream(), copyBuffer.length); InflaterInputStream inflaterIn = - new InflaterInputStream( - compressedDataInputStreamFactory.newStream(), inflater, copyBuffer.length); + new InflaterInputStream(compressedData.openStream(), inflater, copyBuffer.length); DeflaterOutputStream out = new DeflaterOutputStream(matcher, deflater, copyBuffer.length)) { int numRead; while ((numRead = inflaterIn.read(copyBuffer)) >= 0) { diff --git a/generator/src/main/java/com/google/archivepatcher/generator/DeltaFriendlyOldBlobSizeLimiter.java b/generator/src/main/java/com/google/archivepatcher/generator/DeltaFriendlyOldBlobSizeLimiter.java index 5a818aa4..8ec6bcc1 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/DeltaFriendlyOldBlobSizeLimiter.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/DeltaFriendlyOldBlobSizeLimiter.java @@ -14,7 +14,7 @@ package com.google.archivepatcher.generator; -import java.io.File; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -65,7 +65,7 @@ public DeltaFriendlyOldBlobSizeLimiter(long maxSizeBytes) { @Override public List getModifiedPreDiffPlanEntries( - File oldFile, File newFile, List originalEntries) { + ByteSource oldFile, ByteSource newFile, List originalEntries) { List sorted = sortPreDiffPlanEntries(originalEntries); diff --git a/generator/src/main/java/com/google/archivepatcher/generator/DeltaGenerator.java b/generator/src/main/java/com/google/archivepatcher/generator/DeltaGenerator.java index 07135f27..d60a3ea3 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/DeltaGenerator.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/DeltaGenerator.java @@ -14,14 +14,13 @@ package com.google.archivepatcher.generator; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.File; import java.io.IOException; import java.io.OutputStream; -/** - * An interface to be implemented by delta generators. - */ -public interface DeltaGenerator { +/** An interface to be implemented by delta generators. */ +public abstract class DeltaGenerator { /** * Generates a delta in deltaOut that can be applied to oldBlob to produce newBlob. * @@ -32,6 +31,24 @@ public interface DeltaGenerator { * delta output stream * @throws InterruptedException if any thread has interrupted the current thread */ - void generateDelta(File oldBlob, File newBlob, OutputStream deltaOut) + public void generateDelta(File oldBlob, File newBlob, OutputStream deltaOut) + throws IOException, InterruptedException { + try (ByteSource oldByteSource = ByteSource.fromFile(oldBlob); + ByteSource newByteSource = ByteSource.fromFile(newBlob)) { + generateDelta(oldByteSource, newByteSource, deltaOut); + } + } + + /** + * Generates a delta in deltaOut that can be applied to oldBlob to produce newBlob. + * + * @param oldBlob the old blob + * @param newBlob the new blob + * @param deltaOut the stream to write the delta to + * @throws IOException in the event of an I/O error reading the input files or writing to the + * delta output stream + * @throws InterruptedException if any thread has interrupted the current thread + */ + public abstract void generateDelta(ByteSource oldBlob, ByteSource newBlob, OutputStream deltaOut) throws IOException, InterruptedException; } diff --git a/generator/src/main/java/com/google/archivepatcher/generator/FileByFileDeltaGenerator.java b/generator/src/main/java/com/google/archivepatcher/generator/FileByFileDeltaGenerator.java index 4ba21005..4a00cc04 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/FileByFileDeltaGenerator.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/FileByFileDeltaGenerator.java @@ -18,6 +18,7 @@ import com.google.archivepatcher.generator.bsdiff.BsDiffDeltaGenerator; import com.google.archivepatcher.shared.PatchConstants.DeltaFormat; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; @@ -30,7 +31,7 @@ import java.util.Set; /** Generates file-by-file patches. */ -public class FileByFileDeltaGenerator implements DeltaGenerator { +public class FileByFileDeltaGenerator extends DeltaGenerator { /** Modifiers for planning and patch generation. */ private final List preDiffPlanEntryModifiers; @@ -84,14 +85,14 @@ public FileByFileDeltaGenerator( * OutputStream} or in a post-processing step, prior to transmitting the patch to the patch * applier. * - * @param oldFile the original old file to read (will not be modified) - * @param newFile the original new file to read (will not be modified) + * @param oldBlob the original old file to read (will not be modified) + * @param newBlob the original new file to read (will not be modified) * @param patchOut the stream to write the patch to * @throws IOException if unable to complete the operation due to an I/O error * @throws InterruptedException if any thread has interrupted the current thread */ @Override - public void generateDelta(File oldFile, File newFile, OutputStream patchOut) + public void generateDelta(ByteSource oldBlob, ByteSource newBlob, OutputStream patchOut) throws IOException, InterruptedException { try (TempFileHolder deltaFriendlyOldFile = new TempFileHolder(); TempFileHolder deltaFriendlyNewFile = new TempFileHolder(); @@ -100,7 +101,7 @@ public void generateDelta(File oldFile, File newFile, OutputStream patchOut) BufferedOutputStream bufferedDeltaOut = new BufferedOutputStream(deltaFileOut)) { PreDiffPlan preDiffPlan = generatePreDiffPlan( - oldFile, newFile, deltaFriendlyOldFile, deltaFriendlyNewFile, supportedDeltaFormats); + oldBlob, newBlob, deltaFriendlyOldFile, deltaFriendlyNewFile, supportedDeltaFormats); DeltaGenerator deltaGenerator = getDeltaGenerator(); deltaGenerator.generateDelta( deltaFriendlyOldFile.file, deltaFriendlyNewFile.file, bufferedDeltaOut); @@ -124,15 +125,17 @@ public void generateDelta(File oldFile, File newFile, OutputStream patchOut) */ public PreDiffPlan generatePreDiffPlan(File oldFile, File newFile) throws IOException { try (TempFileHolder deltaFriendlyOldFile = new TempFileHolder(); - TempFileHolder deltaFriendlyNewFile = new TempFileHolder()) { + TempFileHolder deltaFriendlyNewFile = new TempFileHolder(); + ByteSource oldBlob = ByteSource.fromFile(oldFile); + ByteSource newBlob = ByteSource.fromFile(newFile)) { return generatePreDiffPlan( - oldFile, newFile, deltaFriendlyOldFile, deltaFriendlyNewFile, supportedDeltaFormats); + oldBlob, newBlob, deltaFriendlyOldFile, deltaFriendlyNewFile, supportedDeltaFormats); } } private PreDiffPlan generatePreDiffPlan( - File oldFile, - File newFile, + ByteSource oldFile, + ByteSource newFile, TempFileHolder deltaFriendlyOldFile, TempFileHolder deltaFriendlyNewFile, Set supportedDeltaFormats) diff --git a/generator/src/main/java/com/google/archivepatcher/generator/MinimalZipArchive.java b/generator/src/main/java/com/google/archivepatcher/generator/MinimalZipArchive.java index d3ae6bfc..4ec0a57c 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/MinimalZipArchive.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/MinimalZipArchive.java @@ -14,9 +14,10 @@ package com.google.archivepatcher.generator; -import com.google.archivepatcher.shared.RandomAccessFileInputStream; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -51,38 +52,44 @@ public int compare(MinimalZipEntry o1, MinimalZipEntry o2) { * @throws IOException if anything goes wrong while reading */ public static List listEntries(File file) throws IOException { - try (RandomAccessFileInputStream in = new RandomAccessFileInputStream(file)) { - return listEntriesInternal(in); + try (ByteSource byteSource = ByteSource.fromFile(file)) { + return listEntries(byteSource); } } /** - * Internal implementation of {@link #listEntries(File)}. - * @param in the input stream to read from - * @return see {@link #listEntries(File)} + * Generate a listing of all of the files in a zip archive in file order and return it. Each entry + * is a {@link MinimalZipEntry}, which has just enough information to generate a patch. + * + * @param data the zip file to read + * @return such a listing * @throws IOException if anything goes wrong while reading */ - private static List listEntriesInternal(RandomAccessFileInputStream in) - throws IOException { + public static List listEntries(ByteSource data) throws IOException { // Step 1: Locate the end-of-central-directory record header. - long offsetOfEocd = MinimalZipParser.locateStartOfEocd(in, 32768); + long offsetOfEocd = MinimalZipParser.locateStartOfEocd(data, 32768); if (offsetOfEocd == -1) { // Archive is weird, abort. throw new ZipException("EOCD record not found in last 32k of archive, giving up"); } // Step 2: Parse the end-of-central-directory data to locate the central directory itself - in.setRange(offsetOfEocd, in.length() - offsetOfEocd); - MinimalCentralDirectoryMetadata centralDirectoryMetadata = MinimalZipParser.parseEocd(in); + MinimalCentralDirectoryMetadata centralDirectoryMetadata; + try (InputStream inputStream = data.sliceFrom(offsetOfEocd).openStream()) { + centralDirectoryMetadata = MinimalZipParser.parseEocd(inputStream); + } // Step 3: Extract a list of all central directory entries (contiguous data stream) - in.setRange( - centralDirectoryMetadata.getOffsetOfCentralDirectory(), - centralDirectoryMetadata.getLengthOfCentralDirectory()); List minimalZipEntries = - new ArrayList(centralDirectoryMetadata.getNumEntriesInCentralDirectory()); - for (int x = 0; x < centralDirectoryMetadata.getNumEntriesInCentralDirectory(); x++) { - minimalZipEntries.add(MinimalZipParser.parseCentralDirectoryEntry(in)); + new ArrayList<>(centralDirectoryMetadata.getNumEntriesInCentralDirectory()); + try (InputStream inputStream = + data.slice( + centralDirectoryMetadata.getOffsetOfCentralDirectory(), + centralDirectoryMetadata.getLengthOfCentralDirectory()) + .openStream()) { + for (int x = 0; x < centralDirectoryMetadata.getNumEntriesInCentralDirectory(); x++) { + minimalZipEntries.add(MinimalZipParser.parseCentralDirectoryEntry(inputStream)); + } } // Step 4: Sort the entries in file order, not central directory order. @@ -100,9 +107,12 @@ private static List listEntriesInternal(RandomAccessFileInputSt offsetOfNextEntry = centralDirectoryMetadata.getOffsetOfCentralDirectory(); } long rangeLength = offsetOfNextEntry - entry.getFileOffsetOfLocalEntry(); - in.setRange(entry.getFileOffsetOfLocalEntry(), rangeLength); - long relativeDataOffset = MinimalZipParser.parseLocalEntryAndGetCompressedDataOffset(in); - entry.setFileOffsetOfCompressedData(entry.getFileOffsetOfLocalEntry() + relativeDataOffset); + try (InputStream inputStream = + data.slice(entry.getFileOffsetOfLocalEntry(), rangeLength).openStream()) { + long relativeDataOffset = + MinimalZipParser.parseLocalEntryAndGetCompressedDataOffset(inputStream); + entry.setFileOffsetOfCompressedData(entry.getFileOffsetOfLocalEntry() + relativeDataOffset); + } } // Done! diff --git a/generator/src/main/java/com/google/archivepatcher/generator/MinimalZipParser.java b/generator/src/main/java/com/google/archivepatcher/generator/MinimalZipParser.java index 0d3f955d..4586bb81 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/MinimalZipParser.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/MinimalZipParser.java @@ -14,7 +14,7 @@ package com.google.archivepatcher.generator; -import com.google.archivepatcher.shared.RandomAccessFileInputStream; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.IOException; import java.io.InputStream; import java.util.zip.ZipException; @@ -224,25 +224,26 @@ public static long parseLocalEntryAndGetCompressedDataOffset(InputStream in) thr // The extra field length can be different here versus in the central directory and is used for // things like zipaligning APKs. This single value is the critical part as it dictates where the // actual DATA for the entry begins. - return 4 + junkLength + 2 + 2 + fileNameLength + extrasLength; + return 4L + junkLength + 2 + 2 + fileNameLength + extrasLength; } /** * Find the end-of-central-directory record by scanning backwards from the end of a file looking * for the signature of the record. + * * @param in the file to read from * @param searchBufferLength the length of the search buffer, starting from the end of the file * @return the offset in the file at which the first byte of the EOCD signature is located, or -1 - * if the signature is not found in the search buffer + * if the signature is not found in the search buffer * @throws IOException if there is a problem reading */ - public static long locateStartOfEocd(RandomAccessFileInputStream in, int searchBufferLength) - throws IOException { + public static long locateStartOfEocd(ByteSource in, int searchBufferLength) throws IOException { final int maxBufferSize = (int) Math.min(searchBufferLength, in.length()); final byte[] buffer = new byte[maxBufferSize]; final long rangeStart = in.length() - buffer.length; - in.setRange(rangeStart, buffer.length); - readOrDie(in, buffer, 0, buffer.length); + try (InputStream inputStream = in.slice(rangeStart, buffer.length).openStream()) { + readOrDie(inputStream, buffer, 0, buffer.length); + } int offset = locateStartOfEocd(buffer); if (offset == -1) { return -1; diff --git a/generator/src/main/java/com/google/archivepatcher/generator/PreDiffExecutor.java b/generator/src/main/java/com/google/archivepatcher/generator/PreDiffExecutor.java index 20e528c0..aa6cbec1 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/PreDiffExecutor.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/PreDiffExecutor.java @@ -19,6 +19,7 @@ import com.google.archivepatcher.shared.JreDeflateParameters; import com.google.archivepatcher.shared.PatchConstants.DeltaFormat; import com.google.archivepatcher.shared.TypedRange; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; @@ -42,8 +43,8 @@ public static final class Builder { private final List preDiffPlanEntryModifiers = new ArrayList<>(); private final Set supportedDeltaFormats = new HashSet<>(); - private File originalOldFile; - private File originalNewFile; + private ByteSource originalOldBlob; + private ByteSource originalNewBlob; private File deltaFriendlyOldFile; private File deltaFriendlyNewFile; @@ -51,15 +52,15 @@ public static final class Builder { * Sets the original, read-only input files to the patch generation process. This has to be * called at least once, and both arguments must be non-null. * - * @param originalOldFile the original old file to read (will not be modified). - * @param originalNewFile the original new file to read (will not be modified). + * @param originalOldBlob the original old blob to read (will not be modified). + * @param originalNewBlob the original new blob to read (will not be modified). */ - public Builder readingOriginalFiles(File originalOldFile, File originalNewFile) { - if (originalOldFile == null || originalNewFile == null) { + public Builder readingOriginalFiles(ByteSource originalOldBlob, ByteSource originalNewBlob) { + if (originalOldBlob == null || originalNewBlob == null) { throw new IllegalStateException("do not set nul original input files"); } - this.originalOldFile = originalOldFile; - this.originalNewFile = originalNewFile; + this.originalOldBlob = originalOldBlob; + this.originalNewBlob = originalNewBlob; return this; } @@ -124,13 +125,13 @@ public Builder addSupportedDeltaFormats(Collection supportedDeltaFo * Builds and returns a {@link PreDiffExecutor} according to the currnet configuration. */ public PreDiffExecutor build() { - if (originalOldFile == null) { + if (originalOldBlob == null) { // readingOriginalFiles() ensures old and new are non-null when called, so check either. throw new IllegalStateException("original input files cannot be null"); } return new PreDiffExecutor( - originalOldFile, - originalNewFile, + originalOldBlob, + originalNewBlob, deltaFriendlyOldFile, deltaFriendlyNewFile, preDiffPlanEntryModifiers, @@ -139,10 +140,10 @@ public PreDiffExecutor build() { } /** The original old file to read (will not be modified). */ - private final File originalOldFile; + private final ByteSource originalOldBlob; /** The original new file to read (will not be modified). */ - private final File originalNewFile; + private final ByteSource originalNewBlob; /** * Optional file to write the delta-friendly version of the original old file to (will be created, @@ -166,14 +167,14 @@ public PreDiffExecutor build() { /** Constructs a new PreDiffExecutor to work with the specified configuration. */ private PreDiffExecutor( - File originalOldFile, - File originalNewFile, + ByteSource originalOldBlob, + ByteSource originalNewBlob, File deltaFriendlyOldFile, File deltaFriendlyNewFile, List preDiffPlanEntryModifiers, Set supportedDeltaFormats) { - this.originalOldFile = originalOldFile; - this.originalNewFile = originalNewFile; + this.originalOldBlob = originalOldBlob; + this.originalNewBlob = originalNewBlob; this.deltaFriendlyOldFile = deltaFriendlyOldFile; this.deltaFriendlyNewFile = deltaFriendlyNewFile; this.preDiffPlanEntryModifiers = preDiffPlanEntryModifiers; @@ -213,12 +214,12 @@ private List> generateDeltaFriendlyFiles(PreDif try (BufferedOutputStream bufferedOut = new BufferedOutputStream(new FileOutputStream(deltaFriendlyOldFile))) { DeltaFriendlyFile.generateDeltaFriendlyFile( - preDiffPlan.getOldFileUncompressionPlan(), originalOldFile, bufferedOut); + preDiffPlan.getOldFileUncompressionPlan(), originalOldBlob, bufferedOut); } try (BufferedOutputStream bufferedOut = new BufferedOutputStream(new FileOutputStream(deltaFriendlyNewFile))) { return DeltaFriendlyFile.generateDeltaFriendlyFile( - preDiffPlan.getNewFileUncompressionPlan(), originalNewFile, bufferedOut); + preDiffPlan.getNewFileUncompressionPlan(), originalNewBlob, bufferedOut); } } @@ -232,7 +233,7 @@ private List> generateDeltaFriendlyFiles(PreDif */ private PreDiffPlan generatePreDiffPlan() throws IOException { List originalOldArchiveZipEntries = - MinimalZipArchive.listEntries(originalOldFile); + MinimalZipArchive.listEntries(originalOldBlob); Map originalOldArchiveZipEntriesByPath = new HashMap(originalOldArchiveZipEntries.size()); for (MinimalZipEntry zipEntry : originalOldArchiveZipEntries) { @@ -241,7 +242,7 @@ private PreDiffPlan generatePreDiffPlan() throws IOException { } List divinationResults = - DefaultDeflateCompressionDiviner.divineDeflateParameters(originalNewFile); + DefaultDeflateCompressionDiviner.divineDeflateParameters(originalNewBlob); Map originalNewArchiveZipEntriesByPath = new HashMap(divinationResults.size()); Map originalNewArchiveJreDeflateParametersByPath = @@ -255,9 +256,9 @@ private PreDiffPlan generatePreDiffPlan() throws IOException { PreDiffPlanner preDiffPlanner = new PreDiffPlanner( - originalOldFile, + originalOldBlob, originalOldArchiveZipEntriesByPath, - originalNewFile, + originalNewBlob, originalNewArchiveZipEntriesByPath, originalNewArchiveJreDeflateParametersByPath, preDiffPlanEntryModifiers, diff --git a/generator/src/main/java/com/google/archivepatcher/generator/PreDiffPlanEntryModifier.java b/generator/src/main/java/com/google/archivepatcher/generator/PreDiffPlanEntryModifier.java index 37e87af6..e6e9e039 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/PreDiffPlanEntryModifier.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/PreDiffPlanEntryModifier.java @@ -14,7 +14,7 @@ package com.google.archivepatcher.generator; -import java.io.File; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.util.List; /** @@ -35,5 +35,5 @@ public interface PreDiffPlanEntryModifier { * @return the updated list of {@link PreDiffPlanEntry} */ List getModifiedPreDiffPlanEntries( - File oldFile, File newFile, List originalEntries); + ByteSource oldFile, ByteSource newFile, List originalEntries); } diff --git a/generator/src/main/java/com/google/archivepatcher/generator/PreDiffPlanner.java b/generator/src/main/java/com/google/archivepatcher/generator/PreDiffPlanner.java index 7d8c202e..95b71367 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/PreDiffPlanner.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/PreDiffPlanner.java @@ -33,10 +33,10 @@ import com.google.archivepatcher.shared.JreDeflateParameters; import com.google.archivepatcher.shared.PatchConstants.DeltaFormat; import com.google.archivepatcher.shared.TypedRange; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -49,15 +49,11 @@ */ class PreDiffPlanner { - /** - * The old archive. - */ - private final File oldFile; + /** The old archive. */ + private final ByteSource oldFile; - /** - * The new archive. - */ - private final File newFile; + /** The new archive. */ + private final ByteSource newFile; /** * The entries in the old archive, with paths as keys. @@ -94,9 +90,9 @@ class PreDiffPlanner { * PreDiffPlan} is generated in {@link #generatePreDiffPlan()}. */ PreDiffPlanner( - File oldFile, + ByteSource oldFile, Map oldArchiveZipEntriesByPath, - File newFile, + ByteSource newFile, Map newArchiveZipEntriesByPath, Map newArchiveJreDeflateParametersByPath, List preDiffPlanEntryModifiers, @@ -359,14 +355,14 @@ private boolean compressedBytesIdentical(MinimalZipEntry oldEntry, MinimalZipEnt // Length is not the same, so content cannot match. return false; } - try (FileInputStream oldFileInputStream = new FileInputStream(oldFile); + try (InputStream oldFileInputStream = + oldFile.sliceFrom(oldEntry.getFileOffsetOfCompressedData()).openStream(); BufferedInputStream oldFileBufferedInputStream = new BufferedInputStream(oldFileInputStream); - FileInputStream newFileInputStream = new FileInputStream(newFile); + InputStream newFileInputStream = + newFile.sliceFrom(newEntry.getFileOffsetOfCompressedData()).openStream(); BufferedInputStream newFileBufferedInputStream = new BufferedInputStream(newFileInputStream)) { - oldFileBufferedInputStream.skip(oldEntry.getFileOffsetOfCompressedData()); - newFileBufferedInputStream.skip(newEntry.getFileOffsetOfCompressedData()); for (int i = 0; i < oldEntry.getCompressedSize(); ++i) { if (oldFileBufferedInputStream.read() != newFileBufferedInputStream.read()) { diff --git a/generator/src/main/java/com/google/archivepatcher/generator/TotalRecompressionLimiter.java b/generator/src/main/java/com/google/archivepatcher/generator/TotalRecompressionLimiter.java index 7df206d6..017450f2 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/TotalRecompressionLimiter.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/TotalRecompressionLimiter.java @@ -14,7 +14,7 @@ package com.google.archivepatcher.generator; -import java.io.File; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -82,7 +82,7 @@ public TotalRecompressionLimiter(long maxBytesToRecompress) { @Override public List getModifiedPreDiffPlanEntries( - File oldFile, File newFile, List originalEntries) { + ByteSource oldFile, ByteSource newFile, List originalEntries) { List sorted = new ArrayList(originalEntries); Collections.sort(sorted, Collections.reverseOrder(COMPARATOR)); diff --git a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiff.java b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiff.java index 8c4f874a..c793d224 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiff.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiff.java @@ -14,7 +14,9 @@ package com.google.archivepatcher.generator.bsdiff; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.IOException; +import java.io.InputStream; /** * A Java implementation of the "bsdiff" algorithm based on the BSD-2 licensed source code available @@ -39,24 +41,22 @@ class BsDiff { * @param newData the new data to scan * @param newStart the position in the new data at which to start the scan * @return the number of matching bytes in the two arrays starting at the specified indices; zero - * if the first byte fails to match + * if the first byte fails to match */ // Visible for testing only static int lengthOfMatch( - final RandomAccessObject oldData, - final int oldStart, - final RandomAccessObject newData, - final int newStart) + final ByteSource oldData, final int oldStart, final ByteSource newData, final int newStart) throws IOException { final int max = Math.min((int) oldData.length() - oldStart, (int) newData.length() - newStart); if (max > 0) { // If max is 0, it's sometimes possible for this seek to seek to length + 1 and throw an // exception unnecessarily. - oldData.seek(oldStart); - newData.seek(newStart); - for (int offset = 0; offset < max; offset++) { - if (oldData.readByte() != newData.readByte()) { - return offset; + try (InputStream oldDataInputStream = oldData.sliceFrom(oldStart).openStream(); + InputStream newDataInputStream = newData.sliceFrom(newStart).openStream()) { + for (int offset = 0; offset < max; offset++) { + if (oldDataInputStream.read() != newDataInputStream.read()) { + return offset; + } } } } @@ -67,8 +67,8 @@ static int lengthOfMatch( // Visible for testing only static Match searchForMatchBaseCase( final RandomAccessObject groupArray, - final RandomAccessObject oldData, - final RandomAccessObject newData, + final ByteSource oldData, + final ByteSource newData, final int newStart, final int oldDataRangeStartA, final int oldDataRangeStartB) @@ -92,8 +92,9 @@ static Match searchForMatchBaseCase( } /** - * Locates the run of bytes in |oldData| which matches the longest prefix of - * newData[newStart ... newData.length - 1]. + * Locates the run of bytes in |oldData| which matches the longest prefix of newData[newStart ... + * newData.length - 1]. + * * @param groupArray * @param oldData the old data to scan * @param newData the new data to scan @@ -101,13 +102,13 @@ static Match searchForMatchBaseCase( * @param oldDataRangeStartA * @param oldDataRangeStartB * @return a Match containing the length of the matching range, and the position at which the - * matching range begins. + * matching range begins. */ // Visible for testing only static Match searchForMatch( final RandomAccessObject groupArray, - final RandomAccessObject oldData, - final RandomAccessObject newData, + final ByteSource oldData, + final ByteSource newData, final int newStart, final int oldDataRangeStartA, final int oldDataRangeStartB) diff --git a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffDeltaGenerator.java b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffDeltaGenerator.java index 5c62de04..82642271 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffDeltaGenerator.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffDeltaGenerator.java @@ -18,15 +18,15 @@ import com.google.archivepatcher.generator.DeltaGenerator; import com.google.archivepatcher.generator.bsdiff.wrapper.BsDiffNativePatchWriter; -import java.io.File; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.IOException; import java.io.OutputStream; /** - * An implementation of {@link DeltaGenerator} that uses {@link BsDiffPatchWriter} to write a - * bsdiff patch that represents the delta between given inputs. + * An implementation of {@link DeltaGenerator} that uses {@link BsDiffPatchWriter} to write a bsdiff + * patch that represents the delta between given inputs. */ -public class BsDiffDeltaGenerator implements DeltaGenerator { +public class BsDiffDeltaGenerator extends DeltaGenerator { /** * The minimum match length to use for bsdiff. */ @@ -44,7 +44,7 @@ public BsDiffDeltaGenerator(boolean useNativeBsDiff) { } @Override - public void generateDelta(File oldBlob, File newBlob, OutputStream deltaOut) + public void generateDelta(ByteSource oldBlob, ByteSource newBlob, OutputStream deltaOut) throws IOException, InterruptedException { if (useNativeBsDiff) { BsDiffNativePatchWriter.generatePatch(oldBlob, newBlob, deltaOut); diff --git a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffMatcher.java b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffMatcher.java index 8a8f90ba..889e72f2 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffMatcher.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffMatcher.java @@ -14,42 +14,43 @@ package com.google.archivepatcher.generator.bsdiff; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.IOException; +import java.io.InputStream; /** - * Implementation of matcher used by BsDiff. Exact matches between newData[a ... a + len - 1] - * and oldData[b ... b + len - 1] are valid if: + * Implementation of matcher used by BsDiff. Exact matches between mNewData[a ... a + len - 1] and + * mOldData[b ... b + len - 1] are valid if: + * *
    - *
  • |len| > mMinimumMatchLength
    • - *
  • The number of matches between newData[a ... a + len - 1] and - * oldData[previous_b ... previous_b + len - 1] < |len| - mMinimumMatchLength where - * |previous_b| is the |b| value of the previous match if there was one and zero otherwise.
    • + *
  • |len| > mMinimumMatchLength + *
  • The number of matches between mNewData[a ... a + len - 1] and mOldData[previous_b ... + * previous_b + len - 1] < |len| - mMinimumMatchLength where |previous_b| is the |b| value of + * the previous match if there was one and zero otherwise. *
*/ class BsDiffMatcher implements Matcher { - private final RandomAccessObject mOldData; - private final RandomAccessObject mNewData; + private final ByteSource mOldData; + private final ByteSource mNewData; /** - * Contains order of the sorted suffixes of |oldData|. The element at mGroupArray[i] contains the - * position of oldData[i ... oldData.length - 1] in the sorted list of suffixes of |oldData|. + * Contains order of the sorted suffixes of |mOldData|. The element at mGroupArray[i] contains the + * position of mOldData[i ... mOldData.length - 1] in the sorted list of suffixes of |mOldData|. */ private final RandomAccessObject mGroupArray; /** - * The index in |oldData| of the first byte of the match. Zero if no matches have been found yet. + * The index in |mOldData| of the first byte of the match. Zero if no matches have been found yet. */ private int mOldPos; /** - * The index in |newData| of the first byte of the match. Zero if no matches have been found yet. + * The index in |mNewData| of the first byte of the match. Zero if no matches have been found yet. * The next match will be searched starting at |mNewPos| + |mMatchLen|. */ private int mNewPos; - /** - * Minimum match length in bytes. - */ + /** Minimum match length in bytes. */ private final int mMinimumMatchLength; /** @@ -59,35 +60,35 @@ class BsDiffMatcher implements Matcher { private final long mTotalMatchLenBudget = 1L << 26; // ~64 million. /** - * The number of bytes, |n|, which match between newData[mNewPos ... mNewPos + n] and - * oldData[mOldPos ... mOldPos + n]. + * The number of bytes, |n|, which match between mNewData[mNewPos ... mNewPos + n] and + * mOldData[mOldPos ... mOldPos + n]. */ private int mMatchLen; /** * Create a standard BsDiffMatcher. + * + * @param oldData + * @param newData * @param minimumMatchLength the minimum "match" (in bytes) for BsDiff to consider between the - * oldData and newData. This can have a significant effect on both the generated patch size and - * the amount of time and memory required to apply the patch. + * mOldData and mNewData. This can have a significant effect on both the generated patch size + * and */ BsDiffMatcher( - RandomAccessObject oldData, - RandomAccessObject newData, + ByteSource oldData, + ByteSource newData, RandomAccessObject groupArray, int minimumMatchLength) { - mOldData = oldData; - mNewData = newData; - mGroupArray = groupArray; - mOldPos = 0; - mMinimumMatchLength = minimumMatchLength; + this.mOldData = oldData; + this.mNewData = newData; + this.mGroupArray = groupArray; + this.mOldPos = 0; + this.mMinimumMatchLength = minimumMatchLength; } @Override public Matcher.NextMatch next() throws IOException, InterruptedException { - RandomAccessObject oldData = mOldData; - RandomAccessObject newData = mNewData; - - // The offset between between the indices in |oldData| and |newData| + // The offset between between the indices in |mOldData| and |mNewData| // of the previous match. int previousOldOffset = mOldPos - mNewPos; @@ -95,8 +96,8 @@ public Matcher.NextMatch next() throws IOException, InterruptedException { mNewPos += mMatchLen; // The number of matching bytes in the forward extension of the previous match: - // oldData[mNewPos + previousOldOffset ... mNewPos + previousOldOffset + mMatchLen - 1] - // and newData[mNewPos ... mNewPos + mMatchLen - 1]. + // mOldData[mNewPos + previousOldOffset ... mNewPos + previousOldOffset + mMatchLen - 1] + // and mNewData[mNewPos ... mNewPos + mMatchLen - 1]. int numMatches = 0; // The size of the range for which |numMatches| has been computed. @@ -105,12 +106,13 @@ public Matcher.NextMatch next() throws IOException, InterruptedException { // Sum over all match lengths encountered, to exit loop if we take too long to compute. long totalMatchLen = 0; - while (mNewPos < newData.length()) { + while (mNewPos < mNewData.length()) { if (Thread.interrupted()) { throw new InterruptedException(); } BsDiff.Match match = - BsDiff.searchForMatch(mGroupArray, oldData, newData, mNewPos, 0, (int) oldData.length()); + BsDiff.searchForMatch( + mGroupArray, mOldData, mNewData, mNewPos, 0, (int) mOldData.length()); mOldPos = match.start; mMatchLen = match.length; totalMatchLen += mMatchLen; @@ -119,12 +121,12 @@ public Matcher.NextMatch next() throws IOException, InterruptedException { for (; matchesCacheSize < mMatchLen; ++matchesCacheSize) { int oldIndex = mNewPos + previousOldOffset + matchesCacheSize; int newIndex = mNewPos + matchesCacheSize; - if (oldIndex < oldData.length()) { - oldData.seek(oldIndex); - newData.seek(newIndex); - - if (oldData.readByte() == newData.readByte()) { - ++numMatches; + if (oldIndex < mOldData.length()) { + try (InputStream oldDataInputStream = mOldData.sliceFrom(oldIndex).openStream(); + InputStream newDataInputStream = mNewData.sliceFrom(newIndex).openStream()) { + if (oldDataInputStream.read() == newDataInputStream.read()) { + ++numMatches; + } } } } @@ -147,14 +149,16 @@ public Matcher.NextMatch next() throws IOException, InterruptedException { // Update |numMatches| for the value of |mNewPos| in the next iteration of the loop. In the // next iteration of the loop, the new value of |numMatches| will be at least // |numMatches - 1| because - // oldData[mNewPos + previousOldOffset + 1 ... mNewPos + previousOldOffset + mMatchLen - 1] - // matches newData[mNewPos + 1 ... mNewPos + mMatchLen - 1]. - if (mNewPos + previousOldOffset < oldData.length()) { - oldData.seek(mNewPos + previousOldOffset); - newData.seek(mNewPos); - - if (oldData.readByte() == newData.readByte()) { - --numMatches; + // mOldData[mNewPos + previousOldOffset + 1 ... mNewPos + previousOldOffset + mMatchLen - 1] + // matches mNewData[mNewPos + 1 ... mNewPos + mMatchLen - 1]. + if (mNewPos + previousOldOffset < mOldData.length()) { + + try (InputStream oldDataInputStream = + mOldData.sliceFrom(mNewPos + previousOldOffset).openStream(); + InputStream newDataInputStream = mNewData.sliceFrom(mNewPos).openStream()) { + if (oldDataInputStream.read() == newDataInputStream.read()) { + --numMatches; + } } } ++mNewPos; diff --git a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffPatchWriter.java b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffPatchWriter.java index a7ad412a..1e57b668 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffPatchWriter.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffPatchWriter.java @@ -14,10 +14,12 @@ package com.google.archivepatcher.generator.bsdiff; +import com.google.archivepatcher.generator.bsdiff.RandomAccessObjectFactory.RandomAccessByteArrayObjectFactory; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; -import java.io.RandomAccessFile; import java.nio.charset.StandardCharsets; // TODO clean up the various generatePatch(...) methods, there are too many. @@ -49,8 +51,8 @@ public class BsDiffPatchWriter { * @throws IOException if unable to read or write data */ private static void writeEntry( - RandomAccessObject newData, - RandomAccessObject oldData, + ByteSource newData, + ByteSource oldData, int newPosition, int oldPosition, int diffLength, @@ -63,23 +65,26 @@ private static void writeEntry( BsUtil.writeFormattedLong(extraLength, outputStream); BsUtil.writeFormattedLong(oldPositionOffsetForNextEntry, outputStream); - newData.seek(newPosition); - oldData.seek(oldPosition); - // Write diff data - for (int i = 0; i < diffLength; ++i) { - // TODO: test using a small buffer to insulate read() calls (and write() for that - // matter). - outputStream.write(newData.readUnsignedByte() - oldData.readUnsignedByte()); + try (InputStream oldDataInputStream = oldData.sliceFrom(oldPosition).openStream(); + InputStream newDataInputStream = newData.sliceFrom(newPosition).openStream()) { + // Write diff data + for (int i = 0; i < diffLength; ++i) { + // TODO: test using a small buffer to insulate read() calls (and write() for that + // matter). + outputStream.write(newDataInputStream.read() - oldDataInputStream.read()); + } } if (extraLength > 0) { // This seek will throw an IOException sometimes, if we try to seek to the byte after // the end of the RandomAccessObject. - newData.seek(newPosition + diffLength); + try (InputStream newDataInputStream = + newData.sliceFrom(newPosition + diffLength).openStream()) { // Write extra data for (int i = 0; i < extraLength; ++i) { - // TODO: same as above - test buffering readByte(). - outputStream.write(newData.readByte()); + // TODO: same as above - test buffering readByte(). + outputStream.write(newDataInputStream.read()); + } } } } @@ -96,10 +101,7 @@ private static void writeEntry( */ // Visible for testing only static void generatePatchWithMatcher( - RandomAccessObject oldData, - RandomAccessObject newData, - Matcher matcher, - OutputStream outputStream) + ByteSource oldData, ByteSource newData, Matcher matcher, OutputStream outputStream) throws IOException, InterruptedException { // Compute the differences, writing ctrl as we go int lastNewPosition = 0; @@ -127,12 +129,13 @@ static void generatePatchWithMatcher( int score = 0; int bestScore = 0; for (int i = 1; newPosition - i >= lastNewPosition && oldPosition >= i; ++i) { - oldData.seek(oldPosition - i); - newData.seek(newPosition - i); - if (oldData.readByte() == newData.readByte()) { - ++score; - } else { - --score; + try (InputStream oldDataInputStream = oldData.sliceFrom(oldPosition - i).openStream(); + InputStream newDataInputStream = newData.sliceFrom(newPosition - i).openStream()) { + if (oldDataInputStream.read() == newDataInputStream.read()) { + ++score; + } else { + --score; + } } if (score > bestScore) { @@ -149,19 +152,20 @@ static void generatePatchWithMatcher( { int score = 0; int bestScore = 0; - oldData.seek(lastOldPosition); - newData.seek(lastNewPosition); - for (int i = 0; - lastNewPosition + i < newPosition && lastOldPosition + i < oldData.length(); - ++i) { - if (oldData.readByte() == newData.readByte()) { - ++score; - } else { - --score; - } - if (score > bestScore) { - bestScore = score; - forwardExtension = i + 1; + try (InputStream oldDataInputStream = oldData.sliceFrom(lastOldPosition).openStream(); + InputStream newDataInputStream = newData.sliceFrom(lastNewPosition).openStream()) { + for (int i = 0; + lastNewPosition + i < newPosition && lastOldPosition + i < oldData.length(); + ++i) { + if (oldDataInputStream.read() == newDataInputStream.read()) { + ++score; + } else { + --score; + } + if (score > bestScore) { + bestScore = score; + forwardExtension = i + 1; + } } } } @@ -174,16 +178,24 @@ static void generatePatchWithMatcher( int bestScore = 0; int backwardExtensionDecrement = 0; for (int i = 0; i < overlap; ++i) { - newData.seek(lastNewPosition + forwardExtension - overlap + i); - oldData.seek(lastOldPosition + forwardExtension - overlap + i); - if (newData.readByte() == oldData.readByte()) { - ++score; + try (InputStream oldDataInputStream = + oldData.sliceFrom(lastOldPosition + forwardExtension - overlap + i).openStream(); + InputStream newDataInputStream = + newData + .sliceFrom(lastNewPosition + forwardExtension - overlap + i) + .openStream()) { + if (newDataInputStream.read() == oldDataInputStream.read()) { + ++score; + } } - newData.seek(newPosition - backwardExtension + i); - oldData.seek(oldPosition - backwardExtension + i); - if (newData.readByte() == oldData.readByte()) { - --score; + try (InputStream oldDataInputStream = + oldData.sliceFrom(oldPosition - backwardExtension + i).openStream(); + InputStream newDataInputStream = + newData.sliceFrom(newPosition - backwardExtension + i).openStream()) { + if (newDataInputStream.read() == oldDataInputStream.read()) { + --score; + } } if (score > bestScore) { bestScore = score; @@ -230,27 +242,6 @@ static void generatePatchWithMatcher( } } - /** - * Generate a diff between the old data and the new, writing to the specified stream. Uses {@link - * #DEFAULT_MINIMUM_MATCH_LENGTH} as the match length. - * - * @param oldData the old data - * @param newData the new data - * @param outputStream where output should be written - * @param randomAccessObjectFactory factory to create auxiliary storage during BsDiff - * @throws IOException if unable to read or write data - * @throws InterruptedException if any thread interrupts this thread - */ - public static void generatePatch( - final RandomAccessObject oldData, - final RandomAccessObject newData, - final OutputStream outputStream, - final RandomAccessObjectFactory randomAccessObjectFactory) - throws IOException, InterruptedException { - generatePatch( - oldData, newData, outputStream, randomAccessObjectFactory, DEFAULT_MINIMUM_MATCH_LENGTH); - } - /** * Generate a diff between the old data and the new, writing to the specified stream. Uses * in-memory byte array storage for ancillary allocations and {@link @@ -287,15 +278,13 @@ public static void generatePatch( final OutputStream outputStream, final int minimumMatchLength) throws IOException, InterruptedException { - try (RandomAccessObject oldDataRAO = - new RandomAccessObject.RandomAccessByteArrayObject(oldData); - RandomAccessObject newDataRAO = - new RandomAccessObject.RandomAccessByteArrayObject(newData); ) { + try (ByteSource oldByteSource = ByteSource.wrap(oldData); + ByteSource newByteSource = ByteSource.wrap(newData)) { generatePatch( - oldDataRAO, - newDataRAO, + oldByteSource, + newByteSource, outputStream, - new RandomAccessObjectFactory.RandomAccessByteArrayObjectFactory(), + new RandomAccessByteArrayObjectFactory(), minimumMatchLength); } } @@ -305,16 +294,19 @@ public static void generatePatch( * file-based storage for ancillary operations and {@link #DEFAULT_MINIMUM_MATCH_LENGTH} as the * match length. * - * @param oldData a file containing the old data - * @param newData a file containing the new data + * @param oldFile a file containing the old data + * @param newFile a file containing the new data * @param outputStream where output should be written * @throws IOException if unable to read or write data * @throws InterruptedException if any thread interrupts this thread */ public static void generatePatch( - final File oldData, final File newData, final OutputStream outputStream) + final File oldFile, final File newFile, final OutputStream outputStream) throws IOException, InterruptedException { - generatePatch(oldData, newData, outputStream, DEFAULT_MINIMUM_MATCH_LENGTH); + try (ByteSource oldData = ByteSource.fromFile(oldFile); + ByteSource newData = ByteSource.fromFile(newFile)) { + generatePatch(oldData, newData, outputStream, DEFAULT_MINIMUM_MATCH_LENGTH); + } } /** @@ -331,30 +323,17 @@ public static void generatePatch( * @throws InterruptedException if any thread interrupts this thread */ public static void generatePatch( - final File oldData, - final File newData, + final ByteSource oldData, + final ByteSource newData, final OutputStream outputStream, final int minimumMatchLength) throws IOException, InterruptedException { - try (RandomAccessFile oldDataRAF = new RandomAccessFile(oldData, "r"); - RandomAccessFile newDataRAF = new RandomAccessFile(newData, "r"); - RandomAccessObject oldDataRAO = - new RandomAccessObject.RandomAccessMmapObject(oldDataRAF, "r"); - RandomAccessObject newDataRAO = - new RandomAccessObject.RandomAccessMmapObject(newDataRAF, "r"); ) { - generatePatch( - oldDataRAO, - newDataRAO, - outputStream, - new RandomAccessObjectFactory.RandomAccessMmapObjectFactory("rw"), - minimumMatchLength); - } - - // Due to a bug in the JVM (http://bugs.java.com/view_bug.do?bug_id=6417205), we need to call - // gc() and runFinalization() explicitly to get rid of any MappedByteBuffers we may have used - // during patch generation. - System.gc(); - System.runFinalization(); + generatePatch( + oldData, + newData, + outputStream, + new RandomAccessObjectFactory.RandomAccessMmapObjectFactory("rw"), + minimumMatchLength); } /** @@ -371,8 +350,8 @@ public static void generatePatch( * @throws InterruptedException if any thread interrupts this thread */ public static void generatePatch( - final RandomAccessObject oldData, - final RandomAccessObject newData, + final ByteSource oldData, + final ByteSource newData, final OutputStream outputStream, final RandomAccessObjectFactory randomAccessObjectFactory, final int minimumMatchLength) diff --git a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsUtil.java b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsUtil.java index 007c0583..89322cb2 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsUtil.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsUtil.java @@ -14,6 +14,7 @@ package com.google.archivepatcher.generator.bsdiff; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -77,27 +78,28 @@ static long readFormattedLong(InputStream inputStream) throws IOException { * @return result of lexicographical compare: negative if the first difference has a lower value * in the first array, positive if the first difference has a lower value in the second array. * If both arrays compare equal until one of them ends, the shorter sequence is - * lexicographically less than the longer one (i.e., it returns len(first array) - - * len(second array)). + * lexicographically less than the longer one (i.e., it returns len(first array) - len(second + * array)). */ static int lexicographicalCompare( - final RandomAccessObject data1, + final ByteSource data1, final int start1, final int length1, - final RandomAccessObject data2, + final ByteSource data2, final int start2, final int length2) throws IOException { int bytesLeft = Math.min(length1, length2); - data1.seek(start1); - data2.seek(start2); - while (bytesLeft-- > 0) { - final int i1 = data1.readUnsignedByte(); - final int i2 = data2.readUnsignedByte(); + try (InputStream data1InputStream = data1.sliceFrom(start1).openStream(); + InputStream data2InputStream = data2.sliceFrom(start2).openStream()) { + while (bytesLeft-- > 0) { + final int i1 = data1InputStream.read(); + final int i2 = data2InputStream.read(); - if (i1 != i2) { - return i1 - i2; + if (i1 != i2) { + return i1 - i2; + } } } diff --git a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/DivSuffixSorter.java b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/DivSuffixSorter.java index 8583327a..c3f3de5d 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/DivSuffixSorter.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/DivSuffixSorter.java @@ -16,7 +16,9 @@ package com.google.archivepatcher.generator.bsdiff; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.IOException; +import java.io.InputStream; /** * Taken from @@ -78,14 +80,14 @@ public final class DivSuffixSorter implements SuffixSorter { private final RandomAccessObjectFactory randomAccessObjectFactory; private RandomAccessObject suffixArray; - private RandomAccessObject input; + private ByteSource input; public DivSuffixSorter(RandomAccessObjectFactory randomAccessObjectFactory) { this.randomAccessObjectFactory = randomAccessObjectFactory; } @Override - public RandomAccessObject suffixSort(RandomAccessObject input) throws IOException, InterruptedException { + public RandomAccessObject suffixSort(ByteSource input) throws IOException, InterruptedException { if (4 * (input.length() + 1) >= Integer.MAX_VALUE) { throw new IllegalArgumentException("Input too large (" + input.length() + " bytes)"); } @@ -2059,8 +2061,9 @@ public TRPartitionResult(int a, int b) { } private int readInput(long pos) throws IOException { - input.seek(pos); - return input.readUnsignedByte(); + try (InputStream in = input.sliceFrom(pos).openStream()) { + return in.read(); + } } private int readSuffixArray(long pos) throws IOException { diff --git a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/SuffixSorter.java b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/SuffixSorter.java index 704b0182..0571c2f9 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/SuffixSorter.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/SuffixSorter.java @@ -14,6 +14,7 @@ package com.google.archivepatcher.generator.bsdiff; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.IOException; /** @@ -26,11 +27,11 @@ public interface SuffixSorter { * Perform a "suffix sort". Note: the returned {@link RandomAccessObject} should be closed by the * caller. * - * @param data the data to sort + * @param input the data to sort * @return the suffix array, as a {@link RandomAccessObject} * @throws IOException if unable to read data * @throws InterruptedException if any thread interrupts this thread */ - RandomAccessObject suffixSort(RandomAccessObject data) throws IOException, InterruptedException; + RandomAccessObject suffixSort(ByteSource input) throws IOException, InterruptedException; } diff --git a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/wrapper/BsDiffNativePatchWriter.java b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/wrapper/BsDiffNativePatchWriter.java index 8c0cdd1d..f6f0aedc 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/wrapper/BsDiffNativePatchWriter.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/wrapper/BsDiffNativePatchWriter.java @@ -14,12 +14,34 @@ package com.google.archivepatcher.generator.bsdiff.wrapper; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; /** Generates BsDiff patches using the native implementation, by using JNI. */ public class BsDiffNativePatchWriter { + /** + * Generate a diff between the old file and the new, writing to the specified stream. Uses the + * default match length of 8. + * + * @param oldFile the old file + * @param newFile the new file + * @param deltaOut where output should be written + * @throws IOException if unable to read or write data + */ + public static void generatePatch(File oldFile, File newFile, OutputStream deltaOut) + throws IOException { + byte[] patch = nativeGeneratePatchFile(oldFile.getPath(), newFile.getPath()); + + if (patch == null) { + throw new IllegalStateException("Unable to generate patch."); + } + + deltaOut.write(patch); + } + /** * Generate a diff between the old data and the new, writing to the specified stream. Uses the * default match length of 8. @@ -29,9 +51,17 @@ public class BsDiffNativePatchWriter { * @param deltaOut where output should be written * @throws IOException if unable to read or write data */ - public static void generatePatch(File oldBlob, File newBlob, OutputStream deltaOut) + public static void generatePatch(ByteSource oldBlob, ByteSource newBlob, OutputStream deltaOut) throws IOException { - byte[] patch = nativeGeneratePatchFile(oldBlob.getPath(), newBlob.getPath()); + byte[] oldData = new byte[(int) oldBlob.length()]; + byte[] newData = new byte[(int) newBlob.length()]; + try (InputStream in = oldBlob.openStream()) { + in.read(oldData); + } + try (InputStream in = newBlob.openStream()) { + in.read(newData); + } + byte[] patch = nativeGeneratePatchData(oldData, newData); if (patch == null) { throw new IllegalStateException("Unable to generate patch."); diff --git a/generator/src/main/java/com/google/archivepatcher/generator/similarity/Crc32SimilarityFinder.java b/generator/src/main/java/com/google/archivepatcher/generator/similarity/Crc32SimilarityFinder.java index 0286e755..17a0e135 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/similarity/Crc32SimilarityFinder.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/similarity/Crc32SimilarityFinder.java @@ -15,7 +15,7 @@ package com.google.archivepatcher.generator.similarity; import com.google.archivepatcher.generator.MinimalZipEntry; -import java.io.File; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -37,10 +37,11 @@ public class Crc32SimilarityFinder extends SimilarityFinder { /** * Constructs a new similarity finder with the specified parameters. + * * @param baseArchive the base archive that contains the entries to be searched * @param baseEntries the entries in the base archive that are eligible to be searched */ - public Crc32SimilarityFinder(File baseArchive, Collection baseEntries) { + public Crc32SimilarityFinder(ByteSource baseArchive, Collection baseEntries) { super(baseArchive, baseEntries); for (MinimalZipEntry oldEntry : baseEntries) { long crc32 = oldEntry.getCrc32OfUncompressedData(); @@ -54,7 +55,7 @@ public Crc32SimilarityFinder(File baseArchive, Collection baseE } @Override - public List findSimilarFiles(File newArchive, MinimalZipEntry newEntry) { + public List findSimilarFiles(ByteSource newArchive, MinimalZipEntry newEntry) { List matchedEntries = baseEntriesByCrc32.get(newEntry.getCrc32OfUncompressedData()); if (matchedEntries == null) { diff --git a/generator/src/main/java/com/google/archivepatcher/generator/similarity/SimilarityFinder.java b/generator/src/main/java/com/google/archivepatcher/generator/similarity/SimilarityFinder.java index 5acf61a4..33471ef4 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/similarity/SimilarityFinder.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/similarity/SimilarityFinder.java @@ -15,7 +15,7 @@ package com.google.archivepatcher.generator.similarity; import com.google.archivepatcher.generator.MinimalZipEntry; -import java.io.File; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.util.Collection; import java.util.List; @@ -24,10 +24,8 @@ */ public abstract class SimilarityFinder { - /** - * The base archive that contains the entries to be searched. - */ - protected final File baseArchive; + /** The base archive that contains the entries to be searched. */ + protected final ByteSource baseArchive; /** * The entries in the base archive that are eligible to be searched. @@ -37,10 +35,11 @@ public abstract class SimilarityFinder { /** * Create a new instance to check for similarity of arbitrary files against the specified entries * in the specified archive. + * * @param baseArchive the base archive that contains the entries to be scored against * @param baseEntries the entries in the base archive that are eligible to be scored against. */ - public SimilarityFinder(File baseArchive, Collection baseEntries) { + public SimilarityFinder(ByteSource baseArchive, Collection baseEntries) { this.baseArchive = baseArchive; this.baseEntries = baseEntries; } @@ -48,11 +47,13 @@ public SimilarityFinder(File baseArchive, Collection baseEntrie /** * Searches for files similar to the specified entry in the specified new archive against all of * the available entries in the base archive. + * * @param newArchive the new archive that contains the new entry * @param newEntry the new entry to compare against the entries in the base archive * @return a {@link List} of {@link MinimalZipEntry} entries (possibly empty but never null) from - * the base archive that are similar to the new archive; if the list has more than one entry, the - * entries should be in order from most similar to least similar. + * the base archive that are similar to the new archive; if the list has more than one entry, + * the entries should be in order from most similar to least similar. */ - public abstract List findSimilarFiles(File newArchive, MinimalZipEntry newEntry); + public abstract List findSimilarFiles( + ByteSource newArchive, MinimalZipEntry newEntry); } diff --git a/generator/src/test/java/com/google/archivepatcher/generator/DefaultDeflateCompressionDivinerTest.java b/generator/src/test/java/com/google/archivepatcher/generator/DefaultDeflateCompressionDivinerTest.java index 0b817952..accb17c0 100644 --- a/generator/src/test/java/com/google/archivepatcher/generator/DefaultDeflateCompressionDivinerTest.java +++ b/generator/src/test/java/com/google/archivepatcher/generator/DefaultDeflateCompressionDivinerTest.java @@ -22,6 +22,7 @@ import com.google.archivepatcher.shared.JreDeflateParameters; import com.google.archivepatcher.shared.UnitTestZipArchive; import com.google.archivepatcher.shared.UnitTestZipEntry; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -68,10 +69,9 @@ private byte[] deflate(JreDeflateParameters parameters) throws IOException { @Test public void testDivineDeflateParameters_JunkData() throws IOException { final byte[] junk = new byte[] {0, 1, 2, 3, 4}; - assertThat( - DefaultDeflateCompressionDiviner.divineDeflateParameters( - new ByteArrayInputStreamFactory(junk))) - .isNull(); + try (ByteSource entry = ByteSource.wrap(junk)) { + assertThat(DefaultDeflateCompressionDiviner.divineDeflateParametersForEntry(entry)).isNull(); + } } @Test @@ -81,9 +81,11 @@ public void testDivineDeflateParameters_AllValidSettings() throws IOException { for (int level : new int[] {1, 2, 3, 4, 5, 6, 7, 8, 9}) { JreDeflateParameters trueParameters = JreDeflateParameters.of(level, strategy, nowrap); final byte[] buffer = deflate(trueParameters); - JreDeflateParameters divinedParameters = - DefaultDeflateCompressionDiviner.divineDeflateParameters( - new ByteArrayInputStreamFactory(buffer)); + JreDeflateParameters divinedParameters; + try (ByteSource byteSource = ByteSource.wrap(buffer)) { + divinedParameters = + DefaultDeflateCompressionDiviner.divineDeflateParametersForEntry(byteSource); + } assertThat(divinedParameters).isNotNull(); // TODO make *CERTAIN 100%( that strategy doesn't matter for level < 4. if (strategy == 1 && level <= 3) { @@ -106,10 +108,11 @@ public void testDivineDeflateParameters_AllValidSettings() throws IOException { public void testDivineDeflateParameters_File() throws IOException { File tempFile = File.createTempFile("ddcdt", "tmp"); tempFile.deleteOnExit(); - try { - UnitTestZipArchive.saveTestZip(tempFile); + UnitTestZipArchive.saveTestZip(tempFile); + + try (ByteSource tempBlob = ByteSource.fromFile(tempFile)) { List results = - DefaultDeflateCompressionDiviner.divineDeflateParameters(tempFile); + DefaultDeflateCompressionDiviner.divineDeflateParameters(tempBlob); assertThat(results).hasSize(UnitTestZipArchive.allEntriesInFileOrder.size()); for (int x = 0; x < results.size(); x++) { UnitTestZipEntry expected = UnitTestZipArchive.allEntriesInFileOrder.get(x); diff --git a/generator/src/test/java/com/google/archivepatcher/generator/DeltaFriendlyOldBlobSizeLimiterTest.java b/generator/src/test/java/com/google/archivepatcher/generator/DeltaFriendlyOldBlobSizeLimiterTest.java index 7f07b2eb..f0fa0d88 100644 --- a/generator/src/test/java/com/google/archivepatcher/generator/DeltaFriendlyOldBlobSizeLimiterTest.java +++ b/generator/src/test/java/com/google/archivepatcher/generator/DeltaFriendlyOldBlobSizeLimiterTest.java @@ -23,6 +23,7 @@ import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.File; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -138,17 +139,20 @@ public void testNegativeLimit() { } } - private File tempFile = null; + private File tempFile; + private ByteSource tempFileBlob = null; @Before public void setup() throws IOException { // Make an empty file to test the recommender's limitation logic tempFile = File.createTempFile("DeltaFriendlyOldBlobSizeLimiterTest", "test"); tempFile.deleteOnExit(); + tempFileBlob = ByteSource.fromFile(tempFile); } @After - public void tearDown() { + public void tearDown() throws Exception { + tempFileBlob.close(); tempFile.delete(); } @@ -163,14 +167,18 @@ public void testZeroLimit() { PRE_DIFF_PLAN_ENTRY_C_300K, PRE_DIFF_PLAN_ENTRY_D_400K)); expected.addAll(ALL_IGNORED_PRE_DIFF_PLAN_ENTRIES); - assertThat(limiter.getModifiedPreDiffPlanEntries(tempFile, tempFile, ALL_PRE_DIFF_PLAN_ENTRIES)) + assertThat( + limiter.getModifiedPreDiffPlanEntries( + tempFileBlob, tempFileBlob, ALL_PRE_DIFF_PLAN_ENTRIES)) .containsExactlyElementsIn(expected); } @Test public void testMaxLimit() { DeltaFriendlyOldBlobSizeLimiter limiter = new DeltaFriendlyOldBlobSizeLimiter(Long.MAX_VALUE); - assertThat(limiter.getModifiedPreDiffPlanEntries(tempFile, tempFile, ALL_PRE_DIFF_PLAN_ENTRIES)) + assertThat( + limiter.getModifiedPreDiffPlanEntries( + tempFileBlob, tempFileBlob, ALL_PRE_DIFF_PLAN_ENTRIES)) .containsExactlyElementsIn(ALL_PRE_DIFF_PLAN_ENTRIES); } @@ -186,7 +194,9 @@ public void testLimit_ExactlySmallest() { suppressed( PRE_DIFF_PLAN_ENTRY_B_200K, PRE_DIFF_PLAN_ENTRY_C_300K, PRE_DIFF_PLAN_ENTRY_D_400K)); expected.addAll(ALL_IGNORED_PRE_DIFF_PLAN_ENTRIES); - assertThat(limiter.getModifiedPreDiffPlanEntries(tempFile, tempFile, ALL_PRE_DIFF_PLAN_ENTRIES)) + assertThat( + limiter.getModifiedPreDiffPlanEntries( + tempFileBlob, tempFileBlob, ALL_PRE_DIFF_PLAN_ENTRIES)) .containsExactlyElementsIn(expected); } @@ -205,7 +215,9 @@ public void testLimit_EdgeUnderSmallest() { PRE_DIFF_PLAN_ENTRY_C_300K, PRE_DIFF_PLAN_ENTRY_D_400K)); expected.addAll(ALL_IGNORED_PRE_DIFF_PLAN_ENTRIES); - assertThat(limiter.getModifiedPreDiffPlanEntries(tempFile, tempFile, ALL_PRE_DIFF_PLAN_ENTRIES)) + assertThat( + limiter.getModifiedPreDiffPlanEntries( + tempFileBlob, tempFileBlob, ALL_PRE_DIFF_PLAN_ENTRIES)) .containsExactlyElementsIn(expected); } @@ -222,7 +234,9 @@ public void testLimit_EdgeOverSmallest() { suppressed( PRE_DIFF_PLAN_ENTRY_B_200K, PRE_DIFF_PLAN_ENTRY_C_300K, PRE_DIFF_PLAN_ENTRY_D_400K)); expected.addAll(ALL_IGNORED_PRE_DIFF_PLAN_ENTRIES); - assertThat(limiter.getModifiedPreDiffPlanEntries(tempFile, tempFile, ALL_PRE_DIFF_PLAN_ENTRIES)) + assertThat( + limiter.getModifiedPreDiffPlanEntries( + tempFileBlob, tempFileBlob, ALL_PRE_DIFF_PLAN_ENTRIES)) .containsExactlyElementsIn(expected); } @@ -238,7 +252,9 @@ public void testLimit_ExactlyLargest() { suppressed( PRE_DIFF_PLAN_ENTRY_A_100K, PRE_DIFF_PLAN_ENTRY_B_200K, PRE_DIFF_PLAN_ENTRY_C_300K)); expected.addAll(ALL_IGNORED_PRE_DIFF_PLAN_ENTRIES); - assertThat(limiter.getModifiedPreDiffPlanEntries(tempFile, tempFile, ALL_PRE_DIFF_PLAN_ENTRIES)) + assertThat( + limiter.getModifiedPreDiffPlanEntries( + tempFileBlob, tempFileBlob, ALL_PRE_DIFF_PLAN_ENTRIES)) .containsExactlyElementsIn(expected); } @@ -255,7 +271,9 @@ public void testLimit_EdgeUnderLargest() { suppressed( PRE_DIFF_PLAN_ENTRY_A_100K, PRE_DIFF_PLAN_ENTRY_B_200K, PRE_DIFF_PLAN_ENTRY_D_400K)); expected.addAll(ALL_IGNORED_PRE_DIFF_PLAN_ENTRIES); - assertThat(limiter.getModifiedPreDiffPlanEntries(tempFile, tempFile, ALL_PRE_DIFF_PLAN_ENTRIES)) + assertThat( + limiter.getModifiedPreDiffPlanEntries( + tempFileBlob, tempFileBlob, ALL_PRE_DIFF_PLAN_ENTRIES)) .containsExactlyElementsIn(expected); } @@ -272,7 +290,9 @@ public void testLimit_EdgeOverLargest() { suppressed( PRE_DIFF_PLAN_ENTRY_A_100K, PRE_DIFF_PLAN_ENTRY_B_200K, PRE_DIFF_PLAN_ENTRY_C_300K)); expected.addAll(ALL_IGNORED_PRE_DIFF_PLAN_ENTRIES); - assertThat(limiter.getModifiedPreDiffPlanEntries(tempFile, tempFile, ALL_PRE_DIFF_PLAN_ENTRIES)) + assertThat( + limiter.getModifiedPreDiffPlanEntries( + tempFileBlob, tempFileBlob, ALL_PRE_DIFF_PLAN_ENTRIES)) .containsExactlyElementsIn(expected); } @@ -293,7 +313,9 @@ public void testLimit_Complex() { expected.add(PRE_DIFF_PLAN_ENTRY_D_400K); expected.addAll(suppressed(PRE_DIFF_PLAN_ENTRY_A_100K, PRE_DIFF_PLAN_ENTRY_C_300K)); expected.addAll(ALL_IGNORED_PRE_DIFF_PLAN_ENTRIES); - assertThat(limiter.getModifiedPreDiffPlanEntries(tempFile, tempFile, ALL_PRE_DIFF_PLAN_ENTRIES)) + assertThat( + limiter.getModifiedPreDiffPlanEntries( + tempFileBlob, tempFileBlob, ALL_PRE_DIFF_PLAN_ENTRIES)) .containsExactlyElementsIn(expected); } } diff --git a/generator/src/test/java/com/google/archivepatcher/generator/MinimalZipParserTest.java b/generator/src/test/java/com/google/archivepatcher/generator/MinimalZipParserTest.java index b7ff55d7..eba5422f 100644 --- a/generator/src/test/java/com/google/archivepatcher/generator/MinimalZipParserTest.java +++ b/generator/src/test/java/com/google/archivepatcher/generator/MinimalZipParserTest.java @@ -16,9 +16,9 @@ import static com.google.common.truth.Truth.assertThat; -import com.google.archivepatcher.shared.RandomAccessFileInputStream; import com.google.archivepatcher.shared.UnitTestZipArchive; import com.google.archivepatcher.shared.UnitTestZipEntry; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileOutputStream; @@ -87,7 +87,7 @@ public void testLocateStartOfEocd_WithFile() throws IOException { } // Now expect to find the EOCD at the right place. - try (RandomAccessFileInputStream in = new RandomAccessFileInputStream(tempFile)) { + try (ByteSource in = ByteSource.fromFile(tempFile)) { long eocdOffset = MinimalZipParser.locateStartOfEocd(in, 32768); assertThat(eocdOffset).isEqualTo(bytesBefore); } @@ -113,7 +113,7 @@ public void testLocateStartOfEocd_WithFile_NoEocd() throws IOException { } // Now expect to find no EOCD. - try (RandomAccessFileInputStream in = new RandomAccessFileInputStream(tempFile)) { + try (ByteSource in = ByteSource.fromFile(tempFile)) { long eocdOffset = MinimalZipParser.locateStartOfEocd(in, 4000); assertThat(eocdOffset).isEqualTo(-1); } diff --git a/generator/src/test/java/com/google/archivepatcher/generator/PreDiffExecutorTest.java b/generator/src/test/java/com/google/archivepatcher/generator/PreDiffExecutorTest.java index 42f4fd5e..c8cacece 100644 --- a/generator/src/test/java/com/google/archivepatcher/generator/PreDiffExecutorTest.java +++ b/generator/src/test/java/com/google/archivepatcher/generator/PreDiffExecutorTest.java @@ -20,6 +20,7 @@ import com.google.archivepatcher.shared.PatchConstants.DeltaFormat; import com.google.archivepatcher.shared.UnitTestZipArchive; import com.google.archivepatcher.shared.UnitTestZipEntry; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.File; @@ -126,12 +127,16 @@ public void testPrepareForDiffing_OneCompressedEntry_Unchanged() throws IOExcept byte[] bytes = UnitTestZipArchive.makeTestZip(Collections.singletonList(ENTRY_LEVEL_6)); File oldFile = store(bytes); File newFile = store(bytes); - PreDiffExecutor executor = - new PreDiffExecutor.Builder() - .readingOriginalFiles(oldFile, newFile) - .writingDeltaFriendlyFiles(deltaFriendlyOldFile, deltaFriendlyNewFile) - .build(); - PreDiffPlan plan = executor.prepareForDiffing(); + PreDiffPlan plan; + try (ByteSource oldBlob = ByteSource.fromFile(oldFile); + ByteSource newBlob = ByteSource.fromFile(newFile)) { + PreDiffExecutor executor = + new PreDiffExecutor.Builder() + .readingOriginalFiles(oldBlob, newBlob) + .writingDeltaFriendlyFiles(deltaFriendlyOldFile, deltaFriendlyNewFile) + .build(); + plan = executor.prepareForDiffing(); + } assertThat(plan).isNotNull(); // The plan should be to leave everything alone because there is no change. assertThat(plan.getOldFileUncompressionPlan()).isEmpty(); @@ -149,12 +154,16 @@ public void testPrepareForDiffing_OneCompressedEntry_Changed() throws IOExceptio File oldFile = store(oldBytes); byte[] newBytes = UnitTestZipArchive.makeTestZip(Collections.singletonList(ENTRY_LEVEL_9)); File newFile = store(newBytes); - PreDiffExecutor executor = - new PreDiffExecutor.Builder() - .readingOriginalFiles(oldFile, newFile) - .writingDeltaFriendlyFiles(deltaFriendlyOldFile, deltaFriendlyNewFile) - .build(); - PreDiffPlan plan = executor.prepareForDiffing(); + PreDiffPlan plan; + try (ByteSource oldBlob = ByteSource.fromFile(oldFile); + ByteSource newBlob = ByteSource.fromFile(newFile)) { + PreDiffExecutor executor = + new PreDiffExecutor.Builder() + .readingOriginalFiles(oldBlob, newBlob) + .writingDeltaFriendlyFiles(deltaFriendlyOldFile, deltaFriendlyNewFile) + .build(); + plan = executor.prepareForDiffing(); + } assertThat(plan).isNotNull(); // The plan should be to uncompress the data in both the old and new files. assertThat(plan.getOldFileUncompressionPlan()).hasSize(1); @@ -219,13 +228,17 @@ public void testPrepareForDiffing_OneCompressedEntry_Changed_Limited() throws IO byte[] newBytes = UnitTestZipArchive.makeTestZip(Collections.singletonList(ENTRY_LEVEL_9)); File newFile = store(newBytes); TotalRecompressionLimiter limiter = new TotalRecompressionLimiter(1); // 1 byte limitation - PreDiffExecutor executor = - new PreDiffExecutor.Builder() - .readingOriginalFiles(oldFile, newFile) - .writingDeltaFriendlyFiles(deltaFriendlyOldFile, deltaFriendlyNewFile) - .addPreDiffPlanEntryModifier(limiter) - .build(); - PreDiffPlan plan = executor.prepareForDiffing(); + PreDiffPlan plan; + try (ByteSource oldBlob = ByteSource.fromFile(oldFile); + ByteSource newBlob = ByteSource.fromFile(newFile)) { + PreDiffExecutor executor = + new PreDiffExecutor.Builder() + .readingOriginalFiles(oldBlob, newBlob) + .writingDeltaFriendlyFiles(deltaFriendlyOldFile, deltaFriendlyNewFile) + .addPreDiffPlanEntryModifier(limiter) + .build(); + plan = executor.prepareForDiffing(); + } assertThat(plan).isNotNull(); // The plan should be to leave everything alone because of the limiter assertThat(plan.getOldFileUncompressionPlan()).isEmpty(); diff --git a/generator/src/test/java/com/google/archivepatcher/generator/PreDiffPlannerTest.java b/generator/src/test/java/com/google/archivepatcher/generator/PreDiffPlannerTest.java index 8f984084..09557172 100644 --- a/generator/src/test/java/com/google/archivepatcher/generator/PreDiffPlannerTest.java +++ b/generator/src/test/java/com/google/archivepatcher/generator/PreDiffPlannerTest.java @@ -28,13 +28,14 @@ import com.google.archivepatcher.shared.DefaultDeflateCompatibilityWindow; import com.google.archivepatcher.shared.JreDeflateParameters; import com.google.archivepatcher.shared.PatchConstants.DeltaFormat; -import com.google.archivepatcher.shared.RandomAccessFileInputStream; import com.google.archivepatcher.shared.TypedRange; import com.google.archivepatcher.shared.UnitTestZipArchive; import com.google.archivepatcher.shared.UnitTestZipEntry; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.RandomAccessFile; import java.io.UnsupportedEncodingException; import java.util.ArrayList; @@ -224,21 +225,35 @@ private void corruptEntryData(File tempFile, UnitTestZipEntry unitTestEntry) thr private void corruptCompressionMethod(File tempFile, UnitTestZipEntry unitTestEntry) throws IOException { long centralDirectoryRecordOffset = -1; - try (RandomAccessFileInputStream rafis = new RandomAccessFileInputStream(tempFile)) { - long startOfEocd = MinimalZipParser.locateStartOfEocd(rafis, 32768); - rafis.setRange(startOfEocd, tempFile.length() - startOfEocd); - MinimalCentralDirectoryMetadata centralDirectoryMetadata = MinimalZipParser.parseEocd(rafis); - int numEntries = centralDirectoryMetadata.getNumEntriesInCentralDirectory(); - rafis.setRange( - centralDirectoryMetadata.getOffsetOfCentralDirectory(), - centralDirectoryMetadata.getLengthOfCentralDirectory()); - for (int x = 0; x < numEntries; x++) { - long recordStartOffset = rafis.getPosition(); - MinimalZipEntry candidate = MinimalZipParser.parseCentralDirectoryEntry(rafis); - if (candidate.getFileName().equals(unitTestEntry.path)) { - // Located! Track offset and bail out. - centralDirectoryRecordOffset = recordStartOffset; - x = numEntries; + try (ByteSource byteSource = ByteSource.fromFile(tempFile)) { + long startOfEocd = MinimalZipParser.locateStartOfEocd(byteSource, 32768); + MinimalCentralDirectoryMetadata centralDirectoryMetadata; + int numEntries; + try (InputStream sliceIn = byteSource.sliceFrom(startOfEocd).openStream()) { + centralDirectoryMetadata = MinimalZipParser.parseEocd(sliceIn); + numEntries = centralDirectoryMetadata.getNumEntriesInCentralDirectory(); + } + + try (InputStream sliceIn = + byteSource + .slice( + centralDirectoryMetadata.getOffsetOfCentralDirectory(), + centralDirectoryMetadata.getLengthOfCentralDirectory()) + .openStream()) { + for (int x = 0; x < numEntries; x++) { + // Here we compute the offset to the start of the file by computing offset to start of + // sliceFrom (i.e. start of central directory) and adding it to the central directory + // offset. + long offsetToStartOfCentralDir = + centralDirectoryMetadata.getLengthOfCentralDirectory() - sliceIn.available(); + long offsetToStartOfFile = + centralDirectoryMetadata.getOffsetOfCentralDirectory() + offsetToStartOfCentralDir; + MinimalZipEntry candidate = MinimalZipParser.parseCentralDirectoryEntry(sliceIn); + if (candidate.getFileName().equals(unitTestEntry.path)) { + // Located! Track offset and bail out. + centralDirectoryRecordOffset = offsetToStartOfFile; + x = numEntries; + } } } } @@ -270,24 +285,28 @@ private PreDiffPlan invokeGeneratePreDiffPlan( originalOldArchiveZipEntriesByPath.put(key, zipEntry); } - for (DivinationResult divinationResult : - DefaultDeflateCompressionDiviner.divineDeflateParameters(newFile)) { - ByteArrayHolder key = new ByteArrayHolder(divinationResult.minimalZipEntry.getFileNameBytes()); + try (ByteSource oldBlob = ByteSource.fromFile(oldFile); + ByteSource newBlob = ByteSource.fromFile(newFile)) { + for (DivinationResult divinationResult : + DefaultDeflateCompressionDiviner.divineDeflateParameters(newBlob)) { + ByteArrayHolder key = + new ByteArrayHolder(divinationResult.minimalZipEntry.getFileNameBytes()); originalNewArchiveZipEntriesByPath.put(key, divinationResult.minimalZipEntry); originalNewArchiveJreDeflateParametersByPath.put(key, divinationResult.divinedParameters); } - PreDiffPlanner preDiffPlanner = - new PreDiffPlanner( - oldFile, - originalOldArchiveZipEntriesByPath, - newFile, - originalNewArchiveZipEntriesByPath, - originalNewArchiveJreDeflateParametersByPath, - preDiffPlanEntryModifiers, - supportedDeltaFormats); + PreDiffPlanner preDiffPlanner = + new PreDiffPlanner( + oldBlob, + originalOldArchiveZipEntriesByPath, + newBlob, + originalNewArchiveZipEntriesByPath, + originalNewArchiveJreDeflateParametersByPath, + preDiffPlanEntryModifiers, + supportedDeltaFormats); return preDiffPlanner.generatePreDiffPlan(); } + } private void checkPreDiffPlanEntry(PreDiffPlan plan, PreDiffPlanEntry... expected) { assertThat(plan.getPreDiffPlanEntries()).isNotNull(); diff --git a/generator/src/test/java/com/google/archivepatcher/generator/TotalRecompressionLimiterTest.java b/generator/src/test/java/com/google/archivepatcher/generator/TotalRecompressionLimiterTest.java index 491ad32d..8af9b9fc 100644 --- a/generator/src/test/java/com/google/archivepatcher/generator/TotalRecompressionLimiterTest.java +++ b/generator/src/test/java/com/google/archivepatcher/generator/TotalRecompressionLimiterTest.java @@ -24,7 +24,7 @@ import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; -import java.io.File; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.Arrays; @@ -39,8 +39,8 @@ @SuppressWarnings("javadoc") public class TotalRecompressionLimiterTest { - private static final File OLD_FILE = null; - private static final File NEW_FILE = null; + private static final ByteSource OLD_FILE = null; + private static final ByteSource NEW_FILE = null; private static final MinimalZipEntry UNIMPORTANT = makeFakeEntry("/unimportant", 1337); private static final MinimalZipEntry ENTRY_A_100K = makeFakeEntry("/a/100k", 100 * 1024); diff --git a/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/BsDiffTest.java b/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/BsDiffTest.java index 6432da35..65665939 100644 --- a/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/BsDiffTest.java +++ b/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/BsDiffTest.java @@ -17,13 +17,17 @@ import static com.google.common.truth.Truth.assertThat; import com.google.archivepatcher.generator.bsdiff.Matcher.NextMatch; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import java.util.Objects; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -41,8 +45,8 @@ public void lengthOfMatchTest() throws IOException { + "then ends didlyiefferently"; byte[] s1b = s1.getBytes(Charset.forName("US-ASCII")); byte[] s2b = s2.getBytes(Charset.forName("US-ASCII")); - RandomAccessObject s1ro = new RandomAccessObject.RandomAccessByteArrayObject(s1b); - RandomAccessObject s2ro = new RandomAccessObject.RandomAccessByteArrayObject(s2b); + ByteSource s1ro = ByteSource.wrap(s1b); + ByteSource s2ro = ByteSource.wrap(s2b); assertThat(BsDiff.lengthOfMatch(s1ro, 0, s2ro, 0)).isEqualTo(36); assertThat(BsDiff.lengthOfMatch(s1ro, 5, s2ro, 0)).isEqualTo(0); @@ -59,8 +63,8 @@ public void searchForMatchBaseCaseShortGroupArrayTest() throws IOException { final String s2 = "hkjl.9999.00vbn,``'=-this should match.9900-mmmnmn,,,.x??'"; final byte[] s1b = s1.getBytes(Charset.forName("US-ASCII")); final byte[] s2b = s2.getBytes(Charset.forName("US-ASCII")); - final RandomAccessObject s1ro = new RandomAccessObject.RandomAccessByteArrayObject(s1b); - final RandomAccessObject s2ro = new RandomAccessObject.RandomAccessByteArrayObject(s2b); + ByteSource s1ro = ByteSource.wrap(s1b); + ByteSource s2ro = ByteSource.wrap(s2b); final RandomAccessObject groupArrayRO = intArrayToRandomAccessObject(BsDiffTestData.SHORT_GROUP_ARRAY); @@ -148,8 +152,8 @@ public void searchForMatchShortGroupArrayTest() throws IOException { final String s2 = "hkjl.9999.00vbn,``'=-this should match.9900-mmmnmn,,,.x??'"; final byte[] s1b = s1.getBytes(Charset.forName("US-ASCII")); final byte[] s2b = s2.getBytes(Charset.forName("US-ASCII")); - final RandomAccessObject s1ro = new RandomAccessObject.RandomAccessByteArrayObject(s1b); - final RandomAccessObject s2ro = new RandomAccessObject.RandomAccessByteArrayObject(s2b); + ByteSource s1ro = ByteSource.wrap(s1b); + ByteSource s2ro = ByteSource.wrap(s2b); final RandomAccessObject groupArrayRO = intArrayToRandomAccessObject(BsDiffTestData.SHORT_GROUP_ARRAY); @@ -212,7 +216,7 @@ public void searchForMatch() throws Exception { for (String testCase : testCases) { int size = testCase.length(); byte[] bytes = testCase.getBytes(StandardCharsets.US_ASCII); - RandomAccessObject input = new RandomAccessObject.RandomAccessByteArrayObject(bytes); + ByteSource input = ByteSource.wrap(bytes); RandomAccessObject suffixArray = new DivSuffixSorter(new RandomAccessObjectFactory.RandomAccessByteArrayObjectFactory()) .suffixSort(input); @@ -223,7 +227,7 @@ public void searchForMatch() throws Exception { byte[] query = Arrays.copyOfRange(bytes, lo, hi); int querySize = query.length; assertThat(hi - lo).isEqualTo(querySize); - RandomAccessObject queryBuf = new RandomAccessObject.RandomAccessByteArrayObject(query); + ByteSource queryBuf = ByteSource.wrap(query); BsDiff.Match match = BsDiff.searchForMatch(suffixArray, input, queryBuf, 0, 0, size); @@ -273,35 +277,34 @@ public void generatePatchWithMatcherTest() throws Exception { // Test that all of the characters are diffed if two strings are identical even if there // is no "valid match" because the strings are too short. CtrlEntry[] expectedCtrlEntries = {new CtrlEntry(2, 0, 0)}; - assertThat(generatePatchAndCheckCtrlEntries("aa", "aa", expectedCtrlEntries)).isTrue(); + generatePatchAndCheckCtrlEntries("aa", "aa", expectedCtrlEntries); } { // Test that all of the characters are diffed if two strings are identical and are long // enough to be considered a "valid match". CtrlEntry[] expectedCtrlEntries = {new CtrlEntry(0, 0, 0), new CtrlEntry(3, 0, 0)}; - assertThat(generatePatchAndCheckCtrlEntries("aaa", "aaa", expectedCtrlEntries)).isTrue(); + generatePatchAndCheckCtrlEntries("aaa", "aaa", expectedCtrlEntries); } { // Test that none of the characters are diffed if the strings do not match. CtrlEntry[] expectedCtrlEntries = {new CtrlEntry(0, 2, 0)}; - assertThat(generatePatchAndCheckCtrlEntries("aa", "bb", expectedCtrlEntries)).isTrue(); + generatePatchAndCheckCtrlEntries("aa", "bb", expectedCtrlEntries); } { // Test that characters are diffed if the beginning of the strings match even if the match // is not long enough to be considered valid. CtrlEntry[] expectedCtrlEntries = {new CtrlEntry(2, 6, 3), new CtrlEntry(3, 0, 0)}; - assertThat(generatePatchAndCheckCtrlEntries("aazzzbbb", "aaayyyyybbb", expectedCtrlEntries)) - .isTrue(); + generatePatchAndCheckCtrlEntries("aazzzbbb", "aaayyyyybbb", expectedCtrlEntries); } { // Test that none of the characters are diffed if the beginning of the strings do not // match and the available match is not long enough to be considered valid. CtrlEntry[] expectedCtrlEntries = {new CtrlEntry(0, 3, 0)}; - assertThat(generatePatchAndCheckCtrlEntries("zzzbb", "abb", expectedCtrlEntries)).isTrue(); + generatePatchAndCheckCtrlEntries("zzzbb", "abb", expectedCtrlEntries); } { @@ -312,10 +315,8 @@ public void generatePatchWithMatcherTest() throws Exception { new CtrlEntry(6, 3, 1), // 012345 | %^& new CtrlEntry(13, 0, 0) // abcdefghijklm | n/a }; - assertThat( - generatePatchAndCheckCtrlEntries( - "@@012345@ab@de@ghijklm", "#012$45%^&abcdefghijklm", expectedCtrlEntries)) - .isTrue(); + generatePatchAndCheckCtrlEntries( + "@@012345@ab@de@ghijklm", "#012$45%^&abcdefghijklm", expectedCtrlEntries); } { @@ -326,10 +327,8 @@ public void generatePatchWithMatcherTest() throws Exception { new CtrlEntry(6, 3, -21), // 012345 | %^& new CtrlEntry(13, 0, 0) // abcdefghijklm | n/a }; - assertThat( - generatePatchAndCheckCtrlEntries( - "@ab@de@ghijklm@@012345", "#012$45%^&abcdefghijklm", expectedCtrlEntries)) - .isTrue(); + generatePatchAndCheckCtrlEntries( + "@ab@de@ghijklm@@012345", "#012$45%^&abcdefghijklm", expectedCtrlEntries); } { @@ -340,12 +339,8 @@ public void generatePatchWithMatcherTest() throws Exception { CtrlEntry[] expectedCtrlEntries = { new CtrlEntry(0, 0, 5), new CtrlEntry(4, 0, 17), new CtrlEntry(13, 0, 0), }; - assertThat( - generatePatchAndCheckCtrlEntries( - "012345678901234567890nexus9nexus5nexus6", - "567@9n1x3s56exus6", - expectedCtrlEntries)) - .isTrue(); + generatePatchAndCheckCtrlEntries( + "012345678901234567890nexus9nexus5nexus6", "567@9n1x3s56exus6", expectedCtrlEntries); } { @@ -354,10 +349,7 @@ public void generatePatchWithMatcherTest() throws Exception { CtrlEntry[] expectedCtrlEntries = { new CtrlEntry(0, 8, 0), new CtrlEntry(3, 0, 3), new CtrlEntry(3, 0, 0), }; - assertThat( - generatePatchAndCheckCtrlEntries( - "aaazzzbbbbbbbb", "bb@bb@bbaaabbb", expectedCtrlEntries)) - .isTrue(); + generatePatchAndCheckCtrlEntries("aaazzzbbbbbbbb", "bb@bb@bbaaabbb", expectedCtrlEntries); } { @@ -366,10 +358,7 @@ public void generatePatchWithMatcherTest() throws Exception { CtrlEntry[] expectedCtrlEntries = { new CtrlEntry(0, 0, 0), new CtrlEntry(3, 0, 11), new CtrlEntry(3, 8, 0), }; - assertThat( - generatePatchAndCheckCtrlEntries( - "aaaaaaaaaaazzzbbb", "aaabbbaa@aa@aa", expectedCtrlEntries)) - .isTrue(); + generatePatchAndCheckCtrlEntries("aaaaaaaaaaazzzbbb", "aaabbbaa@aa@aa", expectedCtrlEntries); } { @@ -377,8 +366,7 @@ public void generatePatchWithMatcherTest() throws Exception { CtrlEntry[] expectedCtrlEntries = { new CtrlEntry(0, 0, 0), new CtrlEntry(9, 0, 0), }; - assertThat(generatePatchAndCheckCtrlEntries("abcdefghi", "ab@def@hi", expectedCtrlEntries)) - .isTrue(); + generatePatchAndCheckCtrlEntries("abcdefghi", "ab@def@hi", expectedCtrlEntries); } } @@ -474,15 +462,36 @@ private BsDiff.Match findLongestMatchInOld(int newStartIndex) { } private static class CtrlEntry { - public int diffLength; - public int extraLength; - public int oldOffset; + public long diffLength; + public long extraLength; + public long oldOffset; - public CtrlEntry(int diffLength, int extraLength, int oldOffset) { + public CtrlEntry(long diffLength, long extraLength, long oldOffset) { this.diffLength = diffLength; this.extraLength = extraLength; this.oldOffset = oldOffset; } + + @Override + public String toString() { + return String.format( + "CtrlEntry(diffLength = %d, extraLength = %d, oldOffset = %d", + diffLength, extraLength, oldOffset); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof CtrlEntry)) { + return false; + } + CtrlEntry o = (CtrlEntry) obj; + return o.diffLength == diffLength && o.extraLength == extraLength && o.oldOffset == oldOffset; + } + + @Override + public int hashCode() { + return Objects.hash(diffLength, extraLength, oldOffset); + } } /** @@ -490,38 +499,32 @@ public CtrlEntry(int diffLength, int extraLength, int oldOffset) { * patch's control data matches |expected|. For the sake of simplicity, assumes that chars are * always 1 byte. * - * @param oldData - * @param newData * @param expected The expected control entries in the generated patch - * @return returns whether the actual control entries in the generated patch match the expected - * ones */ - private boolean generatePatchAndCheckCtrlEntries( + private void generatePatchAndCheckCtrlEntries( String oldData, String newData, CtrlEntry[] expected) throws Exception { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); byte[] oldBytes = oldData.getBytes(Charset.forName("US-ASCII")); byte[] newBytes = newData.getBytes(Charset.forName("US-ASCII")); - RandomAccessObject oldBytesRo = new RandomAccessObject.RandomAccessByteArrayObject(oldBytes); - RandomAccessObject newBytesRo = new RandomAccessObject.RandomAccessByteArrayObject(newBytes); + ByteSource oldBytesRo = ByteSource.wrap(oldBytes); + ByteSource newBytesRo = ByteSource.wrap(newBytes); BsDiffPatchWriter.generatePatchWithMatcher( oldBytesRo, newBytesRo, new NaiveMatcher(oldBytes, newBytes), outputStream); ByteArrayInputStream patchInputStream = new ByteArrayInputStream(outputStream.toByteArray()); - for (CtrlEntry element : expected) { - if (patchInputStream.available() < 24 - || BsUtil.readFormattedLong(patchInputStream) != element.diffLength - || BsUtil.readFormattedLong(patchInputStream) != element.extraLength - || BsUtil.readFormattedLong(patchInputStream) != element.oldOffset) { - return false; - } - patchInputStream.skip(element.diffLength + element.extraLength); + List actualEntries = new ArrayList<>(); + while (patchInputStream.available() >= 24) { + long diffLength = BsUtil.readFormattedLong(patchInputStream); + long extraLength = BsUtil.readFormattedLong(patchInputStream); + long oldOffset = BsUtil.readFormattedLong(patchInputStream); + CtrlEntry entry = new CtrlEntry(diffLength, extraLength, oldOffset); + actualEntries.add(entry); + patchInputStream.skip(diffLength + extraLength); } - if (patchInputStream.available() > 0) { - return false; - } + assertThat(actualEntries).containsExactlyElementsIn(expected).inOrder(); - return true; + assertThat(patchInputStream.available()).isEqualTo(0); } } diff --git a/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/BsDiffTestData.java b/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/BsDiffTestData.java index 11ff7d3e..32ce6a5e 100644 --- a/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/BsDiffTestData.java +++ b/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/BsDiffTestData.java @@ -14,6 +14,7 @@ package com.google.archivepatcher.generator.bsdiff; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.nio.charset.Charset; class BsDiffTestData { @@ -187,10 +188,8 @@ class BsDiffTestData { public static final byte[] LONGER_DATA_349 = LONGER_DATA_349_S.getBytes(Charset.forName("US-ASCII")); - public static final RandomAccessObject LONG_DATA_99_RO = - new RandomAccessObject.RandomAccessByteArrayObject(LONG_DATA_99); - public static final RandomAccessObject LONGER_DATA_349_RO = - new RandomAccessObject.RandomAccessByteArrayObject(LONGER_DATA_349); + public static final ByteSource LONG_DATA_99_RO = ByteSource.wrap(LONG_DATA_99); + public static final ByteSource LONGER_DATA_349_RO = ByteSource.wrap(LONGER_DATA_349); public static final int[] QUICK_SUFFIX_SORT_INIT_TEST_GA_CONTROL = new int[] { @@ -303,8 +302,6 @@ class BsDiffTestData { public static final byte[] LONGER_DATA_354_NEW = LONGER_DATA_354_NEW_S.getBytes(Charset.forName("US-ASCII")); - public static final RandomAccessObject LONG_DATA_104_NEW_RO = - new RandomAccessObject.RandomAccessByteArrayObject(LONG_DATA_104_NEW); - public static final RandomAccessObject LONGER_DATA_354_NEW_RO = - new RandomAccessObject.RandomAccessByteArrayObject(LONGER_DATA_354_NEW); + public static final ByteSource LONG_DATA_104_NEW_RO = ByteSource.wrap(LONG_DATA_104_NEW); + public static final ByteSource LONGER_DATA_354_NEW_RO = ByteSource.wrap(LONGER_DATA_354_NEW); } diff --git a/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/BsUtilTest.java b/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/BsUtilTest.java index ead6cd0a..f24d66ba 100644 --- a/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/BsUtilTest.java +++ b/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/BsUtilTest.java @@ -16,6 +16,7 @@ import static com.google.common.truth.Truth.assertThat; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -103,8 +104,8 @@ public void lexicographicalCompareTest() throws IOException { String s2 = "that was a string"; byte[] s1b = s1.getBytes(Charset.forName("US-ASCII")); byte[] s2b = s2.getBytes(Charset.forName("US-ASCII")); - RandomAccessObject s1ro = new RandomAccessObject.RandomAccessByteArrayObject(s1b); - RandomAccessObject s2ro = new RandomAccessObject.RandomAccessByteArrayObject(s2b); + ByteSource s1ro = ByteSource.wrap(s1b); + ByteSource s2ro = ByteSource.wrap(s2b); int r = BsUtil.lexicographicalCompare(s1ro, 0, s1b.length, s2ro, 0, s2b.length); assertThat(r).isGreaterThan(0); diff --git a/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/SuffixSorterTestBase.java b/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/SuffixSorterTestBase.java index e2f09afc..927c5a3c 100644 --- a/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/SuffixSorterTestBase.java +++ b/generator/src/test/java/com/google/archivepatcher/generator/bsdiff/SuffixSorterTestBase.java @@ -17,6 +17,8 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.fail; +import com.google.archivepatcher.shared.bytesource.ByteSource; +import java.io.InputStream; import java.util.Random; import org.junit.Test; @@ -41,7 +43,7 @@ public void suffixSortShortDataTest() throws Exception { } private void checkSuffixSort(int[] expectedSuffixArray, byte[] inputBytes) throws Exception { - RandomAccessObject input = new RandomAccessObject.RandomAccessByteArrayObject(inputBytes); + ByteSource input = ByteSource.wrap(inputBytes); RandomAccessObject groupArray = getSuffixSorter().suffixSort(input); assertSorted(groupArray, input); @@ -73,17 +75,17 @@ public void suffixSortVeryLongDataTest() throws Exception { public void testRandom() throws Exception { Random rand = new Random(1123458); for (int i = 1; i <= 10; i++) { - RandomAccessObject input = generateRandom(rand, i * 10000); + ByteSource input = generateRandom(rand, i * 10000); RandomAccessObject suffixArray = getSuffixSorter().suffixSort(input); assertSorted(suffixArray, input); } } - private static RandomAccessObject generateRandom(Random rand, int length) { + private static ByteSource generateRandom(Random rand, int length) { byte[] bytes = new byte[length]; rand.nextBytes(bytes); - return new RandomAccessObject.RandomAccessByteArrayObject(bytes); + return ByteSource.wrap(bytes); } protected static RandomAccessObject intArrayToRandomAccessObject(final int[] array) @@ -124,16 +126,20 @@ protected static int[] randomAccessObjectToIntArray(RandomAccessObject randomAcc return ret; } - private static boolean checkSuffixLessThanOrEqual( - RandomAccessObject input, int index1, int index2) throws Exception { + private static boolean checkSuffixLessThanOrEqual(ByteSource input, int index1, int index2) + throws Exception { while (true) { if (index1 == input.length()) { return true; } - input.seek(index1); - int unsignedByte1 = input.readUnsignedByte(); - input.seek(index2); - int unsignedByte2 = input.readUnsignedByte(); + int unsignedByte1; + try (InputStream in = input.sliceFrom(index1).openStream()) { + unsignedByte1 = in.read(); + } + int unsignedByte2; + try (InputStream in = input.sliceFrom(index2).openStream()) { + unsignedByte2 = in.read(); + } if (unsignedByte1 < unsignedByte2) { return true; } @@ -145,7 +151,7 @@ private static boolean checkSuffixLessThanOrEqual( } } - private static void assertSorted(RandomAccessObject suffixArray, RandomAccessObject input) + private static void assertSorted(RandomAccessObject suffixArray, ByteSource input) throws Exception { for (int i = 0; i < input.length(); i++) { suffixArray.seekToIntAligned(i); diff --git a/shared/src/main/java/com/google/archivepatcher/shared/DeltaFriendlyFile.java b/shared/src/main/java/com/google/archivepatcher/shared/DeltaFriendlyFile.java index 59aea7b6..91b55d78 100644 --- a/shared/src/main/java/com/google/archivepatcher/shared/DeltaFriendlyFile.java +++ b/shared/src/main/java/com/google/archivepatcher/shared/DeltaFriendlyFile.java @@ -14,8 +14,10 @@ package com.google.archivepatcher.shared; +import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.List; @@ -31,24 +33,25 @@ public class DeltaFriendlyFile { public static final int DEFAULT_COPY_BUFFER_SIZE = 32768; /** - * Invoke {@link #generateDeltaFriendlyFile(List, File, OutputStream, boolean, int)} with + * Invoke {@link #generateDeltaFriendlyFile(List, ByteSource, OutputStream, boolean, int)} with + * * generateInverse set to true and a copy buffer size of {@link * #DEFAULT_COPY_BUFFER_SIZE}. * * @param the type of the data associated with the ranges * @param rangesToUncompress the ranges to be uncompressed during transformation to a * delta-friendly form - * @param file the file to read from + * @param data the original archive * @param deltaFriendlyOut a stream to write the delta-friendly file to * @return the ranges in the delta-friendly file that correspond to the ranges in the original * file, with identical metadata and in the same order * @throws IOException if anything goes wrong */ public static List> generateDeltaFriendlyFile( - List> rangesToUncompress, File file, OutputStream deltaFriendlyOut) + List> rangesToUncompress, ByteSource data, OutputStream deltaFriendlyOut) throws IOException { return generateDeltaFriendlyFile( - rangesToUncompress, file, deltaFriendlyOut, true, DEFAULT_COPY_BUFFER_SIZE); + rangesToUncompress, data, deltaFriendlyOut, true, DEFAULT_COPY_BUFFER_SIZE); } /** @@ -62,7 +65,7 @@ public static List> generateDeltaFriendlyFile( * @param the type of the data associated with the ranges * @param rangesToUncompress the ranges to be uncompressed during transformation to a * delta-friendly form - * @param file the file to read from + * @param blob the blob to read from * @param deltaFriendlyOut a stream to write the delta-friendly file to * @param generateInverse if true, generate and return a list of inverse ranges in * file order; otherwise, do all the normal work but return null instead of the inverse ranges @@ -74,7 +77,7 @@ public static List> generateDeltaFriendlyFile( */ public static List> generateDeltaFriendlyFile( List> rangesToUncompress, - File file, + ByteSource blob, OutputStream deltaFriendlyOut, boolean generateInverse, int copyBufferSize) @@ -84,25 +87,27 @@ public static List> generateDeltaFriendlyFile( inverseRanges = new ArrayList>(rangesToUncompress.size()); } long lastReadOffset = 0; - RandomAccessFileInputStream oldFileRafis = null; try (PartiallyUncompressingPipe filteredOut = new PartiallyUncompressingPipe(deltaFriendlyOut, copyBufferSize)) { - oldFileRafis = new RandomAccessFileInputStream(file); for (TypedRange rangeToUncompress : rangesToUncompress) { long gap = rangeToUncompress.getOffset() - lastReadOffset; if (gap > 0) { // Copy bytes up to the range start point - oldFileRafis.setRange(lastReadOffset, gap); - filteredOut.pipe(oldFileRafis, PartiallyUncompressingPipe.Mode.COPY); + try (InputStream in = blob.slice(lastReadOffset, gap).openStream()) { + filteredOut.pipe(in, PartiallyUncompressingPipe.Mode.COPY); + } } // Now uncompress the range. - oldFileRafis.setRange(rangeToUncompress.getOffset(), rangeToUncompress.getLength()); long inverseRangeStart = filteredOut.getNumBytesWritten(); - // TODO: Support nowrap=false here? Never encountered in practice. - // This would involve catching the ZipException, checking if numBytesWritten is still zero, - // resetting the stream and trying again. - filteredOut.pipe(oldFileRafis, PartiallyUncompressingPipe.Mode.UNCOMPRESS_NOWRAP); + try (InputStream in = + blob.slice(rangeToUncompress.getOffset(), rangeToUncompress.getLength()).openStream()) { + // TODO: Support nowrap=false here? Never encountered in practice. + // This would involve catching the ZipException, checking if numBytesWritten is still + // zero, + // resetting the stream and trying again. + filteredOut.pipe(in, PartiallyUncompressingPipe.Mode.UNCOMPRESS_NOWRAP); + } lastReadOffset = rangeToUncompress.getOffset() + rangeToUncompress.getLength(); if (generateInverse) { @@ -115,14 +120,47 @@ public static List> generateDeltaFriendlyFile( } } // Finish the final bytes of the file - long bytesLeft = oldFileRafis.length() - lastReadOffset; + long bytesLeft = blob.length() - lastReadOffset; if (bytesLeft > 0) { - oldFileRafis.setRange(lastReadOffset, bytesLeft); - filteredOut.pipe(oldFileRafis, PartiallyUncompressingPipe.Mode.COPY); + try (InputStream in = blob.slice(lastReadOffset, bytesLeft).openStream()) { + filteredOut.pipe(in, PartiallyUncompressingPipe.Mode.COPY); + } } - } finally { - Closeables.closeQuietly(oldFileRafis); } return inverseRanges; } + + /** + * Generate one delta-friendly file and (optionally) return the ranges necessary to invert the + * transform, in file order. There is a 1:1 correspondence between the ranges in the input list + * and the returned list, but the offsets and lengths will be different (the input list represents + * compressed data, the output list represents uncompressed data). The ability to suppress + * generation of the inverse range and to specify the size of the copy buffer are provided for + * clients that desire a minimal memory footprint. + * + * @param the type of the data associated with the ranges + * @param rangesToUncompress the ranges to be uncompressed during transformation to a + * delta-friendly form + * @param blob the blob to read from + * @param deltaFriendlyOut a stream to write the delta-friendly file to + * @param generateInverse if true, generate and return a list of inverse ranges in + * file order; otherwise, do all the normal work but return null instead of the inverse ranges + * @param copyBufferSize the size of the buffer to use for copying bytes between streams + * @return if generateInverse was true, returns the ranges in the delta-friendly file + * that correspond to the ranges in the original file, with identical metadata and in the same + * order; otherwise, return null + * @throws IOException if anything goes wrong + */ + public static List> generateDeltaFriendlyFile( + List> rangesToUncompress, + File blob, + OutputStream deltaFriendlyOut, + boolean generateInverse, + int copyBufferSize) + throws IOException { + try (ByteSource byteSource = ByteSource.fromFile(blob)) { + return generateDeltaFriendlyFile( + rangesToUncompress, byteSource, deltaFriendlyOut, generateInverse, copyBufferSize); + } + } } diff --git a/shared/src/main/java/com/google/archivepatcher/shared/RandomAccessFileInputStream.java b/shared/src/main/java/com/google/archivepatcher/shared/RandomAccessFileInputStream.java index 2b4e45b7..ab8332f1 100644 --- a/shared/src/main/java/com/google/archivepatcher/shared/RandomAccessFileInputStream.java +++ b/shared/src/main/java/com/google/archivepatcher/shared/RandomAccessFileInputStream.java @@ -81,18 +81,33 @@ public RandomAccessFileInputStream(File file) throws IOException { */ public RandomAccessFileInputStream(File file, long rangeOffset, long rangeLength) throws IOException { - raf = getRandomAccessFile(file); + this(getRandomAccessFile(file), rangeOffset, rangeLength); + } + + /** + * Constructs a new stream for the given file, which will be opened in read-only mode for random + * access within a specific range. + * + * @param file the file to read + * @param rangeOffset the offset at which the valid range starts + * @param rangeLength the number of bytes in the range + * @throws IOException if unable to open the file for read + */ + public RandomAccessFileInputStream(RandomAccessFile file, long rangeOffset, long rangeLength) + throws IOException { + raf = file; fileLength = file.length(); setRange(rangeOffset, rangeLength); } /** * Given a {@link File}, get a read-only {@link RandomAccessFile} reference for it. + * * @param file the file * @return as described * @throws IOException if unable to open the file */ - protected RandomAccessFile getRandomAccessFile(File file) throws IOException { + protected static RandomAccessFile getRandomAccessFile(File file) throws IOException { return new RandomAccessFile(file, "r"); } diff --git a/shared/src/main/java/com/google/archivepatcher/shared/bytesource/ByteSource.java b/shared/src/main/java/com/google/archivepatcher/shared/bytesource/ByteSource.java index 50e0ffae..fdee628e 100644 --- a/shared/src/main/java/com/google/archivepatcher/shared/bytesource/ByteSource.java +++ b/shared/src/main/java/com/google/archivepatcher/shared/bytesource/ByteSource.java @@ -15,6 +15,7 @@ package com.google.archivepatcher.shared.bytesource; import java.io.Closeable; +import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -32,8 +33,8 @@ public ByteSource slice(long offset, long length) { length = Math.min(length, length()); return new SlicedByteSource(this, offset, length); } - - public ByteSource sliceFrom(long offset) { + /** Returns a slice of this {@link ByteSource} starting at byte {@code offset}. */ + public ByteSource sliceFrom(long offset) throws IOException { long length = Math.max(length() - offset, 0); return new SlicedByteSource(this, offset, length); } @@ -48,4 +49,14 @@ public synchronized InputStream openStream() throws IOException { * in this {@link ByteSource}. */ protected abstract InputStream openStream(long offset, long length) throws IOException; + + /** Convenience method to obtain a {@link ByteSource} from a {@link File}. */ + public static ByteSource fromFile(File file) throws IOException { + return new RandomAccessFileByteSource(file); + } + + /** Convenience method to obtain a {@link ByteSource} from a byte array. */ + public static ByteSource wrap(byte[] buffer) { + return new ByteArrayByteSource(buffer); + } } diff --git a/shared/src/main/java/com/google/archivepatcher/shared/bytesource/ByteStreams.java b/shared/src/main/java/com/google/archivepatcher/shared/bytesource/ByteStreams.java new file mode 100644 index 00000000..71bc6310 --- /dev/null +++ b/shared/src/main/java/com/google/archivepatcher/shared/bytesource/ByteStreams.java @@ -0,0 +1,55 @@ +// Copyright 2016 Google LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.archivepatcher.shared.bytesource; + +import java.io.IOException; +import java.io.InputStream; + +/** This class contains utility methods related to IO. */ +public class ByteStreams { + + /** + * Read exactly the specified number of bytes into the specified buffer. + * + * @param in the input stream to read from + * @param destination where to write the bytes to + * @param startAt the offset at which to start writing bytes in the destination buffer + * @param numBytes the number of bytes to read + * @throws IOException if reading from the stream fails + */ + public static void readFully( + final InputStream in, final byte[] destination, final int startAt, final int numBytes) + throws IOException { + int numRead = 0; + while (numRead < numBytes) { + int readNow = in.read(destination, startAt + numRead, numBytes - numRead); + if (readNow == -1) { + throw new IOException("truncated input stream"); + } + numRead += readNow; + } + } + + /** + * Read into the buffer from an input stream and fail if not enough data to fill the buffer. + * + * @param in the input stream to read from + * @param destination where to write the bytes to + * @throws IOException if reading from the stream fails + */ + public static void readFully(final InputStream in, final byte[] destination) throws IOException { + readFully(in, destination, 0, destination.length); + } +} diff --git a/shared/src/test/java/com/google/archivepatcher/shared/RandomAccessFileInputStreamTest.java b/shared/src/test/java/com/google/archivepatcher/shared/RandomAccessFileInputStreamTest.java index a53a6052..01ca0c53 100644 --- a/shared/src/test/java/com/google/archivepatcher/shared/RandomAccessFileInputStreamTest.java +++ b/shared/src/test/java/com/google/archivepatcher/shared/RandomAccessFileInputStreamTest.java @@ -217,17 +217,15 @@ public void testReset_NoMarkSet() throws IOException { @Test public void testMark_IOExceptionInRaf() throws IOException { stream = - new RandomAccessFileInputStream(tempFile, 0, testData.length) { - @Override - protected RandomAccessFile getRandomAccessFile(File file) throws IOException { - return new RandomAccessFile(file, "r") { + new RandomAccessFileInputStream( + new RandomAccessFile(tempFile, "r") { @Override public long getFilePointer() throws IOException { throw new IOException("Blah314159"); } - }; - } - }; + }, + 0, + testData.length); try { stream.mark(0); assertWithMessage("Executed code that should have failed.").fail(); diff --git a/shared/src/test/java/com/google/archivepatcher/shared/bytesource/ByteStreamsTest.java b/shared/src/test/java/com/google/archivepatcher/shared/bytesource/ByteStreamsTest.java new file mode 100644 index 00000000..1981b4bb --- /dev/null +++ b/shared/src/test/java/com/google/archivepatcher/shared/bytesource/ByteStreamsTest.java @@ -0,0 +1,65 @@ +// Copyright 2016 Google LLC. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.archivepatcher.shared.bytesource; + +import static com.google.archivepatcher.shared.TestUtils.assertThrows; +import static com.google.archivepatcher.shared.bytesource.ByteStreams.readFully; +import static com.google.common.truth.Truth.assertThat; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ByteStreamsTest { + + byte[] input; + byte[] dst; + InputStream inputStream; + + @Before + public void setUp() throws Exception { + input = "this is a sample string to read".getBytes("UTF-8"); + inputStream = new ByteArrayInputStream(input); + dst = new byte[50]; + } + + @Test + public void readFully_notEnoughBytes() throws Exception { + assertThrows(IOException.class, () -> readFully(inputStream, dst, 0, 50)); + } + + @Test + public void readFully_enoughBytes() throws Exception { + readFully(inputStream, dst, 0, input.length); + assertThat(ByteBuffer.wrap(dst, 0, input.length)).isEqualTo(ByteBuffer.wrap(input)); + } + + @Test + public void readFully_nonZeroOffset() throws Exception { + readFully(inputStream, dst, 40, 10); + assertThat(ByteBuffer.wrap(dst, 40, 10)).isEqualTo(ByteBuffer.wrap(input, 0, 10)); + } + + @Test + public void readFully_outOfBounds() throws IOException { + assertThrows(IndexOutOfBoundsException.class, () -> readFully(inputStream, dst, 45, 11)); + } +}