Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 14 additions & 12 deletions client/src/main/java/com/defold/extender/client/ExtenderClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -279,13 +279,15 @@ private void build_async(String platform, String sdkVersion, HttpEntity entity,
response = httpClient.execute(resultRequest);
if (jobStatus == 1) {
log("Job %s completed successfully. Writing result to %s", jobId, destination);
response.getEntity().writeTo(new FileOutputStream(destination));
try(OutputStream os = new FileOutputStream(destination)) {
response.getEntity().writeTo(os);
}
} else {
log("Job %s did not complete successfully. Writing log to %s", jobId, log);
OutputStream os = new FileOutputStream(log);
os.write(String.format("Job id: %s; traceId: %s", jobId, traceId == null ? "null" : traceId).getBytes());
response.getEntity().writeTo(os);
os.close();
try(OutputStream os = new FileOutputStream(log)) {
os.write(String.format("Job id: %s; traceId: %s", jobId, traceId == null ? "null" : traceId).getBytes());
response.getEntity().writeTo(os);
}
throw new ExtenderClientException(String.format("Failed to build source: jobId - %s, traceId - %s", jobId, traceId == null ? "null" : traceId));
}
} else if (statusCode == HttpStatus.SC_NOT_IMPLEMENTED) {
Expand All @@ -294,17 +296,17 @@ private void build_async(String platform, String sdkVersion, HttpEntity entity,
String body = responseBody != null ? EntityUtils.toString(responseBody) : "(unknown)";
String error = String.format("%s (trace id - %s)", body, traceId == null ? "null" : traceId);
log(error);
OutputStream os = new FileOutputStream(log);
os.write(error.getBytes());
os.close();
try (OutputStream os = new FileOutputStream(log)) {
os.write(error.getBytes());
}
throw new ExtenderClientException(error);
} else{
String result = String.format("Async build request failed with status code %d %s", statusCode, statusLine.getReasonPhrase());
log(result);
OutputStream os = new FileOutputStream(log);
os.write(result.getBytes());
response.getEntity().writeTo(os);
os.close();
try (OutputStream os = new FileOutputStream(log)) {
os.write(result.getBytes());
response.getEntity().writeTo(os);
}
throw new ExtenderClientException("Failed to build source.");
}
} catch (ExtenderClientException exc) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -187,8 +187,8 @@ private static String hash(ExtenderResource extenderResource) throws ExtenderCli
private void saveCache() {
Properties properties = new Properties();
properties.putAll(this.persistentHashes);
try {
properties.store(new FileOutputStream(getCacheFile()), null);
try(OutputStream os = new FileOutputStream(getCacheFile())) {
properties.store(os, null);
} catch (IOException e) {
System.out.println(String.format("Could not store cache to '%s'", getCacheFile().getAbsolutePath()));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -249,9 +249,8 @@ public void merge(File main, File[] libraries, File out) throws RuntimeException
}


try {
try(FileWriter writer = new FileWriter(out)) {
FileHandler handler = new FileHandler(basePlist);
FileWriter writer = new FileWriter(out);
handler.save(writer);
} catch (ConfigurationException | IOException e) {
throw new RuntimeException("Failed to write plist: " + e.toString());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,9 +162,8 @@ public void merge(File main, File[] libraries, File out) throws RuntimeException
}
}

try {
try(ModifyingFileWriter writer = new ModifyingFileWriter(out)) {
FileHandler handler = new FileHandler(basePlist);
ModifyingFileWriter writer = new ModifyingFileWriter(out);
handler.save(writer);
} catch (ConfigurationException | IOException e) {
throw new RuntimeException("Failed to write plist: " + e.toString());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,9 @@ static void receiveUpload(MultipartHttpServletRequest request, File uploadDirect
File sourceCodeArchive = new File(uploadDirectory, ExtenderConst.SOURCE_CODE_ARCHIVE_MAGIC_NAME);
if (sourceCodeArchive.exists()) {
LOGGER.debug("Source code archive found. Unarchiving...");
ZipUtils.unzip(new FileInputStream(sourceCodeArchive), uploadDirectory.toPath());
try (InputStream fis = new FileInputStream(sourceCodeArchive)) {
ZipUtils.unzip(fis, uploadDirectory.toPath());
}
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.HttpURLConnection;
import java.net.URI;
import java.nio.charset.StandardCharsets;
Expand Down Expand Up @@ -236,7 +237,9 @@ public CompletableFuture<DefoldSdk> getRemoteSdk(String hash) {
File tmpSdkDirectory = tempDirectoryPath.toFile(); // Either moved or deleted later by Move()

Files.createDirectories(tempDirectoryPath);
ZipUtils.unzip(new FileInputStream(tmpResponseBody), tmpSdkDirectory.toPath());
try (InputStream is = new FileInputStream(tmpResponseBody)) {
ZipUtils.unzip(is, tmpSdkDirectory.toPath());
}

Files.move(tmpSdkDirectory.toPath(), sdkDirectory.toPath(), StandardCopyOption.ATOMIC_MOVE);
isVerified = true;
Expand Down Expand Up @@ -350,7 +353,9 @@ private CompletableFuture<JSONObject> downloadSdkMappings(String hash) {
LOGGER.info("Downloading platform sdks mappings from {} ...", url);
InputStream body = response.getBody();
JSONParser parser = new JSONParser();
result = (JSONObject)parser.parse(new InputStreamReader(body));
try (Reader reader = new InputStreamReader(body)) {
result = (JSONObject)parser.parse(reader);
}
break;
}
} catch(IOException|ParseException exc) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.FileInputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
Expand Down Expand Up @@ -258,7 +259,9 @@ private File resolveDependencyAAR(File dependency, String name, File jobDir) thr

// use job folder as tmp location
File unpackedTmp = new File(jobDir, dependency.getName() + ".tmp");
ZipUtils.unzip(new FileInputStream(dependency), unpackedTmp.toPath());
try (InputStream fis = new FileInputStream(dependency)) {
ZipUtils.unzip(fis, unpackedTmp.toPath());
}
Move(unpackedTmp.toPath(), unpackedTarget.toPath());
return unpackedTarget;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
Expand Down Expand Up @@ -91,9 +92,9 @@ public static File generateVFSOverlay(PodBuildSpec spec, Map<String, Collection<

public static File mergeVFSOverlays(File overlayA, File overlayB) {
JSONParser parser = new JSONParser();
try {
JSONObject parsedOverlayA = (JSONObject)parser.parse(new FileReader(overlayA));
JSONObject parsedOverlayB = (JSONObject)parser.parse(new FileReader(overlayB));
try(Reader readerA = new FileReader(overlayA); Reader readerB = new FileReader(overlayB)) {
JSONObject parsedOverlayA = (JSONObject)parser.parse(readerA);
JSONObject parsedOverlayB = (JSONObject)parser.parse(readerB);

JSONArray roots = (JSONArray)parsedOverlayA.get("roots");
roots.addAll((JSONArray)parsedOverlayB.get("roots"));
Expand Down
16 changes: 11 additions & 5 deletions server/src/test/java/com/defold/extender/ExtenderUtilTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
Expand Down Expand Up @@ -292,11 +293,16 @@ public void testCreatePlatformConfig() throws ExtenderException {

@Test
public void testSHA256Checksum() throws NoSuchAlgorithmException, FileNotFoundException, IOException {
String calculatedSha = ExtenderUtil.calculateSHA256(new FileInputStream("test-data/checksum_sdk/test_sdk.zip"));
String expectedSha = new String(Files.readAllBytes(Path.of("test-data/checksum_sdk/test_sdk.sha256")), StandardCharsets.UTF_8);
assertEquals(expectedSha, calculatedSha);

assertThrows(FileNotFoundException.class, () -> ExtenderUtil.calculateSHA256(new FileInputStream("test-data/checksum_sdk/non-exist.zip")));
try (InputStream is = new FileInputStream("test-data/checksum_sdk/test_sdk.zip")) {
String calculatedSha = ExtenderUtil.calculateSHA256(is);
String expectedSha = new String(Files.readAllBytes(Path.of("test-data/checksum_sdk/test_sdk.sha256")), StandardCharsets.UTF_8);
assertEquals(expectedSha, calculatedSha);
}
assertThrows(FileNotFoundException.class, () -> {
try(InputStream is = new FileInputStream("test-data/checksum_sdk/non-exist.zip")) {
ExtenderUtil.calculateSHA256(is);
}
});
}

@Test
Expand Down
10 changes: 8 additions & 2 deletions server/src/test/java/com/defold/extender/ZipUtilsTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
Expand All @@ -28,9 +30,13 @@ public void zipAndUnzipFiles() throws IOException {
files.add(sourceFile1.toFile());
files.add(sourceFile2.toFile());

ZipUtils.zip(new FileOutputStream(destinationFile.toFile()), null, files);
try (OutputStream os = new FileOutputStream(destinationFile.toFile())) {
ZipUtils.zip(os, null, files);
}

ZipUtils.unzip(new FileInputStream(destinationFile.toFile()), targetDirectory);
try(InputStream is = new FileInputStream(destinationFile.toFile())) {
ZipUtils.unzip(is, targetDirectory);
}

assertEquals(2, targetDirectory.toFile().listFiles().length);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,9 @@ public void shouldGetCachedObject() throws Exception {
for (CacheEntry entry : TestUtils.CACHE_ENTRIES) {
InputStream inputStream = cache.get(entry.getKey());
File source = getSourceFile(entry.getPath());
assertTrue(IOUtils.contentEquals(new FileInputStream(source), inputStream));
try (InputStream fis = new FileInputStream(source)) {
assertTrue(IOUtils.contentEquals(fis, inputStream));
}
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,20 +52,22 @@ public void parseInputStream() throws Exception {
CacheInfoFileParser parser = new CacheInfoFileParser();
File file = new File(ClassLoader.getSystemResource("upload/ne-cache-info.json").toURI());

CacheInfoWrapper info = parser.parse(new FileInputStream(file));
List<CacheEntry> entries = info.getEntries();

assertEquals(1, info.getVersion());
assertEquals("sha256", info.getHashType());
assertEquals(2, entries.size());

CacheEntry entry1 = entries.get(0);
assertEquals(TestUtils.CACHE_ENTRIES[0].getPath(), entry1.getPath());
assertEquals(TestUtils.CACHE_ENTRIES[0].getKey(), entry1.getKey());

CacheEntry entry2 = entries.get(1);
assertEquals(TestUtils.CACHE_ENTRIES[1].getPath(), entry2.getPath());
assertEquals(TestUtils.CACHE_ENTRIES[1].getKey(), entry2.getKey());
try (InputStream is = new FileInputStream(file)) {
CacheInfoWrapper info = parser.parse(is);
List<CacheEntry> entries = info.getEntries();

assertEquals(1, info.getVersion());
assertEquals("sha256", info.getHashType());
assertEquals(2, entries.size());

CacheEntry entry1 = entries.get(0);
assertEquals(TestUtils.CACHE_ENTRIES[0].getPath(), entry1.getPath());
assertEquals(TestUtils.CACHE_ENTRIES[0].getKey(), entry1.getKey());

CacheEntry entry2 = entries.get(1);
assertEquals(TestUtils.CACHE_ENTRIES[1].getPath(), entry2.getPath());
assertEquals(TestUtils.CACHE_ENTRIES[1].getKey(), entry2.getKey());
}
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -338,61 +338,74 @@ public void testConcurrentMappingsDownloading() throws IOException, InterruptedE

@Test
public void testChecksumVerification() throws IOException {
DefoldSdkServiceConfiguration conf = DefoldSdkServiceConfiguration.builder()
.location(DefoldSDKServiceTest.configuration.getLocation())
.cacheSize(1)
.sdkUrls(new String[] {"http://localhost:" + String.valueOf(serverPort) + "/%s.zip"})
.enableSdkVerification(true)
.maxVerificationRetryCount(3)
.build();
DefoldSdkService sdkService = new DefoldSdkService(conf, new SimpleMeterRegistry());
assertDoesNotThrow(() -> sdkService.getSdk("test_sdk"));
Path tmpLocation = Files.createTempDirectory("defoldsdk_checksum_test");
try {
DefoldSdkServiceConfiguration conf = DefoldSdkServiceConfiguration.builder()
.location(tmpLocation)
.cacheSize(1)
.sdkUrls(new String[] {"http://localhost:" + String.valueOf(serverPort) + "/%s.zip"})
.enableSdkVerification(true)
.maxVerificationRetryCount(3)
.build();
DefoldSdkService sdkService = new DefoldSdkService(conf, new SimpleMeterRegistry());
assertDoesNotThrow(() -> sdkService.getSdk("test_sdk"));
} finally {
FileUtils.deleteDirectory(tmpLocation.toFile());
}
}

@Test
public void testInvalidVerification() throws IOException {
DefoldSdkServiceConfiguration disabledVerificationConf = DefoldSdkServiceConfiguration.builder()
.location(DefoldSDKServiceTest.configuration.getLocation())
.cacheSize(0)
.sdkUrls(new String[] {"http://localhost:" + String.valueOf(serverPort) + "/%s.zip"})
.enableSdkVerification(false)
.maxVerificationRetryCount(3)
.build();
DefoldSdkService sdkService = new DefoldSdkService(disabledVerificationConf, new SimpleMeterRegistry());
assertDoesNotThrow(() -> sdkService.getSdk("test_sdk_invalid"));

DefoldSdkServiceConfiguration enabledVerificationConf = DefoldSdkServiceConfiguration.builder()
.location(DefoldSDKServiceTest.configuration.getLocation())
.cacheSize(0)
.sdkUrls(new String[] {"http://localhost:" + String.valueOf(serverPort) + "/%s.zip"})
.enableSdkVerification(true)
.maxVerificationRetryCount(3)
.build();
DefoldSdkService sdkService1 = new DefoldSdkService(enabledVerificationConf, new SimpleMeterRegistry());
// no exception because sdk folder already exists
assertDoesNotThrow(() -> sdkService.getSdk("test_sdk_invalid"));
// force remove cache
sdkService1.evictCache();

ExtenderException exc = assertThrows(ExtenderException.class, () -> sdkService1.getSdk("test_sdk_invalid"));
assertTrue(exc.getMessage().contains("Sdk verification failed"));
Path tmpLocation = Files.createTempDirectory("defoldsdk_invalid_test");
try {
DefoldSdkServiceConfiguration disabledVerificationConf = DefoldSdkServiceConfiguration.builder()
.location(tmpLocation)
.cacheSize(0)
.sdkUrls(new String[] {"http://localhost:" + String.valueOf(serverPort) + "/%s.zip"})
.enableSdkVerification(false)
.maxVerificationRetryCount(3)
.build();
DefoldSdkService sdkService = new DefoldSdkService(disabledVerificationConf, new SimpleMeterRegistry());
assertDoesNotThrow(() -> sdkService.getSdk("test_sdk_invalid"));

DefoldSdkServiceConfiguration enabledVerificationConf = DefoldSdkServiceConfiguration.builder()
.location(tmpLocation)
.cacheSize(0)
.sdkUrls(new String[] {"http://localhost:" + String.valueOf(serverPort) + "/%s.zip"})
.enableSdkVerification(true)
.maxVerificationRetryCount(3)
.build();
DefoldSdkService sdkService1 = new DefoldSdkService(enabledVerificationConf, new SimpleMeterRegistry());
// no exception because sdk folder already exists
assertDoesNotThrow(() -> sdkService.getSdk("test_sdk_invalid"));
// force remove cache
sdkService1.evictCache();

ExtenderException exc = assertThrows(ExtenderException.class, () -> sdkService1.getSdk("test_sdk_invalid"));
assertTrue(exc.getMessage().contains("Sdk verification failed"));
} finally {
FileUtils.deleteDirectory(tmpLocation.toFile());
}
}

@Test
public void testMissingChecksumVerification() throws IOException {
DefoldSdkServiceConfiguration conf = DefoldSdkServiceConfiguration.builder()
.location(DefoldSDKServiceTest.configuration.getLocation())
.cacheSize(0)
.sdkUrls(new String[] {"http://localhost:" + String.valueOf(serverPort) + "/%s.zip"})
.enableSdkVerification(true)
.maxVerificationRetryCount(3)
.build();
DefoldSdkService sdkService = new DefoldSdkService(conf, new SimpleMeterRegistry());
// ensure nothing is left in cache from previous runs
sdkService.evictCache();

ExtenderException exc = assertThrows(ExtenderException.class, () -> sdkService.getSdk("test_sdk_no_checksum"));
assertTrue(exc.getMessage().contains("Sdk verification failed"));
Path tmpLocation = Files.createTempDirectory("defoldsdk_nochecksum_test");
try {
DefoldSdkServiceConfiguration conf = DefoldSdkServiceConfiguration.builder()
.location(tmpLocation)
.cacheSize(0)
.sdkUrls(new String[] {"http://localhost:" + String.valueOf(serverPort) + "/%s.zip"})
.enableSdkVerification(true)
.maxVerificationRetryCount(3)
.build();
DefoldSdkService sdkService = new DefoldSdkService(conf, new SimpleMeterRegistry());

ExtenderException exc = assertThrows(ExtenderException.class, () -> sdkService.getSdk("test_sdk_no_checksum"));
assertTrue(exc.getMessage().contains("Sdk verification failed"));
} finally {
FileUtils.deleteDirectory(tmpLocation.toFile());
}
}

@Test
Expand Down
Loading