diff --git a/pom.xml b/pom.xml index a0360bb..aefa82e 100644 --- a/pom.xml +++ b/pom.xml @@ -1,7 +1,7 @@ 4.0.0 com.amazonaws - 1.0.1 + 1.0.2-SNAPSHOT dynamodb-import-export-tool jar DynamoDB Import Export Tool @@ -9,6 +9,7 @@ Exports DynamoDB items via parallel scan into a blocking queue, then consumes the queue and import DynamoDB items into a replica table using asynchronous writes. https://github.com/awslabs/dynamodb-import-export-tool.git + 1.0.2-SNAPSHOT 1.10.10 diff --git a/src/main/java/com/amazonaws/dynamodb/bootstrap/AbstractLogConsumer.java b/src/main/java/com/amazonaws/dynamodb/bootstrap/AbstractLogConsumer.java index 960a59d..7f5b35a 100644 --- a/src/main/java/com/amazonaws/dynamodb/bootstrap/AbstractLogConsumer.java +++ b/src/main/java/com/amazonaws/dynamodb/bootstrap/AbstractLogConsumer.java @@ -14,8 +14,10 @@ */ package com.amazonaws.dynamodb.bootstrap; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -30,8 +32,9 @@ */ public abstract class AbstractLogConsumer { - public ExecutorCompletionService exec; - protected ExecutorService threadPool; + // only keep a reference to the thread pool because we need to be able to shut it down + private ExecutorService threadPool; + protected ExecutorCompletionService exec; /** * Logger for the DynamoDBBootstrapWorker. @@ -39,11 +42,23 @@ public abstract class AbstractLogConsumer { private static final Logger LOGGER = LogManager .getLogger(AbstractLogConsumer.class); + + + protected AbstractLogConsumer(int numThreads) { + this.threadPool = Executors.newFixedThreadPool(numThreads); + this.exec = new ExecutorCompletionService(threadPool, new ArrayBlockingQueue>(numThreads)); + } + + protected AbstractLogConsumer(ExecutorCompletionService exec, ExecutorService threadPool) { + this.threadPool = threadPool; + this.exec = exec; + } + /** * Writes the result of a scan to another endpoint asynchronously. Will call * getWorker to determine what job to submit with the result. * - * @param + * @param result * the SegmentedScanResult to asynchronously write to another * endpoint. */ @@ -52,7 +67,7 @@ public abstract class AbstractLogConsumer { /** * Shuts the thread pool down. * - * @param + * @param awaitTermination * If true, this method waits for the threads in the pool to * finish. If false, this thread pool shuts down without * finishing their current tasks. diff --git a/src/main/java/com/amazonaws/dynamodb/bootstrap/BlockingQueueConsumer.java b/src/main/java/com/amazonaws/dynamodb/bootstrap/BlockingQueueConsumer.java index 11cb995..c853bcd 100644 --- a/src/main/java/com/amazonaws/dynamodb/bootstrap/BlockingQueueConsumer.java +++ b/src/main/java/com/amazonaws/dynamodb/bootstrap/BlockingQueueConsumer.java @@ -33,13 +33,8 @@ public class BlockingQueueConsumer extends AbstractLogConsumer { private BlockingQueue queue; public BlockingQueueConsumer(int numThreads) { + super(Math.max(numThreads, Runtime.getRuntime().availableProcessors())); this.queue = new ArrayBlockingQueue(20); - int numProcessors = Runtime.getRuntime().availableProcessors(); - if (numProcessors > numThreads) { - numThreads = numProcessors; - } - this.threadPool = Executors.newFixedThreadPool(numThreads); - this.exec = new ExecutorCompletionService(threadPool); } @Override diff --git a/src/main/java/com/amazonaws/dynamodb/bootstrap/CommandLineInterface.java b/src/main/java/com/amazonaws/dynamodb/bootstrap/CommandLineInterface.java index 67639fc..b8de07e 100644 --- a/src/main/java/com/amazonaws/dynamodb/bootstrap/CommandLineInterface.java +++ b/src/main/java/com/amazonaws/dynamodb/bootstrap/CommandLineInterface.java @@ -91,6 +91,7 @@ public static void main(String[] args) { TableDescription readTableDescription = sourceClient.describeTable( sourceTable).getTable(); + TableDescription writeTableDescription = destinationClient .describeTable(destinationTable).getTable(); int numSegments = 10; @@ -107,11 +108,27 @@ public static void main(String[] args) { final double writeThroughput = calculateThroughput( writeTableDescription, writeThroughputRatio, false); + final double averageItemSize; + if (readTableDescription.getItemCount().equals(Long.valueOf(0L))) { + averageItemSize = 0.0; + } else { + averageItemSize = readTableDescription.getTableSizeBytes().doubleValue() / readTableDescription.getItemCount().doubleValue(); + } + final double averageWcuPerItem = Math.ceil(averageItemSize / 1024.0); + final double averageWcuPerBatchWriteItem = averageWcuPerItem * 25; + final int parallelBatchWriteItems = (int) Math.min(1L, Math.round(Math.ceil(writeThroughput / averageWcuPerBatchWriteItem))); + if (maxWriteThreads > parallelBatchWriteItems) { + LOGGER.warn("Expected WCU per BatchWriteItem call is " + averageWcuPerBatchWriteItem + + " so this configuration could support up to " + parallelBatchWriteItems + + " parallel BatchWriteItem calls. However, maxWriteThreads(" + maxWriteThreads + + ") was greater than this expectation."); + } + try { ExecutorService sourceExec = getSourceThreadPool(numSegments); ExecutorService destinationExec = getDestinationThreadPool(maxWriteThreads); DynamoDBConsumer consumer = new DynamoDBConsumer(destinationClient, - destinationTable, writeThroughput, destinationExec); + destinationTable, writeThroughput, destinationExec, maxWriteThreads); final DynamoDBBootstrapWorker worker = new DynamoDBBootstrapWorker( sourceClient, readThroughput, sourceTable, sourceExec, diff --git a/src/main/java/com/amazonaws/dynamodb/bootstrap/DynamoDBConsumer.java b/src/main/java/com/amazonaws/dynamodb/bootstrap/DynamoDBConsumer.java index a5bfa6c..a828048 100644 --- a/src/main/java/com/amazonaws/dynamodb/bootstrap/DynamoDBConsumer.java +++ b/src/main/java/com/amazonaws/dynamodb/bootstrap/DynamoDBConsumer.java @@ -18,6 +18,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -46,12 +47,11 @@ public class DynamoDBConsumer extends AbstractLogConsumer { * Class to consume logs and write them to a DynamoDB table. */ public DynamoDBConsumer(AmazonDynamoDBClient client, String tableName, - double rateLimit, ExecutorService exec) { + double rateLimit, ExecutorService exec, int parallelBatchWriteItems) { + super(new ExecutorCompletionService(exec, new ArrayBlockingQueue>(parallelBatchWriteItems)), exec); this.client = client; this.tableName = tableName; this.rateLimiter = RateLimiter.create(rateLimit); - super.threadPool = exec; - super.exec = new ExecutorCompletionService(threadPool); } /**