From 46a1d9931c3baa7937f00c6fed4a2189d155463f Mon Sep 17 00:00:00 2001 From: alejandromontero Date: Fri, 20 Oct 2017 11:30:18 +0200 Subject: [PATCH 1/2] Added a new 2-Node test cluster in HDI --- .../cluster_hdil16-D14v2-spark21-293.conf | 3 +- .../conf/cluster_hdil2-D14v2-spark21-307.conf | 29 +++++++++++++++++++ shell/conf/hdinsight_defaults.conf | 2 +- 3 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 shell/conf/cluster_hdil2-D14v2-spark21-307.conf diff --git a/shell/conf/cluster_hdil16-D14v2-spark21-293.conf b/shell/conf/cluster_hdil16-D14v2-spark21-293.conf index 18678faa..e5e497a9 100644 --- a/shell/conf/cluster_hdil16-D14v2-spark21-293.conf +++ b/shell/conf/cluster_hdil16-D14v2-spark21-293.conf @@ -3,8 +3,7 @@ source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/hdinsight_defaults.con #Cluster config defaultProvider="hdinsight" -azureLocation="Central US" -azureLocationShort="centralus" +azureLocation="East US 2" clusterID="$(get_id_cluster "$(basename $BASH_SOURCE)")" clusterName="hdil16-D14v2-spark21-${clusterID}" #no ADLS diff --git a/shell/conf/cluster_hdil2-D14v2-spark21-307.conf b/shell/conf/cluster_hdil2-D14v2-spark21-307.conf new file mode 100644 index 00000000..69d9d859 --- /dev/null +++ b/shell/conf/cluster_hdil2-D14v2-spark21-307.conf @@ -0,0 +1,29 @@ +#load cluster defaults +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/hdinsight_defaults.conf" + +#Cluster config +defaultProvider="hdinsight" +azureLocation="East US 2" + +clusterID="$(get_id_cluster "$(basename $BASH_SOURCE)")" +clusterName="hdil2-D14v2-spark21-${clusterID}" #no ADLS +#storageAccountName +numberOfNodes="2" #starts at 0 (max 99) 0 is assigned to master +#Node config +headnodeSize="Standard_D14_v2" +vmSize='Standard_D14_v2' +vmType='linux' + +attachedVolumes="0" +diskSize="1024" + +#queueJobs="true" #enable on cluster config to queue benchmarks after deploy + +#details +vmCores="16" +vmRAM="112" + +#costs +clusterCostHour="83.71" +clusterType="PaaS" +clusterDescription="" diff --git a/shell/conf/hdinsight_defaults.conf b/shell/conf/hdinsight_defaults.conf index 80c464b0..07f5ce7d 100644 --- a/shell/conf/hdinsight_defaults.conf +++ b/shell/conf/hdinsight_defaults.conf @@ -22,7 +22,7 @@ CLUSTER_DISKS="RR1" #Cost per remote/hour $0.024/GB/month USD 24$ /month Locally Redundant clusterCostDisk="0.17" # 5TB/hr -resourceGroup="Default-Storage-SouthCentralUS" +resourceGroup="alojadlapremium" # We use spark as the type as it already incorporates hadoop and hive hdiType="Spark" #Hadoop # TODO Remove: should not be used From feca5bc296a40a6092b5d61870fa88bebde8dffe Mon Sep 17 00:00:00 2001 From: alejandromontero Date: Thu, 2 Nov 2017 11:23:37 +0100 Subject: [PATCH 2/2] Increased the number of nodes in HDI cluster with ID 295 --- ...2-spark21-295.conf => cluster_hdil64-D14v2-spark21-295.conf} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename shell/conf/{cluster_hdil16-D14v2-spark21-295.conf => cluster_hdil64-D14v2-spark21-295.conf} (90%) diff --git a/shell/conf/cluster_hdil16-D14v2-spark21-295.conf b/shell/conf/cluster_hdil64-D14v2-spark21-295.conf similarity index 90% rename from shell/conf/cluster_hdil16-D14v2-spark21-295.conf rename to shell/conf/cluster_hdil64-D14v2-spark21-295.conf index e5735568..eb49a893 100644 --- a/shell/conf/cluster_hdil16-D14v2-spark21-295.conf +++ b/shell/conf/cluster_hdil64-D14v2-spark21-295.conf @@ -9,7 +9,7 @@ clusterID="$(get_id_cluster "$(basename $BASH_SOURCE)")" clusterName="hdil16-D14v2-spark21-${clusterID}" #no ADLS #storageAccountName storageAccount="hdil295" -numberOfNodes="32" #starts at 0 (max 99) 0 is assigned to master +numberOfNodes="64" #starts at 0 (max 99) 0 is assigned to master #Node config headnodeSize="Standard_D14_v2" vmSize='Standard_D14_v2'