diff --git a/utilities/.DS_Store b/utilities/.DS_Store new file mode 100644 index 0000000..7407ad5 Binary files /dev/null and b/utilities/.DS_Store differ diff --git a/utilities/transfer/dashboard_export.sh b/utilities/transfer/dashboard_export.sh new file mode 100755 index 0000000..c9f5697 --- /dev/null +++ b/utilities/transfer/dashboard_export.sh @@ -0,0 +1,208 @@ +#!/bin/bash + +# Script to export dashboards from a specified cClear as JSON files +# "folderUid" and "folderTitle" are added to each dashboard to help identify the folders to import later by +# running "dashboard_import.sh". +# Modified from Paul Sulistio's scripts. + +# todo: add support to import multiple folders separated by ";" + +OPTSPEC=":hu:p:t:f:" + +# Show help on how to use this script +show_help() { +cat << EOF +Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST_IP] [-f FROM_FOLDER] +Script to export grafana dashboards + -u Required. cClear user to login + -p Required. cClear user password to login + -t Required. The IP of the source cClear host i.e 10.51.10.32 + -f Optional. The name of the folder to export from, double quotes with spaces. Export all folders if not + specified. + -h Display this help and exit. +EOF +} + +# Check script invocation options +while getopts "$OPTSPEC" optchar; do + case "$optchar" in + h) + show_help + exit + ;; + u) + USER="$OPTARG";; + p) + PASSWORD="$OPTARG";; + t) + TARGET_HOST_IP="$OPTARG";; + f) + FROM="$OPTARG";; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +# Check required arguments +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$TARGET_HOST_IP" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|-------------------------------------------------------------------------|\n" + printf "|$1|\n"; + printf "|-------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + +function init() { + DASH_FOLDER="dashboards" + DASH_DIR="$PWD/${DASH_FOLDER}" + if [ ! -d "${DASH_DIR}" ]; then + mkdir -p "${DASH_DIR}" + else + log_title "----------------- A $DASH_DIR directory already exists! -----------------" + log_title "----------------- Rename or remove this directory before continuing -----------------" + exit 1 + fi +} + +# set cookie param for curl command according to login options +STATUS_CODE=$(curl --noproxy '*' -k --write-out '%{http_code}' --silent --output /dev/null --data "uname=$USER&psw=$PASSWORD" "https://$TARGET_HOST_IP/sess/login?rp=/vb/") +if [[ "$STATUS_CODE" -eq 404 ]]; then + HOST="https://$USER:$PASSWORD@$TARGET_HOST_IP" +elif [[ "$STATUS_CODE" -eq 302 ]]; then + HOST="https://$TARGET_HOST_IP" + mycookie="$PWD/mycookie" + LOGIN=$(curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/) + CURL_COOKIE="-b $mycookie" +else + show_help + exit 1 +fi + +# get folders +folder_json=$(curl --noproxy '*' -k $CURL_COOKIE --request "GET" -H "Content-Type:application/json" \ +"$HOST/graph-engine/api/folders") +# From folder specified: +declare -a dashboard_uids +if [ ${#FROM} -gt 0 ]; then + IFS=';' read -ra FROM_LIST <<< "$FROM" + for i in "${FROM_LIST[@]}"; do + folder_i=$(echo $i | xargs) + # Find matching folder from remote (with folder title) + FOLDER_UID=$(echo "$folder_json" | jq -r '.[] | select(.title == "'"${folder_i}"'") | .uid') + # Folder not found, prompt error and exit + if [ -z "$FOLDER_UID" ] ; then + log_failure "Folder ${i} is not found. Please check spelling and double quote with any spaces." + continue + fi + # Folder found: get the collection of dashboard uids in this folder + uids=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST"/graph-engine/api/search\?query\=\& | \ + jq -r '.[] | select(.type | contains("dash-db")) | select(.folderUid != null) | select(.folderUid == "'"$FOLDER_UID"'") | .uid') + dashboard_uids+=${uids[@]} + done +# From all folders: +else + dashboard_uids=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST"/graph-engine/api/search\?query\=\& | \ + jq -r '.[] | select(.type | contains("dash-db")) | .uid') +fi + +#echo "dashboard_uids: "$dashboard_uids + +# exit if nothing to import +#if [[ ${#dashboard_uid[@]} -eq 0 ]]; then +# exit 1 +#fi + +# Export dashboards +init +counter=0 +for dashboard_uid in $dashboard_uids; do + url=$(echo "$HOST/graph-engine/api/dashboards/uid/$dashboard_uid" | tr -d '\r') + dashboard_json=$(curl --noproxy '*' -k $CURL_COOKIE "$url") + dashboard_title=$(echo "$dashboard_json" | jq -r '.dashboard | .title' | sed -r 's/[ \/]+/_/g' ) + dashboard_file=$(echo "$dashboard_title" | tr '[:upper:]' '[:lower:]') + dashboard_version=$(echo "$dashboard_json" | jq -r '.dashboard | .version') + dashboard_folder_raw=$(echo "$dashboard_json" | jq -r '.meta | .folderTitle') + dashboard_folder=$(echo "$dashboard_json" | jq -r '.meta | .folderTitle' | sed -r 's/[ \/]+/_/g' ) + dashboard_folderId=$(echo "$dashboard_json" | jq -r '.meta | .folderId') + + # Find folder uid to save (so that importing can find the right folder to import to) + folder_uid=$(echo "$folder_json" | jq -r '.[] | select(.id=='$dashboard_folderId') | .uid ') + + # create the folder if not existing + if [ ! -d "${DASH_DIR}/${dashboard_folder}" ]; then + mkdir "${DASH_DIR}/${dashboard_folder}" + fi + + counter=$((counter + 1)) + # save dashboard with folder uid and title to help identify folders to import later. + echo "$dashboard_json" | jq '.dashboard | . += {"folderUid":"'$folder_uid'", "folderTitle": "'"$dashboard_folder_raw"'"}' > \ + "$DASH_DIR/${dashboard_folder}/${dashboard_file}_v${dashboard_version}.json" + log_success "Dashboard has been saved\t\t title=\"${dashboard_file}\", uid=\"${dashboard_uid}\", + path=\"${DASH_DIR}/${dashboard_folder}/${dashboard_file}_v${dashboard_version}.json\"." +done + +if [[ ${counter} -gt 0 ]]; then + cclear_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/api/admin/info") + CCLEAR_VERSION="cclear_$(echo $cclear_json | jq '.data.software.build' | tr -d '"')" + grafana_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/graph-engine/api/health") + GRAFANA_VERSION="grafana_$(echo $grafana_json | jq '.version' | tr -d '"')" + DATE_TIME="date_$(date '+%d%m%Y_%H%M%S')" + DASH_FILE_ZIP="${DASH_FOLDER}_${TARGET_HOST_IP}_${CCLEAR_VERSION}_${GRAFANA_VERSION}_${DATE_TIME}" + zip -r -m ${DASH_FILE_ZIP}.zip ${DASH_FOLDER} +fi +rm mycookie 2> /dev/null + +log_title "${counter} dashboards were saved in "$PWD/${DASH_FILE_ZIP}".zip"; +log_title "------------------------------ FINISHED ---------------------------------"; diff --git a/utilities/transfer/dashboard_import.sh b/utilities/transfer/dashboard_import.sh new file mode 100755 index 0000000..40c2e63 --- /dev/null +++ b/utilities/transfer/dashboard_import.sh @@ -0,0 +1,204 @@ +#!/bin/bash + +# Script to import dashboard JSON files into a specified cClear +# Dashboards json files from running "dashboard_export.sh" will get imported to the specified folders. +# Dashboards saved from Grafana import will get imported to the "General" folder. +# Modified from Paul Sulistio's scripts. + +OPTSPEC=":hu:p:t:i:" + +###### Show help on how to use this script ###### +show_help() { +cat << EOF +Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST_IP] [-i IMPORT_PATH] +Script to import dashboards into Grafana + -u Required. cClear user to login + -p Required. cClear user password to login + -t Required. The IP of the destination cClear host i.e 10.51.10.32 + -i Required. Full path to the folder or zip file containing JSON exports of the dashboards + you want to be imported. + -h Display this help and exit. +EOF +} + +###### Check script invocation options ###### +while getopts "$OPTSPEC" optchar; do + case "$optchar" in + h) + show_help + exit + ;; + u) + USER="$OPTARG";; + p) + PASSWORD="$OPTARG";; + t) + TARGET_HOST_IP="$OPTARG";; + i) + IMPORT_PATH="$OPTARG";; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +###### Check required arguments ###### +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$IMPORT_PATH" ] || [ -z "$TARGET_HOST_IP" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[%s] $1\n" "$timestamp" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[%s] $1\n" "$timestamp" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|---------------------------------------------------------------------------------------|\n" + printf "| %s |\n" "$1"; + printf "|---------------------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + + +ZIP_FILE=$(basename $IMPORT_PATH) + +if [[ $ZIP_FILE =~ \.zip$ ]]; then + DASH_DIR=$(unzip -qql $IMPORT_PATH | head -n1 | tr -s ' ' | cut -d' ' -f5-) + unzip -o $IMPORT_PATH + DIR_LENGTH=${#DASH_DIR} + DASH_DIR=${DASH_DIR:0:DIR_LENGTH-1} +else + DASH_DIR=$IMPORT_PATH +fi + +if [ -d "$DASH_DIR" ]; then + DASH_LIST=$(find "$PWD/$DASH_DIR" -mindepth 1 -name \*.json) + + if [ -z "$DASH_LIST" ]; then + log_title "----------------- $DASH_DIR contains no JSON files! -----------------" + log_failure "Directory $DASH_DIR does not appear to contain any JSON files for import. Check your path and try again." + exit 1 + else + FILESTOTAL=$(echo "$DASH_LIST" | wc -l) + log_title "----------------- Starting import of $FILESTOTAL dashboards -----------------" + fi +else + log_title "-------------------- $DASH_DIR directory not found! -----------------" + log_failure "Directory $DASH_DIR does not exist. Check your path and try again." + exit 1 +fi + +# set cookie param for curl command according to login options +STATUS_CODE=$(curl --noproxy '*' -k --write-out '%{http_code}' --silent --output /dev/null --data "uname=$USER&psw=$PASSWORD" "https://$TARGET_HOST_IP/sess/login?rp=/vb/") +if [[ "$STATUS_CODE" -eq 404 ]]; then + HOST="https://$USER:$PASSWORD@$TARGET_HOST_IP" +elif [[ "$STATUS_CODE" -eq 302 ]]; then + HOST="https://$TARGET_HOST_IP" + mycookie="$PWD/mycookie" + LOGIN=$(curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/) + CURL_COOKIE="-b $mycookie" +else + show_help + exit 1 +fi + +NUMSUCCESS=0 +NUMFAILURE=0 +COUNTER=0 +for DASH_FILE in $DASH_LIST; do + COUNTER=$((COUNTER + 1)) + echo "Import $COUNTER/$FILESTOTAL: $DASH_FILE..." + + # Get folder uid and title from dashboard + dashboard=$(cat "$DASH_FILE") + folder_title=$(echo "$dashboard" | jq -r '.folderTitle') + folder_uid=$(echo "$dashboard" | jq -r '.folderUid') + dashboard=$(echo "$dashboard" | jq -r 'del(.folderTitle) | del(.folderUid)') + # shellcheck disable=SC2116 + dashboard=$(echo '{"dashboard": ' "${dashboard}"'}') + + # If folder uid id not provided, import to the "General" folder + if [ ${#folder_uid} -eq 0 ]; then + RESULT=$(echo "$dashboard" | jq -r '. * {overwrite: true, dashboard: {id: null}}' | curl -k $CURL_COOKIE -X POST \ + -H "Content-Type: application/json" $HOST/graph-engine/api/dashboards/db -d @-) + else + # Find the folder id from $HOST with this folder uid + folder_id=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST"/graph-engine/api/folders/$folder_uid | jq -r '.id') + # If not found, try finding it with folder title. + if [ "$folder_id" == "null" ]; then + folder_json=$(curl --noproxy '*' -k $CURL_COOKIE --request "GET" -H "Content-Type:application/json" \ + "$HOST/graph-engine/api/folders") + folder_id=$(echo "$folder_json" | jq -r '.[] | select(.title == "'"$folder_title"'") | .id') + folder_uid=$(echo "$folder_json" | jq -r '.[] | select(.title == "'"$folder_title"'") | .uid') + fi + # If still not found, create a folder with this folder uid and folder title. + if [ "$folder_id" == "null" ] || [ ${#folder_id} -eq 0 ]; then + echo " here" + folder_new=$(echo '{"uid": "'$folder_uid'", "title": "'"$folder_title"'"}' | curl --noproxy '*' -k \ + $CURL_COOKIE -X POST -H "Content-Type: application/json" $HOST/graph-engine/api/folders -d @-) + folder_id=$(echo $folder_new | jq -r '.id') + folder_uid=$(echo $folder_new | jq -r '.uid') + fi + # Import dashboard with folder id, uid, and title + RESULT=$(echo "$dashboard" | jq -r '. * {overwrite: true, dashboard: {id: null}} | . += {folderId:'$folder_id', folderUid:"'$folder_uid'", folderTitle: "'"$folder_title"'"}' \ + | curl --noproxy '*' -k $CURL_COOKIE -X POST -H "Content-Type: application/json" $HOST/graph-engine/api/dashboards/db -d @-) + fi + + # log result + if [[ "$RESULT" == *"success"* ]]; then + log_success "$RESULT" + NUMSUCCESS=$((NUMSUCCESS + 1)) + else + log_failure "$RESULT" + NUMFAILURE=$((NUMFAILURE + 1)) + fi +done + +rm mycookie +rm -rf "$DASH_DIR" + +log_title "Import complete. $NUMSUCCESS dashboards were successfully imported. $NUMFAILURE dashboard imports failed."; +log_title "-------------------------------------FINISHED----------------------------------------"; diff --git a/utilities/transfer/datasource_export.sh b/utilities/transfer/datasource_export.sh new file mode 100755 index 0000000..301af05 --- /dev/null +++ b/utilities/transfer/datasource_export.sh @@ -0,0 +1,148 @@ +#!/bin/bash + +# Script to export datasource from a specified cClear as JSON files. +# Modified from Paul Sulistio's scripts. + +OPTSPEC=":hu:p:t:" + +###### Show help on how to use this script ###### +show_help() { +cat << EOF +Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST_IP] +Script to export grafana datasources + -u Required. cClear user to login + -p Required. cClear user password to login + -t Required. The IP of the target host i.e 10.51.10.32 + -h Display this help and exit. +EOF +} + +###### Check script invocation options ###### +while getopts "$OPTSPEC" optchar; do + case "$optchar" in + h) + show_help + exit + ;; + u) + USER="$OPTARG";; + p) + PASSWORD="$OPTARG";; + t) + TARGET_HOST_IP="$OPTARG";; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +###### Check required arguments ###### +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$TARGET_HOST_IP" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|-------------------------------------------------------------------------|\n" + printf "|$1|\n"; + printf "|-------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + +function init() { + DS_FOLDER="datasources" + DS_DIR="$PWD/${DS_FOLDER}" + echo $DS_DIR + + if [ ! -d "${DS_DIR}" ]; then + mkdir -p "${DS_DIR}" + else + log_title "----------------- A $DS_DIR directory already exists! -----------------" + log_title "----------------- Rename or remove this directory before continuing -----------------" + exit 1 + fi +} + +# set cookie param for curl command according to login options +STATUS_CODE=$(curl --noproxy '*' -k --write-out '%{http_code}' --silent --output /dev/null --data "uname=$USER&psw=$PASSWORD" "https://$TARGET_HOST_IP/sess/login?rp=/vb/") +if [[ "$STATUS_CODE" -eq 404 ]]; then + HOST="https://$USER:$PASSWORD@$TARGET_HOST_IP" +elif [[ "$STATUS_CODE" -eq 302 ]]; then + HOST="https://$TARGET_HOST_IP" + mycookie="$PWD/mycookie" + LOGIN=$(curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/) + CURL_COOKIE="-b $mycookie" +else + show_help + exit 1 +fi + +counter=0 +init +datasource_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/graph-engine/api/datasources") +for id in $(echo $datasource_json | jq -r '.[] | .id'); do + name=$(echo $datasource_json | jq -r '.[] | select(.id == '"$id"') | .name' | sed -r 's/[ \/]+/_/g' | \ + tr '[:upper:]' '[:lower:]') + counter=$((counter + 1)) + curl --noproxy '*' -f -k $CURL_COOKIE "$HOST/graph-engine/api/datasources/${id}" | jq '' > "$DS_DIR/${name}.json" + log_success "Datasource has been saved\t id=\"${id}\", name=\"${name}\", path=\"${DS_DIR}/${name}.json\"." +done + +cclear_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/api/admin/info") +CCLEAR_VERSION="cclear_$(echo $cclear_json | jq '.data.software.build' | tr -d '"')" +grafana_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/graph-engine/api/health") +GRAFANA_VERSION="grafana_$(echo $grafana_json | jq '.version' | tr -d '"')" +DATE_TIME="date_$(date '+%d%m%Y_%H%M%S')" +DS_FILE_ZIP="${DS_FOLDER}_${TARGET_HOST_IP}_${CCLEAR_VERSION}_${GRAFANA_VERSION}_${DATE_TIME}" +zip -r -m "${DS_FILE_ZIP}.zip" "${DS_FOLDER}" +rm mycookie + +log_title "${counter} datasource(s) were saved and zipped in "$PWD/${DS_FILE_ZIP}".zip"; +log_title "------------------------------ FINISHED ---------------------------------"; diff --git a/utilities/transfer/datasource_import.sh b/utilities/transfer/datasource_import.sh new file mode 100755 index 0000000..d6e140b --- /dev/null +++ b/utilities/transfer/datasource_import.sh @@ -0,0 +1,163 @@ +#!/bin/bash + +# Script to import datasource JSON files into a specified cClear +# Tests have been done on exported datasource json files from running "datasource_export.sh". +# Modified from Paul Sulistio's scripts. + +OPTSPEC=":hu:p:t:i:" + +###### Show help on how to use this script ###### +show_help() { +cat << EOF +Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST_IP] [-i IMPORT_PATH] +Script to import datasource into Grafana + -u Required. cClear user to login + -p Required. cClear user password to login + -t Required. The IP of the target host i.e 10.51.10.32 + -i Required. Full path to the zip file containing JSON exports of the datasource you want to be imported. + -h Display this help and exit. +EOF +} + +###### Check script invocation options ###### +while getopts "$OPTSPEC" optchar; do + case "$optchar" in + h) + show_help + exit + ;; + u) + USER="$OPTARG";; + p) + PASSWORD="$OPTARG";; + t) + TARGET_HOST_IP="$OPTARG";; + i) + IMPORT_PATH="$OPTARG";; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +###### Check required arguments ###### +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$IMPORT_PATH" ] || [ -z "$TARGET_HOST_IP" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[%s] $1\n" "$timestamp" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[%s] $1\n" "$timestamp" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|-----------------------------------------------------------------------------------------|\n" + printf "| %s |\n" "$1"; + printf "|-----------------------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + +ZIP_FILE=$(basename $IMPORT_PATH) +echo $ZIP_FILE + +if [[ $ZIP_FILE =~ \.zip$ ]]; then + DS_DIR=$(unzip -qql $IMPORT_PATH | head -n1 | tr -s ' ' | cut -d' ' -f5-) + unzip -o $IMPORT_PATH + DS_DIR=${DS_DIR: : -1} + if [ -d "$DS_DIR" ]; then + DS_LIST=$(find "$PWD/$DS_DIR" -mindepth 1 -name \*.json) + echo $DS_LIST + if [ -z "$DS_LIST" ]; then + log_title "----------------- $DS_DIR contains no JSON files! -----------------" + log_failure "Directory $DS_DIR does not appear to contain any JSON files for import. Check your path and try again." + exit 1 + else + FILESTOTAL=$(echo "$DS_LIST" | wc -l) + log_title "----------------- Starting import of $FILESTOTAL datasource(s) -----------------" + fi + else + log_title "-------------------- $DS_DIR directory not found! -----------------" + log_failure "Directory $DS_DIR does not exist. Check your path and try again." + exit 1 + fi +else + log_title "-------------------- $ZIP_FILE Wrong format! -----------------" + log_failure "$ZIP_FILE is not a zip file. Please enter a correct file" +fi + +# set cookie param for curl command according to login options +STATUS_CODE=$(curl --noproxy '*' -k --write-out '%{http_code}' --silent --output /dev/null --data "uname=$USER&psw=$PASSWORD" "https://$TARGET_HOST_IP/sess/login?rp=/vb/") +if [[ "$STATUS_CODE" -eq 404 ]]; then + HOST="https://$USER:$PASSWORD@$TARGET_HOST_IP" +elif [[ "$STATUS_CODE" -eq 302 ]]; then + HOST="https://$TARGET_HOST_IP" + mycookie="$PWD/mycookie" + LOGIN=$(curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/) + CURL_COOKIE="-b $mycookie" +else + show_help + exit 1 +fi + +NUMSUCCESS=0 +NUMFAILURE=0 +COUNTER=0 +for i in datasources/*; do + RESULT=$(curl --noproxy '*' -k $CURL_COOKIE -X "POST" "$HOST/graph-engine/api/datasources" \ + -H "Content-Type: application/json" --data-binary @$i) + if [[ "$RESULT" == *"Datasource added"* ]]; then + log_success "$RESULT" + NUMSUCCESS=$((NUMSUCCESS + 1)) + else + log_failure "$RESULT" + NUMFAILURE=$((NUMFAILURE + 1)) + fi +done + +rm mycookie + +log_title "Import complete. $NUMSUCCESS datasource(s) successfully imported. $NUMFAILURE datasource(s) imports failed."; +log_title "---------------------------------------FINISHED----------------------------------------"; diff --git a/utilities/transfer/preference_export.sh b/utilities/transfer/preference_export.sh new file mode 100755 index 0000000..38c79a4 --- /dev/null +++ b/utilities/transfer/preference_export.sh @@ -0,0 +1,158 @@ +#!/bin/bash + +# Script to export preferences from a specified cClear as JSON files. +# Modified from Paul Sulistio's scripts. + +###### Show help on how to use this script ###### +OPTSPEC=":hu:p:t:" + +show_help() { +cat << EOF +Usage: $0 [-u USER] [-p PASSWORD] [-f FROM_FOLDER] [-t TARGET_HOST_IP] +Script to export grafana dashboards + -u Required. cClear user to login + -p Required. cClear user password to login + -t Required. The IP of the source cClear host i.e 10.51.10.32 + -h Display this help and exit. +EOF +} + +###### Check script invocation options ###### +while getopts "$OPTSPEC" optchar; do + case "$optchar" in + h) + show_help + exit + ;; + u) + USER="$OPTARG";; + p) + PASSWORD="$OPTARG";; + t) + TARGET_HOST_IP="$OPTARG";; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +###### Check required arguments ###### +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$TARGET_HOST_IP" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|-------------------------------------------------------------------------|\n" + printf "|$1|\n"; + printf "|-------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + +function init() { + PREF_FOLDER="preferences" + PREF_DIR="$PWD/${PREF_FOLDER}" + if [ ! -d "${PREF_DIR}" ]; then + mkdir -p "${PREF_DIR}" + else + log_title "----------------- A $PREF_DIR directory already exists! -----------------" + log_title "----------------- Rename or remove this directory before continuing -----------------" + exit 1 + fi +} + +# set cookie param for curl command according to login options +STATUS_CODE=$(curl --noproxy '*' -k --write-out '%{http_code}' --silent --output /dev/null --data "uname=$USER&psw=$PASSWORD" "https://$TARGET_HOST_IP/sess/login?rp=/vb/") +if [[ "$STATUS_CODE" -eq 404 ]]; then + HOST="https://$USER:$PASSWORD@$TARGET_HOST_IP" +elif [[ "$STATUS_CODE" -eq 302 ]]; then + HOST="https://$TARGET_HOST_IP" + mycookie="$PWD/mycookie" + LOGIN=$(curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/) + CURL_COOKIE="-b $mycookie" +else + show_help + exit 1 +fi + +init +# Get preferences +pref_org_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/graph-engine/api/org/preferences") +pref_user_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/graph-engine/api/user/preferences") + +# log result +if [ -z "$pref_org_json" ] || [ -z "$pref_user_json" ]; then + log_failure "Failed to download preferences. Please check parameters passed in. " + show_help + exit 1 +elif [[ "$pref_org_json" == *"Unauthorized"* ]] || [[ "$pref_user_json" == *"Unauthorized"* ]]; then + log_failure "Failed to login: $pref_org_json; $pref_user_json" + exit 1 +fi + +# Save preferences +PREF_ORG="$PREF_DIR/preferences_org.json" +PREF_USER="$PREF_DIR/preferences_user.json" +echo $pref_org_json | jq '.' > "$PREF_ORG" +log_success "Org. preferences saved: $pref_org_json" +echo $pref_user_json | jq '.' > "$PREF_USER" +log_success "User preferences saved: $pref_user_json" + +cclear_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/api/admin/info") +CCLEAR_VERSION="cclear_$(echo $cclear_json | jq '.data.software.build' | tr -d '"')" +grafana_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/graph-engine/api/health") +GRAFANA_VERSION="grafana_$(echo $grafana_json | jq '.version' | tr -d '"')" +DATE_TIME="date_$(date '+%d%m%Y_%H%M%S')" +PREF_FILE_ZIP="${PREF_FOLDER}_${TARGET_HOST_IP}_${CCLEAR_VERSION}_${GRAFANA_VERSION}_${DATE_TIME}" +zip -r -m "${PREF_FILE_ZIP}.zip" "${PREF_FOLDER}" +rm mycookie + +log_title "Preferences were saved in ${PREF_FILE_ZIP}"; +log_title "------------------------------ FINISHED ---------------------------------"; diff --git a/utilities/transfer/preference_import.sh b/utilities/transfer/preference_import.sh new file mode 100755 index 0000000..e0376c9 --- /dev/null +++ b/utilities/transfer/preference_import.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +# Script to import preference JSON files into a specified cClear +# Tests have been done on exported preference json files from running "preference_export.sh". +# Modified from Paul Sulistio's scripts. + +OPTSPEC=":hu:p:t:i:" + +###### Show help on how to use this script ###### +show_help() { +cat << EOF +Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST_IP] [-z IMPORT_PATH] +Script to import dashboards into Grafana + -u Required. cClear user to login + -p Required. cClear user password to login + -t Required. The IP of the destination cClear host i.e 10.51.10.32 + -i Required. Grafana preferences json file to import from. e.g. preferences.json + -h Display this help and exit. +EOF +} + +###### Check script invocation options ###### +while getopts "$OPTSPEC" optchar; do + case "$optchar" in + h) + show_help + exit + ;; + u) + USER="$OPTARG";; + p) + PASSWORD="$OPTARG";; + t) + TARGET_HOST_IP="$OPTARG";; + i) + IMPORT_PATH="$OPTARG";; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +###### Check required arguments ###### +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$IMPORT_PATH" ] || [ -z "$TARGET_HOST_IP" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[%s] $1\n" "$timestamp" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[%s] $1\n" "$timestamp" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|---------------------------------------------------------------------------------------|\n" + printf "| %s |\n" "$1"; + printf "|---------------------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + +ZIP_FILE=$(basename $IMPORT_PATH) + +if [[ $ZIP_FILE =~ \.zip$ ]]; then + PREF_DIR=$(unzip -qql $IMPORT_PATH | head -n1 | tr -s ' ' | cut -d' ' -f5-) + unzip -o $IMPORT_PATH + DIR_LENGTH=${#PREF_DIR} + PREF_DIR=${PREF_DIR:0:DIR_LENGTH-1} +else + PREF_DIR=$IMPORT_PATH +fi + +if [ -d "$PREF_DIR" ]; then + PREF_LIST=$(find "$PWD/$PREF_DIR" -mindepth 1 -name \*.json) + + if [ -z "$PREF_LIST" ]; then + log_title "----------------- $PREF_DIR contains no JSON files! -----------------" + log_failure "Directory $PREF_DIR does not appear to contain any JSON files for import. Check your path and try again." + exit 1 + else + FILESTOTAL=$(echo "$PREF_LIST" | wc -l) + log_title "----------------- Starting import of $FILESTOTAL dashboards -----------------" + fi +else + log_title "-------------------- $PREF_DIR directory not found! -----------------" + log_failure "Directory $PREF_DIR does not exist. Check your path and try again." + exit 1 +fi + +# set cookie param for curl command according to login options +STATUS_CODE=$(curl --noproxy '*' -k --write-out '%{http_code}' --silent --output /dev/null --data "uname=$USER&psw=$PASSWORD" "https://$TARGET_HOST_IP/sess/login?rp=/vb/") +if [[ "$STATUS_CODE" -eq 404 ]]; then + HOST="https://$USER:$PASSWORD@$TARGET_HOST_IP" +elif [[ "$STATUS_CODE" -eq 302 ]]; then + HOST="https://$TARGET_HOST_IP" + mycookie="$PWD/mycookie" + LOGIN=$(curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/) + CURL_COOKIE="-b $mycookie" +else + show_help + exit 1 +fi + +NUMSUCCESS=0 +NUMFAILURE=0 +COUNTER=0 +for PREF_FILE in $PREF_LIST; do + COUNTER=$((COUNTER + 1)) + echo "Import $COUNTER/$FILESTOTAL: $PREF_FILE..." + RESULT=$(cat "$PREF_FILE" | jq '.' | curl --noproxy '*' -k $CURL_COOKIE -X PUT -H \ + "Content-Type: application/json" "$HOST/graph-engine/api/user/preferences" -d @-) + echo + # log result + if [[ "$RESULT" == *"updated"* ]]; then + log_success "$RESULT" + NUMSUCCESS=$((NUMSUCCESS + 1)) + else + log_failure "$RESULT" + NUMFAILURE=$((NUMFAILURE + 1)) + fi +done +rm mycookie + +log_title "Import complete. $NUMSUCCESS dashboards were successfully imported. $NUMFAILURE dashboard imports failed."; +log_title "-------------------------------------FINISHED----------------------------------------"; diff --git a/utilities/transform/dashboard_field_renamed.py b/utilities/transform/dashboard_field_renamed.py new file mode 100644 index 0000000..89c82c5 --- /dev/null +++ b/utilities/transform/dashboard_field_renamed.py @@ -0,0 +1,79 @@ +import json +import os +import sys + +# The key of property to update value with +CONVERTED_PATH_NAME = "converted" +RENAMED_MAP = {"bytes_per_s": "bytes", + "fragments_per_s": "fragments", + "packets_per_s": "packets", + "resolution_s": "resolution"}; + + +def convert_dashboard(dash_parent_obj, dash_obj_key, dash_obj): + """ + Depth First Search to find any string value and replace the matching old value with its specified new value. With any + value matching and updated, the update is done using dash_parent_obj and dash_obj_key. Thus it doesn't have to bubble + up to its parent for update. + + :param dash_parent_obj: The parent element that owns this element being inspected. + :param dash_obj_key: The key of the element being inspected. + :param dash_obj: The value of the element being inspected. + + :return: + """ + is_updated: bool = False + if type(dash_obj) == list: + for ls in dash_obj: + temp_is_updated = convert_dashboard(dash_parent_obj, dash_obj_key, ls) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == dict: + for key, value in dash_obj.items(): + temp_is_updated = convert_dashboard(dash_obj, key, value) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == str: + # print(f'key: {dash_obj_key}, value: {dash_obj}') + is_matched = False + for old_value, new_value in RENAMED_MAP.items(): + if old_value in dash_obj: + dash_obj = dash_obj.replace(old_value, new_value) + is_matched = True + if is_matched: + dash_parent_obj[dash_obj_key] = dash_obj + is_updated = True + else: + pass # do nothing with other types: int, float, bool, None + return is_updated + + +def convert_file(file): + with open(file, "r") as dash_json: + dash_obj = json.load(dash_json) + is_updated = convert_dashboard(dash_obj, "root", dash_obj) + if is_updated: + if not os.path.exists(("{}/converted".format(os.path.dirname(file)))): + os.mkdir("{}/converted".format(os.path.dirname(file))) + filename = os.path.basename(file).replace(":", "").replace("___", "_").replace("__", "_") + with open(os.path.dirname(file) + "/" + CONVERTED_PATH_NAME + "/" + filename, "w") as f: + json.dump(dash_obj, f, indent=2) + print('Converted to file: {}'.format(f)) + + +def convert_folder(folder): + for root, dirs, files in os.walk(folder): + for file in files: + try: + if str(file).endswith(".json"): + convert_file(root + "/" + file) + except Exception as e: + print(f"Exception converting file {file}: {e}") + continue + for sub_dir in dirs: + convert_folder(sub_dir) + + +# Press the green button in the gutter to run the script. +if __name__ == "__main__": + convert_folder(sys.argv[1]) diff --git a/utilities/transform/dashboard_grafana_9_to_7.py b/utilities/transform/dashboard_grafana_9_to_7.py new file mode 100644 index 0000000..205cbcb --- /dev/null +++ b/utilities/transform/dashboard_grafana_9_to_7.py @@ -0,0 +1,98 @@ +import json +import os +import sys + +# The key of property to update value with +DS_KEY = "datasource" +DS_TYPE_KEY = "type" +DS_UID_KEY = "uid" +DS_GRAFANA_KEY = "grafana" +DS_GRAFANA_VALUE_7 = "-- Grafana --" +CONVERTED_PATH_NAME = "converted" + + +def convertDatasource(dash_obj, key, value): + """ + Update datasource element from Grafana 9 format to Grafana 7. + + :param dash_obj: the parent element that ows this datasource element to update with + :param key: the key of the datasource element ("datasource") + :param value: the value of the datasource element in Grafana 9 format: { "type": , "uid": } + + :return: is_updated: True if successfully replaced, False otherwise. + """ + if value[DS_UID_KEY]: # if (value[DS_TYPE_KEY] or value[DS_UID_KEY]): + # print('original: {}'.format(dash_obj[key])) + if value[DS_UID_KEY] == DS_GRAFANA_KEY: + dash_obj[key] = DS_GRAFANA_VALUE_7 + else: + dash_obj[key] = value[DS_UID_KEY] + # print('converted: {}'.format(dash_obj[key])) + return True + return False + + +def convert_dashboard(dash_obj): + """ + Depth First Search to find any "datasource" element and update its value format from a dict to a string value + as its UID value. + + Returns: + is_updated: if any node gets updated, it bubbles up to root so the dashboard json file will be rewritten. + """ + is_updated = False + if type(dash_obj) == list: + for ls in dash_obj: + temp_is_updated = convert_dashboard(ls) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == dict: + for key, value in dash_obj.items(): + try: + if key == DS_KEY: + temp_is_updated = convertDatasource(dash_obj, key, value) + else: + temp_is_updated = convert_dashboard(value) + if temp_is_updated: + is_updated = True + except TypeError: + # print('Type error: {}:{}'.format(key, value)) + pass + else: + pass # do nothing with other types: int, float, bool, None + return is_updated + + +def convert_file(file): + with open(file, "r") as dash_json: + dash_obj = json.load(dash_json) + is_converted = convert_dashboard(dash_obj) + + if not is_converted: + return + + # rewrite json file + if not os.path.exists(("{}/converted".format(os.path.dirname(file)))): + os.mkdir("{}/converted".format(os.path.dirname(file))) + filename = os.path.basename(file).replace(":", "").replace("___", "_").replace("__", "_") + with open(os.path.dirname(file) + "/" + CONVERTED_PATH_NAME + "/" + filename, "w") as f: + json.dump(dash_obj, f, indent=2) + print('Converted to file: {}'.format(f)) + + +def convert_folder(folder): + for root, dirs, files in os.walk(folder): + for file in files: + try: + if str(file).endswith(".json"): + convert_file(root + "/" + file) + except Exception as e: + print(f"Exception converting file {file}: {e}") + continue + for sub_dir in dirs: + convert_folder(sub_dir) + + +# Press the green button in the gutter to run the script. +if __name__ == "__main__": + convert_folder(sys.argv[1]) diff --git a/utilities/transform/dashboard_remove_default_values.py b/utilities/transform/dashboard_remove_default_values.py new file mode 100644 index 0000000..fcc2e17 --- /dev/null +++ b/utilities/transform/dashboard_remove_default_values.py @@ -0,0 +1,90 @@ +import json +import os +import sys +import traceback + +CONVERTED_PATH_NAME = "converted" +TYPE_KEY = "type" +NAME_KEY = "name" +CURRENT_KEY = "current" +DEFAULT_QUERY = { + "selected": False, + "text": "", + "value": "" +} +DEFAULT_RESOLUTION = { + "selected": True, + "text": "auto", + "value": "$__auto_interval_resolution" +} + + +# Depth First Search to find any matching measurement and back up the tree to update the nearest parent's "datasource" +# property to the matching split database +def convert_dashboard(dash_parent_obj, dash_key, dash_obj): + """ + Loop through template variables and update their default values. + + :param dash_parent_obj: The parent element that owns this element being inspected. + :param dash_key: The key of the element being inspected. + :param dash_obj: The value of the element being inspected. + + :return: + """ + is_updated: bool = False + if type(dash_obj) == list: + for ls in dash_obj: + temp_is_updated = convert_dashboard(dash_parent_obj, dash_key, ls) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == dict: + # for key, value in dash_obj.items(): + try: + if dash_obj[TYPE_KEY] == "query": + dash_obj[CURRENT_KEY] = DEFAULT_QUERY + is_updated = True + elif dash_obj[TYPE_KEY] == "interval" and dash_obj[NAME_KEY] == "resolution": + dash_obj[CURRENT_KEY] = DEFAULT_RESOLUTION + is_updated = True + else: + pass # do nothing about the rest + except TypeError: + print('Type error: {}:{}'.format(dash_key, dash_obj)) + else: + pass # do nothing + return is_updated + + +def convert_file(file): + with open(file, "r") as dash_json: + dash_obj = json.load(dash_json) + inspect_obj = dash_obj['templating']['list'] + inspect_key = "list" + is_updated = convert_dashboard(dash_obj['templating'], inspect_key, inspect_obj) + + if is_updated: + if not os.path.exists(("{}/converted".format(os.path.dirname(file)))): + os.mkdir("{}/converted".format(os.path.dirname(file))) + filename = os.path.basename(file).replace(":", "").replace("___", "_").replace("__", "_") + with open(os.path.dirname(file) + "/" + CONVERTED_PATH_NAME + "/" + filename, "w") as f: + json.dump(dash_obj, f, indent=2) + print('Converted to file: {}'.format(f)) + + +def convert_folder(folder): + for root, dirs, files in os.walk(folder): + for file in files: + try: + if str(file).endswith(".json"): + convert_file(root + "/" + file) + except Exception as e: + print(f"Exception converting file {file}: {e}") + traceback.print_exc() + continue + for sub_dir in dirs: + convert_folder(sub_dir) + + +# Press the green button in the gutter to run the script. +if __name__ == "__main__": + convert_folder(sys.argv[1]) diff --git a/utilities/transform/dashboard_sql_single_to_multi.py b/utilities/transform/dashboard_sql_single_to_multi.py new file mode 100644 index 0000000..d58579a --- /dev/null +++ b/utilities/transform/dashboard_sql_single_to_multi.py @@ -0,0 +1,78 @@ +import json +import os +import sys + +# The key of property to update value with +DS_KEY = "datasource" +DS_TYPE_KEY = "type" +DS_UID_KEY = "uid" +DS_GRAFANA_KEY = "grafana" +CONVERTED_PATH_NAME = "converted" + +# Depth First Search to find any matching measurement and back up the tree to update the nearest parent's "datasource" +# property to the matching split database +def convert_dashboard(dash_obj): + is_updated: bool = False + + if type(dash_obj) == list: + for ls in dash_obj: + temp_is_updated = convert_dashboard(ls) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == dict: + # depth first + for key, value in dash_obj.items(): + try: + if (key == DS_KEY): + if (value[DS_TYPE_KEY] and value[DS_UID_KEY]): + #print('original: {}'.format(dash_obj[key])) + if (value[DS_UID_KEY]==DS_GRAFANA_KEY): + dash_obj[key] = "-- Grafana --" + else: + dash_obj[key] = value[DS_UID_KEY] + #print('converted: {}'.format(dash_obj[key])) + is_updated = True + else: + temp_is_updated = convert_dashboard(value) + if temp_is_updated: + is_updated = True + except TypeError: + #print('Type error: {}:{}'.format(key, value)) + # do nothing + pass + else: + pass # do nothing with other types: int, float, bool, None + return is_updated + + +def convert_file(file): + is_updated = False + #print('file: {}'.format(file)) + with open(file, "r") as dash_json: + dash_obj = json.load(dash_json) + temp_is_updated = convert_dashboard(dash_obj) + if temp_is_updated: + is_updated = temp_is_updated + if is_updated: + if not os.path.exists(("{}/converted".format(os.path.dirname(file)))): + os.mkdir("{}/converted".format(os.path.dirname(file))) + #name process + filename = os.path.basename(file) + filename = filename.replace(":", "") + filename = filename.replace("___", "_") + filename = filename.replace("__", "_") + with open(os.path.dirname(file) + "/" + CONVERTED_PATH_NAME + "/" + filename, "w") as f: + json.dump(dash_obj, f, indent=2) + print('Write to file: {}'.format(f)) +def convert_folder(folder): + for root, dirs, files in os.walk(folder): + for file in files: + if str(file).endswith(".json"): + convert_file(root + "/" + file) + for sub_dir in dirs: + convert_folder(sub_dir) + + +# Press the green button in the gutter to run the script. +if __name__ == "__main__": + convert_folder(sys.argv[1]) diff --git a/utilities/transform/split_datasource.py b/utilities/transform/split_datasource.py new file mode 100644 index 0000000..7d18f75 --- /dev/null +++ b/utilities/transform/split_datasource.py @@ -0,0 +1,119 @@ +import json +import os +import sys + + +# The list of measurements that would map to the "flows +FLOW_MEASUREMENTS = ("flow_data_4_tuple", + "flows_summary_application_port", + "flows_summary_destination_ip", + "flows_summary_source_ip", + "flows_summary_vlan_tag_outer") +TCP_MEASUREMENTS = ("tcp_open_4_tuple", + "tcp_open_summary_application_port", + "tcp_open_summary_client_ip", + "tcp_open_summary_server_ip", + "tcp_open_summary_vlan_tag_outer", + "tcp_timeslice_4_tuple", + "tcp_timeslice_summary_application_port", + "tcp_timeslice_summary_client_ip", + "tcp_timeslice_summary_server_ip", + "tcp_timeslice_summary_vlan_tag_outer") +DS_MM_MAP = {"flows": FLOW_MEASUREMENTS, "tcp": TCP_MEASUREMENTS} +# The key of property to update value with +DS_KEY = "datasource" +# The datasource value to match/replace +DS_ORIGINAL = "indicators" +# The properties to search in json files to search and replace +SEARCH_PROPERTIES = ("annotations", "templating", "panels") +# The extension of the output file to rename to +OUTFILE_EXTENSION = "_converted.json" + + +# replace ds_ori with ds from ds_mm_map where any mm is matched +def find_match(dash_str): + for key, value in DS_MM_MAP.items(): + for mm in value: + if dash_str.count(mm) > 0: + return False, True, key + return False, False, "" + + +def replace(dash_obj, is_matched, ds_matched): + if dash_obj and is_matched: + if DS_KEY in dash_obj.keys() and dash_obj[DS_KEY] == DS_ORIGINAL: + dash_obj[DS_KEY] = ds_matched + return True, False, "" + return False, is_matched, ds_matched + + +def split_datasource_json(dash_obj): + """ + Depth First Search to find any matching measurement and back up the tree to update the nearest parent's "datasource" + property to the matching split database. + + Returns: + is_updated: if any node gets updated, it bubbles up to root so the dashboard json file will be rewritten + is_matched: notifies each node's parent if there's a match to replace. + ds_matched: the name of the matching datasource to replace with. + """ + is_updated: bool = False + is_matched: bool = False + ds_matched: str = "" + + if dash_obj: + if type(dash_obj) == list: + for ls in dash_obj: + temp_is_updated, is_matched, ds_matched = split_datasource_json(ls) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == dict: + # depth first + for key, value in dash_obj.items(): + temp_is_updated, temp_is_matched, temp_ds_matched = split_datasource_json(value) + if temp_is_updated: + is_updated = True + if temp_is_matched: + is_matched = True + ds_matched = temp_ds_matched + # update datasource + temp_is_updated, is_matched, ds_matched = replace(dash_obj, is_matched, ds_matched) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == str: + temp_is_updated, is_matched, ds_matched = find_match(dash_obj) + if temp_is_updated: + is_updated = True + else: + pass # do nothing with other types: int, float, bool, None + return is_updated, is_matched, ds_matched + + +def split_datasource_file(file): + is_updated = False + with open(file, "r") as dash_json: + dash_obj = json.load(dash_json) + for prop in SEARCH_PROPERTIES: + temp_is_updated, is_matched, ds_matched = split_datasource_json(dash_obj[prop]) + if temp_is_updated: + is_updated = True + if is_updated: + with open(str(file)[:file.index(".json")] + OUTFILE_EXTENSION, "w") as f: + json.dump(dash_obj, f, indent=4) + + +def split_datasource_folder(folder): + for root, dirs, files in os.walk(folder): + for file in files: + try: + if str(file).endswith(".json"): + split_datasource_file(root + "/" + file) + except Exception: + continue + for sub_dir in dirs: + split_datasource_folder(sub_dir) + + +# Press the green button in the gutter to run the script. +if __name__ == "__main__": + split_datasource_folder(sys.argv[1])