From a573f2b7adb4efdb25d486fcdf97435eb995013d Mon Sep 17 00:00:00 2001 From: Marina Zheng Date: Fri, 30 Sep 2022 12:08:49 -0700 Subject: [PATCH 1/6] Add dashboard transfer scripts - Add utility scripts to explort dashboards, datasources and preferences from a cclear into local files. - Add utility scripts to import dashboards, datasources and preferences to a cclear from local files. --- utilities/transfer/dashboard_export.sh | 169 +++++++++++++++++++++ utilities/transfer/dashboard_import.sh | 188 ++++++++++++++++++++++++ utilities/transfer/datasource_export.sh | 126 ++++++++++++++++ utilities/transfer/datasource_import.sh | 153 +++++++++++++++++++ utilities/transfer/preference_export.sh | 143 ++++++++++++++++++ utilities/transfer/preference_import.sh | 137 +++++++++++++++++ 6 files changed, 916 insertions(+) create mode 100755 utilities/transfer/dashboard_export.sh create mode 100755 utilities/transfer/dashboard_import.sh create mode 100755 utilities/transfer/datasource_export.sh create mode 100755 utilities/transfer/datasource_import.sh create mode 100755 utilities/transfer/preference_export.sh create mode 100755 utilities/transfer/preference_import.sh diff --git a/utilities/transfer/dashboard_export.sh b/utilities/transfer/dashboard_export.sh new file mode 100755 index 0000000..d48ae5a --- /dev/null +++ b/utilities/transfer/dashboard_export.sh @@ -0,0 +1,169 @@ +#!/bin/bash + +OPTSPEC=":hu:p:t:f:" + +show_help() { +cat << EOF +Usage: $0 [-u USER] [-p PASSWORD] [-f FROM_FOLDER] [-t TARGET_HOST] +Script to export grafana dashboards + -u Required. cClear user to login + -p Required. cClear user password to login + -t Required. The IP of the source cClear host i.e 10.51.10.32 + -f Optional. The name of the folder to export from, double quotes with spaces. Export all folders if not + specified. + -h Display this help and exit. +EOF +} + +###### Check script invocation options ###### +while getopts "$OPTSPEC" optchar; do + case "$optchar" in + h) + show_help + exit + ;; + u) + USER="$OPTARG";; + p) + PASSWORD="$OPTARG";; + t) + HOST="$OPTARG";; + f) + FROM="$OPTARG";; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$HOST" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|-------------------------------------------------------------------------|\n" + printf "|$1|\n"; + printf "|-------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + +mycookie="$PWD/mycookie" +counter=0 + +function init() { + DATE_TIME=$(date '+%d%m%Y_%H%M%S') + DASH_DIR="$PWD/dashboards_${HOST}_${DATE_TIME}" + if [ ! -d "${DASH_DIR}" ]; then + mkdir "${DASH_DIR}" + else + log_title "----------------- A $DASH_DIR directory already exists! -----------------" + fi +} + +init + +# host url +if [[ ! "$HOST" == "https://"* ]]; then + HOST="https://$HOST" +fi + +curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" "$HOST/sess/login?rp=/vb/" + +folder_json=$(curl --noproxy '*' -k -b "$mycookie" "$HOST/graph-engine/api/folders") +# From folder specified: +if [ ${#FROM} -gt 0 ]; then + # Find matching folder from remote (with folder title) + FOLDER_UID=$(echo $folder_json | jq -r '.[] | select(.title == "'"$FROM"'") | .uid') + # Folder not found, prompt error and get out + if [ -z "$FOLDER_UID" ] ; then + log_failure "Folder $FROM is not found. Please check spelling and double quote with any spaces." + exit 1 + fi + # Folder found: get the collection of dashboard uids in this folder + dashboard_uids=$(curl --noproxy '*' -k -b "$mycookie" "$HOST"/graph-engine/api/search\?query\=\& | \ + jq -r '.[] | select(.type | contains("dash-db")) | select(.folderUid != null) | select(.folderUid == "'"$FOLDER_UID"'") | .uid') +# From all folders: +else + dashboard_uids=$(curl --noproxy '*' -k -b "$mycookie" "$HOST"/graph-engine/api/search\?query\=\& | \ + jq -r '.[] | select(.type | contains("dash-db")) | .uid') +fi + +# Export dashboards +for dashboard_uid in $dashboard_uids; do + url=$(echo "$HOST/graph-engine/api/dashboards/uid/$dashboard_uid" | tr -d '\r') + dashboard_json=$(curl --noproxy '*' -k -b "$mycookie" "$url") + dashboard_title=$(echo "$dashboard_json" | jq -r '.dashboard | .title' | sed -r 's/[ \/]+/_/g' ) + dashboard_version=$(echo "$dashboard_json" | jq -r '.dashboard | .version') + dashboard_folder_raw=$(echo "$dashboard_json" | jq -r '.meta | .folderTitle') + dashboard_folder=$(echo "$dashboard_json" | jq -r '.meta | .folderTitle' | sed -r 's/[ \/]+/_/g' ) + dashboard_folderId=$(echo "$dashboard_json" | jq -r '.meta | .folderId') + + # Find folder uid to save (so that importing can find the right folder to import to) + folder_uid=$(echo "$folder_json" | jq -r '.[] | select(.id=='$dashboard_folderId') | .uid ') + + # create the folder if not existing + if [ ! -d "${DASH_DIR}/${dashboard_folder}" ]; then + mkdir "${DASH_DIR}/${dashboard_folder}" + fi + + counter=$((counter + 1)) + # save dashboard with meta, dashboard and folder uid. + echo "$dashboard_json" | jq '.dashboard | . += {"folderUid":"'$folder_uid'", "folderTitle": "'"$dashboard_folder_raw"'"}' > \ + "$DASH_DIR/${dashboard_folder}/${dashboard_title}_v${dashboard_version}.json" + log_success "Dashboard has been saved\t\t title=\"${dashboard_title}\", uid=\"${dashboard_uid}\", + path=\"${DASH_DIR}/${dashboard_folder}/${dashboard_title}_v${dashboard_version}.json\"." +done + +# zip -r -m ${DASH_DIR}.zip ${DASH_DIR} +rm mycookie + +log_title "${counter} dashboards were saved in ${DASH_DIR}"; +log_title "------------------------------ FINISHED ---------------------------------"; diff --git a/utilities/transfer/dashboard_import.sh b/utilities/transfer/dashboard_import.sh new file mode 100755 index 0000000..c18b03d --- /dev/null +++ b/utilities/transfer/dashboard_import.sh @@ -0,0 +1,188 @@ +#!/bin/bash +# todo: Import from dashboard files without .folderTitle...from git repo or manual exported +OPTSPEC=":hu:p:z:t:" + +show_help() { +cat << EOF +Usage: $0 [-u USER] [-p PASSWORD] [-z PATH] [-t TARGET_HOST] +Script to import dashboards into Grafana + -u Required. cClear user to login + -p Required. cClear user password to login + -z Required. Full path to the folder or zip file containing JSON exports of the dashboards + you want to be imported. + -t Required. The IP of the destination cClear host i.e 10.51.10.32 + -h Display this help and exit. +EOF +} + +###### Check script invocation options ###### +while getopts "$OPTSPEC" optchar; do + case "$optchar" in + h) + show_help + exit + ;; + u) + USER="$OPTARG";; + p) + PASSWORD="$OPTARG";; + z) + DASH_PATH="$OPTARG";; + t) + HOST="$OPTARG";; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$DASH_PATH" ] || [ -z "$HOST" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[%s] $1\n" "$timestamp" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[%s] $1\n" "$timestamp" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|---------------------------------------------------------------------------------------|\n" + printf "| %s |\n" "$1"; + printf "|---------------------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + + +ZIP_FILE=$(basename $DASH_PATH) + +if [[ $ZIP_FILE =~ \.zip$ ]]; then + DASH_DIR=$(unzip -qql $DASH_PATH | head -n1 | tr -s ' ' | cut -d' ' -f5-) + unzip $DASH_PATH + DIR_LENGTH=${#DASH_DIR} + DASH_DIR=${DASH_DIR:0:DIR_LENGTH-1} +else + DASH_DIR=$DASH_PATH +fi + +if [ -d "$DASH_DIR" ]; then + DASH_LIST=$(find "$PWD/$DASH_DIR" -mindepth 1 -name \*.json) + + if [ -z "$DASH_LIST" ]; then + log_title "----------------- $DASH_DIR contains no JSON files! -----------------" + log_failure "Directory $DASH_DIR does not appear to contain any JSON files for import. Check your path and try again." + exit 1 + else + FILESTOTAL=$(echo "$DASH_LIST" | wc -l) + log_title "----------------- Starting import of $FILESTOTAL dashboards -----------------" + fi +else + log_title "-------------------- $DASH_DIR directory not found! -----------------" + log_failure "Directory $DASH_DIR does not exist. Check your path and try again." + exit 1 +fi + +NUMSUCCESS=0 +NUMFAILURE=0 +COUNTER=0 + + +# host url +if [[ ! "$HOST" == "http"* ]]; then + HOST="https://$HOST" +fi + +mycookie="$PWD/mycookie" +curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/ + +for DASH_FILE in $DASH_LIST; do + COUNTER=$((COUNTER + 1)) + echo "Import $COUNTER/$FILESTOTAL: $DASH_FILE..." + + # Get folder uid and title from dashboard + dashboard=$(cat "$DASH_FILE") + folder_title=$(echo "$dashboard" | jq -r '.folderTitle') + folder_uid=$(echo "$dashboard" | jq -r '.folderUid') + dashboard=$(echo "$dashboard" | jq -r 'del(.folderTitle) | del(.folderUid)') + # shellcheck disable=SC2116 + dashboard=$(echo '{"dashboard": ' "${dashboard}"'}') + +# echo "$dashboard" + # If folder uid id not provided, import to the "General" folder + if [ ${#folder_uid} -eq 0 ]; then + RESULT=$(echo "$dashboard" | jq -r '. * {overwrite: true, dashboard: {id: null}}' | curl -k -b $mycookie -X POST + -H \ + "Content-Type: application/json" $HOST/graph-engine/api/dashboards/db -d @-) + else + # Find the folder id from $HOST with this folder uid + folder_id=$(curl --noproxy '*' -k -b "$mycookie" "$HOST"/graph-engine/api/folders/$folder_uid | jq -r '.id') + # If not found, create a folder with this folder uid and folder title. + if [ "$folder_id" == "null" ]; then + folder_new=$(echo '{"uid": "'$folder_uid'", "title": "'"$folder_title"'"}' | curl --noproxy '*' -k -b \ + $mycookie -X POST -H "Content-Type: application/json" $HOST/graph-engine/api/folders -d @-) + folder_id=$(echo $folder_new | jq -r '.id') + fi + + # Import dashboard with folder id, uid, and title + RESULT=$(echo "$dashboard" | jq -r '. * {overwrite: true, dashboard: {id: null}} | . += {folderId:'$folder_id', + folderUid: + "'$folder_uid'", folderTitle: "'"$folder_title"'"}' \ + | curl --noproxy '*' -k -b $mycookie -X POST -H "Content-Type: application/json" $HOST/graph-engine/api/dashboards/db -d @-) + fi + + # log result + if [[ "$RESULT" == *"success"* ]]; then + log_success "$RESULT" + NUMSUCCESS=$((NUMSUCCESS + 1)) + else + log_failure "$RESULT" + NUMFAILURE=$((NUMFAILURE + 1)) + fi +done + +rm mycookie + +log_title "Import complete. $NUMSUCCESS dashboards were successfully imported. $NUMFAILURE dashboard imports failed."; +log_title "-------------------------------------FINISHED----------------------------------------"; diff --git a/utilities/transfer/datasource_export.sh b/utilities/transfer/datasource_export.sh new file mode 100755 index 0000000..36cdfdf --- /dev/null +++ b/utilities/transfer/datasource_export.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +OPTSPEC=":hu:p:t:" + +show_help() { +cat << EOF +Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST] +Script to export grafana datasources + -u Required. cClear user to login + -p Required. cClear user password to login + -t Required. The IP of the target host i.e 10.51.10.32 + -h Display this help and exit. +EOF +} + +###### Check script invocation options ###### +while getopts "$OPTSPEC" optchar; do + case "$optchar" in + h) + show_help + exit + ;; + u) + USER="$OPTARG";; + p) + PASSWORD="$OPTARG";; + t) + HOST="$OPTARG";; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$HOST" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|-------------------------------------------------------------------------|\n" + printf "|$1|\n"; + printf "|-------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + +mycookie="$PWD/mycookie" +counter=0 + +function init() { + DS_DIR="$PWD/datasources" + + if [ ! -d "${DS_DIR}" ]; then + mkdir "${DS_DIR}" + else + log_title "----------------- A $DS_DIR directory already exists! -----------------" + fi +} + +init + +# host url +if [[ ! "$HOST" == "https://"* ]]; then + HOST="https://$HOST" +fi + +curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" "$HOST/sess/login?rp=/vb/" +datasource_json=$(curl --noproxy '*' -k -b $mycookie "$HOST/graph-engine/api/datasources") +for id in $(echo $datasource_json | jq -r '.[] | .id'); do + counter=$((counter + 1)) + curl --noproxy '*' -f -k -b $mycookie "$HOST/graph-engine/api/datasources/${id}" | jq '' > "$DS_DIR/${id}.json" + log_success "Datasource has been saved\t id=\"${id}\", path=\"${DS_DIR}/${id}.json\"." +done + +zip -r -m datasources.zip datasources +rm mycookie + +log_title "${counter} datasource(s) were saved and zipped in $PWD/datasources.zip"; +log_title "------------------------------ FINISHED ---------------------------------"; diff --git a/utilities/transfer/datasource_import.sh b/utilities/transfer/datasource_import.sh new file mode 100755 index 0000000..bb77951 --- /dev/null +++ b/utilities/transfer/datasource_import.sh @@ -0,0 +1,153 @@ +#!/bin/bash -x + +OPTSPEC=":hu:p:z:t:" + +show_help() { +cat << EOF +Usage: $0 [-u USER] [-p PASSWORD] [-z PATH] [-t TARGET_HOST] +Script to import datasource into Grafana + -u Required. cClear user to login + -p Required. cClear user password to login + -z Required. Full path to the zip file containing JSON exports of the datasource you want to be imported. + -t Required. The IP of the target host i.e 10.51.10.32 + -h Display this help and exit. +EOF +} + +###### Check script invocation options ###### +while getopts "$OPTSPEC" optchar; do + case "$optchar" in + h) + show_help + exit + ;; + u) + USER="$OPTARG";; + p) + PASSWORD="$OPTARG";; + z) + DS_PATH="$OPTARG";; + t) + HOST="$OPTARG";; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$DS_PATH" ] || [ -z "$HOST" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[%s] $1\n" "$timestamp" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[%s] $1\n" "$timestamp" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|-----------------------------------------------------------------------------------------|\n" + printf "| %s |\n" "$1"; + printf "|-----------------------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + +ZIP_FILE=$(basename $DS_PATH) +echo $ZIP_FILE + +if [[ $ZIP_FILE =~ \.zip$ ]]; then + DS_DIR=$(unzip -qql $DS_PATH | head -n1 | tr -s ' ' | cut -d' ' -f5-) + unzip $DS_PATH + DS_DIR=${DS_DIR: : -1} + if [ -d "$DS_DIR" ]; then + DS_LIST=$(find "$PWD/$DS_DIR" -mindepth 1 -name \*.json) + echo $DS_LIST + if [ -z "$DS_LIST" ]; then + log_title "----------------- $DS_DIR contains no JSON files! -----------------" + log_failure "Directory $DS_DIR does not appear to contain any JSON files for import. Check your path and try again." + exit 1 + else + FILESTOTAL=$(echo "$DS_LIST" | wc -l) + log_title "----------------- Starting import of $FILESTOTAL datasource(s) -----------------" + fi + else + log_title "-------------------- $DS_DIR directory not found! -----------------" + log_failure "Directory $DS_DIR does not exist. Check your path and try again." + exit 1 + fi +else + log_title "-------------------- $ZIP_FILE Wrong format! -----------------" + log_failure "$ZIP_FILE is not a zip file. Please enter a correct file" +fi + +NUMSUCCESS=0 +NUMFAILURE=0 +COUNTER=0 + + +# host url +if [[ ! "$HOST" == "https://"* ]]; then + HOST="https://$HOST" +fi + +mycookie="$PWD/mycookie" +curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" "$HOST/sess/login?rp=/vb/" + +for i in datasources/*; do + RESULT=$(curl --noproxy '*' -k -b $mycookie -X "POST" "$HOST/graph-engine/api/datasources" \ + -H "Content-Type: application/json" --data-binary @$i) + if [[ "$RESULT" == *"Datasource added"* ]]; then + log_success "$RESULT" + NUMSUCCESS=$((NUMSUCCESS + 1)) + else + log_failure "$RESULT" + NUMFAILURE=$((NUMFAILURE + 1)) + fi +done + +rm mycookie + +log_title "Import complete. $NUMSUCCESS datasource(s) successfully imported. $NUMFAILURE datasource(s) imports failed."; +log_title "---------------------------------------FINISHED----------------------------------------"; diff --git a/utilities/transfer/preference_export.sh b/utilities/transfer/preference_export.sh new file mode 100755 index 0000000..5f8fbc2 --- /dev/null +++ b/utilities/transfer/preference_export.sh @@ -0,0 +1,143 @@ +#!/bin/bash +# +# +# + +OPTSPEC=":hu:p:t:f:" + +show_help() { +cat << EOF +Usage: $0 [-u USER] [-p PASSWORD] [-f FROM_FOLDER] [-t TARGET_HOST] +Script to export grafana dashboards + -u Required. cClear user to login + -p Required. cClear user password to login + -t Required. The IP of the source cClear host i.e 10.51.10.32 + -h Display this help and exit. +EOF +} + +###### Check script invocation options ###### +while getopts "$OPTSPEC" optchar; do + case "$optchar" in + h) + show_help + exit + ;; + u) + USER="$OPTARG";; + p) + PASSWORD="$OPTARG";; + t) + HOST="$OPTARG";; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$HOST" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|-------------------------------------------------------------------------|\n" + printf "|$1|\n"; + printf "|-------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + +function init() { + DATE_TIME=$(date '+%d%m%Y_%H%M%S') + DASH_DIR="$PWD/preferences_${HOST}_${DATE_TIME}" + if [ ! -d "${DASH_DIR}" ]; then + mkdir "${DASH_DIR}" + else + log_title "----------------- A $DASH_DIR directory already exists! -----------------" + fi +} + +init + +PREF_ORG="$DASH_DIR/preferences_org.json" +PREF_USER="$DASH_DIR/preferences_user.json" + +# host url +if [[ ! "$HOST" == "https://"* ]]; then + HOST="https://$HOST" +fi + +mycookie="$PWD/mycookie" +curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" "$HOST/sess/login?rp=/vb/" + +# Get preferences +pref_org_json=$(curl --noproxy '*' -k -b "$mycookie" "$HOST/graph-engine/api/org/preferences") +pref_user_json=$(curl --noproxy '*' -k -b "$mycookie" "$HOST/graph-engine/api/user/preferences") + +rm mycookie + +# log result +if [ -z "$pref_org_json" ] || [ -z "$pref_user_json" ]; then + log_failure "Failed to download preferences. Please check parameters passed in. " + show_help + exit 1 +elif [[ "$pref_org_json" == *"Unauthorized"* ]] || [[ "$pref_user_json" == *"Unauthorized"* ]]; then + log_failure "Failed to login: $pref_org_json; $pref_user_json" + exit 1 +fi + +# Save preferences +echo $pref_org_json | jq '.' > "$PREF_ORG" +log_success "Org. preferences saved: $pref_org_json" +echo $pref_user_json | jq '.' > "$PREF_USER" +log_success "User preferences saved: $pref_user_json" + +log_title "Preferences were saved in $DASH_DIR"; +log_title "------------------------------ FINISHED ---------------------------------"; diff --git a/utilities/transfer/preference_import.sh b/utilities/transfer/preference_import.sh new file mode 100755 index 0000000..61b79bd --- /dev/null +++ b/utilities/transfer/preference_import.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# +# todo: ip to both ip and https; from folder to be both zipped or unzipped +# +OPTSPEC=":hu:p:z:t:" + +show_help() { +cat << EOF +Usage: $0 [-u USER] [-p PASSWORD] [-z PATH] [-t TARGET_HOST] +Script to import dashboards into Grafana + -u Required. cClear user to login + -p Required. cClear user password to login + -z Required. Grafana preferences json file to import from. e.g. preferences.json + -t Required. The IP of the destination cClear host i.e 10.51.10.32 + -h Display this help and exit. +EOF +} + +###### Check script invocation options ###### +while getopts "$OPTSPEC" optchar; do + case "$optchar" in + h) + show_help + exit + ;; + u) + USER="$OPTARG";; + p) + PASSWORD="$OPTARG";; + z) + PREF_PATH="$OPTARG";; + t) + HOST="$OPTARG";; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$PREF_PATH" ] || [ -z "$HOST" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[%s] $1\n" "$timestamp" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[%s] $1\n" "$timestamp" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|---------------------------------------------------------------------------------------|\n" + printf "| %s |\n" "$1"; + printf "|---------------------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + +PREF_FILE=$(basename $PREF_PATH) + +if [[ ! $PREF_FILE =~ \.json$ ]]; then + log_title "-------------------- $PREF_FILE Wrong format! -----------------" + log_failure "$PREF_PATH is not a json file. Please enter a correct file" + exit 1 +fi + +if [[ ! -f "$PREF_FILE" ]]; then + log_failure "No such file: $PREF_FILE." + exit 1 +fi + +NUMSUCCESS=0 +NUMFAILURE=0 +COUNTER=0 + + +# host url +if [[ ! "$HOST" == "https://"* ]]; then + HOST="https://$HOST" +fi + +mycookie="$PWD/mycookie" +curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/ + +RESULT=$(cat "$PREF_FILE" | jq '.' | curl --noproxy '*' -k -b $mycookie -X PUT -H \ +"Content-Type: application/json" "$HOST/graph-engine/api/user/preferences" -d @-) + +rm mycookie + +# log result +if [[ "$RESULT" == *"updated"* ]]; then + log_success "$RESULT" +else + log_failure "$RESULT" +fi + +log_title "-------------------- Preferences were successfully imported.-------------------------" From 379c05a6220c04d74d8e2db2c131de111662ca0f Mon Sep 17 00:00:00 2001 From: Marina Zheng Date: Fri, 30 Sep 2022 12:12:32 -0700 Subject: [PATCH 2/6] Add a util to convert dashboards using multiple datasource. - Add script take in a folder of dashboards - Convert each dashboard from using one datasource to using multiple - The appropriate datasource to used is based on the name of the measurements in the queries. --- utilities/transform/split_datasource.py | 100 ++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 utilities/transform/split_datasource.py diff --git a/utilities/transform/split_datasource.py b/utilities/transform/split_datasource.py new file mode 100644 index 0000000..4e44540 --- /dev/null +++ b/utilities/transform/split_datasource.py @@ -0,0 +1,100 @@ +import json +import os +import sys + +# The list of measurements that would map to the "flows +FLOW_MEASUREMENTS = ("flow_data_4_tuple", + "flows_summary_application_port", + "flows_summary_destination_ip", + "flows_summary_source_ip", + "flows_summary_vlan_tag_outer") +TCP_MEASUREMENTS = ("tcp_open_4_tuple", + "tcp_open_summary_application_port", + "tcp_open_summary_client_ip", + "tcp_open_summary_server_ip", + "tcp_open_summary_vlan_tag_outer", + "tcp_timeslice_4_tuple", + "tcp_timeslice_summary_application_port", + "tcp_timeslice_summary_client_ip", + "tcp_timeslice_summary_server_ip", + "tcp_timeslice_summary_vlan_tag_outer") +DS_MM_MAP = {"flows": FLOW_MEASUREMENTS, "tcp": TCP_MEASUREMENTS} +# The key of property to update value with +DS_KEY = "datasource" +# The datasource value to match/replace +DS_ORIGINAL = "indicators" +# The properties to search in json files to search and replace +SEARCH_PROPERTIES = ("annotations", "templating", "panels") +# The extension of the output file to rename to +OUTFILE_EXTENSION = "_converted.json" + + +# replace ds_ori with ds from ds_mm_map where any mm is matched +def find_match(dash_str): + for key, value in DS_MM_MAP.items(): + for mm in value: + if dash_str.count(mm) > 0: + return False, True, key + return False, False, "" + + +# Depth First Search to find any matching measurement and back up the tree to update the nearest parent's "datasource" +# property to the matching split database +def split_datasource_json(dash_obj): + is_updated: bool = False + is_matched: bool = False + ds_matched: str = "" + + if type(dash_obj) == list: + for ls in dash_obj: + temp_is_updated, is_matched, ds_matched = split_datasource_json(ls) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == dict: + # depth first + for key, value in dash_obj.items(): + temp_is_updated, temp_is_matched, temp_ds_matched = split_datasource_json(value) + if temp_is_updated: + is_updated = True + if temp_is_matched: + is_matched = True + ds_matched = temp_ds_matched + # update datasource + if is_matched and DS_KEY in dash_obj.keys() and dash_obj[DS_KEY] == DS_ORIGINAL: + dash_obj[DS_KEY] = ds_matched + is_matched = False + ds_matched = "" + is_updated = True + elif type(dash_obj) == str: + return find_match(dash_obj) + else: + pass # do nothing with other types: int, float, bool, None + return is_updated, is_matched, ds_matched + + +def split_datasource_file(file): + is_updated = False + print(file) + with open(file, "r") as dash_json: + dash_obj = json.load(dash_json) + for prop in SEARCH_PROPERTIES: + temp_is_updated, temp_is_matched, temp_ds_matched = split_datasource_json(dash_obj[prop]) + if temp_is_updated: + is_updated = temp_is_updated + if is_updated: + with open(str(file)[:file.index(".json")] + OUTFILE_EXTENSION, "w") as f: + json.dump(dash_obj, f, indent=4) + + +def split_datasource_folder(folder): + for root, dirs, files in os.walk(folder): + for file in files: + if str(file).endswith(".json"): + split_datasource_file(root + "/" + file) + for sub_dir in dirs: + split_datasource_folder(sub_dir) + + +# Press the green button in the gutter to run the script. +if __name__ == "__main__": + split_datasource_folder(sys.argv[1]) From 636fb08c91aaeb4809143109a76ef1977269ed36 Mon Sep 17 00:00:00 2001 From: Marina Zheng Date: Tue, 27 Jun 2023 07:09:47 -0700 Subject: [PATCH 3/6] Add uitility scripts --- utilities/.DS_Store | Bin 0 -> 6148 bytes utilities/transfer/dashboard_export.sh | 6 +- .../transfer/dashboard_export_basic_auth.sh | 164 ++++++++++++++++++ .../transfer/dashboard_export_updated.sh | 126 ++++++++++++++ .../transfer/dashboard_import_updated.sh | 153 ++++++++++++++++ .../transform/dashboard_field_renamed.py | 66 +++++++ .../transform/dashboard_grafana_9_to_7.py | 78 +++++++++ .../dashboard_remove_default_values.py | 70 ++++++++ .../dashboard_sql_single_to_multi.py | 78 +++++++++ utilities/transform/dashboard_transform.py | 1 + utilities/transform/split_datasource.py | 1 + 11 files changed, 740 insertions(+), 3 deletions(-) create mode 100644 utilities/.DS_Store create mode 100644 utilities/transfer/dashboard_export_basic_auth.sh create mode 100755 utilities/transfer/dashboard_export_updated.sh create mode 100644 utilities/transfer/dashboard_import_updated.sh create mode 100644 utilities/transform/dashboard_field_renamed.py create mode 100644 utilities/transform/dashboard_grafana_9_to_7.py create mode 100644 utilities/transform/dashboard_remove_default_values.py create mode 100644 utilities/transform/dashboard_sql_single_to_multi.py create mode 100644 utilities/transform/dashboard_transform.py diff --git a/utilities/.DS_Store b/utilities/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..7407ad5d6d551987d2cd6a73d09d1b5a1d1dab2e GIT binary patch literal 6148 zcmeHKy-ve05I&a-DJ&gXsJy|N1u=#yJV9Qdq~T{saMgmuz~EQlo#;ETvGE*yXIp8b z7{r7sbSIrZ=X_^hK1X(rh}7(<&WI*NltLLtV>A)ran^y1f`#Lhn`2G4bVIA=E-MG# zV%SFpc<)krq!nG$1MTeJ!n8}5aB?z@`Lw1es>W12c7A%cnI3K~<9_w8{Fe7wxn0v@ zDjN6lGRU#5*NdhyrorbH8|DwMn}jZ2<6gU|e?GtH_m4MDJ+fSDw#onO*-M}1m`+3( z5C(*S{bhg;QAm2^QCeX@7!U?J2Kao?P{ur<_2`ZcG_C{yCNMiepKA$@(E;;-)+0tB z%7+4ds0t;9^5L-S;Fkxq9(_2eP<*JcvI-T7a;sy0?c7P_9;Foqgn>Q-2mW!w_y5)R z^?yG|o`eBm;9oJIlA_X6SS229~Dn5offn9S4m&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$HOST" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|-------------------------------------------------------------------------|\n" + printf "|$1|\n"; + printf "|-------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + +counter=0 + +function init() { + DATE_TIME=$(date '+%d%m%Y_%H%M%S') + DASH_DIR="$PWD/dashboards_${HOST}_${DATE_TIME}" + if [ ! -d "${DASH_DIR}" ]; then + mkdir "${DASH_DIR}" + else + log_title "----------------- A $DASH_DIR directory already exists! -----------------" + fi +} + +init + +# host url +if [[ ! "$HOST" == "https://"* ]]; then + HOST="https://$USER:$PASSWORD@$HOST" +fi + +folder_json=$(curl --noproxy '*' -k --request "GET" -H "Content-Type:application/json" "$HOST/graph-engine/api/folders") +# From folder specified: +if [ ${#FROM} -gt 0 ]; then + # Find matching folder from remote (with folder title) + FOLDER_UID=$(echo $folder_json | jq -r '.[] | select(.title == "'"$FROM"'") | .uid') + # Folder not found, prompt error and get out + if [ -z "$FOLDER_UID" ] ; then + log_failure "Folder $FROM is not found. Please check spelling and double quote with any spaces." + exit 1 + fi + # Folder found: get the collection of dashboard uids in this folder + dashboard_uids=$(curl --noproxy '*' -k "$HOST"/graph-engine/api/search\?query\=\& | \ + jq -r '.[] | select(.type | contains("dash-db")) | select(.folderUid != null) | select(.folderUid == "'"$FOLDER_UID"'") | .uid') +# From all folders: +else + dashboard_uids=$(curl --noproxy '*' -k "$HOST"/graph-engine/api/search\?query\=\& | \ + jq -r '.[] | select(.type | contains("dash-db")) | .uid') +fi + +# Export dashboards +for dashboard_uid in $dashboard_uids; do + url=$(echo "$HOST/graph-engine/api/dashboards/uid/$dashboard_uid" | tr -d '\r') + dashboard_json=$(curl --noproxy '*' -k "$url") + dashboard_title=$(echo "$dashboard_json" | jq -r '.dashboard | .title' | sed -r 's/[ \/]+/_/g' ) + dashboard_file=$(echo "$dashboard_title" | tr '[:upper:]' '[:lower:]') + dashboard_version=$(echo "$dashboard_json" | jq -r '.dashboard | .version') + dashboard_folder_raw=$(echo "$dashboard_json" | jq -r '.meta | .folderTitle') + dashboard_folder=$(echo "$dashboard_json" | jq -r '.meta | .folderTitle' | sed -r 's/[ \/]+/_/g' ) + dashboard_folderId=$(echo "$dashboard_json" | jq -r '.meta | .folderId') + + # Find folder uid to save (so that importing can find the right folder to import to) + folder_uid=$(echo "$folder_json" | jq -r '.[] | select(.id=='$dashboard_folderId') | .uid ') + + # create the folder if not existing + if [ ! -d "${DASH_DIR}/${dashboard_folder}" ]; then + mkdir "${DASH_DIR}/${dashboard_folder}" + fi + + counter=$((counter + 1)) + # save dashboard with meta, dashboard and folder uid. + echo "$dashboard_json" | jq '.dashboard | . += {"folderUid":"'$folder_uid'", "folderTitle": "'"$dashboard_folder_raw"'"}' > \ + "$DASH_DIR/${dashboard_folder}/${dashboard_file}_v${dashboard_version}.json" + log_success "Dashboard has been saved\t\t title=\"${dashboard_file}\", uid=\"${dashboard_uid}\", + path=\"${DASH_DIR}/${dashboard_folder}/${dashboard_file}_v${dashboard_version}.json\"." +done + +log_title "${counter} dashboards were saved in ${DASH_DIR}"; +log_title "------------------------------ FINISHED ---------------------------------"; diff --git a/utilities/transfer/dashboard_export_updated.sh b/utilities/transfer/dashboard_export_updated.sh new file mode 100755 index 0000000..a08adbd --- /dev/null +++ b/utilities/transfer/dashboard_export_updated.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +OPTSPEC=":hu:p:t:" + +show_help() { +cat << EOF +Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST] +Script to export grafana datasources + -u Required. cClear user to login + -p Required. cClear user password to login + -t Required. The IP of the target host i.e 10.51.10.32 + -h Display this help and exit. +EOF +} + +###### Check script invocation options ###### +while getopts "$OPTSPEC" optchar; do + case "$optchar" in + h) + show_help + exit + ;; + u) + USER="$OPTARG";; + p) + PASSWORD="$OPTARG";; + t) + HOST="$OPTARG";; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$HOST" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[${timestamp}] $1\n" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|-------------------------------------------------------------------------|\n" + printf "|$1|\n"; + printf "|-------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + +mycookie="$PWD/mycookie" +counter=0 + +function init() { + DS_DIR="$PWD/datasources" + + if [ ! -d "${DS_DIR}" ]; then + mkdir "${DS_DIR}" + else + log_title "----------------- A $DS_DIR directory already exists! -----------------" + fi +} + +init + +# host url +if [[ ! "$HOST" == "https://"* ]]; then + HOST="https://$HOST" +fi + +curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" "$HOST/sess/login?rp=/vb/" +datasource_json=$(curl --noproxy '*' -k -b $mycookie "$HOST/graph-engine/api/datasources") +for id in $(echo $datasource_json | jq -r '.[] | .id'); do + counter=$((counter + 1)) + curl --noproxy '*' -f -k -b $mycookie "$HOST/graph-engine/api/datasources/${id}" | jq '' > "$DS_DIR/${id}.json" + log_success "Datasource has been saved\t id=\"${id}\", path=\"${DS_DIR}/${id}.json\"." +done + +zip -r -m datasources.zip datasources +rm mycookie + +log_title "${counter} datasource(s) were saved and zipped in $PWD/datasources.zip"; +log_title "------------------------------ FINISHED ---------------------------------"; diff --git a/utilities/transfer/dashboard_import_updated.sh b/utilities/transfer/dashboard_import_updated.sh new file mode 100644 index 0000000..9fae8ca --- /dev/null +++ b/utilities/transfer/dashboard_import_updated.sh @@ -0,0 +1,153 @@ +#!/bin/bash -x + +OPTSPEC=":hu:p:z:t:" + +show_help() { +cat << EOF +Usage: $0 [-u USER] [-p PASSWORD] [-z PATH] [-t TARGET_HOST] +Script to import datasource into Grafana + -u Required. cClear user to login + -p Required. cClear user password to login + -z Required. Full path to the zip file containing JSON exports of the datasource you want to be imported. + -t Required. The IP of the target host i.e 10.51.10.32 + -h Display this help and exit. +EOF +} + +###### Check script invocation options ###### +while getopts "$OPTSPEC" optchar; do + case "$optchar" in + h) + show_help + exit + ;; + u) + USER="$OPTARG";; + p) + PASSWORD="$OPTARG";; + z) + DS_PATH="$OPTARG";; + t) + HOST="$OPTARG";; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$DS_PATH" ] || [ -z "$HOST" ]; then + show_help + exit 1 +fi + +# set some colors for status OK, FAIL and titles +SETCOLOR_SUCCESS="echo -en \\033[0;32m" +SETCOLOR_FAILURE="echo -en \\033[1;31m" +SETCOLOR_NORMAL="echo -en \\033[0;39m" +SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple + +# usage log "string to log" "color option" +function log_success() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_SUCCESS} + printf "[%s] $1\n" "$timestamp" + ${SETCOLOR_NORMAL} +} + +function log_failure() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + echo "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") + + ${SETCOLOR_FAILURE} + printf "[%s] $1\n" "$timestamp" + ${SETCOLOR_NORMAL} +} + +function log_title() { + if [ $# -lt 1 ]; then + ${SETCOLOR_FAILURE} + log_failure "Not enough arguments for log function! Expecting 1 argument got $#" + exit 1 + fi + + ${SETCOLOR_TITLE_PURPLE} + printf "|-----------------------------------------------------------------------------------------|\n" + printf "| %s |\n" "$1"; + printf "|-----------------------------------------------------------------------------------------|\n" + ${SETCOLOR_NORMAL} +} + +ZIP_FILE=$(basename $DS_PATH) +echo $ZIP_FILE + +if [[ $ZIP_FILE =~ \.zip$ ]]; then + DS_DIR=$(unzip -qql $DS_PATH | head -n1 | tr -s ' ' | cut -d' ' -f5-) + unzip $DS_PATH + DS_DIR=${DS_DIR: : -1} + if [ -d "$DS_DIR" ]; then + DS_LIST=$(find "$PWD/$DS_DIR" -mindepth 1 -name \*.json) + echo $DS_LIST + if [ -z "$DS_LIST" ]; then + log_title "----------------- $DS_DIR contains no JSON files! -----------------" + log_failure "Directory $DS_DIR does not appear to contain any JSON files for import. Check your path and try again." + exit 1 + else + FILESTOTAL=$(echo "$DS_LIST" | wc -l) + log_title "----------------- Starting import of $FILESTOTAL datasource(s) -----------------" + fi + else + log_title "-------------------- $DS_DIR directory not found! -----------------" + log_failure "Directory $DS_DIR does not exist. Check your path and try again." + exit 1 + fi +else + log_title "-------------------- $ZIP_FILE Wrong format! -----------------" + log_failure "$ZIP_FILE is not a zip file. Please enter a correct file" +fi + +NUMSUCCESS=0 +NUMFAILURE=0 +COUNTER=0 + + +# host url +if [[ ! "$HOST" == "https://"* ]]; then + HOST="https://$HOST" +fi + +mycookie="$PWD/mycookie" +curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" "$HOST/sess/login?rp=/vb/" + +for i in datasources/*; do + RESULT=$(curl --noproxy '*' -k -b $mycookie -X "POST" "$HOST/graph-engine/api/datasources" \ + -H "Content-Type: application/json" --data-binary @$i) + if [[ "$RESULT" == *"Datasource added"* ]]; then + log_success "$RESULT" + NUMSUCCESS=$((NUMSUCCESS + 1)) + else + log_failure "$RESULT" + NUMFAILURE=$((NUMFAILURE + 1)) + fi +done + +rm mycookie + +log_title "Import complete. $NUMSUCCESS datasource(s) successfully imported. $NUMFAILURE datasource(s) imports failed."; +log_title "---------------------------------------FINISHED----------------------------------------"; diff --git a/utilities/transform/dashboard_field_renamed.py b/utilities/transform/dashboard_field_renamed.py new file mode 100644 index 0000000..1bc9de2 --- /dev/null +++ b/utilities/transform/dashboard_field_renamed.py @@ -0,0 +1,66 @@ +import json +import os +import sys + +# The key of property to update value with +CONVERTED_PATH_NAME = "converted" +RENAMED_MAP = {"bytes_per_s":"bytes", + "fragments_per_s":"fragments", + "packets_per_s":"packets", + "resolution_s":"resolution"}; +# Depth First Search to find any matching measurement and back up the tree to update the nearest parent's "datasource" +# property to the matching split database +def convert_dashboard(dash_obj): + is_updated: bool = False + + if type(dash_obj) == list: + for ls in dash_obj: + temp_is_updated = convert_dashboard(ls) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == dict: + # depth first + for key, value in dash_obj.items(): + temp_is_updated = convert_dashboard(value) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == str: + #todo: rename + RENAMED_MAP. + pass + else: + pass # do nothing with other types: int, float, bool, None + return is_updated + + +def convert_file(file): + is_updated = False + #print('file: {}'.format(file)) + with open(file, "r") as dash_json: + dash_obj = json.load(dash_json) + temp_is_updated = convert_dashboard(dash_obj) + if temp_is_updated: + is_updated = temp_is_updated + if is_updated: + if not os.path.exists(("{}/converted".format(os.path.dirname(file)))): + os.mkdir("{}/converted".format(os.path.dirname(file))) + #name process + filename = os.path.basename(file) + filename = filename.replace(":", "") + filename = filename.replace("___", "_") + filename = filename.replace("__", "_") + with open(os.path.dirname(file) + "/" + CONVERTED_PATH_NAME + "/" + filename, "w") as f: + json.dump(dash_obj, f, indent=2) + print('Write to file: {}'.format(f)) +def convert_folder(folder): + for root, dirs, files in os.walk(folder): + for file in files: + if str(file).endswith(".json"): + convert_file(root + "/" + file) + for sub_dir in dirs: + convert_folder(sub_dir) + + +# Press the green button in the gutter to run the script. +if __name__ == "__main__": + convert_folder(sys.argv[1]) diff --git a/utilities/transform/dashboard_grafana_9_to_7.py b/utilities/transform/dashboard_grafana_9_to_7.py new file mode 100644 index 0000000..17641c6 --- /dev/null +++ b/utilities/transform/dashboard_grafana_9_to_7.py @@ -0,0 +1,78 @@ +import json +import os +import sys + +# The key of property to update value with +DS_KEY = "datasource" +DS_TYPE_KEY = "type" +DS_UID_KEY = "uid" +DS_GRAFANA_KEY = "grafana" +CONVERTED_PATH_NAME = "converted" + +# Depth First Search to find any matching measurement and back up the tree to update the nearest parent's "datasource" +# property to the matching split database +def convert_dashboard(dash_obj): + is_updated: bool = False + + if type(dash_obj) == list: + for ls in dash_obj: + temp_is_updated = convert_dashboard(ls) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == dict: + # depth first + for key, value in dash_obj.items(): + try: + if (key == DS_KEY): + if (value[DS_UID_KEY]): # if (value[DS_TYPE_KEY] or value[DS_UID_KEY]): + #print('original: {}'.format(dash_obj[key])) + if (value[DS_UID_KEY]==DS_GRAFANA_KEY): + dash_obj[key] = "-- Grafana --" + else: + dash_obj[key] = value[DS_UID_KEY] + #print('converted: {}'.format(dash_obj[key])) + is_updated = True + else: + temp_is_updated = convert_dashboard(value) + if temp_is_updated: + is_updated = True + except TypeError: + #print('Type error: {}:{}'.format(key, value)) + # do nothing + pass + else: + pass # do nothing with other types: int, float, bool, None + return is_updated + + +def convert_file(file): + is_updated = False + #print('file: {}'.format(file)) + with open(file, "r") as dash_json: + dash_obj = json.load(dash_json) + temp_is_updated = convert_dashboard(dash_obj) + if temp_is_updated: + is_updated = temp_is_updated + if is_updated: + if not os.path.exists(("{}/converted".format(os.path.dirname(file)))): + os.mkdir("{}/converted".format(os.path.dirname(file))) + #name process + filename = os.path.basename(file) + filename = filename.replace(":", "") + filename = filename.replace("___", "_") + filename = filename.replace("__", "_") + with open(os.path.dirname(file) + "/" + CONVERTED_PATH_NAME + "/" + filename, "w") as f: + json.dump(dash_obj, f, indent=2) + print('Write to file: {}'.format(f)) +def convert_folder(folder): + for root, dirs, files in os.walk(folder): + for file in files: + if str(file).endswith(".json"): + convert_file(root + "/" + file) + for sub_dir in dirs: + convert_folder(sub_dir) + + +# Press the green button in the gutter to run the script. +if __name__ == "__main__": + convert_folder(sys.argv[1]) diff --git a/utilities/transform/dashboard_remove_default_values.py b/utilities/transform/dashboard_remove_default_values.py new file mode 100644 index 0000000..51319a8 --- /dev/null +++ b/utilities/transform/dashboard_remove_default_values.py @@ -0,0 +1,70 @@ +import json +import os +import sys + +CONVERTED_PATH_NAME = "converted" +CURRENT_KEY = "current" + +# Depth First Search to find any matching measurement and back up the tree to update the nearest parent's "datasource" +# property to the matching split database +def convert_dashboard(dash_obj): + is_updated: bool = False + + if type(dash_obj) == list: + for ls in dash_obj: + temp_is_updated = convert_dashboard(ls) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == dict: + # depth first + for key, value in dash_obj.items(): + try: + if (key == CURRENT_KEY): + #print('original: {}'.format(dash_obj[key])) + if (value[CURRENT_KEY]==''): + #print('converted: {}'.format(dash_obj[key])) + is_updated = True + else: + temp_is_updated = convert_dashboard(value) + if temp_is_updated: + is_updated = True + except TypeError: + #print('Type error: {}:{}'.format(key, value)) + # do nothing + pass + else: + pass # do nothing with other types: int, float, bool, None + return is_updated + + +def convert_file(file): + is_updated = False + #print('file: {}'.format(file)) + with open(file, "r") as dash_json: + dash_obj = json.load(dash_json) + temp_is_updated = convert_dashboard(dash_obj['templating']) + if temp_is_updated: + is_updated = temp_is_updated + if is_updated: + if not os.path.exists(("{}/converted".format(os.path.dirname(file)))): + os.mkdir("{}/converted".format(os.path.dirname(file))) + #name process + filename = os.path.basename(file) + filename = filename.replace(":", "") + filename = filename.replace("___", "_") + filename = filename.replace("__", "_") + with open(os.path.dirname(file) + "/" + CONVERTED_PATH_NAME + "/" + filename, "w") as f: + json.dump(dash_obj, f, indent=2) + print('Write to file: {}'.format(f)) +def convert_folder(folder): + for root, dirs, files in os.walk(folder): + for file in files: + if str(file).endswith(".json"): + convert_file(root + "/" + file) + for sub_dir in dirs: + convert_folder(sub_dir) + + +# Press the green button in the gutter to run the script. +if __name__ == "__main__": + convert_folder(sys.argv[1]) diff --git a/utilities/transform/dashboard_sql_single_to_multi.py b/utilities/transform/dashboard_sql_single_to_multi.py new file mode 100644 index 0000000..d58579a --- /dev/null +++ b/utilities/transform/dashboard_sql_single_to_multi.py @@ -0,0 +1,78 @@ +import json +import os +import sys + +# The key of property to update value with +DS_KEY = "datasource" +DS_TYPE_KEY = "type" +DS_UID_KEY = "uid" +DS_GRAFANA_KEY = "grafana" +CONVERTED_PATH_NAME = "converted" + +# Depth First Search to find any matching measurement and back up the tree to update the nearest parent's "datasource" +# property to the matching split database +def convert_dashboard(dash_obj): + is_updated: bool = False + + if type(dash_obj) == list: + for ls in dash_obj: + temp_is_updated = convert_dashboard(ls) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == dict: + # depth first + for key, value in dash_obj.items(): + try: + if (key == DS_KEY): + if (value[DS_TYPE_KEY] and value[DS_UID_KEY]): + #print('original: {}'.format(dash_obj[key])) + if (value[DS_UID_KEY]==DS_GRAFANA_KEY): + dash_obj[key] = "-- Grafana --" + else: + dash_obj[key] = value[DS_UID_KEY] + #print('converted: {}'.format(dash_obj[key])) + is_updated = True + else: + temp_is_updated = convert_dashboard(value) + if temp_is_updated: + is_updated = True + except TypeError: + #print('Type error: {}:{}'.format(key, value)) + # do nothing + pass + else: + pass # do nothing with other types: int, float, bool, None + return is_updated + + +def convert_file(file): + is_updated = False + #print('file: {}'.format(file)) + with open(file, "r") as dash_json: + dash_obj = json.load(dash_json) + temp_is_updated = convert_dashboard(dash_obj) + if temp_is_updated: + is_updated = temp_is_updated + if is_updated: + if not os.path.exists(("{}/converted".format(os.path.dirname(file)))): + os.mkdir("{}/converted".format(os.path.dirname(file))) + #name process + filename = os.path.basename(file) + filename = filename.replace(":", "") + filename = filename.replace("___", "_") + filename = filename.replace("__", "_") + with open(os.path.dirname(file) + "/" + CONVERTED_PATH_NAME + "/" + filename, "w") as f: + json.dump(dash_obj, f, indent=2) + print('Write to file: {}'.format(f)) +def convert_folder(folder): + for root, dirs, files in os.walk(folder): + for file in files: + if str(file).endswith(".json"): + convert_file(root + "/" + file) + for sub_dir in dirs: + convert_folder(sub_dir) + + +# Press the green button in the gutter to run the script. +if __name__ == "__main__": + convert_folder(sys.argv[1]) diff --git a/utilities/transform/dashboard_transform.py b/utilities/transform/dashboard_transform.py new file mode 100644 index 0000000..075dc43 --- /dev/null +++ b/utilities/transform/dashboard_transform.py @@ -0,0 +1 @@ +import subprocess \ No newline at end of file diff --git a/utilities/transform/split_datasource.py b/utilities/transform/split_datasource.py index 4e44540..0f72841 100644 --- a/utilities/transform/split_datasource.py +++ b/utilities/transform/split_datasource.py @@ -2,6 +2,7 @@ import os import sys + # The list of measurements that would map to the "flows FLOW_MEASUREMENTS = ("flow_data_4_tuple", "flows_summary_application_port", From 5f29b9c41a451e6f02c4067f4b4ba247a0e5a86b Mon Sep 17 00:00:00 2001 From: Marina Zheng Date: Sun, 23 Jul 2023 20:44:05 -0700 Subject: [PATCH 4/6] Update utility scripts - Update export/import scripts to work with legacy and basic auth in one file for eaiser maintenance - Update export scripts to inlcude cClear version and Grafana version as exported names. - Add document and exception handling to utility scripts. - Removed unused scripts --- utilities/transfer/dashboard_export.sh | 111 ++++++++---- .../transfer/dashboard_export_basic_auth.sh | 164 ------------------ .../transfer/dashboard_export_updated.sh | 126 -------------- utilities/transfer/dashboard_import.sh | 87 ++++++---- .../transfer/dashboard_import_updated.sh | 153 ---------------- utilities/transfer/datasource_export.sh | 62 ++++--- utilities/transfer/datasource_import.sh | 54 +++--- utilities/transfer/preference_export.sh | 68 +++++--- utilities/transfer/preference_import.sh | 103 +++++++---- .../transform/dashboard_field_renamed.py | 67 ++++--- .../transform/dashboard_grafana_9_to_7.py | 90 ++++++---- .../dashboard_remove_default_values.py | 86 +++++---- utilities/transform/dashboard_transform.py | 1 - utilities/transform/split_datasource.py | 76 ++++---- 14 files changed, 503 insertions(+), 745 deletions(-) delete mode 100644 utilities/transfer/dashboard_export_basic_auth.sh delete mode 100755 utilities/transfer/dashboard_export_updated.sh delete mode 100644 utilities/transfer/dashboard_import_updated.sh delete mode 100644 utilities/transform/dashboard_transform.py diff --git a/utilities/transfer/dashboard_export.sh b/utilities/transfer/dashboard_export.sh index f208338..c9f5697 100755 --- a/utilities/transfer/dashboard_export.sh +++ b/utilities/transfer/dashboard_export.sh @@ -1,10 +1,18 @@ #!/bin/bash +# Script to export dashboards from a specified cClear as JSON files +# "folderUid" and "folderTitle" are added to each dashboard to help identify the folders to import later by +# running "dashboard_import.sh". +# Modified from Paul Sulistio's scripts. + +# todo: add support to import multiple folders separated by ";" + OPTSPEC=":hu:p:t:f:" +# Show help on how to use this script show_help() { cat << EOF -Usage: $0 [-u USER] [-p PASSWORD] [-f FROM_FOLDER] [-t TARGET_HOST] +Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST_IP] [-f FROM_FOLDER] Script to export grafana dashboards -u Required. cClear user to login -p Required. cClear user password to login @@ -15,7 +23,7 @@ Script to export grafana dashboards EOF } -###### Check script invocation options ###### +# Check script invocation options while getopts "$OPTSPEC" optchar; do case "$optchar" in h) @@ -27,7 +35,7 @@ while getopts "$OPTSPEC" optchar; do p) PASSWORD="$OPTARG";; t) - HOST="$OPTARG";; + TARGET_HOST_IP="$OPTARG";; f) FROM="$OPTARG";; \?) @@ -41,7 +49,8 @@ while getopts "$OPTSPEC" optchar; do esac done -if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$HOST" ]; then +# Check required arguments +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$TARGET_HOST_IP" ]; then show_help exit 1 fi @@ -95,52 +104,74 @@ function log_title() { ${SETCOLOR_NORMAL} } -mycookie="$PWD/mycookie" -counter=0 - function init() { - DATE_TIME=$(date '+%d%m%Y_%H%M%S') - DASH_DIR="$PWD/exported_dashboards/dashboards_${HOST}_${DATE_TIME}" + DASH_FOLDER="dashboards" + DASH_DIR="$PWD/${DASH_FOLDER}" if [ ! -d "${DASH_DIR}" ]; then mkdir -p "${DASH_DIR}" else log_title "----------------- A $DASH_DIR directory already exists! -----------------" + log_title "----------------- Rename or remove this directory before continuing -----------------" + exit 1 fi } -init - -# host url -if [[ ! "$HOST" == "https://"* ]]; then - HOST="https://$HOST" +# set cookie param for curl command according to login options +STATUS_CODE=$(curl --noproxy '*' -k --write-out '%{http_code}' --silent --output /dev/null --data "uname=$USER&psw=$PASSWORD" "https://$TARGET_HOST_IP/sess/login?rp=/vb/") +if [[ "$STATUS_CODE" -eq 404 ]]; then + HOST="https://$USER:$PASSWORD@$TARGET_HOST_IP" +elif [[ "$STATUS_CODE" -eq 302 ]]; then + HOST="https://$TARGET_HOST_IP" + mycookie="$PWD/mycookie" + LOGIN=$(curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/) + CURL_COOKIE="-b $mycookie" +else + show_help + exit 1 fi -curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" "$HOST/sess/login?rp=/vb/" - -folder_json=$(curl --noproxy '*' -k -b "$mycookie" "$HOST/graph-engine/api/folders") +# get folders +folder_json=$(curl --noproxy '*' -k $CURL_COOKIE --request "GET" -H "Content-Type:application/json" \ +"$HOST/graph-engine/api/folders") # From folder specified: +declare -a dashboard_uids if [ ${#FROM} -gt 0 ]; then - # Find matching folder from remote (with folder title) - FOLDER_UID=$(echo "$folder_json" | jq -r '.[] | select(.title == "'"$FROM"'") | .uid') - # Folder not found, prompt error and get out - if [ -z "$FOLDER_UID" ] ; then - log_failure "Folder $FROM is not found. Please check spelling and double quote with any spaces." - exit 1 - fi - # Folder found: get the collection of dashboard uids in this folder - dashboard_uids=$(curl --noproxy '*' -k -b "$mycookie" "$HOST"/graph-engine/api/search\?query\=\& | \ - jq -r '.[] | select(.type | contains("dash-db")) | select(.folderUid != null) | select(.folderUid == "'"$FOLDER_UID"'") | .uid') + IFS=';' read -ra FROM_LIST <<< "$FROM" + for i in "${FROM_LIST[@]}"; do + folder_i=$(echo $i | xargs) + # Find matching folder from remote (with folder title) + FOLDER_UID=$(echo "$folder_json" | jq -r '.[] | select(.title == "'"${folder_i}"'") | .uid') + # Folder not found, prompt error and exit + if [ -z "$FOLDER_UID" ] ; then + log_failure "Folder ${i} is not found. Please check spelling and double quote with any spaces." + continue + fi + # Folder found: get the collection of dashboard uids in this folder + uids=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST"/graph-engine/api/search\?query\=\& | \ + jq -r '.[] | select(.type | contains("dash-db")) | select(.folderUid != null) | select(.folderUid == "'"$FOLDER_UID"'") | .uid') + dashboard_uids+=${uids[@]} + done # From all folders: else - dashboard_uids=$(curl --noproxy '*' -k -b "$mycookie" "$HOST"/graph-engine/api/search\?query\=\& | \ + dashboard_uids=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST"/graph-engine/api/search\?query\=\& | \ jq -r '.[] | select(.type | contains("dash-db")) | .uid') fi +#echo "dashboard_uids: "$dashboard_uids + +# exit if nothing to import +#if [[ ${#dashboard_uid[@]} -eq 0 ]]; then +# exit 1 +#fi + # Export dashboards +init +counter=0 for dashboard_uid in $dashboard_uids; do url=$(echo "$HOST/graph-engine/api/dashboards/uid/$dashboard_uid" | tr -d '\r') - dashboard_json=$(curl --noproxy '*' -k -b "$mycookie" "$url") + dashboard_json=$(curl --noproxy '*' -k $CURL_COOKIE "$url") dashboard_title=$(echo "$dashboard_json" | jq -r '.dashboard | .title' | sed -r 's/[ \/]+/_/g' ) + dashboard_file=$(echo "$dashboard_title" | tr '[:upper:]' '[:lower:]') dashboard_version=$(echo "$dashboard_json" | jq -r '.dashboard | .version') dashboard_folder_raw=$(echo "$dashboard_json" | jq -r '.meta | .folderTitle') dashboard_folder=$(echo "$dashboard_json" | jq -r '.meta | .folderTitle' | sed -r 's/[ \/]+/_/g' ) @@ -155,15 +186,23 @@ for dashboard_uid in $dashboard_uids; do fi counter=$((counter + 1)) - # save dashboard with meta, dashboard and folder uid. + # save dashboard with folder uid and title to help identify folders to import later. echo "$dashboard_json" | jq '.dashboard | . += {"folderUid":"'$folder_uid'", "folderTitle": "'"$dashboard_folder_raw"'"}' > \ - "$DASH_DIR/${dashboard_folder}/${dashboard_title}_v${dashboard_version}.json" - log_success "Dashboard has been saved\t\t title=\"${dashboard_title}\", uid=\"${dashboard_uid}\", - path=\"${DASH_DIR}/${dashboard_folder}/${dashboard_title}_v${dashboard_version}.json\"." + "$DASH_DIR/${dashboard_folder}/${dashboard_file}_v${dashboard_version}.json" + log_success "Dashboard has been saved\t\t title=\"${dashboard_file}\", uid=\"${dashboard_uid}\", + path=\"${DASH_DIR}/${dashboard_folder}/${dashboard_file}_v${dashboard_version}.json\"." done -# zip -r -m ${DASH_DIR}.zip ${DASH_DIR} -rm mycookie +if [[ ${counter} -gt 0 ]]; then + cclear_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/api/admin/info") + CCLEAR_VERSION="cclear_$(echo $cclear_json | jq '.data.software.build' | tr -d '"')" + grafana_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/graph-engine/api/health") + GRAFANA_VERSION="grafana_$(echo $grafana_json | jq '.version' | tr -d '"')" + DATE_TIME="date_$(date '+%d%m%Y_%H%M%S')" + DASH_FILE_ZIP="${DASH_FOLDER}_${TARGET_HOST_IP}_${CCLEAR_VERSION}_${GRAFANA_VERSION}_${DATE_TIME}" + zip -r -m ${DASH_FILE_ZIP}.zip ${DASH_FOLDER} +fi +rm mycookie 2> /dev/null -log_title "${counter} dashboards were saved in ${DASH_DIR}"; +log_title "${counter} dashboards were saved in "$PWD/${DASH_FILE_ZIP}".zip"; log_title "------------------------------ FINISHED ---------------------------------"; diff --git a/utilities/transfer/dashboard_export_basic_auth.sh b/utilities/transfer/dashboard_export_basic_auth.sh deleted file mode 100644 index fd4e54b..0000000 --- a/utilities/transfer/dashboard_export_basic_auth.sh +++ /dev/null @@ -1,164 +0,0 @@ -#!/bin/bash - -OPTSPEC=":hu:p:t:f:" - -show_help() { -cat << EOF -Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST] [-f FROM_FOLDER] -Script to export grafana dashboards - -u Required. cClear user to login - -p Required. cClear user password to login - -t Required. The IP of the source cClear host i.e 10.51.10.32 - -f Optional. The name of the folder to export from, double quotes with spaces. Export all folders if not - specified. - -h Display this help and exit. -EOF -} - -###### Check script invocation options ###### -while getopts "$OPTSPEC" optchar; do - case "$optchar" in - h) - show_help - exit - ;; - u) - USER="$OPTARG";; - p) - PASSWORD="$OPTARG";; - t) - HOST="$OPTARG";; - f) - FROM="$OPTARG";; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "Option -$OPTARG requires an argument." >&2 - exit 1 - ;; - esac -done - -if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$HOST" ]; then - show_help - exit 1 -fi - -# set some colors for status OK, FAIL and titles -SETCOLOR_SUCCESS="echo -en \\033[0;32m" -SETCOLOR_FAILURE="echo -en \\033[1;31m" -SETCOLOR_NORMAL="echo -en \\033[0;39m" -SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple - -# usage log "string to log" "color option" -function log_success() { - if [ $# -lt 1 ]; then - ${SETCOLOR_FAILURE} - echo "Not enough arguments for log function! Expecting 1 argument got $#" - exit 1 - fi - - timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") - - ${SETCOLOR_SUCCESS} - printf "[${timestamp}] $1\n" - ${SETCOLOR_NORMAL} -} - -function log_failure() { - if [ $# -lt 1 ]; then - ${SETCOLOR_FAILURE} - echo "Not enough arguments for log function! Expecting 1 argument got $#" - exit 1 - fi - - timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") - - ${SETCOLOR_FAILURE} - printf "[${timestamp}] $1\n" - ${SETCOLOR_NORMAL} -} - -function log_title() { - if [ $# -lt 1 ]; then - ${SETCOLOR_FAILURE} - log_failure "Not enough arguments for log function! Expecting 1 argument got $#" - exit 1 - fi - - ${SETCOLOR_TITLE_PURPLE} - printf "|-------------------------------------------------------------------------|\n" - printf "|$1|\n"; - printf "|-------------------------------------------------------------------------|\n" - ${SETCOLOR_NORMAL} -} - -counter=0 - -function init() { - DATE_TIME=$(date '+%d%m%Y_%H%M%S') - DASH_DIR="$PWD/dashboards_${HOST}_${DATE_TIME}" - if [ ! -d "${DASH_DIR}" ]; then - mkdir "${DASH_DIR}" - else - log_title "----------------- A $DASH_DIR directory already exists! -----------------" - fi -} - -init - -# host url -if [[ ! "$HOST" == "https://"* ]]; then - HOST="https://$USER:$PASSWORD@$HOST" -fi - -folder_json=$(curl --noproxy '*' -k --request "GET" -H "Content-Type:application/json" "$HOST/graph-engine/api/folders") -# From folder specified: -if [ ${#FROM} -gt 0 ]; then - # Find matching folder from remote (with folder title) - FOLDER_UID=$(echo $folder_json | jq -r '.[] | select(.title == "'"$FROM"'") | .uid') - # Folder not found, prompt error and get out - if [ -z "$FOLDER_UID" ] ; then - log_failure "Folder $FROM is not found. Please check spelling and double quote with any spaces." - exit 1 - fi - # Folder found: get the collection of dashboard uids in this folder - dashboard_uids=$(curl --noproxy '*' -k "$HOST"/graph-engine/api/search\?query\=\& | \ - jq -r '.[] | select(.type | contains("dash-db")) | select(.folderUid != null) | select(.folderUid == "'"$FOLDER_UID"'") | .uid') -# From all folders: -else - dashboard_uids=$(curl --noproxy '*' -k "$HOST"/graph-engine/api/search\?query\=\& | \ - jq -r '.[] | select(.type | contains("dash-db")) | .uid') -fi - -# Export dashboards -for dashboard_uid in $dashboard_uids; do - url=$(echo "$HOST/graph-engine/api/dashboards/uid/$dashboard_uid" | tr -d '\r') - dashboard_json=$(curl --noproxy '*' -k "$url") - dashboard_title=$(echo "$dashboard_json" | jq -r '.dashboard | .title' | sed -r 's/[ \/]+/_/g' ) - dashboard_file=$(echo "$dashboard_title" | tr '[:upper:]' '[:lower:]') - dashboard_version=$(echo "$dashboard_json" | jq -r '.dashboard | .version') - dashboard_folder_raw=$(echo "$dashboard_json" | jq -r '.meta | .folderTitle') - dashboard_folder=$(echo "$dashboard_json" | jq -r '.meta | .folderTitle' | sed -r 's/[ \/]+/_/g' ) - dashboard_folderId=$(echo "$dashboard_json" | jq -r '.meta | .folderId') - - # Find folder uid to save (so that importing can find the right folder to import to) - folder_uid=$(echo "$folder_json" | jq -r '.[] | select(.id=='$dashboard_folderId') | .uid ') - - # create the folder if not existing - if [ ! -d "${DASH_DIR}/${dashboard_folder}" ]; then - mkdir "${DASH_DIR}/${dashboard_folder}" - fi - - counter=$((counter + 1)) - # save dashboard with meta, dashboard and folder uid. - echo "$dashboard_json" | jq '.dashboard | . += {"folderUid":"'$folder_uid'", "folderTitle": "'"$dashboard_folder_raw"'"}' > \ - "$DASH_DIR/${dashboard_folder}/${dashboard_file}_v${dashboard_version}.json" - log_success "Dashboard has been saved\t\t title=\"${dashboard_file}\", uid=\"${dashboard_uid}\", - path=\"${DASH_DIR}/${dashboard_folder}/${dashboard_file}_v${dashboard_version}.json\"." -done - -log_title "${counter} dashboards were saved in ${DASH_DIR}"; -log_title "------------------------------ FINISHED ---------------------------------"; diff --git a/utilities/transfer/dashboard_export_updated.sh b/utilities/transfer/dashboard_export_updated.sh deleted file mode 100755 index a08adbd..0000000 --- a/utilities/transfer/dashboard_export_updated.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash - -OPTSPEC=":hu:p:t:" - -show_help() { -cat << EOF -Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST] -Script to export grafana datasources - -u Required. cClear user to login - -p Required. cClear user password to login - -t Required. The IP of the target host i.e 10.51.10.32 - -h Display this help and exit. -EOF -} - -###### Check script invocation options ###### -while getopts "$OPTSPEC" optchar; do - case "$optchar" in - h) - show_help - exit - ;; - u) - USER="$OPTARG";; - p) - PASSWORD="$OPTARG";; - t) - HOST="$OPTARG";; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "Option -$OPTARG requires an argument." >&2 - exit 1 - ;; - esac -done - -if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$HOST" ]; then - show_help - exit 1 -fi - -# set some colors for status OK, FAIL and titles -SETCOLOR_SUCCESS="echo -en \\033[0;32m" -SETCOLOR_FAILURE="echo -en \\033[1;31m" -SETCOLOR_NORMAL="echo -en \\033[0;39m" -SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple - -# usage log "string to log" "color option" -function log_success() { - if [ $# -lt 1 ]; then - ${SETCOLOR_FAILURE} - echo "Not enough arguments for log function! Expecting 1 argument got $#" - exit 1 - fi - - timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") - - ${SETCOLOR_SUCCESS} - printf "[${timestamp}] $1\n" - ${SETCOLOR_NORMAL} -} - -function log_failure() { - if [ $# -lt 1 ]; then - ${SETCOLOR_FAILURE} - echo "Not enough arguments for log function! Expecting 1 argument got $#" - exit 1 - fi - - timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") - - ${SETCOLOR_FAILURE} - printf "[${timestamp}] $1\n" - ${SETCOLOR_NORMAL} -} - -function log_title() { - if [ $# -lt 1 ]; then - ${SETCOLOR_FAILURE} - log_failure "Not enough arguments for log function! Expecting 1 argument got $#" - exit 1 - fi - - ${SETCOLOR_TITLE_PURPLE} - printf "|-------------------------------------------------------------------------|\n" - printf "|$1|\n"; - printf "|-------------------------------------------------------------------------|\n" - ${SETCOLOR_NORMAL} -} - -mycookie="$PWD/mycookie" -counter=0 - -function init() { - DS_DIR="$PWD/datasources" - - if [ ! -d "${DS_DIR}" ]; then - mkdir "${DS_DIR}" - else - log_title "----------------- A $DS_DIR directory already exists! -----------------" - fi -} - -init - -# host url -if [[ ! "$HOST" == "https://"* ]]; then - HOST="https://$HOST" -fi - -curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" "$HOST/sess/login?rp=/vb/" -datasource_json=$(curl --noproxy '*' -k -b $mycookie "$HOST/graph-engine/api/datasources") -for id in $(echo $datasource_json | jq -r '.[] | .id'); do - counter=$((counter + 1)) - curl --noproxy '*' -f -k -b $mycookie "$HOST/graph-engine/api/datasources/${id}" | jq '' > "$DS_DIR/${id}.json" - log_success "Datasource has been saved\t id=\"${id}\", path=\"${DS_DIR}/${id}.json\"." -done - -zip -r -m datasources.zip datasources -rm mycookie - -log_title "${counter} datasource(s) were saved and zipped in $PWD/datasources.zip"; -log_title "------------------------------ FINISHED ---------------------------------"; diff --git a/utilities/transfer/dashboard_import.sh b/utilities/transfer/dashboard_import.sh index c18b03d..a64dc6b 100755 --- a/utilities/transfer/dashboard_import.sh +++ b/utilities/transfer/dashboard_import.sh @@ -1,16 +1,22 @@ #!/bin/bash -# todo: Import from dashboard files without .folderTitle...from git repo or manual exported -OPTSPEC=":hu:p:z:t:" +# Script to import dashboard JSON files into a specified cClear +# Dashboards json files from running "dashboard_export.sh" will get imported to the specified folders. +# Dashboards saved from Grafana import will get imported to the "General" folder. +# Modified from Paul Sulistio's scripts. + +OPTSPEC=":hu:p:t:i:" + +###### Show help on how to use this script ###### show_help() { cat << EOF -Usage: $0 [-u USER] [-p PASSWORD] [-z PATH] [-t TARGET_HOST] +Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST_IP] [-i IMPORT_PATH] Script to import dashboards into Grafana -u Required. cClear user to login -p Required. cClear user password to login - -z Required. Full path to the folder or zip file containing JSON exports of the dashboards - you want to be imported. -t Required. The IP of the destination cClear host i.e 10.51.10.32 + -i Required. Full path to the folder or zip file containing JSON exports of the dashboards + you want to be imported. -h Display this help and exit. EOF } @@ -26,10 +32,10 @@ while getopts "$OPTSPEC" optchar; do USER="$OPTARG";; p) PASSWORD="$OPTARG";; - z) - DASH_PATH="$OPTARG";; t) - HOST="$OPTARG";; + TARGET_HOST_IP="$OPTARG";; + i) + IMPORT_PATH="$OPTARG";; \?) echo "Invalid option: -$OPTARG" >&2 exit 1 @@ -41,7 +47,8 @@ while getopts "$OPTSPEC" optchar; do esac done -if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$DASH_PATH" ] || [ -z "$HOST" ]; then +###### Check required arguments ###### +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$IMPORT_PATH" ] || [ -z "$TARGET_HOST_IP" ]; then show_help exit 1 fi @@ -96,15 +103,15 @@ function log_title() { } -ZIP_FILE=$(basename $DASH_PATH) +ZIP_FILE=$(basename $IMPORT_PATH) if [[ $ZIP_FILE =~ \.zip$ ]]; then - DASH_DIR=$(unzip -qql $DASH_PATH | head -n1 | tr -s ' ' | cut -d' ' -f5-) - unzip $DASH_PATH + DASH_DIR=$(unzip -qql $IMPORT_PATH | head -n1 | tr -s ' ' | cut -d' ' -f5-) + unzip -o $IMPORT_PATH DIR_LENGTH=${#DASH_DIR} DASH_DIR=${DASH_DIR:0:DIR_LENGTH-1} else - DASH_DIR=$DASH_PATH + DASH_DIR=$IMPORT_PATH fi if [ -d "$DASH_DIR" ]; then @@ -124,19 +131,23 @@ else exit 1 fi +# set cookie param for curl command according to login options +STATUS_CODE=$(curl --noproxy '*' -k --write-out '%{http_code}' --silent --output /dev/null --data "uname=$USER&psw=$PASSWORD" "https://$TARGET_HOST_IP/sess/login?rp=/vb/") +if [[ "$STATUS_CODE" -eq 404 ]]; then + HOST="https://$USER:$PASSWORD@$TARGET_HOST_IP" +elif [[ "$STATUS_CODE" -eq 302 ]]; then + HOST="https://$TARGET_HOST_IP" + mycookie="$PWD/mycookie" + LOGIN=$(curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/) + CURL_COOKIE="-b $mycookie" +else + show_help + exit 1 +fi + NUMSUCCESS=0 NUMFAILURE=0 COUNTER=0 - - -# host url -if [[ ! "$HOST" == "http"* ]]; then - HOST="https://$HOST" -fi - -mycookie="$PWD/mycookie" -curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/ - for DASH_FILE in $DASH_LIST; do COUNTER=$((COUNTER + 1)) echo "Import $COUNTER/$FILESTOTAL: $DASH_FILE..." @@ -149,27 +160,31 @@ for DASH_FILE in $DASH_LIST; do # shellcheck disable=SC2116 dashboard=$(echo '{"dashboard": ' "${dashboard}"'}') -# echo "$dashboard" # If folder uid id not provided, import to the "General" folder if [ ${#folder_uid} -eq 0 ]; then - RESULT=$(echo "$dashboard" | jq -r '. * {overwrite: true, dashboard: {id: null}}' | curl -k -b $mycookie -X POST - -H \ - "Content-Type: application/json" $HOST/graph-engine/api/dashboards/db -d @-) + RESULT=$(echo "$dashboard" | jq -r '. * {overwrite: true, dashboard: {id: null}}' | curl -k $CURL_COOKIE -X POST \ + -H "Content-Type: application/json" $HOST/graph-engine/api/dashboards/db -d @-) else # Find the folder id from $HOST with this folder uid - folder_id=$(curl --noproxy '*' -k -b "$mycookie" "$HOST"/graph-engine/api/folders/$folder_uid | jq -r '.id') - # If not found, create a folder with this folder uid and folder title. + folder_id=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST"/graph-engine/api/folders/$folder_uid | jq -r '.id') + # If not found, try finding it with folder title. if [ "$folder_id" == "null" ]; then - folder_new=$(echo '{"uid": "'$folder_uid'", "title": "'"$folder_title"'"}' | curl --noproxy '*' -k -b \ - $mycookie -X POST -H "Content-Type: application/json" $HOST/graph-engine/api/folders -d @-) + folder_json=$(curl --noproxy '*' -k $CURL_COOKIE --request "GET" -H "Content-Type:application/json" \ + "$HOST/graph-engine/api/folders") + folder_id=$(echo "$folder_json" | jq -r '.[] | select(.title == "'"$folder_title"'") | .id') + folder_uid=$(echo "$folder_json" | jq -r '.[] | select(.title == "'"$folder_title"'") | .uid') + fi + # If still not found, create a folder with this folder uid and folder title. + if [ "$folder_id" == "null" ] || [ ${#folder_id} -eq 0 ]; then + echo " here" + folder_new=$(echo '{"uid": "'$folder_uid'", "title": "'"$folder_title"'"}' | curl --noproxy '*' -k \ + $CURL_COOKIE -X POST -H "Content-Type: application/json" $HOST/graph-engine/api/folders -d @-) folder_id=$(echo $folder_new | jq -r '.id') + folder_uid=$(echo $folder_new | jq -r '.uid') fi - # Import dashboard with folder id, uid, and title - RESULT=$(echo "$dashboard" | jq -r '. * {overwrite: true, dashboard: {id: null}} | . += {folderId:'$folder_id', - folderUid: - "'$folder_uid'", folderTitle: "'"$folder_title"'"}' \ - | curl --noproxy '*' -k -b $mycookie -X POST -H "Content-Type: application/json" $HOST/graph-engine/api/dashboards/db -d @-) + RESULT=$(echo "$dashboard" | jq -r '. * {overwrite: true, dashboard: {id: null}} | . += {folderId:'$folder_id', folderUid:"'$folder_uid'", folderTitle: "'"$folder_title"'"}' \ + | curl --noproxy '*' -k $CURL_COOKIE -X POST -H "Content-Type: application/json" $HOST/graph-engine/api/dashboards/db -d @-) fi # log result diff --git a/utilities/transfer/dashboard_import_updated.sh b/utilities/transfer/dashboard_import_updated.sh deleted file mode 100644 index 9fae8ca..0000000 --- a/utilities/transfer/dashboard_import_updated.sh +++ /dev/null @@ -1,153 +0,0 @@ -#!/bin/bash -x - -OPTSPEC=":hu:p:z:t:" - -show_help() { -cat << EOF -Usage: $0 [-u USER] [-p PASSWORD] [-z PATH] [-t TARGET_HOST] -Script to import datasource into Grafana - -u Required. cClear user to login - -p Required. cClear user password to login - -z Required. Full path to the zip file containing JSON exports of the datasource you want to be imported. - -t Required. The IP of the target host i.e 10.51.10.32 - -h Display this help and exit. -EOF -} - -###### Check script invocation options ###### -while getopts "$OPTSPEC" optchar; do - case "$optchar" in - h) - show_help - exit - ;; - u) - USER="$OPTARG";; - p) - PASSWORD="$OPTARG";; - z) - DS_PATH="$OPTARG";; - t) - HOST="$OPTARG";; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "Option -$OPTARG requires an argument." >&2 - exit 1 - ;; - esac -done - -if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$DS_PATH" ] || [ -z "$HOST" ]; then - show_help - exit 1 -fi - -# set some colors for status OK, FAIL and titles -SETCOLOR_SUCCESS="echo -en \\033[0;32m" -SETCOLOR_FAILURE="echo -en \\033[1;31m" -SETCOLOR_NORMAL="echo -en \\033[0;39m" -SETCOLOR_TITLE_PURPLE="echo -en \\033[0;35m" # purple - -# usage log "string to log" "color option" -function log_success() { - if [ $# -lt 1 ]; then - ${SETCOLOR_FAILURE} - echo "Not enough arguments for log function! Expecting 1 argument got $#" - exit 1 - fi - - timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") - - ${SETCOLOR_SUCCESS} - printf "[%s] $1\n" "$timestamp" - ${SETCOLOR_NORMAL} -} - -function log_failure() { - if [ $# -lt 1 ]; then - ${SETCOLOR_FAILURE} - echo "Not enough arguments for log function! Expecting 1 argument got $#" - exit 1 - fi - - timestamp=$(date "+%Y-%m-%d %H:%M:%S %Z") - - ${SETCOLOR_FAILURE} - printf "[%s] $1\n" "$timestamp" - ${SETCOLOR_NORMAL} -} - -function log_title() { - if [ $# -lt 1 ]; then - ${SETCOLOR_FAILURE} - log_failure "Not enough arguments for log function! Expecting 1 argument got $#" - exit 1 - fi - - ${SETCOLOR_TITLE_PURPLE} - printf "|-----------------------------------------------------------------------------------------|\n" - printf "| %s |\n" "$1"; - printf "|-----------------------------------------------------------------------------------------|\n" - ${SETCOLOR_NORMAL} -} - -ZIP_FILE=$(basename $DS_PATH) -echo $ZIP_FILE - -if [[ $ZIP_FILE =~ \.zip$ ]]; then - DS_DIR=$(unzip -qql $DS_PATH | head -n1 | tr -s ' ' | cut -d' ' -f5-) - unzip $DS_PATH - DS_DIR=${DS_DIR: : -1} - if [ -d "$DS_DIR" ]; then - DS_LIST=$(find "$PWD/$DS_DIR" -mindepth 1 -name \*.json) - echo $DS_LIST - if [ -z "$DS_LIST" ]; then - log_title "----------------- $DS_DIR contains no JSON files! -----------------" - log_failure "Directory $DS_DIR does not appear to contain any JSON files for import. Check your path and try again." - exit 1 - else - FILESTOTAL=$(echo "$DS_LIST" | wc -l) - log_title "----------------- Starting import of $FILESTOTAL datasource(s) -----------------" - fi - else - log_title "-------------------- $DS_DIR directory not found! -----------------" - log_failure "Directory $DS_DIR does not exist. Check your path and try again." - exit 1 - fi -else - log_title "-------------------- $ZIP_FILE Wrong format! -----------------" - log_failure "$ZIP_FILE is not a zip file. Please enter a correct file" -fi - -NUMSUCCESS=0 -NUMFAILURE=0 -COUNTER=0 - - -# host url -if [[ ! "$HOST" == "https://"* ]]; then - HOST="https://$HOST" -fi - -mycookie="$PWD/mycookie" -curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" "$HOST/sess/login?rp=/vb/" - -for i in datasources/*; do - RESULT=$(curl --noproxy '*' -k -b $mycookie -X "POST" "$HOST/graph-engine/api/datasources" \ - -H "Content-Type: application/json" --data-binary @$i) - if [[ "$RESULT" == *"Datasource added"* ]]; then - log_success "$RESULT" - NUMSUCCESS=$((NUMSUCCESS + 1)) - else - log_failure "$RESULT" - NUMFAILURE=$((NUMFAILURE + 1)) - fi -done - -rm mycookie - -log_title "Import complete. $NUMSUCCESS datasource(s) successfully imported. $NUMFAILURE datasource(s) imports failed."; -log_title "---------------------------------------FINISHED----------------------------------------"; diff --git a/utilities/transfer/datasource_export.sh b/utilities/transfer/datasource_export.sh index 36cdfdf..301af05 100755 --- a/utilities/transfer/datasource_export.sh +++ b/utilities/transfer/datasource_export.sh @@ -1,10 +1,14 @@ #!/bin/bash +# Script to export datasource from a specified cClear as JSON files. +# Modified from Paul Sulistio's scripts. + OPTSPEC=":hu:p:t:" +###### Show help on how to use this script ###### show_help() { cat << EOF -Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST] +Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST_IP] Script to export grafana datasources -u Required. cClear user to login -p Required. cClear user password to login @@ -25,7 +29,7 @@ while getopts "$OPTSPEC" optchar; do p) PASSWORD="$OPTARG";; t) - HOST="$OPTARG";; + TARGET_HOST_IP="$OPTARG";; \?) echo "Invalid option: -$OPTARG" >&2 exit 1 @@ -37,7 +41,8 @@ while getopts "$OPTSPEC" optchar; do esac done -if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$HOST" ]; then +###### Check required arguments ###### +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$TARGET_HOST_IP" ]; then show_help exit 1 fi @@ -91,36 +96,53 @@ function log_title() { ${SETCOLOR_NORMAL} } -mycookie="$PWD/mycookie" -counter=0 - function init() { - DS_DIR="$PWD/datasources" + DS_FOLDER="datasources" + DS_DIR="$PWD/${DS_FOLDER}" + echo $DS_DIR if [ ! -d "${DS_DIR}" ]; then - mkdir "${DS_DIR}" + mkdir -p "${DS_DIR}" else - log_title "----------------- A $DS_DIR directory already exists! -----------------" + log_title "----------------- A $DS_DIR directory already exists! -----------------" + log_title "----------------- Rename or remove this directory before continuing -----------------" + exit 1 fi } -init - -# host url -if [[ ! "$HOST" == "https://"* ]]; then - HOST="https://$HOST" +# set cookie param for curl command according to login options +STATUS_CODE=$(curl --noproxy '*' -k --write-out '%{http_code}' --silent --output /dev/null --data "uname=$USER&psw=$PASSWORD" "https://$TARGET_HOST_IP/sess/login?rp=/vb/") +if [[ "$STATUS_CODE" -eq 404 ]]; then + HOST="https://$USER:$PASSWORD@$TARGET_HOST_IP" +elif [[ "$STATUS_CODE" -eq 302 ]]; then + HOST="https://$TARGET_HOST_IP" + mycookie="$PWD/mycookie" + LOGIN=$(curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/) + CURL_COOKIE="-b $mycookie" +else + show_help + exit 1 fi -curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" "$HOST/sess/login?rp=/vb/" -datasource_json=$(curl --noproxy '*' -k -b $mycookie "$HOST/graph-engine/api/datasources") +counter=0 +init +datasource_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/graph-engine/api/datasources") for id in $(echo $datasource_json | jq -r '.[] | .id'); do + name=$(echo $datasource_json | jq -r '.[] | select(.id == '"$id"') | .name' | sed -r 's/[ \/]+/_/g' | \ + tr '[:upper:]' '[:lower:]') counter=$((counter + 1)) - curl --noproxy '*' -f -k -b $mycookie "$HOST/graph-engine/api/datasources/${id}" | jq '' > "$DS_DIR/${id}.json" - log_success "Datasource has been saved\t id=\"${id}\", path=\"${DS_DIR}/${id}.json\"." + curl --noproxy '*' -f -k $CURL_COOKIE "$HOST/graph-engine/api/datasources/${id}" | jq '' > "$DS_DIR/${name}.json" + log_success "Datasource has been saved\t id=\"${id}\", name=\"${name}\", path=\"${DS_DIR}/${name}.json\"." done -zip -r -m datasources.zip datasources +cclear_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/api/admin/info") +CCLEAR_VERSION="cclear_$(echo $cclear_json | jq '.data.software.build' | tr -d '"')" +grafana_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/graph-engine/api/health") +GRAFANA_VERSION="grafana_$(echo $grafana_json | jq '.version' | tr -d '"')" +DATE_TIME="date_$(date '+%d%m%Y_%H%M%S')" +DS_FILE_ZIP="${DS_FOLDER}_${TARGET_HOST_IP}_${CCLEAR_VERSION}_${GRAFANA_VERSION}_${DATE_TIME}" +zip -r -m "${DS_FILE_ZIP}.zip" "${DS_FOLDER}" rm mycookie -log_title "${counter} datasource(s) were saved and zipped in $PWD/datasources.zip"; +log_title "${counter} datasource(s) were saved and zipped in "$PWD/${DS_FILE_ZIP}".zip"; log_title "------------------------------ FINISHED ---------------------------------"; diff --git a/utilities/transfer/datasource_import.sh b/utilities/transfer/datasource_import.sh index bb77951..d6e140b 100755 --- a/utilities/transfer/datasource_import.sh +++ b/utilities/transfer/datasource_import.sh @@ -1,15 +1,20 @@ -#!/bin/bash -x +#!/bin/bash -OPTSPEC=":hu:p:z:t:" +# Script to import datasource JSON files into a specified cClear +# Tests have been done on exported datasource json files from running "datasource_export.sh". +# Modified from Paul Sulistio's scripts. +OPTSPEC=":hu:p:t:i:" + +###### Show help on how to use this script ###### show_help() { cat << EOF -Usage: $0 [-u USER] [-p PASSWORD] [-z PATH] [-t TARGET_HOST] +Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST_IP] [-i IMPORT_PATH] Script to import datasource into Grafana -u Required. cClear user to login -p Required. cClear user password to login - -z Required. Full path to the zip file containing JSON exports of the datasource you want to be imported. -t Required. The IP of the target host i.e 10.51.10.32 + -i Required. Full path to the zip file containing JSON exports of the datasource you want to be imported. -h Display this help and exit. EOF } @@ -25,10 +30,10 @@ while getopts "$OPTSPEC" optchar; do USER="$OPTARG";; p) PASSWORD="$OPTARG";; - z) - DS_PATH="$OPTARG";; t) - HOST="$OPTARG";; + TARGET_HOST_IP="$OPTARG";; + i) + IMPORT_PATH="$OPTARG";; \?) echo "Invalid option: -$OPTARG" >&2 exit 1 @@ -40,7 +45,8 @@ while getopts "$OPTSPEC" optchar; do esac done -if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$DS_PATH" ] || [ -z "$HOST" ]; then +###### Check required arguments ###### +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$IMPORT_PATH" ] || [ -z "$TARGET_HOST_IP" ]; then show_help exit 1 fi @@ -94,12 +100,12 @@ function log_title() { ${SETCOLOR_NORMAL} } -ZIP_FILE=$(basename $DS_PATH) +ZIP_FILE=$(basename $IMPORT_PATH) echo $ZIP_FILE if [[ $ZIP_FILE =~ \.zip$ ]]; then - DS_DIR=$(unzip -qql $DS_PATH | head -n1 | tr -s ' ' | cut -d' ' -f5-) - unzip $DS_PATH + DS_DIR=$(unzip -qql $IMPORT_PATH | head -n1 | tr -s ' ' | cut -d' ' -f5-) + unzip -o $IMPORT_PATH DS_DIR=${DS_DIR: : -1} if [ -d "$DS_DIR" ]; then DS_LIST=$(find "$PWD/$DS_DIR" -mindepth 1 -name \*.json) @@ -122,21 +128,25 @@ else log_failure "$ZIP_FILE is not a zip file. Please enter a correct file" fi +# set cookie param for curl command according to login options +STATUS_CODE=$(curl --noproxy '*' -k --write-out '%{http_code}' --silent --output /dev/null --data "uname=$USER&psw=$PASSWORD" "https://$TARGET_HOST_IP/sess/login?rp=/vb/") +if [[ "$STATUS_CODE" -eq 404 ]]; then + HOST="https://$USER:$PASSWORD@$TARGET_HOST_IP" +elif [[ "$STATUS_CODE" -eq 302 ]]; then + HOST="https://$TARGET_HOST_IP" + mycookie="$PWD/mycookie" + LOGIN=$(curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/) + CURL_COOKIE="-b $mycookie" +else + show_help + exit 1 +fi + NUMSUCCESS=0 NUMFAILURE=0 COUNTER=0 - - -# host url -if [[ ! "$HOST" == "https://"* ]]; then - HOST="https://$HOST" -fi - -mycookie="$PWD/mycookie" -curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" "$HOST/sess/login?rp=/vb/" - for i in datasources/*; do - RESULT=$(curl --noproxy '*' -k -b $mycookie -X "POST" "$HOST/graph-engine/api/datasources" \ + RESULT=$(curl --noproxy '*' -k $CURL_COOKIE -X "POST" "$HOST/graph-engine/api/datasources" \ -H "Content-Type: application/json" --data-binary @$i) if [[ "$RESULT" == *"Datasource added"* ]]; then log_success "$RESULT" diff --git a/utilities/transfer/preference_export.sh b/utilities/transfer/preference_export.sh index 5f8fbc2..08bbb34 100755 --- a/utilities/transfer/preference_export.sh +++ b/utilities/transfer/preference_export.sh @@ -1,13 +1,14 @@ #!/bin/bash -# -# -# -OPTSPEC=":hu:p:t:f:" +# Script to export preferences from a specified cClear as JSON files. +# Modified from Paul Sulistio's scripts. + +###### Show help on how to use this script ###### +OPTSPEC=":hu:p:t:" show_help() { cat << EOF -Usage: $0 [-u USER] [-p PASSWORD] [-f FROM_FOLDER] [-t TARGET_HOST] +Usage: $0 [-u USER] [-p PASSWORD] [-f FROM_FOLDER] [-t TARGET_HOST_IP] Script to export grafana dashboards -u Required. cClear user to login -p Required. cClear user password to login @@ -28,7 +29,7 @@ while getopts "$OPTSPEC" optchar; do p) PASSWORD="$OPTARG";; t) - HOST="$OPTARG";; + TARGET_HOST_IP="$OPTARG";; \?) echo "Invalid option: -$OPTARG" >&2 exit 1 @@ -40,7 +41,8 @@ while getopts "$OPTSPEC" optchar; do esac done -if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$HOST" ]; then +###### Check required arguments ###### +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$TARGET_HOST_IP" ]; then show_help exit 1 fi @@ -95,31 +97,35 @@ function log_title() { } function init() { - DATE_TIME=$(date '+%d%m%Y_%H%M%S') - DASH_DIR="$PWD/preferences_${HOST}_${DATE_TIME}" - if [ ! -d "${DASH_DIR}" ]; then - mkdir "${DASH_DIR}" + PREF_FOLDER="preferences" + PREF_DIR="$PWD/${PREF_FOLDER}" + if [ ! -d "${PREF_DIR}" ]; then + mkdir -p "${PREF_DIR}" else - log_title "----------------- A $DASH_DIR directory already exists! -----------------" + log_title "----------------- A $PREF_DIR directory already exists! -----------------" + log_title "----------------- Rename or remove this directory before continuing -----------------" + exit 1 fi } -init - -PREF_ORG="$DASH_DIR/preferences_org.json" -PREF_USER="$DASH_DIR/preferences_user.json" - -# host url -if [[ ! "$HOST" == "https://"* ]]; then - HOST="https://$HOST" +# set cookie param for curl command according to login options +STATUS_CODE=$(curl --noproxy '*' -k --write-out '%{http_code}' --silent --output /dev/null --data "uname=$USER&psw=$PASSWORD" "https://$TARGET_HOST_IP/sess/login?rp=/vb/") +if [[ "$STATUS_CODE" -eq 404 ]]; then + HOST="https://$USER:$PASSWORD@$TARGET_HOST_IP" +elif [[ "$STATUS_CODE" -eq 302 ]]; then + HOST="https://$TARGET_HOST_IP" + mycookie="$PWD/mycookie" + LOGIN=$(curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/) + CURL_COOKIE="-b $mycookie" +else + show_help + exit 1 fi -mycookie="$PWD/mycookie" -curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" "$HOST/sess/login?rp=/vb/" - +init # Get preferences -pref_org_json=$(curl --noproxy '*' -k -b "$mycookie" "$HOST/graph-engine/api/org/preferences") -pref_user_json=$(curl --noproxy '*' -k -b "$mycookie" "$HOST/graph-engine/api/user/preferences") +pref_org_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/graph-engine/api/org/preferences") +pref_user_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/graph-engine/api/user/preferences") rm mycookie @@ -134,10 +140,20 @@ elif [[ "$pref_org_json" == *"Unauthorized"* ]] || [[ "$pref_user_json" == *"Una fi # Save preferences +PREF_ORG="$PREF_DIR/preferences_org.json" +PREF_USER="$PREF_DIR/preferences_user.json" echo $pref_org_json | jq '.' > "$PREF_ORG" log_success "Org. preferences saved: $pref_org_json" echo $pref_user_json | jq '.' > "$PREF_USER" log_success "User preferences saved: $pref_user_json" -log_title "Preferences were saved in $DASH_DIR"; +cclear_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/api/admin/info") +CCLEAR_VERSION="cclear_$(echo $cclear_json | jq '.data.software.build' | tr -d '"')" +grafana_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/graph-engine/api/health") +GRAFANA_VERSION="grafana_$(echo $grafana_json | jq '.version' | tr -d '"')" +DATE_TIME="date_$(date '+%d%m%Y_%H%M%S')" +PREF_FILE_ZIP="${PREF_FOLDER}_${TARGET_HOST_IP}_${CCLEAR_VERSION}_${GRAFANA_VERSION}_${DATE_TIME}" +zip -r -m "${PREF_FILE_ZIP}.zip" "${PREF_FOLDER}" + +log_title "Preferences were saved in ${PREF_FILE_ZIP}"; log_title "------------------------------ FINISHED ---------------------------------"; diff --git a/utilities/transfer/preference_import.sh b/utilities/transfer/preference_import.sh index 61b79bd..e0376c9 100755 --- a/utilities/transfer/preference_import.sh +++ b/utilities/transfer/preference_import.sh @@ -1,17 +1,20 @@ #!/bin/bash -# -# todo: ip to both ip and https; from folder to be both zipped or unzipped -# -OPTSPEC=":hu:p:z:t:" +# Script to import preference JSON files into a specified cClear +# Tests have been done on exported preference json files from running "preference_export.sh". +# Modified from Paul Sulistio's scripts. + +OPTSPEC=":hu:p:t:i:" + +###### Show help on how to use this script ###### show_help() { cat << EOF -Usage: $0 [-u USER] [-p PASSWORD] [-z PATH] [-t TARGET_HOST] +Usage: $0 [-u USER] [-p PASSWORD] [-t TARGET_HOST_IP] [-z IMPORT_PATH] Script to import dashboards into Grafana -u Required. cClear user to login -p Required. cClear user password to login - -z Required. Grafana preferences json file to import from. e.g. preferences.json -t Required. The IP of the destination cClear host i.e 10.51.10.32 + -i Required. Grafana preferences json file to import from. e.g. preferences.json -h Display this help and exit. EOF } @@ -27,10 +30,10 @@ while getopts "$OPTSPEC" optchar; do USER="$OPTARG";; p) PASSWORD="$OPTARG";; - z) - PREF_PATH="$OPTARG";; t) - HOST="$OPTARG";; + TARGET_HOST_IP="$OPTARG";; + i) + IMPORT_PATH="$OPTARG";; \?) echo "Invalid option: -$OPTARG" >&2 exit 1 @@ -42,7 +45,8 @@ while getopts "$OPTSPEC" optchar; do esac done -if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$PREF_PATH" ] || [ -z "$HOST" ]; then +###### Check required arguments ###### +if [ -z "$USER" ] || [ -z "$PASSWORD" ] || [ -z "$IMPORT_PATH" ] || [ -z "$TARGET_HOST_IP" ]; then show_help exit 1 fi @@ -96,42 +100,67 @@ function log_title() { ${SETCOLOR_NORMAL} } -PREF_FILE=$(basename $PREF_PATH) +ZIP_FILE=$(basename $IMPORT_PATH) + +if [[ $ZIP_FILE =~ \.zip$ ]]; then + PREF_DIR=$(unzip -qql $IMPORT_PATH | head -n1 | tr -s ' ' | cut -d' ' -f5-) + unzip -o $IMPORT_PATH + DIR_LENGTH=${#PREF_DIR} + PREF_DIR=${PREF_DIR:0:DIR_LENGTH-1} +else + PREF_DIR=$IMPORT_PATH +fi + +if [ -d "$PREF_DIR" ]; then + PREF_LIST=$(find "$PWD/$PREF_DIR" -mindepth 1 -name \*.json) -if [[ ! $PREF_FILE =~ \.json$ ]]; then - log_title "-------------------- $PREF_FILE Wrong format! -----------------" - log_failure "$PREF_PATH is not a json file. Please enter a correct file" + if [ -z "$PREF_LIST" ]; then + log_title "----------------- $PREF_DIR contains no JSON files! -----------------" + log_failure "Directory $PREF_DIR does not appear to contain any JSON files for import. Check your path and try again." + exit 1 + else + FILESTOTAL=$(echo "$PREF_LIST" | wc -l) + log_title "----------------- Starting import of $FILESTOTAL dashboards -----------------" + fi +else + log_title "-------------------- $PREF_DIR directory not found! -----------------" + log_failure "Directory $PREF_DIR does not exist. Check your path and try again." exit 1 fi -if [[ ! -f "$PREF_FILE" ]]; then - log_failure "No such file: $PREF_FILE." +# set cookie param for curl command according to login options +STATUS_CODE=$(curl --noproxy '*' -k --write-out '%{http_code}' --silent --output /dev/null --data "uname=$USER&psw=$PASSWORD" "https://$TARGET_HOST_IP/sess/login?rp=/vb/") +if [[ "$STATUS_CODE" -eq 404 ]]; then + HOST="https://$USER:$PASSWORD@$TARGET_HOST_IP" +elif [[ "$STATUS_CODE" -eq 302 ]]; then + HOST="https://$TARGET_HOST_IP" + mycookie="$PWD/mycookie" + LOGIN=$(curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/) + CURL_COOKIE="-b $mycookie" +else + show_help exit 1 fi NUMSUCCESS=0 NUMFAILURE=0 COUNTER=0 - - -# host url -if [[ ! "$HOST" == "https://"* ]]; then - HOST="https://$HOST" -fi - -mycookie="$PWD/mycookie" -curl --noproxy '*' -k -c mycookie --data "uname=$USER&psw=$PASSWORD" $HOST/sess/login?rp=/vb/ - -RESULT=$(cat "$PREF_FILE" | jq '.' | curl --noproxy '*' -k -b $mycookie -X PUT -H \ -"Content-Type: application/json" "$HOST/graph-engine/api/user/preferences" -d @-) - +for PREF_FILE in $PREF_LIST; do + COUNTER=$((COUNTER + 1)) + echo "Import $COUNTER/$FILESTOTAL: $PREF_FILE..." + RESULT=$(cat "$PREF_FILE" | jq '.' | curl --noproxy '*' -k $CURL_COOKIE -X PUT -H \ + "Content-Type: application/json" "$HOST/graph-engine/api/user/preferences" -d @-) + echo + # log result + if [[ "$RESULT" == *"updated"* ]]; then + log_success "$RESULT" + NUMSUCCESS=$((NUMSUCCESS + 1)) + else + log_failure "$RESULT" + NUMFAILURE=$((NUMFAILURE + 1)) + fi +done rm mycookie -# log result -if [[ "$RESULT" == *"updated"* ]]; then - log_success "$RESULT" -else - log_failure "$RESULT" -fi - -log_title "-------------------- Preferences were successfully imported.-------------------------" +log_title "Import complete. $NUMSUCCESS dashboards were successfully imported. $NUMFAILURE dashboard imports failed."; +log_title "-------------------------------------FINISHED----------------------------------------"; diff --git a/utilities/transform/dashboard_field_renamed.py b/utilities/transform/dashboard_field_renamed.py index 1bc9de2..89c82c5 100644 --- a/utilities/transform/dashboard_field_renamed.py +++ b/utilities/transform/dashboard_field_renamed.py @@ -4,59 +4,72 @@ # The key of property to update value with CONVERTED_PATH_NAME = "converted" -RENAMED_MAP = {"bytes_per_s":"bytes", - "fragments_per_s":"fragments", - "packets_per_s":"packets", - "resolution_s":"resolution"}; -# Depth First Search to find any matching measurement and back up the tree to update the nearest parent's "datasource" -# property to the matching split database -def convert_dashboard(dash_obj): - is_updated: bool = False +RENAMED_MAP = {"bytes_per_s": "bytes", + "fragments_per_s": "fragments", + "packets_per_s": "packets", + "resolution_s": "resolution"}; + + +def convert_dashboard(dash_parent_obj, dash_obj_key, dash_obj): + """ + Depth First Search to find any string value and replace the matching old value with its specified new value. With any + value matching and updated, the update is done using dash_parent_obj and dash_obj_key. Thus it doesn't have to bubble + up to its parent for update. + + :param dash_parent_obj: The parent element that owns this element being inspected. + :param dash_obj_key: The key of the element being inspected. + :param dash_obj: The value of the element being inspected. + :return: + """ + is_updated: bool = False if type(dash_obj) == list: for ls in dash_obj: - temp_is_updated = convert_dashboard(ls) + temp_is_updated = convert_dashboard(dash_parent_obj, dash_obj_key, ls) if temp_is_updated: is_updated = True elif type(dash_obj) == dict: - # depth first for key, value in dash_obj.items(): - temp_is_updated = convert_dashboard(value) + temp_is_updated = convert_dashboard(dash_obj, key, value) if temp_is_updated: is_updated = True elif type(dash_obj) == str: - #todo: rename - RENAMED_MAP. - pass + # print(f'key: {dash_obj_key}, value: {dash_obj}') + is_matched = False + for old_value, new_value in RENAMED_MAP.items(): + if old_value in dash_obj: + dash_obj = dash_obj.replace(old_value, new_value) + is_matched = True + if is_matched: + dash_parent_obj[dash_obj_key] = dash_obj + is_updated = True else: pass # do nothing with other types: int, float, bool, None return is_updated def convert_file(file): - is_updated = False - #print('file: {}'.format(file)) with open(file, "r") as dash_json: dash_obj = json.load(dash_json) - temp_is_updated = convert_dashboard(dash_obj) - if temp_is_updated: - is_updated = temp_is_updated + is_updated = convert_dashboard(dash_obj, "root", dash_obj) if is_updated: if not os.path.exists(("{}/converted".format(os.path.dirname(file)))): os.mkdir("{}/converted".format(os.path.dirname(file))) - #name process - filename = os.path.basename(file) - filename = filename.replace(":", "") - filename = filename.replace("___", "_") - filename = filename.replace("__", "_") + filename = os.path.basename(file).replace(":", "").replace("___", "_").replace("__", "_") with open(os.path.dirname(file) + "/" + CONVERTED_PATH_NAME + "/" + filename, "w") as f: json.dump(dash_obj, f, indent=2) - print('Write to file: {}'.format(f)) + print('Converted to file: {}'.format(f)) + + def convert_folder(folder): for root, dirs, files in os.walk(folder): for file in files: - if str(file).endswith(".json"): - convert_file(root + "/" + file) + try: + if str(file).endswith(".json"): + convert_file(root + "/" + file) + except Exception as e: + print(f"Exception converting file {file}: {e}") + continue for sub_dir in dirs: convert_folder(sub_dir) diff --git a/utilities/transform/dashboard_grafana_9_to_7.py b/utilities/transform/dashboard_grafana_9_to_7.py index 17641c6..205cbcb 100644 --- a/utilities/transform/dashboard_grafana_9_to_7.py +++ b/utilities/transform/dashboard_grafana_9_to_7.py @@ -7,38 +7,56 @@ DS_TYPE_KEY = "type" DS_UID_KEY = "uid" DS_GRAFANA_KEY = "grafana" +DS_GRAFANA_VALUE_7 = "-- Grafana --" CONVERTED_PATH_NAME = "converted" -# Depth First Search to find any matching measurement and back up the tree to update the nearest parent's "datasource" -# property to the matching split database + +def convertDatasource(dash_obj, key, value): + """ + Update datasource element from Grafana 9 format to Grafana 7. + + :param dash_obj: the parent element that ows this datasource element to update with + :param key: the key of the datasource element ("datasource") + :param value: the value of the datasource element in Grafana 9 format: { "type": , "uid": } + + :return: is_updated: True if successfully replaced, False otherwise. + """ + if value[DS_UID_KEY]: # if (value[DS_TYPE_KEY] or value[DS_UID_KEY]): + # print('original: {}'.format(dash_obj[key])) + if value[DS_UID_KEY] == DS_GRAFANA_KEY: + dash_obj[key] = DS_GRAFANA_VALUE_7 + else: + dash_obj[key] = value[DS_UID_KEY] + # print('converted: {}'.format(dash_obj[key])) + return True + return False + + def convert_dashboard(dash_obj): - is_updated: bool = False + """ + Depth First Search to find any "datasource" element and update its value format from a dict to a string value + as its UID value. + Returns: + is_updated: if any node gets updated, it bubbles up to root so the dashboard json file will be rewritten. + """ + is_updated = False if type(dash_obj) == list: for ls in dash_obj: temp_is_updated = convert_dashboard(ls) if temp_is_updated: is_updated = True elif type(dash_obj) == dict: - # depth first for key, value in dash_obj.items(): try: - if (key == DS_KEY): - if (value[DS_UID_KEY]): # if (value[DS_TYPE_KEY] or value[DS_UID_KEY]): - #print('original: {}'.format(dash_obj[key])) - if (value[DS_UID_KEY]==DS_GRAFANA_KEY): - dash_obj[key] = "-- Grafana --" - else: - dash_obj[key] = value[DS_UID_KEY] - #print('converted: {}'.format(dash_obj[key])) - is_updated = True + if key == DS_KEY: + temp_is_updated = convertDatasource(dash_obj, key, value) else: temp_is_updated = convert_dashboard(value) - if temp_is_updated: - is_updated = True + if temp_is_updated: + is_updated = True except TypeError: - #print('Type error: {}:{}'.format(key, value)) - # do nothing + # print('Type error: {}:{}'.format(key, value)) pass else: pass # do nothing with other types: int, float, bool, None @@ -46,29 +64,31 @@ def convert_dashboard(dash_obj): def convert_file(file): - is_updated = False - #print('file: {}'.format(file)) with open(file, "r") as dash_json: dash_obj = json.load(dash_json) - temp_is_updated = convert_dashboard(dash_obj) - if temp_is_updated: - is_updated = temp_is_updated - if is_updated: - if not os.path.exists(("{}/converted".format(os.path.dirname(file)))): - os.mkdir("{}/converted".format(os.path.dirname(file))) - #name process - filename = os.path.basename(file) - filename = filename.replace(":", "") - filename = filename.replace("___", "_") - filename = filename.replace("__", "_") - with open(os.path.dirname(file) + "/" + CONVERTED_PATH_NAME + "/" + filename, "w") as f: - json.dump(dash_obj, f, indent=2) - print('Write to file: {}'.format(f)) + is_converted = convert_dashboard(dash_obj) + + if not is_converted: + return + + # rewrite json file + if not os.path.exists(("{}/converted".format(os.path.dirname(file)))): + os.mkdir("{}/converted".format(os.path.dirname(file))) + filename = os.path.basename(file).replace(":", "").replace("___", "_").replace("__", "_") + with open(os.path.dirname(file) + "/" + CONVERTED_PATH_NAME + "/" + filename, "w") as f: + json.dump(dash_obj, f, indent=2) + print('Converted to file: {}'.format(f)) + + def convert_folder(folder): for root, dirs, files in os.walk(folder): for file in files: - if str(file).endswith(".json"): - convert_file(root + "/" + file) + try: + if str(file).endswith(".json"): + convert_file(root + "/" + file) + except Exception as e: + print(f"Exception converting file {file}: {e}") + continue for sub_dir in dirs: convert_folder(sub_dir) diff --git a/utilities/transform/dashboard_remove_default_values.py b/utilities/transform/dashboard_remove_default_values.py index 51319a8..fcc2e17 100644 --- a/utilities/transform/dashboard_remove_default_values.py +++ b/utilities/transform/dashboard_remove_default_values.py @@ -1,66 +1,86 @@ import json import os import sys +import traceback CONVERTED_PATH_NAME = "converted" +TYPE_KEY = "type" +NAME_KEY = "name" CURRENT_KEY = "current" +DEFAULT_QUERY = { + "selected": False, + "text": "", + "value": "" +} +DEFAULT_RESOLUTION = { + "selected": True, + "text": "auto", + "value": "$__auto_interval_resolution" +} + # Depth First Search to find any matching measurement and back up the tree to update the nearest parent's "datasource" # property to the matching split database -def convert_dashboard(dash_obj): - is_updated: bool = False +def convert_dashboard(dash_parent_obj, dash_key, dash_obj): + """ + Loop through template variables and update their default values. + + :param dash_parent_obj: The parent element that owns this element being inspected. + :param dash_key: The key of the element being inspected. + :param dash_obj: The value of the element being inspected. + :return: + """ + is_updated: bool = False if type(dash_obj) == list: for ls in dash_obj: - temp_is_updated = convert_dashboard(ls) + temp_is_updated = convert_dashboard(dash_parent_obj, dash_key, ls) if temp_is_updated: is_updated = True elif type(dash_obj) == dict: - # depth first - for key, value in dash_obj.items(): - try: - if (key == CURRENT_KEY): - #print('original: {}'.format(dash_obj[key])) - if (value[CURRENT_KEY]==''): - #print('converted: {}'.format(dash_obj[key])) - is_updated = True - else: - temp_is_updated = convert_dashboard(value) - if temp_is_updated: - is_updated = True - except TypeError: - #print('Type error: {}:{}'.format(key, value)) - # do nothing - pass + # for key, value in dash_obj.items(): + try: + if dash_obj[TYPE_KEY] == "query": + dash_obj[CURRENT_KEY] = DEFAULT_QUERY + is_updated = True + elif dash_obj[TYPE_KEY] == "interval" and dash_obj[NAME_KEY] == "resolution": + dash_obj[CURRENT_KEY] = DEFAULT_RESOLUTION + is_updated = True + else: + pass # do nothing about the rest + except TypeError: + print('Type error: {}:{}'.format(dash_key, dash_obj)) else: - pass # do nothing with other types: int, float, bool, None + pass # do nothing return is_updated def convert_file(file): - is_updated = False - #print('file: {}'.format(file)) with open(file, "r") as dash_json: dash_obj = json.load(dash_json) - temp_is_updated = convert_dashboard(dash_obj['templating']) - if temp_is_updated: - is_updated = temp_is_updated + inspect_obj = dash_obj['templating']['list'] + inspect_key = "list" + is_updated = convert_dashboard(dash_obj['templating'], inspect_key, inspect_obj) + if is_updated: if not os.path.exists(("{}/converted".format(os.path.dirname(file)))): os.mkdir("{}/converted".format(os.path.dirname(file))) - #name process - filename = os.path.basename(file) - filename = filename.replace(":", "") - filename = filename.replace("___", "_") - filename = filename.replace("__", "_") + filename = os.path.basename(file).replace(":", "").replace("___", "_").replace("__", "_") with open(os.path.dirname(file) + "/" + CONVERTED_PATH_NAME + "/" + filename, "w") as f: json.dump(dash_obj, f, indent=2) - print('Write to file: {}'.format(f)) + print('Converted to file: {}'.format(f)) + + def convert_folder(folder): for root, dirs, files in os.walk(folder): for file in files: - if str(file).endswith(".json"): - convert_file(root + "/" + file) + try: + if str(file).endswith(".json"): + convert_file(root + "/" + file) + except Exception as e: + print(f"Exception converting file {file}: {e}") + traceback.print_exc() + continue for sub_dir in dirs: convert_folder(sub_dir) diff --git a/utilities/transform/dashboard_transform.py b/utilities/transform/dashboard_transform.py deleted file mode 100644 index 075dc43..0000000 --- a/utilities/transform/dashboard_transform.py +++ /dev/null @@ -1 +0,0 @@ -import subprocess \ No newline at end of file diff --git a/utilities/transform/split_datasource.py b/utilities/transform/split_datasource.py index 0f72841..7d18f75 100644 --- a/utilities/transform/split_datasource.py +++ b/utilities/transform/split_datasource.py @@ -39,49 +39,64 @@ def find_match(dash_str): return False, False, "" -# Depth First Search to find any matching measurement and back up the tree to update the nearest parent's "datasource" -# property to the matching split database +def replace(dash_obj, is_matched, ds_matched): + if dash_obj and is_matched: + if DS_KEY in dash_obj.keys() and dash_obj[DS_KEY] == DS_ORIGINAL: + dash_obj[DS_KEY] = ds_matched + return True, False, "" + return False, is_matched, ds_matched + + def split_datasource_json(dash_obj): + """ + Depth First Search to find any matching measurement and back up the tree to update the nearest parent's "datasource" + property to the matching split database. + + Returns: + is_updated: if any node gets updated, it bubbles up to root so the dashboard json file will be rewritten + is_matched: notifies each node's parent if there's a match to replace. + ds_matched: the name of the matching datasource to replace with. + """ is_updated: bool = False is_matched: bool = False ds_matched: str = "" - if type(dash_obj) == list: - for ls in dash_obj: - temp_is_updated, is_matched, ds_matched = split_datasource_json(ls) + if dash_obj: + if type(dash_obj) == list: + for ls in dash_obj: + temp_is_updated, is_matched, ds_matched = split_datasource_json(ls) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == dict: + # depth first + for key, value in dash_obj.items(): + temp_is_updated, temp_is_matched, temp_ds_matched = split_datasource_json(value) + if temp_is_updated: + is_updated = True + if temp_is_matched: + is_matched = True + ds_matched = temp_ds_matched + # update datasource + temp_is_updated, is_matched, ds_matched = replace(dash_obj, is_matched, ds_matched) + if temp_is_updated: + is_updated = True + elif type(dash_obj) == str: + temp_is_updated, is_matched, ds_matched = find_match(dash_obj) if temp_is_updated: is_updated = True - elif type(dash_obj) == dict: - # depth first - for key, value in dash_obj.items(): - temp_is_updated, temp_is_matched, temp_ds_matched = split_datasource_json(value) - if temp_is_updated: - is_updated = True - if temp_is_matched: - is_matched = True - ds_matched = temp_ds_matched - # update datasource - if is_matched and DS_KEY in dash_obj.keys() and dash_obj[DS_KEY] == DS_ORIGINAL: - dash_obj[DS_KEY] = ds_matched - is_matched = False - ds_matched = "" - is_updated = True - elif type(dash_obj) == str: - return find_match(dash_obj) - else: - pass # do nothing with other types: int, float, bool, None + else: + pass # do nothing with other types: int, float, bool, None return is_updated, is_matched, ds_matched def split_datasource_file(file): is_updated = False - print(file) with open(file, "r") as dash_json: dash_obj = json.load(dash_json) for prop in SEARCH_PROPERTIES: - temp_is_updated, temp_is_matched, temp_ds_matched = split_datasource_json(dash_obj[prop]) + temp_is_updated, is_matched, ds_matched = split_datasource_json(dash_obj[prop]) if temp_is_updated: - is_updated = temp_is_updated + is_updated = True if is_updated: with open(str(file)[:file.index(".json")] + OUTFILE_EXTENSION, "w") as f: json.dump(dash_obj, f, indent=4) @@ -90,8 +105,11 @@ def split_datasource_file(file): def split_datasource_folder(folder): for root, dirs, files in os.walk(folder): for file in files: - if str(file).endswith(".json"): - split_datasource_file(root + "/" + file) + try: + if str(file).endswith(".json"): + split_datasource_file(root + "/" + file) + except Exception: + continue for sub_dir in dirs: split_datasource_folder(sub_dir) From 17f109148169c2c4f9448b6b417b277206089201 Mon Sep 17 00:00:00 2001 From: Marina Zheng Date: Sun, 23 Jul 2023 22:13:57 -0700 Subject: [PATCH 5/6] Update to remove cookie file after all curl commands are done. --- utilities/transfer/preference_export.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utilities/transfer/preference_export.sh b/utilities/transfer/preference_export.sh index 08bbb34..38c79a4 100755 --- a/utilities/transfer/preference_export.sh +++ b/utilities/transfer/preference_export.sh @@ -127,8 +127,6 @@ init pref_org_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/graph-engine/api/org/preferences") pref_user_json=$(curl --noproxy '*' -k $CURL_COOKIE "$HOST/graph-engine/api/user/preferences") -rm mycookie - # log result if [ -z "$pref_org_json" ] || [ -z "$pref_user_json" ]; then log_failure "Failed to download preferences. Please check parameters passed in. " @@ -154,6 +152,7 @@ GRAFANA_VERSION="grafana_$(echo $grafana_json | jq '.version' | tr -d '"')" DATE_TIME="date_$(date '+%d%m%Y_%H%M%S')" PREF_FILE_ZIP="${PREF_FOLDER}_${TARGET_HOST_IP}_${CCLEAR_VERSION}_${GRAFANA_VERSION}_${DATE_TIME}" zip -r -m "${PREF_FILE_ZIP}.zip" "${PREF_FOLDER}" +rm mycookie log_title "Preferences were saved in ${PREF_FILE_ZIP}"; log_title "------------------------------ FINISHED ---------------------------------"; From fccb8fc5dfc313528ceb767fcbd84063eb7a0f6a Mon Sep 17 00:00:00 2001 From: Marina Zheng Date: Fri, 18 Aug 2023 17:02:21 -0700 Subject: [PATCH 6/6] Remove the unzipped folder after import, otherwise it would reimport on the next interation. --- utilities/transfer/dashboard_import.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/utilities/transfer/dashboard_import.sh b/utilities/transfer/dashboard_import.sh index a64dc6b..40c2e63 100755 --- a/utilities/transfer/dashboard_import.sh +++ b/utilities/transfer/dashboard_import.sh @@ -198,6 +198,7 @@ for DASH_FILE in $DASH_LIST; do done rm mycookie +rm -rf "$DASH_DIR" log_title "Import complete. $NUMSUCCESS dashboards were successfully imported. $NUMFAILURE dashboard imports failed."; log_title "-------------------------------------FINISHED----------------------------------------";