diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/README.md b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/README.md index 1ccccffa..2eff4dbe 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/README.md +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/README.md @@ -6,6 +6,16 @@ Prerequisites: Note: List of the commands executed during the data upgrade can be found in upgrade_commands.txt. One command per line. Commands using data-uploader.py script cannot be executed after it is successfully executed once. It should be commented for the next execution(upgrade.sh ignores the commented lines). +Authentication: +-> The scripts authenticate via /v1/authmanager/authenticate/clientidsecretkey using client-id/secret credentials. Set the following properties in upgrade.properties before running: + - AUTH_APP_ID : app id for authentication (default: admin) + - CLIENT_ID : auth client id + - CLIENT_SECRET_KEY : auth client secret key +-> Do NOT hardcode the secret in the scripts; it stays in upgrade.properties (out of source control). +-> The client secret is NOT passed as a command-line argument (which would expose it in process listings / ps). upgrade.sh exports the properties as environment variables, and the scripts read CLIENT_SECRET_KEY from the environment. AUTH_APP_ID and CLIENT_ID (non-sensitive) are still passed as arguments. +-> If you run a script directly (without upgrade.sh), export the secret first: `export CLIENT_SECRET_KEY=...` +-> The configured client must have GLOBAL_ADMIN and REGISTRATION_ADMIN roles, since the uispec/publish and bulkupload endpoints require them. + 1. Migration of dynamic field table data. Dynamic value was stored as jsonarray in version 1.1.5*, now in 1.2.0.1 we store it as json object. one entry for each language of the field. diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py index 6b3074fb..48c1b8b5 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py @@ -10,6 +10,7 @@ import requests import json import sys +import os import time import psycopg2 from psycopg2 import sql @@ -29,15 +30,25 @@ parser.add_argument("--dbpassword", type=str, required=False, help="DB username") parser.add_argument("--dbhost", type=str, required=False, help="DB hostname") parser.add_argument("--dbport", type=str, required=False, help="DB port number") +parser.add_argument("--appId", type=str, required=False, default="admin", help="App ID for client-id/secret authentication, eg: admin") +parser.add_argument("--clientId", type=str, required=True, help="Client ID for client-id/secret authentication") +# NOTE: the client secret is intentionally NOT a command-line argument. Passing it +# on argv would expose it in process listings (ps/argv). It is read from the +# CLIENT_SECRET_KEY environment variable instead (see getAccessToken below). args = parser.parse_args() ## Values to be updated as per the deployment -authURL='https://'+args.domain+'/v1/authmanager/authenticate/useridPwd' +authURL='https://'+args.domain+'/v1/authmanager/authenticate/clientidsecretkey' uploadURL='https://'+args.domain+'/v1/admin/bulkupload' uploadStatusURL='https://'+args.domain+'/v1/admin/bulkupload/transcation/' username=args.username password=args.password +appId=args.appId +clientId=args.clientId +secretKey=os.environ.get("CLIENT_SECRET_KEY") +if not secretKey: + sys.exit("CLIENT_SECRET_KEY environment variable is not set. Set it in upgrade.properties (upgrade.sh exports it) and re-run.") def getCurrentDateTime(): dt_now = datetime.now(timezone.utc) @@ -114,14 +125,19 @@ def getAccessToken(): 'id': 'string', 'metadata': {}, 'request': { - 'appId': 'admin', - 'password': password, - 'userName': username + 'appId': appId, + 'clientId': clientId, + 'secretKey': secretKey }, 'requesttime': getCurrentDateTime(), 'version': 'string' } - authresponse=requests.post(authURL, json= auth_req_data) + try: + authresponse=requests.post(authURL, json= auth_req_data, timeout=30) + except requests.exceptions.RequestException as e: + sys.exit("Authentication request failed (network error). URL: " + authURL + " Error: " + str(e)) + if authresponse.status_code != 200 or 'authorization' not in authresponse.headers: + sys.exit("Authentication failed (HTTP " + str(authresponse.status_code) + "). URL: " + authURL + " Response: " + authresponse.text) print(json.dumps(authresponse.json())) return authresponse.headers["authorization"] @@ -133,7 +149,7 @@ def uploadFile(): data = {'category': 'masterdata', 'operation': args.operation, 'tableName': args.table} files = {'files': open(args.file, 'rb')} - uploadResponse = requests.post(uploadURL, data=data, files=files, headers=req_headers, verify=True) + uploadResponse = requests.post(uploadURL, data=data, files=files, headers=req_headers, verify=True, timeout=60) uploadResponse_json = uploadResponse.json() response = uploadResponse_json['response'] print(json.dumps(uploadResponse_json)) @@ -141,7 +157,7 @@ def uploadFile(): def getTransactionStatus(transactionId): - statusResponse = requests.get(uploadStatusURL+transactionId, headers=req_headers, verify=True) + statusResponse = requests.get(uploadStatusURL+transactionId, headers=req_headers, verify=True, timeout=30) statusResponse_json = statusResponse.json() response = statusResponse_json['response'] return response diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py index ead4b3a3..17f0512a 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py @@ -10,6 +10,7 @@ import requests import json import sys +import os parser = argparse.ArgumentParser(description='This is UI spec migration script. Migrates 1.1.5.5 UI spec to 1.2.0 compatible SPEC and the same is published to the server. This script should be executed after DB upgrade and 1.2.0.* masterdata-service deployment.') parser.add_argument("-d", "--domain", type=str, required=True, help="Server domain name, eg: dev.mosip.net") @@ -21,12 +22,17 @@ parser.add_argument("--ageGroupConfig", type=str, required=True, help="Age group configuration") parser.add_argument("--infantAgeGroup", type=str, required=True, help="Infant Age group name") parser.add_argument("--allowedBioAttributes", type=str, required=True, help="Comma separated list of allowed biometric attributes") +parser.add_argument("--appId", type=str, required=False, default="admin", help="App ID for client-id/secret authentication, eg: admin") +parser.add_argument("--clientId", type=str, required=True, help="Client ID for client-id/secret authentication") +# NOTE: the client secret is intentionally NOT a command-line argument. Passing it +# on argv would expose it in process listings (ps/argv). It is read from the +# CLIENT_SECRET_KEY environment variable instead (see getAccessToken below). args = parser.parse_args() ## Values to be updated as per the deployment -authURL='https://'+args.domain+'/v1/authmanager/authenticate/useridPwd' +authURL='https://'+args.domain+'/v1/authmanager/authenticate/clientidsecretkey' schemaURL='https://'+args.domain+'/v1/syncdata/latestidschema?schemaVersion=0' uispecURL='https://'+args.domain+'/v1/masterdata/uispec' uispecPublishURL='https://'+args.domain+'/v1/masterdata/uispec/publish' @@ -34,6 +40,11 @@ secondaryLang=args.secondaryLanguage username=args.username password=args.password +appId=args.appId +clientId=args.clientId +secretKey=os.environ.get("CLIENT_SECRET_KEY") +if not secretKey: + sys.exit("CLIENT_SECRET_KEY environment variable is not set. Set it in upgrade.properties (upgrade.sh exports it) and re-run.") agegroup_config=args.ageGroupConfig infantAgeGroup = args.infantAgeGroup.strip() allBioAttributes= args.allowedBioAttributes.strip().split(",") @@ -124,15 +135,20 @@ def getAccessToken(): 'id': 'string', 'metadata': {}, 'request': { - 'appId': 'admin', - 'password': password, - 'userName': username + 'appId': appId, + 'clientId': clientId, + 'secretKey': secretKey }, 'requesttime': getCurrentDateTime(), 'version': 'string' } - authresponse=requests.post(authURL, json= auth_req_data) + try: + authresponse=requests.post(authURL, json= auth_req_data, timeout=30) + except requests.exceptions.RequestException as e: + sys.exit("Authentication request failed (network error). URL: " + authURL + " Error: " + str(e)) print(authresponse) + if authresponse.status_code != 200 or 'authorization' not in authresponse.headers: + sys.exit("Authentication failed (HTTP " + str(authresponse.status_code) + "). URL: " + authURL + " Response: " + authresponse.text) return authresponse.headers["authorization"] @@ -156,7 +172,7 @@ def publish_spec(domain, spec_type, specjson): "jsonspec": json.loads(spec) } } - spec_resp = requests.post(uispecURL, json=request_json, headers=req_headers) + spec_resp = requests.post(uispecURL, json=request_json, headers=req_headers, timeout=30) spec_resp_json = spec_resp.json() #print("UI spec POST response : " + json.dumps(spec_resp_json)) @@ -179,7 +195,7 @@ def publish_spec(domain, spec_type, specjson): } } print("UI spec publish request : " + json.dumps(publish_spec_req)) - publish_resp = requests.put(uispecPublishURL, json=publish_spec_req, headers=req_headers) + publish_resp = requests.put(uispecPublishURL, json=publish_spec_req, headers=req_headers, timeout=30) publish_resp_json = publish_resp.json() print("UI spec published : " + json.dumps(publish_resp_json)) @@ -728,10 +744,12 @@ def buildLostRegistrationSpec(demographic_fields, document_fields, biometric_fie # invoke syncdata service with authtoken in headers req_headers={'Cookie' : 'Authorization='+getAccessToken()} -get_schema_resp=requests.get(schemaURL, headers=req_headers) +get_schema_resp=requests.get(schemaURL, headers=req_headers, timeout=30) print(get_schema_resp) schema_resp_json=get_schema_resp.json() schema_resp=schema_resp_json['response'] +if schema_resp is None: + sys.exit("Failed to fetch identity schema from " + schemaURL + ". Response: " + json.dumps(schema_resp_json)) identity_schema_id=schema_resp['id'] cur_schema=schema_resp['schema'] domain='registration-client' @@ -763,7 +781,7 @@ def buildLostRegistrationSpec(demographic_fields, document_fields, biometric_fie #set all the required field mappings -response = requests.get(args.identityMappingJsonUrl) +response = requests.get(args.identityMappingJsonUrl, timeout=30) data = json.loads(response.text) identity_mapping_json = data['identity'] if(identity_mapping_json.get('individualBiometrics') != None): diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.properties b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.properties index b3d06187..343d139f 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.properties +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.properties @@ -6,6 +6,9 @@ ACTION=upgrade UPGRADE_DOMAIN_NAME= GLOBAL_ADMIN_USER= GLOBAL_ADMIN_USER_PWD= +AUTH_APP_ID=admin +CLIENT_ID= +CLIENT_SECRET_KEY= PRIMARY_LANGUAGE_CODE= SECONDARY_LANGUAGE_CODE= GENDER_DYNAMIC_FIELD_NAME=gender diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.sh b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.sh index b1d8306f..3ee2014e 100755 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.sh +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.sh @@ -11,6 +11,7 @@ then key=$(echo "$key" | tr '.' '_') if [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then printf -v "$key" '%s' "$value" + export "$key" else echo "WARNING: Skipping invalid property key: $key" fi diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade_commands.txt b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade_commands.txt index cbf37391..369dfa47 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade_commands.txt +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade_commands.txt @@ -1,7 +1,7 @@ -python3 migration-ui_spec.py -d "$UPGRADE_DOMAIN_NAME" -u "$GLOBAL_ADMIN_USER" -p "$GLOBAL_ADMIN_USER_PWD" -pl "$PRIMARY_LANGUAGE_CODE" -sl "$SECONDARY_LANGUAGE_CODE" --identityMappingJsonUrl "$IDENTITY_MAPPING_JSON_URL" --ageGroupConfig "$AGEGROUP_CONFIG" --infantAgeGroup "$INFANT_AGEGROUP" --allowedBioAttributes "$ALLOWED_BIO_ATTRIBUTES" +python3 migration-ui_spec.py -d "$UPGRADE_DOMAIN_NAME" -u "$GLOBAL_ADMIN_USER" -p "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" -pl "$PRIMARY_LANGUAGE_CODE" -sl "$SECONDARY_LANGUAGE_CODE" --identityMappingJsonUrl "$IDENTITY_MAPPING_JSON_URL" --ageGroupConfig "$AGEGROUP_CONFIG" --infantAgeGroup "$INFANT_AGEGROUP" --allowedBioAttributes "$ALLOWED_BIO_ATTRIBUTES" python3 migration-dynamicfield.py "$SU_USER" "$SU_USER_PWD" "$DB_SERVERIP" "$DB_PORT" "$GENDER_DYNAMIC_FIELD_NAME" "$INDIVIDUAL_TYPE_DYNAMIC_FIELD_NAME" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "template_type" --operation "Insert" --autogen 0 --file "template_type_delta.xlsx" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "template" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname Sheet1 --idcolumn A --autogen 1 --file "template_delta.xlsx" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "machine_type" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_type --autogen 0 --file "machine_type_delta.xlsx" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "machine_spec" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_spec --autogen 0 --file "machine_spec_delta.xlsx" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "zone_user" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname zone_user --autogen 0 --file "zone_user_delta.xlsx" \ No newline at end of file +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "template_type" --operation "Insert" --autogen 0 --file "template_type_delta.xlsx" +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "template" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname Sheet1 --idcolumn A --autogen 1 --file "template_delta.xlsx" +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "machine_type" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_type --autogen 0 --file "machine_type_delta.xlsx" +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "machine_spec" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_spec --autogen 0 --file "machine_spec_delta.xlsx" +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "zone_user" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname zone_user --autogen 0 --file "zone_user_delta.xlsx" \ No newline at end of file