From cd7a1266750152f9bffebc75b9ba39181fe39152 Mon Sep 17 00:00:00 2001 From: kameshsr Date: Mon, 22 Jun 2026 11:33:31 +0530 Subject: [PATCH 1/5] =?UTF-8?q?MOSIP-39952=20fix(data-upgrade):=20use=20cl?= =?UTF-8?q?ientidsecretkey=20auth=20for=201.1.5.5=E2=86=921.2.0.1=20migrat?= =?UTF-8?q?ion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: kameshsr --- .../1.1.5.5_to_1.2.0.1/data-uploader.py | 16 ++++++++++++---- .../1.1.5.5_to_1.2.0.1/migration-ui_spec.py | 18 ++++++++++++++---- .../1.1.5.5_to_1.2.0.1/upgrade.properties | 3 +++ .../1.1.5.5_to_1.2.0.1/upgrade_commands.txt | 12 ++++++------ 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py index 6b3074fb..1765c539 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py @@ -29,15 +29,21 @@ parser.add_argument("--dbpassword", type=str, required=False, help="DB username") parser.add_argument("--dbhost", type=str, required=False, help="DB hostname") parser.add_argument("--dbport", type=str, required=False, help="DB port number") +parser.add_argument("--appId", type=str, required=False, default="admin", help="App ID for client-id/secret authentication, eg: admin") +parser.add_argument("--clientId", type=str, required=True, help="Client ID for client-id/secret authentication") +parser.add_argument("--secretKey", type=str, required=True, help="Client secret key for client-id/secret authentication") args = parser.parse_args() ## Values to be updated as per the deployment -authURL='https://'+args.domain+'/v1/authmanager/authenticate/useridPwd' +authURL='https://'+args.domain+'/v1/authmanager/authenticate/clientidsecretkey' uploadURL='https://'+args.domain+'/v1/admin/bulkupload' uploadStatusURL='https://'+args.domain+'/v1/admin/bulkupload/transcation/' username=args.username password=args.password +appId=args.appId +clientId=args.clientId +secretKey=args.secretKey def getCurrentDateTime(): dt_now = datetime.now(timezone.utc) @@ -114,15 +120,17 @@ def getAccessToken(): 'id': 'string', 'metadata': {}, 'request': { - 'appId': 'admin', - 'password': password, - 'userName': username + 'appId': appId, + 'clientId': clientId, + 'secretKey': secretKey }, 'requesttime': getCurrentDateTime(), 'version': 'string' } authresponse=requests.post(authURL, json= auth_req_data) print(json.dumps(authresponse.json())) + if authresponse.status_code != 200 or 'authorization' not in authresponse.headers: + sys.exit("Authentication failed (HTTP " + str(authresponse.status_code) + "). URL: " + authURL + " Response: " + authresponse.text) return authresponse.headers["authorization"] diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py index ead4b3a3..e96bbbd6 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py @@ -21,12 +21,15 @@ parser.add_argument("--ageGroupConfig", type=str, required=True, help="Age group configuration") parser.add_argument("--infantAgeGroup", type=str, required=True, help="Infant Age group name") parser.add_argument("--allowedBioAttributes", type=str, required=True, help="Comma separated list of allowed biometric attributes") +parser.add_argument("--appId", type=str, required=False, default="admin", help="App ID for client-id/secret authentication, eg: admin") +parser.add_argument("--clientId", type=str, required=True, help="Client ID for client-id/secret authentication") +parser.add_argument("--secretKey", type=str, required=True, help="Client secret key for client-id/secret authentication") args = parser.parse_args() ## Values to be updated as per the deployment -authURL='https://'+args.domain+'/v1/authmanager/authenticate/useridPwd' +authURL='https://'+args.domain+'/v1/authmanager/authenticate/clientidsecretkey' schemaURL='https://'+args.domain+'/v1/syncdata/latestidschema?schemaVersion=0' uispecURL='https://'+args.domain+'/v1/masterdata/uispec' uispecPublishURL='https://'+args.domain+'/v1/masterdata/uispec/publish' @@ -34,6 +37,9 @@ secondaryLang=args.secondaryLanguage username=args.username password=args.password +appId=args.appId +clientId=args.clientId +secretKey=args.secretKey agegroup_config=args.ageGroupConfig infantAgeGroup = args.infantAgeGroup.strip() allBioAttributes= args.allowedBioAttributes.strip().split(",") @@ -124,15 +130,17 @@ def getAccessToken(): 'id': 'string', 'metadata': {}, 'request': { - 'appId': 'admin', - 'password': password, - 'userName': username + 'appId': appId, + 'clientId': clientId, + 'secretKey': secretKey }, 'requesttime': getCurrentDateTime(), 'version': 'string' } authresponse=requests.post(authURL, json= auth_req_data) print(authresponse) + if authresponse.status_code != 200 or 'authorization' not in authresponse.headers: + sys.exit("Authentication failed (HTTP " + str(authresponse.status_code) + "). URL: " + authURL + " Response: " + authresponse.text) return authresponse.headers["authorization"] @@ -732,6 +740,8 @@ def buildLostRegistrationSpec(demographic_fields, document_fields, biometric_fie print(get_schema_resp) schema_resp_json=get_schema_resp.json() schema_resp=schema_resp_json['response'] +if schema_resp is None: + sys.exit("Failed to fetch identity schema from " + schemaURL + ". Response: " + json.dumps(schema_resp_json)) identity_schema_id=schema_resp['id'] cur_schema=schema_resp['schema'] domain='registration-client' diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.properties b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.properties index b3d06187..343d139f 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.properties +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.properties @@ -6,6 +6,9 @@ ACTION=upgrade UPGRADE_DOMAIN_NAME= GLOBAL_ADMIN_USER= GLOBAL_ADMIN_USER_PWD= +AUTH_APP_ID=admin +CLIENT_ID= +CLIENT_SECRET_KEY= PRIMARY_LANGUAGE_CODE= SECONDARY_LANGUAGE_CODE= GENDER_DYNAMIC_FIELD_NAME=gender diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade_commands.txt b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade_commands.txt index cbf37391..06c6145c 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade_commands.txt +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade_commands.txt @@ -1,7 +1,7 @@ -python3 migration-ui_spec.py -d "$UPGRADE_DOMAIN_NAME" -u "$GLOBAL_ADMIN_USER" -p "$GLOBAL_ADMIN_USER_PWD" -pl "$PRIMARY_LANGUAGE_CODE" -sl "$SECONDARY_LANGUAGE_CODE" --identityMappingJsonUrl "$IDENTITY_MAPPING_JSON_URL" --ageGroupConfig "$AGEGROUP_CONFIG" --infantAgeGroup "$INFANT_AGEGROUP" --allowedBioAttributes "$ALLOWED_BIO_ATTRIBUTES" +python3 migration-ui_spec.py -d "$UPGRADE_DOMAIN_NAME" -u "$GLOBAL_ADMIN_USER" -p "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --secretKey "$CLIENT_SECRET_KEY" -pl "$PRIMARY_LANGUAGE_CODE" -sl "$SECONDARY_LANGUAGE_CODE" --identityMappingJsonUrl "$IDENTITY_MAPPING_JSON_URL" --ageGroupConfig "$AGEGROUP_CONFIG" --infantAgeGroup "$INFANT_AGEGROUP" --allowedBioAttributes "$ALLOWED_BIO_ATTRIBUTES" python3 migration-dynamicfield.py "$SU_USER" "$SU_USER_PWD" "$DB_SERVERIP" "$DB_PORT" "$GENDER_DYNAMIC_FIELD_NAME" "$INDIVIDUAL_TYPE_DYNAMIC_FIELD_NAME" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "template_type" --operation "Insert" --autogen 0 --file "template_type_delta.xlsx" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "template" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname Sheet1 --idcolumn A --autogen 1 --file "template_delta.xlsx" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "machine_type" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_type --autogen 0 --file "machine_type_delta.xlsx" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "machine_spec" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_spec --autogen 0 --file "machine_spec_delta.xlsx" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "zone_user" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname zone_user --autogen 0 --file "zone_user_delta.xlsx" \ No newline at end of file +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --secretKey "$CLIENT_SECRET_KEY" --table "template_type" --operation "Insert" --autogen 0 --file "template_type_delta.xlsx" +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --secretKey "$CLIENT_SECRET_KEY" --table "template" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname Sheet1 --idcolumn A --autogen 1 --file "template_delta.xlsx" +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --secretKey "$CLIENT_SECRET_KEY" --table "machine_type" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_type --autogen 0 --file "machine_type_delta.xlsx" +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --secretKey "$CLIENT_SECRET_KEY" --table "machine_spec" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_spec --autogen 0 --file "machine_spec_delta.xlsx" +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --secretKey "$CLIENT_SECRET_KEY" --table "zone_user" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname zone_user --autogen 0 --file "zone_user_delta.xlsx" \ No newline at end of file From 6d89bfa321fcfc91374a0a7f7972e6b716e4ae2b Mon Sep 17 00:00:00 2001 From: kameshsr Date: Mon, 22 Jun 2026 11:39:20 +0530 Subject: [PATCH 2/5] MOSIP-39952 Corrected Readme.md Signed-off-by: kameshsr --- mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/README.md b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/README.md index 1ccccffa..1c531457 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/README.md +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/README.md @@ -6,6 +6,14 @@ Prerequisites: Note: List of the commands executed during the data upgrade can be found in upgrade_commands.txt. One command per line. Commands using data-uploader.py script cannot be executed after it is successfully executed once. It should be commented for the next execution(upgrade.sh ignores the commented lines). +Authentication: +-> The scripts authenticate via /v1/authmanager/authenticate/clientidsecretkey using client-id/secret credentials. Set the following properties in upgrade.properties before running: + - AUTH_APP_ID : app id for authentication (default: admin) + - CLIENT_ID : auth client id + - CLIENT_SECRET_KEY : auth client secret key +-> Do NOT hardcode the secret in the scripts; it is read from upgrade.properties so it stays out of source control. +-> The configured client must have GLOBAL_ADMIN and REGISTRATION_ADMIN roles, since the uispec/publish and bulkupload endpoints require them. + 1. Migration of dynamic field table data. Dynamic value was stored as jsonarray in version 1.1.5*, now in 1.2.0.1 we store it as json object. one entry for each language of the field. From 7827f05128ba4fbcfca45c7b83e0e42331999227 Mon Sep 17 00:00:00 2001 From: kameshsr Date: Mon, 22 Jun 2026 12:13:45 +0530 Subject: [PATCH 3/5] fix(data-upgrade): add timeouts to all HTTP calls and harden auth network error handling Signed-off-by: kameshsr --- .../1.1.5.5_to_1.2.0.1/data-uploader.py | 9 ++++++--- .../1.1.5.5_to_1.2.0.1/migration-ui_spec.py | 13 ++++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py index 1765c539..3f44d83f 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py @@ -127,7 +127,10 @@ def getAccessToken(): 'requesttime': getCurrentDateTime(), 'version': 'string' } - authresponse=requests.post(authURL, json= auth_req_data) + try: + authresponse=requests.post(authURL, json= auth_req_data, timeout=30) + except requests.exceptions.RequestException as e: + sys.exit("Authentication request failed (network error). URL: " + authURL + " Error: " + str(e)) print(json.dumps(authresponse.json())) if authresponse.status_code != 200 or 'authorization' not in authresponse.headers: sys.exit("Authentication failed (HTTP " + str(authresponse.status_code) + "). URL: " + authURL + " Response: " + authresponse.text) @@ -141,7 +144,7 @@ def uploadFile(): data = {'category': 'masterdata', 'operation': args.operation, 'tableName': args.table} files = {'files': open(args.file, 'rb')} - uploadResponse = requests.post(uploadURL, data=data, files=files, headers=req_headers, verify=True) + uploadResponse = requests.post(uploadURL, data=data, files=files, headers=req_headers, verify=True, timeout=60) uploadResponse_json = uploadResponse.json() response = uploadResponse_json['response'] print(json.dumps(uploadResponse_json)) @@ -149,7 +152,7 @@ def uploadFile(): def getTransactionStatus(transactionId): - statusResponse = requests.get(uploadStatusURL+transactionId, headers=req_headers, verify=True) + statusResponse = requests.get(uploadStatusURL+transactionId, headers=req_headers, verify=True, timeout=30) statusResponse_json = statusResponse.json() response = statusResponse_json['response'] return response diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py index e96bbbd6..01b0a170 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py @@ -137,7 +137,10 @@ def getAccessToken(): 'requesttime': getCurrentDateTime(), 'version': 'string' } - authresponse=requests.post(authURL, json= auth_req_data) + try: + authresponse=requests.post(authURL, json= auth_req_data, timeout=30) + except requests.exceptions.RequestException as e: + sys.exit("Authentication request failed (network error). URL: " + authURL + " Error: " + str(e)) print(authresponse) if authresponse.status_code != 200 or 'authorization' not in authresponse.headers: sys.exit("Authentication failed (HTTP " + str(authresponse.status_code) + "). URL: " + authURL + " Response: " + authresponse.text) @@ -164,7 +167,7 @@ def publish_spec(domain, spec_type, specjson): "jsonspec": json.loads(spec) } } - spec_resp = requests.post(uispecURL, json=request_json, headers=req_headers) + spec_resp = requests.post(uispecURL, json=request_json, headers=req_headers, timeout=30) spec_resp_json = spec_resp.json() #print("UI spec POST response : " + json.dumps(spec_resp_json)) @@ -187,7 +190,7 @@ def publish_spec(domain, spec_type, specjson): } } print("UI spec publish request : " + json.dumps(publish_spec_req)) - publish_resp = requests.put(uispecPublishURL, json=publish_spec_req, headers=req_headers) + publish_resp = requests.put(uispecPublishURL, json=publish_spec_req, headers=req_headers, timeout=30) publish_resp_json = publish_resp.json() print("UI spec published : " + json.dumps(publish_resp_json)) @@ -736,7 +739,7 @@ def buildLostRegistrationSpec(demographic_fields, document_fields, biometric_fie # invoke syncdata service with authtoken in headers req_headers={'Cookie' : 'Authorization='+getAccessToken()} -get_schema_resp=requests.get(schemaURL, headers=req_headers) +get_schema_resp=requests.get(schemaURL, headers=req_headers, timeout=30) print(get_schema_resp) schema_resp_json=get_schema_resp.json() schema_resp=schema_resp_json['response'] @@ -773,7 +776,7 @@ def buildLostRegistrationSpec(demographic_fields, document_fields, biometric_fie #set all the required field mappings -response = requests.get(args.identityMappingJsonUrl) +response = requests.get(args.identityMappingJsonUrl, timeout=30) data = json.loads(response.text) identity_mapping_json = data['identity'] if(identity_mapping_json.get('individualBiometrics') != None): From e678247e3bea3c9ec272594e2fe8e9e930dc7081 Mon Sep 17 00:00:00 2001 From: kameshsr Date: Mon, 22 Jun 2026 12:16:23 +0530 Subject: [PATCH 4/5] MOSIP-39952 fix(data-upgrade): guard JSON parse behind auth success check Signed-off-by: kameshsr --- mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py index 3f44d83f..5a6a4645 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py @@ -131,9 +131,9 @@ def getAccessToken(): authresponse=requests.post(authURL, json= auth_req_data, timeout=30) except requests.exceptions.RequestException as e: sys.exit("Authentication request failed (network error). URL: " + authURL + " Error: " + str(e)) - print(json.dumps(authresponse.json())) if authresponse.status_code != 200 or 'authorization' not in authresponse.headers: sys.exit("Authentication failed (HTTP " + str(authresponse.status_code) + "). URL: " + authURL + " Response: " + authresponse.text) + print(json.dumps(authresponse.json())) return authresponse.headers["authorization"] From 11edee62b1d5d34ce0e7725cc8c2ed28e42acb02 Mon Sep 17 00:00:00 2001 From: kameshsr Date: Mon, 22 Jun 2026 12:23:49 +0530 Subject: [PATCH 5/5] MOSIP-39952 fix(data-upgrade): read client secret from env instead of argv Signed-off-by: kameshsr --- .../data_upgrade/1.1.5.5_to_1.2.0.1/README.md | 4 +++- .../data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py | 9 +++++++-- .../1.1.5.5_to_1.2.0.1/migration-ui_spec.py | 9 +++++++-- .../data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.sh | 1 + .../1.1.5.5_to_1.2.0.1/upgrade_commands.txt | 12 ++++++------ 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/README.md b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/README.md index 1c531457..2eff4dbe 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/README.md +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/README.md @@ -11,7 +11,9 @@ Authentication: - AUTH_APP_ID : app id for authentication (default: admin) - CLIENT_ID : auth client id - CLIENT_SECRET_KEY : auth client secret key --> Do NOT hardcode the secret in the scripts; it is read from upgrade.properties so it stays out of source control. +-> Do NOT hardcode the secret in the scripts; it stays in upgrade.properties (out of source control). +-> The client secret is NOT passed as a command-line argument (which would expose it in process listings / ps). upgrade.sh exports the properties as environment variables, and the scripts read CLIENT_SECRET_KEY from the environment. AUTH_APP_ID and CLIENT_ID (non-sensitive) are still passed as arguments. +-> If you run a script directly (without upgrade.sh), export the secret first: `export CLIENT_SECRET_KEY=...` -> The configured client must have GLOBAL_ADMIN and REGISTRATION_ADMIN roles, since the uispec/publish and bulkupload endpoints require them. diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py index 5a6a4645..48c1b8b5 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py @@ -10,6 +10,7 @@ import requests import json import sys +import os import time import psycopg2 from psycopg2 import sql @@ -31,7 +32,9 @@ parser.add_argument("--dbport", type=str, required=False, help="DB port number") parser.add_argument("--appId", type=str, required=False, default="admin", help="App ID for client-id/secret authentication, eg: admin") parser.add_argument("--clientId", type=str, required=True, help="Client ID for client-id/secret authentication") -parser.add_argument("--secretKey", type=str, required=True, help="Client secret key for client-id/secret authentication") +# NOTE: the client secret is intentionally NOT a command-line argument. Passing it +# on argv would expose it in process listings (ps/argv). It is read from the +# CLIENT_SECRET_KEY environment variable instead (see getAccessToken below). args = parser.parse_args() @@ -43,7 +46,9 @@ password=args.password appId=args.appId clientId=args.clientId -secretKey=args.secretKey +secretKey=os.environ.get("CLIENT_SECRET_KEY") +if not secretKey: + sys.exit("CLIENT_SECRET_KEY environment variable is not set. Set it in upgrade.properties (upgrade.sh exports it) and re-run.") def getCurrentDateTime(): dt_now = datetime.now(timezone.utc) diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py index 01b0a170..17f0512a 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py @@ -10,6 +10,7 @@ import requests import json import sys +import os parser = argparse.ArgumentParser(description='This is UI spec migration script. Migrates 1.1.5.5 UI spec to 1.2.0 compatible SPEC and the same is published to the server. This script should be executed after DB upgrade and 1.2.0.* masterdata-service deployment.') parser.add_argument("-d", "--domain", type=str, required=True, help="Server domain name, eg: dev.mosip.net") @@ -23,7 +24,9 @@ parser.add_argument("--allowedBioAttributes", type=str, required=True, help="Comma separated list of allowed biometric attributes") parser.add_argument("--appId", type=str, required=False, default="admin", help="App ID for client-id/secret authentication, eg: admin") parser.add_argument("--clientId", type=str, required=True, help="Client ID for client-id/secret authentication") -parser.add_argument("--secretKey", type=str, required=True, help="Client secret key for client-id/secret authentication") +# NOTE: the client secret is intentionally NOT a command-line argument. Passing it +# on argv would expose it in process listings (ps/argv). It is read from the +# CLIENT_SECRET_KEY environment variable instead (see getAccessToken below). args = parser.parse_args() @@ -39,7 +42,9 @@ password=args.password appId=args.appId clientId=args.clientId -secretKey=args.secretKey +secretKey=os.environ.get("CLIENT_SECRET_KEY") +if not secretKey: + sys.exit("CLIENT_SECRET_KEY environment variable is not set. Set it in upgrade.properties (upgrade.sh exports it) and re-run.") agegroup_config=args.ageGroupConfig infantAgeGroup = args.infantAgeGroup.strip() allBioAttributes= args.allowedBioAttributes.strip().split(",") diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.sh b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.sh index b1d8306f..3ee2014e 100755 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.sh +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.sh @@ -11,6 +11,7 @@ then key=$(echo "$key" | tr '.' '_') if [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then printf -v "$key" '%s' "$value" + export "$key" else echo "WARNING: Skipping invalid property key: $key" fi diff --git a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade_commands.txt b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade_commands.txt index 06c6145c..369dfa47 100644 --- a/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade_commands.txt +++ b/mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade_commands.txt @@ -1,7 +1,7 @@ -python3 migration-ui_spec.py -d "$UPGRADE_DOMAIN_NAME" -u "$GLOBAL_ADMIN_USER" -p "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --secretKey "$CLIENT_SECRET_KEY" -pl "$PRIMARY_LANGUAGE_CODE" -sl "$SECONDARY_LANGUAGE_CODE" --identityMappingJsonUrl "$IDENTITY_MAPPING_JSON_URL" --ageGroupConfig "$AGEGROUP_CONFIG" --infantAgeGroup "$INFANT_AGEGROUP" --allowedBioAttributes "$ALLOWED_BIO_ATTRIBUTES" +python3 migration-ui_spec.py -d "$UPGRADE_DOMAIN_NAME" -u "$GLOBAL_ADMIN_USER" -p "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" -pl "$PRIMARY_LANGUAGE_CODE" -sl "$SECONDARY_LANGUAGE_CODE" --identityMappingJsonUrl "$IDENTITY_MAPPING_JSON_URL" --ageGroupConfig "$AGEGROUP_CONFIG" --infantAgeGroup "$INFANT_AGEGROUP" --allowedBioAttributes "$ALLOWED_BIO_ATTRIBUTES" python3 migration-dynamicfield.py "$SU_USER" "$SU_USER_PWD" "$DB_SERVERIP" "$DB_PORT" "$GENDER_DYNAMIC_FIELD_NAME" "$INDIVIDUAL_TYPE_DYNAMIC_FIELD_NAME" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --secretKey "$CLIENT_SECRET_KEY" --table "template_type" --operation "Insert" --autogen 0 --file "template_type_delta.xlsx" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --secretKey "$CLIENT_SECRET_KEY" --table "template" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname Sheet1 --idcolumn A --autogen 1 --file "template_delta.xlsx" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --secretKey "$CLIENT_SECRET_KEY" --table "machine_type" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_type --autogen 0 --file "machine_type_delta.xlsx" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --secretKey "$CLIENT_SECRET_KEY" --table "machine_spec" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_spec --autogen 0 --file "machine_spec_delta.xlsx" -python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --secretKey "$CLIENT_SECRET_KEY" --table "zone_user" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname zone_user --autogen 0 --file "zone_user_delta.xlsx" \ No newline at end of file +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "template_type" --operation "Insert" --autogen 0 --file "template_type_delta.xlsx" +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "template" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname Sheet1 --idcolumn A --autogen 1 --file "template_delta.xlsx" +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "machine_type" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_type --autogen 0 --file "machine_type_delta.xlsx" +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "machine_spec" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_spec --autogen 0 --file "machine_spec_delta.xlsx" +python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "zone_user" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname zone_user --autogen 0 --file "zone_user_delta.xlsx" \ No newline at end of file