MOSIP-39952 Fix authorization failure in 1.1.5.5→1.2.0.1 country data migration

mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/README.md

@@ -6,6 +6,16 @@ Prerequisites:
 
 Note: List of the commands executed during the data upgrade can be found in upgrade_commands.txt. One command per line. Commands using data-uploader.py script cannot be executed after it is successfully executed once. It should be commented for the next execution(upgrade.sh ignores the commented lines).
 
+Authentication:
+-> The scripts authenticate via /v1/authmanager/authenticate/clientidsecretkey using client-id/secret credentials. Set the following properties in upgrade.properties before running:
+   - AUTH_APP_ID        : app id for authentication (default: admin)
+   - CLIENT_ID          : auth client id
+   - CLIENT_SECRET_KEY  : auth client secret key
+-> Do NOT hardcode the secret in the scripts; it stays in upgrade.properties (out of source control).
+-> The client secret is NOT passed as a command-line argument (which would expose it in process listings / ps). upgrade.sh exports the properties as environment variables, and the scripts read CLIENT_SECRET_KEY from the environment. AUTH_APP_ID and CLIENT_ID (non-sensitive) are still passed as arguments.
+-> If you run a script directly (without upgrade.sh), export the secret first: `export CLIENT_SECRET_KEY=...`
+-> The configured client must have GLOBAL_ADMIN and REGISTRATION_ADMIN roles, since the uispec/publish and bulkupload endpoints require them.
+
 
 1. Migration of dynamic field table data.
 	Dynamic value was stored as jsonarray in version 1.1.5*, now in 1.2.0.1 we store it as json object. one entry for each language of the field.

mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/data-uploader.py

@@ -10,6 +10,7 @@
 import requests
 import json
 import sys
+import os
 import time
 import psycopg2
 from psycopg2 import sql
@@ -29,15 +30,25 @@
 parser.add_argument("--dbpassword", type=str, required=False, help="DB username")
 parser.add_argument("--dbhost", type=str, required=False, help="DB hostname")
 parser.add_argument("--dbport", type=str, required=False, help="DB port number")
+parser.add_argument("--appId", type=str, required=False, default="admin", help="App ID for client-id/secret authentication, eg: admin")
+parser.add_argument("--clientId", type=str, required=True, help="Client ID for client-id/secret authentication")
+# NOTE: the client secret is intentionally NOT a command-line argument. Passing it
+# on argv would expose it in process listings (ps/argv). It is read from the
+# CLIENT_SECRET_KEY environment variable instead (see getAccessToken below).
 
 args = parser.parse_args()
 
 ## Values to be updated as per the deployment
-authURL='https://'+args.domain+'/v1/authmanager/authenticate/useridPwd'
+authURL='https://'+args.domain+'/v1/authmanager/authenticate/clientidsecretkey'
 uploadURL='https://'+args.domain+'/v1/admin/bulkupload'
 uploadStatusURL='https://'+args.domain+'/v1/admin/bulkupload/transcation/'
 username=args.username
 password=args.password
+appId=args.appId
+clientId=args.clientId
+secretKey=os.environ.get("CLIENT_SECRET_KEY")
+if not secretKey:
+  sys.exit("CLIENT_SECRET_KEY environment variable is not set. Set it in upgrade.properties (upgrade.sh exports it) and re-run.")
 
 def getCurrentDateTime():
   dt_now = datetime.now(timezone.utc)
@@ -114,14 +125,19 @@ def getAccessToken():
     'id': 'string',
     'metadata': {},
     'request': {
-      'appId': 'admin',
-      'password': password,
-      'userName': username
+      'appId': appId,
+      'clientId': clientId,
+      'secretKey': secretKey
     },
     'requesttime': getCurrentDateTime(),
     'version': 'string'
   }
-  authresponse=requests.post(authURL, json= auth_req_data)
+  try:
+    authresponse=requests.post(authURL, json= auth_req_data, timeout=30)
+  except requests.exceptions.RequestException as e:
+    sys.exit("Authentication request failed (network error). URL: " + authURL + " Error: " + str(e))
+  if authresponse.status_code != 200 or 'authorization' not in authresponse.headers:
+    sys.exit("Authentication failed (HTTP " + str(authresponse.status_code) + "). URL: " + authURL + " Response: " + authresponse.text)
   print(json.dumps(authresponse.json()))
   return authresponse.headers["authorization"]
 
@@ -133,15 +149,15 @@ def uploadFile():
 
   data = {'category': 'masterdata', 'operation': args.operation, 'tableName': args.table}
   files = {'files': open(args.file, 'rb')}
-  uploadResponse = requests.post(uploadURL, data=data, files=files, headers=req_headers, verify=True)
+  uploadResponse = requests.post(uploadURL, data=data, files=files, headers=req_headers, verify=True, timeout=60)
   uploadResponse_json = uploadResponse.json()
   response = uploadResponse_json['response']
   print(json.dumps(uploadResponse_json))
   return response['transcationId']
 
 
 def getTransactionStatus(transactionId):
-  statusResponse = requests.get(uploadStatusURL+transactionId, headers=req_headers, verify=True)
+  statusResponse = requests.get(uploadStatusURL+transactionId, headers=req_headers, verify=True, timeout=30)
   statusResponse_json = statusResponse.json()
   response = statusResponse_json['response']
   return response

mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/migration-ui_spec.py

@@ -10,6 +10,7 @@
 import requests
 import json
 import sys
+import os
 
 parser = argparse.ArgumentParser(description='This is UI spec migration script. Migrates 1.1.5.5 UI spec to 1.2.0 compatible SPEC and the same is published to the server. This script should be executed after DB upgrade and 1.2.0.* masterdata-service deployment.')
 parser.add_argument("-d", "--domain", type=str, required=True, help="Server domain name, eg: dev.mosip.net")
@@ -21,19 +22,29 @@
 parser.add_argument("--ageGroupConfig", type=str, required=True, help="Age group configuration")
 parser.add_argument("--infantAgeGroup", type=str, required=True, help="Infant Age group name")
 parser.add_argument("--allowedBioAttributes", type=str, required=True, help="Comma separated list of allowed biometric attributes")
+parser.add_argument("--appId", type=str, required=False, default="admin", help="App ID for client-id/secret authentication, eg: admin")
+parser.add_argument("--clientId", type=str, required=True, help="Client ID for client-id/secret authentication")
+# NOTE: the client secret is intentionally NOT a command-line argument. Passing it
+# on argv would expose it in process listings (ps/argv). It is read from the
+# CLIENT_SECRET_KEY environment variable instead (see getAccessToken below).
 
 args = parser.parse_args()
 
 
 ## Values to be updated as per the deployment
-authURL='https://'+args.domain+'/v1/authmanager/authenticate/useridPwd'
+authURL='https://'+args.domain+'/v1/authmanager/authenticate/clientidsecretkey'
 schemaURL='https://'+args.domain+'/v1/syncdata/latestidschema?schemaVersion=0'
 uispecURL='https://'+args.domain+'/v1/masterdata/uispec'
 uispecPublishURL='https://'+args.domain+'/v1/masterdata/uispec/publish'
 primaryLang=args.primaryLanguage
 secondaryLang=args.secondaryLanguage
 username=args.username
 password=args.password
+appId=args.appId
+clientId=args.clientId
+secretKey=os.environ.get("CLIENT_SECRET_KEY")
+if not secretKey:
+	sys.exit("CLIENT_SECRET_KEY environment variable is not set. Set it in upgrade.properties (upgrade.sh exports it) and re-run.")
 agegroup_config=args.ageGroupConfig
 infantAgeGroup = args.infantAgeGroup.strip()
 allBioAttributes= args.allowedBioAttributes.strip().split(",")
@@ -124,15 +135,20 @@ def getAccessToken():
     'id': 'string',
     'metadata': {},
     'request': {
-      'appId': 'admin',
-      'password': password,
-      'userName': username
+      'appId': appId,
+      'clientId': clientId,
+      'secretKey': secretKey
     },
     'requesttime': getCurrentDateTime(),
     'version': 'string'
   }
-  authresponse=requests.post(authURL, json= auth_req_data)
+  try:
+    authresponse=requests.post(authURL, json= auth_req_data, timeout=30)
+  except requests.exceptions.RequestException as e:
+    sys.exit("Authentication request failed (network error). URL: " + authURL + " Error: " + str(e))
   print(authresponse)
+  if authresponse.status_code != 200 or 'authorization' not in authresponse.headers:
+    sys.exit("Authentication failed (HTTP " + str(authresponse.status_code) + "). URL: " + authURL + " Response: " + authresponse.text)
   return authresponse.headers["authorization"]
 
 
@@ -156,7 +172,7 @@ def publish_spec(domain, spec_type, specjson):
 									    "jsonspec": json.loads(spec)
 									  }
 									}
-	spec_resp = requests.post(uispecURL, json=request_json, headers=req_headers)
+	spec_resp = requests.post(uispecURL, json=request_json, headers=req_headers, timeout=30)
 	spec_resp_json = spec_resp.json()
 	#print("UI spec POST response : " + json.dumps(spec_resp_json))
 
@@ -179,7 +195,7 @@ def publish_spec(domain, spec_type, specjson):
 											  }
 											}
 	print("UI spec publish request : " + json.dumps(publish_spec_req))
-	publish_resp = requests.put(uispecPublishURL, json=publish_spec_req, headers=req_headers)
+	publish_resp = requests.put(uispecPublishURL, json=publish_spec_req, headers=req_headers, timeout=30)
 	publish_resp_json = publish_resp.json()
 	print("UI spec published : " + json.dumps(publish_resp_json))
 
@@ -728,10 +744,12 @@ def buildLostRegistrationSpec(demographic_fields, document_fields, biometric_fie
 
 # invoke syncdata service with authtoken in headers
 req_headers={'Cookie' : 'Authorization='+getAccessToken()}
-get_schema_resp=requests.get(schemaURL, headers=req_headers)
+get_schema_resp=requests.get(schemaURL, headers=req_headers, timeout=30)
 print(get_schema_resp)
 schema_resp_json=get_schema_resp.json()
 schema_resp=schema_resp_json['response']
+if schema_resp is None:
+	sys.exit("Failed to fetch identity schema from " + schemaURL + ". Response: " + json.dumps(schema_resp_json))
 identity_schema_id=schema_resp['id']
 cur_schema=schema_resp['schema']
 domain='registration-client'
@@ -763,7 +781,7 @@ def buildLostRegistrationSpec(demographic_fields, document_fields, biometric_fie
 
 
 #set all the required field mappings
-response = requests.get(args.identityMappingJsonUrl)
+response = requests.get(args.identityMappingJsonUrl, timeout=30)
 data = json.loads(response.text)
 identity_mapping_json = data['identity']
 if(identity_mapping_json.get('individualBiometrics') != None):

mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.properties

@@ -6,6 +6,9 @@ ACTION=upgrade
 UPGRADE_DOMAIN_NAME=
 GLOBAL_ADMIN_USER=
 GLOBAL_ADMIN_USER_PWD=
+AUTH_APP_ID=admin
+CLIENT_ID=
+CLIENT_SECRET_KEY=
 PRIMARY_LANGUAGE_CODE=
 SECONDARY_LANGUAGE_CODE=
 GENDER_DYNAMIC_FIELD_NAME=gender

mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade.sh

@@ -11,6 +11,7 @@ then
         key=$(echo "$key" | tr '.' '_')
         if [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
             printf -v "$key" '%s' "$value"
+            export "$key"
         else
             echo "WARNING: Skipping invalid property key: $key"
         fi

mosip_master/data_upgrade/1.1.5.5_to_1.2.0.1/upgrade_commands.txt

@@ -1,7 +1,7 @@
-python3 migration-ui_spec.py -d "$UPGRADE_DOMAIN_NAME" -u "$GLOBAL_ADMIN_USER" -p "$GLOBAL_ADMIN_USER_PWD" -pl "$PRIMARY_LANGUAGE_CODE" -sl "$SECONDARY_LANGUAGE_CODE" --identityMappingJsonUrl "$IDENTITY_MAPPING_JSON_URL" --ageGroupConfig "$AGEGROUP_CONFIG" --infantAgeGroup "$INFANT_AGEGROUP" --allowedBioAttributes "$ALLOWED_BIO_ATTRIBUTES"
+python3 migration-ui_spec.py -d "$UPGRADE_DOMAIN_NAME" -u "$GLOBAL_ADMIN_USER" -p "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" -pl "$PRIMARY_LANGUAGE_CODE" -sl "$SECONDARY_LANGUAGE_CODE" --identityMappingJsonUrl "$IDENTITY_MAPPING_JSON_URL" --ageGroupConfig "$AGEGROUP_CONFIG" --infantAgeGroup "$INFANT_AGEGROUP" --allowedBioAttributes "$ALLOWED_BIO_ATTRIBUTES"
 python3 migration-dynamicfield.py "$SU_USER" "$SU_USER_PWD" "$DB_SERVERIP" "$DB_PORT" "$GENDER_DYNAMIC_FIELD_NAME" "$INDIVIDUAL_TYPE_DYNAMIC_FIELD_NAME"
-python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "template_type" --operation "Insert" --autogen 0 --file "template_type_delta.xlsx"
-python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "template" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname Sheet1 --idcolumn A --autogen 1 --file "template_delta.xlsx"
-python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "machine_type" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_type --autogen 0 --file "machine_type_delta.xlsx"
-python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "machine_spec" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_spec --autogen 0 --file "machine_spec_delta.xlsx"
-python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --table "zone_user" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname zone_user --autogen 0 --file "zone_user_delta.xlsx"
+python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "template_type" --operation "Insert" --autogen 0 --file "template_type_delta.xlsx"
+python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "template" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname Sheet1 --idcolumn A --autogen 1 --file "template_delta.xlsx"
+python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "machine_type" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_type --autogen 0 --file "machine_type_delta.xlsx"
+python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "machine_spec" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname machine_spec --autogen 0 --file "machine_spec_delta.xlsx"
+python3 data-uploader.py --domain "$UPGRADE_DOMAIN_NAME" --username "$GLOBAL_ADMIN_USER" --password "$GLOBAL_ADMIN_USER_PWD" --appId "$AUTH_APP_ID" --clientId "$CLIENT_ID" --table "zone_user" --operation "Insert" --dbusername "$SU_USER" --dbpassword "$SU_USER_PWD" --dbhost "$DB_SERVERIP" --dbport "$DB_PORT" --sheetname zone_user --autogen 0 --file "zone_user_delta.xlsx"