Develop

.github/workflows/push-trigger.yml

@@ -0,0 +1,161 @@
+name: Setup Postgresql & Run postgres_init for MOSIP_Master & Run docker container
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+  workflow_dispatch:
+    inputs:
+      message:
+        description: 'Message for manually triggering'
+        required: false
+        default: 'Triggered for Updates'
+        type: string
+  push:
+    branches:
+      - develop
+      - 1.2.0.1
+      - MOSIP*
+env:
+  DBUSER_PWD: mosip123
+  SU_USER_PWD: abc123
+jobs:
+  setup-and-run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+      - name: Create Docker Network
+        run: docker network create mosip_network
+      - name: Setup PostgreSQL Container
+        run: |
+          echo "[DB] Starting PostgreSQL container..."
+          docker run -d \
+            --name postgresql \
+            --network mosip_network \
+            -p 5432:5432 \
+            -e POSTGRES_USER=postgres \
+            -e POSTGRES_PASSWORD=${{ env.SU_USER_PWD }} \
+            -e POSTGRES_HOST_AUTH_METHOD=trust \
+            postgres:16
+          echo "[DB] PostgreSQL container started."
+
+      - name: Wait for PostgreSQL to be ready
+        run: |
+          echo "[DB] Waiting for PostgreSQL to accept connections..."
+          for i in $(seq 1 30); do
+            if docker exec postgresql pg_isready -U postgres -q; then
+              echo "[DB] PostgreSQL is ready after ${i} attempt(s)."
+              break
+            fi
+            echo "[DB] Attempt $i/30 - not ready yet, retrying in 2s..."
+            sleep 2
+          done
+          docker exec postgresql pg_isready -U postgres || \
+            (echo "[DB] ERROR: PostgreSQL did not become ready in time." && exit 1)
+
+      - name: Initialise mosip_master DB
+        run: |
+          echo "[DB] Starting mosip_master_container (postgres-init)..."
+          docker run -itd \
+            --name mosip_master_container \
+            --network mosip_network \
+            -e MOSIP_DB_NAME=mosip_master \
+            -e DB_SERVERIP=postgresql \
+            -e DB_PORT=5432 \
+            -e DEFAULT_DB_NAME=postgres \
+            -e DB_UNAME=masteruser \
+            -e DB_SCRIPTS_DIR=mosip_master \
+            -e SU_USER=postgres \
+            -e DML_FLAG=1 \
+            -e GIT_REPO_URL=https://github.com/mosip/admin-services.git \
+            -e GIT_BRANCH=develop \
+            -e SU_USER_PWD=${{ env.SU_USER_PWD }} \
+            -e DBUSER_PWD=${{ env.DBUSER_PWD }} \
+            mosipdev/postgres-init:develop
+          echo "[DB] mosip_master_container launched."
+
+      - name: docker ps command
+        run: docker ps -a
+
+      - name: Stream mosip_master_container logs
+        run: |
+          echo "[DB] Streaming mosip_master_container logs (live)..."
+          docker logs -f mosip_master_container
+          echo "[DB] mosip_master_container has exited."
+
+      - name: Verify mosip_master_container exit code
+        run: |
+          EXIT_CODE=$(docker inspect mosip_master_container --format='{{.State.ExitCode}}')
+          echo "[DB] mosip_master_container exit code: $EXIT_CODE"
+          if [ "$EXIT_CODE" != "0" ]; then
+            echo "[DB] ERROR: mosip_master_container exited with non-zero code $EXIT_CODE"
+            exit 1
+          fi
+
+      - name: Check whether db_init has errors in Logs
+        run: |
+          echo "[DB] Scanning mosip_master_container logs for errors..."
+          if docker logs mosip_master_container 2>&1 | grep -qiE "\berror\b|\bfatal\b"; then
+            echo "[DB] ERROR: Problem keywords found in mosip_master_container logs:"
+            docker logs mosip_master_container 2>&1 | grep -iE "\berror\b|\bfatal\b"
+            exit 1
+          else
+            echo "[DB] No errors found in mosip_master_container logs."
+          fi
+
+      - name: Verify masteruser role exists
+        run: |
+          echo "[DB] Verifying masteruser role exists in PostgreSQL..."
+          ROLE=$(docker exec postgresql psql -U postgres -tAc "SELECT rolname FROM pg_roles WHERE rolname='masteruser';")
+          if [ "$ROLE" = "masteruser" ]; then
+            echo "[DB] masteruser role confirmed."
+          else
+            echo "[DB] ERROR: masteruser role not found. DB init may have failed."
+            echo "[DB] Existing roles:"
+            docker exec postgresql psql -U postgres -c "\du"
+            exit 1
+          fi
+
+
+      - name: Set environment variables
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            REPO_URL="${{ github.event.pull_request.head.repo.clone_url }}"
+            BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
+          else
+            REPO_URL="https://github.com/${{ github.repository }}.git"
+            BRANCH_NAME="${{ github.ref_name }}"
+          fi
+          echo "REPO_URL=${REPO_URL}" >> $GITHUB_ENV
+          echo "BRANCH_NAME=${BRANCH_NAME}" >> $GITHUB_ENV
+
+      - name: Run masterdata-loader
+        run: |
+          docker run -itd \
+            --name masterdata_loader \
+            --network mosip_network \
+            -e DB_HOST=postgresql \
+            -e DB_USER=masteruser \
+            -e DB_PORT=5432 \
+            -e DB_PWD=${{ env.DBUSER_PWD }} \
+            -e REPOSITORY=${{ env.REPO_URL }}\
+            -e BRANCH=${{ env.BRANCH_NAME }} \
+            mosipdev/masterdata-loader:develop
+
+      - name: docker ps command
+        run: docker ps -a
+      - name: Print Logs of masterdata_loader
+        run: |
+          # Display masterdata_loader logs
+          docker logs -f masterdata_loader
+
+      - name: Check whether masterdata_loader has errors in Logs
+        run: |
+          echo "[DB] Scanning masterdata_loader logs for errors..."
+          if docker logs masterdata_loader 2>&1 | grep -qiE "\berror\b|\bfatal\b"; then
+            echo "[DB] ERROR: Problem keywords found in masterdata_loader logs:"
+            docker logs masterdata_loader 2>&1 | grep -iE "\berror\b|\bfatal\b"
+            exit 1
+          else
+            echo "[DB] No errors found in masterdata_loader logs."
+          fi   

.github/workflows/xlsx-to-csv.yml

@@ -0,0 +1,80 @@
+name: xlsx to csv conversion
+on:
+  pull_request_target:
+    types: [opened, reopened, synchronize]
+  workflow_dispatch:
+    inputs:
+      message:
+        description: 'Message for manually triggering'
+        required: false
+        default: 'Triggered for Updates'
+        type: string
+
+jobs:
+  convert_xlsx_to_csv:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check for xlsx changes
+      id: check_xlsx
+      run: |
+        PR_NUMBER=${{ github.event.number }}
+        FILES=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+          -H "Accept: application/vnd.github.v3+json" \
+          "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER/files")
+        XLSX_CHANGED=$(echo "$FILES" | jq '[.[].filename | select(endswith(".xlsx"))] | length')
+        echo "xlsx_changed=$XLSX_CHANGED" >> $GITHUB_OUTPUT
+
+    - name: Skip if no xlsx changes
+      if: steps.check_xlsx.outputs.xlsx_changed == '0' && github.event_name != 'workflow_dispatch'
+      run: |
+        echo "No xlsx files changed. Skipping workflow."
+        exit 0
+
+    - name: Clone the repository
+      if: steps.check_xlsx.outputs.xlsx_changed != '0' || github.event_name == 'workflow_dispatch'
+      run: |
+        git clone -b ${{ github.event.pull_request.head.ref }} https://github.com/${{ github.event.pull_request.head.repo.full_name }}.git
+
+    - name: Get author email
+      if: steps.check_xlsx.outputs.xlsx_changed != '0' || github.event_name == 'workflow_dispatch'
+      run: |
+        PR_NUMBER=${{ github.event.number }}
+        commits=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+          -H "Accept: application/vnd.github.v3+json" \
+          "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER/commits")
+        email=$(echo "$commits" | jq -r '.[0].commit.author.email // "${{ github.event.pull_request.user.login }}@users.noreply.github.com"')
+        echo "author_email=$email" >> $GITHUB_ENV
+
+    - name: Set up Git identity
+      if: steps.check_xlsx.outputs.xlsx_changed != '0' || github.event_name == 'workflow_dispatch'
+      run: |
+        git config --global user.name "${{ github.event.pull_request.user.login }}"
+        git config --global user.email "${{ env.author_email }}"
+
+    - name: Install xlsx2csv
+      if: steps.check_xlsx.outputs.xlsx_changed != '0' || github.event_name == 'workflow_dispatch'
+      run: sudo apt-get install -y xlsx2csv
+
+    - name: Create output directory
+      if: steps.check_xlsx.outputs.xlsx_changed != '0' || github.event_name == 'workflow_dispatch'
+      run: |
+        rm -rf mosip-data/mosip_master_csv/csv
+        mkdir -p mosip-data/mosip_master_csv/csv
+
+    - name: Convert all XLSX to CSV
+      if: steps.check_xlsx.outputs.xlsx_changed != '0' || github.event_name == 'workflow_dispatch'
+      run: |
+        for xlsx_file in mosip-data/mosip_master/xlsx/*.xlsx; do
+          csv_file="mosip-data/mosip_master_csv/csv/$(basename "${xlsx_file%.xlsx}.csv")"
+          xlsx2csv "$xlsx_file" > "$csv_file"
+          echo "Converted $xlsx_file to $csv_file"
+        done
+
+    - name: Commit and push changes
+      if: steps.check_xlsx.outputs.xlsx_changed != '0' || github.event_name == 'workflow_dispatch'
+      run: |
+        cd mosip-data
+        git branch
+        git add .
+        git commit -s -m "Added converted CSV files" || echo "No changes to commit"
+        git push "https://x-access-token:${{ secrets.ACTION_PAT }}@github.com/${{ github.event.pull_request.head.repo.full_name }}.git" HEAD:${{ github.event.pull_request.head.ref }}

.gitignore

@@ -1 +1,3 @@
 .DS_Store
+.idea
+.project

AGENTS.md

@@ -0,0 +1,72 @@
+# AGENTS.md
+
+This file provides guidance to AI agents when working with code in this repository.
+
+## What This Repository Is
+
+This is the **MOSIP Master Data Repository** — a data-only repo containing sample/sandbox master data for the [MOSIP identity platform](https://docs.mosip.io/1.2.0/id-lifecycle-management). It has no application code, no build system, and no test suite. Changes here feed directly into MOSIP database initialization and upgrades.
+
+## Data Layout
+
+```
+mosip_master/xlsx/       ← Canonical source: 37 Excel files (edit these)
+mosip_master/csv/        ← Primary CSV source used by the masterdata-loader
+mosip_master_csv/csv/    ← Auto-generated from xlsx via CI (do not edit manually)
+mosip_master/data_upgrade/ ← Version-to-version delta migration scripts
+```
+
+**Key rule:** Only edit files in `mosip_master/xlsx/` or `mosip_master/csv/`. The `mosip_master_csv/csv/` directory is regenerated automatically by the `xlsx-to-csv` GitHub Actions workflow on every PR — manual edits there will be overwritten.
+
+## Workflows
+
+### xlsx-to-csv (`.github/workflows/xlsx-to-csv.yml`)
+Triggers on PR open/sync or manual `workflow_dispatch`. Converts every `.xlsx` in `mosip_master/xlsx/` to CSV using `xlsx2csv` and commits the result into `mosip_master_csv/csv/` on the PR branch. Requires the `ACTION_PAT` secret.
+
+To trigger manually: GitHub → Actions → "xlsx-to-csv" → Run workflow.
+
+### push-trigger (`.github/workflows/push-trigger.yml`)
+Validates the full data load pipeline end-to-end:
+1. Spins up PostgreSQL 16 in Docker
+2. Runs `mosipdev/postgres-init:develop` to create the `mosip_master` schema
+3. Runs `mosipdev/masterdata-loader:develop` to load CSV data from this repo
+4. Checks for loader errors
+
+This is the closest thing to a "test" for this repo. A passing run confirms the data is loadable.
+
+## Data Entities
+
+The master data covers:
+
+| Domain | Key files |
+|---|---|
+| Identity | `identity_schema`, `ui_spec`, `dynamic_field` |
+| Templates | `template`, `template_type` |
+| Devices | `device_master`, `device_spec`, `device_type` |
+| Machines | `machine_master`, `machine_spec`, `machine_type` |
+| Locations | `location`, `loc_hierarchy_list`, `zone`, `zone_user` |
+| Registration | `registration_center`, `reg_center_type`, `reg_working_nonworking` |
+| Documents | `valid_document`, `doc_category`, `doc_type`, `applicant_valid_document` |
+| Lookup | `language`, `title`, `reason_list`, `blocklisted_words` |
+
+**`identity_schema.csv`** is the most complex file — it embeds full JSON Schema Draft-07 definitions inside CSV cells. Each row is one schema version per language (e.g. `eng, 1001, 0.1, ...`). Schema ID 1001 = standard identity; 1002 = Mosip Identity for handle.
+
+## Data Upgrade Scripts
+
+`mosip_master/data_upgrade/` contains **delta** migrations (not full datasets) between MOSIP versions:
+
+- `1.1.5.5_to_1.2.0.1/` — Dynamic field format changed (array → object per language); UI spec split into separate table; new machine/template types
+- `1.2.0.1_to_1.3.0/` — Template additions and updates only
+
+**To run an upgrade:**
+```bash
+cd mosip_master/data_upgrade/<version>/
+cp upgrade.properties upgrade.local.properties
+# Fill in DB_SERVERIP, SU_USER_PWD, UPGRADE_DOMAIN_NAME, admin creds, language codes
+bash upgrade.sh upgrade.local.properties
+```
+
+The Python helper scripts (`data-uploader.py`, `migration-ui_spec.py`, `migration-dynamicfield.py`) authenticate against the MOSIP Admin API and/or connect directly to PostgreSQL. They are invoked by `upgrade.sh` via `upgrade_commands.txt`.
+
+## Masterdata Reference
+
+Full field definitions and valid values: https://docs.mosip.io/1.2.0/id-lifecycle-management/support-systems/administration/masterdata-guide

NOTICE

@@ -0,0 +1,30 @@
+THIRD-PARTY NOTICES
+
+Copyright © MOSIP.
+All rights reserved.
+
+This project includes third-party components distributed under licenses
+that require attribution. The following sections list those components
+and their respective licenses.
+
+===========================
+Mozilla Public License 2.0
+==========================
+
+• MOSIP Data (mosip-data)
+Version: release-1.3.x
+Homepage: [https://github.com/Rakshithasai123/mosip-data](https://github.com/Rakshithasai123/mosip-data)
+
+===========================
+MIT License
+===========
+
+• actions/checkout
+Version: 2.*.*
+Homepage: [https://github.com/actions/checkout](https://github.com/actions/checkout)
+
+---
+
+Full license texts for all components are available in the
+`license/` directory of this repository.
+----------------------------------------

README.md

@@ -1,7 +1,19 @@
 # Sample Data
 
 ## Overview
-This repo contains data that is uploaded into MOSIP during [sandbox installation](https://docs.mosip.io/1.2.0/deployment/sandbox-deployment). The data needs to be reviewed and modified for a country specific deployment. Refer to [Masterdata Guide](https://docs.mosip.io/1.2.0/deployment/masterdata-guide). 
+This repo contains data that is uploaded into MOSIP during [sandbox installation](https://docs.mosip.io/1.2.0/setup/deploymentnew/getting-started#mosip-installations). The data needs to be reviewed and modified for a country specific deployment. Refer to [Masterdata Guide](https://docs.mosip.io/1.2.0/id-lifecycle-management/support-systems/administration/masterdata-guide). 
+
+## For Build and Run
+Data initialization is performed through the **Master Data Loader** as part of the [postgres-init](https://github.com/mosip/postgres-init/tree/release-1.3.x) repository.
+
+## Contribution & Community
+
+• To learn how you can contribute code to this application, [click here](https://docs.mosip.io/1.2.0/community/code-contributions).
+
+• If you have questions or encounter issues, visit the [MOSIP Community](https://community.mosip.io/) for support.
+
+• For any GitHub issues: [Report here](https://github.com/mosip/mosip-data/issues)
 
 ## License
 This project is licensed under the terms of [Mozilla Public License 2.0](LICENSE).
+

THIRD-PARTY-NOTICES.txt

@@ -0,0 +1,19 @@
+THIRD-PARTY-NOTICES
+
+This project includes third-party packages that are distributed under various open-source licenses. Below is a list of packages and their associated licenses.
+
+================================================================================
+Package: actions/checkout
+Version: 2.*.*
+License: MIT License
+Homepage: https://github.com/actions/checkout
+================================================================================
+
+================================================================================
+Package: PostgreSQL (Docker Official Image)
+Version: 16
+License: PostgreSQL License
+Homepage: https://hub.docker.com/_/postgres
+================================================================================
+
+Full license texts and additional details for each of the above packages are available in the license/ directory of this repository. Please refer to those files or the original source of each package for complete legal terms and conditions.