From bb68513d73777f0e4200170dc347d5c94b42ad9c Mon Sep 17 00:00:00 2001 From: Eduardo Peredo Rivero Date: Tue, 21 Apr 2026 20:28:23 -0500 Subject: [PATCH 1/2] generate script to remove paired dataElements --- src/models/ProjectsList.ts | 6 +- .../unique-periods/UniquePeriodsForm.tsx | 16 +- src/scripts/remove-paired-dataelement.ts | 138 ++++++++++++++++++ src/utils/date.ts | 17 ++- 4 files changed, 160 insertions(+), 17 deletions(-) create mode 100644 src/scripts/remove-paired-dataelement.ts diff --git a/src/models/ProjectsList.ts b/src/models/ProjectsList.ts index a3cff1cc..2a7d5466 100644 --- a/src/models/ProjectsList.ts +++ b/src/models/ProjectsList.ts @@ -1,5 +1,4 @@ import _ from "lodash"; -import { TableSorting } from "@eyeseetea/d2-ui-components"; import { D2Api, D2OrganisationUnitSchema, SelectedPick, Id, Pager, Ref } from "../types/d2-api"; import { Config } from "./Config"; import moment, { Moment } from "moment"; @@ -399,3 +398,8 @@ function getOrgUnitsFilter(filters: FiltersForList, currentUser: User) { : userCountryIds; return filterCountryIds ? { "parent.id": { in: filterCountryIds } } : {}; } + +export type TableSorting = { + field: keyof T; + order: "asc" | "desc"; +}; diff --git a/src/pages/unique-periods/UniquePeriodsForm.tsx b/src/pages/unique-periods/UniquePeriodsForm.tsx index 198ba202..7717754e 100644 --- a/src/pages/unique-periods/UniquePeriodsForm.tsx +++ b/src/pages/unique-periods/UniquePeriodsForm.tsx @@ -10,6 +10,7 @@ import { import i18n from "../../locales"; import { Maybe } from "../../types/utils"; import { getErrors } from "../../domain/entities/generic/Errors"; +import { months } from "../../utils/date"; export type UniquePeriodsFormProps = { existingPeriod?: UniqueBeneficiariesPeriodsAttrs; @@ -17,21 +18,6 @@ export type UniquePeriodsFormProps = { onSubmit: (uniquePeriods: UniqueBeneficiariesPeriod) => void; }; -export const months = [ - { value: "1", text: i18n.t("January") }, - { value: "2", text: i18n.t("February") }, - { value: "3", text: i18n.t("March") }, - { value: "4", text: i18n.t("April") }, - { value: "5", text: i18n.t("May") }, - { value: "6", text: i18n.t("June") }, - { value: "7", text: i18n.t("July") }, - { value: "8", text: i18n.t("August") }, - { value: "9", text: i18n.t("September") }, - { value: "10", text: i18n.t("October") }, - { value: "11", text: i18n.t("November") }, - { value: "12", text: i18n.t("December") }, -]; - function getValueByAttribute( value: string, attribute: keyof UniqueBeneficiariesPeriod diff --git a/src/scripts/remove-paired-dataelement.ts b/src/scripts/remove-paired-dataelement.ts new file mode 100644 index 00000000..bdace879 --- /dev/null +++ b/src/scripts/remove-paired-dataelement.ts @@ -0,0 +1,138 @@ +import _ from "lodash"; +import parse from "parse-typed-args"; +import { promiseMap } from "../migrations/utils"; +import { getConfig } from "../models/Config"; +import { D2Api } from "../types/d2-api"; +import fs from "fs"; + +async function main() { + const parser = parse({ + opts: { + url: {}, + auth: {}, + codes: {}, + persist: { switch: true }, + }, + }); + const { opts } = parser(process.argv); + const { url, auth, codes, persist } = opts; + + const usage = + "npx tsx remove-paired-dataelement --url= [--auth=user:pass] [--codes=deCode1,deCode2] [--persist]"; + if (!url) { + console.error(usage); + process.exit(1); + } + + if (!codes || codes.length === 0) { + console.error(usage); + process.exit(1); + } + + const [username, password] = auth ? auth.split(":") : ["", ""]; + const api = new D2Api({ baseUrl: url, auth: { password, username }, agent: {} }); + const config = await getConfig(api); + + // TODO: request in chunks + const responseDes = await api.models.dataElements + .get({ + fields: { id: true, attributeValues: { attribute: { id: true }, value: true } }, + filter: { code: { in: codes.split(",") } }, + paging: false, + }) + .getData(); + + if (responseDes.objects.length === 0) { + console.debug("No data elements found with the provided codes"); + return; + } + + const dataElements = responseDes.objects.map((de): DataElement => { + return { + id: de.id, + pairedDataElement: de.attributeValues?.find( + av => av.attribute.id === config.attributes.pairedDataElement.id + )?.value, + }; + }); + + const dataElementsWithPaired = dataElements.filter(de => de.pairedDataElement); + + console.debug(`Found ${dataElementsWithPaired.length} dataElements with paired data elements`); + + const allDeIds = dataElementsWithPaired.map(de => de.id); + + const allStats = await promiseMap(_(allDeIds).chunk(100).value(), async dataElementIds => { + const response = await api.models.dataElements + .get({ + fields: { $owner: true }, + filter: { id: { in: dataElementIds } }, + paging: false, + }) + .getData(); + + const dataElementsToUpdate = dataElementIds.map(dataElementId => { + const existingDe = response.objects.find(de => de.id === dataElementId); + + const dataElement = dataElementsWithPaired.find(de => de.id === dataElementId); + if (!dataElement) { + throw Error("Cannot find dataElement"); + } + + const newAttributeValues = existingDe?.attributeValues.map(d2Attribute => { + if (d2Attribute.attribute.id === config.attributes.pairedDataElement.id) { + return { + ...d2Attribute, + value: "", + }; + } + return d2Attribute; + }); + + return { ...(existingDe || {}), attributeValues: newAttributeValues }; + }); + + const persistResponse = await api.metadata + .post( + { dataElements: dataElementsToUpdate }, + { importMode: persist ? "COMMIT" : "VALIDATE", importStrategy: "UPDATE" } + ) + .getData(); + + // @ts-ignore + const stats = persistResponse.response.stats; + return { stats: stats, payload: dataElementsToUpdate }; + }); + + console.debug( + "Stats: ", + allStats.reduce( + (acum, stat) => { + return { + created: (acum.created || 0) + (stat.stats.created || 0), + updated: (acum.updated || 0) + (stat.stats.updated || 0), + deleted: (acum.deleted || 0) + (stat.stats.deleted || 0), + ignored: (acum.ignored || 0) + (stat.stats.ignored || 0), + }; + }, + { created: 0, updated: 0, deleted: 0, ignored: 0 } + ) + ); + + // save payload to disk + fs.writeFileSync( + "data_elements_paired_dataelements.json", + JSON.stringify( + allStats.flatMap(stat => stat.payload), + null, + 2 + ) + ); +} + +type DataElement = { + id: string; + pairedDataElement?: string; +}; + +main(); diff --git a/src/utils/date.ts b/src/utils/date.ts index 8180d29c..35bb52d7 100644 --- a/src/utils/date.ts +++ b/src/utils/date.ts @@ -1,5 +1,5 @@ import moment, { Moment } from "moment"; -import { months } from "../pages/unique-periods/UniquePeriodsForm"; +import i18n from "../locales"; export function toISOString(date: Moment) { return date.format("YYYY-MM-DDTHH:mm:ss"); @@ -58,3 +58,18 @@ export function buildMonthYearFormatDate(dateIsoString: string): string { export function getMonthNameFromNumber(monthNumber: string | number): string { return months.find(month => month.value === monthNumber.toString())?.text || ""; } + +export const months = [ + { value: "1", text: i18n.t("January") }, + { value: "2", text: i18n.t("February") }, + { value: "3", text: i18n.t("March") }, + { value: "4", text: i18n.t("April") }, + { value: "5", text: i18n.t("May") }, + { value: "6", text: i18n.t("June") }, + { value: "7", text: i18n.t("July") }, + { value: "8", text: i18n.t("August") }, + { value: "9", text: i18n.t("September") }, + { value: "10", text: i18n.t("October") }, + { value: "11", text: i18n.t("November") }, + { value: "12", text: i18n.t("December") }, +]; From 89c68ed16b3068e1179b76adc81f6f94c17e6855 Mon Sep 17 00:00:00 2001 From: Eduardo Peredo Rivero Date: Tue, 21 Apr 2026 20:30:49 -0500 Subject: [PATCH 2/2] improve error message --- src/scripts/remove-paired-dataelement.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/remove-paired-dataelement.ts b/src/scripts/remove-paired-dataelement.ts index bdace879..f853febd 100644 --- a/src/scripts/remove-paired-dataelement.ts +++ b/src/scripts/remove-paired-dataelement.ts @@ -18,7 +18,7 @@ async function main() { const { url, auth, codes, persist } = opts; const usage = - "npx tsx remove-paired-dataelement --url= [--auth=user:pass] [--codes=deCode1,deCode2] [--persist]"; + "npx tsx src/scripts/remove-paired-dataelement.ts --url= [--auth=user:pass] [--codes=deCode1,deCode2] [--persist]"; if (!url) { console.error(usage); process.exit(1);