From 406c6b96244aab1cf58658707d10b9c391f854fc Mon Sep 17 00:00:00 2001 From: Declan Joseph Date: Mon, 20 Jul 2026 15:48:25 +0300 Subject: [PATCH 1/3] enhancement/fecth-sources-whose-dataset-is-approved For the source list page, only show users the sources whose dataset has been approved. Performs joins across reference<->occurrence<->dataset. --- ...en-citation-year-unique-contraint-logic.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/API/src/db/migrations/1784294706153-fix-broken-citation-year-unique-contraint-logic.ts diff --git a/src/API/src/db/migrations/1784294706153-fix-broken-citation-year-unique-contraint-logic.ts b/src/API/src/db/migrations/1784294706153-fix-broken-citation-year-unique-contraint-logic.ts new file mode 100644 index 00000000..271b2538 --- /dev/null +++ b/src/API/src/db/migrations/1784294706153-fix-broken-citation-year-unique-contraint-logic.ts @@ -0,0 +1,22 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class FixBrokenCitationYearUniqueContraintLogic1784294706153 implements MigrationInterface { + name = 'FixBrokenCitationYearUniqueContraintLogic1784294706153' + + public async up(queryRunner: QueryRunner): Promise { + // Drop the old broken unique constraint on (citation, year) - this enforced uniqueness even with empty citation + await queryRunner.query(`ALTER TABLE "reference" DROP CONSTRAINT IF EXISTS "UQ_30733720a40f8d4bb3b83d06735"`); + + // Create a new partial unique index on (citation, year) that only enforces uniqueness + // when BOTH citation is non-empty AND year is NOT NULL + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_reference_citation_year_unique_both_set" ON "reference" (citation, "year") WHERE citation <> '' AND "year" IS NOT NULL`); + } + + public async down(queryRunner: QueryRunner): Promise { + // Drop the new partial unique index + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_reference_citation_year_unique_both_set"`); + + // Restore the old constraint on (citation, year) which was too strict (enforced even with empty citation) + await queryRunner.query(`ALTER TABLE "reference" ADD CONSTRAINT "UQ_30733720a40f8d4bb3b83d06735" UNIQUE (citation, year)`); + } +} From e562b5082be3c318b7fdd04fd95f26556d202f6b Mon Sep 17 00:00:00 2001 From: Declan Joseph Date: Mon, 20 Jul 2026 15:49:37 +0300 Subject: [PATCH 2/3] run-lint-fix-build --- src/API/src/db/shared/reference.service.ts | 40 ++++++++++++++++------ 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/src/API/src/db/shared/reference.service.ts b/src/API/src/db/shared/reference.service.ts index 6bd1bde3..12434202 100644 --- a/src/API/src/db/shared/reference.service.ts +++ b/src/API/src/db/shared/reference.service.ts @@ -1,6 +1,8 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { Reference } from './entities/reference.entity'; +import { Occurrence } from '../occurrence/entities/occurrence.entity'; +import { Dataset } from './entities/dataset.entity'; import { Repository } from 'typeorm'; @Injectable() @@ -35,35 +37,51 @@ export class ReferenceService { textFilter: string, ): Promise<{ items: Reference[]; total: number }> { const nonStringCols = ['num_id', 'year', 'published', 'v_data']; - const orderByString = nonStringCols.includes(orderBy) - ? `reference.${orderBy}` - : `LOWER(reference.${orderBy})`; + + // Create base query builder for reference table let query = this.referenceRepository.createQueryBuilder('reference'); + // Join through occurrence to dataset using entity relationships + // This ensures we only return references that are linked to occurrences in approved datasets + query = query + .innerJoin(Occurrence, 'occ', 'occ.referenceId = reference.id') + .innerJoin(Dataset, 'ds', 'ds.id = occ.datasetId') + .andWhere('ds.status = :status', { status: 'Approved' }) + .distinct(true); + + // Apply optional filters if (startId && !isNaN(startId)) { - query = query.andWhere('"reference"."num_id" >= :startId', { + query = query.andWhere('reference.num_id >= :startId', { startId, }); } if (endId && !isNaN(endId)) { - query = query.andWhere('"reference"."num_id" <= :endId', { + query = query.andWhere('reference.num_id <= :endId', { endId, }); } if (textFilter) { query = query.andWhere( - 'LOWER("reference"."article_title") LIKE :textFilter', + 'LOWER(reference.article_title) LIKE :textFilter', { textFilter: `%${textFilter.toLocaleLowerCase()}%`, }, ); } - const [items, total] = await query - .orderBy(orderByString, order) - .skip(skip) - .take(take) - .getManyAndCount(); + // Apply ordering + // For string columns, we add a computed LOWER column to SELECT and order by its alias + // This avoids TypeORM's expression parser issues with LOWER() in ORDER BY + if (nonStringCols.includes(orderBy)) { + query = query.addOrderBy(`"reference"."${orderBy}"`, order); + } else { + const lowerAlias = `lower_${orderBy}`; + query = query.addSelect(`LOWER("reference"."${orderBy}")`, lowerAlias); + query = query.addOrderBy(lowerAlias, order); + } + + // Execute query with pagination + const [items, total] = await query.skip(skip).take(take).getManyAndCount(); return { items, total }; } From c7a38d84bda73847c95ddecac0535563fddfd6ea Mon Sep 17 00:00:00 2001 From: Declan Joseph Date: Mon, 20 Jul 2026 16:04:49 +0300 Subject: [PATCH 3/3] add-pagination-support --- src/API/src/db/shared/reference.service.ts | 91 ++++++++++++++++------ 1 file changed, 67 insertions(+), 24 deletions(-) diff --git a/src/API/src/db/shared/reference.service.ts b/src/API/src/db/shared/reference.service.ts index 12434202..0fa4d2d6 100644 --- a/src/API/src/db/shared/reference.service.ts +++ b/src/API/src/db/shared/reference.service.ts @@ -38,50 +38,93 @@ export class ReferenceService { ): Promise<{ items: Reference[]; total: number }> { const nonStringCols = ['num_id', 'year', 'published', 'v_data']; - // Create base query builder for reference table - let query = this.referenceRepository.createQueryBuilder('reference'); + // Build filter conditions that will be applied to both queries + const filterParams: Record = { status: 'Approved' }; - // Join through occurrence to dataset using entity relationships - // This ensures we only return references that are linked to occurrences in approved datasets - query = query + if (startId && !isNaN(startId)) { + filterParams.startId = startId; + } + if (endId && !isNaN(endId)) { + filterParams.endId = endId; + } + if (textFilter) { + filterParams.textFilter = `%${textFilter.toLocaleLowerCase()}%`; + } + + // ============================================ + // QUERY 1: Get paginated items + // ============================================ + let itemsQuery = this.referenceRepository + .createQueryBuilder('reference') .innerJoin(Occurrence, 'occ', 'occ.referenceId = reference.id') .innerJoin(Dataset, 'ds', 'ds.id = occ.datasetId') - .andWhere('ds.status = :status', { status: 'Approved' }) + .andWhere('ds.status = :status', filterParams) .distinct(true); - // Apply optional filters + // Apply filters to items query if (startId && !isNaN(startId)) { - query = query.andWhere('reference.num_id >= :startId', { - startId, - }); + itemsQuery = itemsQuery.andWhere( + 'reference.num_id >= :startId', + filterParams, + ); } if (endId && !isNaN(endId)) { - query = query.andWhere('reference.num_id <= :endId', { - endId, - }); + itemsQuery = itemsQuery.andWhere( + 'reference.num_id <= :endId', + filterParams, + ); } if (textFilter) { - query = query.andWhere( + itemsQuery = itemsQuery.andWhere( 'LOWER(reference.article_title) LIKE :textFilter', - { - textFilter: `%${textFilter.toLocaleLowerCase()}%`, - }, + filterParams, ); } // Apply ordering - // For string columns, we add a computed LOWER column to SELECT and order by its alias - // This avoids TypeORM's expression parser issues with LOWER() in ORDER BY if (nonStringCols.includes(orderBy)) { - query = query.addOrderBy(`"reference"."${orderBy}"`, order); + itemsQuery = itemsQuery.addOrderBy(`"reference"."${orderBy}"`, order); } else { const lowerAlias = `lower_${orderBy}`; - query = query.addSelect(`LOWER("reference"."${orderBy}")`, lowerAlias); - query = query.addOrderBy(lowerAlias, order); + itemsQuery = itemsQuery.addSelect( + `LOWER("reference"."${orderBy}")`, + lowerAlias, + ); + itemsQuery = itemsQuery.addOrderBy(lowerAlias, order); + } + + const items = await itemsQuery.skip(skip).take(take).getMany(); + + // ============================================ + // QUERY 2: Get DISTINCT count + // ============================================ + let countQuery = this.referenceRepository + .createQueryBuilder('reference') + .select('COUNT(DISTINCT reference.id)', 'count') + .innerJoin(Occurrence, 'occ2', 'occ2.referenceId = reference.id') + .innerJoin(Dataset, 'ds2', 'ds2.id = occ2.datasetId') + .andWhere('ds2.status = :status', { status: 'Approved' }); + + // Apply same filters to count query + if (startId && !isNaN(startId)) { + countQuery = countQuery.andWhere('reference.num_id >= :startId', { + startId, + }); + } + if (endId && !isNaN(endId)) { + countQuery = countQuery.andWhere('reference.num_id <= :endId', { endId }); + } + if (textFilter) { + countQuery = countQuery.andWhere( + 'LOWER(reference.article_title) LIKE :textFilter', + { + textFilter: `%${textFilter.toLocaleLowerCase()}%`, + }, + ); } - // Execute query with pagination - const [items, total] = await query.skip(skip).take(take).getManyAndCount(); + const countResult = await countQuery.getRawOne(); + const total = parseInt(countResult.count, 10) || 0; return { items, total }; }