Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions app/src/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,19 @@ export const FOREVER = '2100-12-31';
export const BASE_URL = 'https://api.policyengine.org';
export const CURRENT_YEAR = '2026';

// Certified default US microdata dataset URI, pinned to the revision that
// policyengine.py's bundle manifest resolves as data_releases.us.default_dataset_uri.
// The legacy `policyengine-us-data` Hugging Face repo is deprecated/archived, so any
// generated "Reproduce in Python" snippet for a US national run must point here rather
// than at a `policyengine-us-data` path. Pinned (rather than tracking a floating branch)
// so a copied snippet reproduces the exact certified dataset the app ran against.
// Ported from policyengine-app v1 (POPULACE_US_DEFAULT_DATASET_URI, PR #2846).
// NOTE: subnational (state/CD) and place fallbacks still reference `policyengine-us-data`
// pending Populace place/geo scoping — tracked in policyengine-app-v2#1079 — and are
// intentionally NOT switched to this national URI here.
export const POPULACE_US_DEFAULT_DATASET_URI =
'hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-sparse-l0-refit-57k-71a0887-national-only-20260701';

// App URLs for the split website/calculator architecture
// In dev mode, these are set via VITE_* env vars to localhost URLs
// In production, they fall back to the prod URLs
Expand Down
8 changes: 5 additions & 3 deletions app/src/tests/unit/utils/reproducibilityCode.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { describe, expect, test, vi } from 'vitest';
import { POPULACE_US_DEFAULT_DATASET_URI } from '@/constants';
import { TEST_COUNTRIES } from '@/tests/fixtures/constants';
import {
BASELINE_AND_REFORM_POLICY,
Expand Down Expand Up @@ -596,9 +597,10 @@ describe('reproducibilityCode', () => {
);
const code = lines.join('\n');

// Then
expect(code).toContain('enhanced_cps_2024.h5');
expect(code).toContain('dataset=');
// Then - a bare US national dataset name resolves to the certified
// populace-us URI, never the deprecated policyengine-us-data repo.
expect(code).toContain(`dataset="${POPULACE_US_DEFAULT_DATASET_URI}"`);
expect(code).not.toContain('policyengine-us-data');
});

test('given full dataset url then uses it verbatim', () => {
Expand Down
27 changes: 22 additions & 5 deletions app/src/utils/reproducibilityCode.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
* https://github.com/PolicyEngine/policyengine-app/blob/main/src/data/reformDefinitionCode.js
*/

import { CURRENT_YEAR } from '@/constants';
import { CURRENT_YEAR, POPULACE_US_DEFAULT_DATASET_URI } from '@/constants';
import type { Household } from '@/models/Household';
import {
addVariationAxesToPythonPackageHouseholdData,
Expand Down Expand Up @@ -38,15 +38,21 @@ function normalizeDatasetUrlForReproducibility(countryId: string, datasetName: s

/**
* Build a HuggingFace dataset URL for a US national-level dataset.
* Dataset files follow the pattern: {name}_{year}.h5
*
* A fully-qualified URI (e.g. the exact `hf://...` the backend ran against) is
* returned verbatim. For a bare US national dataset name, emit the certified
* `populace-us` URI rather than a legacy `policyengine-us-data/{name}_{year}.h5`
* path: that repo is deprecated/archived and policyengine.py no longer resolves
* it as the default, so a copied snippet must point at the certified dataset.
* Ported from policyengine-app v1 PR #2846.
*/
function getDatasetUrl(countryId: string, datasetName: string, year: number): string | null {
function getDatasetUrl(countryId: string, datasetName: string): string | null {
if (datasetName.includes('://')) {
return normalizeDatasetUrlForReproducibility(countryId, datasetName);
}

if (countryId === 'us') {
return `hf://policyengine/policyengine-us-data/${datasetName}_${year}.h5`;
return POPULACE_US_DEFAULT_DATASET_URI;
}
return null;
}
Expand All @@ -57,6 +63,12 @@ function getDatasetUrl(countryId: string, datasetName: string, year: number): st
* - "state/ca" → states/CA.h5
* - "congressional_district/CA-01" → districts/CA-01.h5
* Note: place/ regions are handled separately via getPlaceStateDatasetUrl.
*
* These `policyengine-us-data/{states,districts}/*.h5` paths are the geography-
* scoped fallback used only when a non-default subnational dataset is selected.
* They intentionally still reference the legacy repo pending Populace geo scoping;
* see policyengine-app-v2#1079. Default state/CD runs already use national Populace
* scoping via getScopedUsRegionImplementationCode, so they do not hit this path.
*/
function getSubnationalDatasetUrl(region: string): string | null {
for (const [prefix, folder] of Object.entries(US_REGION_PREFIX_TO_FOLDER)) {
Expand Down Expand Up @@ -90,6 +102,11 @@ function isDefaultUsScopedRegion(
/**
* For place/ regions, get the parent state's dataset URL.
* "place/NJ-57000" → states/NJ.h5
*
* Places load the parent state's `policyengine-us-data` H5 and filter by
* place_fips because the certified Populace export does not yet carry place
* geography. This legacy reference is deliberately retained pending Populace
* place scoping; see policyengine-app-v2#1079.
*/
function getPlaceStateDatasetUrl(region: string): string | null {
if (!region.startsWith('place/')) {
Expand Down Expand Up @@ -318,7 +335,7 @@ function getImplementationCode(
const isNational = region === countryId;
const year = timePeriod || DEFAULT_YEAR;
const resolvedDatasetUrl =
dataset && !isDefaultDataset ? getDatasetUrl(countryId, dataset, year) : null;
dataset && !isDefaultDataset ? getDatasetUrl(countryId, dataset) : null;

if (isDefaultUsScopedRegion(countryId, region, dataset, isDefaultDataset)) {
return getScopedUsRegionImplementationCode(region, year, hasBaseline, hasReform);
Expand Down
2 changes: 2 additions & 0 deletions changelog_entry.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@
changed:
- Proxy UK Python package docs at /uk/python and UK API docs at /uk/api so the country-parameterized "Python" and "API" nav links work on UK pages
- Align the calculator app header (e.g. /us/reports) with the main website header — hover-open dropdowns, per-item underline, Model dropdown with sub-items, Python link, and Events under About
fixed:
- Point "Reproduce in Python" snippets for a US national run at the certified populace-us dataset instead of the deprecated policyengine-us-data repo
Loading