From 4f6f0fff2ddc59269fe6285f3a149c18ce549191 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 15:14:56 -0600 Subject: [PATCH 01/33] cleaned up instructions, next is PR2 --- spec_driven_EDA_plan/docs/START_HERE.md | 157 +++++++++++------- .../docs/codex/review-checklist.md | 1 + .../docs/codex/revised-pr-plan-tdd-first.md | 7 +- .../codex/tdd-first-codex-instructions.md | 57 +++++++ 4 files changed, 162 insertions(+), 60 deletions(-) diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/spec_driven_EDA_plan/docs/START_HERE.md index 3fd7bc1..1606690 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/spec_driven_EDA_plan/docs/START_HERE.md @@ -1,59 +1,115 @@ # START HERE: episcout SDD/TDD development guide -This is the operational entry point for the specification-first EDA work in +This is the live control panel for the specification-first EDA work in `episcout`. -## Purpose +Use this file first to see: -Use this file to see: +- current status; +- the active PR-sized task; +- which instruction block to follow; +- files that are in and out of scope; +- expected test state; +- required closeout updates. -- what has already been planned; -- what should happen next; -- what is out of scope for the immediate work; -- which Codex instruction block to follow. +Design rationale lives in the SDD and ADR files. The full PR sequence lives in +`spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md`. This file is the +operational source of truth for what happens next. -Detailed design rationale lives in the SDD and ADR files. This file should stay -short and action-oriented. - -## Current status +## Current Status | Work item | Status | Next action | |---|---|---| | Phase 0 documentation baseline | Done | Keep docs in sync as scope changes. | -| PR 1 external fixture files | Next | Follow Instruction 1 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | -| PR 2 fixture-backed failing tests | Not started | Add tests after PR 1 fixtures exist. | +| PR 1 external fixture files | Done | Fixture files are present under `tests/testthat/fixtures/blood_storage/`. | +| PR 2 fixture-backed failing tests | Active | Follow Instruction 2 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | | PR 3 spec, schema and missingness implementation | Not started | Implement only after PR 2 tests exist. | | Later synthetic, summary, plot, report and template work | Not started | Continue through the PR plan after PR 3. | -## Immediate Codex starting point +## Active PR + +```text +PR 2: Add failing tests for specification, schema and missingness +``` + +Instruction: + +```text +Follow Instruction 2 in: +spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md +``` + +Required reading: + +```text +AGENTS.MD +spec_driven_EDA_plan/docs/START_HERE.md +spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md +spec_driven_EDA_plan/docs/sdd/data-dictionary-spec.md +spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md +spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md +spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md +spec_driven_EDA_plan/docs/codex/review-checklist.md +``` + +## PR 2 Scope + +Must edit: + +```text +tests/testthat/test-eda_spec-fixtures.R +tests/testthat/test-eda_schema-fixtures.R +tests/testthat/test-eda_missing-fixtures.R +spec_driven_EDA_plan/docs/START_HERE.md +``` + +May read: + +```text +tests/testthat/fixtures/blood_storage/blood_storage.csv +tests/testthat/fixtures/blood_storage/blood_storage_spec.csv +tests/testthat/fixtures/blood_storage/expected_schema.csv +tests/testthat/fixtures/blood_storage/expected_missing.csv +``` -For the next PR, start Codex with this: +Must not edit: ```text -Read AGENTS.MD and spec_driven_EDA_plan/docs/START_HERE.md. +R/eda_spec.R +R/eda_schema.R +R/eda_missing.R +DESCRIPTION +NAMESPACE +man/ +``` + +Expected test state: -Then read: -- spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md -- spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md -- spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md -- spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md -- spec_driven_EDA_plan/docs/codex/review-checklist.md +```text +Some new tests may fail because eda_spec(), validate_eda_spec(), +check_schema() and profile_missing() are intentionally not implemented until +PR 3. +``` -Then follow Instruction 1 in spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md. +## Closeout Rule -Scope: -- add external fixture files only -- no new package behaviour -- no DESCRIPTION changes unless a Suggests entry is needed for fixture regeneration -- no NAMESPACE changes -- no generated man/ files +Every PR must update this file before it is considered complete. -Before making changes, summarise what you plan to edit. +At closeout: -Keep the change small and reviewable. +```text +1. Mark the completed PR as Done. +2. Mark the next PR as Active. +3. Update the Active PR section. +4. Update must-edit and must-not-edit lists. +5. Record whether failing tests are expected. +6. Leave SDD and ADR files unchanged unless the design changed. ``` -## Active development principle +If a PR intentionally does not update `START_HERE.md`, record that exception in +the PR description. + +## Development Principle Use this order: @@ -69,26 +125,20 @@ For every new specification-first EDA function, use: fixture data + fixture specification + independently computed expected output -→ failing tests -→ implementation +-> failing tests +-> implementation ``` -PR 2 may intentionally introduce failing tests. CI does not need to stay green -for that red-test PR. PR 3 is expected to implement the functions needed to make -those tests pass. +## Source of Truth -## Immediate scope - -The next PR should include only: +- Live status and next action: `spec_driven_EDA_plan/docs/START_HERE.md` +- Executable Codex instructions: `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md` +- Full PR sequence: `spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md` +- Design rationale: `spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md` +- Fixture TDD rationale: `spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md` +- Review checklist: `spec_driven_EDA_plan/docs/codex/review-checklist.md` -```text -blood_storage fixture -data dictionary/spec -expected schema output -expected missingness output -fixture provenance -fixture regeneration script -``` +## Out of Scope Until Later PRs Do not start with: @@ -104,15 +154,7 @@ full plot/report system These belong in later phases. -## Source of truth for next steps - -- Executable Codex instructions: `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md` -- Full PR sequence: `spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md` -- Design rationale: `spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md` -- Fixture TDD rationale: `spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md` -- Review checklist: `spec_driven_EDA_plan/docs/codex/review-checklist.md` - -## Review rule +## Review Rule Before merging any PR, check: @@ -126,4 +168,5 @@ Did it add heavy dependencies unnecessarily? Does it preserve backwards compatibility? Are expected outputs independently computed? Are synthetic-data outputs clearly labelled where relevant? +Was START_HERE.md updated for the next PR? ``` diff --git a/spec_driven_EDA_plan/docs/codex/review-checklist.md b/spec_driven_EDA_plan/docs/codex/review-checklist.md index 99f5680..2cbeac9 100644 --- a/spec_driven_EDA_plan/docs/codex/review-checklist.md +++ b/spec_driven_EDA_plan/docs/codex/review-checklist.md @@ -49,6 +49,7 @@ Use this checklist for every PR. ## Documentation +- [ ] `START_HERE.md` reflects the completed PR and next active PR. - [ ] SDD remains accurate. - [ ] ADR is added or updated if architectural decision changed. - [ ] Roadmap is updated if scope changed. diff --git a/spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md b/spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md index fc63d15..c867eb5 100644 --- a/spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md +++ b/spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md @@ -2,10 +2,11 @@ ## Purpose -This is the active PR sequence for the specification-first EDA layer. +This is the stable PR sequence for the specification-first EDA layer. -`START_HERE.md` is the operational entry point. This file defines the full PR -sequence after the current next step. +This file is not the live status dashboard. Use +`spec_driven_EDA_plan/docs/START_HERE.md` for current status, the active PR, +allowed edits, expected test state and closeout requirements. The rule is: diff --git a/spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md b/spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md index 0384c88..0122ea8 100644 --- a/spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md +++ b/spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md @@ -10,6 +10,25 @@ Repeat this often: Keep the change small and reviewable. Do not perform broad refactoring. If you find additional issues, document them rather than fixing them in this PR. ``` +## Standing closeout rule + +Every instruction must end by updating +`spec_driven_EDA_plan/docs/START_HERE.md`. + +At closeout: + +```text +1. Mark the completed PR as Done. +2. Mark the next PR as Active. +3. Update the Active PR section. +4. Update must-edit and must-not-edit lists. +5. Record whether failing tests are expected. +6. Leave SDD and ADR files unchanged unless the design changed. +``` + +If a PR intentionally does not update `START_HERE.md`, record that exception in +the PR description. + ## Instruction 1: Add external fixture files ```text @@ -44,6 +63,9 @@ Requirements: 5. Do not add medicaldata to Imports. 6. If medicaldata is needed, use it only in the regeneration script or Suggests. 7. Add no new package behaviour in this PR. + +Closeout: +Update START_HERE.md to mark PR 1 Done and PR 2 Active. ``` ## Instruction 2: Add failing tests for spec, schema and missingness @@ -69,6 +91,10 @@ Requirements: 3. Tests must not depend on private data. 4. Tests must compare against independently computed expected outputs. 5. Do not implement eda_spec(), check_schema() or profile_missing() in this PR. + +Closeout: +Update START_HERE.md to mark PR 2 Done and PR 3 Active. Record that PR 2 may +leave tests failing until PR 3 implements the missing functions. ``` ## Instruction 3: Implement spec, schema and missingness @@ -94,6 +120,11 @@ Requirements: 3. Return machine-readable tibbles or data frames. 4. Add roxygen2 documentation. 5. Do not implement synthetic data, summaries, plots, reports or run_eda() in this PR. + +Closeout: +Update START_HERE.md to mark PR 3 Done and PR 4 Active. Tests should no longer +fail because eda_spec(), validate_eda_spec(), check_schema() or +profile_missing() are missing. ``` ## Instruction 4: Add failing synthetic-data tests @@ -115,6 +146,10 @@ Requirements: 4. Synthetic values respect min/max where provided. 5. Fixed seed gives identical output. 6. Do not implement generate_synthetic_data() in this PR. + +Closeout: +Update START_HERE.md to mark PR 4 Done and PR 5 Active. Record that PR 4 may +leave tests failing until PR 5 implements generate_synthetic_data(). ``` ## Instruction 5: Implement synthetic-data generation @@ -132,6 +167,9 @@ Requirements: 3. Use deterministic output with a fixed seed. 4. Do not add heavy dependencies. 5. Do not implement summaries, plots, reports or run_eda() in this PR. + +Closeout: +Update START_HERE.md to mark PR 5 Done and PR 6 Active. ``` ## Instruction 6: Add failing summary and plot tests @@ -152,6 +190,11 @@ Requirements: 1. Expected summary outputs must be independently computed. 2. Plot tests should check object classes and names, not visual appearance. 3. Do not implement profile_summaries() or profile_plots() in this PR. + +Closeout: +Update START_HERE.md to mark PR 6 Done and PR 7 Active. Record that PR 6 may +leave tests failing until PR 7 implements profile_summaries() and +profile_plots(). ``` ## Instruction 7: Implement summaries and plots @@ -174,6 +217,9 @@ Requirements: 3. Return machine-readable outputs. 4. Return plot objects without printing them. 5. Do not implement run_eda() or reporting in this PR. + +Closeout: +Update START_HERE.md to mark PR 7 Done and PR 8 Active. ``` ## Instruction 8: Add failing run_eda tests @@ -191,6 +237,10 @@ Requirements: 3. Use temporary directories for output-file tests. 4. Check expected named components: metadata, schema, missing, summaries, plots. 5. Do not implement run_eda() in this PR. + +Closeout: +Update START_HERE.md to mark PR 8 Done and PR 9 Active. Record that PR 8 may +leave tests failing until PR 9 implements run_eda(). ``` ## Instruction 9: Implement run_eda() @@ -214,6 +264,9 @@ Requirements: 9. Support mode = "real" and mode = "synthetic". 10. Clearly label synthetic-mode outputs. 11. Do not render reports in this PR. + +Closeout: +Update START_HERE.md to mark PR 9 Done and PR 10 Active. ``` ## Instruction 10: Report template and project template @@ -229,4 +282,8 @@ Next steps: 4. Add large-data design note. Follow spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md. + +Closeout: +Update START_HERE.md after each remaining PR so it stays the live status +dashboard. ``` From 061361312aaef16a762ab1c4fc7676328530a14a Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 15:27:41 -0600 Subject: [PATCH 02/33] active tests for branches --- .github/workflows/r-cmd-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/r-cmd-check.yml b/.github/workflows/r-cmd-check.yml index 1b423b0..a3ac3ea 100644 --- a/.github/workflows/r-cmd-check.yml +++ b/.github/workflows/r-cmd-check.yml @@ -10,7 +10,7 @@ on: push: - branches: [main, master] + branches: ["**"] #[main, master] pull_request: branches: [main, master] From a061f83618a0e7ae3c4301ab6e53323a08ab2329 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 15:44:46 -0600 Subject: [PATCH 03/33] added instructions for a second EDA based external truth test set, penguins raw --- .../codex-pr-add-penguins-raw-fixture.md | 458 ++++++++++++++++++ ...eview-checklist-external-truth-fixtures.md | 149 ++++++ 2 files changed, 607 insertions(+) create mode 100644 spec_driven_EDA_plan/future/codex-pr-add-penguins-raw-fixture.md create mode 100644 spec_driven_EDA_plan/future/codex-review-checklist-external-truth-fixtures.md diff --git a/spec_driven_EDA_plan/future/codex-pr-add-penguins-raw-fixture.md b/spec_driven_EDA_plan/future/codex-pr-add-penguins-raw-fixture.md new file mode 100644 index 0000000..3b33285 --- /dev/null +++ b/spec_driven_EDA_plan/future/codex-pr-add-penguins-raw-fixture.md @@ -0,0 +1,458 @@ +# Codex PR instruction: add `penguins_raw` as an external EDA truth fixture + +## Intended use + +Pass this file directly to Codex as the PR instruction. + +This PR should add a second external fixture for standard EDA validation, using `palmerpenguins::penguins_raw`. The aim is not to replace the existing `medicaldata::blood_storage` fixture. The aim is to complement it with a widely known public EDA dataset that has stable external documentation, common examples, mixed variable types, raw column names, and missingness. + +## Repository context + +Repository: + +```text +antoniojbt/episcout +``` + +Target branch: + +```text +episcout2 +``` + +Existing design principle: + +```text +external fixture and expected output first +failing fixture-backed tests second +minimal implementation third +``` + +Keep that sequence intact. + +The existing `blood_storage` fixture is appropriate for early biomedical EDA tests. It should remain the primary biomedical fixture for schema, missingness, and clinical-data realism. The new `penguins_raw` fixture should be added as a second external truth fixture for broader standard EDA behaviour, especially summaries, missingness, categorical handling, dates, labels, plots, and report examples. + +## Why this PR is needed + +The current `blood_storage` fixture is useful because it is real biomedical data, small, public, and already has committed expected schema and missingness outputs. It is less ideal as the only standard EDA truth fixture because it is not as widely used for canonical EDA examples as `palmerpenguins`. + +`palmerpenguins::penguins_raw` is useful because it provides: + +- a well-known public dataset; +- external package documentation and examples; +- 344 rows and 17 variables; +- raw-style column names, including spaces and punctuation; +- mixed variable types; +- dates; +- character/categorical variables; +- numeric morphometric variables; +- real missingness; +- common scatterplot, histogram, boxplot, grouped-summary and faceting examples. + +Use source documentation from: + +```text +https://allisonhorst.github.io/palmerpenguins/reference/penguins_raw.html +https://allisonhorst.github.io/palmerpenguins/articles/examples.html +``` + +## PR goal + +Create a small, reviewable PR that adds `penguins_raw` as a pinned external fixture and adds anti-circularity guardrails so future agent-generated implementation code cannot simply validate itself. + +Do not implement new EDA functions in this PR. + +## Scope + +### Must create + +```text +data-raw/test-fixtures/make_penguins_raw_fixture.R +tests/testthat/fixtures/penguins_raw/SOURCE.md +tests/testthat/fixtures/penguins_raw/penguins_raw.csv +tests/testthat/fixtures/penguins_raw/penguins_raw_spec.csv +tests/testthat/fixtures/penguins_raw/expected_missing.csv +tests/testthat/fixtures/penguins_raw/expected_summary_numeric.csv +tests/testthat/fixtures/penguins_raw/expected_summary_categorical.csv +tests/testthat/fixtures/penguins_raw/expected_plot_inventory.csv +``` + +### Should update + +```text +spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md +spec_driven_EDA_plan/docs/codex/review-checklist.md +tests/testthat/fixtures/README.md +``` + +### May create + +```text +tests/testthat/test-fixture-generation-guardrails.R +``` + +Only create this test if it is lightweight and does not run fixture-regeneration scripts. + +### Must not edit + +Do not edit implementation files for this PR: + +```text +R/eda_spec.R +R/eda_schema.R +R/eda_missing.R +R/eda_summaries.R +R/eda_plots.R +R/run_eda.R +``` + +Do not add or modify package behaviour. + +Do not add `palmerpenguins` to `Imports`. + +Do not make routine tests require internet access. + +Do not make routine tests require `palmerpenguins` to be installed. + +## Dependency policy + +`palmerpenguins` may be used by the manual regeneration script only. + +The committed fixture CSV files must be used by routine tests. + +The regeneration script must fail clearly if `palmerpenguins` is not installed: + +```r +if (!requireNamespace("palmerpenguins", quietly = TRUE)) { + stop( + "Package 'palmerpenguins' is required to regenerate penguins_raw fixtures. ", + "Install it with install.packages('palmerpenguins').", + call. = FALSE + ) +} +``` + +## Fixture-generation requirements + +The regeneration script must: + +1. Load `palmerpenguins::penguins_raw`. +2. Write `penguins_raw.csv` to `tests/testthat/fixtures/penguins_raw/`. +3. Write `SOURCE.md` with provenance. +4. Write a manually reviewed `penguins_raw_spec.csv`. +5. Write expected outputs using only base R or simple transparent calculations. +6. Avoid all `episcout` calls. +7. Avoid all new EDA function calls. +8. Avoid internet access. +9. Be manually run only; do not call it from routine tests. + +The script must not call any of these: + +```text +library(episcout) +episcout:: +eda_spec( +validate_eda_spec( +check_schema( +profile_missing( +profile_summaries( +profile_plots( +run_eda( +generate_synthetic_data( +``` + +## Expected outputs + +### `expected_missing.csv` + +Compute independently with base R: + +```r +expected_missing <- data.frame( + name = names(penguins_raw), + n = nrow(penguins_raw), + n_missing = vapply(penguins_raw, function(x) sum(is.na(x)), integer(1)), + p_missing = vapply(penguins_raw, function(x) mean(is.na(x)), numeric(1)), + stringsAsFactors = FALSE +) +``` + +### `expected_summary_numeric.csv` + +For numeric and integer variables, include at least: + +```text +name +class +n +n_missing +mean +sd +median +q25 +q75 +min +max +``` + +Use base R functions only: + +```r +numeric_cols <- names(penguins_raw)[vapply(penguins_raw, is.numeric, logical(1))] + +summarise_numeric <- function(x, name) { + non_missing <- x[!is.na(x)] + data.frame( + name = name, + class = class(x)[1], + n = length(x), + n_missing = sum(is.na(x)), + mean = mean(non_missing), + sd = stats::sd(non_missing), + median = stats::median(non_missing), + q25 = unname(stats::quantile(non_missing, 0.25, names = FALSE)), + q75 = unname(stats::quantile(non_missing, 0.75, names = FALSE)), + min = min(non_missing), + max = max(non_missing), + stringsAsFactors = FALSE + ) +} + +expected_summary_numeric <- do.call( + rbind, + Map(function(nm) summarise_numeric(penguins_raw[[nm]], nm), numeric_cols) +) +``` + +### `expected_summary_categorical.csv` + +For character, factor, logical and non-numeric categorical variables, include at least: + +```text +name +level +n +p +n_missing +``` + +Use base R only. Missing values should be counted separately or documented clearly. + +### `expected_plot_inventory.csv` + +This is not a visual snapshot. It is a plain expected contract for later plot-dispatch tests. + +Include at least: + +```text +name +expected_plot_type +reason +``` + +Suggested values: + +- numeric/integer continuous variables: `histogram` or `density`; +- categorical variables: `bar`; +- date variables: `time` or `skip` with a reason; +- free-text/comment variables: `skip` with a reason; +- unsupported variables: `skip` with a reason. + +Do not test visual appearance in this PR. + +## `penguins_raw_spec.csv` guidance + +Create a manually reviewed fixture data dictionary. Use the existing spec format already used by `blood_storage_spec.csv` where practical: + +```text +name +label +type +role +units +levels +min +max +missing_codes +required +group +description +``` + +Use source variable names exactly as they appear in `penguins_raw`. Do not silently clean column names in the fixture. + +Recommended type mapping: + +- character identifiers and labels: `categorical` or `text`, depending intended EDA handling; +- sex: `categorical`; +- island/species/region/study/stage: `categorical`; +- clutch completion: `binary` or `categorical`, but document the chosen convention; +- dates: `date`; +- body size measurements: `numeric`; +- egg count: `integer`; +- comments: `text`. + +The spec should be manually reviewed. Do not infer it entirely from the future implementation. + +## `SOURCE.md` minimum contents + +Include: + +```markdown +# penguins_raw fixture provenance + +## Source + +- Dataset: `penguins_raw` +- Source package: `palmerpenguins` +- Source package version used for fixture generation: +- Observations: 344 +- Variables: 17 +- Dataset type: ecological observational dataset +- Purpose: standard EDA truth fixture for specification-first EDA tests + +## Source documentation + +- https://allisonhorst.github.io/palmerpenguins/reference/penguins_raw.html +- https://allisonhorst.github.io/palmerpenguins/articles/examples.html + +## Fixture files + +- `penguins_raw.csv`: pinned data exported from `palmerpenguins::penguins_raw`. +- `penguins_raw_spec.csv`: manually reviewed fixture data dictionary. +- `expected_missing.csv`: independently computed expected missingness result. +- `expected_summary_numeric.csv`: independently computed numeric summaries. +- `expected_summary_categorical.csv`: independently computed categorical summaries. +- `expected_plot_inventory.csv`: independently defined plot-dispatch expectation. + +## Regeneration + +Run from the repository root: + +```sh +Rscript data-raw/test-fixtures/make_penguins_raw_fixture.R +``` + +The script computes expected outputs with base R and does not call `episcout` EDA functions. +``` + +## Documentation updates + +Update `spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md` to state: + +- `blood_storage` remains the first biomedical fixture. +- `scurvy` remains useful for tiny small-n edge-case tests. +- `penguins_raw` is added as the second standard EDA truth fixture. +- `penguins_raw` is intended for summary, missingness, plot-dispatch and report examples. +- `penguins_raw` should not displace `blood_storage` for biomedical tests. +- `penguins_raw` should not be used to justify implementation by matching implementation-generated output. + +Update `spec_driven_EDA_plan/docs/codex/review-checklist.md` to add explicit anti-circularity checks: + +```markdown +## External truth fixtures + +- [ ] Expected outputs were committed before implementation code that consumes them. +- [ ] Expected outputs were computed without calling `episcout` functions under test. +- [ ] Regeneration scripts do not call `episcout::`, `library(episcout)`, or new EDA functions. +- [ ] Fixture `SOURCE.md` records source package, version, row count, column count and source documentation. +- [ ] Routine tests use committed fixture files, not live downloads. +- [ ] Plot tests check contracts first, not visual appearance, unless a later visual-regression test is explicitly introduced. +``` + +Update `tests/testthat/fixtures/README.md` to include `penguins_raw` and explain the intended distinction: + +```text +blood_storage: biomedical fixture for clinical schema, missingness and workflow tests. +penguins_raw: standard public EDA fixture for mixed raw columns, common summaries, missingness and plot-dispatch contracts. +``` + +## Optional guardrail test + +If adding `tests/testthat/test-fixture-generation-guardrails.R`, keep it simple and deterministic. It should scan regeneration scripts and fail if they call package code under test. + +Example: + +```r +test_that("fixture regeneration scripts do not call episcout EDA functions", { + scripts <- c( + "data-raw/test-fixtures/make_external_fixtures.R", + "data-raw/test-fixtures/make_penguins_raw_fixture.R" + ) + scripts <- scripts[file.exists(scripts)] + + text <- unlist(lapply(scripts, readLines, warn = FALSE), use.names = FALSE) + + forbidden <- c( + "library\\(episcout\\)", + "episcout::", + "eda_spec\\(", + "validate_eda_spec\\(", + "check_schema\\(", + "profile_missing\\(", + "profile_summaries\\(", + "profile_plots\\(", + "run_eda\\(", + "generate_synthetic_data\\(" + ) + + for (pattern in forbidden) { + expect_false( + any(grepl(pattern, text)), + info = paste("Forbidden fixture-generation dependency found:", pattern) + ) + } +}) +``` + +Do not run regeneration scripts in this test. + +## Acceptance criteria + +The PR is acceptable if: + +1. `penguins_raw` fixture files are committed. +2. Fixture provenance is documented. +3. Expected missingness, numeric summaries, categorical summaries and plot inventory are committed. +4. Expected outputs are generated without `episcout`. +5. Routine tests do not require internet or `palmerpenguins`. +6. No EDA implementation files are changed. +7. Documentation clearly explains why `blood_storage` and `penguins_raw` both exist. +8. Review checklist includes anti-circularity checks. +9. `R CMD check` is not made worse by this PR. + +## PR title + +```text +Add penguins_raw external EDA truth fixture +``` + +## PR description template + +```markdown +## Summary + +Adds `palmerpenguins::penguins_raw` as a pinned external EDA truth fixture for specification-first EDA tests. + +This complements the existing `medicaldata::blood_storage` biomedical fixture. `blood_storage` remains the primary clinical fixture; `penguins_raw` is added for standard public EDA behaviour, including mixed raw variable names, missingness, numeric summaries, categorical summaries and plot-dispatch contracts. + +## What changed + +- Added a manual regeneration script for `penguins_raw` fixtures. +- Added pinned `penguins_raw` CSV fixture. +- Added manually reviewed `penguins_raw_spec.csv`. +- Added independently computed expected missingness, numeric summaries and categorical summaries. +- Added a plot-dispatch inventory for later plot tests. +- Updated fixture documentation and review guardrails. + +## What did not change + +- No new EDA implementation code. +- No changes to public package behaviour. +- No runtime dependency on `palmerpenguins`. +- No internet access required by routine tests. + +## Anti-circularity + +Expected outputs were computed with base R or manually reviewed contracts. The regeneration script does not call `episcout` EDA functions or implementation code under test. +``` diff --git a/spec_driven_EDA_plan/future/codex-review-checklist-external-truth-fixtures.md b/spec_driven_EDA_plan/future/codex-review-checklist-external-truth-fixtures.md new file mode 100644 index 0000000..207b76e --- /dev/null +++ b/spec_driven_EDA_plan/future/codex-review-checklist-external-truth-fixtures.md @@ -0,0 +1,149 @@ +# Review checklist: external truth fixtures for agent-generated EDA code + +Use this checklist when reviewing PRs that add or consume EDA fixture data. + +## Core rule + +The validation chain must be: + +```text +external fixture data ++ manually reviewed fixture specification ++ independently computed expected output +-> failing tests +-> implementation +``` + +Do not accept the inverse pattern: + +```text +implementation +-> generated output +-> tests that compare future output to that same implementation +``` + +That is circular validation. + +## Fixture source + +- [ ] The fixture comes from a public, documented source. +- [ ] The source is appropriate for the test purpose. +- [ ] The fixture is small enough for routine unit tests. +- [ ] The fixture is pinned as committed files under `tests/testthat/fixtures/`. +- [ ] Routine tests do not download live data. +- [ ] Routine tests do not require internet access. +- [ ] Routine tests do not require optional source packages to be installed. + +## Provenance + +Each fixture directory should include `SOURCE.md` with: + +- [ ] Dataset name. +- [ ] Source package or source institution. +- [ ] Source package version, if applicable. +- [ ] Source documentation URL. +- [ ] Row count. +- [ ] Column count. +- [ ] Dataset type and purpose. +- [ ] Regeneration script path. +- [ ] Statement that expected outputs were not computed with `episcout` functions under test. + +## Expected outputs + +- [ ] Expected files are committed beside the fixture data. +- [ ] Expected outputs are readable CSV or another simple, reviewable format. +- [ ] Expected outputs were generated before implementation code that consumes them. +- [ ] Expected outputs were computed with base R, simple transparent code, or a clearly independent reference. +- [ ] Expected outputs do not call package functions under test. +- [ ] Numeric summaries specify definitions for `sd`, quantiles and missing-value handling. +- [ ] Categorical summaries specify how missing values are counted. +- [ ] Plot expectations test contract and dispatch before visual appearance. + +## Regeneration scripts + +- [ ] Regeneration scripts are manual utilities, not routine test dependencies. +- [ ] Scripts fail clearly if optional source packages are unavailable. +- [ ] Scripts do not call `library(episcout)`. +- [ ] Scripts do not call `episcout::`. +- [ ] Scripts do not call EDA implementation functions under test. +- [ ] Scripts use deterministic output ordering. +- [ ] Scripts avoid hidden randomness; any randomness has a fixed seed. + +Forbidden calls in fixture-generation scripts: + +```text +library(episcout) +episcout:: +eda_spec( +validate_eda_spec( +check_schema( +profile_missing( +profile_summaries( +profile_plots( +run_eda( +generate_synthetic_data( +``` + +## Dataset roles + +Use fixtures for distinct purposes: + +```text +blood_storage +``` + +Biomedical fixture. Best for clinical schema, missingness, outcome/exposure roles, survival-style variables and realistic epidemiological data. + +```text +penguins_raw +``` + +Standard public EDA fixture. Best for raw column names, mixed variable types, missingness, grouped numeric summaries, categorical summaries, dates, common plots and report examples. + +```text +scurvy +``` + +Tiny fixture. Best for small-n edge cases and categorical handling. + +Do not expect one fixture to cover all behaviour. + +## PR review questions + +Before merging, answer: + +1. Does this PR add external truth, or does it merely preserve current implementation behaviour? +2. Could an agent have generated both the code and expected output from the same implementation path? +3. Are fixture outputs understandable by inspection? +4. Are there at least a few hard-coded sentinel values that can be checked manually? +5. Does the PR avoid broad refactoring? +6. Does the PR avoid changing implementation files when it is meant to add fixtures only? +7. Does the PR preserve the TDD sequence? + +## Minimum sentinel checks + +For each fixture, include at least: + +- [ ] Row count. +- [ ] Column count. +- [ ] Selected missing counts. +- [ ] Selected categorical counts. +- [ ] Selected numeric summaries. +- [ ] At least one unsupported or skipped variable, if plot dispatch is being tested. + +For `penguins_raw`, suitable sentinel checks include: + +```text +rows: 344 +columns: 17 +``` + +Additional sentinel values should be generated from the committed fixture using independent base R and then committed as expected CSV files. Do not compute them with the implementation under test. + +## Suggested PR label + +```text +testing +fixtures +spec-first-EDA +``` From 137afe3f2072c54c8acaf2de677f818986053bd6 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 16:02:08 -0600 Subject: [PATCH 04/33] prompt instructions, agent md, local R checks script --- spec_driven_EDA_plan/docs/AGENTS.md | 38 +++++++++++++++++++ .../future/scripts/check-local.sh | 6 +++ spec_driven_EDA_plan/prompt_loop.md | 36 ++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 spec_driven_EDA_plan/docs/AGENTS.md create mode 100644 spec_driven_EDA_plan/future/scripts/check-local.sh create mode 100644 spec_driven_EDA_plan/prompt_loop.md diff --git a/spec_driven_EDA_plan/docs/AGENTS.md b/spec_driven_EDA_plan/docs/AGENTS.md new file mode 100644 index 0000000..18c2a81 --- /dev/null +++ b/spec_driven_EDA_plan/docs/AGENTS.md @@ -0,0 +1,38 @@ +# AGENTS.md + +This file is specifically for work outside of the main branch. +It is for work contained only within this directory, `spec_driven_EDA_plan`. + +## Scope + +This repository is an R package. Follow the specification-first EDA plan. + +## Required reading + +Before editing files, read: + +- spec_driven_EDA_plan/docs/START_HERE.md +- spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md +- spec_driven_EDA_plan/docs/codex/review-checklist.md + +Treat START_HERE.md as the live source of truth. + +## Working rules + +- Complete only the active PR-sized task. +- Do not broaden scope. +- Do not perform unrelated refactoring. +- Respect must-edit and must-not-edit lists. +- Do not use episcout functions to generate expected fixture outputs. +- Use external fixtures and independently computed expected outputs. +- Update START_HERE.md at closeout. + +## Required checks + +Run where practical: + +```bash +Rscript -e "devtools::document()" +Rscript -e "devtools::test(reporter = 'summary')" +Rscript -e "devtools::check(manual = FALSE)" +R CMD check --no-manual . diff --git a/spec_driven_EDA_plan/future/scripts/check-local.sh b/spec_driven_EDA_plan/future/scripts/check-local.sh new file mode 100644 index 0000000..510d6f9 --- /dev/null +++ b/spec_driven_EDA_plan/future/scripts/check-local.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +Rscript -e 'devtools::document()' +Rscript -e 'devtools::test(reporter = "summary")' +Rscript -e 'devtools::check(manual = FALSE, error_on = "warning")' diff --git a/spec_driven_EDA_plan/prompt_loop.md b/spec_driven_EDA_plan/prompt_loop.md new file mode 100644 index 0000000..993b83c --- /dev/null +++ b/spec_driven_EDA_plan/prompt_loop.md @@ -0,0 +1,36 @@ +----------- +Follow Instruction 2 in: + +spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md + +for PR 2: Add failing tests for specification, schema and missingness +----------- + + +----------- +You are running locally in my Positron terminal. + +Repository: antoniojbt/episcout +Base branch: episcout2 + +Task: +Complete only the active PR-sized task in START_HERE.md. + +Required sequence: +1. Confirm the git worktree is clean. +2. Create a feature branch from episcout2. +3. Read AGENTS.md and START_HERE.md. +4. Make only the scoped changes. +5. Run: + - Rscript -e "devtools::document()" + - Rscript -e "devtools::test(reporter = 'summary')" + - Rscript -e "devtools::check(manual = FALSE)" +6. Fix errors, warnings and notes caused by this PR. +7. Do not fix unrelated pre-existing issues. +8. Commit the final change. +9. Push the branch. +10. Create a PR against episcout2 with gh. +11. Run gh pr checks --watch --fail-fast. +12. If CI fails because of this PR, fix, commit and push once. +13. Stop and summarise files changed, checks run and remaining issues. +----------- From 2c21e5131a5e6b01fd749184abe1a8c51cde667b Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 16:11:48 -0600 Subject: [PATCH 05/33] safer prompt loop --- spec_driven_EDA_plan/prompt_loop.md | 44 ++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/spec_driven_EDA_plan/prompt_loop.md b/spec_driven_EDA_plan/prompt_loop.md index 993b83c..07154b8 100644 --- a/spec_driven_EDA_plan/prompt_loop.md +++ b/spec_driven_EDA_plan/prompt_loop.md @@ -14,23 +14,41 @@ Repository: antoniojbt/episcout Base branch: episcout2 Task: -Complete only the active PR-sized task in START_HERE.md. +Complete only the active PR-sized task in spec_driven_EDA_plan/docs/START_HERE.md. +For the current active task, PR 2, expected failures from intentionally missing +EDA functions are allowed and must not be fixed in this PR. Required sequence: 1. Confirm the git worktree is clean. -2. Create a feature branch from episcout2. -3. Read AGENTS.md and START_HERE.md. -4. Make only the scoped changes. -5. Run: +2. Confirm GitHub CLI availability: + - command -v gh + - gh auth status + If gh is unavailable or unauthenticated, continue through local changes and + commit, then stop before push/PR/check steps and summarise what remains manual. +3. Ensure the current branch is episcout2 and up to date with origin/episcout2. +4. Create a feature branch from episcout2 using feature/. +5. Read: + - spec_driven_EDA_plan/docs/AGENTS.md + - spec_driven_EDA_plan/docs/START_HERE.md + - all files listed as required reading in START_HERE.md +6. Make only the scoped changes from START_HERE.md. +7. Respect must-edit and must-not-edit lists exactly. +8. Run: - Rscript -e "devtools::document()" - Rscript -e "devtools::test(reporter = 'summary')" - Rscript -e "devtools::check(manual = FALSE)" -6. Fix errors, warnings and notes caused by this PR. -7. Do not fix unrelated pre-existing issues. -8. Commit the final change. -9. Push the branch. -10. Create a PR against episcout2 with gh. -11. Run gh pr checks --watch --fail-fast. -12. If CI fails because of this PR, fix, commit and push once. -13. Stop and summarise files changed, checks run and remaining issues. +9. Fix only errors/warnings/notes caused by this PR, except expected failures + explicitly documented in START_HERE.md. +10. Do not implement functions reserved for later PRs. +11. Do not commit changes to files listed as must-not-edit. +12. Commit the final scoped change. +13. If gh is available and authenticated: + - push the branch + - create a PR against episcout2 with gh + - run gh pr checks --watch --fail-fast +14. If CI fails only because of expected PR 2 failing tests, record that and stop. +15. If CI fails because of an unintended issue introduced by this PR, fix, + commit and push once. +16. Stop and summarise files changed, checks run, expected failures, and any + remaining unrelated/pre-existing issues or manual GitHub steps. ----------- From 9abd72f201a0a490dd678c8ea777c3fbbd3e95d4 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 16:18:06 -0600 Subject: [PATCH 06/33] Add fixture-backed EDA contract tests --- spec_driven_EDA_plan/docs/START_HERE.md | 37 +++++++------- tests/testthat/test-eda_missing-fixtures.R | 56 +++++++++++++++++++++ tests/testthat/test-eda_schema-fixtures.R | 58 ++++++++++++++++++++++ tests/testthat/test-eda_spec-fixtures.R | 49 ++++++++++++++++++ 4 files changed, 183 insertions(+), 17 deletions(-) create mode 100644 tests/testthat/test-eda_missing-fixtures.R create mode 100644 tests/testthat/test-eda_schema-fixtures.R create mode 100644 tests/testthat/test-eda_spec-fixtures.R diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/spec_driven_EDA_plan/docs/START_HERE.md index 1606690..178db58 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/spec_driven_EDA_plan/docs/START_HERE.md @@ -22,20 +22,20 @@ operational source of truth for what happens next. |---|---|---| | Phase 0 documentation baseline | Done | Keep docs in sync as scope changes. | | PR 1 external fixture files | Done | Fixture files are present under `tests/testthat/fixtures/blood_storage/`. | -| PR 2 fixture-backed failing tests | Active | Follow Instruction 2 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | -| PR 3 spec, schema and missingness implementation | Not started | Implement only after PR 2 tests exist. | +| PR 2 fixture-backed failing tests | Done | Tests are present and may fail until PR 3 implements the missing functions. | +| PR 3 spec, schema and missingness implementation | Active | Follow Instruction 3 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | | Later synthetic, summary, plot, report and template work | Not started | Continue through the PR plan after PR 3. | ## Active PR ```text -PR 2: Add failing tests for specification, schema and missingness +PR 3: Implement spec, schema and missingness ``` Instruction: ```text -Follow Instruction 2 in: +Follow Instruction 3 in: spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md ``` @@ -52,14 +52,14 @@ spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md spec_driven_EDA_plan/docs/codex/review-checklist.md ``` -## PR 2 Scope +## PR 3 Scope Must edit: ```text -tests/testthat/test-eda_spec-fixtures.R -tests/testthat/test-eda_schema-fixtures.R -tests/testthat/test-eda_missing-fixtures.R +R/eda_spec.R +R/eda_schema.R +R/eda_missing.R spec_driven_EDA_plan/docs/START_HERE.md ``` @@ -70,25 +70,28 @@ tests/testthat/fixtures/blood_storage/blood_storage.csv tests/testthat/fixtures/blood_storage/blood_storage_spec.csv tests/testthat/fixtures/blood_storage/expected_schema.csv tests/testthat/fixtures/blood_storage/expected_missing.csv +tests/testthat/test-eda_spec-fixtures.R +tests/testthat/test-eda_schema-fixtures.R +tests/testthat/test-eda_missing-fixtures.R ``` Must not edit: ```text -R/eda_spec.R -R/eda_schema.R -R/eda_missing.R -DESCRIPTION -NAMESPACE -man/ +R/eda_synthetic.R +R/eda_summaries.R +R/eda_plots.R +R/run_eda.R +R/eda_report.R +inst/report-template/ +inst/project-template/ ``` Expected test state: ```text -Some new tests may fail because eda_spec(), validate_eda_spec(), -check_schema() and profile_missing() are intentionally not implemented until -PR 3. +PR 3 should make the PR 2 fixture-backed tests pass by implementing only +eda_spec(), validate_eda_spec(), check_schema() and profile_missing(). ``` ## Closeout Rule diff --git a/tests/testthat/test-eda_missing-fixtures.R b/tests/testthat/test-eda_missing-fixtures.R new file mode 100644 index 0000000..5aa1386 --- /dev/null +++ b/tests/testthat/test-eda_missing-fixtures.R @@ -0,0 +1,56 @@ +context("fixture-backed EDA missingness tests") + +library(testthat) +library(episcout) + +fixture_dir <- file.path("fixtures", "blood_storage") +data_path <- file.path(fixture_dir, "blood_storage.csv") +spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") +expected_missing_path <- file.path(fixture_dir, "expected_missing.csv") + +test_that("profile_missing matches independently computed expected missingness", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + expected <- read.csv(expected_missing_path, stringsAsFactors = FALSE) + + observed <- profile_missing(data, spec) + + expect_equal( + as.data.frame(observed), + expected, + tolerance = 1e-12, + ignore_attr = TRUE + ) +}) + +test_that("profile_missing reports the correct denominator for every variable", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + + observed <- profile_missing(data, spec) + + expect_true(all(observed$n == nrow(data))) +}) + +test_that("profile_missing percentages are stable with tolerance", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + expected <- read.csv(expected_missing_path, stringsAsFactors = FALSE) + + observed <- profile_missing(data, spec) + + expect_equal(observed$p_missing, expected$p_missing, tolerance = 1e-12) +}) + +test_that("profile_missing reports zero missingness for complete variables", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + expected <- read.csv(expected_missing_path, stringsAsFactors = FALSE) + complete_names <- expected$name[expected$n_missing == 0] + + observed <- profile_missing(data, spec) + complete_rows <- observed[observed$name %in% complete_names, ] + + expect_true(all(complete_rows$n_missing == 0)) + expect_true(all(complete_rows$p_missing == 0)) +}) diff --git a/tests/testthat/test-eda_schema-fixtures.R b/tests/testthat/test-eda_schema-fixtures.R new file mode 100644 index 0000000..bfc5e59 --- /dev/null +++ b/tests/testthat/test-eda_schema-fixtures.R @@ -0,0 +1,58 @@ +context("fixture-backed EDA schema tests") + +library(testthat) +library(episcout) + +fixture_dir <- file.path("fixtures", "blood_storage") +data_path <- file.path(fixture_dir, "blood_storage.csv") +spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") +expected_schema_path <- file.path(fixture_dir, "expected_schema.csv") + +test_that("check_schema matches independently computed expected schema", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + expected <- read.csv(expected_schema_path, stringsAsFactors = FALSE) + + observed <- check_schema(data, spec) + + expect_equal(as.data.frame(observed), expected, ignore_attr = TRUE) +}) + +test_that("check_schema flags missing expected variables", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + data$Age <- NULL + + observed <- check_schema(data, spec) + age_row <- observed[observed$name == "Age", ] + + expect_equal(nrow(age_row), 1) + expect_true(age_row$expected_present) + expect_false(age_row$observed_present) + expect_equal(age_row$status, "missing") +}) + +test_that("check_schema flags unexpected observed variables", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + data$UnexpectedVariable <- seq_len(nrow(data)) + + observed <- check_schema(data, spec) + unexpected_row <- observed[observed$name == "UnexpectedVariable", ] + + expect_equal(nrow(unexpected_row), 1) + expect_false(unexpected_row$expected_present) + expect_true(unexpected_row$observed_present) + expect_equal(unexpected_row$status, "unexpected") +}) + +test_that("all expected variables are present in the unmodified fixture", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + + observed <- check_schema(data, spec) + expected_rows <- observed[observed$expected_present, ] + + expect_true(all(expected_rows$observed_present)) + expect_false(any(expected_rows$status == "missing")) +}) diff --git a/tests/testthat/test-eda_spec-fixtures.R b/tests/testthat/test-eda_spec-fixtures.R new file mode 100644 index 0000000..19f39d9 --- /dev/null +++ b/tests/testthat/test-eda_spec-fixtures.R @@ -0,0 +1,49 @@ +context("fixture-backed EDA specification tests") + +library(testthat) +library(episcout) + +fixture_dir <- file.path("fixtures", "blood_storage") +spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") + +test_that("blood_storage_spec.csv loads as an EDA specification", { + raw_spec <- read.csv(spec_path, check.names = FALSE, stringsAsFactors = FALSE) + + spec <- eda_spec(spec_path) + + expect_s3_class(spec, "data.frame") + expect_equal(spec$name, raw_spec$name) + expect_true(all(c("name", "label", "type", "role") %in% names(spec))) +}) + +test_that("blood_storage_spec.csv has required specification columns", { + spec <- eda_spec(spec_path) + + expect_true(all(c("name", "label", "type", "role") %in% names(spec))) +}) + +test_that("blood_storage specification variable names are unique", { + spec <- eda_spec(spec_path) + + expect_false(anyDuplicated(spec$name)) +}) + +test_that("invalid specification type fails clearly", { + spec <- read.csv(spec_path, check.names = FALSE, stringsAsFactors = FALSE) + spec$type[1] <- "unsupported_type" + + expect_error( + validate_eda_spec(spec), + regexp = "[Tt]ype|[Uu]nsupported|[Ii]nvalid" + ) +}) + +test_that("duplicate specification variable name fails clearly", { + spec <- read.csv(spec_path, check.names = FALSE, stringsAsFactors = FALSE) + spec$name[2] <- spec$name[1] + + expect_error( + validate_eda_spec(spec), + regexp = "[Dd]uplicate|[Uu]nique|[Nn]ame" + ) +}) From 3444788d6bca4ce6831baef0a9cea5df668aa000 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 16:33:17 -0600 Subject: [PATCH 07/33] codex instructions --- spec_driven_EDA_plan/prompt_loop.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec_driven_EDA_plan/prompt_loop.md b/spec_driven_EDA_plan/prompt_loop.md index 07154b8..14246d3 100644 --- a/spec_driven_EDA_plan/prompt_loop.md +++ b/spec_driven_EDA_plan/prompt_loop.md @@ -18,6 +18,8 @@ Complete only the active PR-sized task in spec_driven_EDA_plan/docs/START_HERE.m For the current active task, PR 2, expected failures from intentionally missing EDA functions are allowed and must not be fixed in this PR. +Use `~/apps/miniforge3/envs/dsci/bin/Rscript` for all R commands. + Required sequence: 1. Confirm the git worktree is clean. 2. Confirm GitHub CLI availability: From 81b5ce70e3b4b2615a9ad20a817353413b336678 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 16:44:50 -0600 Subject: [PATCH 08/33] added mamba env and dev tools --- .Rbuildignore | 3 +++ README.md | 38 +++++++++++++++++++++++++++++++++ environment.yml | 48 ++++++++++++++++++++++++++++++++++++++++++ scripts/check-cran.sh | 32 ++++++++++++++++++++++++++++ scripts/check-local.sh | 12 +++++++++++ scripts/rscript | 23 ++++++++++++++++++++ 6 files changed, 156 insertions(+) create mode 100644 environment.yml create mode 100755 scripts/check-cran.sh create mode 100755 scripts/check-local.sh create mode 100755 scripts/rscript diff --git a/.Rbuildignore b/.Rbuildignore index dc43015..921a91e 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -23,4 +23,7 @@ ^AGENTS\.MD$ ^fishers_results\.txt$ ^spec_driven_EDA_plan$ +^environment\.yml$ +^scripts$ +^build$ # .Rbuildignore diff --git a/README.md b/README.md index 0ebc894..cc0081f 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,44 @@ library(devtools) install_github("AntonioJBT/episcout") ``` +## Development + +Use the repository development environment so local checks run with the same R +tooling in Positron, Codex and shell sessions. Create it once with: + +``` bash +mamba env create -f environment.yml +``` + +Update an existing environment with: + +``` bash +mamba env update -n episcout -f environment.yml --prune +``` + +Run package checks through the repository wrapper, not bare `Rscript`: + +``` bash +scripts/rscript -e "cat(R.home())" +scripts/check-local.sh +scripts/check-cran.sh +``` + +Set `EPISCOUT_RSCRIPT=/path/to/Rscript` if you need to use a different R +binary. + +CRAN does not require `renv`; it requires a source tarball from `R CMD build` +that passes `R CMD check --as-cran` without errors, warnings or significant +notes. Strong dependencies should be available from CRAN or Bioconductor, +suggested packages should be used conditionally in examples and tests, and +tests/examples should avoid internet requirements, unwanted filesystem writes +and excessive runtime or parallelism. See the CRAN Repository Policy, CRAN +submission checklist and Writing R Extensions for the current source of truth: + +- +- +- + ## Getting Started Functions are grouped by purpose, e.g.: diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..4a1bebc --- /dev/null +++ b/environment.yml @@ -0,0 +1,48 @@ +name: episcout +channels: + - conda-forge +dependencies: + - r-base + - pandoc + - qpdf + - ghostscript + - r-broom + - r-compare + - r-covr + - r-cowplot + - r-data.table + - r-devtools + - r-dofuture + - r-dplyr + - r-e1071 + - r-foreach + - r-future + - r-ggplot2 + - r-ggthemes + - r-hmisc + - r-iterators + - r-knitr + - r-lintr + - r-lubridate + - r-magrittr + - r-mice + - r-openxlsx + - r-purrr + - r-rcmdcheck + - r-remotes + - r-renv + - r-reshape2 + - r-rlang + - r-rmarkdown + - r-roxygen2 + - r-scales + - r-stringi + - r-stringr + - r-styler + - r-survival + - r-testthat + - r-tibble + - r-tidyr + - r-usethis + - r-vdiffr + - r-withr diff --git a/scripts/check-cran.sh b/scripts/check-cran.sh new file mode 100755 index 0000000..ee24c48 --- /dev/null +++ b/scripts/check-cran.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "${script_dir}/.." && pwd)" +rscript="${script_dir}/rscript" +check_dir="${repo_root}/build/cran-check" + +cd "$repo_root" + +mkdir -p "$check_dir" + +r_home="$("$rscript" -e 'cat(R.home())')" +r_bin="${r_home}/bin/R" + +if [[ ! -x "$r_bin" ]]; then + echo "R executable not found at ${r_bin}" >&2 + exit 127 +fi + +rm -f "${check_dir}"/*.tar.gz + +( + cd "$check_dir" + "$r_bin" CMD build "$repo_root" --no-resave-data --compact-vignettes=gs+qpdf + tarball="$(find . -maxdepth 1 -name 'episcout_*.tar.gz' -print -quit)" + if [[ -z "$tarball" ]]; then + echo "R CMD build did not produce an episcout source tarball" >&2 + exit 1 + fi + "$r_bin" CMD check --as-cran "$tarball" +) diff --git a/scripts/check-local.sh b/scripts/check-local.sh new file mode 100755 index 0000000..4708baa --- /dev/null +++ b/scripts/check-local.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "${script_dir}/.." && pwd)" +rscript="${script_dir}/rscript" + +cd "$repo_root" + +"$rscript" -e 'devtools::document()' +"$rscript" -e 'devtools::test(reporter = "summary")' +"$rscript" -e 'devtools::check(manual = FALSE)' diff --git a/scripts/rscript b/scripts/rscript new file mode 100755 index 0000000..ff51f3b --- /dev/null +++ b/scripts/rscript @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +default_rscript="${HOME}/apps/miniforge3/envs/episcout/bin/Rscript" +rscript="${EPISCOUT_RSCRIPT:-$default_rscript}" + +if [[ ! -x "$rscript" ]]; then + cat >&2 < Date: Sun, 14 Jun 2026 16:55:32 -0600 Subject: [PATCH 09/33] local env for checks --- scripts/check-cran.sh | 2 ++ scripts/rscript | 3 +++ spec_driven_EDA_plan/prompt_loop.md | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/check-cran.sh b/scripts/check-cran.sh index ee24c48..b445618 100755 --- a/scripts/check-cran.sh +++ b/scripts/check-cran.sh @@ -10,6 +10,8 @@ cd "$repo_root" mkdir -p "$check_dir" +export PATH="$("$rscript" -e 'cat(Sys.getenv("PATH"))')" + r_home="$("$rscript" -e 'cat(R.home())')" r_bin="${r_home}/bin/R" diff --git a/scripts/rscript b/scripts/rscript index ff51f3b..9157017 100755 --- a/scripts/rscript +++ b/scripts/rscript @@ -20,4 +20,7 @@ EOF exit 127 fi +env_bin="$(cd "$(dirname "$rscript")" && pwd)" +export PATH="${env_bin}:${PATH}" + exec "$rscript" "$@" diff --git a/spec_driven_EDA_plan/prompt_loop.md b/spec_driven_EDA_plan/prompt_loop.md index 14246d3..01e7fad 100644 --- a/spec_driven_EDA_plan/prompt_loop.md +++ b/spec_driven_EDA_plan/prompt_loop.md @@ -18,7 +18,7 @@ Complete only the active PR-sized task in spec_driven_EDA_plan/docs/START_HERE.m For the current active task, PR 2, expected failures from intentionally missing EDA functions are allowed and must not be fixed in this PR. -Use `~/apps/miniforge3/envs/dsci/bin/Rscript` for all R commands. +A local mamba env episcout should be available. Required sequence: 1. Confirm the git worktree is clean. From 71050ed694524dfe95c98048cf8962d9157e6990 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 17:11:46 -0600 Subject: [PATCH 10/33] PR3 complete for SDD for EDA flow --- NAMESPACE | 4 + R/eda_missing.R | 37 +++++++ R/eda_schema.R | 100 +++++++++++++++++++ R/eda_spec.R | 126 ++++++++++++++++++++++++ man/check_schema.Rd | 20 ++++ man/eda_spec.Rd | 18 ++++ man/profile_missing.Rd | 20 ++++ man/validate_eda_spec.Rd | 18 ++++ spec_driven_EDA_plan/docs/START_HERE.md | 24 ++--- tests/testthat/test-eda_spec-fixtures.R | 2 +- 10 files changed, 354 insertions(+), 15 deletions(-) create mode 100644 R/eda_missing.R create mode 100644 R/eda_schema.R create mode 100644 R/eda_spec.R create mode 100644 man/check_schema.Rd create mode 100644 man/eda_spec.Rd create mode 100644 man/profile_missing.Rd create mode 100644 man/validate_eda_spec.Rd diff --git a/NAMESPACE b/NAMESPACE index edb576e..eb94590 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,6 +3,8 @@ export("%>%") export(":=") export(.data) +export(check_schema) +export(eda_spec) export(enexpr) export(enexprs) export(enquo) @@ -86,12 +88,14 @@ export(expr) export(epi_grob_to_excel) export(palette_IMSS) export(palette_IMSS_accessible) +export(profile_missing) export(quo) export(quo_name) export(quos) export(rename_contingency_2x2_cols) export(sym) export(syms) +export(validate_eda_spec) importFrom(broom,tidy) importFrom(dplyr,bind_rows) importFrom(dplyr,group_by) diff --git a/R/eda_missing.R b/R/eda_missing.R new file mode 100644 index 0000000..5e92182 --- /dev/null +++ b/R/eda_missing.R @@ -0,0 +1,37 @@ +#' Profile missingness using an EDA specification +#' +#' Count standard missing values for variables listed in an EDA specification. +#' +#' @param data A data frame to profile. +#' @param spec An EDA specification data frame or CSV path. +#' +#' @return A data frame with variable names, row counts, missing counts and +#' missing proportions. +#' +#' @export +profile_missing <- function(data, spec) { + if (!is.data.frame(data)) { + stop("Data must be a data frame.", call. = FALSE) + } + + spec <- eda_spec(spec) + n <- nrow(data) + n_missing <- unname(vapply( + spec$name, + function(name) { + if (!name %in% names(data)) { + return(NA_integer_) + } + sum(is.na(data[[name]])) + }, + integer(1) + )) + + data.frame( + name = spec$name, + n = n, + n_missing = n_missing, + p_missing = n_missing / n, + stringsAsFactors = FALSE + ) +} diff --git a/R/eda_schema.R b/R/eda_schema.R new file mode 100644 index 0000000..11b3eed --- /dev/null +++ b/R/eda_schema.R @@ -0,0 +1,100 @@ +#' Check observed data against an EDA specification +#' +#' Compare expected variables in an EDA specification with variables observed in +#' a data frame. +#' +#' @param data A data frame to check. +#' @param spec An EDA specification data frame or CSV path. +#' +#' @return A data frame with one row per expected or unexpected variable. +#' +#' @export +check_schema <- function(data, spec) { + if (!is.data.frame(data)) { + stop("Data must be a data frame.", call. = FALSE) + } + + spec <- eda_spec(spec) + data_names <- names(data) + expected_names <- spec$name + + expected <- data.frame( + name = expected_names, + expected_type = spec$type, + observed_type = unname(vapply( + expected_names, + function(name) { + if (name %in% data_names) { + eda_observed_type(data[[name]]) + } else { + NA_character_ + } + }, + character(1) + )), + expected_present = TRUE, + observed_present = expected_names %in% data_names, + stringsAsFactors = FALSE + ) + expected$status <- ifelse(expected$observed_present, "present", "missing") + + unexpected_names <- setdiff(data_names, expected_names) + if (length(unexpected_names) == 0) { + return(expected[, schema_columns()]) + } + + unexpected <- data.frame( + name = unexpected_names, + expected_type = NA_character_, + observed_type = unname(vapply( + unexpected_names, + function(name) eda_observed_type(data[[name]]), + character(1) + )), + expected_present = FALSE, + observed_present = TRUE, + status = "unexpected", + stringsAsFactors = FALSE + ) + + rbind(expected[, schema_columns()], unexpected[, schema_columns()]) +} + +schema_columns <- function() { + c( + "name", + "expected_type", + "observed_type", + "expected_present", + "observed_present", + "status" + ) +} + +eda_observed_type <- function(x) { + if (inherits(x, "POSIXct") || inherits(x, "POSIXlt")) { + return("datetime") + } + + if (inherits(x, "Date")) { + return("date") + } + + if (is.factor(x)) { + return("categorical") + } + + if (is.numeric(x) || is.integer(x)) { + return("numeric") + } + + if (is.character(x)) { + return("text") + } + + if (is.logical(x)) { + return("binary") + } + + class(x)[1] +} diff --git a/R/eda_spec.R b/R/eda_spec.R new file mode 100644 index 0000000..f407297 --- /dev/null +++ b/R/eda_spec.R @@ -0,0 +1,126 @@ +#' Read an EDA specification +#' +#' Read and validate a specification-first EDA data dictionary. +#' +#' @param path_or_data A path to a CSV specification file, or a data frame +#' containing the specification. +#' +#' @return A validated data frame specification. +#' +#' @export +eda_spec <- function(path_or_data) { + if (is.character(path_or_data) && length(path_or_data) == 1) { + spec <- utils::read.csv( + path_or_data, + check.names = FALSE, + stringsAsFactors = FALSE + ) + } else if (is.data.frame(path_or_data)) { + spec <- as.data.frame(path_or_data, stringsAsFactors = FALSE) + } else { + stop("EDA specification must be a CSV path or a data frame.", call. = FALSE) + } + + validate_eda_spec(spec) +} + +#' Validate an EDA specification +#' +#' Validate the columns and basic values required by a specification-first EDA +#' data dictionary. +#' +#' @param spec A data frame containing an EDA specification. +#' +#' @return The validated specification as a data frame. +#' +#' @export +validate_eda_spec <- function(spec) { + if (!is.data.frame(spec)) { + stop("EDA specification must be a data frame.", call. = FALSE) + } + + spec <- as.data.frame(spec, stringsAsFactors = FALSE) + required_cols <- c("name", "label", "type", "role") + missing_cols <- setdiff(required_cols, names(spec)) + + if (length(missing_cols) > 0) { + stop( + "EDA specification is missing required columns: ", + paste(missing_cols, collapse = ", "), + call. = FALSE + ) + } + + spec$name <- as.character(spec$name) + empty_names <- is.na(spec$name) | trimws(spec$name) == "" + + if (any(empty_names)) { + stop("EDA specification variable names must be non-empty.", call. = FALSE) + } + + if (anyDuplicated(spec$name)) { + stop("Duplicate variable name found in EDA specification.", call. = FALSE) + } + + allowed_types <- c( + "numeric", + "integer", + "categorical", + "binary", + "date", + "datetime", + "text" + ) + spec$type <- tolower(as.character(spec$type)) + invalid_type <- is.na(spec$type) | !(spec$type %in% allowed_types) + + if (any(invalid_type)) { + stop( + "Invalid type in EDA specification: ", + paste(unique(spec$type[invalid_type]), collapse = ", "), + call. = FALSE + ) + } + + if ("required" %in% names(spec)) { + spec$required <- parse_eda_spec_logical(spec$required) + } + + validate_eda_spec_ranges(spec) + spec +} + +parse_eda_spec_logical <- function(x) { + if (is.logical(x)) { + return(x) + } + + values <- trimws(tolower(as.character(x))) + missing_values <- is.na(x) | values == "" + out <- rep(NA, length(values)) + out[values %in% c("true", "t", "1", "yes", "y")] <- TRUE + out[values %in% c("false", "f", "0", "no", "n")] <- FALSE + invalid <- is.na(out) & !missing_values + + if (any(invalid)) { + stop("Invalid required value in EDA specification.", call. = FALSE) + } + + out +} + +validate_eda_spec_ranges <- function(spec) { + if (!all(c("min", "max") %in% names(spec))) { + return(invisible(TRUE)) + } + + min_value <- suppressWarnings(as.numeric(spec$min)) + max_value <- suppressWarnings(as.numeric(spec$max)) + comparable <- !is.na(min_value) & !is.na(max_value) + + if (any(comparable & min_value > max_value)) { + stop("EDA specification min must not be greater than max.", call. = FALSE) + } + + invisible(TRUE) +} diff --git a/man/check_schema.Rd b/man/check_schema.Rd new file mode 100644 index 0000000..04ed79b --- /dev/null +++ b/man/check_schema.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/eda_schema.R +\name{check_schema} +\alias{check_schema} +\title{Check observed data against an EDA specification} +\usage{ +check_schema(data, spec) +} +\arguments{ +\item{data}{A data frame to check.} + +\item{spec}{An EDA specification data frame or CSV path.} +} +\value{ +A data frame with one row per expected or unexpected variable. +} +\description{ +Compare expected variables in an EDA specification with variables observed in +a data frame. +} diff --git a/man/eda_spec.Rd b/man/eda_spec.Rd new file mode 100644 index 0000000..5bcf5cc --- /dev/null +++ b/man/eda_spec.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/eda_spec.R +\name{eda_spec} +\alias{eda_spec} +\title{Read an EDA specification} +\usage{ +eda_spec(path_or_data) +} +\arguments{ +\item{path_or_data}{A path to a CSV specification file, or a data frame +containing the specification.} +} +\value{ +A validated data frame specification. +} +\description{ +Read and validate a specification-first EDA data dictionary. +} diff --git a/man/profile_missing.Rd b/man/profile_missing.Rd new file mode 100644 index 0000000..56ba465 --- /dev/null +++ b/man/profile_missing.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/eda_missing.R +\name{profile_missing} +\alias{profile_missing} +\title{Profile missingness using an EDA specification} +\usage{ +profile_missing(data, spec) +} +\arguments{ +\item{data}{A data frame to profile.} + +\item{spec}{An EDA specification data frame or CSV path.} +} +\value{ +A data frame with variable names, row counts, missing counts and +missing proportions. +} +\description{ +Count standard missing values for variables listed in an EDA specification. +} diff --git a/man/validate_eda_spec.Rd b/man/validate_eda_spec.Rd new file mode 100644 index 0000000..e13cc34 --- /dev/null +++ b/man/validate_eda_spec.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/eda_spec.R +\name{validate_eda_spec} +\alias{validate_eda_spec} +\title{Validate an EDA specification} +\usage{ +validate_eda_spec(spec) +} +\arguments{ +\item{spec}{A data frame containing an EDA specification.} +} +\value{ +The validated specification as a data frame. +} +\description{ +Validate the columns and basic values required by a specification-first EDA +data dictionary. +} diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/spec_driven_EDA_plan/docs/START_HERE.md index 178db58..afab459 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/spec_driven_EDA_plan/docs/START_HERE.md @@ -23,19 +23,20 @@ operational source of truth for what happens next. | Phase 0 documentation baseline | Done | Keep docs in sync as scope changes. | | PR 1 external fixture files | Done | Fixture files are present under `tests/testthat/fixtures/blood_storage/`. | | PR 2 fixture-backed failing tests | Done | Tests are present and may fail until PR 3 implements the missing functions. | -| PR 3 spec, schema and missingness implementation | Active | Follow Instruction 3 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | +| PR 3 spec, schema and missingness implementation | Done | Spec, schema and missingness functions are implemented. | +| PR 4 synthetic-data failing tests | Active | Follow Instruction 4 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | | Later synthetic, summary, plot, report and template work | Not started | Continue through the PR plan after PR 3. | ## Active PR ```text -PR 3: Implement spec, schema and missingness +PR 4: Add failing synthetic-data tests ``` Instruction: ```text -Follow Instruction 3 in: +Follow Instruction 4 in: spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md ``` @@ -52,14 +53,12 @@ spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md spec_driven_EDA_plan/docs/codex/review-checklist.md ``` -## PR 3 Scope +## PR 4 Scope Must edit: ```text -R/eda_spec.R -R/eda_schema.R -R/eda_missing.R +tests/testthat/test-eda_synthetic-fixtures.R spec_driven_EDA_plan/docs/START_HERE.md ``` @@ -68,17 +67,15 @@ May read: ```text tests/testthat/fixtures/blood_storage/blood_storage.csv tests/testthat/fixtures/blood_storage/blood_storage_spec.csv -tests/testthat/fixtures/blood_storage/expected_schema.csv -tests/testthat/fixtures/blood_storage/expected_missing.csv -tests/testthat/test-eda_spec-fixtures.R -tests/testthat/test-eda_schema-fixtures.R -tests/testthat/test-eda_missing-fixtures.R ``` Must not edit: ```text R/eda_synthetic.R +R/eda_spec.R +R/eda_schema.R +R/eda_missing.R R/eda_summaries.R R/eda_plots.R R/run_eda.R @@ -90,8 +87,7 @@ inst/project-template/ Expected test state: ```text -PR 3 should make the PR 2 fixture-backed tests pass by implementing only -eda_spec(), validate_eda_spec(), check_schema() and profile_missing(). +PR 4 may leave tests failing until PR 5 implements generate_synthetic_data(). ``` ## Closeout Rule diff --git a/tests/testthat/test-eda_spec-fixtures.R b/tests/testthat/test-eda_spec-fixtures.R index 19f39d9..9b0fc89 100644 --- a/tests/testthat/test-eda_spec-fixtures.R +++ b/tests/testthat/test-eda_spec-fixtures.R @@ -25,7 +25,7 @@ test_that("blood_storage_spec.csv has required specification columns", { test_that("blood_storage specification variable names are unique", { spec <- eda_spec(spec_path) - expect_false(anyDuplicated(spec$name)) + expect_equal(anyDuplicated(spec$name), 0L) }) test_that("invalid specification type fails clearly", { From dd40d228c78683895f49e4f68a4c1991082afe42 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 17:30:31 -0600 Subject: [PATCH 11/33] phase 1 wasn't done, will do later, SDD here --- ...codex-pr-deferred-phase-1-stabilization.md | 262 ++++++++++++++++++ .../future/phase-1-stabilization-sdd-tdd.md | 122 ++++++++ spec_driven_EDA_plan/prompt_loop.md | 37 ++- 3 files changed, 418 insertions(+), 3 deletions(-) create mode 100644 spec_driven_EDA_plan/future/codex-pr-deferred-phase-1-stabilization.md create mode 100644 spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md diff --git a/spec_driven_EDA_plan/future/codex-pr-deferred-phase-1-stabilization.md b/spec_driven_EDA_plan/future/codex-pr-deferred-phase-1-stabilization.md new file mode 100644 index 0000000..57a29b4 --- /dev/null +++ b/spec_driven_EDA_plan/future/codex-pr-deferred-phase-1-stabilization.md @@ -0,0 +1,262 @@ +# Codex PR instruction: deferred Phase 1 stabilization + +## Intended use + +Pass this file directly to Codex only when you want to activate the deferred +Phase 1 stabilization work. + +This instruction is not part of the live PR sequence. Do not run it unless the +user explicitly asks to do deferred Phase 1 stabilization. + +## Repository context + +Repository: + +```text +antoniojbt/episcout +``` + +Target branch: + +```text +episcout2 +``` + +Current live workflow: + +```text +spec_driven_EDA_plan/docs/START_HERE.md +``` + +The live workflow remains the source of truth for the active spec-first PR +sequence. This deferred Phase 1 PR should not change the active PR unless the +user explicitly requests that. + +## When to use this + +Use this instruction when one of these triggers is present: + +- `profile_summaries()` depends on `epi_stats_numeric()` or + `epi_stats_summary()`. +- `run_eda()` depends on older `epi_*` helpers. +- CI, `R CMD check`, or local tests expose helper failures. +- A release or broader package hardening pass is being prepared. +- The user explicitly asks to activate deferred Phase 1 stabilization. + +If none of these triggers is present, continue the active PR-sized task in +`START_HERE.md` instead. + +## Required reading + +Read these files before making changes: + +```text +AGENTS.MD +spec_driven_EDA_plan/docs/START_HERE.md +spec_driven_EDA_plan/docs/roadmap.md +spec_driven_EDA_plan/docs/repository-audit.md +spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md +spec_driven_EDA_plan/docs/codex/review-checklist.md +``` + +## PR goal + +Create a small TDD-first hardening PR for the highest-risk existing helper edge +cases without changing the current spec-first EDA PR sequence. + +The first deferred Phase 1 PR should focus on: + +```text +epi_stats_numeric() +epi_stats_na_perc() +baseline local check/test status +``` + +Do not broaden the PR into package-wide modernization. + +## Scope + +This PR should: + +- record baseline local test/check status before behaviour changes; +- add failing tests for selected helper edge cases; +- implement only the helper changes needed to pass those tests; +- preserve backwards compatibility where practical; +- document pre-existing or unrelated failures rather than fixing them. + +## Must edit + +Edit only the files needed for the selected helper tests and implementation. + +Expected files for the first deferred Phase 1 PR: + +```text +tests/testthat/test-epi_stats_numeric.R +tests/testthat/test-stats.R +R/epi_stats_numeric.R +R/epi_stats_na_perc.R +``` + +If a narrower test file is clearer, create or edit a focused test file under: + +```text +tests/testthat/ +``` + +## May edit + +Only if required by roxygen/documentation updates: + +```text +man/ +NAMESPACE +``` + +Only if the PR needs to record the deferred Phase 1 closeout: + +```text +spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md +``` + +## Must not edit + +Do not edit current spec-first implementation files unless a failing test proves +that the deferred hardening PR directly requires it: + +```text +R/eda_spec.R +R/eda_schema.R +R/eda_missing.R +R/eda_synthetic.R +R/eda_summaries.R +R/eda_plots.R +R/run_eda.R +R/eda_report.R +``` + +Do not edit fixture files: + +```text +tests/testthat/fixtures/ +``` + +Do not edit report or project templates: + +```text +inst/report-template/ +inst/project-template/ +``` + +Do not edit active PR task files from: + +```text +spec_driven_EDA_plan/docs/START_HERE.md +``` + +unless the user explicitly asks to change the live workflow. + +## TDD requirements + +Add tests before implementation for the selected edge cases. + +Initial test targets: + +- `epi_stats_numeric()` handles all-missing numeric vectors. +- `epi_stats_numeric()` handles zero-length numeric vectors. +- `epi_stats_numeric()` returns a stable coefficient of variation when mean is + zero. +- `epi_stats_numeric()` skips normality, skewness, and kurtosis safely when + there are insufficient usable values. +- `epi_stats_na_perc()` rejects invalid `margin` with a clear error. +- `epi_stats_na_perc()` handles mixed-type data frames without coercion-related + count errors. +- `epi_stats_na_perc()` keeps a stable return shape for row and column modes. + +Tests should define the desired contract explicitly. Do not write tests that +only preserve accidental current behaviour. + +## Baseline commands + +Run these before implementation where practical: + +```bash +Rscript -e "devtools::test(reporter = 'summary')" +Rscript -e "devtools::check(manual = FALSE)" +Rscript -e "covr::package_coverage()" +``` + +If required packages are missing locally, do not install them unless the user +approves. Record the missing dependency and continue with focused tests that can +run in the available environment. + +Run focused tests after adding failing tests and after implementation. Examples: + +```bash +Rscript -e "testthat::test_file('tests/testthat/test-epi_stats_numeric.R')" +Rscript -e "testthat::test_file('tests/testthat/test-stats.R')" +``` + +Run broader checks after implementation where practical: + +```bash +Rscript -e "devtools::document()" +Rscript -e "devtools::test(reporter = 'summary')" +Rscript -e "devtools::check(manual = FALSE)" +``` + +## Implementation constraints + +Keep the change small and reviewable. + +Preserve: + +- exported function names; +- argument names where practical; +- data-frame-like return values; +- existing successful tests; +- backwards compatibility unless an old behaviour is clearly erroneous and now + covered by tests. + +Avoid: + +- package-wide refactoring; +- broad `select_if()` modernization; +- adding heavy dependencies; +- changing the spec-first EDA API; +- changing fixtures or expected fixture outputs; +- fixing unrelated audit findings in the same PR. + +## Acceptance criteria + +The PR is ready when: + +- selected edge-case tests fail before implementation and pass after; +- focused helper tests pass; +- broader package tests/checks are run where practical; +- any missing local dependencies or pre-existing failures are recorded; +- no files outside the deferred Phase 1 scope are changed; +- the current `START_HERE.md` active PR is not changed unless explicitly + requested. + +## Closeout + +Summarise: + +- why deferred Phase 1 was activated; +- files changed; +- tests added; +- helper behaviours hardened; +- checks run and results; +- expected or pre-existing failures; +- any follow-up Phase 1 work left out of scope. + +Do not mark Phase 1 as globally complete unless all roadmap Phase 1 goals are +actually satisfied: + +```text +run R CMD check +run test suite +record baseline coverage +identify fragile functions +fix clear edge cases +``` diff --git a/spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md b/spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md new file mode 100644 index 0000000..963383c --- /dev/null +++ b/spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md @@ -0,0 +1,122 @@ +# Deferred Phase 1 stabilization plan + +## Status + +Phase 1 in `spec_driven_EDA_plan/docs/roadmap.md` is not complete. + +`spec_driven_EDA_plan/docs/repository-audit.md` partially satisfies the audit +portion by identifying fragile existing helpers, but there is no completed +Phase 1 PR sequence, baseline check record, coverage baseline, or targeted +hardening PR. + +This work is intentionally deferred. It must not block the active PR-sized task +in `spec_driven_EDA_plan/docs/START_HERE.md`. + +## Decision + +Do not interrupt the current specification-first TDD sequence to complete Phase +1. Continue the active PR flow unless the user explicitly asks to activate this +deferred stabilization plan. + +When activated, Phase 1 stabilization should be a small, TDD-first hardening PR +for existing helper behaviour, not a broad modernization of the package. + +## Why this is deferred + +The current spec-first EDA work is already organized as small TDD slices: + +```text +external fixture and expected output first +failing fixture-backed tests second +minimal implementation third +``` + +The active spec-first tasks do not yet require broad changes to older `epi_*` +helpers. Completing Phase 1 now would add scope and review risk without being +necessary for the current PR. + +The older helpers still matter, but they should be hardened when new work +depends on them or when package checks expose concrete failures. + +## Reopen triggers + +Reopen this deferred Phase 1 work when any of the following is true: + +- `profile_summaries()` depends on `epi_stats_numeric()` or + `epi_stats_summary()`. +- `run_eda()` depends on older `epi_*` helpers. +- CI, `R CMD check`, or local tests expose helper failures. +- The package is being prepared for release or broader hardening. +- The user explicitly asks to run the deferred Phase 1 stabilization PR. + +## SDD intent + +The intent is to preserve existing public behaviour while making fragile helper +edge cases explicit and testable. + +Phase 1 stabilization should: + +- keep existing exported function names and return shapes stable where + practical; +- improve clear failure modes for invalid inputs; +- avoid hidden state and broad refactoring; +- document any intentional behaviour changes in tests and roxygen comments; +- leave the specification-first API sequence intact unless a direct dependency + requires coordination. + +## TDD sequence + +Use this sequence when the deferred work is activated: + +1. Record the local baseline status for tests/checks before changing behaviour. +2. Add failing tests for the selected helper edge cases. +3. Implement the smallest code change that passes those tests. +4. Run focused tests for the changed helpers. +5. Run broader package tests/checks where practical. +6. Fix only failures caused by this PR. +7. Document any pre-existing or unrelated failures in the PR summary. + +Do not write tests that merely lock in current broken behaviour. Tests should +describe the intended stable contract. + +## Initial edge cases + +Start with these edge cases: + +- `epi_stats_numeric()` with all-missing numeric vectors. +- `epi_stats_numeric()` with zero-length vectors. +- `epi_stats_numeric()` coefficient of variation when mean is zero. +- `epi_stats_numeric()` normality, skewness, and kurtosis behaviour with + insufficient usable values. +- `epi_stats_na_perc()` with invalid `margin`. +- `epi_stats_na_perc()` with mixed-type data frames. +- `epi_stats_na_perc()` return shape for row and column modes. + +Prefer one small PR for `epi_stats_numeric()` and `epi_stats_na_perc()` before +expanding to other helpers. + +## Acceptance criteria + +The deferred Phase 1 PR is complete when: + +- failing tests are added before implementation changes; +- `epi_stats_numeric()` and `epi_stats_na_perc()` handle the selected edge cases + deterministically; +- public function names remain unchanged; +- return objects remain data frames or tibbles compatible with existing tests; +- invalid input errors are clear and actionable; +- baseline and final local check/test status are recorded in the PR summary; +- unrelated helper modernization is not included. + +## Out of scope + +Do not include these unless a failing test or check makes them necessary: + +- broad replacement of all `select_if()` calls; +- redesign of the public `epi_*` API; +- removal or renaming of exported functions; +- changes to fixture files; +- changes to spec-first implementation files; +- report or project-template work; +- large-data backend implementation; +- package-wide style rewrites. diff --git a/spec_driven_EDA_plan/prompt_loop.md b/spec_driven_EDA_plan/prompt_loop.md index 01e7fad..84ebe6d 100644 --- a/spec_driven_EDA_plan/prompt_loop.md +++ b/spec_driven_EDA_plan/prompt_loop.md @@ -1,13 +1,44 @@ ----------- -Follow Instruction 2 in: +Reading order and mapping for SDD/TDD in this directory: +- START_HERE.md for what to do now. +- revised-pr-plan-tdd-first.md for where the current PR sits in the sequence. +- tdd-first-codex-instructions.md for the exact task block. +- roadmap.md for why this work exists and which phase it belongs to. +- review-checklist.md before closing any PR. -spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md -for PR 2: Add failing tests for specification, schema and missingness + +The mapping is hierarchical, not one-to-one: + +**Control Files** +`roadmap.md` is the strategic map: phases and broad intent. + +`START_HERE.md` is the live operational state. It explicitly says it is the “operational source of truth” for what happens next, including active PR, allowed files, and expected test state ([START_HERE.md]()). + +`revised-pr-plan-tdd-first.md` is the stable PR sequence. It translates roadmap phases into PR-sized work and enforces the pattern: fixture/expected output, failing tests, implementation ([revised-pr-plan-tdd-first.md]()). + +`tdd-first-codex-instructions.md` is the executable instruction bank. Each instruction corresponds roughly to one PR, especially PRs 1-9. Later work is collapsed into Instruction 10 ([tdd-first-codex-instructions.md]()). + +`review-checklist.md` applies to every PR before closeout/review ([review-checklist.md]()). + +**Phase Mapping** + +| Roadmap phase | Actual operational mapping | +|---|---| +| Phase 0: Documentation and baseline | Done. This produced SDD, ADRs, roadmap, Codex instructions, and PR plan. Tracked in `START_HERE.md` as done. | +| Phase 1: Stabilise current package | Not clearly mapped to PRs. It has audit material, but no active PR sequence or completion evidence. Looks bypassed/deferred. | +| Phase 2: TDD fixtures for spec-first EDA | PR 1 and PR 2. PR 1 adds external fixtures and expected outputs. PR 2 adds failing tests for spec/schema/missingness. | +| Phase 3: Specification-first MVP | PR 3 through PR 9, but in test-first slices: implement spec/schema/missingness, test synthetic data, implement synthetic data, test summaries/plots, implement summaries/plots, test `run_eda()`, implement `run_eda()`. Current active PR is PR 4. | +| Phase 4: Reporting and project template | PR 10, PR 11, PR 12. Report tests, report implementation, project template. | +| Phase 5: Large-data optimisation | PR 13 only documents large-data strategy. Actual `data.table`, DuckDB, Arrow implementation is explicitly not planned yet. | +| Phase 6: Biomedical extensions | No PRs/instructions yet. It is future scope only. | + ----------- ----------- +Prompt that can be re-used at each stage/step: + You are running locally in my Positron terminal. Repository: antoniojbt/episcout From 54132dc59e324b3f2e01f3d1c8b385391597113d Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 17:34:08 -0600 Subject: [PATCH 12/33] clarified what 'fragile' helper functions means --- .../future/phase-1-stabilization-sdd-tdd.md | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md b/spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md index 963383c..8654d11 100644 --- a/spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md +++ b/spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md @@ -12,6 +12,33 @@ hardening PR. This work is intentionally deferred. It must not block the active PR-sized task in `spec_driven_EDA_plan/docs/START_HERE.md`. +Fragile was identified as edge cases which can produce warnings, unclear errors, or unstable numeric results. The functions are likely fine for common data, but their contracts are under-specified at the boundaries. TDD hardening would make those boundaries explicit. + +For instance: + +For `epi_stats_numeric()`: + +- **All-missing vectors:** calls like `min(x, na.rm = TRUE)`, `max()`, `quantile()`, `IQR()`, `sd()`, `var()`, `skewness()`, and `kurtosis()` do not all have clean, consistent behavior when there are no usable values. +- **Zero-length vectors:** `n = 0` makes `NA_percentage <- (NA_count / n) * 100` become division by zero/`NaN`. +- **Mean is zero:** `CV = sd / mean` can become `Inf`, `-Inf`, or `NaN`. That needs an explicit contract. +- **Shapiro-Wilk handling:** the code checks `n_nonNA`, but passes the original vector to `shapiro.test()`. If there are `NA`s, it relies on `tryCatch()` rather than intentionally passing cleaned values. +- **Repeated calculations:** `sd()` and `mean()` are recalculated multiple times, increasing room for inconsistent handling when edge cases are added. +- **Dependency behavior:** it hard-stops if `e1071` is unavailable, even though some simpler stats could still be computed. Since `e1071` is in `Imports`, that is acceptable in installed-package use, but it makes local/source testing brittle if dependencies are not installed. + +For `epi_stats_na_perc()`: + +- **Uses `apply()` on data frames:** `apply()` coerces data frames to a matrix first. With mixed types, that can silently coerce numbers, characters, dates, factors, etc. before counting. +- **Invalid `margin`:** if `margin` is not `1` or `2`, the function falls through and returns an undefined object error: `object 'na_perc_all' not found`. +- **Return shape differs by mode:** column mode returns transposed row names as variable labels; row mode returns a plain data frame from an unnamed vector. That can be okay, but it needs a stable tested contract. +- **No input validation:** `df = NULL`, non-data-frame input, zero-row/zero-column inputs, and invalid margins are not handled deliberately. + +For `epi_stats_summary()`: + +- **Older tidyverse idioms:** `select_if()` still works in many setups, but it is superseded and more likely to become noisy or awkward in future dplyr versions. +- **`expression()` + `eval()` dispatch:** harder to reason about, harder to test, and easier to break when refactoring. +- **Depends on lower helpers:** if `epi_stats_numeric()` is fragile, then numeric summary mode inherits that fragility. + + ## Decision Do not interrupt the current specification-first TDD sequence to complete Phase From 7089f0693f6351f75e047ece0349ed31d8b3d5e1 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 22:55:26 -0600 Subject: [PATCH 13/33] Add synthetic data fixture tests --- spec_driven_EDA_plan/docs/START_HERE.md | 18 ++--- tests/testthat/test-eda_synthetic-fixtures.R | 71 ++++++++++++++++++++ 2 files changed, 80 insertions(+), 9 deletions(-) create mode 100644 tests/testthat/test-eda_synthetic-fixtures.R diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/spec_driven_EDA_plan/docs/START_HERE.md index afab459..8745fd9 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/spec_driven_EDA_plan/docs/START_HERE.md @@ -24,19 +24,20 @@ operational source of truth for what happens next. | PR 1 external fixture files | Done | Fixture files are present under `tests/testthat/fixtures/blood_storage/`. | | PR 2 fixture-backed failing tests | Done | Tests are present and may fail until PR 3 implements the missing functions. | | PR 3 spec, schema and missingness implementation | Done | Spec, schema and missingness functions are implemented. | -| PR 4 synthetic-data failing tests | Active | Follow Instruction 4 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | -| Later synthetic, summary, plot, report and template work | Not started | Continue through the PR plan after PR 3. | +| PR 4 synthetic-data failing tests | Done | Tests are present and may fail until PR 5 implements `generate_synthetic_data()`. | +| PR 5 synthetic-data implementation | Active | Follow Instruction 5 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | +| Later summary, plot, report and template work | Not started | Continue through the PR plan after PR 5. | ## Active PR ```text -PR 4: Add failing synthetic-data tests +PR 5: Implement synthetic-data generation ``` Instruction: ```text -Follow Instruction 4 in: +Follow Instruction 5 in: spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md ``` @@ -53,26 +54,25 @@ spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md spec_driven_EDA_plan/docs/codex/review-checklist.md ``` -## PR 4 Scope +## PR 5 Scope Must edit: ```text -tests/testthat/test-eda_synthetic-fixtures.R +R/eda_synthetic.R spec_driven_EDA_plan/docs/START_HERE.md ``` May read: ```text -tests/testthat/fixtures/blood_storage/blood_storage.csv +tests/testthat/test-eda_synthetic-fixtures.R tests/testthat/fixtures/blood_storage/blood_storage_spec.csv ``` Must not edit: ```text -R/eda_synthetic.R R/eda_spec.R R/eda_schema.R R/eda_missing.R @@ -87,7 +87,7 @@ inst/project-template/ Expected test state: ```text -PR 4 may leave tests failing until PR 5 implements generate_synthetic_data(). +PR 5 should make the synthetic-data tests pass. ``` ## Closeout Rule diff --git a/tests/testthat/test-eda_synthetic-fixtures.R b/tests/testthat/test-eda_synthetic-fixtures.R new file mode 100644 index 0000000..a1bca77 --- /dev/null +++ b/tests/testthat/test-eda_synthetic-fixtures.R @@ -0,0 +1,71 @@ +context("fixture-backed EDA synthetic-data tests") + +library(testthat) +library(episcout) + +fixture_dir <- file.path("fixtures", "blood_storage") +spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") + +spec <- eda_spec(spec_path) + +split_levels <- function(x) { + strsplit(x, ";", fixed = TRUE)[[1]] +} + +test_that("synthetic data has the same variable names as the specification", { + synthetic <- generate_synthetic_data(spec, n = 25, seed = 1) + + expect_s3_class(synthetic, "data.frame") + expect_identical(names(synthetic), spec$name) +}) + +test_that("synthetic data has the requested row count", { + synthetic <- generate_synthetic_data(spec, n = 37, seed = 1) + + expect_equal(nrow(synthetic), 37L) +}) + +test_that("synthetic categorical and binary values respect specification levels", { + synthetic <- generate_synthetic_data(spec, n = 100, seed = 1) + + categorical_spec <- spec[spec$type %in% c("categorical", "binary") & + !is.na(spec$levels) & spec$levels != "", , drop = FALSE] + + for (i in seq_len(nrow(categorical_spec))) { + name <- categorical_spec$name[i] + allowed_levels <- split_levels(categorical_spec$levels[i]) + observed_values <- unique(as.character(stats::na.omit(synthetic[[name]]))) + + expect_true( + all(observed_values %in% allowed_levels), + info = paste(name, "contains values outside the specification levels") + ) + } +}) + +test_that("synthetic numeric and integer values respect specification min and max", { + synthetic <- generate_synthetic_data(spec, n = 100, seed = 1) + + ranged_spec <- spec[spec$type %in% c("numeric", "integer") & + !is.na(spec$min) & spec$min != "" & + !is.na(spec$max) & spec$max != "", , drop = FALSE] + + for (i in seq_len(nrow(ranged_spec))) { + name <- ranged_spec$name[i] + min_value <- as.numeric(ranged_spec$min[i]) + max_value <- as.numeric(ranged_spec$max[i]) + observed_values <- stats::na.omit(synthetic[[name]]) + + expect_true( + all(observed_values >= min_value & observed_values <= max_value), + info = paste(name, "contains values outside the specification range") + ) + } +}) + +test_that("fixed seed gives identical synthetic output", { + synthetic_a <- generate_synthetic_data(spec, n = 25, seed = 2024) + synthetic_b <- generate_synthetic_data(spec, n = 25, seed = 2024) + + expect_identical(synthetic_a, synthetic_b) +}) From ba59d3edb3a4a1fa356dc2c096b1ace8cb5335e0 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 23:11:33 -0600 Subject: [PATCH 14/33] Implement EDA synthetic data generation --- NAMESPACE | 1 + R/eda_synthetic.R | 169 ++++++++++++++++++++++++ man/generate_synthetic_data.Rd | 26 ++++ spec_driven_EDA_plan/docs/START_HERE.md | 21 +-- 4 files changed, 209 insertions(+), 8 deletions(-) create mode 100644 R/eda_synthetic.R create mode 100644 man/generate_synthetic_data.Rd diff --git a/NAMESPACE b/NAMESPACE index eb94590..47e66c8 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -85,6 +85,7 @@ export(epi_utils_session) export(epi_write) export(epi_write_df) export(expr) +export(generate_synthetic_data) export(epi_grob_to_excel) export(palette_IMSS) export(palette_IMSS_accessible) diff --git a/R/eda_synthetic.R b/R/eda_synthetic.R new file mode 100644 index 0000000..838af6f --- /dev/null +++ b/R/eda_synthetic.R @@ -0,0 +1,169 @@ +#' Generate synthetic data from an EDA specification +#' +#' Generate simple deterministic synthetic data from a validated +#' specification-first EDA data dictionary. Synthetic data are intended for +#' pipeline preparation and testing only, not for inference or disclosure +#' control. +#' +#' @param spec A data frame containing an EDA specification, or a path accepted +#' by [eda_spec()]. +#' @param n Number of rows to generate. +#' @param seed Optional random seed. When supplied, repeated calls with the same +#' specification, row count and seed return identical data. +#' +#' @return A data frame with one column per specification variable and `n` rows. +#' +#' @export +generate_synthetic_data <- function(spec, n = 100, seed = NULL) { + spec <- if (is.data.frame(spec)) { + validate_eda_spec(spec) + } else { + eda_spec(spec) + } + + if (!is.numeric(n) || length(n) != 1 || is.na(n) || n < 0 || n != floor(n)) { + stop("n must be a non-negative whole number.", call. = FALSE) + } + n <- as.integer(n) + + if (!is.null(seed)) { + if (!is.numeric(seed) || length(seed) != 1 || is.na(seed)) { + stop("seed must be NULL or a single numeric value.", call. = FALSE) + } + old_seed <- if (exists(".Random.seed", envir = .GlobalEnv, inherits = FALSE)) { + get(".Random.seed", envir = .GlobalEnv, inherits = FALSE) + } else { + NULL + } + on.exit({ + if (is.null(old_seed)) { + if (exists(".Random.seed", envir = .GlobalEnv, inherits = FALSE)) { + rm(".Random.seed", envir = .GlobalEnv) + } + } else { + assign(".Random.seed", old_seed, envir = .GlobalEnv) + } + }, add = TRUE) + set.seed(seed) + } + + columns <- stats::setNames( + lapply(seq_len(nrow(spec)), function(i) generate_synthetic_column(spec[i, , drop = FALSE], n)), + spec$name + ) + + as.data.frame(columns, stringsAsFactors = FALSE, check.names = FALSE) +} + +generate_synthetic_column <- function(row, n) { + type <- row$type[[1]] + + switch( + type, + numeric = generate_synthetic_numeric(row, n), + integer = generate_synthetic_integer(row, n), + categorical = generate_synthetic_categorical(row, n), + binary = generate_synthetic_binary(row, n), + date = generate_synthetic_date(row, n), + datetime = generate_synthetic_datetime(row, n), + text = generate_synthetic_text(row, n), + stop("Unsupported EDA specification type: ", type, call. = FALSE) + ) +} + +generate_synthetic_numeric <- function(row, n) { + bounds <- synthetic_numeric_bounds(row, default_min = 0, default_max = 100) + stats::runif(n, min = bounds[["min"]], max = bounds[["max"]]) +} + +generate_synthetic_integer <- function(row, n) { + bounds <- synthetic_numeric_bounds(row, default_min = 0, default_max = 100) + sample(seq.int(ceiling(bounds[["min"]]), floor(bounds[["max"]])), n, replace = TRUE) +} + +generate_synthetic_categorical <- function(row, n) { + levels <- synthetic_levels(row) + if (length(levels) == 0) { + levels <- c("Level 1", "Level 2", "Level 3") + } + sample(levels, n, replace = TRUE) +} + +generate_synthetic_binary <- function(row, n) { + levels <- synthetic_levels(row) + if (length(levels) == 0) { + levels <- c("0", "1") + } + sample(levels, n, replace = TRUE) +} + +generate_synthetic_date <- function(row, n) { + bounds <- synthetic_date_bounds(row) + offsets <- sample(seq.int(0, as.integer(bounds[["max"]] - bounds[["min"]])), n, replace = TRUE) + bounds[["min"]] + offsets +} + +generate_synthetic_datetime <- function(row, n) { + bounds <- synthetic_datetime_bounds(row) + offsets <- stats::runif(n, min = 0, max = as.numeric(difftime(bounds[["max"]], bounds[["min"]], units = "secs"))) + bounds[["min"]] + offsets +} + +generate_synthetic_text <- function(row, n) { + if (n == 0) { + return(character()) + } + paste0(row$name[[1]], "_synthetic_", seq_len(n)) +} + +synthetic_levels <- function(row) { + if (!("levels" %in% names(row)) || is.na(row$levels[[1]]) || trimws(row$levels[[1]]) == "") { + return(character()) + } + trimws(strsplit(as.character(row$levels[[1]]), ";", fixed = TRUE)[[1]]) +} + +synthetic_numeric_bounds <- function(row, default_min, default_max) { + min_value <- if ("min" %in% names(row)) suppressWarnings(as.numeric(row$min[[1]])) else NA_real_ + max_value <- if ("max" %in% names(row)) suppressWarnings(as.numeric(row$max[[1]])) else NA_real_ + if (is.na(min_value)) min_value <- default_min + if (is.na(max_value)) max_value <- default_max + if (min_value > max_value) { + stop("Synthetic numeric bounds are invalid for ", row$name[[1]], ".", call. = FALSE) + } + c(min = min_value, max = max_value) +} + +synthetic_date_bounds <- function(row) { + min_value <- synthetic_parse_date_bound(row, "min", as.Date("2000-01-01")) + max_value <- synthetic_parse_date_bound(row, "max", as.Date("2000-12-31")) + if (min_value > max_value) { + stop("Synthetic date bounds are invalid for ", row$name[[1]], ".", call. = FALSE) + } + c(min = min_value, max = max_value) +} + +synthetic_datetime_bounds <- function(row) { + min_value <- synthetic_parse_datetime_bound(row, "min", as.POSIXct("2000-01-01 00:00:00", tz = "UTC")) + max_value <- synthetic_parse_datetime_bound(row, "max", as.POSIXct("2000-12-31 23:59:59", tz = "UTC")) + if (min_value > max_value) { + stop("Synthetic datetime bounds are invalid for ", row$name[[1]], ".", call. = FALSE) + } + c(min = min_value, max = max_value) +} + +synthetic_parse_date_bound <- function(row, column, default) { + if (!(column %in% names(row)) || is.na(row[[column]][[1]]) || trimws(row[[column]][[1]]) == "") { + return(default) + } + value <- as.Date(row[[column]][[1]]) + if (is.na(value)) default else value +} + +synthetic_parse_datetime_bound <- function(row, column, default) { + if (!(column %in% names(row)) || is.na(row[[column]][[1]]) || trimws(row[[column]][[1]]) == "") { + return(default) + } + value <- as.POSIXct(row[[column]][[1]], tz = "UTC") + if (is.na(value)) default else value +} diff --git a/man/generate_synthetic_data.Rd b/man/generate_synthetic_data.Rd new file mode 100644 index 0000000..648c3fc --- /dev/null +++ b/man/generate_synthetic_data.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/eda_synthetic.R +\name{generate_synthetic_data} +\alias{generate_synthetic_data} +\title{Generate synthetic data from an EDA specification} +\usage{ +generate_synthetic_data(spec, n = 100, seed = NULL) +} +\arguments{ +\item{spec}{A data frame containing an EDA specification, or a path accepted +by \code{\link[=eda_spec]{eda_spec()}}.} + +\item{n}{Number of rows to generate.} + +\item{seed}{Optional random seed. When supplied, repeated calls with the same +specification, row count and seed return identical data.} +} +\value{ +A data frame with one column per specification variable and \code{n} rows. +} +\description{ +Generate simple deterministic synthetic data from a validated +specification-first EDA data dictionary. Synthetic data are intended for +pipeline preparation and testing only, not for inference or disclosure +control. +} diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/spec_driven_EDA_plan/docs/START_HERE.md index 8745fd9..8ce2d44 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/spec_driven_EDA_plan/docs/START_HERE.md @@ -25,19 +25,20 @@ operational source of truth for what happens next. | PR 2 fixture-backed failing tests | Done | Tests are present and may fail until PR 3 implements the missing functions. | | PR 3 spec, schema and missingness implementation | Done | Spec, schema and missingness functions are implemented. | | PR 4 synthetic-data failing tests | Done | Tests are present and may fail until PR 5 implements `generate_synthetic_data()`. | -| PR 5 synthetic-data implementation | Active | Follow Instruction 5 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | -| Later summary, plot, report and template work | Not started | Continue through the PR plan after PR 5. | +| PR 5 synthetic-data implementation | Done | `generate_synthetic_data()` is implemented. | +| PR 6 summary and plot failing tests | Active | Follow Instruction 6 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | +| Later summary, plot, report and template work | Not started | Continue through the PR plan after PR 6. | ## Active PR ```text -PR 5: Implement synthetic-data generation +PR 6: Add failing summary and plot tests ``` Instruction: ```text -Follow Instruction 5 in: +Follow Instruction 6 in: spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md ``` @@ -54,20 +55,23 @@ spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md spec_driven_EDA_plan/docs/codex/review-checklist.md ``` -## PR 5 Scope +## PR 6 Scope Must edit: ```text -R/eda_synthetic.R +tests/testthat/test-eda_summaries-fixtures.R +tests/testthat/test-eda_plots-fixtures.R spec_driven_EDA_plan/docs/START_HERE.md ``` May read: ```text -tests/testthat/test-eda_synthetic-fixtures.R +tests/testthat/fixtures/blood_storage/blood_storage.csv tests/testthat/fixtures/blood_storage/blood_storage_spec.csv +R/eda_summaries.R +R/eda_plots.R ``` Must not edit: @@ -76,6 +80,7 @@ Must not edit: R/eda_spec.R R/eda_schema.R R/eda_missing.R +R/eda_synthetic.R R/eda_summaries.R R/eda_plots.R R/run_eda.R @@ -87,7 +92,7 @@ inst/project-template/ Expected test state: ```text -PR 5 should make the synthetic-data tests pass. +PR 6 may leave tests failing until PR 7 implements `profile_summaries()` and `profile_plots()`. ``` ## Closeout Rule From e19d4d761cb683e4bef129dc51b9fb89202505d8 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 23:22:55 -0600 Subject: [PATCH 15/33] Add fixture tests for EDA summaries and plots --- spec_driven_EDA_plan/docs/START_HERE.md | 23 +++-- tests/testthat/test-eda_plots-fixtures.R | 44 ++++++++++ tests/testthat/test-eda_summaries-fixtures.R | 92 ++++++++++++++++++++ 3 files changed, 147 insertions(+), 12 deletions(-) create mode 100644 tests/testthat/test-eda_plots-fixtures.R create mode 100644 tests/testthat/test-eda_summaries-fixtures.R diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/spec_driven_EDA_plan/docs/START_HERE.md index 8ce2d44..96f4222 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/spec_driven_EDA_plan/docs/START_HERE.md @@ -26,19 +26,20 @@ operational source of truth for what happens next. | PR 3 spec, schema and missingness implementation | Done | Spec, schema and missingness functions are implemented. | | PR 4 synthetic-data failing tests | Done | Tests are present and may fail until PR 5 implements `generate_synthetic_data()`. | | PR 5 synthetic-data implementation | Done | `generate_synthetic_data()` is implemented. | -| PR 6 summary and plot failing tests | Active | Follow Instruction 6 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | -| Later summary, plot, report and template work | Not started | Continue through the PR plan after PR 6. | +| PR 6 summary and plot failing tests | Done | Tests are present and may fail until PR 7 implements `profile_summaries()` and `profile_plots()`. | +| PR 7 summary and plot implementation | Active | Follow Instruction 7 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | +| Later run, report and template work | Not started | Continue through the PR plan after PR 7. | ## Active PR ```text -PR 6: Add failing summary and plot tests +PR 7: Implement summaries and plots ``` Instruction: ```text -Follow Instruction 6 in: +Follow Instruction 7 in: spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md ``` @@ -55,23 +56,23 @@ spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md spec_driven_EDA_plan/docs/codex/review-checklist.md ``` -## PR 6 Scope +## PR 7 Scope Must edit: ```text -tests/testthat/test-eda_summaries-fixtures.R -tests/testthat/test-eda_plots-fixtures.R +R/eda_summaries.R +R/eda_plots.R spec_driven_EDA_plan/docs/START_HERE.md ``` May read: ```text +tests/testthat/test-eda_summaries-fixtures.R +tests/testthat/test-eda_plots-fixtures.R tests/testthat/fixtures/blood_storage/blood_storage.csv tests/testthat/fixtures/blood_storage/blood_storage_spec.csv -R/eda_summaries.R -R/eda_plots.R ``` Must not edit: @@ -81,8 +82,6 @@ R/eda_spec.R R/eda_schema.R R/eda_missing.R R/eda_synthetic.R -R/eda_summaries.R -R/eda_plots.R R/run_eda.R R/eda_report.R inst/report-template/ @@ -92,7 +91,7 @@ inst/project-template/ Expected test state: ```text -PR 6 may leave tests failing until PR 7 implements `profile_summaries()` and `profile_plots()`. +PR 7 is expected to make the PR 6 summary and plot tests pass by implementing `profile_summaries()` and `profile_plots()`. ``` ## Closeout Rule diff --git a/tests/testthat/test-eda_plots-fixtures.R b/tests/testthat/test-eda_plots-fixtures.R new file mode 100644 index 0000000..a893af4 --- /dev/null +++ b/tests/testthat/test-eda_plots-fixtures.R @@ -0,0 +1,44 @@ +context("fixture-backed EDA plot tests") + +library(testthat) +library(episcout) + +fixture_dir <- file.path("fixtures", "blood_storage") +data_path <- file.path(fixture_dir, "blood_storage.csv") +spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") + +test_that("profile_plots returns one named plot object per specified variable", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + + observed <- profile_plots(data, spec) + + expect_type(observed, "list") + expect_named(observed, spec$name) + expect_equal(length(observed), nrow(spec)) +}) + +test_that("profile_plots returns ggplot objects without checking visual appearance", { + skip_if_not_installed("ggplot2") + + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + + observed <- profile_plots(data, spec) + + expect_true(all(vapply(observed, inherits, logical(1), what = "ggplot"))) +}) + +test_that("profile_plots dispatches numeric and categorical fixture variables", { + skip_if_not_installed("ggplot2") + + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + + observed <- profile_plots(data, spec) + + expect_s3_class(observed$Age, "ggplot") + expect_s3_class(observed$Units, "ggplot") + expect_s3_class(observed$RBC.Age.Group, "ggplot") + expect_s3_class(observed$Recurrence, "ggplot") +}) diff --git a/tests/testthat/test-eda_summaries-fixtures.R b/tests/testthat/test-eda_summaries-fixtures.R new file mode 100644 index 0000000..86674f0 --- /dev/null +++ b/tests/testthat/test-eda_summaries-fixtures.R @@ -0,0 +1,92 @@ +context("fixture-backed EDA summary tests") + +library(testthat) +library(episcout) + +fixture_dir <- file.path("fixtures", "blood_storage") +data_path <- file.path(fixture_dir, "blood_storage.csv") +spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") + +make_expected_numeric_summary <- function(data, spec) { + numeric_spec <- spec[spec$type %in% c("numeric", "integer"), , drop = FALSE] + + rows <- lapply(numeric_spec$name, function(name) { + values <- data[[name]] + data.frame( + name = name, + n = length(values), + n_missing = sum(is.na(values)), + mean = mean(values, na.rm = TRUE), + sd = stats::sd(values, na.rm = TRUE), + median = stats::median(values, na.rm = TRUE), + min = min(values, na.rm = TRUE), + max = max(values, na.rm = TRUE), + stringsAsFactors = FALSE + ) + }) + + do.call(rbind, rows) +} + +make_expected_categorical_summary <- function(data, spec) { + categorical_spec <- spec[spec$type %in% c("categorical", "binary"), , drop = FALSE] + + rows <- lapply(categorical_spec$name, function(name) { + values <- data[[name]] + observed <- as.character(values[!is.na(values)]) + levels <- strsplit(categorical_spec$levels[categorical_spec$name == name], ";", fixed = TRUE)[[1]] + counts <- vapply(levels, function(level) sum(observed == level), integer(1)) + + data.frame( + name = name, + level = levels, + n = as.integer(counts), + p = as.numeric(counts) / length(values), + stringsAsFactors = FALSE + ) + }) + + do.call(rbind, rows) +} + +test_that("profile_summaries returns numeric and categorical summary components", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + + observed <- profile_summaries(data, spec) + + expect_type(observed, "list") + expect_named(observed, c("numeric", "categorical")) + expect_s3_class(observed$numeric, "data.frame") + expect_s3_class(observed$categorical, "data.frame") +}) + +test_that("profile_summaries numeric output matches independent fixture calculations", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + expected <- make_expected_numeric_summary(data, spec) + + observed <- profile_summaries(data, spec) + + expect_equal( + as.data.frame(observed$numeric), + expected, + tolerance = 1e-12, + ignore_attr = TRUE + ) +}) + +test_that("profile_summaries categorical output matches independent fixture calculations", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + expected <- make_expected_categorical_summary(data, spec) + + observed <- profile_summaries(data, spec) + + expect_equal( + as.data.frame(observed$categorical), + expected, + tolerance = 1e-12, + ignore_attr = TRUE + ) +}) From 56a39cb69ee15c63f0aa9a7f12b679fef0644489 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Sun, 14 Jun 2026 23:36:32 -0600 Subject: [PATCH 16/33] Implement EDA summaries and plots --- NAMESPACE | 2 + R/eda_plots.R | 64 ++++++++++++++++ R/eda_summaries.R | 99 +++++++++++++++++++++++++ man/profile_plots.Rd | 21 ++++++ man/profile_summaries.Rd | 20 +++++ spec_driven_EDA_plan/docs/START_HERE.md | 25 ++++--- 6 files changed, 221 insertions(+), 10 deletions(-) create mode 100644 R/eda_plots.R create mode 100644 R/eda_summaries.R create mode 100644 man/profile_plots.Rd create mode 100644 man/profile_summaries.Rd diff --git a/NAMESPACE b/NAMESPACE index 47e66c8..16d2d9c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -90,6 +90,8 @@ export(epi_grob_to_excel) export(palette_IMSS) export(palette_IMSS_accessible) export(profile_missing) +export(profile_plots) +export(profile_summaries) export(quo) export(quo_name) export(quos) diff --git a/R/eda_plots.R b/R/eda_plots.R new file mode 100644 index 0000000..3593f0a --- /dev/null +++ b/R/eda_plots.R @@ -0,0 +1,64 @@ +#' Profile plots using an EDA specification +#' +#' Create one basic ggplot object for each variable listed in a +#' specification-first EDA data dictionary. Plot objects are returned without +#' being printed. +#' +#' @param data A data frame containing observed data. +#' @param spec An EDA specification data frame or CSV path. +#' +#' @return A named list of ggplot objects, one per specified variable. +#' +#' @export +profile_plots <- function(data, spec) { + if (!is.data.frame(data)) { + stop("EDA data must be a data frame.", call. = FALSE) + } + + if (!requireNamespace("ggplot2", quietly = TRUE)) { + stop("The ggplot2 package is required for profile_plots().", call. = FALSE) + } + + spec <- eda_spec(spec) + missing_vars <- setdiff(spec$name, names(data)) + + if (length(missing_vars) > 0) { + stop( + "EDA data is missing specified variables: ", + paste(missing_vars, collapse = ", "), + call. = FALSE + ) + } + + plots <- lapply(seq_len(nrow(spec)), function(i) { + profile_plot_variable(data, spec$name[[i]], spec$type[[i]], spec$label[[i]]) + }) + names(plots) <- spec$name + plots +} + +profile_plot_variable <- function(data, name, type, label) { + plot_data <- data.frame(value = data[[name]]) + axis_label <- if (!is.na(label) && nzchar(label)) label else name + + if (type %in% c("numeric", "integer")) { + ggplot2::ggplot(plot_data, ggplot2::aes(x = .data$value)) + + ggplot2::geom_histogram(bins = 30, na.rm = TRUE) + + ggplot2::labs(x = axis_label, y = "Count", title = axis_label) + + ggplot2::theme_minimal() + } else if (type %in% c("categorical", "binary", "text")) { + plot_data$value <- as.factor(plot_data$value) + ggplot2::ggplot(plot_data, ggplot2::aes(x = .data$value)) + + ggplot2::geom_bar(na.rm = TRUE) + + ggplot2::labs(x = axis_label, y = "Count", title = axis_label) + + ggplot2::theme_minimal() + } else if (type %in% c("date", "datetime")) { + plot_data$value <- as.POSIXct(plot_data$value) + ggplot2::ggplot(plot_data, ggplot2::aes(x = .data$value)) + + ggplot2::geom_histogram(bins = 30, na.rm = TRUE) + + ggplot2::labs(x = axis_label, y = "Count", title = axis_label) + + ggplot2::theme_minimal() + } else { + stop("Unsupported EDA plot type: ", type, call. = FALSE) + } +} diff --git a/R/eda_summaries.R b/R/eda_summaries.R new file mode 100644 index 0000000..dfd411c --- /dev/null +++ b/R/eda_summaries.R @@ -0,0 +1,99 @@ +#' Profile summaries using an EDA specification +#' +#' Produce machine-readable descriptive summaries for variables listed in a +#' specification-first EDA data dictionary. +#' +#' @param data A data frame containing observed data. +#' @param spec An EDA specification data frame or CSV path. +#' +#' @return A named list with `numeric` and `categorical` data frames. +#' +#' @export +profile_summaries <- function(data, spec) { + if (!is.data.frame(data)) { + stop("EDA data must be a data frame.", call. = FALSE) + } + + spec <- eda_spec(spec) + missing_vars <- setdiff(spec$name, names(data)) + + if (length(missing_vars) > 0) { + stop( + "EDA data is missing specified variables: ", + paste(missing_vars, collapse = ", "), + call. = FALSE + ) + } + + list( + numeric = profile_summaries_numeric(data, spec), + categorical = profile_summaries_categorical(data, spec) + ) +} + +profile_summaries_numeric <- function(data, spec) { + numeric_spec <- spec[spec$type %in% c("numeric", "integer"), , drop = FALSE] + + if (nrow(numeric_spec) == 0) { + return(data.frame( + name = character(), + n = integer(), + n_missing = integer(), + mean = numeric(), + sd = numeric(), + median = numeric(), + min = numeric(), + max = numeric(), + stringsAsFactors = FALSE + )) + } + + rows <- lapply(numeric_spec$name, function(name) { + values <- data[[name]] + data.frame( + name = name, + n = length(values), + n_missing = sum(is.na(values)), + mean = mean(values, na.rm = TRUE), + sd = stats::sd(values, na.rm = TRUE), + median = stats::median(values, na.rm = TRUE), + min = min(values, na.rm = TRUE), + max = max(values, na.rm = TRUE), + stringsAsFactors = FALSE + ) + }) + + do.call(rbind, rows) +} + +profile_summaries_categorical <- function(data, spec) { + categorical_spec <- spec[spec$type %in% c("categorical", "binary"), , drop = FALSE] + + if (nrow(categorical_spec) == 0) { + return(data.frame( + name = character(), + level = character(), + n = integer(), + p = numeric(), + stringsAsFactors = FALSE + )) + } + + rows <- lapply(categorical_spec$name, function(name) { + values <- data[[name]] + observed <- as.character(values[!is.na(values)]) + levels_value <- categorical_spec$levels[categorical_spec$name == name] + levels <- strsplit(levels_value, ";", fixed = TRUE)[[1]] + counts <- vapply(levels, function(level) sum(observed == level), integer(1)) + + data.frame( + name = name, + level = levels, + n = as.integer(counts), + p = as.numeric(counts) / length(values), + stringsAsFactors = FALSE + ) + }) + + do.call(rbind, rows) +} diff --git a/man/profile_plots.Rd b/man/profile_plots.Rd new file mode 100644 index 0000000..4816507 --- /dev/null +++ b/man/profile_plots.Rd @@ -0,0 +1,21 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/eda_plots.R +\name{profile_plots} +\alias{profile_plots} +\title{Profile plots using an EDA specification} +\usage{ +profile_plots(data, spec) +} +\arguments{ +\item{data}{A data frame containing observed data.} + +\item{spec}{An EDA specification data frame or CSV path.} +} +\value{ +A named list of ggplot objects, one per specified variable. +} +\description{ +Create one basic ggplot object for each variable listed in a +specification-first EDA data dictionary. Plot objects are returned without +being printed. +} diff --git a/man/profile_summaries.Rd b/man/profile_summaries.Rd new file mode 100644 index 0000000..1deec60 --- /dev/null +++ b/man/profile_summaries.Rd @@ -0,0 +1,20 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/eda_summaries.R +\name{profile_summaries} +\alias{profile_summaries} +\title{Profile summaries using an EDA specification} +\usage{ +profile_summaries(data, spec) +} +\arguments{ +\item{data}{A data frame containing observed data.} + +\item{spec}{An EDA specification data frame or CSV path.} +} +\value{ +A named list with \code{numeric} and \code{categorical} data frames. +} +\description{ +Produce machine-readable descriptive summaries for variables listed in a +specification-first EDA data dictionary. +} diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/spec_driven_EDA_plan/docs/START_HERE.md index 96f4222..2bc7857 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/spec_driven_EDA_plan/docs/START_HERE.md @@ -27,19 +27,20 @@ operational source of truth for what happens next. | PR 4 synthetic-data failing tests | Done | Tests are present and may fail until PR 5 implements `generate_synthetic_data()`. | | PR 5 synthetic-data implementation | Done | `generate_synthetic_data()` is implemented. | | PR 6 summary and plot failing tests | Done | Tests are present and may fail until PR 7 implements `profile_summaries()` and `profile_plots()`. | -| PR 7 summary and plot implementation | Active | Follow Instruction 7 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | -| Later run, report and template work | Not started | Continue through the PR plan after PR 7. | +| PR 7 summary and plot implementation | Done | `profile_summaries()` and `profile_plots()` are implemented. | +| PR 8 run_eda failing tests | Active | Follow Instruction 8 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | +| Later run, report and template work | Not started | Continue through the PR plan after PR 8. | ## Active PR ```text -PR 7: Implement summaries and plots +PR 8: Add failing run_eda tests ``` Instruction: ```text -Follow Instruction 7 in: +Follow Instruction 8 in: spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md ``` @@ -56,23 +57,25 @@ spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md spec_driven_EDA_plan/docs/codex/review-checklist.md ``` -## PR 7 Scope +## PR 8 Scope Must edit: ```text -R/eda_summaries.R -R/eda_plots.R +tests/testthat/test-run_eda-fixtures.R spec_driven_EDA_plan/docs/START_HERE.md ``` May read: ```text -tests/testthat/test-eda_summaries-fixtures.R -tests/testthat/test-eda_plots-fixtures.R tests/testthat/fixtures/blood_storage/blood_storage.csv tests/testthat/fixtures/blood_storage/blood_storage_spec.csv +tests/testthat/test-eda_spec-fixtures.R +tests/testthat/test-eda_schema-fixtures.R +tests/testthat/test-eda_missing-fixtures.R +tests/testthat/test-eda_summaries-fixtures.R +tests/testthat/test-eda_plots-fixtures.R ``` Must not edit: @@ -82,6 +85,8 @@ R/eda_spec.R R/eda_schema.R R/eda_missing.R R/eda_synthetic.R +R/eda_summaries.R +R/eda_plots.R R/run_eda.R R/eda_report.R inst/report-template/ @@ -91,7 +96,7 @@ inst/project-template/ Expected test state: ```text -PR 7 is expected to make the PR 6 summary and plot tests pass by implementing `profile_summaries()` and `profile_plots()`. +PR 8 may leave tests failing until PR 9 implements `run_eda()`. ``` ## Closeout Rule From 3f736f0137ae57180a0c63e18671f6970831205d Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Mon, 15 Jun 2026 09:25:41 -0600 Subject: [PATCH 17/33] Add run_eda fixture workflow tests --- spec_driven_EDA_plan/docs/START_HERE.md | 17 +++--- tests/testthat/test-run_eda-fixtures.R | 81 +++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 8 deletions(-) create mode 100644 tests/testthat/test-run_eda-fixtures.R diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/spec_driven_EDA_plan/docs/START_HERE.md index 2bc7857..6b55844 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/spec_driven_EDA_plan/docs/START_HERE.md @@ -28,19 +28,20 @@ operational source of truth for what happens next. | PR 5 synthetic-data implementation | Done | `generate_synthetic_data()` is implemented. | | PR 6 summary and plot failing tests | Done | Tests are present and may fail until PR 7 implements `profile_summaries()` and `profile_plots()`. | | PR 7 summary and plot implementation | Done | `profile_summaries()` and `profile_plots()` are implemented. | -| PR 8 run_eda failing tests | Active | Follow Instruction 8 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | -| Later run, report and template work | Not started | Continue through the PR plan after PR 8. | +| PR 8 run_eda failing tests | Done | Tests are present and may fail until PR 9 implements `run_eda()`. | +| PR 9 run_eda implementation | Active | Follow Instruction 9 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | +| Later report and template work | Not started | Continue through the PR plan after PR 9. | ## Active PR ```text -PR 8: Add failing run_eda tests +PR 9: Implement run_eda() ``` Instruction: ```text -Follow Instruction 8 in: +Follow Instruction 9 in: spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md ``` @@ -57,12 +58,12 @@ spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md spec_driven_EDA_plan/docs/codex/review-checklist.md ``` -## PR 8 Scope +## PR 9 Scope Must edit: ```text -tests/testthat/test-run_eda-fixtures.R +R/run_eda.R spec_driven_EDA_plan/docs/START_HERE.md ``` @@ -76,6 +77,7 @@ tests/testthat/test-eda_schema-fixtures.R tests/testthat/test-eda_missing-fixtures.R tests/testthat/test-eda_summaries-fixtures.R tests/testthat/test-eda_plots-fixtures.R +tests/testthat/test-run_eda-fixtures.R ``` Must not edit: @@ -87,7 +89,6 @@ R/eda_missing.R R/eda_synthetic.R R/eda_summaries.R R/eda_plots.R -R/run_eda.R R/eda_report.R inst/report-template/ inst/project-template/ @@ -96,7 +97,7 @@ inst/project-template/ Expected test state: ```text -PR 8 may leave tests failing until PR 9 implements `run_eda()`. +PR 9 should make the run_eda fixture tests pass. ``` ## Closeout Rule diff --git a/tests/testthat/test-run_eda-fixtures.R b/tests/testthat/test-run_eda-fixtures.R new file mode 100644 index 0000000..a8b4bcf --- /dev/null +++ b/tests/testthat/test-run_eda-fixtures.R @@ -0,0 +1,81 @@ +context("fixture-backed run_eda workflow tests") + +library(testthat) +library(episcout) + +fixture_dir <- file.path("fixtures", "blood_storage") +data_path <- file.path(fixture_dir, "blood_storage.csv") +spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") + +expected_components <- c("metadata", "schema", "missing", "summaries", "plots") + +test_that("run_eda returns expected components for real fixture data", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + + observed <- run_eda(data = data, spec = spec) + + expect_type(observed, "list") + expect_named(observed, expected_components) + expect_s3_class(observed$metadata, "data.frame") + expect_s3_class(observed$schema, "data.frame") + expect_s3_class(observed$missing, "data.frame") + expect_type(observed$summaries, "list") + expect_type(observed$plots, "list") +}) + +test_that("run_eda real-data workflow matches component functions", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + + observed <- run_eda(data = data, spec = spec) + + expect_equal(observed$schema, check_schema(data, spec), ignore_attr = TRUE) + expect_equal(observed$missing, profile_missing(data, spec), ignore_attr = TRUE) + expect_equal( + observed$summaries, + profile_summaries(data, spec), + tolerance = 1e-12, + ignore_attr = TRUE + ) + expect_named(observed$plots, spec$name) +}) + +test_that("run_eda supports synthetic fixture workflow", { + spec <- eda_spec(spec_path) + + observed <- run_eda( + data = NULL, + spec = spec, + synthetic = TRUE, + n = 25, + seed = 2024 + ) + + expect_type(observed, "list") + expect_named(observed, expected_components) + expect_equal(observed$metadata$synthetic, TRUE) + expect_equal(observed$metadata$n_rows, 25L) + expect_equal(observed$schema$status, rep("present", nrow(spec))) + expect_named(observed$plots, spec$name) +}) + +test_that("run_eda writes workflow outputs to a temporary output directory", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + output_dir <- tempfile("run-eda-fixture-") + dir.create(output_dir) + + observed <- run_eda(data = data, spec = spec, output_dir = output_dir) + + expect_true(dir.exists(output_dir)) + expect_true(file.exists(file.path(output_dir, "metadata.csv"))) + expect_true(file.exists(file.path(output_dir, "schema.csv"))) + expect_true(file.exists(file.path(output_dir, "missing.csv"))) + expect_true(file.exists(file.path(output_dir, "summary_numeric.csv"))) + expect_true(file.exists(file.path(output_dir, "summary_categorical.csv"))) + expect_equal( + names(utils::read.csv(file.path(output_dir, "schema.csv"), check.names = FALSE)), + names(observed$schema) + ) +}) From 3eae684b9170e4e84072bf9a885efb072867d05d Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Mon, 15 Jun 2026 09:44:06 -0600 Subject: [PATCH 18/33] Implement run_eda orchestration --- NAMESPACE | 1 + R/run_eda.R | 114 ++++++++++++++++++++++++ man/run_eda.Rd | 35 ++++++++ spec_driven_EDA_plan/docs/START_HERE.md | 26 +++--- 4 files changed, 162 insertions(+), 14 deletions(-) create mode 100644 R/run_eda.R create mode 100644 man/run_eda.Rd diff --git a/NAMESPACE b/NAMESPACE index 16d2d9c..ae84b0d 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -96,6 +96,7 @@ export(quo) export(quo_name) export(quos) export(rename_contingency_2x2_cols) +export(run_eda) export(sym) export(syms) export(validate_eda_spec) diff --git a/R/run_eda.R b/R/run_eda.R new file mode 100644 index 0000000..df599ed --- /dev/null +++ b/R/run_eda.R @@ -0,0 +1,114 @@ +#' Run a specification-first EDA workflow +#' +#' Orchestrate the specification-first EDA helpers for either observed data or +#' deterministic synthetic data. The function validates the specification, runs +#' schema, missingness, summary and plot profiling, optionally writes +#' machine-readable outputs, and returns all results as a named list. +#' +#' @param data A data frame containing observed data. Required when +#' `synthetic = FALSE`; ignored when `synthetic = TRUE`. +#' @param spec An EDA specification data frame or CSV path accepted by +#' [eda_spec()]. +#' @param output_dir Optional directory where machine-readable CSV outputs are +#' written. The directory must already exist. +#' @param synthetic Logical; when `TRUE`, generate synthetic data from `spec` +#' before running the workflow. +#' @param n Number of synthetic rows to generate when `synthetic = TRUE`. +#' @param seed Optional random seed passed to [generate_synthetic_data()]. +#' +#' @return A named list with `metadata`, `schema`, `missing`, `summaries` and +#' `plots` components. +#' +#' @export +run_eda <- function(data, + spec, + output_dir = NULL, + synthetic = FALSE, + n = 100, + seed = NULL) { + synthetic <- validate_run_eda_synthetic(synthetic) + spec <- eda_spec(spec) + + if (synthetic) { + data <- generate_synthetic_data(spec = spec, n = n, seed = seed) + } else if (!is.data.frame(data)) { + stop("data must be a data frame when synthetic is FALSE.", call. = FALSE) + } + + if (!is.null(output_dir)) { + validate_run_eda_output_dir(output_dir) + } + + results <- list( + metadata = run_eda_metadata(data, spec, synthetic = synthetic), + schema = check_schema(data, spec), + missing = profile_missing(data, spec), + summaries = profile_summaries(data, spec), + plots = profile_plots(data, spec) + ) + + if (!is.null(output_dir)) { + write_run_eda_outputs(results, output_dir) + } + + results +} + +validate_run_eda_synthetic <- function(synthetic) { + if (!is.logical(synthetic) || length(synthetic) != 1 || is.na(synthetic)) { + stop("synthetic must be TRUE or FALSE.", call. = FALSE) + } + synthetic +} + +validate_run_eda_output_dir <- function(output_dir) { + if (!is.character(output_dir) || length(output_dir) != 1 || is.na(output_dir)) { + stop("output_dir must be NULL or a single directory path.", call. = FALSE) + } + + if (!dir.exists(output_dir)) { + stop("output_dir must exist before run_eda() writes outputs.", call. = FALSE) + } + + invisible(TRUE) +} + +run_eda_metadata <- function(data, spec, synthetic) { + data.frame( + synthetic = synthetic, + n_rows = as.integer(nrow(data)), + n_columns = as.integer(ncol(data)), + n_spec_variables = as.integer(nrow(spec)), + stringsAsFactors = FALSE + ) +} + +write_run_eda_outputs <- function(results, output_dir) { + utils::write.csv( + results$metadata, + file.path(output_dir, "metadata.csv"), + row.names = FALSE + ) + utils::write.csv( + results$schema, + file.path(output_dir, "schema.csv"), + row.names = FALSE + ) + utils::write.csv( + results$missing, + file.path(output_dir, "missing.csv"), + row.names = FALSE + ) + utils::write.csv( + results$summaries$numeric, + file.path(output_dir, "summary_numeric.csv"), + row.names = FALSE + ) + utils::write.csv( + results$summaries$categorical, + file.path(output_dir, "summary_categorical.csv"), + row.names = FALSE + ) + + invisible(TRUE) +} diff --git a/man/run_eda.Rd b/man/run_eda.Rd new file mode 100644 index 0000000..ad0e926 --- /dev/null +++ b/man/run_eda.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/run_eda.R +\name{run_eda} +\alias{run_eda} +\title{Run a specification-first EDA workflow} +\usage{ +run_eda(data, spec, output_dir = NULL, synthetic = FALSE, n = 100, seed = NULL) +} +\arguments{ +\item{data}{A data frame containing observed data. Required when +\code{synthetic = FALSE}; ignored when \code{synthetic = TRUE}.} + +\item{spec}{An EDA specification data frame or CSV path accepted by +\code{\link[=eda_spec]{eda_spec()}}.} + +\item{output_dir}{Optional directory where machine-readable CSV outputs are +written. The directory must already exist.} + +\item{synthetic}{Logical; when \code{TRUE}, generate synthetic data from \code{spec} +before running the workflow.} + +\item{n}{Number of synthetic rows to generate when \code{synthetic = TRUE}.} + +\item{seed}{Optional random seed passed to \code{\link[=generate_synthetic_data]{generate_synthetic_data()}}.} +} +\value{ +A named list with \code{metadata}, \code{schema}, \code{missing}, \code{summaries} and +\code{plots} components. +} +\description{ +Orchestrate the specification-first EDA helpers for either observed data or +deterministic synthetic data. The function validates the specification, runs +schema, missingness, summary and plot profiling, optionally writes +machine-readable outputs, and returns all results as a named list. +} diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/spec_driven_EDA_plan/docs/START_HERE.md index 6b55844..f501936 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/spec_driven_EDA_plan/docs/START_HERE.md @@ -29,19 +29,20 @@ operational source of truth for what happens next. | PR 6 summary and plot failing tests | Done | Tests are present and may fail until PR 7 implements `profile_summaries()` and `profile_plots()`. | | PR 7 summary and plot implementation | Done | `profile_summaries()` and `profile_plots()` are implemented. | | PR 8 run_eda failing tests | Done | Tests are present and may fail until PR 9 implements `run_eda()`. | -| PR 9 run_eda implementation | Active | Follow Instruction 9 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md`. | -| Later report and template work | Not started | Continue through the PR plan after PR 9. | +| PR 9 run_eda implementation | Done | `run_eda()` is implemented. | +| PR 10 report-template tests | Active | Follow Instruction 10 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md` and add failing report-rendering tests from the PR plan. | +| Later report and template work | Not started | Continue through the PR plan after PR 10. | ## Active PR ```text -PR 9: Implement run_eda() +PR 10: Add report-template tests ``` Instruction: ```text -Follow Instruction 9 in: +Follow Instruction 10 in: spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md ``` @@ -58,25 +59,23 @@ spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md spec_driven_EDA_plan/docs/codex/review-checklist.md ``` -## PR 9 Scope +## PR 10 Scope Must edit: ```text -R/run_eda.R +tests/testthat/test-eda_report.R spec_driven_EDA_plan/docs/START_HERE.md ``` May read: ```text +R/run_eda.R +R/eda_report.R +inst/report-template/ tests/testthat/fixtures/blood_storage/blood_storage.csv tests/testthat/fixtures/blood_storage/blood_storage_spec.csv -tests/testthat/test-eda_spec-fixtures.R -tests/testthat/test-eda_schema-fixtures.R -tests/testthat/test-eda_missing-fixtures.R -tests/testthat/test-eda_summaries-fixtures.R -tests/testthat/test-eda_plots-fixtures.R tests/testthat/test-run_eda-fixtures.R ``` @@ -89,15 +88,14 @@ R/eda_missing.R R/eda_synthetic.R R/eda_summaries.R R/eda_plots.R -R/eda_report.R -inst/report-template/ +R/run_eda.R inst/project-template/ ``` Expected test state: ```text -PR 9 should make the run_eda fixture tests pass. +PR 10 may leave report-rendering tests failing until PR 11 implements render_eda_report(). ``` ## Closeout Rule From 86217cec63a01ae4415dae5e455867e3847bd9a6 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Mon, 15 Jun 2026 11:32:05 -0600 Subject: [PATCH 19/33] PR10 plus corrections to env calling and naming conventions --- AGENTS.MD | 63 +++++++++- README.md | 2 +- scripts/check-cran.sh | 2 +- scripts/check-local.sh | 2 +- scripts/{rscript => rscript_env_caller.R} | 2 +- spec_driven_EDA_plan/docs/START_HERE.md | 18 +-- .../future/scripts/check-local.sh | 6 - tests/testthat/test-eda_report.R | 112 ++++++++++++++++++ 8 files changed, 187 insertions(+), 20 deletions(-) rename scripts/{rscript => rscript_env_caller.R} (88%) delete mode 100644 spec_driven_EDA_plan/future/scripts/check-local.sh create mode 100644 tests/testthat/test-eda_report.R diff --git a/AGENTS.MD b/AGENTS.MD index 8fbd441..ae2f435 100644 --- a/AGENTS.MD +++ b/AGENTS.MD @@ -17,6 +17,68 @@ It standardises formatting, documentation, testing, and pull request workflows. --- +## R Environment + +Use the repo-local R wrapper for all R commands: + +```bash +scripts/rscript_env_caller.R -e "R.home(); .libPaths()" +``` + +The wrapper resolves to the project mamba environment by default: + +```text +~/apps/miniforge3/envs/episcout/bin/Rscript +``` + +Do not use bare `Rscript` unless you have verified it resolves to the +`episcout` mamba environment. To use a different R binary, set +`EPISCOUT_RSCRIPT`; see `scripts/rscript_env_caller.R`. + +## Repo Script Discovery and Naming + +Before adding or changing helper commands, inspect existing project entry +points: + +```text +scripts/ +Makefile +.github/workflows/ +AGENTS.MD +``` + +Do not add a new helper script if an existing script already covers the task. +If an existing helper has an unclear or misleading name, propose a rename or +targeted cleanup instead of adding another wrapper. + +Script names must describe the project action, not merely mirror a system +binary. Avoid names that are visually or semantically confusable with common +commands, such as: + +```text +rscript +python +R +test +check +``` + +Prefer explicit project or task names such as: + +```text +scripts/rscript_env_caller.R +scripts/check-local.sh +scripts/check-cran.sh +scripts/render-report.sh +``` + +Before adding a new script, confirm: + +1. No existing script already does the job. +2. The name is unambiguous in chat, docs, logs and CI. +3. The script follows the repo's existing shell/R conventions. + + ## Coding Conventions * Use `snake_case` for object and function names. @@ -173,4 +235,3 @@ This AGENTS.md should be respected by OpenAI Codex or other automated agents whe * Running checks (use scripts provided above) --- - diff --git a/README.md b/README.md index cc0081f..1948a24 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ mamba env update -n episcout -f environment.yml --prune Run package checks through the repository wrapper, not bare `Rscript`: ``` bash -scripts/rscript -e "cat(R.home())" +scripts/rscript_env_caller.R -e "cat(R.home())" scripts/check-local.sh scripts/check-cran.sh ``` diff --git a/scripts/check-cran.sh b/scripts/check-cran.sh index b445618..0a45938 100755 --- a/scripts/check-cran.sh +++ b/scripts/check-cran.sh @@ -3,7 +3,7 @@ set -euo pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(cd "${script_dir}/.." && pwd)" -rscript="${script_dir}/rscript" +rscript="${script_dir}/rscript_env_caller.R" check_dir="${repo_root}/build/cran-check" cd "$repo_root" diff --git a/scripts/check-local.sh b/scripts/check-local.sh index 4708baa..f09f3d3 100755 --- a/scripts/check-local.sh +++ b/scripts/check-local.sh @@ -3,7 +3,7 @@ set -euo pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(cd "${script_dir}/.." && pwd)" -rscript="${script_dir}/rscript" +rscript="${script_dir}/rscript_env_caller.R" cd "$repo_root" diff --git a/scripts/rscript b/scripts/rscript_env_caller.R similarity index 88% rename from scripts/rscript rename to scripts/rscript_env_caller.R index 9157017..b60a421 100755 --- a/scripts/rscript +++ b/scripts/rscript_env_caller.R @@ -15,7 +15,7 @@ Create or update the dedicated package-development environment with: mamba env update -n episcout -f environment.yml --prune To use a different R binary, set EPISCOUT_RSCRIPT: - EPISCOUT_RSCRIPT=/path/to/Rscript scripts/rscript -e 'R.version.string' + EPISCOUT_RSCRIPT=/path/to/Rscript scripts/rscript_env_caller.R -e 'R.version.string' EOF exit 127 fi diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/spec_driven_EDA_plan/docs/START_HERE.md index f501936..a8e933e 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/spec_driven_EDA_plan/docs/START_HERE.md @@ -30,13 +30,14 @@ operational source of truth for what happens next. | PR 7 summary and plot implementation | Done | `profile_summaries()` and `profile_plots()` are implemented. | | PR 8 run_eda failing tests | Done | Tests are present and may fail until PR 9 implements `run_eda()`. | | PR 9 run_eda implementation | Done | `run_eda()` is implemented. | -| PR 10 report-template tests | Active | Follow Instruction 10 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md` and add failing report-rendering tests from the PR plan. | -| Later report and template work | Not started | Continue through the PR plan after PR 10. | +| PR 10 report-template tests | Done | Report-rendering tests are present and may fail until PR 11 implements `render_eda_report()`. | +| PR 11 report rendering implementation | Active | Follow Instruction 10 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md` and implement report rendering from the PR plan. | +| Later project template work | Not started | Continue through the PR plan after PR 11. | ## Active PR ```text -PR 10: Add report-template tests +PR 11: Implement report rendering ``` Instruction: @@ -59,12 +60,13 @@ spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md spec_driven_EDA_plan/docs/codex/review-checklist.md ``` -## PR 10 Scope +## PR 11 Scope Must edit: ```text -tests/testthat/test-eda_report.R +R/eda_report.R +inst/report-template/eda.qmd spec_driven_EDA_plan/docs/START_HERE.md ``` @@ -72,10 +74,9 @@ May read: ```text R/run_eda.R -R/eda_report.R -inst/report-template/ tests/testthat/fixtures/blood_storage/blood_storage.csv tests/testthat/fixtures/blood_storage/blood_storage_spec.csv +tests/testthat/test-eda_report.R tests/testthat/test-run_eda-fixtures.R ``` @@ -95,7 +96,7 @@ inst/project-template/ Expected test state: ```text -PR 10 may leave report-rendering tests failing until PR 11 implements render_eda_report(). +PR 11 should make the report-rendering fixture tests pass. ``` ## Closeout Rule @@ -152,7 +153,6 @@ Do not start with: ```text Arrow DuckDB -Quarto report project templates multiple fixtures large-data optimisation diff --git a/spec_driven_EDA_plan/future/scripts/check-local.sh b/spec_driven_EDA_plan/future/scripts/check-local.sh deleted file mode 100644 index 510d6f9..0000000 --- a/spec_driven_EDA_plan/future/scripts/check-local.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -Rscript -e 'devtools::document()' -Rscript -e 'devtools::test(reporter = "summary")' -Rscript -e 'devtools::check(manual = FALSE, error_on = "warning")' diff --git a/tests/testthat/test-eda_report.R b/tests/testthat/test-eda_report.R new file mode 100644 index 0000000..d160b95 --- /dev/null +++ b/tests/testthat/test-eda_report.R @@ -0,0 +1,112 @@ +context("fixture-backed EDA report-rendering tests") + +library(testthat) +library(episcout) + +fixture_dir <- file.path("fixtures", "blood_storage") +data_path <- file.path(fixture_dir, "blood_storage.csv") +spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") + +read_report_text <- function(path) { + paste(readLines(path, warn = FALSE), collapse = "\n") +} + +expect_report_file <- function(path) { + expect_type(path, "character") + expect_length(path, 1L) + expect_true(file.exists(path)) + expect_match(basename(path), "\\.html?$") +} + +test_that("EDA report template is bundled with report sections", { + template_path <- system.file( + "report-template", + "eda.qmd", + package = "episcout" + ) + + expect_true(nzchar(template_path)) + if (!nzchar(template_path)) { + return(invisible()) + } + + template_text <- read_report_text(template_path) + expect_match(template_text, "schema|Schema") + expect_match(template_text, "missing|Missing") + expect_match(template_text, "summar|Summar") + expect_match(template_text, "plot|Plot") +}) + +test_that("render_eda_report renders a real fixture-data report", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + output_dir <- tempfile("eda-report-real-") + dir.create(output_dir) + + report_path <- render_eda_report( + data = data, + spec = spec, + output_dir = output_dir + ) + + expect_report_file(report_path) + + report_text <- read_report_text(report_path) + expect_match(report_text, "schema|Schema") + expect_match(report_text, "missing|Missing") + expect_match(report_text, "summar|Summar") + expect_match(report_text, "plot|Plot") + expect_match(report_text, "real|Real") +}) + +test_that("render_eda_report labels synthetic fixture reports", { + spec <- eda_spec(spec_path) + output_dir <- tempfile("eda-report-synthetic-") + dir.create(output_dir) + + report_path <- render_eda_report( + data = NULL, + spec = spec, + output_dir = output_dir, + synthetic = TRUE, + n = 25, + seed = 2024 + ) + + expect_report_file(report_path) + expect_match(read_report_text(report_path), "synthetic|Synthetic") +}) + +test_that("render_eda_report writes machine-readable workflow outputs", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + output_dir <- tempfile("eda-report-outputs-") + dir.create(output_dir) + + render_eda_report( + data = data, + spec = spec, + output_dir = output_dir + ) + + expect_true(file.exists(file.path(output_dir, "metadata.csv"))) + expect_true(file.exists(file.path(output_dir, "schema.csv"))) + expect_true(file.exists(file.path(output_dir, "missing.csv"))) + expect_true(file.exists(file.path(output_dir, "summary_numeric.csv"))) + expect_true(file.exists(file.path(output_dir, "summary_categorical.csv"))) +}) + +test_that("render_eda_report requires an existing output directory", { + data <- read.csv(data_path, check.names = FALSE) + spec <- eda_spec(spec_path) + output_dir <- tempfile("eda-report-missing-") + + expect_error( + render_eda_report( + data = data, + spec = spec, + output_dir = output_dir + ), + regexp = "output_dir|directory|exist" + ) +}) From a4f99d55f74310b68ba4f93516ce087071bb8b6d Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Mon, 15 Jun 2026 11:44:19 -0600 Subject: [PATCH 20/33] PR11 EDA SDD reporting --- NAMESPACE | 1 + R/eda_report.R | 70 +++++++++++++++++++++++++ inst/report-template/eda.qmd | 56 ++++++++++++++++++++ man/render_eda_report.Rd | 43 +++++++++++++++ spec_driven_EDA_plan/docs/START_HERE.md | 23 ++++---- 5 files changed, 181 insertions(+), 12 deletions(-) create mode 100644 R/eda_report.R create mode 100644 inst/report-template/eda.qmd create mode 100644 man/render_eda_report.Rd diff --git a/NAMESPACE b/NAMESPACE index ae84b0d..9a205d5 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -96,6 +96,7 @@ export(quo) export(quo_name) export(quos) export(rename_contingency_2x2_cols) +export(render_eda_report) export(run_eda) export(sym) export(syms) diff --git a/R/eda_report.R b/R/eda_report.R new file mode 100644 index 0000000..00aab3f --- /dev/null +++ b/R/eda_report.R @@ -0,0 +1,70 @@ +#' Render a specification-first EDA report +#' +#' Run the specification-first EDA workflow and render a bundled Quarto HTML +#' report from the workflow outputs. +#' +#' @param data A data frame containing observed data. Required when +#' `synthetic = FALSE`; ignored when `synthetic = TRUE`. +#' @param spec An EDA specification data frame or CSV path accepted by +#' [eda_spec()]. +#' @param output_dir Directory where machine-readable workflow outputs and the +#' rendered report are written. The directory must already exist. +#' @param synthetic Logical; when `TRUE`, generate synthetic data from `spec` +#' before running the workflow. +#' @param n Number of synthetic rows to generate when `synthetic = TRUE`. +#' @param seed Optional random seed passed to [generate_synthetic_data()]. +#' @param quiet Logical; passed to [rmarkdown::render()] to control render +#' output. +#' +#' @return A single character string containing the rendered HTML report path. +#' +#' @export +render_eda_report <- function(data, + spec, + output_dir, + synthetic = FALSE, + n = 100, + seed = NULL, + quiet = TRUE) { + validate_run_eda_output_dir(output_dir) + + if (!requireNamespace("rmarkdown", quietly = TRUE)) { + stop("The rmarkdown package is required for render_eda_report().", call. = FALSE) + } + + template_path <- system.file( + "report-template", + "eda.qmd", + package = "episcout" + ) + + if (!nzchar(template_path) || !file.exists(template_path)) { + stop("The bundled EDA report template could not be found.", call. = FALSE) + } + + results <- run_eda( + data = data, + spec = spec, + output_dir = output_dir, + synthetic = synthetic, + n = n, + seed = seed + ) + + render_input <- file.path(tempdir(), "episcout-eda-report.qmd") + file.copy(template_path, render_input, overwrite = TRUE) + + report_path <- rmarkdown::render( + input = render_input, + output_file = "eda-report.html", + output_dir = output_dir, + params = list( + results = results, + synthetic = isTRUE(results$metadata$synthetic[[1]]) + ), + quiet = quiet, + envir = new.env(parent = globalenv()) + ) + + normalizePath(report_path, winslash = "/", mustWork = TRUE) +} diff --git a/inst/report-template/eda.qmd b/inst/report-template/eda.qmd new file mode 100644 index 0000000..2f6fb90 --- /dev/null +++ b/inst/report-template/eda.qmd @@ -0,0 +1,56 @@ +--- +title: "Specification-first EDA Report" +format: html +params: + results: null + synthetic: false +--- + +```{r setup, include=FALSE} +results <- params$results +mode_label <- if (isTRUE(params$synthetic)) "Synthetic" else "Real" +``` + +# EDA Report + +Mode: `r mode_label` + +```{r metadata, echo=FALSE} +knitr::kable(results$metadata, caption = "Workflow metadata") +``` + +## Schema + +```{r schema, echo=FALSE} +knitr::kable(results$schema, caption = "Schema checks") +``` + +## Missingness + +```{r missingness, echo=FALSE} +knitr::kable(results$missing, caption = "Missing-value profile") +``` + +## Summaries + +### Numeric Summaries + +```{r numeric-summaries, echo=FALSE} +knitr::kable(results$summaries$numeric, caption = "Numeric summaries") +``` + +### Categorical Summaries + +```{r categorical-summaries, echo=FALSE} +knitr::kable(results$summaries$categorical, caption = "Categorical summaries") +``` + +## Plots + +```{r plots, echo=FALSE, fig.width=7, fig.height=4} +for (name in names(results$plots)) { + cat("### ", name, "\n\n", sep = "") + print(results$plots[[name]]) + cat("\n\n") +} +``` diff --git a/man/render_eda_report.Rd b/man/render_eda_report.Rd new file mode 100644 index 0000000..3cf8b04 --- /dev/null +++ b/man/render_eda_report.Rd @@ -0,0 +1,43 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/eda_report.R +\name{render_eda_report} +\alias{render_eda_report} +\title{Render a specification-first EDA report} +\usage{ +render_eda_report( + data, + spec, + output_dir, + synthetic = FALSE, + n = 100, + seed = NULL, + quiet = TRUE +) +} +\arguments{ +\item{data}{A data frame containing observed data. Required when +\code{synthetic = FALSE}; ignored when \code{synthetic = TRUE}.} + +\item{spec}{An EDA specification data frame or CSV path accepted by +\code{\link[=eda_spec]{eda_spec()}}.} + +\item{output_dir}{Directory where machine-readable workflow outputs and the +rendered report are written. The directory must already exist.} + +\item{synthetic}{Logical; when \code{TRUE}, generate synthetic data from \code{spec} +before running the workflow.} + +\item{n}{Number of synthetic rows to generate when \code{synthetic = TRUE}.} + +\item{seed}{Optional random seed passed to \code{\link[=generate_synthetic_data]{generate_synthetic_data()}}.} + +\item{quiet}{Logical; passed to \code{\link[rmarkdown:render]{rmarkdown::render()}} to control render +output.} +} +\value{ +A single character string containing the rendered HTML report path. +} +\description{ +Run the specification-first EDA workflow and render a bundled Quarto HTML +report from the workflow outputs. +} diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/spec_driven_EDA_plan/docs/START_HERE.md index a8e933e..68c679f 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/spec_driven_EDA_plan/docs/START_HERE.md @@ -31,13 +31,13 @@ operational source of truth for what happens next. | PR 8 run_eda failing tests | Done | Tests are present and may fail until PR 9 implements `run_eda()`. | | PR 9 run_eda implementation | Done | `run_eda()` is implemented. | | PR 10 report-template tests | Done | Report-rendering tests are present and may fail until PR 11 implements `render_eda_report()`. | -| PR 11 report rendering implementation | Active | Follow Instruction 10 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md` and implement report rendering from the PR plan. | -| Later project template work | Not started | Continue through the PR plan after PR 11. | +| PR 11 report rendering implementation | Done | `render_eda_report()` and the bundled Quarto report template are implemented. | +| PR 12 project template work | Active | Follow Instruction 10 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md` and add the project template from the PR plan. | ## Active PR ```text -PR 11: Implement report rendering +PR 12: Add project template ``` Instruction: @@ -60,24 +60,22 @@ spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md spec_driven_EDA_plan/docs/codex/review-checklist.md ``` -## PR 11 Scope +## PR 12 Scope Must edit: ```text -R/eda_report.R -inst/report-template/eda.qmd +inst/project-template/ spec_driven_EDA_plan/docs/START_HERE.md ``` May read: ```text -R/run_eda.R -tests/testthat/fixtures/blood_storage/blood_storage.csv -tests/testthat/fixtures/blood_storage/blood_storage_spec.csv +R/eda_report.R +inst/report-template/eda.qmd tests/testthat/test-eda_report.R -tests/testthat/test-run_eda-fixtures.R +spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md ``` Must not edit: @@ -90,13 +88,14 @@ R/eda_synthetic.R R/eda_summaries.R R/eda_plots.R R/run_eda.R -inst/project-template/ +R/eda_report.R +inst/report-template/eda.qmd ``` Expected test state: ```text -PR 11 should make the report-rendering fixture tests pass. +PR 12 should add project-template coverage without regressing PR 11 report tests. ``` ## Closeout Rule From c007a14772fa3b30d2e8773acbe10502ad03294b Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Mon, 15 Jun 2026 12:15:26 -0600 Subject: [PATCH 21/33] split pr12 for tdd then coding, pr14 is now large data, deferred --- spec_driven_EDA_plan/docs/START_HERE.md | 20 +++-- .../docs/codex/revised-pr-plan-tdd-first.md | 50 +++++++++-- .../codex/tdd-first-codex-instructions.md | 49 ++++++++++- tests/testthat/test-project-template.R | 84 +++++++++++++++++++ 4 files changed, 190 insertions(+), 13 deletions(-) create mode 100644 tests/testthat/test-project-template.R diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/spec_driven_EDA_plan/docs/START_HERE.md index 68c679f..8f8b235 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/spec_driven_EDA_plan/docs/START_HERE.md @@ -32,18 +32,20 @@ operational source of truth for what happens next. | PR 9 run_eda implementation | Done | `run_eda()` is implemented. | | PR 10 report-template tests | Done | Report-rendering tests are present and may fail until PR 11 implements `render_eda_report()`. | | PR 11 report rendering implementation | Done | `render_eda_report()` and the bundled Quarto report template are implemented. | -| PR 12 project template work | Active | Follow Instruction 10 in `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md` and add the project template from the PR plan. | +| PR 12 project-template contract tests | Done | Failing tests define the project template and `use_episcout_project()` contract. | +| PR 13 project-template implementation | Active | Implement `inst/project-template/` and `use_episcout_project()` to satisfy PR 12 tests. | +| PR 14 large-data design note | Deferred | Document large-data backend strategy after the project-template PRs. | ## Active PR ```text -PR 12: Add project template +PR 13: Implement project template ``` Instruction: ```text -Follow Instruction 10 in: +Follow Instruction 13 in: spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md ``` @@ -60,12 +62,15 @@ spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md spec_driven_EDA_plan/docs/codex/review-checklist.md ``` -## PR 12 Scope +## PR 13 Scope Must edit: ```text inst/project-template/ +R/use_episcout_project.R +NAMESPACE +man/ spec_driven_EDA_plan/docs/START_HERE.md ``` @@ -74,8 +79,11 @@ May read: ```text R/eda_report.R inst/report-template/eda.qmd +tests/testthat/test-project-template.R tests/testthat/test-eda_report.R +tests/testthat/test-run_eda-fixtures.R spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md +spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md ``` Must not edit: @@ -95,7 +103,8 @@ inst/report-template/eda.qmd Expected test state: ```text -PR 12 should add project-template coverage without regressing PR 11 report tests. +PR 13 should make the PR 12 project-template tests pass without regressing +the PR 11 report-rendering tests. ``` ## Closeout Rule @@ -152,7 +161,6 @@ Do not start with: ```text Arrow DuckDB -project templates multiple fixtures large-data optimisation full plot/report system diff --git a/spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md b/spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md index c867eb5..3241878 100644 --- a/spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md +++ b/spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md @@ -236,25 +236,65 @@ Function: render_eda_report() ``` -### PR 12: Add project template +### PR 12: Add project-template contract tests Scope: -- add reusable project scaffold. +- add failing tests for the reusable project scaffold; +- add failing tests for the user-facing project setup helper; +- do not add the template or implement the helper yet. + +Files: + +```text +tests/testthat/test-project-template.R +``` + +Future function contract: + +```r +use_episcout_project(path, overwrite = FALSE) +``` + +Expected template entries: + +```text +metadata/data_dictionary.csv +config/eda.yml +_targets.R +reports/eda.qmd +R/project-derivations.R +outputs/ +``` + +Expected state: + +- tests may fail because `inst/project-template/` and + `use_episcout_project()` are not implemented yet; +- PR 13 is expected to restore passing tests by implementing the missing + scaffold and helper. + +### PR 13: Implement project template + +Scope: + +- add reusable project scaffold; +- implement the user-facing project setup helper. Files: ```text inst/project-template/ +R/use_episcout_project.R ``` -Optional function: +Function: ```r -use_episcout_project() +use_episcout_project(path, overwrite = FALSE) ``` -### PR 13: Add large-data design note +### PR 14: Add large-data design note Scope: diff --git a/spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md b/spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md index 0122ea8..1a7d950 100644 --- a/spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md +++ b/spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md @@ -278,8 +278,9 @@ Continue with the revised PR plan. Next steps: 1. Add failing report-rendering tests. 2. Implement render_eda_report(). -3. Add project template. -4. Add large-data design note. +3. Add failing project-template and use_episcout_project() contract tests. +4. Implement project template and use_episcout_project(). +5. Add large-data design note. Follow spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md. @@ -287,3 +288,47 @@ Closeout: Update START_HERE.md after each remaining PR so it stays the live status dashboard. ``` + +## Instruction 12: Add project-template contract tests + +```text +Task: +Add failing tests for the project template and use_episcout_project() contract. + +Add: +- tests/testthat/test-project-template.R + +Requirements: +1. Tests define the bundled inst/project-template/ scaffold. +2. Tests define use_episcout_project(path, overwrite = FALSE). +3. Tests require reports/eda.qmd to culminate in render_eda_report(). +4. Do not add inst/project-template/ in this PR. +5. Do not implement use_episcout_project() in this PR. + +Closeout: +Update START_HERE.md to mark PR 12 Done and PR 13 Active. Record that PR 12 +may leave tests failing until PR 13 implements the scaffold and helper. +``` + +## Instruction 13: Implement project template + +```text +Task: +Implement the project template and use_episcout_project(). + +Add: +- inst/project-template/ +- R/use_episcout_project.R + +Requirements: +1. Include metadata/data_dictionary.csv, config/eda.yml, _targets.R, + reports/eda.qmd, R/project-derivations.R and outputs/. +2. use_episcout_project(path, overwrite = FALSE) creates the destination + project and copies the scaffold. +3. Refuse to overwrite existing files unless overwrite = TRUE. +4. Return the normalized project path invisibly. +5. Keep the report path connected to render_eda_report(). + +Closeout: +Update START_HERE.md to mark PR 13 Done and PR 14 Active. +``` diff --git a/tests/testthat/test-project-template.R b/tests/testthat/test-project-template.R new file mode 100644 index 0000000..f150f5a --- /dev/null +++ b/tests/testthat/test-project-template.R @@ -0,0 +1,84 @@ +context("project template contract tests") + +library(testthat) +library(episcout) + +expected_project_template_entries <- c( + "metadata/data_dictionary.csv", + "config/eda.yml", + "_targets.R", + "reports/eda.qmd", + "R/project-derivations.R", + "outputs" +) + +expect_project_template_entries <- function(root) { + for (entry in expected_project_template_entries) { + expect_true( + file.exists(file.path(root, entry)), + info = paste("Missing project template entry:", entry) + ) + } +} + +read_project_report_template <- function(root) { + paste( + readLines(file.path(root, "reports", "eda.qmd"), warn = FALSE), + collapse = "\n" + ) +} + +test_that("project template is bundled with the expected scaffold", { + template_path <- system.file("project-template", package = "episcout") + + expect_true(nzchar(template_path)) + if (!nzchar(template_path)) { + return(invisible()) + } + + expect_true(dir.exists(template_path)) + expect_project_template_entries(template_path) + expect_match(read_project_report_template(template_path), "render_eda_report") +}) + +test_that("use_episcout_project creates the expected scaffold", { + project_path <- tempfile("episcout-project-") + + returned_path <- use_episcout_project(project_path) + + expect_equal( + normalizePath(returned_path, winslash = "/", mustWork = TRUE), + normalizePath(project_path, winslash = "/", mustWork = TRUE) + ) + expect_project_template_entries(project_path) + expect_match(read_project_report_template(project_path), "render_eda_report") +}) + +test_that("use_episcout_project refuses to overwrite existing files by default", { + project_path <- tempfile("episcout-project-existing-") + dir.create(file.path(project_path, "metadata"), recursive = TRUE) + existing_file <- file.path(project_path, "metadata", "data_dictionary.csv") + writeLines("existing", existing_file) + + expect_error( + use_episcout_project(project_path), + regexp = "overwrite|exist" + ) + expect_equal(readLines(existing_file, warn = FALSE), "existing") +}) + +test_that("use_episcout_project can overwrite existing files when requested", { + project_path <- tempfile("episcout-project-overwrite-") + dir.create(file.path(project_path, "metadata"), recursive = TRUE) + existing_file <- file.path(project_path, "metadata", "data_dictionary.csv") + writeLines("existing", existing_file) + + returned_path <- use_episcout_project(project_path, overwrite = TRUE) + + expect_equal( + normalizePath(returned_path, winslash = "/", mustWork = TRUE), + normalizePath(project_path, winslash = "/", mustWork = TRUE) + ) + expect_project_template_entries(project_path) + expect_false(identical(readLines(existing_file, warn = FALSE), "existing")) +}) From bdd52b2c369cb356cb7df5efacd7b01e5b307469 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Mon, 15 Jun 2026 12:36:50 -0600 Subject: [PATCH 22/33] pr13 for reporting template --- NAMESPACE | 1 + R/use_episcout_project.R | 103 ++++++++++++++++++ inst/project-template/R/project-derivations.R | 3 + inst/project-template/_targets.R | 26 +++++ inst/project-template/config/eda.yml | 15 +++ .../metadata/data_dictionary.csv | 2 + inst/project-template/outputs/.gitkeep | 1 + inst/project-template/reports/eda.qmd | 29 +++++ man/use_episcout_project.Rd | 23 ++++ spec_driven_EDA_plan/docs/START_HERE.md | 34 +++--- 10 files changed, 220 insertions(+), 17 deletions(-) create mode 100644 R/use_episcout_project.R create mode 100644 inst/project-template/R/project-derivations.R create mode 100644 inst/project-template/_targets.R create mode 100644 inst/project-template/config/eda.yml create mode 100644 inst/project-template/metadata/data_dictionary.csv create mode 100644 inst/project-template/outputs/.gitkeep create mode 100644 inst/project-template/reports/eda.qmd create mode 100644 man/use_episcout_project.Rd diff --git a/NAMESPACE b/NAMESPACE index 9a205d5..7b40cfe 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -100,6 +100,7 @@ export(render_eda_report) export(run_eda) export(sym) export(syms) +export(use_episcout_project) export(validate_eda_spec) importFrom(broom,tidy) importFrom(dplyr,bind_rows) diff --git a/R/use_episcout_project.R b/R/use_episcout_project.R new file mode 100644 index 0000000..c3e6027 --- /dev/null +++ b/R/use_episcout_project.R @@ -0,0 +1,103 @@ +#' Create an episcout project scaffold +#' +#' Copy the bundled specification-first EDA project template into a destination +#' directory. +#' +#' @param path A single directory path where the project scaffold should be +#' created. +#' @param overwrite Logical; when `FALSE`, existing destination files are left +#' untouched and cause an error. When `TRUE`, existing destination files may +#' be replaced by template files. +#' +#' @return The normalized project path, invisibly. +#' +#' @export +use_episcout_project <- function(path, overwrite = FALSE) { + path <- validate_episcout_project_path(path) + overwrite <- validate_episcout_project_overwrite(overwrite) + + template_path <- system.file("project-template", package = "episcout") + if (!nzchar(template_path) || !dir.exists(template_path)) { + stop("The bundled episcout project template could not be found.", call. = FALSE) + } + + if (file.exists(path) && !dir.exists(path)) { + stop("path exists and is not a directory.", call. = FALSE) + } + + template_files <- list.files( + template_path, + all.files = TRUE, + no.. = TRUE, + recursive = TRUE, + full.names = FALSE, + include.dirs = FALSE + ) + + destination_files <- file.path(path, template_files) + existing_directories <- destination_files[dir.exists(destination_files)] + if (length(existing_directories) > 0) { + stop( + "Destination paths exist as directories and cannot be replaced by files: ", + paste(existing_directories, collapse = ", "), + call. = FALSE + ) + } + + existing_files <- destination_files[file.exists(destination_files)] + if (length(existing_files) > 0 && !overwrite) { + stop( + "Destination files already exist; use overwrite = TRUE to replace them: ", + paste(existing_files, collapse = ", "), + call. = FALSE + ) + } + + if (!dir.exists(path)) { + created <- dir.create(path, recursive = TRUE, showWarnings = FALSE) + if (!created) { + stop("Could not create project directory.", call. = FALSE) + } + } + + for (relative_file in template_files) { + source_file <- file.path(template_path, relative_file) + destination_file <- file.path(path, relative_file) + destination_dir <- dirname(destination_file) + + if (!dir.exists(destination_dir)) { + created <- dir.create(destination_dir, recursive = TRUE, showWarnings = FALSE) + if (!created) { + stop("Could not create project template directory.", call. = FALSE) + } + } + + copied <- file.copy( + from = source_file, + to = destination_file, + overwrite = overwrite, + copy.date = TRUE + ) + if (!isTRUE(copied)) { + stop("Could not copy project template file: ", relative_file, call. = FALSE) + } + } + + invisible(normalizePath(path, winslash = "/", mustWork = TRUE)) +} + +validate_episcout_project_path <- function(path) { + if (!is.character(path) || length(path) != 1 || is.na(path) || !nzchar(path)) { + stop("path must be a single non-empty directory path.", call. = FALSE) + } + + path.expand(path) +} + +validate_episcout_project_overwrite <- function(overwrite) { + if (!is.logical(overwrite) || length(overwrite) != 1 || is.na(overwrite)) { + stop("overwrite must be TRUE or FALSE.", call. = FALSE) + } + + overwrite +} diff --git a/inst/project-template/R/project-derivations.R b/inst/project-template/R/project-derivations.R new file mode 100644 index 0000000..bb7d817 --- /dev/null +++ b/inst/project-template/R/project-derivations.R @@ -0,0 +1,3 @@ +derive_project_data <- function(data) { + data +} diff --git a/inst/project-template/_targets.R b/inst/project-template/_targets.R new file mode 100644 index 0000000..2c5ce1c --- /dev/null +++ b/inst/project-template/_targets.R @@ -0,0 +1,26 @@ +library(targets) +library(episcout) + +source("R/project-derivations.R") + +tar_option_set(packages = c("episcout")) + +list( + tar_target( + raw_data, + read.csv("data/input.csv", stringsAsFactors = FALSE) + ), + tar_target( + analysis_data, + derive_project_data(raw_data) + ), + tar_target( + eda_report, + render_eda_report( + data = analysis_data, + spec = "metadata/data_dictionary.csv", + output_dir = "outputs" + ), + format = "file" + ) +) diff --git a/inst/project-template/config/eda.yml b/inst/project-template/config/eda.yml new file mode 100644 index 0000000..a7d0289 --- /dev/null +++ b/inst/project-template/config/eda.yml @@ -0,0 +1,15 @@ +project: + name: episcout-eda-project + +inputs: + data: data/input.csv + spec: metadata/data_dictionary.csv + +outputs: + directory: outputs + report: outputs/eda-report.html + +synthetic: + enabled: false + n: 100 + seed: 1 diff --git a/inst/project-template/metadata/data_dictionary.csv b/inst/project-template/metadata/data_dictionary.csv new file mode 100644 index 0000000..bc47e9a --- /dev/null +++ b/inst/project-template/metadata/data_dictionary.csv @@ -0,0 +1,2 @@ +"name","label","type","role","units","levels","min","max","missing_codes","required","group","description" +"example_numeric","Example numeric variable","numeric","covariate","units","",0,100,"",TRUE,"example","Replace this row with your project data dictionary." diff --git a/inst/project-template/outputs/.gitkeep b/inst/project-template/outputs/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/inst/project-template/outputs/.gitkeep @@ -0,0 +1 @@ + diff --git a/inst/project-template/reports/eda.qmd b/inst/project-template/reports/eda.qmd new file mode 100644 index 0000000..7495f30 --- /dev/null +++ b/inst/project-template/reports/eda.qmd @@ -0,0 +1,29 @@ +--- +title: "Project EDA Report" +format: html +params: + data_path: data/input.csv + spec_path: metadata/data_dictionary.csv + output_dir: outputs + synthetic: false + n: 100 + seed: 1 +--- + +# Project EDA Report + +Render the specification-first EDA report for this project with +`episcout::render_eda_report()`. + +```{r render-eda-report, eval=FALSE} +data <- read.csv(params$data_path, stringsAsFactors = FALSE) + +episcout::render_eda_report( + data = data, + spec = params$spec_path, + output_dir = params$output_dir, + synthetic = params$synthetic, + n = params$n, + seed = params$seed +) +``` diff --git a/man/use_episcout_project.Rd b/man/use_episcout_project.Rd new file mode 100644 index 0000000..17964d9 --- /dev/null +++ b/man/use_episcout_project.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/use_episcout_project.R +\name{use_episcout_project} +\alias{use_episcout_project} +\title{Create an episcout project scaffold} +\usage{ +use_episcout_project(path, overwrite = FALSE) +} +\arguments{ +\item{path}{A single directory path where the project scaffold should be +created.} + +\item{overwrite}{Logical; when \code{FALSE}, existing destination files are left +untouched and cause an error. When \code{TRUE}, existing destination files may +be replaced by template files.} +} +\value{ +The normalized project path, invisibly. +} +\description{ +Copy the bundled specification-first EDA project template into a destination +directory. +} diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/spec_driven_EDA_plan/docs/START_HERE.md index 8f8b235..ed2b287 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/spec_driven_EDA_plan/docs/START_HERE.md @@ -33,20 +33,20 @@ operational source of truth for what happens next. | PR 10 report-template tests | Done | Report-rendering tests are present and may fail until PR 11 implements `render_eda_report()`. | | PR 11 report rendering implementation | Done | `render_eda_report()` and the bundled Quarto report template are implemented. | | PR 12 project-template contract tests | Done | Failing tests define the project template and `use_episcout_project()` contract. | -| PR 13 project-template implementation | Active | Implement `inst/project-template/` and `use_episcout_project()` to satisfy PR 12 tests. | -| PR 14 large-data design note | Deferred | Document large-data backend strategy after the project-template PRs. | +| PR 13 project-template implementation | Done | `inst/project-template/` and `use_episcout_project()` are implemented. | +| PR 14 large-data design note | Active | Document large-data backend strategy without implementing backends. | ## Active PR ```text -PR 13: Implement project template +PR 14: Add large-data design note ``` Instruction: ```text -Follow Instruction 13 in: -spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md +Follow PR 14 in: +spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md ``` Required reading: @@ -62,26 +62,23 @@ spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md spec_driven_EDA_plan/docs/codex/review-checklist.md ``` -## PR 13 Scope +## PR 14 Scope Must edit: ```text -inst/project-template/ -R/use_episcout_project.R -NAMESPACE -man/ +spec_driven_EDA_plan/docs/sdd/large-data-backend-strategy.md spec_driven_EDA_plan/docs/START_HERE.md ``` May read: ```text -R/eda_report.R -inst/report-template/eda.qmd -tests/testthat/test-project-template.R -tests/testthat/test-eda_report.R -tests/testthat/test-run_eda-fixtures.R +spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md +spec_driven_EDA_plan/docs/sdd/mvp-scope.md +spec_driven_EDA_plan/docs/adr/0003-r-not-pure-base-r.md +spec_driven_EDA_plan/docs/roadmap.md +spec_driven_EDA_plan/docs/repository-audit.md spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md ``` @@ -98,13 +95,16 @@ R/eda_plots.R R/run_eda.R R/eda_report.R inst/report-template/eda.qmd +inst/project-template/ +NAMESPACE +man/ ``` Expected test state: ```text -PR 13 should make the PR 12 project-template tests pass without regressing -the PR 11 report-rendering tests. +PR 14 is documentation-only. No failing tests are expected, and no large-data +backend implementation should be added. ``` ## Closeout Rule From f76e88fa8af9a8b04a0630191cb970231886c37c Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:05:21 -0600 Subject: [PATCH 23/33] docs for eda sdd flow --- DESCRIPTION | 12 +- NAMESPACE | 28 +-- R/eda_missing.R | 4 +- R/eda_plots.R | 6 +- R/eda_report.R | 22 +-- R/eda_schema.R | 4 +- R/eda_spec.R | 6 +- R/eda_summaries.R | 4 +- R/eda_synthetic.R | 8 +- R/run_eda.R | 30 +-- R/use_episcout_project.R | 2 +- README.md | 98 ++++++++-- inst/project-template/README.md | 59 ++++++ inst/project-template/_targets.R | 2 +- inst/project-template/data/.gitkeep | 1 + inst/project-template/reports/eda.qmd | 4 +- ...heck_schema.Rd => epi_eda_check_schema.Rd} | 6 +- ...t_project.Rd => epi_eda_create_project.Rd} | 6 +- ....Rd => epi_eda_generate_synthetic_data.Rd} | 8 +- ..._missing.Rd => epi_eda_profile_missing.Rd} | 6 +- ...file_plots.Rd => epi_eda_profile_plots.Rd} | 6 +- ...maries.Rd => epi_eda_profile_summaries.Rd} | 6 +- ...eda_report.Rd => epi_eda_render_report.Rd} | 10 +- man/{run_eda.Rd => epi_eda_run.Rd} | 17 +- man/{eda_spec.Rd => epi_eda_spec.Rd} | 6 +- ...e_eda_spec.Rd => epi_eda_validate_spec.Rd} | 6 +- spec_driven_EDA_plan/docs/START_HERE.md | 24 +-- .../docs/adr/0001-build-on-episcout.md | 4 +- .../adr/0002-specification-first-workflow.md | 18 +- .../adr/0004-synthetic-data-before-access.md | 2 +- .../docs/codex/revised-pr-plan-tdd-first.md | 46 ++--- .../codex/tdd-first-codex-instructions.md | 78 ++++---- spec_driven_EDA_plan/docs/repository-audit.md | 20 +- spec_driven_EDA_plan/docs/roadmap.md | 22 +-- .../docs/sdd/0001-spec-first-eda.md | 28 +-- .../docs/sdd/data-dictionary-spec.md | 4 +- .../docs/sdd/tdd-external-fixtures.md | 28 +-- .../codex-pr-add-penguins-raw-fixture.md | 36 ++-- ...codex-pr-deferred-phase-1-stabilization.md | 8 +- ...eview-checklist-external-truth-fixtures.md | 16 +- .../future/phase-1-stabilization-sdd-tdd.md | 4 +- spec_driven_EDA_plan/prompt_loop.md | 2 +- tests/testthat/test-eda-api.R | 73 +++++++ tests/testthat/test-eda_missing-fixtures.R | 24 +-- tests/testthat/test-eda_plots-fixtures.R | 18 +- tests/testthat/test-eda_report.R | 24 +-- tests/testthat/test-eda_schema-fixtures.R | 22 +-- tests/testthat/test-eda_spec-fixtures.R | 10 +- tests/testthat/test-eda_summaries-fixtures.R | 18 +- tests/testthat/test-eda_synthetic-fixtures.R | 14 +- tests/testthat/test-project-template.R | 31 ++- tests/testthat/test-run_eda-fixtures.R | 32 +-- vignettes/introduction_episcout.Rmd | 11 +- vignettes/specification-first-eda.Rmd | 185 ++++++++++++++++++ 54 files changed, 792 insertions(+), 377 deletions(-) create mode 100644 inst/project-template/README.md create mode 100644 inst/project-template/data/.gitkeep rename man/{check_schema.Rd => epi_eda_check_schema.Rd} (83%) rename man/{use_episcout_project.Rd => epi_eda_create_project.Rd} (84%) rename man/{generate_synthetic_data.Rd => epi_eda_generate_synthetic_data.Rd} (79%) rename man/{profile_missing.Rd => epi_eda_profile_missing.Rd} (82%) rename man/{profile_plots.Rd => epi_eda_profile_plots.Rd} (84%) rename man/{profile_summaries.Rd => epi_eda_profile_summaries.Rd} (82%) rename man/{render_eda_report.Rd => epi_eda_render_report.Rd} (81%) rename man/{run_eda.Rd => epi_eda_run.Rd} (77%) rename man/{eda_spec.Rd => epi_eda_spec.Rd} (84%) rename man/{validate_eda_spec.Rd => epi_eda_validate_spec.Rd} (81%) create mode 100644 tests/testthat/test-eda-api.R create mode 100644 vignettes/specification-first-eda.Rmd diff --git a/DESCRIPTION b/DESCRIPTION index c0bdf01..c48de38 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -8,7 +8,12 @@ Authors@R: email = "antoniojberlanga@gmail.com") Author: Antonio Berlanga-Taylor [aut, cre] Maintainer: Antonio Berlanga-Taylor -Description: Facilitates cleaning, exploring and visualising large-ish datasets (hundreds of thousands to millions of observations with tens to hundreds of variables). These are mostly wrapper and convenience functions to pre-process (wrangle, explore, clean, etc.) data-sets. Assumes you're happy with tidyverse and the basics of data.table. +Description: Facilitates cleaning, exploring and visualising large-ish + epidemiological datasets (hundreds of thousands to millions of + observations with tens to hundreds of variables). Provides helper functions + for preprocessing and descriptive work, plus specification-first + exploratory data analysis workflows for epidemiological datasets. Assumes + you're happy with tidyverse and the basics of data.table. License: GPL-3 Encoding: UTF-8 LazyData: true @@ -52,7 +57,8 @@ Suggests: covr, withr, roxygen2, - mice + mice, + targets Roxygen: list(markdown = TRUE) VignetteBuilder: knitr -RoxygenNote: 7.3.2 +Config/roxygen2/version: 8.0.0 diff --git a/NAMESPACE b/NAMESPACE index 7b40cfe..1329b2b 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,8 +3,6 @@ export("%>%") export(":=") export(.data) -export(check_schema) -export(eda_spec) export(enexpr) export(enexprs) export(enquo) @@ -31,6 +29,17 @@ export(epi_clean_replace_value) export(epi_clean_spread_repeated) export(epi_clean_transpose) export(epi_create_dir) +export(epi_eda_check_schema) +export(epi_eda_create_project) +export(epi_eda_generate_synthetic_data) +export(epi_eda_profile_missing) +export(epi_eda_profile_plots) +export(epi_eda_profile_summaries) +export(epi_eda_render_report) +export(epi_eda_run) +export(epi_eda_spec) +export(epi_eda_validate_spec) +export(epi_grob_to_excel) export(epi_head_and_tail) export(epi_list_head) export(epi_list_tail) @@ -49,8 +58,8 @@ export(epi_plot_parallel) export(epi_plot_save_parallel) export(epi_plot_theme_1) export(epi_plot_theme_imss) -export(epi_plots_to_grid) export(epi_plot_volcano) +export(epi_plots_to_grid) export(epi_read) export(epi_stats_2x2_all) export(epi_stats_2x2_cols) @@ -85,23 +94,14 @@ export(epi_utils_session) export(epi_write) export(epi_write_df) export(expr) -export(generate_synthetic_data) -export(epi_grob_to_excel) export(palette_IMSS) export(palette_IMSS_accessible) -export(profile_missing) -export(profile_plots) -export(profile_summaries) export(quo) export(quo_name) export(quos) export(rename_contingency_2x2_cols) -export(render_eda_report) -export(run_eda) export(sym) export(syms) -export(use_episcout_project) -export(validate_eda_spec) importFrom(broom,tidy) importFrom(dplyr,bind_rows) importFrom(dplyr,group_by) @@ -115,6 +115,8 @@ importFrom(e1071,kurtosis) importFrom(e1071,skewness) importFrom(foreach,`%dopar%`) importFrom(grDevices,boxplot.stats) +importFrom(graphics,abline) +importFrom(graphics,plot) importFrom(magrittr,"%>%") importFrom(magrittr,`%>%`) importFrom(rlang,":=") @@ -151,5 +153,3 @@ importFrom(utils,capture.output) importFrom(utils,head) importFrom(utils,sessionInfo) importFrom(utils,tail) -importFrom(graphics,abline) -importFrom(graphics,plot) diff --git a/R/eda_missing.R b/R/eda_missing.R index 5e92182..e7e56fc 100644 --- a/R/eda_missing.R +++ b/R/eda_missing.R @@ -9,12 +9,12 @@ #' missing proportions. #' #' @export -profile_missing <- function(data, spec) { +epi_eda_profile_missing <- function(data, spec) { if (!is.data.frame(data)) { stop("Data must be a data frame.", call. = FALSE) } - spec <- eda_spec(spec) + spec <- epi_eda_spec(spec) n <- nrow(data) n_missing <- unname(vapply( spec$name, diff --git a/R/eda_plots.R b/R/eda_plots.R index 3593f0a..2915350 100644 --- a/R/eda_plots.R +++ b/R/eda_plots.R @@ -10,16 +10,16 @@ #' @return A named list of ggplot objects, one per specified variable. #' #' @export -profile_plots <- function(data, spec) { +epi_eda_profile_plots <- function(data, spec) { if (!is.data.frame(data)) { stop("EDA data must be a data frame.", call. = FALSE) } if (!requireNamespace("ggplot2", quietly = TRUE)) { - stop("The ggplot2 package is required for profile_plots().", call. = FALSE) + stop("The ggplot2 package is required for epi_eda_profile_plots().", call. = FALSE) } - spec <- eda_spec(spec) + spec <- epi_eda_spec(spec) missing_vars <- setdiff(spec$name, names(data)) if (length(missing_vars) > 0) { diff --git a/R/eda_report.R b/R/eda_report.R index 00aab3f..7c66f3e 100644 --- a/R/eda_report.R +++ b/R/eda_report.R @@ -6,30 +6,30 @@ #' @param data A data frame containing observed data. Required when #' `synthetic = FALSE`; ignored when `synthetic = TRUE`. #' @param spec An EDA specification data frame or CSV path accepted by -#' [eda_spec()]. +#' [epi_eda_spec()]. #' @param output_dir Directory where machine-readable workflow outputs and the #' rendered report are written. The directory must already exist. #' @param synthetic Logical; when `TRUE`, generate synthetic data from `spec` #' before running the workflow. #' @param n Number of synthetic rows to generate when `synthetic = TRUE`. -#' @param seed Optional random seed passed to [generate_synthetic_data()]. +#' @param seed Optional random seed passed to [epi_eda_generate_synthetic_data()]. #' @param quiet Logical; passed to [rmarkdown::render()] to control render #' output. #' #' @return A single character string containing the rendered HTML report path. #' #' @export -render_eda_report <- function(data, - spec, - output_dir, - synthetic = FALSE, - n = 100, - seed = NULL, - quiet = TRUE) { +epi_eda_render_report <- function(data, + spec, + output_dir, + synthetic = FALSE, + n = 100, + seed = NULL, + quiet = TRUE) { validate_run_eda_output_dir(output_dir) if (!requireNamespace("rmarkdown", quietly = TRUE)) { - stop("The rmarkdown package is required for render_eda_report().", call. = FALSE) + stop("The rmarkdown package is required for epi_eda_render_report().", call. = FALSE) } template_path <- system.file( @@ -42,7 +42,7 @@ render_eda_report <- function(data, stop("The bundled EDA report template could not be found.", call. = FALSE) } - results <- run_eda( + results <- epi_eda_run( data = data, spec = spec, output_dir = output_dir, diff --git a/R/eda_schema.R b/R/eda_schema.R index 11b3eed..352863e 100644 --- a/R/eda_schema.R +++ b/R/eda_schema.R @@ -9,12 +9,12 @@ #' @return A data frame with one row per expected or unexpected variable. #' #' @export -check_schema <- function(data, spec) { +epi_eda_check_schema <- function(data, spec) { if (!is.data.frame(data)) { stop("Data must be a data frame.", call. = FALSE) } - spec <- eda_spec(spec) + spec <- epi_eda_spec(spec) data_names <- names(data) expected_names <- spec$name diff --git a/R/eda_spec.R b/R/eda_spec.R index f407297..7b88452 100644 --- a/R/eda_spec.R +++ b/R/eda_spec.R @@ -8,7 +8,7 @@ #' @return A validated data frame specification. #' #' @export -eda_spec <- function(path_or_data) { +epi_eda_spec <- function(path_or_data) { if (is.character(path_or_data) && length(path_or_data) == 1) { spec <- utils::read.csv( path_or_data, @@ -21,7 +21,7 @@ eda_spec <- function(path_or_data) { stop("EDA specification must be a CSV path or a data frame.", call. = FALSE) } - validate_eda_spec(spec) + epi_eda_validate_spec(spec) } #' Validate an EDA specification @@ -34,7 +34,7 @@ eda_spec <- function(path_or_data) { #' @return The validated specification as a data frame. #' #' @export -validate_eda_spec <- function(spec) { +epi_eda_validate_spec <- function(spec) { if (!is.data.frame(spec)) { stop("EDA specification must be a data frame.", call. = FALSE) } diff --git a/R/eda_summaries.R b/R/eda_summaries.R index dfd411c..3e18506 100644 --- a/R/eda_summaries.R +++ b/R/eda_summaries.R @@ -9,12 +9,12 @@ #' @return A named list with `numeric` and `categorical` data frames. #' #' @export -profile_summaries <- function(data, spec) { +epi_eda_profile_summaries <- function(data, spec) { if (!is.data.frame(data)) { stop("EDA data must be a data frame.", call. = FALSE) } - spec <- eda_spec(spec) + spec <- epi_eda_spec(spec) missing_vars <- setdiff(spec$name, names(data)) if (length(missing_vars) > 0) { diff --git a/R/eda_synthetic.R b/R/eda_synthetic.R index 838af6f..9a98ddb 100644 --- a/R/eda_synthetic.R +++ b/R/eda_synthetic.R @@ -6,7 +6,7 @@ #' control. #' #' @param spec A data frame containing an EDA specification, or a path accepted -#' by [eda_spec()]. +#' by [epi_eda_spec()]. #' @param n Number of rows to generate. #' @param seed Optional random seed. When supplied, repeated calls with the same #' specification, row count and seed return identical data. @@ -14,11 +14,11 @@ #' @return A data frame with one column per specification variable and `n` rows. #' #' @export -generate_synthetic_data <- function(spec, n = 100, seed = NULL) { +epi_eda_generate_synthetic_data <- function(spec, n = 100, seed = NULL) { spec <- if (is.data.frame(spec)) { - validate_eda_spec(spec) + epi_eda_validate_spec(spec) } else { - eda_spec(spec) + epi_eda_spec(spec) } if (!is.numeric(n) || length(n) != 1 || is.na(n) || n < 0 || n != floor(n)) { diff --git a/R/run_eda.R b/R/run_eda.R index df599ed..810f34d 100644 --- a/R/run_eda.R +++ b/R/run_eda.R @@ -8,29 +8,29 @@ #' @param data A data frame containing observed data. Required when #' `synthetic = FALSE`; ignored when `synthetic = TRUE`. #' @param spec An EDA specification data frame or CSV path accepted by -#' [eda_spec()]. +#' [epi_eda_spec()]. #' @param output_dir Optional directory where machine-readable CSV outputs are #' written. The directory must already exist. #' @param synthetic Logical; when `TRUE`, generate synthetic data from `spec` #' before running the workflow. #' @param n Number of synthetic rows to generate when `synthetic = TRUE`. -#' @param seed Optional random seed passed to [generate_synthetic_data()]. +#' @param seed Optional random seed passed to [epi_eda_generate_synthetic_data()]. #' #' @return A named list with `metadata`, `schema`, `missing`, `summaries` and #' `plots` components. #' #' @export -run_eda <- function(data, - spec, - output_dir = NULL, - synthetic = FALSE, - n = 100, - seed = NULL) { +epi_eda_run <- function(data, + spec, + output_dir = NULL, + synthetic = FALSE, + n = 100, + seed = NULL) { synthetic <- validate_run_eda_synthetic(synthetic) - spec <- eda_spec(spec) + spec <- epi_eda_spec(spec) if (synthetic) { - data <- generate_synthetic_data(spec = spec, n = n, seed = seed) + data <- epi_eda_generate_synthetic_data(spec = spec, n = n, seed = seed) } else if (!is.data.frame(data)) { stop("data must be a data frame when synthetic is FALSE.", call. = FALSE) } @@ -41,10 +41,10 @@ run_eda <- function(data, results <- list( metadata = run_eda_metadata(data, spec, synthetic = synthetic), - schema = check_schema(data, spec), - missing = profile_missing(data, spec), - summaries = profile_summaries(data, spec), - plots = profile_plots(data, spec) + schema = epi_eda_check_schema(data, spec), + missing = epi_eda_profile_missing(data, spec), + summaries = epi_eda_profile_summaries(data, spec), + plots = epi_eda_profile_plots(data, spec) ) if (!is.null(output_dir)) { @@ -67,7 +67,7 @@ validate_run_eda_output_dir <- function(output_dir) { } if (!dir.exists(output_dir)) { - stop("output_dir must exist before run_eda() writes outputs.", call. = FALSE) + stop("output_dir must exist before epi_eda_run() writes outputs.", call. = FALSE) } invisible(TRUE) diff --git a/R/use_episcout_project.R b/R/use_episcout_project.R index c3e6027..d63eb02 100644 --- a/R/use_episcout_project.R +++ b/R/use_episcout_project.R @@ -12,7 +12,7 @@ #' @return The normalized project path, invisibly. #' #' @export -use_episcout_project <- function(path, overwrite = FALSE) { +epi_eda_create_project <- function(path, overwrite = FALSE) { path <- validate_episcout_project_path(path) overwrite <- validate_episcout_project_overwrite(overwrite) diff --git a/README.md b/README.md index 1948a24..cd3507c 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,19 @@ # episcout -episcout provides helper functions for cleaning, exploring and visualising large datasets. It wraps common preprocessing and descriptive tasks so you can focus on analysis. The package builds on the **tidyverse** and **data.table** ecosystems for fast and flexible data manipulation. +episcout provides helper functions for cleaning, exploring and visualising large +epidemiological datasets. It also supports specification-first exploratory data +analysis workflows for epidemiological datasets, where a data dictionary drives +schema checks, missingness summaries, descriptive summaries, plots and optional +HTML reports. ## Features -* **Cleaning** – `epi_clean_*` functions tidy raw data and detect issues such as duplicates or inconsistent labels. -* **Statistics** – `epi_stats_*` functions create summary tables and descriptive statistics in a single call. -* **Plotting** – `epi_plot_*` wrappers make it straightforward to produce common graphs with *ggplot2* and *cowplot*. -* **Utilities** – `epi_utils_*` helpers cover tasks like parallel processing and logging. +* **Cleaning** - `epi_clean_*` functions tidy raw data and detect issues such as duplicates or inconsistent labels. +* **Statistics** - `epi_stats_*` functions create summary tables and descriptive statistics in a single call. +* **Plotting** - `epi_plot_*` wrappers produce common graphs with *ggplot2* and *cowplot*. +* **Specification-first EDA** - `epi_eda_*` functions use a data dictionary to run repeatable EDA on synthetic or real data. +* **Utilities** - `epi_utils_*` helpers cover tasks like parallel processing and logging. ## Installation @@ -70,16 +75,17 @@ submission checklist and Writing R Extensions for the current source of truth: ## Getting Started -Functions are grouped by purpose, e.g.: -epi_clean_* for data wrangling/cleanup. -epi_stats_* for generating descriptive statistics and contingency tables. -epi_plot_* for plotting (wrappers around ggplot2 and cowplot). -epi_utils_* for utilities such as parallel processing, logging, etc. -Miscellaneous helpers such as epi_read/epi_write. +There are two main ways to use episcout: -## Example +* Use lower-level helpers directly: `epi_clean_*`, `epi_stats_*`, `epi_plot_*` + and `epi_utils_*`. +* Use the specification-first EDA workflow: `epi_eda_spec()`, + `epi_eda_generate_synthetic_data()`, `epi_eda_run()` and + `epi_eda_render_report()`. -This is a basic example of things you can do with episcout: +### Helper functions + +This is a basic example of the lower-level helper API: ``` r library(episcout) @@ -113,6 +119,72 @@ desc_stats # And many more functions for cleaning, stats and plotting that do things a bit faster or more conveniently and I couldn't easily find in other packages. ``` +### Specification-first EDA quickstart + +Start from a data dictionary with at least these columns: + +``` csv +name,label,type,role,units,levels,min,max,missing_codes,required,group,description +age,Age at baseline,numeric,covariate,years,,18,110,,TRUE,demographics,Age in years +sex,Sex at birth,categorical,covariate,,"Female;Male;Unknown",,,,TRUE,demographics,Recorded sex +death,Death during follow-up,binary,outcome,,"0;1",0,1,,TRUE,outcomes,Outcome indicator +``` + +You can prepare the workflow before real data arrive by generating synthetic +data from the same specification: + +``` r +library(episcout) + +spec <- epi_eda_spec("metadata/data_dictionary.csv") + +results <- epi_eda_run( + data = NULL, + spec = spec, + synthetic = TRUE, + n = 100, + seed = 1 +) + +names(results) +results$metadata +``` + +When real data are available, keep the same specification and change only the +data source: + +``` r +data <- read.csv("data/input.csv", stringsAsFactors = FALSE) +dir.create("outputs", showWarnings = FALSE) + +results <- epi_eda_run( + data = data, + spec = spec, + output_dir = "outputs" +) +``` + +Render the optional HTML report when `rmarkdown` is installed: + +``` r +epi_eda_render_report( + data = data, + spec = spec, + output_dir = "outputs" +) +``` + +To create a starter project scaffold: + +``` r +epi_eda_create_project("my-eda-project") +``` + +Current EDA workflow limits: summaries and plots are deliberately basic, the +synthetic data generator is for pipeline preparation and testing only, generated +synthetic data are not suitable for inference or disclosure control, and the MVP +does not yet include Arrow, DuckDB or data.table large-data backends. + ## Contribute - [Issue Tracker](https://github.com/AntonioJBT/episcout/issues) diff --git a/inst/project-template/README.md b/inst/project-template/README.md new file mode 100644 index 0000000..6f9459c --- /dev/null +++ b/inst/project-template/README.md @@ -0,0 +1,59 @@ +# episcout EDA project + +This scaffold is a starting point for specification-first exploratory data +analysis with `episcout`. + +## Files + +- `metadata/data_dictionary.csv`: edit this first. It defines the expected + variables and drives synthetic data, schema checks, summaries and plots. +- `data/input.csv`: place the real input dataset here when it is available. +- `outputs/`: generated CSV outputs and the HTML report are written here. +- `R/project-derivations.R`: add project-specific derived-variable code. +- `_targets.R`: optional targets pipeline. +- `reports/eda.qmd`: direct report-rendering example. + +## Direct use + +```r +library(episcout) + +data <- read.csv("data/input.csv", stringsAsFactors = FALSE) + +epi_eda_render_report( + data = data, + spec = "metadata/data_dictionary.csv", + output_dir = "outputs" +) +``` + +## Synthetic-data preparation + +Use synthetic data while access to real data is pending: + +```r +library(episcout) + +epi_eda_render_report( + data = NULL, + spec = "metadata/data_dictionary.csv", + output_dir = "outputs", + synthetic = TRUE, + n = 100, + seed = 1 +) +``` + +Synthetic data are for pipeline preparation and testing only. They are not for +inference or disclosure control. + +## Optional targets workflow + +If `targets` is installed, run: + +```r +targets::tar_make() +``` + +The targets pipeline reads `data/input.csv`, applies `derive_project_data()`, +and renders the EDA report. diff --git a/inst/project-template/_targets.R b/inst/project-template/_targets.R index 2c5ce1c..688f877 100644 --- a/inst/project-template/_targets.R +++ b/inst/project-template/_targets.R @@ -16,7 +16,7 @@ list( ), tar_target( eda_report, - render_eda_report( + epi_eda_render_report( data = analysis_data, spec = "metadata/data_dictionary.csv", output_dir = "outputs" diff --git a/inst/project-template/data/.gitkeep b/inst/project-template/data/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/inst/project-template/data/.gitkeep @@ -0,0 +1 @@ + diff --git a/inst/project-template/reports/eda.qmd b/inst/project-template/reports/eda.qmd index 7495f30..464dba8 100644 --- a/inst/project-template/reports/eda.qmd +++ b/inst/project-template/reports/eda.qmd @@ -13,12 +13,12 @@ params: # Project EDA Report Render the specification-first EDA report for this project with -`episcout::render_eda_report()`. +`episcout::epi_eda_render_report()`. ```{r render-eda-report, eval=FALSE} data <- read.csv(params$data_path, stringsAsFactors = FALSE) -episcout::render_eda_report( +episcout::epi_eda_render_report( data = data, spec = params$spec_path, output_dir = params$output_dir, diff --git a/man/check_schema.Rd b/man/epi_eda_check_schema.Rd similarity index 83% rename from man/check_schema.Rd rename to man/epi_eda_check_schema.Rd index 04ed79b..fd46350 100644 --- a/man/check_schema.Rd +++ b/man/epi_eda_check_schema.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/eda_schema.R -\name{check_schema} -\alias{check_schema} +\name{epi_eda_check_schema} +\alias{epi_eda_check_schema} \title{Check observed data against an EDA specification} \usage{ -check_schema(data, spec) +epi_eda_check_schema(data, spec) } \arguments{ \item{data}{A data frame to check.} diff --git a/man/use_episcout_project.Rd b/man/epi_eda_create_project.Rd similarity index 84% rename from man/use_episcout_project.Rd rename to man/epi_eda_create_project.Rd index 17964d9..a0e925b 100644 --- a/man/use_episcout_project.Rd +++ b/man/epi_eda_create_project.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/use_episcout_project.R -\name{use_episcout_project} -\alias{use_episcout_project} +\name{epi_eda_create_project} +\alias{epi_eda_create_project} \title{Create an episcout project scaffold} \usage{ -use_episcout_project(path, overwrite = FALSE) +epi_eda_create_project(path, overwrite = FALSE) } \arguments{ \item{path}{A single directory path where the project scaffold should be diff --git a/man/generate_synthetic_data.Rd b/man/epi_eda_generate_synthetic_data.Rd similarity index 79% rename from man/generate_synthetic_data.Rd rename to man/epi_eda_generate_synthetic_data.Rd index 648c3fc..1989241 100644 --- a/man/generate_synthetic_data.Rd +++ b/man/epi_eda_generate_synthetic_data.Rd @@ -1,14 +1,14 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/eda_synthetic.R -\name{generate_synthetic_data} -\alias{generate_synthetic_data} +\name{epi_eda_generate_synthetic_data} +\alias{epi_eda_generate_synthetic_data} \title{Generate synthetic data from an EDA specification} \usage{ -generate_synthetic_data(spec, n = 100, seed = NULL) +epi_eda_generate_synthetic_data(spec, n = 100, seed = NULL) } \arguments{ \item{spec}{A data frame containing an EDA specification, or a path accepted -by \code{\link[=eda_spec]{eda_spec()}}.} +by \code{\link[=epi_eda_spec]{epi_eda_spec()}}.} \item{n}{Number of rows to generate.} diff --git a/man/profile_missing.Rd b/man/epi_eda_profile_missing.Rd similarity index 82% rename from man/profile_missing.Rd rename to man/epi_eda_profile_missing.Rd index 56ba465..d3fc728 100644 --- a/man/profile_missing.Rd +++ b/man/epi_eda_profile_missing.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/eda_missing.R -\name{profile_missing} -\alias{profile_missing} +\name{epi_eda_profile_missing} +\alias{epi_eda_profile_missing} \title{Profile missingness using an EDA specification} \usage{ -profile_missing(data, spec) +epi_eda_profile_missing(data, spec) } \arguments{ \item{data}{A data frame to profile.} diff --git a/man/profile_plots.Rd b/man/epi_eda_profile_plots.Rd similarity index 84% rename from man/profile_plots.Rd rename to man/epi_eda_profile_plots.Rd index 4816507..40a5a9a 100644 --- a/man/profile_plots.Rd +++ b/man/epi_eda_profile_plots.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/eda_plots.R -\name{profile_plots} -\alias{profile_plots} +\name{epi_eda_profile_plots} +\alias{epi_eda_profile_plots} \title{Profile plots using an EDA specification} \usage{ -profile_plots(data, spec) +epi_eda_profile_plots(data, spec) } \arguments{ \item{data}{A data frame containing observed data.} diff --git a/man/profile_summaries.Rd b/man/epi_eda_profile_summaries.Rd similarity index 82% rename from man/profile_summaries.Rd rename to man/epi_eda_profile_summaries.Rd index 1deec60..8cc413a 100644 --- a/man/profile_summaries.Rd +++ b/man/epi_eda_profile_summaries.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/eda_summaries.R -\name{profile_summaries} -\alias{profile_summaries} +\name{epi_eda_profile_summaries} +\alias{epi_eda_profile_summaries} \title{Profile summaries using an EDA specification} \usage{ -profile_summaries(data, spec) +epi_eda_profile_summaries(data, spec) } \arguments{ \item{data}{A data frame containing observed data.} diff --git a/man/render_eda_report.Rd b/man/epi_eda_render_report.Rd similarity index 81% rename from man/render_eda_report.Rd rename to man/epi_eda_render_report.Rd index 3cf8b04..8b839b5 100644 --- a/man/render_eda_report.Rd +++ b/man/epi_eda_render_report.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/eda_report.R -\name{render_eda_report} -\alias{render_eda_report} +\name{epi_eda_render_report} +\alias{epi_eda_render_report} \title{Render a specification-first EDA report} \usage{ -render_eda_report( +epi_eda_render_report( data, spec, output_dir, @@ -19,7 +19,7 @@ render_eda_report( \code{synthetic = FALSE}; ignored when \code{synthetic = TRUE}.} \item{spec}{An EDA specification data frame or CSV path accepted by -\code{\link[=eda_spec]{eda_spec()}}.} +\code{\link[=epi_eda_spec]{epi_eda_spec()}}.} \item{output_dir}{Directory where machine-readable workflow outputs and the rendered report are written. The directory must already exist.} @@ -29,7 +29,7 @@ before running the workflow.} \item{n}{Number of synthetic rows to generate when \code{synthetic = TRUE}.} -\item{seed}{Optional random seed passed to \code{\link[=generate_synthetic_data]{generate_synthetic_data()}}.} +\item{seed}{Optional random seed passed to \code{\link[=epi_eda_generate_synthetic_data]{epi_eda_generate_synthetic_data()}}.} \item{quiet}{Logical; passed to \code{\link[rmarkdown:render]{rmarkdown::render()}} to control render output.} diff --git a/man/run_eda.Rd b/man/epi_eda_run.Rd similarity index 77% rename from man/run_eda.Rd rename to man/epi_eda_run.Rd index ad0e926..1127d2d 100644 --- a/man/run_eda.Rd +++ b/man/epi_eda_run.Rd @@ -1,17 +1,24 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/run_eda.R -\name{run_eda} -\alias{run_eda} +\name{epi_eda_run} +\alias{epi_eda_run} \title{Run a specification-first EDA workflow} \usage{ -run_eda(data, spec, output_dir = NULL, synthetic = FALSE, n = 100, seed = NULL) +epi_eda_run( + data, + spec, + output_dir = NULL, + synthetic = FALSE, + n = 100, + seed = NULL +) } \arguments{ \item{data}{A data frame containing observed data. Required when \code{synthetic = FALSE}; ignored when \code{synthetic = TRUE}.} \item{spec}{An EDA specification data frame or CSV path accepted by -\code{\link[=eda_spec]{eda_spec()}}.} +\code{\link[=epi_eda_spec]{epi_eda_spec()}}.} \item{output_dir}{Optional directory where machine-readable CSV outputs are written. The directory must already exist.} @@ -21,7 +28,7 @@ before running the workflow.} \item{n}{Number of synthetic rows to generate when \code{synthetic = TRUE}.} -\item{seed}{Optional random seed passed to \code{\link[=generate_synthetic_data]{generate_synthetic_data()}}.} +\item{seed}{Optional random seed passed to \code{\link[=epi_eda_generate_synthetic_data]{epi_eda_generate_synthetic_data()}}.} } \value{ A named list with \code{metadata}, \code{schema}, \code{missing}, \code{summaries} and diff --git a/man/eda_spec.Rd b/man/epi_eda_spec.Rd similarity index 84% rename from man/eda_spec.Rd rename to man/epi_eda_spec.Rd index 5bcf5cc..0f395fb 100644 --- a/man/eda_spec.Rd +++ b/man/epi_eda_spec.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/eda_spec.R -\name{eda_spec} -\alias{eda_spec} +\name{epi_eda_spec} +\alias{epi_eda_spec} \title{Read an EDA specification} \usage{ -eda_spec(path_or_data) +epi_eda_spec(path_or_data) } \arguments{ \item{path_or_data}{A path to a CSV specification file, or a data frame diff --git a/man/validate_eda_spec.Rd b/man/epi_eda_validate_spec.Rd similarity index 81% rename from man/validate_eda_spec.Rd rename to man/epi_eda_validate_spec.Rd index e13cc34..e3a99a4 100644 --- a/man/validate_eda_spec.Rd +++ b/man/epi_eda_validate_spec.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/eda_spec.R -\name{validate_eda_spec} -\alias{validate_eda_spec} +\name{epi_eda_validate_spec} +\alias{epi_eda_validate_spec} \title{Validate an EDA specification} \usage{ -validate_eda_spec(spec) +epi_eda_validate_spec(spec) } \arguments{ \item{spec}{A data frame containing an EDA specification.} diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/spec_driven_EDA_plan/docs/START_HERE.md index ed2b287..78d06d4 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/spec_driven_EDA_plan/docs/START_HERE.md @@ -24,16 +24,16 @@ operational source of truth for what happens next. | PR 1 external fixture files | Done | Fixture files are present under `tests/testthat/fixtures/blood_storage/`. | | PR 2 fixture-backed failing tests | Done | Tests are present and may fail until PR 3 implements the missing functions. | | PR 3 spec, schema and missingness implementation | Done | Spec, schema and missingness functions are implemented. | -| PR 4 synthetic-data failing tests | Done | Tests are present and may fail until PR 5 implements `generate_synthetic_data()`. | -| PR 5 synthetic-data implementation | Done | `generate_synthetic_data()` is implemented. | -| PR 6 summary and plot failing tests | Done | Tests are present and may fail until PR 7 implements `profile_summaries()` and `profile_plots()`. | -| PR 7 summary and plot implementation | Done | `profile_summaries()` and `profile_plots()` are implemented. | -| PR 8 run_eda failing tests | Done | Tests are present and may fail until PR 9 implements `run_eda()`. | -| PR 9 run_eda implementation | Done | `run_eda()` is implemented. | -| PR 10 report-template tests | Done | Report-rendering tests are present and may fail until PR 11 implements `render_eda_report()`. | -| PR 11 report rendering implementation | Done | `render_eda_report()` and the bundled Quarto report template are implemented. | -| PR 12 project-template contract tests | Done | Failing tests define the project template and `use_episcout_project()` contract. | -| PR 13 project-template implementation | Done | `inst/project-template/` and `use_episcout_project()` are implemented. | +| PR 4 synthetic-data failing tests | Done | Tests are present and may fail until PR 5 implements `epi_eda_generate_synthetic_data()`. | +| PR 5 synthetic-data implementation | Done | `epi_eda_generate_synthetic_data()` is implemented. | +| PR 6 summary and plot failing tests | Done | Tests are present and may fail until PR 7 implements `epi_eda_profile_summaries()` and `epi_eda_profile_plots()`. | +| PR 7 summary and plot implementation | Done | `epi_eda_profile_summaries()` and `epi_eda_profile_plots()` are implemented. | +| PR 8 epi_eda_run failing tests | Done | Tests are present and may fail until PR 9 implements `epi_eda_run()`. | +| PR 9 epi_eda_run implementation | Done | `epi_eda_run()` is implemented. | +| PR 10 report-template tests | Done | Report-rendering tests are present and may fail until PR 11 implements `epi_eda_render_report()`. | +| PR 11 report rendering implementation | Done | `epi_eda_render_report()` and the bundled Quarto report template are implemented. | +| PR 12 project-template contract tests | Done | Failing tests define the project template and `epi_eda_create_project()` contract. | +| PR 13 project-template implementation | Done | `inst/project-template/` and `epi_eda_create_project()` are implemented. | | PR 14 large-data design note | Active | Document large-data backend strategy without implementing backends. | ## Active PR @@ -86,13 +86,13 @@ spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md Must not edit: ```text -R/eda_spec.R +R/epi_eda_spec.R R/eda_schema.R R/eda_missing.R R/eda_synthetic.R R/eda_summaries.R R/eda_plots.R -R/run_eda.R +R/epi_eda_run.R R/eda_report.R inst/report-template/eda.qmd inst/project-template/ diff --git a/spec_driven_EDA_plan/docs/adr/0001-build-on-episcout.md b/spec_driven_EDA_plan/docs/adr/0001-build-on-episcout.md index 4ec00eb..82e2b90 100644 --- a/spec_driven_EDA_plan/docs/adr/0001-build-on-episcout.md +++ b/spec_driven_EDA_plan/docs/adr/0001-build-on-episcout.md @@ -37,14 +37,14 @@ Negative: Add new files for the new layer: ```text -R/eda_spec.R +R/epi_eda_spec.R R/eda_schema.R R/eda_synthetic.R R/eda_missing.R R/eda_summaries.R R/eda_plots.R R/eda_report.R -R/run_eda.R +R/epi_eda_run.R ``` Do not remove existing exported functions during the MVP. diff --git a/spec_driven_EDA_plan/docs/adr/0002-specification-first-workflow.md b/spec_driven_EDA_plan/docs/adr/0002-specification-first-workflow.md index c677df4..d2b394e 100644 --- a/spec_driven_EDA_plan/docs/adr/0002-specification-first-workflow.md +++ b/spec_driven_EDA_plan/docs/adr/0002-specification-first-workflow.md @@ -46,15 +46,15 @@ Negative: Initial functions: ```r -eda_spec() -validate_eda_spec() -generate_synthetic_data() -check_schema() -profile_missing() -profile_summaries() -profile_plots() -run_eda() -render_eda_report() +epi_eda_spec() +epi_eda_validate_spec() +epi_eda_generate_synthetic_data() +epi_eda_check_schema() +epi_eda_profile_missing() +epi_eda_profile_summaries() +epi_eda_profile_plots() +epi_eda_run() +epi_eda_render_report() ``` The package should also allow a dataset-only convenience path later, where a draft specification can be inferred from real data. diff --git a/spec_driven_EDA_plan/docs/adr/0004-synthetic-data-before-access.md b/spec_driven_EDA_plan/docs/adr/0004-synthetic-data-before-access.md index 60c1701..7d9cce8 100644 --- a/spec_driven_EDA_plan/docs/adr/0004-synthetic-data-before-access.md +++ b/spec_driven_EDA_plan/docs/adr/0004-synthetic-data-before-access.md @@ -14,7 +14,7 @@ Synthetic data generated from the specification can support code development, re ## Decision -Add `generate_synthetic_data()` as a first-class part of the specification-first workflow. +Add `epi_eda_generate_synthetic_data()` as a first-class part of the specification-first workflow. Synthetic data should be generated from the same EDA specification used to validate real data. diff --git a/spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md b/spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md index 3241878..c9a45fd 100644 --- a/spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md +++ b/spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md @@ -54,7 +54,7 @@ Scope: Files: ```text -tests/testthat/test-eda_spec-fixtures.R +tests/testthat/test-epi_eda_spec-fixtures.R tests/testthat/test-eda_schema-fixtures.R tests/testthat/test-eda_missing-fixtures.R ``` @@ -74,7 +74,7 @@ Scope: Files: ```text -R/eda_spec.R +R/epi_eda_spec.R R/eda_schema.R R/eda_missing.R ``` @@ -82,10 +82,10 @@ R/eda_missing.R Functions: ```r -eda_spec() -validate_eda_spec() -check_schema() -profile_missing() +epi_eda_spec() +epi_eda_validate_spec() +epi_eda_check_schema() +epi_eda_profile_missing() ``` ### PR 4: Add failing synthetic-data tests @@ -93,7 +93,7 @@ profile_missing() Scope: - add fixture-backed tests for synthetic data generation; -- do not implement `generate_synthetic_data()` yet. +- do not implement `epi_eda_generate_synthetic_data()` yet. File: @@ -122,7 +122,7 @@ R/eda_synthetic.R Function: ```r -generate_synthetic_data() +epi_eda_generate_synthetic_data() ``` ### PR 6: Add failing summary and plot tests @@ -130,7 +130,7 @@ generate_synthetic_data() Scope: - add fixture-backed tests for summaries and plots; -- do not implement `profile_summaries()` or `profile_plots()` yet. +- do not implement `epi_eda_profile_summaries()` or `epi_eda_profile_plots()` yet. Files: @@ -162,21 +162,21 @@ R/eda_plots.R Functions: ```r -profile_summaries() -profile_plots() +epi_eda_profile_summaries() +epi_eda_profile_plots() ``` -### PR 8: Add failing run_eda end-to-end tests +### PR 8: Add failing epi_eda_run end-to-end tests Scope: - add fixture-backed end-to-end workflow tests; -- do not implement `run_eda()` yet. +- do not implement `epi_eda_run()` yet. File: ```text -tests/testthat/test-run_eda-fixtures.R +tests/testthat/test-epi_eda_run-fixtures.R ``` Tests: @@ -186,7 +186,7 @@ Tests: - temporary output directory; - expected named components. -### PR 9: Implement run_eda() +### PR 9: Implement epi_eda_run() Scope: @@ -195,13 +195,13 @@ Scope: File: ```text -R/run_eda.R +R/epi_eda_run.R ``` Function: ```r -run_eda() +epi_eda_run() ``` ### PR 10: Add report-template tests @@ -209,7 +209,7 @@ run_eda() Scope: - add tests for report rendering; -- do not implement `render_eda_report()` yet. +- do not implement `epi_eda_render_report()` yet. File: @@ -233,7 +233,7 @@ inst/report-template/eda.qmd Function: ```r -render_eda_report() +epi_eda_render_report() ``` ### PR 12: Add project-template contract tests @@ -253,7 +253,7 @@ tests/testthat/test-project-template.R Future function contract: ```r -use_episcout_project(path, overwrite = FALSE) +epi_eda_create_project(path, overwrite = FALSE) ``` Expected template entries: @@ -270,7 +270,7 @@ outputs/ Expected state: - tests may fail because `inst/project-template/` and - `use_episcout_project()` are not implemented yet; + `epi_eda_create_project()` are not implemented yet; - PR 13 is expected to restore passing tests by implementing the missing scaffold and helper. @@ -285,13 +285,13 @@ Files: ```text inst/project-template/ -R/use_episcout_project.R +R/epi_eda_create_project.R ``` Function: ```r -use_episcout_project(path, overwrite = FALSE) +epi_eda_create_project(path, overwrite = FALSE) ``` ### PR 14: Add large-data design note diff --git a/spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md b/spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md index 1a7d950..783732c 100644 --- a/spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md +++ b/spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md @@ -75,7 +75,7 @@ Task: Add fixture-backed tests before implementing the new functions. Add: -- tests/testthat/test-eda_spec-fixtures.R +- tests/testthat/test-epi_eda_spec-fixtures.R - tests/testthat/test-eda_schema-fixtures.R - tests/testthat/test-eda_missing-fixtures.R @@ -90,7 +90,7 @@ Requirements: 2. Tests must not require internet. 3. Tests must not depend on private data. 4. Tests must compare against independently computed expected outputs. -5. Do not implement eda_spec(), check_schema() or profile_missing() in this PR. +5. Do not implement epi_eda_spec(), epi_eda_check_schema() or epi_eda_profile_missing() in this PR. Closeout: Update START_HERE.md to mark PR 2 Done and PR 3 Active. Record that PR 2 may @@ -104,27 +104,27 @@ Task: Implement only the functions required to pass the fixture-backed tests from the previous PR. Add: -- R/eda_spec.R +- R/epi_eda_spec.R - R/eda_schema.R - R/eda_missing.R Functions: -- eda_spec() -- validate_eda_spec() -- check_schema() -- profile_missing() +- epi_eda_spec() +- epi_eda_validate_spec() +- epi_eda_check_schema() +- epi_eda_profile_missing() Requirements: 1. Keep the API small. 2. Validate inputs clearly. 3. Return machine-readable tibbles or data frames. 4. Add roxygen2 documentation. -5. Do not implement synthetic data, summaries, plots, reports or run_eda() in this PR. +5. Do not implement synthetic data, summaries, plots, reports or epi_eda_run() in this PR. Closeout: Update START_HERE.md to mark PR 3 Done and PR 4 Active. Tests should no longer -fail because eda_spec(), validate_eda_spec(), check_schema() or -profile_missing() are missing. +fail because epi_eda_spec(), epi_eda_validate_spec(), epi_eda_check_schema() or +epi_eda_profile_missing() are missing. ``` ## Instruction 4: Add failing synthetic-data tests @@ -145,18 +145,18 @@ Requirements: 3. Synthetic values respect levels where provided. 4. Synthetic values respect min/max where provided. 5. Fixed seed gives identical output. -6. Do not implement generate_synthetic_data() in this PR. +6. Do not implement epi_eda_generate_synthetic_data() in this PR. Closeout: Update START_HERE.md to mark PR 4 Done and PR 5 Active. Record that PR 4 may -leave tests failing until PR 5 implements generate_synthetic_data(). +leave tests failing until PR 5 implements epi_eda_generate_synthetic_data(). ``` ## Instruction 5: Implement synthetic-data generation ```text Task: -Implement generate_synthetic_data() only. +Implement epi_eda_generate_synthetic_data() only. Add: - R/eda_synthetic.R @@ -166,7 +166,7 @@ Requirements: 2. Support numeric, integer, categorical, binary, date, datetime and text variables. 3. Use deterministic output with a fixed seed. 4. Do not add heavy dependencies. -5. Do not implement summaries, plots, reports or run_eda() in this PR. +5. Do not implement summaries, plots, reports or epi_eda_run() in this PR. Closeout: Update START_HERE.md to mark PR 5 Done and PR 6 Active. @@ -189,12 +189,12 @@ Optional expected outputs: Requirements: 1. Expected summary outputs must be independently computed. 2. Plot tests should check object classes and names, not visual appearance. -3. Do not implement profile_summaries() or profile_plots() in this PR. +3. Do not implement epi_eda_profile_summaries() or epi_eda_profile_plots() in this PR. Closeout: Update START_HERE.md to mark PR 6 Done and PR 7 Active. Record that PR 6 may -leave tests failing until PR 7 implements profile_summaries() and -profile_plots(). +leave tests failing until PR 7 implements epi_eda_profile_summaries() and +epi_eda_profile_plots(). ``` ## Instruction 7: Implement summaries and plots @@ -208,49 +208,49 @@ Add: - R/eda_plots.R Functions: -- profile_summaries() -- profile_plots() +- epi_eda_profile_summaries() +- epi_eda_profile_plots() Requirements: 1. Use the EDA specification to decide variable handling. 2. Use existing epi_stats_* helpers where suitable. 3. Return machine-readable outputs. 4. Return plot objects without printing them. -5. Do not implement run_eda() or reporting in this PR. +5. Do not implement epi_eda_run() or reporting in this PR. Closeout: Update START_HERE.md to mark PR 7 Done and PR 8 Active. ``` -## Instruction 8: Add failing run_eda tests +## Instruction 8: Add failing epi_eda_run tests ```text Task: -Add fixture-backed end-to-end tests for run_eda(). +Add fixture-backed end-to-end tests for epi_eda_run(). Add: -- tests/testthat/test-run_eda-fixtures.R +- tests/testthat/test-epi_eda_run-fixtures.R Requirements: 1. Test real fixture data workflow. 2. Test synthetic data workflow. 3. Use temporary directories for output-file tests. 4. Check expected named components: metadata, schema, missing, summaries, plots. -5. Do not implement run_eda() in this PR. +5. Do not implement epi_eda_run() in this PR. Closeout: Update START_HERE.md to mark PR 8 Done and PR 9 Active. Record that PR 8 may -leave tests failing until PR 9 implements run_eda(). +leave tests failing until PR 9 implements epi_eda_run(). ``` -## Instruction 9: Implement run_eda() +## Instruction 9: Implement epi_eda_run() ```text Task: -Implement run_eda(). +Implement epi_eda_run(). Add: -- R/run_eda.R +- R/epi_eda_run.R Requirements: 1. Accept data and spec. @@ -277,9 +277,9 @@ Continue with the revised PR plan. Next steps: 1. Add failing report-rendering tests. -2. Implement render_eda_report(). -3. Add failing project-template and use_episcout_project() contract tests. -4. Implement project template and use_episcout_project(). +2. Implement epi_eda_render_report(). +3. Add failing project-template and epi_eda_create_project() contract tests. +4. Implement project template and epi_eda_create_project(). 5. Add large-data design note. Follow spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md. @@ -293,17 +293,17 @@ dashboard. ```text Task: -Add failing tests for the project template and use_episcout_project() contract. +Add failing tests for the project template and epi_eda_create_project() contract. Add: - tests/testthat/test-project-template.R Requirements: 1. Tests define the bundled inst/project-template/ scaffold. -2. Tests define use_episcout_project(path, overwrite = FALSE). -3. Tests require reports/eda.qmd to culminate in render_eda_report(). +2. Tests define epi_eda_create_project(path, overwrite = FALSE). +3. Tests require reports/eda.qmd to culminate in epi_eda_render_report(). 4. Do not add inst/project-template/ in this PR. -5. Do not implement use_episcout_project() in this PR. +5. Do not implement epi_eda_create_project() in this PR. Closeout: Update START_HERE.md to mark PR 12 Done and PR 13 Active. Record that PR 12 @@ -314,20 +314,20 @@ may leave tests failing until PR 13 implements the scaffold and helper. ```text Task: -Implement the project template and use_episcout_project(). +Implement the project template and epi_eda_create_project(). Add: - inst/project-template/ -- R/use_episcout_project.R +- R/epi_eda_create_project.R Requirements: 1. Include metadata/data_dictionary.csv, config/eda.yml, _targets.R, reports/eda.qmd, R/project-derivations.R and outputs/. -2. use_episcout_project(path, overwrite = FALSE) creates the destination +2. epi_eda_create_project(path, overwrite = FALSE) creates the destination project and copies the scaffold. 3. Refuse to overwrite existing files unless overwrite = TRUE. 4. Return the normalized project path invisibly. -5. Keep the report path connected to render_eda_report(). +5. Keep the report path connected to epi_eda_render_report(). Closeout: Update START_HERE.md to mark PR 13 Done and PR 14 Active. diff --git a/spec_driven_EDA_plan/docs/repository-audit.md b/spec_driven_EDA_plan/docs/repository-audit.md index 92ef809..94da6d3 100644 --- a/spec_driven_EDA_plan/docs/repository-audit.md +++ b/spec_driven_EDA_plan/docs/repository-audit.md @@ -20,7 +20,7 @@ The package should not be rewritten from scratch. The preferred approach is to k - No specification-first workflow. - No data dictionary parser as central project contract. - No synthetic-data generator driven by expected variable metadata. -- No single high-level `run_eda()` orchestration function. +- No single high-level `epi_eda_run()` orchestration function. - No standard EDA report template. - No project template for reproducible analysis. - No large-data backend abstraction. @@ -38,15 +38,15 @@ Existing layer: - epi_utils_* New layer: -- eda_spec() -- validate_eda_spec() -- generate_synthetic_data() -- check_schema() -- profile_missing() -- profile_summaries() -- profile_plots() -- run_eda() -- render_eda_report() +- epi_eda_spec() +- epi_eda_validate_spec() +- epi_eda_generate_synthetic_data() +- epi_eda_check_schema() +- epi_eda_profile_missing() +- epi_eda_profile_summaries() +- epi_eda_profile_plots() +- epi_eda_run() +- epi_eda_render_report() ``` ## Immediate code review targets diff --git a/spec_driven_EDA_plan/docs/roadmap.md b/spec_driven_EDA_plan/docs/roadmap.md index 4c79213..a927514 100644 --- a/spec_driven_EDA_plan/docs/roadmap.md +++ b/spec_driven_EDA_plan/docs/roadmap.md @@ -60,28 +60,28 @@ No new EDA API should be implemented before the first fixture-backed tests are i New files: ```text -R/eda_spec.R +R/epi_eda_spec.R R/eda_schema.R R/eda_synthetic.R R/eda_missing.R R/eda_summaries.R R/eda_plots.R R/eda_report.R -R/run_eda.R +R/epi_eda_run.R ``` Initial public functions: ```r -eda_spec() -validate_eda_spec() -generate_synthetic_data() -check_schema() -profile_missing() -profile_summaries() -profile_plots() -run_eda() -render_eda_report() +epi_eda_spec() +epi_eda_validate_spec() +epi_eda_generate_synthetic_data() +epi_eda_check_schema() +epi_eda_profile_missing() +epi_eda_profile_summaries() +epi_eda_profile_plots() +epi_eda_run() +epi_eda_render_report() ``` ## Phase 4: Reporting and project template diff --git a/spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md b/spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md index b67fd5d..95aad71 100644 --- a/spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md +++ b/spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md @@ -77,10 +77,10 @@ The same specification should be used to: ### Workflow A: real data already available ```r -spec <- eda_spec("metadata/data_dictionary.csv") +spec <- epi_eda_spec("metadata/data_dictionary.csv") data <- read_project_data("data/raw/data.parquet") -run_eda( +epi_eda_run( data = data, spec = spec, out_dir = "outputs/eda_real", @@ -101,10 +101,10 @@ Expected outputs: ### Workflow B: data access pending ```r -spec <- eda_spec("metadata/data_dictionary.csv") -data <- generate_synthetic_data(spec, n = 10000, seed = 1) +spec <- epi_eda_spec("metadata/data_dictionary.csv") +data <- epi_eda_generate_synthetic_data(spec, n = 10000, seed = 1) -run_eda( +epi_eda_run( data = data, spec = spec, out_dir = "outputs/eda_synthetic", @@ -157,15 +157,15 @@ The package should not immediately optimise every operation for very large data. Initial public functions: ```r -eda_spec() -validate_eda_spec() -generate_synthetic_data() -check_schema() -profile_missing() -profile_summaries() -profile_plots() -run_eda() -render_eda_report() +epi_eda_spec() +epi_eda_validate_spec() +epi_eda_generate_synthetic_data() +epi_eda_check_schema() +epi_eda_profile_missing() +epi_eda_profile_summaries() +epi_eda_profile_plots() +epi_eda_run() +epi_eda_render_report() ``` Existing `epi_*` functions should remain available initially. Over time, they may be refactored into lower-level helpers if this can be done without breaking existing users. diff --git a/spec_driven_EDA_plan/docs/sdd/data-dictionary-spec.md b/spec_driven_EDA_plan/docs/sdd/data-dictionary-spec.md index 27cff4e..e5cd98a 100644 --- a/spec_driven_EDA_plan/docs/sdd/data-dictionary-spec.md +++ b/spec_driven_EDA_plan/docs/sdd/data-dictionary-spec.md @@ -69,7 +69,7 @@ death,Death during follow-up,binary,outcome,,"0;1",,,TRUE,outcomes ## Validation rules -`validate_eda_spec()` should check: +`epi_eda_validate_spec()` should check: - required columns exist; - variable names are non-empty; @@ -100,7 +100,7 @@ These should be standardised to `NA` during EDA, with a record of what was chang ## Synthetic-data use -The data dictionary should drive `generate_synthetic_data()`. +The data dictionary should drive `epi_eda_generate_synthetic_data()`. For MVP: diff --git a/spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md b/spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md index a20d5e7..b936ce2 100644 --- a/spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md +++ b/spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md @@ -176,7 +176,7 @@ Expected values must be independently computed. Bad: ```r -expected_missing <- profile_missing(data, spec) +expected_missing <- epi_eda_profile_missing(data, spec) ``` Good: @@ -199,7 +199,7 @@ This avoids circular tests. Test file: ```text -tests/testthat/test-eda_spec-fixtures.R +tests/testthat/test-epi_eda_spec-fixtures.R ``` Tests: @@ -220,7 +220,7 @@ tests/testthat/test-eda_schema-fixtures.R Tests: -- `check_schema(blood_storage, blood_storage_spec)` matches `expected_schema.csv`; +- `epi_eda_check_schema(blood_storage, blood_storage_spec)` matches `expected_schema.csv`; - missing variable is flagged; - unexpected variable is flagged; - all expected variables are present in the unmodified fixture. @@ -255,7 +255,7 @@ Tests: - synthetic values respect levels where provided; - synthetic values respect min/max where provided; - fixed seed gives identical output; -- synthetic data can be passed into `run_eda()` once available. +- synthetic data can be passed into `epi_eda_run()` once available. ### Summary tests @@ -292,18 +292,18 @@ Tests: Test file: ```text -tests/testthat/test-run_eda-fixtures.R +tests/testthat/test-epi_eda_run-fixtures.R ``` Tests: -- `run_eda()` returns a named list; +- `epi_eda_run()` returns a named list; - required components exist: metadata, schema, missing, summaries, plots; -- `run_eda()` works on real fixture data; -- `run_eda()` works on synthetic data generated from the same spec; +- `epi_eda_run()` works on real fixture data; +- `epi_eda_run()` works on synthetic data generated from the same spec; - output files are created in a temporary directory. -## Expected named output from `run_eda()` +## Expected named output from `epi_eda_run()` The first stable contract should be: @@ -376,14 +376,14 @@ Recommended order: 2. Add fixture generation script. 3. Commit blood_storage CSV fixture and spec. 4. Commit independently computed expected schema and missingness outputs. -5. Add failing tests for eda_spec(), check_schema(), profile_missing(). +5. Add failing tests for epi_eda_spec(), epi_eda_check_schema(), epi_eda_profile_missing(). 6. Implement or adjust functions until tests pass. 7. Add failing synthetic-data tests. -8. Implement generate_synthetic_data(). +8. Implement epi_eda_generate_synthetic_data(). 9. Add failing summary and plot tests. -10. Implement profile_summaries() and profile_plots(). -11. Add failing run_eda() fixture tests. -12. Implement run_eda(). +10. Implement epi_eda_profile_summaries() and epi_eda_profile_plots(). +11. Add failing epi_eda_run() fixture tests. +12. Implement epi_eda_run(). 13. Add scurvy fixture later for categorical and small-n testing. 14. Add covid_testing later for larger-data smoke testing. ``` diff --git a/spec_driven_EDA_plan/future/codex-pr-add-penguins-raw-fixture.md b/spec_driven_EDA_plan/future/codex-pr-add-penguins-raw-fixture.md index 3b33285..dc88ec2 100644 --- a/spec_driven_EDA_plan/future/codex-pr-add-penguins-raw-fixture.md +++ b/spec_driven_EDA_plan/future/codex-pr-add-penguins-raw-fixture.md @@ -98,12 +98,12 @@ Only create this test if it is lightweight and does not run fixture-regeneration Do not edit implementation files for this PR: ```text -R/eda_spec.R +R/epi_eda_spec.R R/eda_schema.R R/eda_missing.R R/eda_summaries.R R/eda_plots.R -R/run_eda.R +R/epi_eda_run.R ``` Do not add or modify package behaviour. @@ -151,14 +151,14 @@ The script must not call any of these: ```text library(episcout) episcout:: -eda_spec( -validate_eda_spec( -check_schema( -profile_missing( -profile_summaries( -profile_plots( -run_eda( -generate_synthetic_data( +epi_eda_spec( +epi_eda_validate_spec( +epi_eda_check_schema( +epi_eda_profile_missing( +epi_eda_profile_summaries( +epi_eda_profile_plots( +epi_eda_run( +epi_eda_generate_synthetic_data( ``` ## Expected outputs @@ -386,14 +386,14 @@ test_that("fixture regeneration scripts do not call episcout EDA functions", { forbidden <- c( "library\\(episcout\\)", "episcout::", - "eda_spec\\(", - "validate_eda_spec\\(", - "check_schema\\(", - "profile_missing\\(", - "profile_summaries\\(", - "profile_plots\\(", - "run_eda\\(", - "generate_synthetic_data\\(" + "epi_eda_spec\\(", + "epi_eda_validate_spec\\(", + "epi_eda_check_schema\\(", + "epi_eda_profile_missing\\(", + "epi_eda_profile_summaries\\(", + "epi_eda_profile_plots\\(", + "epi_eda_run\\(", + "epi_eda_generate_synthetic_data\\(" ) for (pattern in forbidden) { diff --git a/spec_driven_EDA_plan/future/codex-pr-deferred-phase-1-stabilization.md b/spec_driven_EDA_plan/future/codex-pr-deferred-phase-1-stabilization.md index 57a29b4..590c963 100644 --- a/spec_driven_EDA_plan/future/codex-pr-deferred-phase-1-stabilization.md +++ b/spec_driven_EDA_plan/future/codex-pr-deferred-phase-1-stabilization.md @@ -36,9 +36,9 @@ user explicitly requests that. Use this instruction when one of these triggers is present: -- `profile_summaries()` depends on `epi_stats_numeric()` or +- `epi_eda_profile_summaries()` depends on `epi_stats_numeric()` or `epi_stats_summary()`. -- `run_eda()` depends on older `epi_*` helpers. +- `epi_eda_run()` depends on older `epi_*` helpers. - CI, `R CMD check`, or local tests expose helper failures. - A release or broader package hardening pass is being prepared. - The user explicitly asks to activate deferred Phase 1 stabilization. @@ -124,13 +124,13 @@ Do not edit current spec-first implementation files unless a failing test proves that the deferred hardening PR directly requires it: ```text -R/eda_spec.R +R/epi_eda_spec.R R/eda_schema.R R/eda_missing.R R/eda_synthetic.R R/eda_summaries.R R/eda_plots.R -R/run_eda.R +R/epi_eda_run.R R/eda_report.R ``` diff --git a/spec_driven_EDA_plan/future/codex-review-checklist-external-truth-fixtures.md b/spec_driven_EDA_plan/future/codex-review-checklist-external-truth-fixtures.md index 207b76e..6eeec9a 100644 --- a/spec_driven_EDA_plan/future/codex-review-checklist-external-truth-fixtures.md +++ b/spec_driven_EDA_plan/future/codex-review-checklist-external-truth-fixtures.md @@ -74,14 +74,14 @@ Forbidden calls in fixture-generation scripts: ```text library(episcout) episcout:: -eda_spec( -validate_eda_spec( -check_schema( -profile_missing( -profile_summaries( -profile_plots( -run_eda( -generate_synthetic_data( +epi_eda_spec( +epi_eda_validate_spec( +epi_eda_check_schema( +epi_eda_profile_missing( +epi_eda_profile_summaries( +epi_eda_profile_plots( +epi_eda_run( +epi_eda_generate_synthetic_data( ``` ## Dataset roles diff --git a/spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md b/spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md index 8654d11..2528fda 100644 --- a/spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md +++ b/spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md @@ -69,9 +69,9 @@ depends on them or when package checks expose concrete failures. Reopen this deferred Phase 1 work when any of the following is true: -- `profile_summaries()` depends on `epi_stats_numeric()` or +- `epi_eda_profile_summaries()` depends on `epi_stats_numeric()` or `epi_stats_summary()`. -- `run_eda()` depends on older `epi_*` helpers. +- `epi_eda_run()` depends on older `epi_*` helpers. - CI, `R CMD check`, or local tests expose helper failures. - The package is being prepared for release or broader hardening. - The user explicitly asks to run the deferred Phase 1 stabilization PR. diff --git a/spec_driven_EDA_plan/prompt_loop.md b/spec_driven_EDA_plan/prompt_loop.md index 84ebe6d..e03b1dc 100644 --- a/spec_driven_EDA_plan/prompt_loop.md +++ b/spec_driven_EDA_plan/prompt_loop.md @@ -28,7 +28,7 @@ The mapping is hierarchical, not one-to-one: | Phase 0: Documentation and baseline | Done. This produced SDD, ADRs, roadmap, Codex instructions, and PR plan. Tracked in `START_HERE.md` as done. | | Phase 1: Stabilise current package | Not clearly mapped to PRs. It has audit material, but no active PR sequence or completion evidence. Looks bypassed/deferred. | | Phase 2: TDD fixtures for spec-first EDA | PR 1 and PR 2. PR 1 adds external fixtures and expected outputs. PR 2 adds failing tests for spec/schema/missingness. | -| Phase 3: Specification-first MVP | PR 3 through PR 9, but in test-first slices: implement spec/schema/missingness, test synthetic data, implement synthetic data, test summaries/plots, implement summaries/plots, test `run_eda()`, implement `run_eda()`. Current active PR is PR 4. | +| Phase 3: Specification-first MVP | PR 3 through PR 9, but in test-first slices: implement spec/schema/missingness, test synthetic data, implement synthetic data, test summaries/plots, implement summaries/plots, test `epi_eda_run()`, implement `epi_eda_run()`. Current active PR is PR 4. | | Phase 4: Reporting and project template | PR 10, PR 11, PR 12. Report tests, report implementation, project template. | | Phase 5: Large-data optimisation | PR 13 only documents large-data strategy. Actual `data.table`, DuckDB, Arrow implementation is explicitly not planned yet. | | Phase 6: Biomedical extensions | No PRs/instructions yet. It is future scope only. | diff --git a/tests/testthat/test-eda-api.R b/tests/testthat/test-eda-api.R new file mode 100644 index 0000000..b085636 --- /dev/null +++ b/tests/testthat/test-eda-api.R @@ -0,0 +1,73 @@ +context("EDA API naming tests") + +library(testthat) +library(episcout) + +find_package_root <- function() { + candidates <- c( + normalizePath(getwd(), winslash = "/", mustWork = TRUE), + normalizePath(file.path(getwd(), ".."), winslash = "/", mustWork = FALSE), + normalizePath(file.path(getwd(), "..", ".."), winslash = "/", mustWork = FALSE) + ) + for (candidate in candidates) { + if (file.exists(file.path(candidate, "DESCRIPTION"))) { + return(candidate) + } + } + NA_character_ +} + +read_if_exists <- function(path) { + if (!file.exists(path)) { + return(NULL) + } + paste(readLines(path, warn = FALSE), collapse = "\n") +} + +test_that("EDA workflow exports use the epi_eda prefix", { + exports <- getNamespaceExports("episcout") + expected <- c( + "epi_eda_spec", + "epi_eda_validate_spec", + "epi_eda_check_schema", + "epi_eda_profile_missing", + "epi_eda_profile_summaries", + "epi_eda_profile_plots", + "epi_eda_generate_synthetic_data", + "epi_eda_run", + "epi_eda_render_report", + "epi_eda_create_project" + ) + old <- c( + "eda_spec", + "validate_eda_spec", + "check_schema", + "profile_missing", + "profile_summaries", + "profile_plots", + "generate_synthetic_data", + "run_eda", + "render_eda_report", + "use_episcout_project" + ) + + expect_true(all(expected %in% exports)) + expect_false(any(old %in% exports)) +}) + +test_that("README and EDA vignette document the epi_eda workflow", { + root <- find_package_root() + skip_if(is.na(root), "Package root not available.") + + readme <- read_if_exists(file.path(root, "README.md")) + vignette <- read_if_exists(file.path(root, "vignettes", "specification-first-eda.Rmd")) + skip_if(is.null(readme), "README.md not available.") + skip_if(is.null(vignette), "specification-first-eda vignette not available.") + + expect_match(readme, "epi_eda_run") + expect_match(readme, "epi_eda_render_report") + expect_match(readme, "specification-first exploratory data\\s+analysis workflows") + expect_match(vignette, "epi_eda_spec") + expect_match(vignette, "epi_eda_run") + expect_match(vignette, "epi_eda_render_report") +}) diff --git a/tests/testthat/test-eda_missing-fixtures.R b/tests/testthat/test-eda_missing-fixtures.R index 5aa1386..2f625bf 100644 --- a/tests/testthat/test-eda_missing-fixtures.R +++ b/tests/testthat/test-eda_missing-fixtures.R @@ -8,12 +8,12 @@ data_path <- file.path(fixture_dir, "blood_storage.csv") spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") expected_missing_path <- file.path(fixture_dir, "expected_missing.csv") -test_that("profile_missing matches independently computed expected missingness", { +test_that("epi_eda_profile_missing matches independently computed expected missingness", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) expected <- read.csv(expected_missing_path, stringsAsFactors = FALSE) - observed <- profile_missing(data, spec) + observed <- epi_eda_profile_missing(data, spec) expect_equal( as.data.frame(observed), @@ -23,32 +23,32 @@ test_that("profile_missing matches independently computed expected missingness", ) }) -test_that("profile_missing reports the correct denominator for every variable", { +test_that("epi_eda_profile_missing reports the correct denominator for every variable", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) - observed <- profile_missing(data, spec) + observed <- epi_eda_profile_missing(data, spec) expect_true(all(observed$n == nrow(data))) }) -test_that("profile_missing percentages are stable with tolerance", { +test_that("epi_eda_profile_missing percentages are stable with tolerance", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) expected <- read.csv(expected_missing_path, stringsAsFactors = FALSE) - observed <- profile_missing(data, spec) + observed <- epi_eda_profile_missing(data, spec) expect_equal(observed$p_missing, expected$p_missing, tolerance = 1e-12) }) -test_that("profile_missing reports zero missingness for complete variables", { +test_that("epi_eda_profile_missing reports zero missingness for complete variables", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) expected <- read.csv(expected_missing_path, stringsAsFactors = FALSE) complete_names <- expected$name[expected$n_missing == 0] - observed <- profile_missing(data, spec) + observed <- epi_eda_profile_missing(data, spec) complete_rows <- observed[observed$name %in% complete_names, ] expect_true(all(complete_rows$n_missing == 0)) diff --git a/tests/testthat/test-eda_plots-fixtures.R b/tests/testthat/test-eda_plots-fixtures.R index a893af4..a4cf05d 100644 --- a/tests/testthat/test-eda_plots-fixtures.R +++ b/tests/testthat/test-eda_plots-fixtures.R @@ -7,35 +7,35 @@ fixture_dir <- file.path("fixtures", "blood_storage") data_path <- file.path(fixture_dir, "blood_storage.csv") spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") -test_that("profile_plots returns one named plot object per specified variable", { +test_that("epi_eda_profile_plots returns one named plot object per specified variable", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) - observed <- profile_plots(data, spec) + observed <- epi_eda_profile_plots(data, spec) expect_type(observed, "list") expect_named(observed, spec$name) expect_equal(length(observed), nrow(spec)) }) -test_that("profile_plots returns ggplot objects without checking visual appearance", { +test_that("epi_eda_profile_plots returns ggplot objects without checking visual appearance", { skip_if_not_installed("ggplot2") data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) - observed <- profile_plots(data, spec) + observed <- epi_eda_profile_plots(data, spec) expect_true(all(vapply(observed, inherits, logical(1), what = "ggplot"))) }) -test_that("profile_plots dispatches numeric and categorical fixture variables", { +test_that("epi_eda_profile_plots dispatches numeric and categorical fixture variables", { skip_if_not_installed("ggplot2") data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) - observed <- profile_plots(data, spec) + observed <- epi_eda_profile_plots(data, spec) expect_s3_class(observed$Age, "ggplot") expect_s3_class(observed$Units, "ggplot") diff --git a/tests/testthat/test-eda_report.R b/tests/testthat/test-eda_report.R index d160b95..3d0bcb9 100644 --- a/tests/testthat/test-eda_report.R +++ b/tests/testthat/test-eda_report.R @@ -37,13 +37,13 @@ test_that("EDA report template is bundled with report sections", { expect_match(template_text, "plot|Plot") }) -test_that("render_eda_report renders a real fixture-data report", { +test_that("epi_eda_render_report renders a real fixture-data report", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) output_dir <- tempfile("eda-report-real-") dir.create(output_dir) - report_path <- render_eda_report( + report_path <- epi_eda_render_report( data = data, spec = spec, output_dir = output_dir @@ -59,12 +59,12 @@ test_that("render_eda_report renders a real fixture-data report", { expect_match(report_text, "real|Real") }) -test_that("render_eda_report labels synthetic fixture reports", { - spec <- eda_spec(spec_path) +test_that("epi_eda_render_report labels synthetic fixture reports", { + spec <- epi_eda_spec(spec_path) output_dir <- tempfile("eda-report-synthetic-") dir.create(output_dir) - report_path <- render_eda_report( + report_path <- epi_eda_render_report( data = NULL, spec = spec, output_dir = output_dir, @@ -77,13 +77,13 @@ test_that("render_eda_report labels synthetic fixture reports", { expect_match(read_report_text(report_path), "synthetic|Synthetic") }) -test_that("render_eda_report writes machine-readable workflow outputs", { +test_that("epi_eda_render_report writes machine-readable workflow outputs", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) output_dir <- tempfile("eda-report-outputs-") dir.create(output_dir) - render_eda_report( + epi_eda_render_report( data = data, spec = spec, output_dir = output_dir @@ -96,13 +96,13 @@ test_that("render_eda_report writes machine-readable workflow outputs", { expect_true(file.exists(file.path(output_dir, "summary_categorical.csv"))) }) -test_that("render_eda_report requires an existing output directory", { +test_that("epi_eda_render_report requires an existing output directory", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) output_dir <- tempfile("eda-report-missing-") expect_error( - render_eda_report( + epi_eda_render_report( data = data, spec = spec, output_dir = output_dir diff --git a/tests/testthat/test-eda_schema-fixtures.R b/tests/testthat/test-eda_schema-fixtures.R index bfc5e59..38c0f41 100644 --- a/tests/testthat/test-eda_schema-fixtures.R +++ b/tests/testthat/test-eda_schema-fixtures.R @@ -8,22 +8,22 @@ data_path <- file.path(fixture_dir, "blood_storage.csv") spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") expected_schema_path <- file.path(fixture_dir, "expected_schema.csv") -test_that("check_schema matches independently computed expected schema", { +test_that("epi_eda_check_schema matches independently computed expected schema", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) expected <- read.csv(expected_schema_path, stringsAsFactors = FALSE) - observed <- check_schema(data, spec) + observed <- epi_eda_check_schema(data, spec) expect_equal(as.data.frame(observed), expected, ignore_attr = TRUE) }) -test_that("check_schema flags missing expected variables", { +test_that("epi_eda_check_schema flags missing expected variables", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) data$Age <- NULL - observed <- check_schema(data, spec) + observed <- epi_eda_check_schema(data, spec) age_row <- observed[observed$name == "Age", ] expect_equal(nrow(age_row), 1) @@ -32,12 +32,12 @@ test_that("check_schema flags missing expected variables", { expect_equal(age_row$status, "missing") }) -test_that("check_schema flags unexpected observed variables", { +test_that("epi_eda_check_schema flags unexpected observed variables", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) data$UnexpectedVariable <- seq_len(nrow(data)) - observed <- check_schema(data, spec) + observed <- epi_eda_check_schema(data, spec) unexpected_row <- observed[observed$name == "UnexpectedVariable", ] expect_equal(nrow(unexpected_row), 1) @@ -48,9 +48,9 @@ test_that("check_schema flags unexpected observed variables", { test_that("all expected variables are present in the unmodified fixture", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) - observed <- check_schema(data, spec) + observed <- epi_eda_check_schema(data, spec) expected_rows <- observed[observed$expected_present, ] expect_true(all(expected_rows$observed_present)) diff --git a/tests/testthat/test-eda_spec-fixtures.R b/tests/testthat/test-eda_spec-fixtures.R index 9b0fc89..1480804 100644 --- a/tests/testthat/test-eda_spec-fixtures.R +++ b/tests/testthat/test-eda_spec-fixtures.R @@ -9,7 +9,7 @@ spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") test_that("blood_storage_spec.csv loads as an EDA specification", { raw_spec <- read.csv(spec_path, check.names = FALSE, stringsAsFactors = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) expect_s3_class(spec, "data.frame") expect_equal(spec$name, raw_spec$name) @@ -17,13 +17,13 @@ test_that("blood_storage_spec.csv loads as an EDA specification", { }) test_that("blood_storage_spec.csv has required specification columns", { - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) expect_true(all(c("name", "label", "type", "role") %in% names(spec))) }) test_that("blood_storage specification variable names are unique", { - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) expect_equal(anyDuplicated(spec$name), 0L) }) @@ -33,7 +33,7 @@ test_that("invalid specification type fails clearly", { spec$type[1] <- "unsupported_type" expect_error( - validate_eda_spec(spec), + epi_eda_validate_spec(spec), regexp = "[Tt]ype|[Uu]nsupported|[Ii]nvalid" ) }) @@ -43,7 +43,7 @@ test_that("duplicate specification variable name fails clearly", { spec$name[2] <- spec$name[1] expect_error( - validate_eda_spec(spec), + epi_eda_validate_spec(spec), regexp = "[Dd]uplicate|[Uu]nique|[Nn]ame" ) }) diff --git a/tests/testthat/test-eda_summaries-fixtures.R b/tests/testthat/test-eda_summaries-fixtures.R index 86674f0..9bef13d 100644 --- a/tests/testthat/test-eda_summaries-fixtures.R +++ b/tests/testthat/test-eda_summaries-fixtures.R @@ -49,11 +49,11 @@ make_expected_categorical_summary <- function(data, spec) { do.call(rbind, rows) } -test_that("profile_summaries returns numeric and categorical summary components", { +test_that("epi_eda_profile_summaries returns numeric and categorical summary components", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) - observed <- profile_summaries(data, spec) + observed <- epi_eda_profile_summaries(data, spec) expect_type(observed, "list") expect_named(observed, c("numeric", "categorical")) @@ -61,12 +61,12 @@ test_that("profile_summaries returns numeric and categorical summary components" expect_s3_class(observed$categorical, "data.frame") }) -test_that("profile_summaries numeric output matches independent fixture calculations", { +test_that("epi_eda_profile_summaries numeric output matches independent fixture calculations", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) expected <- make_expected_numeric_summary(data, spec) - observed <- profile_summaries(data, spec) + observed <- epi_eda_profile_summaries(data, spec) expect_equal( as.data.frame(observed$numeric), @@ -76,12 +76,12 @@ test_that("profile_summaries numeric output matches independent fixture calculat ) }) -test_that("profile_summaries categorical output matches independent fixture calculations", { +test_that("epi_eda_profile_summaries categorical output matches independent fixture calculations", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) expected <- make_expected_categorical_summary(data, spec) - observed <- profile_summaries(data, spec) + observed <- epi_eda_profile_summaries(data, spec) expect_equal( as.data.frame(observed$categorical), diff --git a/tests/testthat/test-eda_synthetic-fixtures.R b/tests/testthat/test-eda_synthetic-fixtures.R index a1bca77..7052794 100644 --- a/tests/testthat/test-eda_synthetic-fixtures.R +++ b/tests/testthat/test-eda_synthetic-fixtures.R @@ -6,27 +6,27 @@ library(episcout) fixture_dir <- file.path("fixtures", "blood_storage") spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") -spec <- eda_spec(spec_path) +spec <- epi_eda_spec(spec_path) split_levels <- function(x) { strsplit(x, ";", fixed = TRUE)[[1]] } test_that("synthetic data has the same variable names as the specification", { - synthetic <- generate_synthetic_data(spec, n = 25, seed = 1) + synthetic <- epi_eda_generate_synthetic_data(spec, n = 25, seed = 1) expect_s3_class(synthetic, "data.frame") expect_identical(names(synthetic), spec$name) }) test_that("synthetic data has the requested row count", { - synthetic <- generate_synthetic_data(spec, n = 37, seed = 1) + synthetic <- epi_eda_generate_synthetic_data(spec, n = 37, seed = 1) expect_equal(nrow(synthetic), 37L) }) test_that("synthetic categorical and binary values respect specification levels", { - synthetic <- generate_synthetic_data(spec, n = 100, seed = 1) + synthetic <- epi_eda_generate_synthetic_data(spec, n = 100, seed = 1) categorical_spec <- spec[spec$type %in% c("categorical", "binary") & !is.na(spec$levels) & spec$levels != "", , drop = FALSE] @@ -44,7 +44,7 @@ test_that("synthetic categorical and binary values respect specification levels" }) test_that("synthetic numeric and integer values respect specification min and max", { - synthetic <- generate_synthetic_data(spec, n = 100, seed = 1) + synthetic <- epi_eda_generate_synthetic_data(spec, n = 100, seed = 1) ranged_spec <- spec[spec$type %in% c("numeric", "integer") & !is.na(spec$min) & spec$min != "" & @@ -64,8 +64,8 @@ test_that("synthetic numeric and integer values respect specification min and ma }) test_that("fixed seed gives identical synthetic output", { - synthetic_a <- generate_synthetic_data(spec, n = 25, seed = 2024) - synthetic_b <- generate_synthetic_data(spec, n = 25, seed = 2024) + synthetic_a <- epi_eda_generate_synthetic_data(spec, n = 25, seed = 2024) + synthetic_b <- epi_eda_generate_synthetic_data(spec, n = 25, seed = 2024) expect_identical(synthetic_a, synthetic_b) }) diff --git a/tests/testthat/test-project-template.R b/tests/testthat/test-project-template.R index f150f5a..dd4953a 100644 --- a/tests/testthat/test-project-template.R +++ b/tests/testthat/test-project-template.R @@ -4,12 +4,14 @@ library(testthat) library(episcout) expected_project_template_entries <- c( + "README.md", "metadata/data_dictionary.csv", "config/eda.yml", + "data/.gitkeep", "_targets.R", "reports/eda.qmd", "R/project-derivations.R", - "outputs" + "outputs/.gitkeep" ) expect_project_template_entries <- function(root) { @@ -28,6 +30,13 @@ read_project_report_template <- function(root) { ) } +read_project_targets <- function(root) { + paste( + readLines(file.path(root, "_targets.R"), warn = FALSE), + collapse = "\n" + ) +} + test_that("project template is bundled with the expected scaffold", { template_path <- system.file("project-template", package = "episcout") @@ -38,42 +47,46 @@ test_that("project template is bundled with the expected scaffold", { expect_true(dir.exists(template_path)) expect_project_template_entries(template_path) - expect_match(read_project_report_template(template_path), "render_eda_report") + expect_match(read_project_report_template(template_path), "epi_eda_render_report") + expect_match(read_project_targets(template_path), "epi_eda_render_report") + expect_no_match(read_project_report_template(template_path), "\\brender_eda_report\\b") + expect_no_match(read_project_targets(template_path), "\\brender_eda_report\\b") }) -test_that("use_episcout_project creates the expected scaffold", { +test_that("epi_eda_create_project creates the expected scaffold", { project_path <- tempfile("episcout-project-") - returned_path <- use_episcout_project(project_path) + returned_path <- epi_eda_create_project(project_path) expect_equal( normalizePath(returned_path, winslash = "/", mustWork = TRUE), normalizePath(project_path, winslash = "/", mustWork = TRUE) ) expect_project_template_entries(project_path) - expect_match(read_project_report_template(project_path), "render_eda_report") + expect_match(read_project_report_template(project_path), "epi_eda_render_report") + expect_match(read_project_targets(project_path), "epi_eda_render_report") }) -test_that("use_episcout_project refuses to overwrite existing files by default", { +test_that("epi_eda_create_project refuses to overwrite existing files by default", { project_path <- tempfile("episcout-project-existing-") dir.create(file.path(project_path, "metadata"), recursive = TRUE) existing_file <- file.path(project_path, "metadata", "data_dictionary.csv") writeLines("existing", existing_file) expect_error( - use_episcout_project(project_path), + epi_eda_create_project(project_path), regexp = "overwrite|exist" ) expect_equal(readLines(existing_file, warn = FALSE), "existing") }) -test_that("use_episcout_project can overwrite existing files when requested", { +test_that("epi_eda_create_project can overwrite existing files when requested", { project_path <- tempfile("episcout-project-overwrite-") dir.create(file.path(project_path, "metadata"), recursive = TRUE) existing_file <- file.path(project_path, "metadata", "data_dictionary.csv") writeLines("existing", existing_file) - returned_path <- use_episcout_project(project_path, overwrite = TRUE) + returned_path <- epi_eda_create_project(project_path, overwrite = TRUE) expect_equal( normalizePath(returned_path, winslash = "/", mustWork = TRUE), diff --git a/tests/testthat/test-run_eda-fixtures.R b/tests/testthat/test-run_eda-fixtures.R index a8b4bcf..4f2f99f 100644 --- a/tests/testthat/test-run_eda-fixtures.R +++ b/tests/testthat/test-run_eda-fixtures.R @@ -1,4 +1,4 @@ -context("fixture-backed run_eda workflow tests") +context("fixture-backed epi_eda_run workflow tests") library(testthat) library(episcout) @@ -9,11 +9,11 @@ spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") expected_components <- c("metadata", "schema", "missing", "summaries", "plots") -test_that("run_eda returns expected components for real fixture data", { +test_that("epi_eda_run returns expected components for real fixture data", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) - observed <- run_eda(data = data, spec = spec) + observed <- epi_eda_run(data = data, spec = spec) expect_type(observed, "list") expect_named(observed, expected_components) @@ -24,27 +24,27 @@ test_that("run_eda returns expected components for real fixture data", { expect_type(observed$plots, "list") }) -test_that("run_eda real-data workflow matches component functions", { +test_that("epi_eda_run real-data workflow matches component functions", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) - observed <- run_eda(data = data, spec = spec) + observed <- epi_eda_run(data = data, spec = spec) - expect_equal(observed$schema, check_schema(data, spec), ignore_attr = TRUE) - expect_equal(observed$missing, profile_missing(data, spec), ignore_attr = TRUE) + expect_equal(observed$schema, epi_eda_check_schema(data, spec), ignore_attr = TRUE) + expect_equal(observed$missing, epi_eda_profile_missing(data, spec), ignore_attr = TRUE) expect_equal( observed$summaries, - profile_summaries(data, spec), + epi_eda_profile_summaries(data, spec), tolerance = 1e-12, ignore_attr = TRUE ) expect_named(observed$plots, spec$name) }) -test_that("run_eda supports synthetic fixture workflow", { - spec <- eda_spec(spec_path) +test_that("epi_eda_run supports synthetic fixture workflow", { + spec <- epi_eda_spec(spec_path) - observed <- run_eda( + observed <- epi_eda_run( data = NULL, spec = spec, synthetic = TRUE, @@ -60,13 +60,13 @@ test_that("run_eda supports synthetic fixture workflow", { expect_named(observed$plots, spec$name) }) -test_that("run_eda writes workflow outputs to a temporary output directory", { +test_that("epi_eda_run writes workflow outputs to a temporary output directory", { data <- read.csv(data_path, check.names = FALSE) - spec <- eda_spec(spec_path) + spec <- epi_eda_spec(spec_path) output_dir <- tempfile("run-eda-fixture-") dir.create(output_dir) - observed <- run_eda(data = data, spec = spec, output_dir = output_dir) + observed <- epi_eda_run(data = data, spec = spec, output_dir = output_dir) expect_true(dir.exists(output_dir)) expect_true(file.exists(file.path(output_dir, "metadata.csv"))) diff --git a/vignettes/introduction_episcout.Rmd b/vignettes/introduction_episcout.Rmd index f2280c8..475a9b7 100644 --- a/vignettes/introduction_episcout.Rmd +++ b/vignettes/introduction_episcout.Rmd @@ -55,9 +55,9 @@ install.packages(c( Many parameters are given as defaults, admittedly with quite a bit of personal preference, but with the aim of processing hundreds of thousands of observations from hundreds of variables from multiple data sets a bit faster. Hence, convenience and standardisation are often preferred. This may not be your case however but it is simple enough to defer to your preferred R packages. -Below are a number of examples of how to use episcout functions. - -All functions start with "epi\_". +Below are a number of examples of how to use lower-level episcout helper +functions. For a complete specification-first EDA workflow, see the +`specification-first-eda` vignette. ```{r library, message=FALSE} library(episcout) @@ -68,6 +68,7 @@ Currently there are functions for: - pre-processing: epi\_clean\_\* - descriptive statistics: epi\_stats\_\* - visualising: epi\_plot\_\* +- specification-first EDA workflows: epi\_eda\_\* - various: epi\_read() ; epi\_write() etc. ## A few examples with dummy data @@ -89,8 +90,7 @@ df `epi_clean_get_dups()` identifies duplicate rows based on a chosen column. ```{r} -epi_clean_get_dups(df, "id", 1) -epi_clean_get_dups(df, "id", 1) +epi_clean_get_dups(df, "var_id", 1) ``` ## Numeric summaries @@ -111,4 +111,3 @@ epi_plot_hist(df, "x") ``` This vignette only scratches the surface of what `episcout` can do. For more details see the function documentation. - diff --git a/vignettes/specification-first-eda.Rmd b/vignettes/specification-first-eda.Rmd new file mode 100644 index 0000000..7bcbcd1 --- /dev/null +++ b/vignettes/specification-first-eda.Rmd @@ -0,0 +1,185 @@ +--- +title: "Specification-first exploratory data analysis" +author: "Antonio Berlanga-Taylor" +date: "`r Sys.Date()`" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Specification-first exploratory data analysis} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r setup, include = FALSE} +has_ggplot2 <- requireNamespace("ggplot2", quietly = TRUE) +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>", + eval = has_ggplot2 +) +``` + +```{r eval = !has_ggplot2, echo = FALSE, results = "asis"} +cat("This vignette needs ggplot2 to run the plotting step. Install ggplot2 to run the examples.") +``` + +Specification-first EDA starts with a data dictionary rather than with ad hoc +analysis code. The same specification can be used while data access is pending, +then reused with real data once they arrive. + +The MVP workflow is designed to: + +- validate expected variables; +- generate simple synthetic data for pipeline preparation; +- profile missingness; +- produce descriptive summaries; +- create one basic plot per variable; +- optionally write machine-readable outputs and render an HTML report. + +Synthetic data generated by `episcout` are for workflow preparation and testing +only. They are not suitable for inference, publication, privacy protection or +external data release. + +## Create a specification + +The specification is usually stored as `metadata/data_dictionary.csv`. It must +include `name`, `label`, `type` and `role`. + +```{r} +library(episcout) + +spec_data <- data.frame( + name = c("age", "sex", "death", "follow_up_days"), + label = c( + "Age at baseline", + "Sex at birth", + "Death during follow-up", + "Follow-up time" + ), + type = c("numeric", "categorical", "binary", "integer"), + role = c("covariate", "covariate", "outcome", "outcome"), + units = c("years", "", "", "days"), + levels = c("", "Female;Male;Unknown", "0;1", ""), + min = c(18, NA, 0, 0), + max = c(110, NA, 1, 3650), + missing_codes = c("", "", "", ""), + required = c(TRUE, TRUE, TRUE, TRUE), + group = c("demographics", "demographics", "outcomes", "outcomes"), + description = c( + "Age in years at study baseline.", + "Recorded sex at birth.", + "Binary outcome indicator.", + "Observed follow-up time in days." + ), + stringsAsFactors = FALSE +) + +spec <- epi_eda_spec(spec_data) +spec +``` + +## Prepare the workflow before data access + +Use the specification to generate deterministic synthetic data, then run the +same workflow that will later be used for real data. + +```{r} +synthetic_results <- epi_eda_run( + data = NULL, + spec = spec, + synthetic = TRUE, + n = 100, + seed = 1 +) + +names(synthetic_results) +synthetic_results$metadata +synthetic_results$schema +head(synthetic_results$missing) +``` + +The `summaries` component contains separate machine-readable tables for numeric +and categorical variables. + +```{r} +synthetic_results$summaries$numeric +synthetic_results$summaries$categorical +``` + +Plots are returned as ggplot objects without being printed by the workflow. + +```{r, fig.width = 5, fig.height = 3} +synthetic_results$plots$age +``` + +## Replace synthetic data with real data + +When real data are available, keep the same specification and change the data +source. This example creates a small in-memory data frame to show the shape of +the call. + +```{r} +real_data <- data.frame( + age = c(42, 57, 61, NA), + sex = c("Female", "Male", "Female", "Unknown"), + death = c("0", "1", "0", "0"), + follow_up_days = c(365L, 420L, 800L, 90L), + stringsAsFactors = FALSE +) + +real_results <- epi_eda_run( + data = real_data, + spec = spec +) + +real_results$metadata +real_results$missing +``` + +## Write outputs and render a report + +`epi_eda_run()` writes CSV outputs when `output_dir` is supplied. The directory +must already exist. + +```{r} +output_dir <- file.path(tempdir(), "episcout-eda") +dir.create(output_dir, showWarnings = FALSE) + +epi_eda_run( + data = real_data, + spec = spec, + output_dir = output_dir +) + +list.files(output_dir) +``` + +If `rmarkdown` is installed, `epi_eda_render_report()` runs the workflow and +renders a bundled HTML report from the workflow outputs. + +```{r eval = FALSE} +epi_eda_render_report( + data = real_data, + spec = spec, + output_dir = output_dir +) +``` + +## Start a project scaffold + +For a file-based project, create a starter scaffold and then edit the copied +data dictionary. + +```{r eval = FALSE} +epi_eda_create_project("my-eda-project") +``` + +The scaffold includes: + +- `metadata/data_dictionary.csv`; +- `data/` for real input data; +- `outputs/` for generated workflow files; +- `_targets.R` for optional targets-based execution; +- `reports/eda.qmd` for direct report rendering. + +The reporting and scaffold workflow is optional. You can use the core +`epi_eda_*` functions directly in ordinary R scripts. From 7c12c2a20fee598fdd8f48cc8f707277e56c172c Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:56:30 -0600 Subject: [PATCH 24/33] cran pass and rd updates --- .Rbuildignore | 4 + R/epi_clean_make_names.R | 2 +- R/epi_plot_colour_palettes.R | 8 +- R/epi_plot_hist.R | 2 +- R/epi_plot_parallel.R | 1 - R/epi_plot_themes.R | 5 +- R/epi_stats_numeric.R | 20 +-- R/utils-tidy-eval.R | 16 +-- README.md | 4 +- man/epi_clean_compare_str.Rd | 25 ++-- man/epi_clean_cond_numeric.Rd | 2 +- man/epi_clean_label.Rd | 3 - man/epi_clean_make_names.Rd | 2 +- man/epi_clean_merge_nested_dfs.Rd | 2 +- man/epi_clean_replace_value.Rd | 2 +- man/epi_clean_spread_repeated.Rd | 2 +- man/epi_head_and_tail.Rd | 8 +- man/epi_output_name.Rd | 1 + man/epi_plot_box.Rd | 28 ++-- man/epi_plot_heatmap_triangle.Rd | 5 +- man/epi_plot_hist.Rd | 2 +- man/epi_plot_km.Rd | 3 +- man/epi_plot_theme_1.Rd | 5 +- man/epi_plot_volcano.Rd | 8 +- man/epi_stats_corr.Rd | 2 +- man/epi_stats_dates.Rd | 10 +- man/epi_stats_factors.Rd | 9 +- man/epi_stats_numeric.Rd | 221 ++++++++---------------------- man/epi_survfit_to_df.Rd | 2 +- man/episcout-package.Rd | 7 +- man/palette_IMSS.Rd | 7 +- man/palette_IMSS_accessible.Rd | 11 +- man/tidyeval.Rd | 4 +- scripts/check-cran.sh | 20 ++- 34 files changed, 179 insertions(+), 274 deletions(-) diff --git a/.Rbuildignore b/.Rbuildignore index 921a91e..f35d876 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,6 +1,9 @@ +^future$ +^future/ ^codecov\.yml$ ^cran-comments\.md$ ^\.travis\.yml$ +^\.DS_Store$ ^data-raw$ ^LICENSE\.md$ ^episcout\.Rproj$ @@ -26,4 +29,5 @@ ^environment\.yml$ ^scripts$ ^build$ +^legacy$ # .Rbuildignore diff --git a/R/epi_clean_make_names.R b/R/epi_clean_make_names.R index 5dd5e52..590817b 100644 --- a/R/epi_clean_make_names.R +++ b/R/epi_clean_make_names.R @@ -17,7 +17,7 @@ #' #' @seealso \code{\link[base]{make.names}}, #' \href{https://stackoverflow.com/questions/8396577/check-if-character-value-is-a-valid-r-object-name/8396658#8396658}{stackoverflow test if valid}, -#' \href{https://www.r-bloggers.com/testing-for-valid-variable-names/}{r-bloggers test valid names}. +#' \href{https://www.r-bloggers.com/2011/07/testing-for-valid-variable-names/}{r-bloggers test valid names}. #' #' #' @examples diff --git a/R/epi_plot_colour_palettes.R b/R/epi_plot_colour_palettes.R index fe53dfa..5d0fe78 100644 --- a/R/epi_plot_colour_palettes.R +++ b/R/epi_plot_colour_palettes.R @@ -1,4 +1,4 @@ -#' IMSS “official” colour palette +#' IMSS "official" colour palette #' #' Vector of IMSS brand colours. #' @@ -18,10 +18,10 @@ palette_IMSS <- c( ) -#' Colour-blind–friendly IMSS palette +#' Colour-blind-friendly IMSS palette #' -#' A ten‐hex vector that’s easier to distinguish for viewers -#' with colour‐vision deficiencies. +#' A ten-hex vector that's easier to distinguish for viewers +#' with colour-vision deficiencies. #' #' @return A character vector of hex colour codes. #' @export diff --git a/R/epi_plot_hist.R b/R/epi_plot_hist.R index c5e84a4..dcdf1b4 100644 --- a/R/epi_plot_hist.R +++ b/R/epi_plot_hist.R @@ -52,7 +52,7 @@ #' my_hist_plot <- my_hist_plot + geom_density(col = 2) #' my_hist_plot #' # Histogram overlaid with kernel density curve: -#' # http://www.cookbook-r.com/Graphs/Plotting_distributions_(ggplot2)/ +#' # https://ggplot2.tidyverse.org/reference/geom_histogram.html #' my_hist_plot <- my_hist_plot + #' # Density instead of count on y-axis: #' geom_histogram(aes(y = ..density..), diff --git a/R/epi_plot_parallel.R b/R/epi_plot_parallel.R index 0f8c998..a953045 100644 --- a/R/epi_plot_parallel.R +++ b/R/epi_plot_parallel.R @@ -92,7 +92,6 @@ epi_plot_parallel <- function(df, #' @param file_prefix Prefix used to create file names. #' @param plot_type File type passed to [cowplot::save_plot()]. #' @param plot_step Number of plots per file. -#' @inheritParams epi_plot_parallel #' #' @return A character vector of file names with an attribute `workers` #' reporting the number of workers used. diff --git a/R/epi_plot_themes.R b/R/epi_plot_themes.R index a7d1c4c..c644bc5 100644 --- a/R/epi_plot_themes.R +++ b/R/epi_plot_themes.R @@ -19,10 +19,9 @@ #' @note For examples see eg \code{\link{epi_plot_heatmap_triangle}} and others. #' If creating your own themes see for instance: #' \href{https://github.com/jrnold/ggthemes}{ggthemes}, -#' \href{http://sape.inf.usi.ch/quick-reference/ggplot2/themes}{ggplot2 themes}. +#' \href{https://ggplot2.tidyverse.org/reference/ggtheme.html}{ggplot2 themes}. #' Some colour codes: -#' \href{http://www.cookbook-r.com/Graphs/Colors_(ggplot2)/}{Colors ggplot2}, -#' \href{http://sape.inf.usi.ch/quick-reference/ggplot2/colour}{ggplot2 Quick Reference} +#' \href{https://ggplot2.tidyverse.org/reference/scale_colour_discrete.html}{ggplot2 colour scales} #' #' @author Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> #' diff --git a/R/epi_stats_numeric.R b/R/epi_stats_numeric.R index 67c3c4e..ca0dbb1 100644 --- a/R/epi_stats_numeric.R +++ b/R/epi_stats_numeric.R @@ -6,16 +6,16 @@ #' @param num_vec Numeric vector to summarise. #' @param na.rm Logical; should \code{NA} values be removed prior to computations? #' Default is \code{TRUE}. -#' @param coef Numeric; multiplier for the IQR in Tukey’s outlier rule. +#' @param coef Numeric; multiplier for the IQR in Tukey's outlier rule. #' Values more than \code{coef * IQR} below \code{Q1} or above \code{Q3} are #' counted as outliers. Default is \code{1.5}. #' @param ... Additional arguments passed to \code{\link[e1071]{skewness}} and #' \code{\link[e1071]{kurtosis}} (e.g., \code{type}). #' -#' @return A one‐row \code{data.frame} with the following columns: +#' @return A one-row \code{data.frame} with the following columns: #' \describe{ #' \item{\code{n}}{Total length of the input vector.} -#' \item{\code{n_nonNA}}{Number of non‐\code{NA} observations.} +#' \item{\code{n_nonNA}}{Number of non-\code{NA} observations.} #' \item{\code{NA_count}}{Count of \code{NA} values.} #' \item{\code{NA_percentage}}{Percentage of values that are \code{NA}.} #' \item{\code{sum}}{Sum of values.} @@ -25,26 +25,26 @@ #' \item{\code{median}}{Median.} #' \item{\code{quantile_75}}{75th percentile (\code{Q3}).} #' \item{\code{max}}{Maximum value.} -#' \item{\code{IQR}}{Interquartile range (\code{Q3} – \code{Q1}).} +#' \item{\code{IQR}}{Interquartile range (\code{Q3 - Q1}).} #' \item{\code{SD}}{Standard deviation.} #' \item{\code{CV}}{Coefficient of variation (\code{SD / mean}).} #' \item{\code{variance}}{Variance.} #' \item{\code{sem}}{Standard error of the mean (\code{SD / sqrt(n_nonNA)}).} #' \item{\code{skewness}}{Sample skewness (via \pkg{e1071}).} #' \item{\code{kurtosis}}{Sample kurtosis (via \pkg{e1071}).} -#' \item{\code{Shapiro_Wilk_p_value}}{P‐value from Shapiro–Wilk test (if 3 < \code{n_nonNA} < 5000; else \code{NA}).} -#' \item{\code{lower_fence}}{Lower Tukey fence (\code{Q1} − \code{coef * IQR}).} +#' \item{\code{Shapiro_Wilk_p_value}}{P-value from Shapiro-Wilk test (if 3 < \code{n_nonNA} < 5000; else \code{NA}).} +#' \item{\code{lower_fence}}{Lower Tukey fence (\code{Q1 - coef * IQR}).} #' \item{\code{upper_fence}}{Upper Tukey fence (\code{Q3} + \code{coef * IQR}).} #' \item{\code{n_below_lower}}{Count of values < \code{lower_fence}.} #' \item{\code{n_above_upper}}{Count of values > \code{upper_fence}.} #' \item{\code{outlier_count}}{Total number of Tukey outliers.} -#' \item{\code{outlier_percentage}}{Percentage of non‐\code{NA} values flagged as outliers.} +#' \item{\code{outlier_percentage}}{Percentage of non-\code{NA} values flagged as outliers.} #' } #' #' @details -#' Missing values are dropped when \code{na.rm = TRUE}. The Shapiro–Wilk test +#' Missing values are dropped when \code{na.rm = TRUE}. The Shapiro-Wilk test #' for normality is only run for sample sizes between 4 and 4999; otherwise -#' its p‐value is reported as \code{NA}. For skewness: negative/longer left tail, +#' its p-value is reported as \code{NA}. For skewness: negative/longer left tail, #' positive/longer right tail, values above 1 usually means non-normality. #' For kurtosis consider lower values, broader shape and longer tails (platy ~<3), #' normal (meso ~3) and slender/no tails (lepto ~<3). @@ -92,7 +92,7 @@ epi_stats_numeric <- function(num_vec = NULL, outlier_count <- epi_stats_count_outliers(num_vec, coef = coef) outlier_percentage <- if (n_nonNA > 0) outlier_count / n_nonNA * 100 else NA_real_ - # Shapiro–Wilk normality + # Shapiro-Wilk normality normality <- NA_real_ if (n_nonNA > 3 && n_nonNA < 5000) { normality <- tryCatch( diff --git a/R/utils-tidy-eval.R b/R/utils-tidy-eval.R index e6b3e18..165e4b3 100644 --- a/R/utils-tidy-eval.R +++ b/R/utils-tidy-eval.R @@ -25,19 +25,17 @@ #' therefore one should always use `rlang::exprs()`. #' #' To learn more about tidy eval and how to use these tools, visit -#' and the [Metaprogramming -#' section](http://adv-r.had.co.nz/Computing-on-the-language.html) of [Advanced -#' R](https://adv-r.hadley.nz). +#' and [Advanced R](https://adv-r.hadley.nz). #' #' @md #' @name tidyeval #' @keywords internal -#' @importFrom rlang quo quos enquo enquos quo_name sym ensym syms -#' ensyms expr exprs enexpr enexprs .data := -#' @aliases quo quos enquo enquos quo_name -#' sym ensym syms ensyms -#' expr exprs enexpr enexprs -#' .data := +#' @importFrom rlang quo quos enquo enquos quo_name sym ensym syms ensyms +#' @importFrom rlang expr exprs enexpr enexprs .data := +#' @aliases quo quos enquo enquos quo_name +#' @aliases sym ensym syms ensyms +#' @aliases expr exprs enexpr enexprs +#' @aliases .data := #' @export quo quos enquo enquos quo_name #' @export sym ensym syms ensyms #' @export expr enexpr enexprs diff --git a/README.md b/README.md index cd3507c..dd90566 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -[![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) +[![Project Status: Active - The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) [![R](https://github.com/antoniojbt/episcout/actions/workflows/r-cmd-check.yml/badge.svg)](https://github.com/antoniojbt/episcout/actions/workflows/r-cmd-check.yml) -[![codecov](https://codecov.io/gh/AntonioJBT/episcout/branch/master/graph/badge.svg)](https://codecov.io/gh/AntonioJBT/episcout) +[![codecov](https://codecov.io/gh/AntonioJBT/episcout/branch/master/graph/badge.svg)](https://app.codecov.io/gh/AntonioJBT/episcout) # episcout diff --git a/man/epi_clean_compare_str.Rd b/man/epi_clean_compare_str.Rd index df24668..3487066 100644 --- a/man/epi_clean_compare_str.Rd +++ b/man/epi_clean_compare_str.Rd @@ -43,20 +43,23 @@ as fixed characters, not with regex. The value in string_col will be the character vector with strings to search in. } \examples{ - \dontrun{ -letts <- paste(letters, collapse = ' ') -other_letts <- toupper(paste(letters, collapse = ' ')) -df_comp <- data.frame ('sub' = rep(x = substr(letts, 1, 5), 10), - 'str' = rep(x = substr(letts, 1, 5), 10), - stringsAsFactors = FALSE) -df2_comp <- data.frame ('sub' = rep(x = substr(letts, 1, 5), 10), - 'str' = rep(x = substr(other_letts, 6, 10), 10), - stringsAsFactors = FALSE) +letts <- paste(letters, collapse = " ") +other_letts <- toupper(paste(letters, collapse = " ")) +df_comp <- data.frame( + "sub" = rep(x = substr(letts, 1, 5), 10), + "str" = rep(x = substr(letts, 1, 5), 10), + stringsAsFactors = FALSE +) +df2_comp <- data.frame( + "sub" = rep(x = substr(letts, 1, 5), 10), + "str" = rep(x = substr(other_letts, 6, 10), 10), + stringsAsFactors = FALSE +) # Create a new data frame and rename columns: df3 <- rbind(df_comp, df2_comp) -col_1 <- 'sub' -col_2 <- 'str' +col_1 <- "sub" +col_2 <- "str" val_id <- 1 # df3[val_id, c(col_1, col_2)] # Should evaluate to TRUE: diff --git a/man/epi_clean_cond_numeric.Rd b/man/epi_clean_cond_numeric.Rd index 0606bed..874668f 100644 --- a/man/epi_clean_cond_numeric.Rd +++ b/man/epi_clean_cond_numeric.Rd @@ -42,5 +42,5 @@ epi_clean_cond_numeric(df[, "x"]) # should be TRUE \code{\link[base]{is.integer}}, \code{\link[base]{is.numeric}} } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_clean_label.Rd b/man/epi_clean_label.Rd index 32f7375..5bc2de5 100644 --- a/man/epi_clean_label.Rd +++ b/man/epi_clean_label.Rd @@ -24,9 +24,6 @@ with levels and labels as defined in \code{lookup_df}. This function takes a dataframe and a lookup dataframe that contains the mapping of factors' levels and labels for specific variables. It applies these levels and labels to the corresponding variables in the target dataframe. -An error is thrown if variables referenced in \code{lookup_df} are missing from -\code{data_df}. Warnings are issued when levels present in one but not the -other are encountered. } \examples{ # Assume data_df is already loaded and contains 'ORIGEN' and 'SECTOR' columns diff --git a/man/epi_clean_make_names.Rd b/man/epi_clean_make_names.Rd index 99ec2aa..572c032 100644 --- a/man/epi_clean_make_names.Rd +++ b/man/epi_clean_make_names.Rd @@ -40,7 +40,7 @@ epi_clean_make_names(valid_names) # There shouldn't be anything to correct \seealso{ \code{\link[base]{make.names}}, \href{https://stackoverflow.com/questions/8396577/check-if-character-value-is-a-valid-r-object-name/8396658#8396658}{stackoverflow test if valid}, -\href{https://www.r-bloggers.com/testing-for-valid-variable-names/}{r-bloggers test valid names}. +\href{https://www.r-bloggers.com/2011/07/testing-for-valid-variable-names/}{r-bloggers test valid names}. } \author{ Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} diff --git a/man/epi_clean_merge_nested_dfs.Rd b/man/epi_clean_merge_nested_dfs.Rd index ea25897..2e0a5e5 100644 --- a/man/epi_clean_merge_nested_dfs.Rd +++ b/man/epi_clean_merge_nested_dfs.Rd @@ -143,5 +143,5 @@ View(all_merged) \code{\link{epi_clean_transpose}}, \code{\link[base]{merge}}. } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_clean_replace_value.Rd b/man/epi_clean_replace_value.Rd index 44e43c1..0b75795 100644 --- a/man/epi_clean_replace_value.Rd +++ b/man/epi_clean_replace_value.Rd @@ -60,5 +60,5 @@ df_factor$date_col \code{\link[stringr]{str_detect}}. } \author{ -Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> +Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} } diff --git a/man/epi_clean_spread_repeated.Rd b/man/epi_clean_spread_repeated.Rd index 81e8622..b6bd8ca 100644 --- a/man/epi_clean_spread_repeated.Rd +++ b/man/epi_clean_spread_repeated.Rd @@ -47,5 +47,5 @@ df_spread \code{\link{epi_clean_transpose}} } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_head_and_tail.Rd b/man/epi_head_and_tail.Rd index 89db5de..87145ec 100644 --- a/man/epi_head_and_tail.Rd +++ b/man/epi_head_and_tail.Rd @@ -14,10 +14,14 @@ epi_head_and_tail(df = NULL, rows = 5, cols = 5, last_cols = FALSE) \item{cols}{Number of columns to print, default is 5} \item{last_cols}{Print the last columns instead of the first few, -default is FALSE} +default is FALSE + +Row and column counts outside the available range are clamped to the +data dimensions.} } \value{ -A data.frame containing the selected head and tail rows and columns. +A data.frame containing the selected head and tail rows and +columns. The value is returned invisibly after printing. } \description{ epi_head_and_tail() prints the first and last few rows of a data.frame diff --git a/man/epi_output_name.Rd b/man/epi_output_name.Rd index e7735b2..2241d87 100644 --- a/man/epi_output_name.Rd +++ b/man/epi_output_name.Rd @@ -18,6 +18,7 @@ string value \description{ Generate a simple output name, given a string (filename) splits at the last '.', drops the current suffix and adds the one provided. +If \code{input_name} lacks a dot, the whole string is used as is. } \examples{ \dontrun{ diff --git a/man/epi_plot_box.Rd b/man/epi_plot_box.Rd index 46f0c57..892a5a9 100644 --- a/man/epi_plot_box.Rd +++ b/man/epi_plot_box.Rd @@ -80,34 +80,34 @@ var_x and var_y are passed to ggplot2::aes. For colour and fill see ggplot2::fill for further information. } \examples{ - \dontrun{ -Boxplot of one variable: +# Boxplot of one variable: library(ggplot2) library(ggthemes) set.seed(12345) n <- 20 -df <- data.frame(var_id = rep(1:(n / 2), each = 2), - var_to_rep = rep(c("Pre", "Post"), n / 2), - x = rnorm(n), - y = rbinom(n, 1, 0.50), - z = rpois(n, 2) - ) +df <- data.frame( + var_id = rep(1:(n / 2), each = 2), + var_to_rep = rep(c("Pre", "Post"), n / 2), + x = rnorm(n), + y = rbinom(n, 1, 0.50), + z = rpois(n, 2) +) df df$x # continuous variable -epi_plot_box(df, var_y = 'x') +epi_plot_box(df, var_y = "x") # Add notch: -epi_plot_box(df, var_y = 'x', notch = TRUE) +epi_plot_box(df, var_y = "x", notch = TRUE) # Boxplot for x and y variables: df$x # continuous variable df$var_to_rep # factor -epi_plot_box(df, var_x = 'var_to_rep', var_y = 'x') +epi_plot_box(df, var_x = "var_to_rep", var_y = "x") # Change colours, remove legend, etc.: -my_boxplot <- epi_plot_box(df, var_x = 'var_to_rep', var_y = 'x') +my_boxplot <- epi_plot_box(df, var_x = "var_to_rep", var_y = "x") my_boxplot + # scale_fill_grey() + -scale_fill_brewer(palette = "Blues") + + scale_fill_brewer(palette = "Blues") + # scale_fill_brewer(palette = "Dark2") + theme(legend.position = "none") # Remove legend # dev.off() @@ -122,5 +122,5 @@ scale_fill_brewer(palette = "Blues") + \code{\link[ggplot2]{geom_boxplot}}. } \author{ -Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> +Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} } diff --git a/man/epi_plot_heatmap_triangle.Rd b/man/epi_plot_heatmap_triangle.Rd index 145fa58..d8e65d3 100644 --- a/man/epi_plot_heatmap_triangle.Rd +++ b/man/epi_plot_heatmap_triangle.Rd @@ -24,7 +24,8 @@ Default is 'Spearman'} \item{show_values}{Values to show in plot, 'pval' or 'corr'. Default is 'pval'.} -\item{show_labels}{Logical indicating whether numeric labels are displayed. Default is TRUE.} +\item{show_labels}{Logical indicating whether numeric labels should be +displayed inside the tiles. Default is \code{TRUE}.} } \value{ Returns a heatmap as a ggplot2 object @@ -83,5 +84,5 @@ ggsave("epi_heatmap_triangle.svg", \code{\link{epi_stats_corr_rename}}, \code{\link{epi_plot_heatmap}} } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_plot_hist.Rd b/man/epi_plot_hist.Rd index deb6ea4..d0d4aa7 100644 --- a/man/epi_plot_hist.Rd +++ b/man/epi_plot_hist.Rd @@ -56,7 +56,7 @@ my_hist_plot my_hist_plot <- my_hist_plot + geom_density(col = 2) my_hist_plot # Histogram overlaid with kernel density curve: -# http://www.cookbook-r.com/Graphs/Plotting_distributions_(ggplot2)/ +# https://ggplot2.tidyverse.org/reference/geom_histogram.html my_hist_plot <- my_hist_plot + # Density instead of count on y-axis: geom_histogram(aes(y = ..density..), diff --git a/man/epi_plot_km.Rd b/man/epi_plot_km.Rd index 1c3fd21..3e6e92f 100644 --- a/man/epi_plot_km.Rd +++ b/man/epi_plot_km.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/KM_plot.R +% Please edit documentation in R/epi_plot_km.R \name{epi_plot_km} \alias{epi_plot_km} \title{Plot Kaplan-Meier survival curves} @@ -39,4 +39,5 @@ library(survival) fit <- survfit(Surv(time, status) ~ sex, data = lung) epi_plot_km(fit, save_path = "km.png") } + } diff --git a/man/epi_plot_theme_1.Rd b/man/epi_plot_theme_1.Rd index 73d9926..fa75bec 100644 --- a/man/epi_plot_theme_1.Rd +++ b/man/epi_plot_theme_1.Rd @@ -47,10 +47,9 @@ one with your preferences or modify directly in a ggplot2 call. For examples see eg \code{\link{epi_plot_heatmap_triangle}} and others. If creating your own themes see for instance: \href{https://github.com/jrnold/ggthemes}{ggthemes}, -\href{http://sape.inf.usi.ch/quick-reference/ggplot2/themes}{ggplot2 themes}. +\href{https://ggplot2.tidyverse.org/reference/ggtheme.html}{ggplot2 themes}. Some colour codes: -\href{http://www.cookbook-r.com/Graphs/Colors_(ggplot2)/}{Colors ggplot2}, -\href{http://sape.inf.usi.ch/quick-reference/ggplot2/colour}{ggplot2 Quick Reference} +\href{https://ggplot2.tidyverse.org/reference/scale_colour_discrete.html}{ggplot2 colour scales} } \author{ Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} diff --git a/man/epi_plot_volcano.Rd b/man/epi_plot_volcano.Rd index fd79dca..e425b17 100644 --- a/man/epi_plot_volcano.Rd +++ b/man/epi_plot_volcano.Rd @@ -20,15 +20,15 @@ epi_plot_volcano( \item{p_val}{Numeric vector of raw p-values (will be transformed to \code{-log10(p_val)}).} -\item{main}{Main title passed to \code{\link[graphics:plot.default]{graphics::plot()}}.} +\item{main}{Main title passed to \code{\link[graphics:plot]{graphics::plot()}}.} -\item{pch}{Plotting character passed to \code{\link[graphics:plot.default]{graphics::plot()}}.} +\item{pch}{Plotting character passed to \code{\link[graphics:plot]{graphics::plot()}}.} \item{h_abline}{Horizontal line position passed to \code{\link[graphics:abline]{graphics::abline()}}.} \item{v_abline}{Vertical line positions passed to \code{\link[graphics:abline]{graphics::abline()}}.} -\item{...}{Additional arguments passed to \code{\link[graphics:plot.default]{graphics::plot()}}.} +\item{...}{Additional arguments passed to \code{\link[graphics:plot]{graphics::plot()}}.} } \value{ Invisibly returns \code{NULL} and draws the plot to the active device. @@ -50,7 +50,7 @@ epi_plot_volcano(log2fc, pvals) } \seealso{ \code{\link[=epi_plot_list]{epi_plot_list()}}, \code{\link[=epi_plots_to_grid]{epi_plots_to_grid()}}, -\code{\link[=epi_plot_cow_save]{epi_plot_cow_save()}}, \code{\link[graphics:plot.default]{graphics::plot()}} +\code{\link[=epi_plot_cow_save]{epi_plot_cow_save()}}, \code{\link[graphics:plot]{graphics::plot()}} } \author{ Antonio Berlanga-Taylor \url{https://github.com/AntonioJBT/episcout} diff --git a/man/epi_stats_corr.Rd b/man/epi_stats_corr.Rd index 8149c97..2a806f4 100644 --- a/man/epi_stats_corr.Rd +++ b/man/epi_stats_corr.Rd @@ -54,5 +54,5 @@ class(cormat_all) \code{\link[data.table]{melt}}. } \author{ -Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> +Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} } diff --git a/man/epi_stats_dates.Rd b/man/epi_stats_dates.Rd index 5e5d6c6..2d57496 100644 --- a/man/epi_stats_dates.Rd +++ b/man/epi_stats_dates.Rd @@ -22,15 +22,15 @@ is compatible with both \code{Date} and \code{IDate} objects. The default origin for numeric date conversion in R is "1970-01-01". } \examples{ -sample_dates <- as.Date(c("2020-01-01", "2020-05-15", "2020-12-31", "2021-01-01", "2021-07-15")) -date_stats <- epi_stats_dates(sample_dates) -print(date_stats) +sample_dates <- as.Date(c("2020-01-01", "2020-05-15", "2020-12-31", "2021-01-01")) +epi_stats_dates(sample_dates) + } \seealso{ \code{\link{epi_stats_summary}}, \code{\link{epi_stats_format}}, -\code{\link{epi_stats_numeric}}. +\code{\link{epi_stats_numeric}} } \author{ -Antonio J Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> +Antonio J Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} } diff --git a/man/epi_stats_factors.Rd b/man/epi_stats_factors.Rd index d6bb4ff..8a4f360 100644 --- a/man/epi_stats_factors.Rd +++ b/man/epi_stats_factors.Rd @@ -13,10 +13,10 @@ epi_stats_factors(df) A tibble in wide format with one row per factor column. } \description{ -Produce summary metrics for all factor columns in a data frame. The -output includes counts of missing values, proportion of complete cases, -whether the factor is ordered, the number of unique levels and the -three most frequent levels with their counts. +Produce summary metrics for all factor columns in a data frame. The output +includes counts of missing values, proportion of complete cases, whether the +factor is ordered, the number of unique levels and the three most frequent +levels with their counts. } \examples{ df <- data.frame( @@ -24,6 +24,7 @@ df <- data.frame( f2 = factor(c("x", "x", "y", "y", "y")) ) epi_stats_factors(df) + } \seealso{ \code{\link{epi_stats_chars}}, \code{\link{epi_stats_numeric}}, diff --git a/man/epi_stats_numeric.Rd b/man/epi_stats_numeric.Rd index c7f04d2..f160054 100644 --- a/man/epi_stats_numeric.Rd +++ b/man/epi_stats_numeric.Rd @@ -2,187 +2,74 @@ % Please edit documentation in R/epi_stats_numeric.R \name{epi_stats_numeric} \alias{epi_stats_numeric} -\title{Get summary descriptive statistics for numeric columns.} +\title{Summarise Numeric Vector with Descriptive Statistics} \usage{ epi_stats_numeric(num_vec = NULL, na.rm = TRUE, coef = 1.5, ...) } \arguments{ -\item{num_vec}{Numeric vector to test.} +\item{num_vec}{Numeric vector to summarise.} -\item{na.rm}{Remove NA values, default is TRUE} +\item{na.rm}{Logical; should \code{NA} values be removed prior to computations? +Default is \code{TRUE}.} -\item{coef}{Coefficient for outlier detection, default is 1.5} +\item{coef}{Numeric; multiplier for the IQR in Tukey's outlier rule. +Values more than \code{coef * IQR} below \code{Q1} or above \code{Q3} are +counted as outliers. Default is \code{1.5}.} -\item{...}{is passed to skewness() and kurtosis()} +\item{...}{Additional arguments passed to \code{\link[e1071]{skewness}} and +\code{\link[e1071]{kurtosis}} (e.g., \code{type}).} } \value{ -Data.frame with results from tests performed +A one-row \code{data.frame} with the following columns: +\describe{ +\item{\code{n}}{Total length of the input vector.} +\item{\code{n_nonNA}}{Number of non-\code{NA} observations.} +\item{\code{NA_count}}{Count of \code{NA} values.} +\item{\code{NA_percentage}}{Percentage of values that are \code{NA}.} +\item{\code{sum}}{Sum of values.} +\item{\code{min}}{Minimum value.} +\item{\code{quantile_25}}{25th percentile (\code{Q1}).} +\item{\code{mean}}{Arithmetic mean.} +\item{\code{median}}{Median.} +\item{\code{quantile_75}}{75th percentile (\code{Q3}).} +\item{\code{max}}{Maximum value.} +\item{\code{IQR}}{Interquartile range (\code{Q3 - Q1}).} +\item{\code{SD}}{Standard deviation.} +\item{\code{CV}}{Coefficient of variation (\code{SD / mean}).} +\item{\code{variance}}{Variance.} +\item{\code{sem}}{Standard error of the mean (\code{SD / sqrt(n_nonNA)}).} +\item{\code{skewness}}{Sample skewness (via \pkg{e1071}).} +\item{\code{kurtosis}}{Sample kurtosis (via \pkg{e1071}).} +\item{\code{Shapiro_Wilk_p_value}}{P-value from Shapiro-Wilk test (if 3 < \code{n_nonNA} < 5000; else \code{NA}).} +\item{\code{lower_fence}}{Lower Tukey fence (\code{Q1 - coef * IQR}).} +\item{\code{upper_fence}}{Upper Tukey fence (\code{Q3} + \code{coef * IQR}).} +\item{\code{n_below_lower}}{Count of values < \code{lower_fence}.} +\item{\code{n_above_upper}}{Count of values > \code{upper_fence}.} +\item{\code{outlier_count}}{Total number of Tukey outliers.} +\item{\code{outlier_percentage}}{Percentage of non-\code{NA} values flagged as outliers.} } -\description{ -epi_stats_numeric() generates several descriptive summary statistics for numeric -variables. } -\note{ -Normality is tested with Shapiro-Wilk (small values indicate -non-normality). Testing normality can be contentious, likely uninformative -and with Shapiro-Wilk can only be done for sample size between 3-5000. -The package e1071 is used for skewness and kurtosis. -For skewness: negative/longer left tail, positive/longer right tail, -values above 1 usually means non-normality. For kurtosis consider lower values, -broader shape and longer tails (platy ~<3), normal (meso ~3) -and slender/no tails (lepto ~<3). Outliers are detected with the Tukey -method (above and below 1.5 * IQR). na.rm is TRUE by default for all tests. +\description{ +Compute a comprehensive set of descriptive statistics for a numeric vector, +including measures of location, dispersion, shape, normality, and outliers. } -\examples{ -##### -# Load libraries needed: -library(episcout) -library(dplyr) -library(purrr) -library(e1071) -library(tibble) -library(tidyr) -##### - -##### -# Generate a data frame: -n <- 1000 -df <- data.frame( - var_id = rep(1:(n / 2), each = 2), - var_to_rep = rep(c("Pre", "Post"), n / 2), - x = rnorm(n), - y = rbinom(n, 1, 0.50), - z = rpois(n, 2) -) - -# Explore first and last rows for first columns: -epi_head_and_tail(df) - -# Add character/factor columns: -col_chr <- data.frame( - "chr1" = rep(c("A", "B"), length.out = 1000), - "chr2" = rep(c("C", "D"), length.out = 1000) -) -dim(col_chr) -df_cont_chr <- tibble::as.tibble(cbind(df, col_chr)) -epi_head_and_tail(df_cont_chr) -epi_head_and_tail(df_cont_chr, last_cols = TRUE) - -# Check variable types are what you expect: -epi_clean_count_classes(df_cont_chr) -str(df_cont_chr) -dim(df_cont_chr) -# var_id, y and z can be treated as factors or characters. -summary(as.factor(df_cont_chr$y)) -summary(as.factor(df_cont_chr$z)) -# Here we'll only transform y though: -df_cont_chr$y <- as.factor(df_cont_chr$y) -epi_clean_count_classes(df_cont_chr) -str(df_cont_chr) - -# Designate some values as codes to be counted separately: -codes <- c("Pre", "A", "C", "1", "3") -##### - -##### -# Count when codes are present, pass these as character or factor, specify -# action is to count codes only: -stat_sum1 <- epi_stats_summary( - df = df_cont_chr, - codes = codes, - class_type = "chr_fct", - action = "codes_only" -) -class(stat_sum1) -stat_sum1 -##### - -##### -# Add total for percentage calculation and order column to tidy up results: -perc_n <- nrow(df_cont_chr) -order_by <- "percent" -stat_sum_tidy <- epi_stats_tidy( - sum_df = stat_sum1, - order_by = order_by, - perc_n = perc_n -) -stat_sum_tidy -# Format them if needed: -epi_stats_format(stat_sum_tidy, digits = 0) -epi_stats_format(stat_sum_tidy, digits = 2) -##### - -##### -# Count integer or numeric codes: -stat_sum2 <- epi_stats_summary(df_cont_chr, - codes = codes, - class_type = "int_num", - action = "codes_only" -) -stat_sum2 -# Tidy and format them: -stat_sum_tidy <- epi_stats_tidy( - sum_df = stat_sum2, - order_by = order_by, - perc_n = perc_n -) -stat_sum_tidy -epi_stats_format(stat_sum_tidy, digits = 0) -epi_stats_format(stat_sum_tidy, digits = 2, skip = c(2, 3)) -##### - -##### -# Get summary stats excluding contingency codes for character and factor columns: -stat_sum3 <- epi_stats_summary(df_cont_chr, - codes = codes, - class_type = "chr_fct", - action = "exclude" -) -stat_sum3 -# Tidy and format: -stat_sum_tidy <- epi_stats_tidy( - sum_df = stat_sum3, - order_by = order_by, - perc_n = perc_n -) -stat_sum_tidy -epi_stats_format(stat_sum_tidy, digits = 0) -epi_stats_format(stat_sum_tidy, digits = 1) -##### - -##### -# Get summary stats for numeric/integer columns -# while excluding certain codes/values: -stat_sum4 <- epi_stats_summary( - df = df_cont_chr, - codes = codes, - class_type = "int_num", - action = "exclude" -) -class(stat_sum4) -stat_sum4 -# Numeric data summary doesn't need tidying but could be formatted: -epi_stats_format(stat_sum4, digits = 2) -##### - -##### -# If there are no codes to return the result is an empty data.frame (tibble): -codes <- c("Per", "X", "55") -stat_sum_zero <- epi_stats_summary(df_cont_chr, - codes = codes, - class_type = "chr_fct", - action = "codes_only" -) -stat_sum_zero -##### +\details{ +Missing values are dropped when \code{na.rm = TRUE}. The Shapiro-Wilk test +for normality is only run for sample sizes between 4 and 4999; otherwise +its p-value is reported as \code{NA}. For skewness: negative/longer left tail, +positive/longer right tail, values above 1 usually means non-normality. +For kurtosis consider lower values, broader shape and longer tails (platy ~<3), +normal (meso ~3) and slender/no tails (lepto ~<3). +Outliers are detected with the Tukey method (above and below 1.5 * IQR) or +using the multiplier \code{coef}. Coefficient of variation (\code{CV}) is +calculated as \code{SD / mean}. You may pass further arguments (e.g., +\code{type}) to the skewness and kurtosis functions from \pkg{e1071}. +na.rm is TRUE by default for all tests. } \seealso{ -\code{\link{epi_stats_summary}}, -\code{\link{epi_stats_format}}, -\code{\link{epi_stats_tidy}}, -\code{\link[e1071]{skewness}}, -\code{\link[e1071]{kurtosis}}. +\code{\link{epi_stats_count_outliers}}, \code{\link[e1071]{skewness}}, +\code{\link[e1071]{kurtosis}}, \code{\link[stats]{shapiro.test}} } \author{ -Antonio J Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio J. Berlanga-Taylor } diff --git a/man/epi_survfit_to_df.Rd b/man/epi_survfit_to_df.Rd index 151945c..7be37ed 100644 --- a/man/epi_survfit_to_df.Rd +++ b/man/epi_survfit_to_df.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/KM_plot.R +% Please edit documentation in R/epi_plot_km.R \name{epi_survfit_to_df} \alias{epi_survfit_to_df} \title{Convert survfit object to data frame} diff --git a/man/episcout-package.Rd b/man/episcout-package.Rd index 5d2cb80..bb24219 100644 --- a/man/episcout-package.Rd +++ b/man/episcout-package.Rd @@ -6,10 +6,15 @@ \alias{episcout-package} \title{episcout: Quickly Clean, Explore and Visualise Large Epidemiological Datasets} \description{ -Facilitates cleaning, exploring and visualising large-ish datasets (hundreds of thousands to millions of observations with tens to hundreds of variables). These are mostly wrapper and convenience functions to pre-process (wrangle, explore, clean, etc.) data-sets. Assumes you're happy with tidyverse and the basics of data.table. +Facilitates cleaning, exploring and visualising large-ish epidemiological datasets (hundreds of thousands to millions of observations with tens to hundreds of variables). Provides helper functions for preprocessing and descriptive work, plus specification-first exploratory data analysis workflows for epidemiological datasets. Assumes you're happy with tidyverse and the basics of data.table. } \author{ \strong{Maintainer}: Antonio Berlanga-Taylor \email{antoniojberlanga@gmail.com} +Authors: +\itemize{ + \item Antonio Berlanga-Taylor \email{antoniojberlanga@gmail.com} +} + } \keyword{internal} diff --git a/man/palette_IMSS.Rd b/man/palette_IMSS.Rd index 77f2112..d835344 100644 --- a/man/palette_IMSS.Rd +++ b/man/palette_IMSS.Rd @@ -1,12 +1,8 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/epi_plot_colour_palettes.R -\docType{data} \name{palette_IMSS} \alias{palette_IMSS} -\title{IMSS “official” colour palette} -\format{ -An object of class \code{character} of length 10. -} +\title{IMSS "official" colour palette} \usage{ palette_IMSS } @@ -16,4 +12,3 @@ A character vector of hex colour codes. \description{ Vector of IMSS brand colours. } -\keyword{datasets} diff --git a/man/palette_IMSS_accessible.Rd b/man/palette_IMSS_accessible.Rd index 56586fe..ca168ca 100644 --- a/man/palette_IMSS_accessible.Rd +++ b/man/palette_IMSS_accessible.Rd @@ -1,12 +1,8 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/epi_plot_colour_palettes.R -\docType{data} \name{palette_IMSS_accessible} \alias{palette_IMSS_accessible} -\title{Colour-blind–friendly IMSS palette} -\format{ -An object of class \code{character} of length 10. -} +\title{Colour-blind-friendly IMSS palette} \usage{ palette_IMSS_accessible } @@ -14,7 +10,6 @@ palette_IMSS_accessible A character vector of hex colour codes. } \description{ -A ten‐hex vector that’s easier to distinguish for viewers -with colour‐vision deficiencies. +A ten-hex vector that's easier to distinguish for viewers +with colour-vision deficiencies. } -\keyword{datasets} diff --git a/man/tidyeval.Rd b/man/tidyeval.Rd index f91bbef..bc4172f 100644 --- a/man/tidyeval.Rd +++ b/man/tidyeval.Rd @@ -26,7 +26,7 @@ character vector. \item \code{\link[rlang]{expr}()} and \code{\link[rlang]{quo}()} quote one expression. \code{quo()} wraps the quoted expression in a quosure. -The plural variants \code{\link[rlang:defusing-advanced]{rlang::exprs()}} and +The plural variants \code{\link[rlang:exprs]{rlang::exprs()}} and \code{\link[rlang]{quos}()} return a list of quoted expressions or quosures. \item \code{\link[rlang]{enexpr}()} and \code{\link[rlang]{enquo}()} @@ -42,6 +42,6 @@ capture multiple expressions supplied as arguments, including therefore one should always use \code{rlang::exprs()}. To learn more about tidy eval and how to use these tools, visit -\url{http://rlang.r-lib.org} and the \href{http://adv-r.had.co.nz/Computing-on-the-language.html}{Metaprogramming section} of \href{https://adv-r.hadley.nz}{Advanced R}. +\url{https://rlang.r-lib.org} and \href{https://adv-r.hadley.nz}{Advanced R}. } \keyword{internal} diff --git a/scripts/check-cran.sh b/scripts/check-cran.sh index 0a45938..69e8ecc 100755 --- a/scripts/check-cran.sh +++ b/scripts/check-cran.sh @@ -20,10 +20,26 @@ if [[ ! -x "$r_bin" ]]; then exit 127 fi -rm -f "${check_dir}"/*.tar.gz +work_dir="$(mktemp -d "${TMPDIR:-/tmp}/episcout-cran-check.XXXXXX")" + +copy_check_artifacts() { + local status=$? + + rm -f "${check_dir}"/*.tar.gz + rm -rf "${check_dir}/episcout.Rcheck" + cp "${work_dir}"/episcout_*.tar.gz "${check_dir}/" 2>/dev/null || true + if [[ -d "${work_dir}/episcout.Rcheck" ]]; then + cp -R "${work_dir}/episcout.Rcheck" "${check_dir}/" + fi + rm -rf "$work_dir" + + exit "$status" +} + +trap copy_check_artifacts EXIT ( - cd "$check_dir" + cd "$work_dir" "$r_bin" CMD build "$repo_root" --no-resave-data --compact-vignettes=gs+qpdf tarball="$(find . -maxdepth 1 -name 'episcout_*.tar.gz' -print -quit)" if [[ -z "$tarball" ]]; then From c1adfefaa3cbdec210209bce5d3ddd8c09eeeefa Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Mon, 15 Jun 2026 15:18:20 -0600 Subject: [PATCH 25/33] re-org future work with sdd tdd, archived first phase --- .Rbuildignore | 2 + AGENTS.MD | 24 +- NEWS.md | 37 ++ archive/eda_sdd_tdd_r1_archive/AGENTS.md | 38 ++ archive/eda_sdd_tdd_r1_archive/CHANGELOG.md | 24 + archive/eda_sdd_tdd_r1_archive/README.md | 36 ++ .../eda_sdd_tdd_r1_archive}/START_HERE.md | 92 ++-- archive/eda_sdd_tdd_r1_archive/SUMMARY.md | 50 ++ .../adr/0001-build-on-episcout.md | 0 .../adr/0002-specification-first-workflow.md | 0 .../adr/0003-r-not-pure-base-r.md | 0 .../adr/0004-synthetic-data-before-access.md | 0 .../adr/0005-external-fixture-tdd.md | 0 .../eda_sdd_tdd_r1_archive}/adr/README.md | 0 .../codex/review-checklist.md | 6 +- .../codex/revised-pr-plan-tdd-first.md | 2 +- .../codex/tdd-first-codex-instructions.md | 16 +- archive/eda_sdd_tdd_r1_archive/prompt_loop.md | 30 ++ .../repository-audit.md | 0 .../eda_sdd_tdd_r1_archive}/roadmap.md | 2 +- .../sdd/0001-spec-first-eda.md | 0 .../sdd/data-dictionary-spec.md | 0 .../eda_sdd_tdd_r1_archive}/sdd/mvp-scope.md | 2 +- .../sdd/tdd-external-fixtures.md | 2 +- future/README.md | 51 ++ future/TODOs.md | 46 ++ future/changelog.md | 12 + future/decisions.md | 24 + future/prompts/refactor-review.md | 12 + future/prompts/senior-r-package-review.md | 115 +++++ future/prompts/test-design-review.md | 12 + future/references/spec-driven-eda-archive.md | 19 + future/reviews/README.md | 5 + future/scratch/README.md | 6 + .../acceptance.md | 13 + .../001-phase-1-helper-stabilization/brief.md | 36 ++ .../manifest.yml | 32 ++ .../review.md | 16 + .../001-phase-1-helper-stabilization/sdd.md | 35 ++ .../001-phase-1-helper-stabilization/tdd.md | 42 ++ .../002-penguins-raw-fixture/acceptance.md | 15 + .../specs/002-penguins-raw-fixture/brief.md | 29 ++ .../002-penguins-raw-fixture/manifest.yml | 31 ++ .../specs/002-penguins-raw-fixture/review.md | 15 + future/specs/002-penguins-raw-fixture/sdd.md | 46 ++ future/specs/002-penguins-raw-fixture/tdd.md | 48 ++ .../acceptance.md | 12 + .../003-large-data-backend-strategy/brief.md | 26 + .../manifest.yml | 29 ++ .../003-large-data-backend-strategy/review.md | 15 + .../003-large-data-backend-strategy/sdd.md | 37 ++ .../003-large-data-backend-strategy/tdd.md | 24 + future/specs/template/acceptance.md | 12 + future/specs/template/brief.md | 26 + future/specs/template/manifest.yml | 29 ++ future/specs/template/review.md | 16 + future/specs/template/sdd.md | 39 ++ future/specs/template/tdd.md | 33 ++ spec_driven_EDA_plan/docs/AGENTS.md | 38 -- .../codex-pr-add-penguins-raw-fixture.md | 458 ------------------ ...codex-pr-deferred-phase-1-stabilization.md | 262 ---------- ...eview-checklist-external-truth-fixtures.md | 149 ------ .../future/phase-1-stabilization-sdd-tdd.md | 149 ------ spec_driven_EDA_plan/prompt_loop.md | 87 ---- 64 files changed, 1253 insertions(+), 1211 deletions(-) create mode 100644 NEWS.md create mode 100644 archive/eda_sdd_tdd_r1_archive/AGENTS.md create mode 100644 archive/eda_sdd_tdd_r1_archive/CHANGELOG.md create mode 100644 archive/eda_sdd_tdd_r1_archive/README.md rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/START_HERE.md (60%) create mode 100644 archive/eda_sdd_tdd_r1_archive/SUMMARY.md rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/adr/0001-build-on-episcout.md (100%) rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/adr/0002-specification-first-workflow.md (100%) rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/adr/0003-r-not-pure-base-r.md (100%) rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/adr/0004-synthetic-data-before-access.md (100%) rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/adr/0005-external-fixture-tdd.md (100%) rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/adr/README.md (100%) rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/codex/review-checklist.md (90%) rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/codex/revised-pr-plan-tdd-first.md (98%) rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/codex/tdd-first-codex-instructions.md (95%) create mode 100644 archive/eda_sdd_tdd_r1_archive/prompt_loop.md rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/repository-audit.md (100%) rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/roadmap.md (97%) rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/sdd/0001-spec-first-eda.md (100%) rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/sdd/data-dictionary-spec.md (100%) rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/sdd/mvp-scope.md (97%) rename {spec_driven_EDA_plan/docs => archive/eda_sdd_tdd_r1_archive}/sdd/tdd-external-fixtures.md (99%) create mode 100644 future/README.md create mode 100644 future/TODOs.md create mode 100644 future/changelog.md create mode 100644 future/decisions.md create mode 100644 future/prompts/refactor-review.md create mode 100644 future/prompts/senior-r-package-review.md create mode 100644 future/prompts/test-design-review.md create mode 100644 future/references/spec-driven-eda-archive.md create mode 100644 future/reviews/README.md create mode 100644 future/scratch/README.md create mode 100644 future/specs/001-phase-1-helper-stabilization/acceptance.md create mode 100644 future/specs/001-phase-1-helper-stabilization/brief.md create mode 100644 future/specs/001-phase-1-helper-stabilization/manifest.yml create mode 100644 future/specs/001-phase-1-helper-stabilization/review.md create mode 100644 future/specs/001-phase-1-helper-stabilization/sdd.md create mode 100644 future/specs/001-phase-1-helper-stabilization/tdd.md create mode 100644 future/specs/002-penguins-raw-fixture/acceptance.md create mode 100644 future/specs/002-penguins-raw-fixture/brief.md create mode 100644 future/specs/002-penguins-raw-fixture/manifest.yml create mode 100644 future/specs/002-penguins-raw-fixture/review.md create mode 100644 future/specs/002-penguins-raw-fixture/sdd.md create mode 100644 future/specs/002-penguins-raw-fixture/tdd.md create mode 100644 future/specs/003-large-data-backend-strategy/acceptance.md create mode 100644 future/specs/003-large-data-backend-strategy/brief.md create mode 100644 future/specs/003-large-data-backend-strategy/manifest.yml create mode 100644 future/specs/003-large-data-backend-strategy/review.md create mode 100644 future/specs/003-large-data-backend-strategy/sdd.md create mode 100644 future/specs/003-large-data-backend-strategy/tdd.md create mode 100644 future/specs/template/acceptance.md create mode 100644 future/specs/template/brief.md create mode 100644 future/specs/template/manifest.yml create mode 100644 future/specs/template/review.md create mode 100644 future/specs/template/sdd.md create mode 100644 future/specs/template/tdd.md delete mode 100644 spec_driven_EDA_plan/docs/AGENTS.md delete mode 100644 spec_driven_EDA_plan/future/codex-pr-add-penguins-raw-fixture.md delete mode 100644 spec_driven_EDA_plan/future/codex-pr-deferred-phase-1-stabilization.md delete mode 100644 spec_driven_EDA_plan/future/codex-review-checklist-external-truth-fixtures.md delete mode 100644 spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md delete mode 100644 spec_driven_EDA_plan/prompt_loop.md diff --git a/.Rbuildignore b/.Rbuildignore index f35d876..125fcff 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,3 +1,5 @@ +^archive$ +^archive/ ^future$ ^future/ ^codecov\.yml$ diff --git a/AGENTS.MD b/AGENTS.MD index ae2f435..c3d1123 100644 --- a/AGENTS.MD +++ b/AGENTS.MD @@ -3,6 +3,18 @@ This file defines conventions and instructions to guide OpenAI Codex and other agents in contributing to this R package. It standardises formatting, documentation, testing, and pull request workflows. +It should be respected by automated agents when: + +* Writing new functions (must include roxygen2 and tests) +* Refactoring existing code (preserve naming and indentation) +* Editing documentation (run `devtools::document()`) +* Opening a PR (follow title/description/branch rules) +* Running checks (use scripts provided above) + +Do not overcomplicate and over-engineer. Prefer the simpler solution. Make it easy for a human to review, supervise and monitor work. Do not bloat file content or create numerous files when one would suffice. e.g. use one `TODOs.md`, not a dir with `now.md`, `next.md`, `later.md`. + +--- + ## 📁 Project Structure | Folder | Purpose | @@ -223,15 +235,3 @@ act -j check # if act is installed for local testing ``` --- - -## Agent-Specific Instructions - -This AGENTS.md should be respected by OpenAI Codex or other automated agents when: - -* Writing new functions (must include roxygen2 and tests) -* Refactoring existing code (preserve naming and indentation) -* Editing documentation (run `devtools::document()`) -* Opening a PR (follow title/description/branch rules) -* Running checks (use scripts provided above) - ---- diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 0000000..03fa482 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,37 @@ +# episcout News + +## Development version + +- Added a specification-first EDA workflow driven by data dictionaries: + `epi_eda_spec()`, `epi_eda_check_schema()`, `epi_eda_profile_missing()`, + `epi_eda_profile_summaries()`, `epi_eda_profile_plots()`, + `epi_eda_generate_synthetic_data()`, `epi_eda_run()`, and + `epi_eda_render_report()`. +- Added `epi_eda_create_project()` and a reusable project scaffold under + `inst/project-template/`. +- Added fixture-backed tests for the EDA workflow using a pinned + `blood_storage` fixture. +- Added an HTML EDA report template. +- Documented current MVP limits: basic summaries and plots, synthetic data for + pipeline preparation only, and no Arrow, DuckDB or data.table EDA backends yet. +- Cleaned up Rd documentation and CRAN-check issues on the development branch. + +## 0.1.4 + +- Added `epi_plot_theme_imss` and colour palette helpers. +- Added `epi_plot_add_var_labels`. +- Reworked `epi_stats_*` summary functions. + +## 0.1.3 + +- Improved coverage tests. +- Added helper wrappers. +- Improved documentation. + +## 0.1.2 + +- Minor bug fixes and internal improvements. + +## 0.1.1 + +- First release. diff --git a/archive/eda_sdd_tdd_r1_archive/AGENTS.md b/archive/eda_sdd_tdd_r1_archive/AGENTS.md new file mode 100644 index 0000000..69eb13b --- /dev/null +++ b/archive/eda_sdd_tdd_r1_archive/AGENTS.md @@ -0,0 +1,38 @@ +# Archived AGENTS.md + +This file is historical guidance for the archived specification-first EDA +planning flow. Use root `AGENTS.MD` and `future/README.md` for new work. + +## Scope + +This repository is an R package. The specification-first EDA plan in this +directory is archived reference material. + +## Required reading + +For historical context, read: + +- archive/eda_sdd_tdd_r1_archive/START_HERE.md +- archive/eda_sdd_tdd_r1_archive/codex/tdd-first-codex-instructions.md +- archive/eda_sdd_tdd_r1_archive/codex/review-checklist.md + +Do not treat archived `START_HERE.md` as the live source of truth. + +## Working rules + +- Use the root `future/` workspace for active PR-sized tasks. +- Do not broaden scope. +- Do not perform unrelated refactoring. +- Respect must-edit and must-not-edit lists. +- Do not use episcout functions to generate expected fixture outputs. +- Use external fixtures and independently computed expected outputs. +- Do not update archived `START_HERE.md` for new work. + +## Required checks + +Run where practical: + +```bash +scripts/rscript_env_caller.R -e "devtools::document()" +scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" diff --git a/archive/eda_sdd_tdd_r1_archive/CHANGELOG.md b/archive/eda_sdd_tdd_r1_archive/CHANGELOG.md new file mode 100644 index 0000000..f5aff74 --- /dev/null +++ b/archive/eda_sdd_tdd_r1_archive/CHANGELOG.md @@ -0,0 +1,24 @@ +# Specification-first EDA Planning Changelog + +## 2026-06-15 + +- Archived the completed SDD/TDD planning docs under + `archive/eda_sdd_tdd_r1_archive/`. +- Moved remaining next-phase work into root-level `future/specs/`. +- Added root-level `future/` SDD/TDD workspace for future development. + +## Branch History Summary + +- Added the specification-first EDA SDD, ADRs, roadmap, TDD instructions and + review checklist. +- Added external fixture files and fixture-backed tests for the initial EDA + contracts. +- Implemented specification loading, validation, schema checks and missingness + profiling. +- Added synthetic-data tests and implemented synthetic data generation. +- Added summary and plot tests and implemented summary/plot profiling. +- Added `epi_eda_run()` workflow tests and implementation. +- Added report-rendering tests and `epi_eda_render_report()`. +- Added project-template tests, `inst/project-template/`, and + `epi_eda_create_project()`. +- Completed CRAN-check and Rd documentation cleanup on the branch. diff --git a/archive/eda_sdd_tdd_r1_archive/README.md b/archive/eda_sdd_tdd_r1_archive/README.md new file mode 100644 index 0000000..777b1e5 --- /dev/null +++ b/archive/eda_sdd_tdd_r1_archive/README.md @@ -0,0 +1,36 @@ +# Specification-first EDA Planning Archive + +This directory contains the archived SDD/TDD planning record for the +specification-first EDA work in `episcout`. + +The original live workflow has been closed. Historical design material now +lives under: + +```text +archive/eda_sdd_tdd_r1_archive/ +``` + +Use the root-level `future/` directory for next-phase planning: + +```text +future/README.md +future/backlog.md +future/specs/ +``` + +## What Is Archived Here + +- SDD and ADR files for the specification-first EDA layer. +- The original TDD-first PR sequence. +- Repository audit notes and review checklists. +- Historical Codex instruction blocks. +- Summary and changelog for the completed EDA planning flow. + +## What Moved To `future/` + +- Deferred Phase 1 helper stabilization. +- The proposed `penguins_raw` external fixture. +- Large-data backend strategy follow-up. + +Archived files are reference material. Do not treat archived `START_HERE.md` as +the current source of truth. diff --git a/spec_driven_EDA_plan/docs/START_HERE.md b/archive/eda_sdd_tdd_r1_archive/START_HERE.md similarity index 60% rename from spec_driven_EDA_plan/docs/START_HERE.md rename to archive/eda_sdd_tdd_r1_archive/START_HERE.md index 78d06d4..b9d7213 100644 --- a/spec_driven_EDA_plan/docs/START_HERE.md +++ b/archive/eda_sdd_tdd_r1_archive/START_HERE.md @@ -1,9 +1,23 @@ -# START HERE: episcout SDD/TDD development guide +# ARCHIVED: episcout SDD/TDD development guide -This is the live control panel for the specification-first EDA work in +This file is historical. The original specification-first EDA PR sequence has +been completed or moved into the root `future/` workspace for later work. + +Do not use this file as the live control panel for new work. Use: + +```text +future/README.md +future/backlog.md +future/specs/ +``` + +The archived SDD, ADR, roadmap and Codex instruction files remain useful as +design references. + +This was the live control panel for the specification-first EDA work in `episcout`. -Use this file first to see: +This file originally tracked: - current status; - the active PR-sized task; @@ -12,9 +26,9 @@ Use this file first to see: - expected test state; - required closeout updates. -Design rationale lives in the SDD and ADR files. The full PR sequence lives in -`spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md`. This file is the -operational source of truth for what happens next. +Design rationale lives in the SDD and ADR files. The full historical PR +sequence lives in +`archive/eda_sdd_tdd_r1_archive/codex/revised-pr-plan-tdd-first.md`. ## Current Status @@ -34,53 +48,49 @@ operational source of truth for what happens next. | PR 11 report rendering implementation | Done | `epi_eda_render_report()` and the bundled Quarto report template are implemented. | | PR 12 project-template contract tests | Done | Failing tests define the project template and `epi_eda_create_project()` contract. | | PR 13 project-template implementation | Done | `inst/project-template/` and `epi_eda_create_project()` are implemented. | -| PR 14 large-data design note | Active | Document large-data backend strategy without implementing backends. | - -## Active PR +| PR 14 large-data design note | Moved to future | See `future/specs/003-large-data-backend-strategy/`. | -```text -PR 14: Add large-data design note -``` +## Archived Active PR -Instruction: +There is no active PR in this archived workflow. The last open design thread +was moved to: ```text -Follow PR 14 in: -spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md +future/specs/003-large-data-backend-strategy/ ``` Required reading: ```text AGENTS.MD -spec_driven_EDA_plan/docs/START_HERE.md -spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md -spec_driven_EDA_plan/docs/sdd/data-dictionary-spec.md -spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md -spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md -spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md -spec_driven_EDA_plan/docs/codex/review-checklist.md +archive/eda_sdd_tdd_r1_archive/START_HERE.md +archive/eda_sdd_tdd_r1_archive/sdd/0001-spec-first-eda.md +archive/eda_sdd_tdd_r1_archive/sdd/data-dictionary-spec.md +archive/eda_sdd_tdd_r1_archive/sdd/tdd-external-fixtures.md +archive/eda_sdd_tdd_r1_archive/codex/tdd-first-codex-instructions.md +archive/eda_sdd_tdd_r1_archive/codex/revised-pr-plan-tdd-first.md +archive/eda_sdd_tdd_r1_archive/codex/review-checklist.md ``` -## PR 14 Scope +## Historical PR 14 Scope Must edit: ```text -spec_driven_EDA_plan/docs/sdd/large-data-backend-strategy.md -spec_driven_EDA_plan/docs/START_HERE.md +archive/eda_sdd_tdd_r1_archive/sdd/large-data-backend-strategy.md +archive/eda_sdd_tdd_r1_archive/START_HERE.md ``` May read: ```text -spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md -spec_driven_EDA_plan/docs/sdd/mvp-scope.md -spec_driven_EDA_plan/docs/adr/0003-r-not-pure-base-r.md -spec_driven_EDA_plan/docs/roadmap.md -spec_driven_EDA_plan/docs/repository-audit.md -spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md -spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md +archive/eda_sdd_tdd_r1_archive/sdd/0001-spec-first-eda.md +archive/eda_sdd_tdd_r1_archive/sdd/mvp-scope.md +archive/eda_sdd_tdd_r1_archive/adr/0003-r-not-pure-base-r.md +archive/eda_sdd_tdd_r1_archive/roadmap.md +archive/eda_sdd_tdd_r1_archive/repository-audit.md +archive/eda_sdd_tdd_r1_archive/codex/revised-pr-plan-tdd-first.md +archive/eda_sdd_tdd_r1_archive/codex/tdd-first-codex-instructions.md ``` Must not edit: @@ -107,9 +117,9 @@ PR 14 is documentation-only. No failing tests are expected, and no large-data backend implementation should be added. ``` -## Closeout Rule +## Historical Closeout Rule -Every PR must update this file before it is considered complete. +This rule applied while the archived workflow was active. At closeout: @@ -145,14 +155,14 @@ fixture data -> implementation ``` -## Source of Truth +## Historical References -- Live status and next action: `spec_driven_EDA_plan/docs/START_HERE.md` -- Executable Codex instructions: `spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md` -- Full PR sequence: `spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md` -- Design rationale: `spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md` -- Fixture TDD rationale: `spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md` -- Review checklist: `spec_driven_EDA_plan/docs/codex/review-checklist.md` +- Archived status and next action: `archive/eda_sdd_tdd_r1_archive/START_HERE.md` +- Executable Codex instructions: `archive/eda_sdd_tdd_r1_archive/codex/tdd-first-codex-instructions.md` +- Full PR sequence: `archive/eda_sdd_tdd_r1_archive/codex/revised-pr-plan-tdd-first.md` +- Design rationale: `archive/eda_sdd_tdd_r1_archive/sdd/0001-spec-first-eda.md` +- Fixture TDD rationale: `archive/eda_sdd_tdd_r1_archive/sdd/tdd-external-fixtures.md` +- Review checklist: `archive/eda_sdd_tdd_r1_archive/codex/review-checklist.md` ## Out of Scope Until Later PRs diff --git a/archive/eda_sdd_tdd_r1_archive/SUMMARY.md b/archive/eda_sdd_tdd_r1_archive/SUMMARY.md new file mode 100644 index 0000000..9730423 --- /dev/null +++ b/archive/eda_sdd_tdd_r1_archive/SUMMARY.md @@ -0,0 +1,50 @@ +# Specification-first EDA Work Summary + +## Outcome + +The branch introduced a specification-first EDA workflow for `episcout` while +preserving the existing lower-level `epi_*` helper API. + +The completed work added: + +- external fixture-backed tests using `medicaldata::blood_storage`; +- data dictionary loading and validation through `epi_eda_spec()`; +- schema checks with `epi_eda_check_schema()`; +- missingness summaries with `epi_eda_profile_missing()`; +- synthetic data generation with `epi_eda_generate_synthetic_data()`; +- variable summaries and plot dispatch through `epi_eda_profile_summaries()` and + `epi_eda_profile_plots()`; +- workflow orchestration through `epi_eda_run()`; +- optional report rendering through `epi_eda_render_report()`; +- a reusable project scaffold through `epi_eda_create_project()`; +- README, vignette, Rd and CRAN-check cleanup around the new workflow. + +## Design Pattern + +The work used this sequence: + +```text +external fixture and expected output first +failing fixture-backed tests second +minimal implementation third +``` + +Expected fixture outputs were intended to be independently computed, not +generated from the implementation under test. + +## Current Status + +The package-facing workflow is now described in `README.md` and the package +vignette. The archived planning files remain as rationale and review history. + +Remaining work has moved to root-level `future/specs/`: + +- `001-phase-1-helper-stabilization` +- `002-penguins-raw-fixture` +- `003-large-data-backend-strategy` + +## Limits Preserved + +The MVP intentionally keeps summaries and plots basic. Synthetic data are for +pipeline preparation and testing only. Large-data backends such as Arrow, +DuckDB and data.table remain future work. diff --git a/spec_driven_EDA_plan/docs/adr/0001-build-on-episcout.md b/archive/eda_sdd_tdd_r1_archive/adr/0001-build-on-episcout.md similarity index 100% rename from spec_driven_EDA_plan/docs/adr/0001-build-on-episcout.md rename to archive/eda_sdd_tdd_r1_archive/adr/0001-build-on-episcout.md diff --git a/spec_driven_EDA_plan/docs/adr/0002-specification-first-workflow.md b/archive/eda_sdd_tdd_r1_archive/adr/0002-specification-first-workflow.md similarity index 100% rename from spec_driven_EDA_plan/docs/adr/0002-specification-first-workflow.md rename to archive/eda_sdd_tdd_r1_archive/adr/0002-specification-first-workflow.md diff --git a/spec_driven_EDA_plan/docs/adr/0003-r-not-pure-base-r.md b/archive/eda_sdd_tdd_r1_archive/adr/0003-r-not-pure-base-r.md similarity index 100% rename from spec_driven_EDA_plan/docs/adr/0003-r-not-pure-base-r.md rename to archive/eda_sdd_tdd_r1_archive/adr/0003-r-not-pure-base-r.md diff --git a/spec_driven_EDA_plan/docs/adr/0004-synthetic-data-before-access.md b/archive/eda_sdd_tdd_r1_archive/adr/0004-synthetic-data-before-access.md similarity index 100% rename from spec_driven_EDA_plan/docs/adr/0004-synthetic-data-before-access.md rename to archive/eda_sdd_tdd_r1_archive/adr/0004-synthetic-data-before-access.md diff --git a/spec_driven_EDA_plan/docs/adr/0005-external-fixture-tdd.md b/archive/eda_sdd_tdd_r1_archive/adr/0005-external-fixture-tdd.md similarity index 100% rename from spec_driven_EDA_plan/docs/adr/0005-external-fixture-tdd.md rename to archive/eda_sdd_tdd_r1_archive/adr/0005-external-fixture-tdd.md diff --git a/spec_driven_EDA_plan/docs/adr/README.md b/archive/eda_sdd_tdd_r1_archive/adr/README.md similarity index 100% rename from spec_driven_EDA_plan/docs/adr/README.md rename to archive/eda_sdd_tdd_r1_archive/adr/README.md diff --git a/spec_driven_EDA_plan/docs/codex/review-checklist.md b/archive/eda_sdd_tdd_r1_archive/codex/review-checklist.md similarity index 90% rename from spec_driven_EDA_plan/docs/codex/review-checklist.md rename to archive/eda_sdd_tdd_r1_archive/codex/review-checklist.md index 2cbeac9..60060ce 100644 --- a/spec_driven_EDA_plan/docs/codex/review-checklist.md +++ b/archive/eda_sdd_tdd_r1_archive/codex/review-checklist.md @@ -60,7 +60,7 @@ Use this checklist for every PR. Run where practical: ```bash -Rscript -e "devtools::document()" -Rscript -e "devtools::test(reporter = 'summary')" -Rscript -e "devtools::check(manual = FALSE)" +scripts/rscript_env_caller.R -e "devtools::document()" +scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" ``` diff --git a/spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md b/archive/eda_sdd_tdd_r1_archive/codex/revised-pr-plan-tdd-first.md similarity index 98% rename from spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md rename to archive/eda_sdd_tdd_r1_archive/codex/revised-pr-plan-tdd-first.md index c9a45fd..169e897 100644 --- a/spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md +++ b/archive/eda_sdd_tdd_r1_archive/codex/revised-pr-plan-tdd-first.md @@ -5,7 +5,7 @@ This is the stable PR sequence for the specification-first EDA layer. This file is not the live status dashboard. Use -`spec_driven_EDA_plan/docs/START_HERE.md` for current status, the active PR, +`archive/eda_sdd_tdd_r1_archive/START_HERE.md` for current status, the active PR, allowed edits, expected test state and closeout requirements. The rule is: diff --git a/spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md b/archive/eda_sdd_tdd_r1_archive/codex/tdd-first-codex-instructions.md similarity index 95% rename from spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md rename to archive/eda_sdd_tdd_r1_archive/codex/tdd-first-codex-instructions.md index 783732c..a142b77 100644 --- a/spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md +++ b/archive/eda_sdd_tdd_r1_archive/codex/tdd-first-codex-instructions.md @@ -13,7 +13,7 @@ Keep the change small and reviewable. Do not perform broad refactoring. If you f ## Standing closeout rule Every instruction must end by updating -`spec_driven_EDA_plan/docs/START_HERE.md`. +`archive/eda_sdd_tdd_r1_archive/START_HERE.md`. At closeout: @@ -34,12 +34,12 @@ the PR description. ```text Read: - AGENTS.MD -- spec_driven_EDA_plan/docs/START_HERE.md -- spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md -- spec_driven_EDA_plan/docs/sdd/data-dictionary-spec.md -- spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md -- spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md -- spec_driven_EDA_plan/docs/codex/review-checklist.md +- archive/eda_sdd_tdd_r1_archive/START_HERE.md +- archive/eda_sdd_tdd_r1_archive/sdd/0001-spec-first-eda.md +- archive/eda_sdd_tdd_r1_archive/sdd/data-dictionary-spec.md +- archive/eda_sdd_tdd_r1_archive/sdd/tdd-external-fixtures.md +- archive/eda_sdd_tdd_r1_archive/codex/revised-pr-plan-tdd-first.md +- archive/eda_sdd_tdd_r1_archive/codex/review-checklist.md Task: Add external real-data test fixtures for specification-first EDA. @@ -282,7 +282,7 @@ Next steps: 4. Implement project template and epi_eda_create_project(). 5. Add large-data design note. -Follow spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md. +Follow archive/eda_sdd_tdd_r1_archive/codex/revised-pr-plan-tdd-first.md. Closeout: Update START_HERE.md after each remaining PR so it stays the live status diff --git a/archive/eda_sdd_tdd_r1_archive/prompt_loop.md b/archive/eda_sdd_tdd_r1_archive/prompt_loop.md new file mode 100644 index 0000000..bce6973 --- /dev/null +++ b/archive/eda_sdd_tdd_r1_archive/prompt_loop.md @@ -0,0 +1,30 @@ +# Archived Prompt Loop Notes + +This file used to hold reusable prompts for the original specification-first +EDA PR loop. That loop is now archived. + +Use these files for historical context: + +- `archive/eda_sdd_tdd_r1_archive/START_HERE.md` +- `archive/eda_sdd_tdd_r1_archive/codex/revised-pr-plan-tdd-first.md` +- `archive/eda_sdd_tdd_r1_archive/codex/tdd-first-codex-instructions.md` +- `archive/eda_sdd_tdd_r1_archive/codex/review-checklist.md` + +Use the root-level future workspace for new work: + +- `future/README.md` +- `future/backlog.md` +- `future/specs/` + +## Historical Pattern + +The archived PR loop used this sequence: + +```text +external fixture and expected output first +failing fixture-backed tests second +minimal implementation third +``` + +Future specs should preserve that pattern where fixture-backed behaviour is +being added or changed. diff --git a/spec_driven_EDA_plan/docs/repository-audit.md b/archive/eda_sdd_tdd_r1_archive/repository-audit.md similarity index 100% rename from spec_driven_EDA_plan/docs/repository-audit.md rename to archive/eda_sdd_tdd_r1_archive/repository-audit.md diff --git a/spec_driven_EDA_plan/docs/roadmap.md b/archive/eda_sdd_tdd_r1_archive/roadmap.md similarity index 97% rename from spec_driven_EDA_plan/docs/roadmap.md rename to archive/eda_sdd_tdd_r1_archive/roadmap.md index a927514..d1f9cde 100644 --- a/spec_driven_EDA_plan/docs/roadmap.md +++ b/archive/eda_sdd_tdd_r1_archive/roadmap.md @@ -4,7 +4,7 @@ Deliverables: -- `spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md` +- `archive/eda_sdd_tdd_r1_archive/sdd/0001-spec-first-eda.md` - ADR index - roadmap - Codex instructions diff --git a/spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md b/archive/eda_sdd_tdd_r1_archive/sdd/0001-spec-first-eda.md similarity index 100% rename from spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md rename to archive/eda_sdd_tdd_r1_archive/sdd/0001-spec-first-eda.md diff --git a/spec_driven_EDA_plan/docs/sdd/data-dictionary-spec.md b/archive/eda_sdd_tdd_r1_archive/sdd/data-dictionary-spec.md similarity index 100% rename from spec_driven_EDA_plan/docs/sdd/data-dictionary-spec.md rename to archive/eda_sdd_tdd_r1_archive/sdd/data-dictionary-spec.md diff --git a/spec_driven_EDA_plan/docs/sdd/mvp-scope.md b/archive/eda_sdd_tdd_r1_archive/sdd/mvp-scope.md similarity index 97% rename from spec_driven_EDA_plan/docs/sdd/mvp-scope.md rename to archive/eda_sdd_tdd_r1_archive/sdd/mvp-scope.md index bb7e9cb..066a939 100644 --- a/spec_driven_EDA_plan/docs/sdd/mvp-scope.md +++ b/archive/eda_sdd_tdd_r1_archive/sdd/mvp-scope.md @@ -117,7 +117,7 @@ outputs/ ## Development order -Use the TDD-first order in `spec_driven_EDA_plan/docs/codex/revised-pr-plan-tdd-first.md`. +Use the TDD-first order in `archive/eda_sdd_tdd_r1_archive/codex/revised-pr-plan-tdd-first.md`. In short: diff --git a/spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md b/archive/eda_sdd_tdd_r1_archive/sdd/tdd-external-fixtures.md similarity index 99% rename from spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md rename to archive/eda_sdd_tdd_r1_archive/sdd/tdd-external-fixtures.md index b936ce2..87dcabf 100644 --- a/spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md +++ b/archive/eda_sdd_tdd_r1_archive/sdd/tdd-external-fixtures.md @@ -372,7 +372,7 @@ All new code should test against the external fixture where practical. Recommended order: ```text -1. Add spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md. +1. Add archive/eda_sdd_tdd_r1_archive/sdd/tdd-external-fixtures.md. 2. Add fixture generation script. 3. Commit blood_storage CSV fixture and spec. 4. Commit independently computed expected schema and missingness outputs. diff --git a/future/README.md b/future/README.md new file mode 100644 index 0000000..1603ab1 --- /dev/null +++ b/future/README.md @@ -0,0 +1,51 @@ +# Future SDD/TDD Workspace + +This directory is the active planning workspace for next-phase `episcout` +development. It is committed to Git but excluded from R package builds by +`.Rbuildignore`. + +Use this workspace for internal design notes, TDD plans, review prompts and +agent handoff material. Keep executable package tests in `tests/testthat/`. + +## Directory Structure + +```text +future/ +├── README.md +├── backlog.md +├── decisions.md +├── changelog.md +├── prompts/ +├── references/ +├── reviews/ +├── specs/ +└── scratch/ +``` + +## Workflow + +1. Add candidate work to `backlog.md`. +2. Promote active work to a numbered directory under `specs/`. +3. Fill in `brief.md`, `sdd.md`, `tdd.md`, `acceptance.md` and + `manifest.yml` before implementation. +4. Implement executable tests under `tests/testthat/`. +5. Implement package code under `R/`, `inst/` or other package directories. +6. Record review notes in the spec `review.md`. +7. Update acceptance status and `changelog.md`. + +## R Command Policy + +Use the repo-local wrapper for R commands: + +```bash +scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" +``` + +Do not use bare `Rscript` in future specs or check instructions. + +## Active Specs + +- `001-phase-1-helper-stabilization` +- `002-penguins-raw-fixture` +- `003-large-data-backend-strategy` diff --git a/future/TODOs.md b/future/TODOs.md new file mode 100644 index 0000000..6dd6b9e --- /dev/null +++ b/future/TODOs.md @@ -0,0 +1,46 @@ +# TODOs + +Keep all future work here with a priority order. +Do not create additional files such as backlog.md or equivalent. +Centralise task list here for clarity and for easy human review. + +Use this template and place the task under the appropriate priority heading: + +```markdown +- Short title: + - Problem: + - Goal: + - User need: + - Proposed scope: + - Out of scope: + - Candidate files: + - Risks: + - Suggested spec ID: +``` + +For simple tasks only use e.g. `- [ ] xxx` + +## Priority 1 + +- [ ] Decide which future spec to activate first. +- [ ] For the activated spec, record baseline package test/check status before + package-code changes. +- [ ] Keep implementation work scoped to one numbered spec at a time. + + +## Priority 2 + +- [ ] Convert any new candidate work into a numbered spec before coding. +- [ ] Review fixture anti-circularity guardrails before adding new expected + outputs. +- [ ] Keep README and `NEWS.md` aligned with user-facing workflow changes. + + + +## Priority 3 + +- [ ] Revisit large-data backend implementation after the design spec is + accepted. +- [ ] Consider visual-regression strategy for EDA plots only after plot + contracts are stable. +- [ ] Add biomedical EDA extensions as separate numbered specs. diff --git a/future/changelog.md b/future/changelog.md new file mode 100644 index 0000000..2690d6c --- /dev/null +++ b/future/changelog.md @@ -0,0 +1,12 @@ +# Future Workspace Changelog + +## 2026-06-15 + +- Created root-level `future/` SDD/TDD workspace. +- Migrated deferred Phase 1 stabilization into + `specs/001-phase-1-helper-stabilization/`. +- Migrated the proposed `penguins_raw` fixture PR into + `specs/002-penguins-raw-fixture/`. +- Added large-data backend design follow-up under + `specs/003-large-data-backend-strategy/`. +- Added templates, prompts, todos, references and review folders. diff --git a/future/decisions.md b/future/decisions.md new file mode 100644 index 0000000..542968a --- /dev/null +++ b/future/decisions.md @@ -0,0 +1,24 @@ +# Decisions + +Record decisions that affect package scope, architecture, testing or review +standards. + +| Date | Decision | Rationale | Consequences | +|---|---|---|---| +| 2026-06-15 | Use root-level `future/` for next-phase planning. | `.Rbuildignore` already excludes `future/`, and it separates active planning from the archived SDD sequence. | Active SDD/TDD specs live outside `spec_driven_EDA_plan/`. | +| 2026-06-15 | Rename the completed SDD docs folder to `archive/eda_sdd_tdd_r1_archive/`. | The original control files are historical, not live instructions. | Archived files are references only. | +| 2026-06-15 | Keep executable tests under `tests/testthat/`. | Tests should run through normal R package tooling and CI. | `future/specs/*/tdd.md` defines test intent only. | +| 2026-06-15 | Use `scripts/rscript_env_caller.R` in future check commands. | The repo requires the project mamba R environment. | Future specs avoid bare `Rscript`. | + +## Decision Template + +```markdown +## YYYY-MM-DD - + +- Context: +- Options considered: +- Decision: +- Rationale: +- Consequences: +- Review date: +``` diff --git a/future/prompts/refactor-review.md b/future/prompts/refactor-review.md new file mode 100644 index 0000000..9049434 --- /dev/null +++ b/future/prompts/refactor-review.md @@ -0,0 +1,12 @@ +# Refactor Review Prompt + +Review this refactor for scope control and maintainability. + +Check: + +- exported function names and argument names remain stable unless explicitly + changed by the active spec; +- unrelated modernization is not included; +- helper extraction reduces real duplication or complexity; +- tests protect public behaviour before implementation changes; +- docs and examples still match behaviour. diff --git a/future/prompts/senior-r-package-review.md b/future/prompts/senior-r-package-review.md new file mode 100644 index 0000000..018d9e1 --- /dev/null +++ b/future/prompts/senior-r-package-review.md @@ -0,0 +1,115 @@ +# Senior R Package Review Prompt + +You are a senior R package reviewer with expertise in statistics, epidemiology/data science workflows, software engineering, and LLM-assisted coding. + +Your task is to review this repository as if preparing it for long-term human maintenance by a research/data science team. + +Focus on: + +- public API stability; +- CRAN check risk; +- dependency policy; +- backwards compatibility; +- test coverage for changed behaviour; +- clarity of errors and warnings; +- whether the change is scoped to the active spec. + +Lead with concrete findings and file references. Do not rewrite code. + +Context: +- The package was developed using SDD and TDD. +- Some or most code may have been generated or refactored by LLM agents. +- The goal is not only to check whether tests pass, but whether the package is statistically sound, readable, maintainable, idiomatic, and safe to extend. +- Treat the repository as a human-owned codebase, not an LLM artefact. + +Review priorities: + +1. Package purpose and scope +- Is the package purpose clear from README, DESCRIPTION, vignettes, and function names? +- Are public functions coherent and limited in scope? +- Are there signs of unnecessary abstraction, premature generalisation, duplicated logic, or “agentic sprawl”? + +2. Statistical and analytical correctness +- Check whether implemented summaries, tests, estimates, transformations, and plots are statistically appropriate. +- Identify silent assumptions, edge cases, inappropriate defaults, and unclear handling of missing data. +- Check whether outputs are reproducible and interpretable. +- Flag any method that looks plausible but is statistically weak, misleading, or insufficiently documented. + +3. R package quality +- Review DESCRIPTION, NAMESPACE, Roxygen documentation, examples, tests, vignettes, and package structure. +- Check dependency choices. Flag unnecessary, heavy, unstable, or poorly justified dependencies. +- Prefer simple, idiomatic R over over-engineered patterns. +- Check compatibility with CRAN-like checks where relevant. + +4. Code clarity and maintainability +- Is the code readable by a competent R developer who did not write it? +- Are functions small, named clearly, and organised logically? +- Are inputs validated with clear errors? +- Are side effects explicit? +- Are comments useful rather than noisy? +- Is there dead code, unused parameters, generic boilerplate, or suspicious LLM-style verbosity? + +5. TDD and test quality +- Check whether tests verify behaviour rather than implementation details. +- Identify brittle, redundant, superficial, or snapshot-heavy tests. +- Look for missing tests around edge cases, invalid inputs, missing data, small datasets, large datasets, grouped data, non-standard names, and numerical stability. +- Check that tests would catch realistic analytical errors. + +6. SDD alignment +- Compare specification/design documents with the implemented code. +- Identify drift between stated requirements, tests, and implementation. +- Flag requirements that were implemented without tests or tests that encode behaviour not present in the design. + +7. LLM-assisted coding risks +- Look for hallucinated functions, unused helper layers, invented terminology, excessive indirection, inconsistent style, duplicated implementations, and code that passes tests without solving the underlying problem. +- Check for defensive code that hides errors rather than exposing them. +- Identify places where the implementation is “technically working” but not something a human maintainer would want to own. + +8. Performance and scalability +- Check whether the package will behave reasonably on large epidemiological or biomedical datasets. +- Identify avoidable copies, rowwise operations, repeated expensive calculations, unnecessary conversions, and memory risks. +- Do not optimise prematurely; focus on clear bottlenecks or design choices that will block scale. + +9. User experience +- Are error messages actionable? +- Are defaults safe? +- Are returned objects predictable and documented? +- Are examples realistic? +- Can a new user understand the main workflow within 10 minutes? + +Output format, provide an md file placed in `future/reviews`: + +A. Executive summary +- Overall judgement: ready / mostly ready / needs focused revision / not ready. +- Main risks. +- Highest-value fixes. + +B. Findings table +Use columns: +- Area +- Severity: blocker / major / minor / note +- Location +- Issue +- Why it matters +- Suggested fix + +C. Test gaps +List missing or weak tests, grouped by function or module. + +D. Statistical concerns +Separate confirmed issues from assumptions needing clarification. + +E. Maintainability concerns +Focus on code humans will struggle to understand, trust, or extend. + +F. Suggested next commits +Give a short, ordered list of concrete commits. Each commit should be small, reviewable, and testable. + +Review rules: +- Be specific. Cite files, functions, and line numbers where possible. +- Do not rewrite the whole package. +- Do not add features unless they address a clear defect. +- Prefer deletion and simplification over abstraction. +- Distinguish confirmed defects from judgement calls. +- Do not assume passing tests mean the package is correct. +- Do not assume LLM-generated code is wrong; inspect it on its merits. diff --git a/future/prompts/test-design-review.md b/future/prompts/test-design-review.md new file mode 100644 index 0000000..672c378 --- /dev/null +++ b/future/prompts/test-design-review.md @@ -0,0 +1,12 @@ +# Test Design Review Prompt + +Review the test plan before implementation. + +Check: + +- tests define desired behaviour rather than current accidental behaviour; +- expected outputs are independently computed where fixtures are used; +- routine tests avoid internet access and optional source packages; +- edge cases and invalid inputs are covered; +- tests are placed under `tests/testthat/`; +- future spec files remain planning documents only. diff --git a/future/references/spec-driven-eda-archive.md b/future/references/spec-driven-eda-archive.md new file mode 100644 index 0000000..2f32cef --- /dev/null +++ b/future/references/spec-driven-eda-archive.md @@ -0,0 +1,19 @@ +# Spec-driven EDA Archive Reference + +The completed specification-first EDA planning record lives under: + +```text +archive/eda_sdd_tdd_r1_archive +``` + +Useful entry points: + +- `archive/eda_sdd_tdd_r1_archive/SUMMARY.md` +- `archive/eda_sdd_tdd_r1_archive/CHANGELOG.md` +- `archive/eda_sdd_tdd_r1_archive/sdd/0001-spec-first-eda.md` +- `archive/eda_sdd_tdd_r1_archive/sdd/tdd-external-fixtures.md` +- `archive/eda_sdd_tdd_r1_archive/codex/revised-pr-plan-tdd-first.md` +- `archive/eda_sdd_tdd_r1_archive/codex/review-checklist.md` + +Archived files are reference material. The active future planning workspace is +`future/`. diff --git a/future/reviews/README.md b/future/reviews/README.md new file mode 100644 index 0000000..8fcae99 --- /dev/null +++ b/future/reviews/README.md @@ -0,0 +1,5 @@ +# Reviews + +Use this directory for review notes that apply across multiple specs. + +Spec-specific reviews should live in each spec directory as `review.md`. diff --git a/future/scratch/README.md b/future/scratch/README.md new file mode 100644 index 0000000..e7e981c --- /dev/null +++ b/future/scratch/README.md @@ -0,0 +1,6 @@ +# Scratch + +Temporary notes can go here while shaping a future spec. + +Do not place executable tests, generated package outputs or large data files in +this directory. diff --git a/future/specs/001-phase-1-helper-stabilization/acceptance.md b/future/specs/001-phase-1-helper-stabilization/acceptance.md new file mode 100644 index 0000000..9868b8b --- /dev/null +++ b/future/specs/001-phase-1-helper-stabilization/acceptance.md @@ -0,0 +1,13 @@ +# Acceptance + +Spec ID: `001-phase-1-helper-stabilization` +Status: Ready for activation + +- [ ] Baseline local test/check status is recorded before behaviour changes. +- [ ] Failing tests are added before implementation. +- [ ] Selected edge-case tests pass after implementation. +- [ ] Public function names remain unchanged. +- [ ] Return objects remain data-frame-like and compatible with existing tests. +- [ ] Invalid input errors are clear and actionable. +- [ ] No spec-first EDA implementation files are changed. +- [ ] Pre-existing unrelated failures are documented rather than fixed. diff --git a/future/specs/001-phase-1-helper-stabilization/brief.md b/future/specs/001-phase-1-helper-stabilization/brief.md new file mode 100644 index 0000000..870291f --- /dev/null +++ b/future/specs/001-phase-1-helper-stabilization/brief.md @@ -0,0 +1,36 @@ +# Brief + +Spec ID: `001-phase-1-helper-stabilization` +Status: Ready for activation + +## Problem + +Some existing helper functions have under-specified edge-case behaviour. The +highest-risk examples are `epi_stats_numeric()` and `epi_stats_na_perc()`. + +## Goal + +Create a small TDD-first hardening PR that makes selected edge cases +deterministic without redesigning the public helper API. + +## Non-goals + +- Package-wide modernization. +- Renaming exported functions. +- Broad replacement of older tidyverse idioms. +- Changes to specification-first EDA functions. +- Changes to fixtures or report/project templates. + +## Candidate Files + +- `R/epi_stats_numeric.R` +- `R/epi_stats_na_perc.R` +- `tests/testthat/test-epi_stats_numeric.R` +- `tests/testthat/test-stats.R` + +## Activation Triggers + +- CI or local checks expose helper failures. +- Release preparation requires helper hardening. +- A future EDA implementation depends directly on these helpers. +- The user explicitly activates this spec. diff --git a/future/specs/001-phase-1-helper-stabilization/manifest.yml b/future/specs/001-phase-1-helper-stabilization/manifest.yml new file mode 100644 index 0000000..57ef561 --- /dev/null +++ b/future/specs/001-phase-1-helper-stabilization/manifest.yml @@ -0,0 +1,32 @@ +id: "001" +slug: "phase-1-helper-stabilization" +status: ready +owner: TBD +base_branch: episcout2 +working_branch: TBD +created: 2026-06-15 +updated: 2026-06-15 + +related_files: + - R/epi_stats_numeric.R + - R/epi_stats_na_perc.R + - tests/testthat/test-epi_stats_numeric.R + - tests/testthat/test-stats.R + +documents: + brief: brief.md + sdd: sdd.md + tdd: tdd.md + acceptance: acceptance.md + review: review.md + +review_required: true + +check_commands: + - scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" + - scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" + - scripts/rscript_env_caller.R -e "covr::package_coverage()" + +notes: + - Add tests before implementation. + - Do not broaden into package-wide modernization. diff --git a/future/specs/001-phase-1-helper-stabilization/review.md b/future/specs/001-phase-1-helper-stabilization/review.md new file mode 100644 index 0000000..9fa0edf --- /dev/null +++ b/future/specs/001-phase-1-helper-stabilization/review.md @@ -0,0 +1,16 @@ +# Review Notes + +Spec ID: `001-phase-1-helper-stabilization` +Status: Ready for activation + +## Review Focus + +- Does the PR preserve public helper API compatibility? +- Do tests define intended stable behaviour rather than accidental current + behaviour? +- Are changes limited to selected helper edge cases? +- Were baseline and final check results recorded? + +## Findings + +None yet. diff --git a/future/specs/001-phase-1-helper-stabilization/sdd.md b/future/specs/001-phase-1-helper-stabilization/sdd.md new file mode 100644 index 0000000..e14f106 --- /dev/null +++ b/future/specs/001-phase-1-helper-stabilization/sdd.md @@ -0,0 +1,35 @@ +# Software Design + +Spec ID: `001-phase-1-helper-stabilization` +Status: Ready for activation + +## Scope + +Harden `epi_stats_numeric()` and `epi_stats_na_perc()` while preserving existing +function names, argument names and data-frame-like return values where +practical. + +## Desired Behaviour + +- `epi_stats_numeric()` handles all-missing numeric vectors deterministically. +- `epi_stats_numeric()` handles zero-length numeric vectors deterministically. +- Coefficient of variation is stable when the mean is zero. +- Normality, skewness and kurtosis are skipped safely when usable values are + insufficient. +- `epi_stats_na_perc()` rejects invalid `margin` values with a clear error. +- `epi_stats_na_perc()` counts missing values in mixed-type data frames without + matrix coercion changing values. +- Row and column modes keep stable return shapes. + +## Compatibility + +Preserve backwards compatibility unless an old behaviour is clearly erroneous +and now covered by tests. Document intentional behaviour changes in tests and +roxygen comments. + +## Out Of Scope + +- Redesigning the `epi_stats_*` API. +- Updating every superseded tidyverse idiom. +- Editing `R/eda_*`, `R/run_eda.R` or report/project templates. +- Adding new dependencies. diff --git a/future/specs/001-phase-1-helper-stabilization/tdd.md b/future/specs/001-phase-1-helper-stabilization/tdd.md new file mode 100644 index 0000000..8bfb823 --- /dev/null +++ b/future/specs/001-phase-1-helper-stabilization/tdd.md @@ -0,0 +1,42 @@ +# Test Design + +Spec ID: `001-phase-1-helper-stabilization` +Status: Ready for activation + +## Baseline Commands + +Run before implementation where practical: + +```bash +scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" +scripts/rscript_env_caller.R -e "covr::package_coverage()" +``` + +## Test Files + +- `tests/testthat/test-epi_stats_numeric.R` +- `tests/testthat/test-stats.R` + +Use a narrower new test file only if it makes the edge-case contracts clearer. + +## Required Tests + +- [ ] `epi_stats_numeric()` handles all-missing numeric vectors. +- [ ] `epi_stats_numeric()` handles zero-length numeric vectors. +- [ ] `epi_stats_numeric()` returns stable coefficient-of-variation output when + mean is zero. +- [ ] `epi_stats_numeric()` handles insufficient usable values for normality, + skewness and kurtosis. +- [ ] `epi_stats_na_perc()` rejects invalid `margin` with a clear error. +- [ ] `epi_stats_na_perc()` handles mixed-type data frames without + coercion-related count errors. +- [ ] `epi_stats_na_perc()` keeps stable row-mode and column-mode return shapes. + +## Acceptance Commands + +```bash +scripts/rscript_env_caller.R -e "testthat::test_file('tests/testthat/test-epi_stats_numeric.R')" +scripts/rscript_env_caller.R -e "testthat::test_file('tests/testthat/test-stats.R')" +scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" +``` diff --git a/future/specs/002-penguins-raw-fixture/acceptance.md b/future/specs/002-penguins-raw-fixture/acceptance.md new file mode 100644 index 0000000..eb3f818 --- /dev/null +++ b/future/specs/002-penguins-raw-fixture/acceptance.md @@ -0,0 +1,15 @@ +# Acceptance + +Spec ID: `002-penguins-raw-fixture` +Status: Ready for activation + +- [ ] `penguins_raw` fixture files are committed. +- [ ] Fixture provenance is documented. +- [ ] Expected missingness, numeric summaries, categorical summaries and plot + inventory are committed. +- [ ] Expected outputs are generated without `episcout`. +- [ ] Routine tests do not require internet or `palmerpenguins`. +- [ ] No EDA implementation files are changed. +- [ ] Documentation explains why `blood_storage` and `penguins_raw` both exist. +- [ ] Review checklist includes anti-circularity checks. +- [ ] R package checks are not made worse. diff --git a/future/specs/002-penguins-raw-fixture/brief.md b/future/specs/002-penguins-raw-fixture/brief.md new file mode 100644 index 0000000..af771fb --- /dev/null +++ b/future/specs/002-penguins-raw-fixture/brief.md @@ -0,0 +1,29 @@ +# Brief + +Spec ID: `002-penguins-raw-fixture` +Status: Ready for activation + +## Problem + +The current EDA tests use `blood_storage`, which is useful for biomedical +schema and missingness testing but is not a broadly recognised standard EDA +fixture. + +## Goal + +Add `palmerpenguins::penguins_raw` as a pinned external truth fixture for +standard EDA behaviour. + +## Non-goals + +- Implementing new EDA functions. +- Replacing `blood_storage`. +- Adding `palmerpenguins` to `Imports`. +- Making routine tests require internet access or `palmerpenguins`. + +## Candidate Files + +- `data-raw/test-fixtures/make_penguins_raw_fixture.R` +- `tests/testthat/fixtures/penguins_raw/` +- `tests/testthat/fixtures/README.md` +- `tests/testthat/test-fixture-generation-guardrails.R` diff --git a/future/specs/002-penguins-raw-fixture/manifest.yml b/future/specs/002-penguins-raw-fixture/manifest.yml new file mode 100644 index 0000000..80b2f6e --- /dev/null +++ b/future/specs/002-penguins-raw-fixture/manifest.yml @@ -0,0 +1,31 @@ +id: "002" +slug: "penguins-raw-fixture" +status: ready +owner: TBD +base_branch: episcout2 +working_branch: TBD +created: 2026-06-15 +updated: 2026-06-15 + +related_files: + - data-raw/test-fixtures/make_penguins_raw_fixture.R + - tests/testthat/fixtures/penguins_raw/ + - tests/testthat/fixtures/README.md + - tests/testthat/test-fixture-generation-guardrails.R + +documents: + brief: brief.md + sdd: sdd.md + tdd: tdd.md + acceptance: acceptance.md + review: review.md + +review_required: true + +check_commands: + - scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" + - scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" + +notes: + - Do not add palmerpenguins to Imports. + - Routine tests must use committed fixture files. diff --git a/future/specs/002-penguins-raw-fixture/review.md b/future/specs/002-penguins-raw-fixture/review.md new file mode 100644 index 0000000..d1ee358 --- /dev/null +++ b/future/specs/002-penguins-raw-fixture/review.md @@ -0,0 +1,15 @@ +# Review Notes + +Spec ID: `002-penguins-raw-fixture` +Status: Ready for activation + +## Review Focus + +- Are expected outputs independent of implementation code? +- Are fixture files small, committed and deterministic? +- Do routine tests avoid internet access and optional source packages? +- Does the PR avoid implementation-file changes? + +## Findings + +None yet. diff --git a/future/specs/002-penguins-raw-fixture/sdd.md b/future/specs/002-penguins-raw-fixture/sdd.md new file mode 100644 index 0000000..ae8e93d --- /dev/null +++ b/future/specs/002-penguins-raw-fixture/sdd.md @@ -0,0 +1,46 @@ +# Software Design + +Spec ID: `002-penguins-raw-fixture` +Status: Ready for activation + +## Scope + +Add a second external fixture for standard EDA contracts using +`palmerpenguins::penguins_raw`. + +The fixture should complement `blood_storage`: + +- `blood_storage`: biomedical fixture for clinical schema, missingness and + workflow tests. +- `penguins_raw`: standard public EDA fixture for raw column names, mixed + variable types, missingness, summaries, plot-dispatch contracts and report + examples. + +## Required Fixture Files + +```text +tests/testthat/fixtures/penguins_raw/SOURCE.md +tests/testthat/fixtures/penguins_raw/penguins_raw.csv +tests/testthat/fixtures/penguins_raw/penguins_raw_spec.csv +tests/testthat/fixtures/penguins_raw/expected_missing.csv +tests/testthat/fixtures/penguins_raw/expected_summary_numeric.csv +tests/testthat/fixtures/penguins_raw/expected_summary_categorical.csv +tests/testthat/fixtures/penguins_raw/expected_plot_inventory.csv +``` + +## Regeneration Script + +Create: + +```text +data-raw/test-fixtures/make_penguins_raw_fixture.R +``` + +The script may require `palmerpenguins`, but routine tests must use committed +CSV files and must not require `palmerpenguins` or internet access. + +## Anti-circularity + +Expected outputs must be computed with base R, simple transparent code or +manual review. The regeneration script must not call `episcout`, `episcout::` +or any `epi_eda_*` function under test. diff --git a/future/specs/002-penguins-raw-fixture/tdd.md b/future/specs/002-penguins-raw-fixture/tdd.md new file mode 100644 index 0000000..fba6fa8 --- /dev/null +++ b/future/specs/002-penguins-raw-fixture/tdd.md @@ -0,0 +1,48 @@ +# Test Design + +Spec ID: `002-penguins-raw-fixture` +Status: Ready for activation + +## Test Files + +- `tests/testthat/test-fixture-generation-guardrails.R` if lightweight. +- Existing fixture-backed EDA tests may consume the committed files in later + implementation specs. + +## Fixture Expectations + +- [ ] `penguins_raw.csv` has 344 rows and 17 columns. +- [ ] `SOURCE.md` records source package, package version, row count, column + count, documentation URLs and regeneration command. +- [ ] `penguins_raw_spec.csv` is manually reviewed and uses source variable + names exactly as provided. +- [ ] `expected_missing.csv` is independently computed. +- [ ] `expected_summary_numeric.csv` is independently computed. +- [ ] `expected_summary_categorical.csv` is independently computed. +- [ ] `expected_plot_inventory.csv` records plot-dispatch expectations without + visual snapshots. + +## Guardrail Test + +If added, the guardrail test should scan fixture-generation scripts and fail on +forbidden calls: + +- `library(episcout)` +- `episcout::` +- `epi_eda_spec(` +- `epi_eda_validate_spec(` +- `epi_eda_check_schema(` +- `epi_eda_profile_missing(` +- `epi_eda_profile_summaries(` +- `epi_eda_profile_plots(` +- `epi_eda_run(` +- `epi_eda_generate_synthetic_data(` + +Do not run regeneration scripts from routine tests. + +## Acceptance Commands + +```bash +scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" +``` diff --git a/future/specs/003-large-data-backend-strategy/acceptance.md b/future/specs/003-large-data-backend-strategy/acceptance.md new file mode 100644 index 0000000..2fd8e30 --- /dev/null +++ b/future/specs/003-large-data-backend-strategy/acceptance.md @@ -0,0 +1,12 @@ +# Acceptance + +Spec ID: `003-large-data-backend-strategy` +Status: Draft + +- [ ] Backend strategy is documented before implementation. +- [ ] Existing data-frame/tibble workflow remains the baseline. +- [ ] Optional dependency policy is explicit. +- [ ] Plot sampling and full-summary expectations are documented. +- [ ] Cache/recompute policy is documented. +- [ ] No Arrow, DuckDB or data.table backend implementation is included in the + design PR. diff --git a/future/specs/003-large-data-backend-strategy/brief.md b/future/specs/003-large-data-backend-strategy/brief.md new file mode 100644 index 0000000..22ad9f5 --- /dev/null +++ b/future/specs/003-large-data-backend-strategy/brief.md @@ -0,0 +1,26 @@ +# Brief + +Spec ID: `003-large-data-backend-strategy` +Status: Draft + +## Problem + +The EDA MVP works with ordinary data frames and tibbles. The package describes +large-ish epidemiological data, but the new EDA workflow does not yet define how +larger backends should be introduced. + +## Goal + +Write a design-first plan for large-data backend support before implementation. + +## Non-goals + +- Implementing Arrow, DuckDB or data.table EDA backends in the design PR. +- Rewriting the existing EDA workflow. +- Adding heavy dependencies without an implementation spec. + +## Candidate Backends + +- data.table for fast in-memory aggregation. +- DuckDB for local SQL-style analysis. +- Arrow for multi-file Parquet datasets. diff --git a/future/specs/003-large-data-backend-strategy/manifest.yml b/future/specs/003-large-data-backend-strategy/manifest.yml new file mode 100644 index 0000000..e4b497c --- /dev/null +++ b/future/specs/003-large-data-backend-strategy/manifest.yml @@ -0,0 +1,29 @@ +id: "003" +slug: "large-data-backend-strategy" +status: draft +owner: TBD +base_branch: episcout2 +working_branch: TBD +created: 2026-06-15 +updated: 2026-06-15 + +related_files: + - README.md + - future/specs/003-large-data-backend-strategy/ + +documents: + brief: brief.md + sdd: sdd.md + tdd: tdd.md + acceptance: acceptance.md + review: review.md + +review_required: true + +check_commands: + - scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" + - scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" + +notes: + - Design-only first. + - Do not implement Arrow, DuckDB or data.table backends in this spec. diff --git a/future/specs/003-large-data-backend-strategy/review.md b/future/specs/003-large-data-backend-strategy/review.md new file mode 100644 index 0000000..403465d --- /dev/null +++ b/future/specs/003-large-data-backend-strategy/review.md @@ -0,0 +1,15 @@ +# Review Notes + +Spec ID: `003-large-data-backend-strategy` +Status: Draft + +## Review Focus + +- Does the design avoid premature backend implementation? +- Does it preserve current user-facing APIs? +- Are dependency and caching policies clear enough for a later implementation + spec? + +## Findings + +None yet. diff --git a/future/specs/003-large-data-backend-strategy/sdd.md b/future/specs/003-large-data-backend-strategy/sdd.md new file mode 100644 index 0000000..4b33e41 --- /dev/null +++ b/future/specs/003-large-data-backend-strategy/sdd.md @@ -0,0 +1,37 @@ +# Software Design + +Spec ID: `003-large-data-backend-strategy` +Status: Draft + +## Scope + +Define how the specification-first EDA workflow should later support larger +datasets while preserving the current data-frame/tibble API. + +## Design Direction + +- Keep data frames and tibbles as the baseline backend. +- Add backend support incrementally and behind stable user-facing functions. +- Compute summaries on full data where practical. +- Collect small, deterministic samples for plotting. +- Cache machine-readable outputs so reports do not recompute expensive steps. +- Keep optional backend dependencies optional unless implementation proves they + should move to `Imports`. + +## Candidate Sequence + +1. Document backend interface expectations. +2. Add data.table-backed summaries if profiling shows a need. +3. Add DuckDB support for local SQL-style analysis. +4. Add Arrow support for multi-file Parquet datasets. +5. Add cache invalidation and sampled plotting contracts. + +## Compatibility + +Existing calls to `epi_eda_run()`, `epi_eda_profile_missing()`, +`epi_eda_profile_summaries()` and `epi_eda_profile_plots()` should continue to +work on ordinary data frames. + +## Out Of Scope + +No backend implementation in the first design PR. diff --git a/future/specs/003-large-data-backend-strategy/tdd.md b/future/specs/003-large-data-backend-strategy/tdd.md new file mode 100644 index 0000000..5c4293f --- /dev/null +++ b/future/specs/003-large-data-backend-strategy/tdd.md @@ -0,0 +1,24 @@ +# Test Design + +Spec ID: `003-large-data-backend-strategy` +Status: Draft + +## Design-only First Step + +The first PR for this spec should be documentation-only. No executable tests are +required unless package-facing behaviour changes. + +## Future Test Categories + +- [ ] Backend selection preserves existing data-frame results. +- [ ] Summary calculations match fixture-backed data-frame expectations. +- [ ] Plot sampling is deterministic. +- [ ] Reports consume cached outputs rather than recomputing summaries. +- [ ] Optional backend packages are skipped cleanly when unavailable. + +## Future Acceptance Commands + +```bash +scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" +``` diff --git a/future/specs/template/acceptance.md b/future/specs/template/acceptance.md new file mode 100644 index 0000000..8ebc942 --- /dev/null +++ b/future/specs/template/acceptance.md @@ -0,0 +1,12 @@ +# Acceptance + +Spec ID: `` +Status: Draft + +- [ ] SDD is complete before implementation. +- [ ] TDD plan is complete before implementation. +- [ ] Executable tests are under `tests/testthat/`. +- [ ] Package code changes are scoped to this spec. +- [ ] Documentation is updated where user-facing behaviour changes. +- [ ] Checks listed in `manifest.yml` are run or explicitly deferred. +- [ ] Review notes are recorded in `review.md`. diff --git a/future/specs/template/brief.md b/future/specs/template/brief.md new file mode 100644 index 0000000..bf2d25c --- /dev/null +++ b/future/specs/template/brief.md @@ -0,0 +1,26 @@ +# Brief + +Spec ID: `` +Status: Draft +Owner: TBD + +## Problem + +TBD. + +## Goal + +TBD. + +## Non-goals + +- TBD. + +## Candidate Files + +- `R/.R` +- `tests/testthat/test-.R` + +## Risks + +- TBD. diff --git a/future/specs/template/manifest.yml b/future/specs/template/manifest.yml new file mode 100644 index 0000000..5e25b36 --- /dev/null +++ b/future/specs/template/manifest.yml @@ -0,0 +1,29 @@ +id: "" +slug: "" +status: draft +owner: TBD +base_branch: episcout2 +working_branch: TBD +created: 2026-06-15 +updated: 2026-06-15 + +related_files: + - R/.R + - tests/testthat/test-.R + +documents: + brief: brief.md + sdd: sdd.md + tdd: tdd.md + acceptance: acceptance.md + review: review.md + +review_required: true + +check_commands: + - scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" + - scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" + +notes: + - Keep executable tests under tests/testthat/. + - Keep this spec directory for planning and review only. diff --git a/future/specs/template/review.md b/future/specs/template/review.md new file mode 100644 index 0000000..a7e7b63 --- /dev/null +++ b/future/specs/template/review.md @@ -0,0 +1,16 @@ +# Review Notes + +Spec ID: `` +Status: Draft + +## Findings + +None yet. + +## Open Questions + +None yet. + +## Closeout Notes + +TBD. diff --git a/future/specs/template/sdd.md b/future/specs/template/sdd.md new file mode 100644 index 0000000..61dd556 --- /dev/null +++ b/future/specs/template/sdd.md @@ -0,0 +1,39 @@ +# Software Design + +Spec ID: `` +Status: Draft + +## Scope + +TBD. + +## Public API + +TBD. + +## Inputs And Outputs + +TBD. + +## Data Flow + +1. Validate inputs. +2. Compute or transform data. +3. Return stable, documented outputs. + +## Edge Cases + +- Empty data. +- Zero-row data. +- All-missing values. +- Unsupported classes. +- Non-syntactic names. + +## Errors And Warnings + +Errors should be clear, actionable and covered by tests. + +## Dependencies + +List any new or optional dependencies. Avoid adding dependencies unless the spec +requires them. diff --git a/future/specs/template/tdd.md b/future/specs/template/tdd.md new file mode 100644 index 0000000..59f2c63 --- /dev/null +++ b/future/specs/template/tdd.md @@ -0,0 +1,33 @@ +# Test Design + +Spec ID: `` +Status: Draft + +## Test Files + +- `tests/testthat/test-.R` + +## Baseline Commands + +```bash +scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" +``` + +## Behaviour Tests + +- [ ] TBD. + +## Edge-case Tests + +- [ ] TBD. + +## Failure Tests + +- [ ] TBD. + +## Acceptance Commands + +```bash +scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" +``` diff --git a/spec_driven_EDA_plan/docs/AGENTS.md b/spec_driven_EDA_plan/docs/AGENTS.md deleted file mode 100644 index 18c2a81..0000000 --- a/spec_driven_EDA_plan/docs/AGENTS.md +++ /dev/null @@ -1,38 +0,0 @@ -# AGENTS.md - -This file is specifically for work outside of the main branch. -It is for work contained only within this directory, `spec_driven_EDA_plan`. - -## Scope - -This repository is an R package. Follow the specification-first EDA plan. - -## Required reading - -Before editing files, read: - -- spec_driven_EDA_plan/docs/START_HERE.md -- spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md -- spec_driven_EDA_plan/docs/codex/review-checklist.md - -Treat START_HERE.md as the live source of truth. - -## Working rules - -- Complete only the active PR-sized task. -- Do not broaden scope. -- Do not perform unrelated refactoring. -- Respect must-edit and must-not-edit lists. -- Do not use episcout functions to generate expected fixture outputs. -- Use external fixtures and independently computed expected outputs. -- Update START_HERE.md at closeout. - -## Required checks - -Run where practical: - -```bash -Rscript -e "devtools::document()" -Rscript -e "devtools::test(reporter = 'summary')" -Rscript -e "devtools::check(manual = FALSE)" -R CMD check --no-manual . diff --git a/spec_driven_EDA_plan/future/codex-pr-add-penguins-raw-fixture.md b/spec_driven_EDA_plan/future/codex-pr-add-penguins-raw-fixture.md deleted file mode 100644 index dc88ec2..0000000 --- a/spec_driven_EDA_plan/future/codex-pr-add-penguins-raw-fixture.md +++ /dev/null @@ -1,458 +0,0 @@ -# Codex PR instruction: add `penguins_raw` as an external EDA truth fixture - -## Intended use - -Pass this file directly to Codex as the PR instruction. - -This PR should add a second external fixture for standard EDA validation, using `palmerpenguins::penguins_raw`. The aim is not to replace the existing `medicaldata::blood_storage` fixture. The aim is to complement it with a widely known public EDA dataset that has stable external documentation, common examples, mixed variable types, raw column names, and missingness. - -## Repository context - -Repository: - -```text -antoniojbt/episcout -``` - -Target branch: - -```text -episcout2 -``` - -Existing design principle: - -```text -external fixture and expected output first -failing fixture-backed tests second -minimal implementation third -``` - -Keep that sequence intact. - -The existing `blood_storage` fixture is appropriate for early biomedical EDA tests. It should remain the primary biomedical fixture for schema, missingness, and clinical-data realism. The new `penguins_raw` fixture should be added as a second external truth fixture for broader standard EDA behaviour, especially summaries, missingness, categorical handling, dates, labels, plots, and report examples. - -## Why this PR is needed - -The current `blood_storage` fixture is useful because it is real biomedical data, small, public, and already has committed expected schema and missingness outputs. It is less ideal as the only standard EDA truth fixture because it is not as widely used for canonical EDA examples as `palmerpenguins`. - -`palmerpenguins::penguins_raw` is useful because it provides: - -- a well-known public dataset; -- external package documentation and examples; -- 344 rows and 17 variables; -- raw-style column names, including spaces and punctuation; -- mixed variable types; -- dates; -- character/categorical variables; -- numeric morphometric variables; -- real missingness; -- common scatterplot, histogram, boxplot, grouped-summary and faceting examples. - -Use source documentation from: - -```text -https://allisonhorst.github.io/palmerpenguins/reference/penguins_raw.html -https://allisonhorst.github.io/palmerpenguins/articles/examples.html -``` - -## PR goal - -Create a small, reviewable PR that adds `penguins_raw` as a pinned external fixture and adds anti-circularity guardrails so future agent-generated implementation code cannot simply validate itself. - -Do not implement new EDA functions in this PR. - -## Scope - -### Must create - -```text -data-raw/test-fixtures/make_penguins_raw_fixture.R -tests/testthat/fixtures/penguins_raw/SOURCE.md -tests/testthat/fixtures/penguins_raw/penguins_raw.csv -tests/testthat/fixtures/penguins_raw/penguins_raw_spec.csv -tests/testthat/fixtures/penguins_raw/expected_missing.csv -tests/testthat/fixtures/penguins_raw/expected_summary_numeric.csv -tests/testthat/fixtures/penguins_raw/expected_summary_categorical.csv -tests/testthat/fixtures/penguins_raw/expected_plot_inventory.csv -``` - -### Should update - -```text -spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md -spec_driven_EDA_plan/docs/codex/review-checklist.md -tests/testthat/fixtures/README.md -``` - -### May create - -```text -tests/testthat/test-fixture-generation-guardrails.R -``` - -Only create this test if it is lightweight and does not run fixture-regeneration scripts. - -### Must not edit - -Do not edit implementation files for this PR: - -```text -R/epi_eda_spec.R -R/eda_schema.R -R/eda_missing.R -R/eda_summaries.R -R/eda_plots.R -R/epi_eda_run.R -``` - -Do not add or modify package behaviour. - -Do not add `palmerpenguins` to `Imports`. - -Do not make routine tests require internet access. - -Do not make routine tests require `palmerpenguins` to be installed. - -## Dependency policy - -`palmerpenguins` may be used by the manual regeneration script only. - -The committed fixture CSV files must be used by routine tests. - -The regeneration script must fail clearly if `palmerpenguins` is not installed: - -```r -if (!requireNamespace("palmerpenguins", quietly = TRUE)) { - stop( - "Package 'palmerpenguins' is required to regenerate penguins_raw fixtures. ", - "Install it with install.packages('palmerpenguins').", - call. = FALSE - ) -} -``` - -## Fixture-generation requirements - -The regeneration script must: - -1. Load `palmerpenguins::penguins_raw`. -2. Write `penguins_raw.csv` to `tests/testthat/fixtures/penguins_raw/`. -3. Write `SOURCE.md` with provenance. -4. Write a manually reviewed `penguins_raw_spec.csv`. -5. Write expected outputs using only base R or simple transparent calculations. -6. Avoid all `episcout` calls. -7. Avoid all new EDA function calls. -8. Avoid internet access. -9. Be manually run only; do not call it from routine tests. - -The script must not call any of these: - -```text -library(episcout) -episcout:: -epi_eda_spec( -epi_eda_validate_spec( -epi_eda_check_schema( -epi_eda_profile_missing( -epi_eda_profile_summaries( -epi_eda_profile_plots( -epi_eda_run( -epi_eda_generate_synthetic_data( -``` - -## Expected outputs - -### `expected_missing.csv` - -Compute independently with base R: - -```r -expected_missing <- data.frame( - name = names(penguins_raw), - n = nrow(penguins_raw), - n_missing = vapply(penguins_raw, function(x) sum(is.na(x)), integer(1)), - p_missing = vapply(penguins_raw, function(x) mean(is.na(x)), numeric(1)), - stringsAsFactors = FALSE -) -``` - -### `expected_summary_numeric.csv` - -For numeric and integer variables, include at least: - -```text -name -class -n -n_missing -mean -sd -median -q25 -q75 -min -max -``` - -Use base R functions only: - -```r -numeric_cols <- names(penguins_raw)[vapply(penguins_raw, is.numeric, logical(1))] - -summarise_numeric <- function(x, name) { - non_missing <- x[!is.na(x)] - data.frame( - name = name, - class = class(x)[1], - n = length(x), - n_missing = sum(is.na(x)), - mean = mean(non_missing), - sd = stats::sd(non_missing), - median = stats::median(non_missing), - q25 = unname(stats::quantile(non_missing, 0.25, names = FALSE)), - q75 = unname(stats::quantile(non_missing, 0.75, names = FALSE)), - min = min(non_missing), - max = max(non_missing), - stringsAsFactors = FALSE - ) -} - -expected_summary_numeric <- do.call( - rbind, - Map(function(nm) summarise_numeric(penguins_raw[[nm]], nm), numeric_cols) -) -``` - -### `expected_summary_categorical.csv` - -For character, factor, logical and non-numeric categorical variables, include at least: - -```text -name -level -n -p -n_missing -``` - -Use base R only. Missing values should be counted separately or documented clearly. - -### `expected_plot_inventory.csv` - -This is not a visual snapshot. It is a plain expected contract for later plot-dispatch tests. - -Include at least: - -```text -name -expected_plot_type -reason -``` - -Suggested values: - -- numeric/integer continuous variables: `histogram` or `density`; -- categorical variables: `bar`; -- date variables: `time` or `skip` with a reason; -- free-text/comment variables: `skip` with a reason; -- unsupported variables: `skip` with a reason. - -Do not test visual appearance in this PR. - -## `penguins_raw_spec.csv` guidance - -Create a manually reviewed fixture data dictionary. Use the existing spec format already used by `blood_storage_spec.csv` where practical: - -```text -name -label -type -role -units -levels -min -max -missing_codes -required -group -description -``` - -Use source variable names exactly as they appear in `penguins_raw`. Do not silently clean column names in the fixture. - -Recommended type mapping: - -- character identifiers and labels: `categorical` or `text`, depending intended EDA handling; -- sex: `categorical`; -- island/species/region/study/stage: `categorical`; -- clutch completion: `binary` or `categorical`, but document the chosen convention; -- dates: `date`; -- body size measurements: `numeric`; -- egg count: `integer`; -- comments: `text`. - -The spec should be manually reviewed. Do not infer it entirely from the future implementation. - -## `SOURCE.md` minimum contents - -Include: - -```markdown -# penguins_raw fixture provenance - -## Source - -- Dataset: `penguins_raw` -- Source package: `palmerpenguins` -- Source package version used for fixture generation: -- Observations: 344 -- Variables: 17 -- Dataset type: ecological observational dataset -- Purpose: standard EDA truth fixture for specification-first EDA tests - -## Source documentation - -- https://allisonhorst.github.io/palmerpenguins/reference/penguins_raw.html -- https://allisonhorst.github.io/palmerpenguins/articles/examples.html - -## Fixture files - -- `penguins_raw.csv`: pinned data exported from `palmerpenguins::penguins_raw`. -- `penguins_raw_spec.csv`: manually reviewed fixture data dictionary. -- `expected_missing.csv`: independently computed expected missingness result. -- `expected_summary_numeric.csv`: independently computed numeric summaries. -- `expected_summary_categorical.csv`: independently computed categorical summaries. -- `expected_plot_inventory.csv`: independently defined plot-dispatch expectation. - -## Regeneration - -Run from the repository root: - -```sh -Rscript data-raw/test-fixtures/make_penguins_raw_fixture.R -``` - -The script computes expected outputs with base R and does not call `episcout` EDA functions. -``` - -## Documentation updates - -Update `spec_driven_EDA_plan/docs/sdd/tdd-external-fixtures.md` to state: - -- `blood_storage` remains the first biomedical fixture. -- `scurvy` remains useful for tiny small-n edge-case tests. -- `penguins_raw` is added as the second standard EDA truth fixture. -- `penguins_raw` is intended for summary, missingness, plot-dispatch and report examples. -- `penguins_raw` should not displace `blood_storage` for biomedical tests. -- `penguins_raw` should not be used to justify implementation by matching implementation-generated output. - -Update `spec_driven_EDA_plan/docs/codex/review-checklist.md` to add explicit anti-circularity checks: - -```markdown -## External truth fixtures - -- [ ] Expected outputs were committed before implementation code that consumes them. -- [ ] Expected outputs were computed without calling `episcout` functions under test. -- [ ] Regeneration scripts do not call `episcout::`, `library(episcout)`, or new EDA functions. -- [ ] Fixture `SOURCE.md` records source package, version, row count, column count and source documentation. -- [ ] Routine tests use committed fixture files, not live downloads. -- [ ] Plot tests check contracts first, not visual appearance, unless a later visual-regression test is explicitly introduced. -``` - -Update `tests/testthat/fixtures/README.md` to include `penguins_raw` and explain the intended distinction: - -```text -blood_storage: biomedical fixture for clinical schema, missingness and workflow tests. -penguins_raw: standard public EDA fixture for mixed raw columns, common summaries, missingness and plot-dispatch contracts. -``` - -## Optional guardrail test - -If adding `tests/testthat/test-fixture-generation-guardrails.R`, keep it simple and deterministic. It should scan regeneration scripts and fail if they call package code under test. - -Example: - -```r -test_that("fixture regeneration scripts do not call episcout EDA functions", { - scripts <- c( - "data-raw/test-fixtures/make_external_fixtures.R", - "data-raw/test-fixtures/make_penguins_raw_fixture.R" - ) - scripts <- scripts[file.exists(scripts)] - - text <- unlist(lapply(scripts, readLines, warn = FALSE), use.names = FALSE) - - forbidden <- c( - "library\\(episcout\\)", - "episcout::", - "epi_eda_spec\\(", - "epi_eda_validate_spec\\(", - "epi_eda_check_schema\\(", - "epi_eda_profile_missing\\(", - "epi_eda_profile_summaries\\(", - "epi_eda_profile_plots\\(", - "epi_eda_run\\(", - "epi_eda_generate_synthetic_data\\(" - ) - - for (pattern in forbidden) { - expect_false( - any(grepl(pattern, text)), - info = paste("Forbidden fixture-generation dependency found:", pattern) - ) - } -}) -``` - -Do not run regeneration scripts in this test. - -## Acceptance criteria - -The PR is acceptable if: - -1. `penguins_raw` fixture files are committed. -2. Fixture provenance is documented. -3. Expected missingness, numeric summaries, categorical summaries and plot inventory are committed. -4. Expected outputs are generated without `episcout`. -5. Routine tests do not require internet or `palmerpenguins`. -6. No EDA implementation files are changed. -7. Documentation clearly explains why `blood_storage` and `penguins_raw` both exist. -8. Review checklist includes anti-circularity checks. -9. `R CMD check` is not made worse by this PR. - -## PR title - -```text -Add penguins_raw external EDA truth fixture -``` - -## PR description template - -```markdown -## Summary - -Adds `palmerpenguins::penguins_raw` as a pinned external EDA truth fixture for specification-first EDA tests. - -This complements the existing `medicaldata::blood_storage` biomedical fixture. `blood_storage` remains the primary clinical fixture; `penguins_raw` is added for standard public EDA behaviour, including mixed raw variable names, missingness, numeric summaries, categorical summaries and plot-dispatch contracts. - -## What changed - -- Added a manual regeneration script for `penguins_raw` fixtures. -- Added pinned `penguins_raw` CSV fixture. -- Added manually reviewed `penguins_raw_spec.csv`. -- Added independently computed expected missingness, numeric summaries and categorical summaries. -- Added a plot-dispatch inventory for later plot tests. -- Updated fixture documentation and review guardrails. - -## What did not change - -- No new EDA implementation code. -- No changes to public package behaviour. -- No runtime dependency on `palmerpenguins`. -- No internet access required by routine tests. - -## Anti-circularity - -Expected outputs were computed with base R or manually reviewed contracts. The regeneration script does not call `episcout` EDA functions or implementation code under test. -``` diff --git a/spec_driven_EDA_plan/future/codex-pr-deferred-phase-1-stabilization.md b/spec_driven_EDA_plan/future/codex-pr-deferred-phase-1-stabilization.md deleted file mode 100644 index 590c963..0000000 --- a/spec_driven_EDA_plan/future/codex-pr-deferred-phase-1-stabilization.md +++ /dev/null @@ -1,262 +0,0 @@ -# Codex PR instruction: deferred Phase 1 stabilization - -## Intended use - -Pass this file directly to Codex only when you want to activate the deferred -Phase 1 stabilization work. - -This instruction is not part of the live PR sequence. Do not run it unless the -user explicitly asks to do deferred Phase 1 stabilization. - -## Repository context - -Repository: - -```text -antoniojbt/episcout -``` - -Target branch: - -```text -episcout2 -``` - -Current live workflow: - -```text -spec_driven_EDA_plan/docs/START_HERE.md -``` - -The live workflow remains the source of truth for the active spec-first PR -sequence. This deferred Phase 1 PR should not change the active PR unless the -user explicitly requests that. - -## When to use this - -Use this instruction when one of these triggers is present: - -- `epi_eda_profile_summaries()` depends on `epi_stats_numeric()` or - `epi_stats_summary()`. -- `epi_eda_run()` depends on older `epi_*` helpers. -- CI, `R CMD check`, or local tests expose helper failures. -- A release or broader package hardening pass is being prepared. -- The user explicitly asks to activate deferred Phase 1 stabilization. - -If none of these triggers is present, continue the active PR-sized task in -`START_HERE.md` instead. - -## Required reading - -Read these files before making changes: - -```text -AGENTS.MD -spec_driven_EDA_plan/docs/START_HERE.md -spec_driven_EDA_plan/docs/roadmap.md -spec_driven_EDA_plan/docs/repository-audit.md -spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md -spec_driven_EDA_plan/docs/codex/review-checklist.md -``` - -## PR goal - -Create a small TDD-first hardening PR for the highest-risk existing helper edge -cases without changing the current spec-first EDA PR sequence. - -The first deferred Phase 1 PR should focus on: - -```text -epi_stats_numeric() -epi_stats_na_perc() -baseline local check/test status -``` - -Do not broaden the PR into package-wide modernization. - -## Scope - -This PR should: - -- record baseline local test/check status before behaviour changes; -- add failing tests for selected helper edge cases; -- implement only the helper changes needed to pass those tests; -- preserve backwards compatibility where practical; -- document pre-existing or unrelated failures rather than fixing them. - -## Must edit - -Edit only the files needed for the selected helper tests and implementation. - -Expected files for the first deferred Phase 1 PR: - -```text -tests/testthat/test-epi_stats_numeric.R -tests/testthat/test-stats.R -R/epi_stats_numeric.R -R/epi_stats_na_perc.R -``` - -If a narrower test file is clearer, create or edit a focused test file under: - -```text -tests/testthat/ -``` - -## May edit - -Only if required by roxygen/documentation updates: - -```text -man/ -NAMESPACE -``` - -Only if the PR needs to record the deferred Phase 1 closeout: - -```text -spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md -``` - -## Must not edit - -Do not edit current spec-first implementation files unless a failing test proves -that the deferred hardening PR directly requires it: - -```text -R/epi_eda_spec.R -R/eda_schema.R -R/eda_missing.R -R/eda_synthetic.R -R/eda_summaries.R -R/eda_plots.R -R/epi_eda_run.R -R/eda_report.R -``` - -Do not edit fixture files: - -```text -tests/testthat/fixtures/ -``` - -Do not edit report or project templates: - -```text -inst/report-template/ -inst/project-template/ -``` - -Do not edit active PR task files from: - -```text -spec_driven_EDA_plan/docs/START_HERE.md -``` - -unless the user explicitly asks to change the live workflow. - -## TDD requirements - -Add tests before implementation for the selected edge cases. - -Initial test targets: - -- `epi_stats_numeric()` handles all-missing numeric vectors. -- `epi_stats_numeric()` handles zero-length numeric vectors. -- `epi_stats_numeric()` returns a stable coefficient of variation when mean is - zero. -- `epi_stats_numeric()` skips normality, skewness, and kurtosis safely when - there are insufficient usable values. -- `epi_stats_na_perc()` rejects invalid `margin` with a clear error. -- `epi_stats_na_perc()` handles mixed-type data frames without coercion-related - count errors. -- `epi_stats_na_perc()` keeps a stable return shape for row and column modes. - -Tests should define the desired contract explicitly. Do not write tests that -only preserve accidental current behaviour. - -## Baseline commands - -Run these before implementation where practical: - -```bash -Rscript -e "devtools::test(reporter = 'summary')" -Rscript -e "devtools::check(manual = FALSE)" -Rscript -e "covr::package_coverage()" -``` - -If required packages are missing locally, do not install them unless the user -approves. Record the missing dependency and continue with focused tests that can -run in the available environment. - -Run focused tests after adding failing tests and after implementation. Examples: - -```bash -Rscript -e "testthat::test_file('tests/testthat/test-epi_stats_numeric.R')" -Rscript -e "testthat::test_file('tests/testthat/test-stats.R')" -``` - -Run broader checks after implementation where practical: - -```bash -Rscript -e "devtools::document()" -Rscript -e "devtools::test(reporter = 'summary')" -Rscript -e "devtools::check(manual = FALSE)" -``` - -## Implementation constraints - -Keep the change small and reviewable. - -Preserve: - -- exported function names; -- argument names where practical; -- data-frame-like return values; -- existing successful tests; -- backwards compatibility unless an old behaviour is clearly erroneous and now - covered by tests. - -Avoid: - -- package-wide refactoring; -- broad `select_if()` modernization; -- adding heavy dependencies; -- changing the spec-first EDA API; -- changing fixtures or expected fixture outputs; -- fixing unrelated audit findings in the same PR. - -## Acceptance criteria - -The PR is ready when: - -- selected edge-case tests fail before implementation and pass after; -- focused helper tests pass; -- broader package tests/checks are run where practical; -- any missing local dependencies or pre-existing failures are recorded; -- no files outside the deferred Phase 1 scope are changed; -- the current `START_HERE.md` active PR is not changed unless explicitly - requested. - -## Closeout - -Summarise: - -- why deferred Phase 1 was activated; -- files changed; -- tests added; -- helper behaviours hardened; -- checks run and results; -- expected or pre-existing failures; -- any follow-up Phase 1 work left out of scope. - -Do not mark Phase 1 as globally complete unless all roadmap Phase 1 goals are -actually satisfied: - -```text -run R CMD check -run test suite -record baseline coverage -identify fragile functions -fix clear edge cases -``` diff --git a/spec_driven_EDA_plan/future/codex-review-checklist-external-truth-fixtures.md b/spec_driven_EDA_plan/future/codex-review-checklist-external-truth-fixtures.md deleted file mode 100644 index 6eeec9a..0000000 --- a/spec_driven_EDA_plan/future/codex-review-checklist-external-truth-fixtures.md +++ /dev/null @@ -1,149 +0,0 @@ -# Review checklist: external truth fixtures for agent-generated EDA code - -Use this checklist when reviewing PRs that add or consume EDA fixture data. - -## Core rule - -The validation chain must be: - -```text -external fixture data -+ manually reviewed fixture specification -+ independently computed expected output --> failing tests --> implementation -``` - -Do not accept the inverse pattern: - -```text -implementation --> generated output --> tests that compare future output to that same implementation -``` - -That is circular validation. - -## Fixture source - -- [ ] The fixture comes from a public, documented source. -- [ ] The source is appropriate for the test purpose. -- [ ] The fixture is small enough for routine unit tests. -- [ ] The fixture is pinned as committed files under `tests/testthat/fixtures/`. -- [ ] Routine tests do not download live data. -- [ ] Routine tests do not require internet access. -- [ ] Routine tests do not require optional source packages to be installed. - -## Provenance - -Each fixture directory should include `SOURCE.md` with: - -- [ ] Dataset name. -- [ ] Source package or source institution. -- [ ] Source package version, if applicable. -- [ ] Source documentation URL. -- [ ] Row count. -- [ ] Column count. -- [ ] Dataset type and purpose. -- [ ] Regeneration script path. -- [ ] Statement that expected outputs were not computed with `episcout` functions under test. - -## Expected outputs - -- [ ] Expected files are committed beside the fixture data. -- [ ] Expected outputs are readable CSV or another simple, reviewable format. -- [ ] Expected outputs were generated before implementation code that consumes them. -- [ ] Expected outputs were computed with base R, simple transparent code, or a clearly independent reference. -- [ ] Expected outputs do not call package functions under test. -- [ ] Numeric summaries specify definitions for `sd`, quantiles and missing-value handling. -- [ ] Categorical summaries specify how missing values are counted. -- [ ] Plot expectations test contract and dispatch before visual appearance. - -## Regeneration scripts - -- [ ] Regeneration scripts are manual utilities, not routine test dependencies. -- [ ] Scripts fail clearly if optional source packages are unavailable. -- [ ] Scripts do not call `library(episcout)`. -- [ ] Scripts do not call `episcout::`. -- [ ] Scripts do not call EDA implementation functions under test. -- [ ] Scripts use deterministic output ordering. -- [ ] Scripts avoid hidden randomness; any randomness has a fixed seed. - -Forbidden calls in fixture-generation scripts: - -```text -library(episcout) -episcout:: -epi_eda_spec( -epi_eda_validate_spec( -epi_eda_check_schema( -epi_eda_profile_missing( -epi_eda_profile_summaries( -epi_eda_profile_plots( -epi_eda_run( -epi_eda_generate_synthetic_data( -``` - -## Dataset roles - -Use fixtures for distinct purposes: - -```text -blood_storage -``` - -Biomedical fixture. Best for clinical schema, missingness, outcome/exposure roles, survival-style variables and realistic epidemiological data. - -```text -penguins_raw -``` - -Standard public EDA fixture. Best for raw column names, mixed variable types, missingness, grouped numeric summaries, categorical summaries, dates, common plots and report examples. - -```text -scurvy -``` - -Tiny fixture. Best for small-n edge cases and categorical handling. - -Do not expect one fixture to cover all behaviour. - -## PR review questions - -Before merging, answer: - -1. Does this PR add external truth, or does it merely preserve current implementation behaviour? -2. Could an agent have generated both the code and expected output from the same implementation path? -3. Are fixture outputs understandable by inspection? -4. Are there at least a few hard-coded sentinel values that can be checked manually? -5. Does the PR avoid broad refactoring? -6. Does the PR avoid changing implementation files when it is meant to add fixtures only? -7. Does the PR preserve the TDD sequence? - -## Minimum sentinel checks - -For each fixture, include at least: - -- [ ] Row count. -- [ ] Column count. -- [ ] Selected missing counts. -- [ ] Selected categorical counts. -- [ ] Selected numeric summaries. -- [ ] At least one unsupported or skipped variable, if plot dispatch is being tested. - -For `penguins_raw`, suitable sentinel checks include: - -```text -rows: 344 -columns: 17 -``` - -Additional sentinel values should be generated from the committed fixture using independent base R and then committed as expected CSV files. Do not compute them with the implementation under test. - -## Suggested PR label - -```text -testing -fixtures -spec-first-EDA -``` diff --git a/spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md b/spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md deleted file mode 100644 index 2528fda..0000000 --- a/spec_driven_EDA_plan/future/phase-1-stabilization-sdd-tdd.md +++ /dev/null @@ -1,149 +0,0 @@ -# Deferred Phase 1 stabilization plan - -## Status - -Phase 1 in `spec_driven_EDA_plan/docs/roadmap.md` is not complete. - -`spec_driven_EDA_plan/docs/repository-audit.md` partially satisfies the audit -portion by identifying fragile existing helpers, but there is no completed -Phase 1 PR sequence, baseline check record, coverage baseline, or targeted -hardening PR. - -This work is intentionally deferred. It must not block the active PR-sized task -in `spec_driven_EDA_plan/docs/START_HERE.md`. - -Fragile was identified as edge cases which can produce warnings, unclear errors, or unstable numeric results. The functions are likely fine for common data, but their contracts are under-specified at the boundaries. TDD hardening would make those boundaries explicit. - -For instance: - -For `epi_stats_numeric()`: - -- **All-missing vectors:** calls like `min(x, na.rm = TRUE)`, `max()`, `quantile()`, `IQR()`, `sd()`, `var()`, `skewness()`, and `kurtosis()` do not all have clean, consistent behavior when there are no usable values. -- **Zero-length vectors:** `n = 0` makes `NA_percentage <- (NA_count / n) * 100` become division by zero/`NaN`. -- **Mean is zero:** `CV = sd / mean` can become `Inf`, `-Inf`, or `NaN`. That needs an explicit contract. -- **Shapiro-Wilk handling:** the code checks `n_nonNA`, but passes the original vector to `shapiro.test()`. If there are `NA`s, it relies on `tryCatch()` rather than intentionally passing cleaned values. -- **Repeated calculations:** `sd()` and `mean()` are recalculated multiple times, increasing room for inconsistent handling when edge cases are added. -- **Dependency behavior:** it hard-stops if `e1071` is unavailable, even though some simpler stats could still be computed. Since `e1071` is in `Imports`, that is acceptable in installed-package use, but it makes local/source testing brittle if dependencies are not installed. - -For `epi_stats_na_perc()`: - -- **Uses `apply()` on data frames:** `apply()` coerces data frames to a matrix first. With mixed types, that can silently coerce numbers, characters, dates, factors, etc. before counting. -- **Invalid `margin`:** if `margin` is not `1` or `2`, the function falls through and returns an undefined object error: `object 'na_perc_all' not found`. -- **Return shape differs by mode:** column mode returns transposed row names as variable labels; row mode returns a plain data frame from an unnamed vector. That can be okay, but it needs a stable tested contract. -- **No input validation:** `df = NULL`, non-data-frame input, zero-row/zero-column inputs, and invalid margins are not handled deliberately. - -For `epi_stats_summary()`: - -- **Older tidyverse idioms:** `select_if()` still works in many setups, but it is superseded and more likely to become noisy or awkward in future dplyr versions. -- **`expression()` + `eval()` dispatch:** harder to reason about, harder to test, and easier to break when refactoring. -- **Depends on lower helpers:** if `epi_stats_numeric()` is fragile, then numeric summary mode inherits that fragility. - - -## Decision - -Do not interrupt the current specification-first TDD sequence to complete Phase -1. Continue the active PR flow unless the user explicitly asks to activate this -deferred stabilization plan. - -When activated, Phase 1 stabilization should be a small, TDD-first hardening PR -for existing helper behaviour, not a broad modernization of the package. - -## Why this is deferred - -The current spec-first EDA work is already organized as small TDD slices: - -```text -external fixture and expected output first -failing fixture-backed tests second -minimal implementation third -``` - -The active spec-first tasks do not yet require broad changes to older `epi_*` -helpers. Completing Phase 1 now would add scope and review risk without being -necessary for the current PR. - -The older helpers still matter, but they should be hardened when new work -depends on them or when package checks expose concrete failures. - -## Reopen triggers - -Reopen this deferred Phase 1 work when any of the following is true: - -- `epi_eda_profile_summaries()` depends on `epi_stats_numeric()` or - `epi_stats_summary()`. -- `epi_eda_run()` depends on older `epi_*` helpers. -- CI, `R CMD check`, or local tests expose helper failures. -- The package is being prepared for release or broader hardening. -- The user explicitly asks to run the deferred Phase 1 stabilization PR. - -## SDD intent - -The intent is to preserve existing public behaviour while making fragile helper -edge cases explicit and testable. - -Phase 1 stabilization should: - -- keep existing exported function names and return shapes stable where - practical; -- improve clear failure modes for invalid inputs; -- avoid hidden state and broad refactoring; -- document any intentional behaviour changes in tests and roxygen comments; -- leave the specification-first API sequence intact unless a direct dependency - requires coordination. - -## TDD sequence - -Use this sequence when the deferred work is activated: - -1. Record the local baseline status for tests/checks before changing behaviour. -2. Add failing tests for the selected helper edge cases. -3. Implement the smallest code change that passes those tests. -4. Run focused tests for the changed helpers. -5. Run broader package tests/checks where practical. -6. Fix only failures caused by this PR. -7. Document any pre-existing or unrelated failures in the PR summary. - -Do not write tests that merely lock in current broken behaviour. Tests should -describe the intended stable contract. - -## Initial edge cases - -Start with these edge cases: - -- `epi_stats_numeric()` with all-missing numeric vectors. -- `epi_stats_numeric()` with zero-length vectors. -- `epi_stats_numeric()` coefficient of variation when mean is zero. -- `epi_stats_numeric()` normality, skewness, and kurtosis behaviour with - insufficient usable values. -- `epi_stats_na_perc()` with invalid `margin`. -- `epi_stats_na_perc()` with mixed-type data frames. -- `epi_stats_na_perc()` return shape for row and column modes. - -Prefer one small PR for `epi_stats_numeric()` and `epi_stats_na_perc()` before -expanding to other helpers. - -## Acceptance criteria - -The deferred Phase 1 PR is complete when: - -- failing tests are added before implementation changes; -- `epi_stats_numeric()` and `epi_stats_na_perc()` handle the selected edge cases - deterministically; -- public function names remain unchanged; -- return objects remain data frames or tibbles compatible with existing tests; -- invalid input errors are clear and actionable; -- baseline and final local check/test status are recorded in the PR summary; -- unrelated helper modernization is not included. - -## Out of scope - -Do not include these unless a failing test or check makes them necessary: - -- broad replacement of all `select_if()` calls; -- redesign of the public `epi_*` API; -- removal or renaming of exported functions; -- changes to fixture files; -- changes to spec-first implementation files; -- report or project-template work; -- large-data backend implementation; -- package-wide style rewrites. diff --git a/spec_driven_EDA_plan/prompt_loop.md b/spec_driven_EDA_plan/prompt_loop.md deleted file mode 100644 index e03b1dc..0000000 --- a/spec_driven_EDA_plan/prompt_loop.md +++ /dev/null @@ -1,87 +0,0 @@ ------------ -Reading order and mapping for SDD/TDD in this directory: -- START_HERE.md for what to do now. -- revised-pr-plan-tdd-first.md for where the current PR sits in the sequence. -- tdd-first-codex-instructions.md for the exact task block. -- roadmap.md for why this work exists and which phase it belongs to. -- review-checklist.md before closing any PR. - - - -The mapping is hierarchical, not one-to-one: - -**Control Files** -`roadmap.md` is the strategic map: phases and broad intent. - -`START_HERE.md` is the live operational state. It explicitly says it is the “operational source of truth” for what happens next, including active PR, allowed files, and expected test state ([START_HERE.md]()). - -`revised-pr-plan-tdd-first.md` is the stable PR sequence. It translates roadmap phases into PR-sized work and enforces the pattern: fixture/expected output, failing tests, implementation ([revised-pr-plan-tdd-first.md]()). - -`tdd-first-codex-instructions.md` is the executable instruction bank. Each instruction corresponds roughly to one PR, especially PRs 1-9. Later work is collapsed into Instruction 10 ([tdd-first-codex-instructions.md]()). - -`review-checklist.md` applies to every PR before closeout/review ([review-checklist.md]()). - -**Phase Mapping** - -| Roadmap phase | Actual operational mapping | -|---|---| -| Phase 0: Documentation and baseline | Done. This produced SDD, ADRs, roadmap, Codex instructions, and PR plan. Tracked in `START_HERE.md` as done. | -| Phase 1: Stabilise current package | Not clearly mapped to PRs. It has audit material, but no active PR sequence or completion evidence. Looks bypassed/deferred. | -| Phase 2: TDD fixtures for spec-first EDA | PR 1 and PR 2. PR 1 adds external fixtures and expected outputs. PR 2 adds failing tests for spec/schema/missingness. | -| Phase 3: Specification-first MVP | PR 3 through PR 9, but in test-first slices: implement spec/schema/missingness, test synthetic data, implement synthetic data, test summaries/plots, implement summaries/plots, test `epi_eda_run()`, implement `epi_eda_run()`. Current active PR is PR 4. | -| Phase 4: Reporting and project template | PR 10, PR 11, PR 12. Report tests, report implementation, project template. | -| Phase 5: Large-data optimisation | PR 13 only documents large-data strategy. Actual `data.table`, DuckDB, Arrow implementation is explicitly not planned yet. | -| Phase 6: Biomedical extensions | No PRs/instructions yet. It is future scope only. | - ------------ - - ------------ -Prompt that can be re-used at each stage/step: - -You are running locally in my Positron terminal. - -Repository: antoniojbt/episcout -Base branch: episcout2 - -Task: -Complete only the active PR-sized task in spec_driven_EDA_plan/docs/START_HERE.md. -For the current active task, PR 2, expected failures from intentionally missing -EDA functions are allowed and must not be fixed in this PR. - -A local mamba env episcout should be available. - -Required sequence: -1. Confirm the git worktree is clean. -2. Confirm GitHub CLI availability: - - command -v gh - - gh auth status - If gh is unavailable or unauthenticated, continue through local changes and - commit, then stop before push/PR/check steps and summarise what remains manual. -3. Ensure the current branch is episcout2 and up to date with origin/episcout2. -4. Create a feature branch from episcout2 using feature/. -5. Read: - - spec_driven_EDA_plan/docs/AGENTS.md - - spec_driven_EDA_plan/docs/START_HERE.md - - all files listed as required reading in START_HERE.md -6. Make only the scoped changes from START_HERE.md. -7. Respect must-edit and must-not-edit lists exactly. -8. Run: - - Rscript -e "devtools::document()" - - Rscript -e "devtools::test(reporter = 'summary')" - - Rscript -e "devtools::check(manual = FALSE)" -9. Fix only errors/warnings/notes caused by this PR, except expected failures - explicitly documented in START_HERE.md. -10. Do not implement functions reserved for later PRs. -11. Do not commit changes to files listed as must-not-edit. -12. Commit the final scoped change. -13. If gh is available and authenticated: - - push the branch - - create a PR against episcout2 with gh - - run gh pr checks --watch --fail-fast -14. If CI fails only because of expected PR 2 failing tests, record that and stop. -15. If CI fails because of an unintended issue introduced by this PR, fix, - commit and push once. -16. Stop and summarise files changed, checks run, expected failures, and any - remaining unrelated/pre-existing issues or manual GitHub steps. ------------ From 7e038a609b42ab0f2960b2712ff9249b75d053e1 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Mon, 15 Jun 2026 15:27:20 -0600 Subject: [PATCH 26/33] fixed 'backlog.md' remains --- archive/eda_sdd_tdd_r1_archive/README.md | 2 +- archive/eda_sdd_tdd_r1_archive/START_HERE.md | 2 +- archive/eda_sdd_tdd_r1_archive/prompt_loop.md | 2 +- future/README.md | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/archive/eda_sdd_tdd_r1_archive/README.md b/archive/eda_sdd_tdd_r1_archive/README.md index 777b1e5..6f60dec 100644 --- a/archive/eda_sdd_tdd_r1_archive/README.md +++ b/archive/eda_sdd_tdd_r1_archive/README.md @@ -14,7 +14,7 @@ Use the root-level `future/` directory for next-phase planning: ```text future/README.md -future/backlog.md +future/TODOs.md future/specs/ ``` diff --git a/archive/eda_sdd_tdd_r1_archive/START_HERE.md b/archive/eda_sdd_tdd_r1_archive/START_HERE.md index b9d7213..25dc74f 100644 --- a/archive/eda_sdd_tdd_r1_archive/START_HERE.md +++ b/archive/eda_sdd_tdd_r1_archive/START_HERE.md @@ -7,7 +7,7 @@ Do not use this file as the live control panel for new work. Use: ```text future/README.md -future/backlog.md +future/TODOs.md future/specs/ ``` diff --git a/archive/eda_sdd_tdd_r1_archive/prompt_loop.md b/archive/eda_sdd_tdd_r1_archive/prompt_loop.md index bce6973..f210b9c 100644 --- a/archive/eda_sdd_tdd_r1_archive/prompt_loop.md +++ b/archive/eda_sdd_tdd_r1_archive/prompt_loop.md @@ -13,7 +13,7 @@ Use these files for historical context: Use the root-level future workspace for new work: - `future/README.md` -- `future/backlog.md` +- `future/TODOs.md` - `future/specs/` ## Historical Pattern diff --git a/future/README.md b/future/README.md index 1603ab1..2ffead4 100644 --- a/future/README.md +++ b/future/README.md @@ -12,7 +12,7 @@ agent handoff material. Keep executable package tests in `tests/testthat/`. ```text future/ ├── README.md -├── backlog.md +├── TODOs.md ├── decisions.md ├── changelog.md ├── prompts/ @@ -24,7 +24,7 @@ future/ ## Workflow -1. Add candidate work to `backlog.md`. +1. Add candidate work to `TODOs.md`. 2. Promote active work to a numbered directory under `specs/`. 3. Fill in `brief.md`, `sdd.md`, `tdd.md`, `acceptance.md` and `manifest.yml` before implementation. From f85d61d64315b0ca342b861ccd5396db4b7e7f19 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Mon, 15 Jun 2026 16:16:09 -0600 Subject: [PATCH 27/33] tests for edge cases for epi_stats functions --- R/epi_stats_na_perc.R | 40 +++++++--- R/epi_stats_numeric.R | 80 ++++++++++++++----- future/TODOs.md | 53 ++++++++---- future/changelog.md | 2 + .../acceptance.md | 35 +++++--- .../001-phase-1-helper-stabilization/brief.md | 2 +- .../manifest.yml | 2 +- .../review.md | 13 ++- .../001-phase-1-helper-stabilization/sdd.md | 2 +- .../001-phase-1-helper-stabilization/tdd.md | 20 ++--- man/epi_stats_na_perc.Rd | 1 + man/epi_stats_numeric.Rd | 8 +- tests/testthat/test-epi_stats_numeric.R | 50 ++++++++++++ tests/testthat/test-stats.R | 31 +++++++ 14 files changed, 269 insertions(+), 70 deletions(-) diff --git a/R/epi_stats_na_perc.R b/R/epi_stats_na_perc.R index 442bb72..be17963 100644 --- a/R/epi_stats_na_perc.R +++ b/R/epi_stats_na_perc.R @@ -8,6 +8,7 @@ #' @param margin 2 for columns, 1 for rows. Default is columns #' #' @return A dataframe object with counts and percentage of NAs for each column or row. +#' Invalid margins raise a clear error. #' #' @author Antonio J Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> #' @@ -36,20 +37,41 @@ epi_stats_na_perc <- function(df = NULL, margin = 2 # 2 for columns, 1 for rows ) { + if (!is.numeric(margin) || length(margin) != 1 || !margin %in% c(1, 2)) { + stop("margin must be either 1 (rows) or 2 (columns)", call. = FALSE) + } + + df <- as.data.frame(df) + # For columns: if (margin == 2) { - na_perc_all <- as.list(apply(X = df, MARGIN = margin, function(x) sum(is.na(x)))) - na_perc_all <- as.data.frame(na_perc_all) - na_perc_all <- as.data.frame(t(na_perc_all)) - names(na_perc_all)[1] <- "na_counts" - na_perc_all$na_perc <- (na_perc_all$na_counts / dim(df)[1]) * 100 + na_counts <- vapply(df, function(x) sum(is.na(x)), integer(1)) + na_perc <- if (nrow(df) > 0) { + (na_counts / nrow(df)) * 100 + } else { + rep(NA_real_, length(na_counts)) + } + na_perc_all <- data.frame( + na_counts = as.integer(na_counts), + na_perc = as.numeric(na_perc), + row.names = names(df), + check.names = FALSE + ) } # For rows: else if (margin == 1) { - na_perc_all <- apply(X = df, MARGIN = margin, function(x) sum(is.na(x))) - na_perc_all <- as.data.frame(na_perc_all) - names(na_perc_all)[1] <- "na_counts" - na_perc_all$na_perc <- (na_perc_all$na_counts / dim(df)[2]) * 100 + na_counts <- rowSums(is.na(df)) + na_perc <- if (ncol(df) > 0) { + (na_counts / ncol(df)) * 100 + } else { + rep(NA_real_, length(na_counts)) + } + na_perc_all <- data.frame( + na_counts = as.integer(na_counts), + na_perc = as.numeric(na_perc), + row.names = rownames(df), + check.names = FALSE + ) } na_perc_all } diff --git a/R/epi_stats_numeric.R b/R/epi_stats_numeric.R index ca0dbb1..77805b1 100644 --- a/R/epi_stats_numeric.R +++ b/R/epi_stats_numeric.R @@ -17,7 +17,7 @@ #' \item{\code{n}}{Total length of the input vector.} #' \item{\code{n_nonNA}}{Number of non-\code{NA} observations.} #' \item{\code{NA_count}}{Count of \code{NA} values.} -#' \item{\code{NA_percentage}}{Percentage of values that are \code{NA}.} +#' \item{\code{NA_percentage}}{Percentage of values that are \code{NA}; \code{NA} for zero-length inputs.} #' \item{\code{sum}}{Sum of values.} #' \item{\code{min}}{Minimum value.} #' \item{\code{quantile_25}}{25th percentile (\code{Q1}).} @@ -27,7 +27,7 @@ #' \item{\code{max}}{Maximum value.} #' \item{\code{IQR}}{Interquartile range (\code{Q3 - Q1}).} #' \item{\code{SD}}{Standard deviation.} -#' \item{\code{CV}}{Coefficient of variation (\code{SD / mean}).} +#' \item{\code{CV}}{Coefficient of variation (\code{SD / mean}); \code{NA} when the mean is zero or unavailable.} #' \item{\code{variance}}{Variance.} #' \item{\code{sem}}{Standard error of the mean (\code{SD / sqrt(n_nonNA)}).} #' \item{\code{skewness}}{Sample skewness (via \pkg{e1071}).} @@ -44,7 +44,9 @@ #' @details #' Missing values are dropped when \code{na.rm = TRUE}. The Shapiro-Wilk test #' for normality is only run for sample sizes between 4 and 4999; otherwise -#' its p-value is reported as \code{NA}. For skewness: negative/longer left tail, +#' its p-value is reported as \code{NA}. Empty and all-missing inputs return one +#' row with unavailable summaries reported as \code{NA}. For skewness: +#' negative/longer left tail, #' positive/longer right tail, values above 1 usually means non-normality. #' For kurtosis consider lower values, broader shape and longer tails (platy ~<3), #' normal (meso ~3) and slender/no tails (lepto ~<3). @@ -77,12 +79,52 @@ epi_stats_numeric <- function(num_vec = NULL, n <- length(num_vec) n_nonNA <- sum(!is.na(num_vec)) NA_count <- n - n_nonNA - NA_percentage <- (NA_count / n) * 100 + NA_percentage <- if (n > 0) (NA_count / n) * 100 else NA_real_ + + safe_scalar <- function(expr) { + value <- tryCatch( + expr, + warning = function(w) NA_real_, + error = function(e) NA_real_ + ) + if (length(value) == 0 || is.nan(value)) { + return(NA_real_) + } + as.numeric(value) + } # Summaries - q1 <- quantile(num_vec, 0.25, na.rm = na.rm, names = FALSE) - q3 <- quantile(num_vec, 0.75, na.rm = na.rm, names = FALSE) - iqr_val <- IQR(num_vec, na.rm = na.rm) + has_non_missing <- n_nonNA > 0 + sum_val <- sum(num_vec, na.rm = na.rm) + min_val <- if (has_non_missing) safe_scalar(min(num_vec, na.rm = na.rm)) else NA_real_ + q1 <- if (has_non_missing) { + safe_scalar(quantile(num_vec, 0.25, na.rm = na.rm, names = FALSE)) + } else { + NA_real_ + } + mean_val <- if (has_non_missing) safe_scalar(mean(num_vec, na.rm = na.rm)) else NA_real_ + median_val <- if (has_non_missing) safe_scalar(median(num_vec, na.rm = na.rm)) else NA_real_ + q3 <- if (has_non_missing) { + safe_scalar(quantile(num_vec, 0.75, na.rm = na.rm, names = FALSE)) + } else { + NA_real_ + } + max_val <- if (has_non_missing) safe_scalar(max(num_vec, na.rm = na.rm)) else NA_real_ + iqr_val <- if (has_non_missing) safe_scalar(IQR(num_vec, na.rm = na.rm)) else NA_real_ + sd_val <- if (has_non_missing) safe_scalar(sd(num_vec, na.rm = na.rm)) else NA_real_ + variance_val <- if (has_non_missing) safe_scalar(var(num_vec, na.rm = na.rm)) else NA_real_ + cv_val <- if (!is.na(mean_val) && mean_val != 0) sd_val / mean_val else NA_real_ + sem_val <- if (n_nonNA > 0) sd_val / sqrt(n_nonNA) else NA_real_ + skewness_val <- if (n_nonNA >= 3) { + safe_scalar(e1071::skewness(num_vec, na.rm = na.rm, ...)) + } else { + NA_real_ + } + kurtosis_val <- if (n_nonNA >= 3) { + safe_scalar(e1071::kurtosis(num_vec, na.rm = na.rm, ...)) + } else { + NA_real_ + } # Tukey fences & outlier counts lower_fence <- q1 - coef * iqr_val @@ -96,7 +138,7 @@ epi_stats_numeric <- function(num_vec = NULL, normality <- NA_real_ if (n_nonNA > 3 && n_nonNA < 5000) { normality <- tryCatch( - shapiro.test(num_vec)$p.value, + shapiro.test(num_vec[!is.na(num_vec)])$p.value, error = function(e) NA_real_ ) } @@ -108,20 +150,20 @@ epi_stats_numeric <- function(num_vec = NULL, n_nonNA = n_nonNA, NA_count = NA_count, NA_percentage = NA_percentage, - sum = sum(num_vec, na.rm = na.rm), - min = min(num_vec, na.rm = na.rm), + sum = sum_val, + min = min_val, quantile_25 = q1, - mean = mean(num_vec, na.rm = na.rm), - median = median(num_vec, na.rm = na.rm), + mean = mean_val, + median = median_val, quantile_75 = q3, - max = max(num_vec, na.rm = na.rm), + max = max_val, IQR = iqr_val, - SD = sd(num_vec, na.rm = na.rm), - CV = sd(num_vec, na.rm = na.rm) / mean(num_vec, na.rm = na.rm), - variance = var(num_vec, na.rm = na.rm), - sem = sd(num_vec, na.rm = na.rm) / sqrt(n_nonNA), - skewness = e1071::skewness(num_vec, na.rm = na.rm, ...), - kurtosis = e1071::kurtosis(num_vec, na.rm = na.rm, ...), + SD = sd_val, + CV = cv_val, + variance = variance_val, + sem = sem_val, + skewness = skewness_val, + kurtosis = kurtosis_val, Shapiro_Wilk_p_value = normality, lower_fence = lower_fence, upper_fence = upper_fence, diff --git a/future/TODOs.md b/future/TODOs.md index 6dd6b9e..447fa8a 100644 --- a/future/TODOs.md +++ b/future/TODOs.md @@ -1,8 +1,15 @@ # TODOs -Keep all future work here with a priority order. -Do not create additional files such as backlog.md or equivalent. -Centralise task list here for clarity and for easy human review. +- Keep all future work here with a priority order. +- Do not create additional files such as `backlog.md` or equivalent. +- Centralise task list here for clarity and for easy human review. +- Decide which future spec to activate first. +- Convert any new candidate work into a numbered spec before coding. +- If a task in this file does not have the needed SDD-TDD approach and a numbered spec, prompt me to clarify and create. Tasks that require multiple steps, substantial judgement calls, or may compromise existing code must have an SDD-TDD specification. If unclear ask first before proceeding with any write work. +- For the activated spec, record baseline package test/check status before package-code changes. +- Keep implementation work scoped to one numbered spec at a time. +- Review fixture anti-circularity guardrails before adding new expected outputs. +- Keep README and `NEWS.md` aligned with user-facing workflow changes. Use this template and place the task under the appropriate priority heading: @@ -20,27 +27,45 @@ Use this template and place the task under the appropriate priority heading: For simple tasks only use e.g. `- [ ] xxx` -## Priority 1 +## Task list -- [ ] Decide which future spec to activate first. -- [ ] For the activated spec, record baseline package test/check status before - package-code changes. -- [ ] Keep implementation work scoped to one numbered spec at a time. +### Priority 1 +#TODO: - [ ] thorough code review, use prompt in `future/prompts/senior-r-package-review.md` + - correct small/important aspects + - plan improvements and specify in `future` -## Priority 2 -- [ ] Convert any new candidate work into a numbered spec before coding. -- [ ] Review fixture anti-circularity guardrails before adding new expected - outputs. -- [ ] Keep README and `NEWS.md` aligned with user-facing workflow changes. +- [ ] get instructions to download, install, episcout2 branch +- [ ] human live walkthrough, no agent needed here +- [ ] carry out changes needed from human review +- [ ] run devtools and CRAN checks +- [ ] merge with main +- [ ] tag v0.2.0 -## Priority 3 +### Priority 2 +- [ ] Add functions to create a data dictionary when real data already exists but does not have a dictionary. +- [ ] Sanitise dictionaries so that R, QGIS, SQL/MariaDB/postgreSQL can easily use them as input - [ ] Revisit large-data backend implementation after the design spec is accepted. + +### Priority 3 + - [ ] Consider visual-regression strategy for EDA plots only after plot contracts are stable. - [ ] Add biomedical EDA extensions as separate numbered specs. + +## Done + +- [x] Completed spec: `001-phase-1-helper-stabilization` + - Baseline recorded before package-code changes. + - TDD edge-case tests added before implementation. + - Implementation scoped to `epi_stats_numeric()` and `epi_stats_na_perc()`. + - Final `devtools::test(reporter = 'summary')` passed. +- [x] check future work has been setup properly for SDD-TDD: + - PR14 + - add penguins_raw + - ? diff --git a/future/changelog.md b/future/changelog.md index 2690d6c..aa2c30a 100644 --- a/future/changelog.md +++ b/future/changelog.md @@ -10,3 +10,5 @@ - Added large-data backend design follow-up under `specs/003-large-data-backend-strategy/`. - Added templates, prompts, todos, references and review folders. +- Implemented `specs/001-phase-1-helper-stabilization/` with TDD coverage for + numeric summary and missingness helper edge cases. diff --git a/future/specs/001-phase-1-helper-stabilization/acceptance.md b/future/specs/001-phase-1-helper-stabilization/acceptance.md index 9868b8b..5f3538b 100644 --- a/future/specs/001-phase-1-helper-stabilization/acceptance.md +++ b/future/specs/001-phase-1-helper-stabilization/acceptance.md @@ -1,13 +1,28 @@ # Acceptance Spec ID: `001-phase-1-helper-stabilization` -Status: Ready for activation - -- [ ] Baseline local test/check status is recorded before behaviour changes. -- [ ] Failing tests are added before implementation. -- [ ] Selected edge-case tests pass after implementation. -- [ ] Public function names remain unchanged. -- [ ] Return objects remain data-frame-like and compatible with existing tests. -- [ ] Invalid input errors are clear and actionable. -- [ ] No spec-first EDA implementation files are changed. -- [ ] Pre-existing unrelated failures are documented rather than fixed. +Status: Implemented + +- [x] Baseline local test/check status is recorded before behaviour changes. +- [x] Failing tests are added before implementation. +- [x] Selected edge-case tests pass after implementation. +- [x] Public function names remain unchanged. +- [x] Return objects remain data-frame-like and compatible with existing tests. +- [x] Invalid input errors are clear and actionable. +- [x] No spec-first EDA implementation files are changed. +- [x] Pre-existing unrelated failures are documented rather than fixed. + +## Results + +- Baseline `devtools::test(reporter = 'summary')`: passed with 2 skips and 1 + pre-existing parallel-worker warning. +- Baseline `devtools::check(manual = FALSE)`: 0 errors, 0 warnings, 1 + pre-existing NOTE for `.gitkeep` files under `inst/project-template`. +- TDD run before implementation: new tests failed for all-missing/empty + numeric summaries, zero-mean CV and invalid `margin` messaging. +- Final targeted tests: passed after `devtools::load_all()` plus + `testthat::test_file()`. +- Final `devtools::test(reporter = 'summary')`: passed with the same 2 skips + and 1 parallel-worker warning as baseline. +- Final `devtools::check(manual = FALSE)`: 0 errors, 0 warnings, 1 + pre-existing NOTE for `.gitkeep` files under `inst/project-template`. diff --git a/future/specs/001-phase-1-helper-stabilization/brief.md b/future/specs/001-phase-1-helper-stabilization/brief.md index 870291f..dbb3747 100644 --- a/future/specs/001-phase-1-helper-stabilization/brief.md +++ b/future/specs/001-phase-1-helper-stabilization/brief.md @@ -1,7 +1,7 @@ # Brief Spec ID: `001-phase-1-helper-stabilization` -Status: Ready for activation +Status: Implemented ## Problem diff --git a/future/specs/001-phase-1-helper-stabilization/manifest.yml b/future/specs/001-phase-1-helper-stabilization/manifest.yml index 57ef561..0f024ca 100644 --- a/future/specs/001-phase-1-helper-stabilization/manifest.yml +++ b/future/specs/001-phase-1-helper-stabilization/manifest.yml @@ -1,6 +1,6 @@ id: "001" slug: "phase-1-helper-stabilization" -status: ready +status: implemented owner: TBD base_branch: episcout2 working_branch: TBD diff --git a/future/specs/001-phase-1-helper-stabilization/review.md b/future/specs/001-phase-1-helper-stabilization/review.md index 9fa0edf..9bcafdc 100644 --- a/future/specs/001-phase-1-helper-stabilization/review.md +++ b/future/specs/001-phase-1-helper-stabilization/review.md @@ -1,7 +1,7 @@ # Review Notes Spec ID: `001-phase-1-helper-stabilization` -Status: Ready for activation +Status: Implemented; review required ## Review Focus @@ -13,4 +13,13 @@ Status: Ready for activation ## Findings -None yet. +- No blocking findings from the implementation pass. +- Public function names and arguments were preserved. +- Code changes stayed limited to `epi_stats_numeric()` and + `epi_stats_na_perc()`. +- `testthat::test_file()` is not standalone for these files unless the package + is loaded first; targeted verification used `devtools::load_all()`. +- `devtools::test()` removed skipped vdiffr snapshot files as a side effect; + those unrelated deletions were restored. +- The pre-existing `R CMD check` NOTE about `.gitkeep` files in + `inst/project-template` was documented and not fixed in this spec. diff --git a/future/specs/001-phase-1-helper-stabilization/sdd.md b/future/specs/001-phase-1-helper-stabilization/sdd.md index e14f106..b640a1b 100644 --- a/future/specs/001-phase-1-helper-stabilization/sdd.md +++ b/future/specs/001-phase-1-helper-stabilization/sdd.md @@ -1,7 +1,7 @@ # Software Design Spec ID: `001-phase-1-helper-stabilization` -Status: Ready for activation +Status: Implemented ## Scope diff --git a/future/specs/001-phase-1-helper-stabilization/tdd.md b/future/specs/001-phase-1-helper-stabilization/tdd.md index 8bfb823..9153595 100644 --- a/future/specs/001-phase-1-helper-stabilization/tdd.md +++ b/future/specs/001-phase-1-helper-stabilization/tdd.md @@ -1,7 +1,7 @@ # Test Design Spec ID: `001-phase-1-helper-stabilization` -Status: Ready for activation +Status: Implemented ## Baseline Commands @@ -22,21 +22,21 @@ Use a narrower new test file only if it makes the edge-case contracts clearer. ## Required Tests -- [ ] `epi_stats_numeric()` handles all-missing numeric vectors. -- [ ] `epi_stats_numeric()` handles zero-length numeric vectors. -- [ ] `epi_stats_numeric()` returns stable coefficient-of-variation output when +- [x] `epi_stats_numeric()` handles all-missing numeric vectors. +- [x] `epi_stats_numeric()` handles zero-length numeric vectors. +- [x] `epi_stats_numeric()` returns stable coefficient-of-variation output when mean is zero. -- [ ] `epi_stats_numeric()` handles insufficient usable values for normality, +- [x] `epi_stats_numeric()` handles insufficient usable values for normality, skewness and kurtosis. -- [ ] `epi_stats_na_perc()` rejects invalid `margin` with a clear error. -- [ ] `epi_stats_na_perc()` handles mixed-type data frames without +- [x] `epi_stats_na_perc()` rejects invalid `margin` with a clear error. +- [x] `epi_stats_na_perc()` handles mixed-type data frames without coercion-related count errors. -- [ ] `epi_stats_na_perc()` keeps stable row-mode and column-mode return shapes. +- [x] `epi_stats_na_perc()` keeps stable row-mode and column-mode return shapes. ## Acceptance Commands ```bash -scripts/rscript_env_caller.R -e "testthat::test_file('tests/testthat/test-epi_stats_numeric.R')" -scripts/rscript_env_caller.R -e "testthat::test_file('tests/testthat/test-stats.R')" +scripts/rscript_env_caller.R -e "devtools::load_all(); testthat::test_file('tests/testthat/test-epi_stats_numeric.R')" +scripts/rscript_env_caller.R -e "devtools::load_all(); testthat::test_file('tests/testthat/test-stats.R')" scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" ``` diff --git a/man/epi_stats_na_perc.Rd b/man/epi_stats_na_perc.Rd index 2988973..c6d5d85 100644 --- a/man/epi_stats_na_perc.Rd +++ b/man/epi_stats_na_perc.Rd @@ -13,6 +13,7 @@ epi_stats_na_perc(df = NULL, margin = 2) } \value{ A dataframe object with counts and percentage of NAs for each column or row. +Invalid margins raise a clear error. } \description{ epi_stats_na_perc() gets the total counts and percentage for missing values diff --git a/man/epi_stats_numeric.Rd b/man/epi_stats_numeric.Rd index f160054..68bbce2 100644 --- a/man/epi_stats_numeric.Rd +++ b/man/epi_stats_numeric.Rd @@ -25,7 +25,7 @@ A one-row \code{data.frame} with the following columns: \item{\code{n}}{Total length of the input vector.} \item{\code{n_nonNA}}{Number of non-\code{NA} observations.} \item{\code{NA_count}}{Count of \code{NA} values.} -\item{\code{NA_percentage}}{Percentage of values that are \code{NA}.} +\item{\code{NA_percentage}}{Percentage of values that are \code{NA}; \code{NA} for zero-length inputs.} \item{\code{sum}}{Sum of values.} \item{\code{min}}{Minimum value.} \item{\code{quantile_25}}{25th percentile (\code{Q1}).} @@ -35,7 +35,7 @@ A one-row \code{data.frame} with the following columns: \item{\code{max}}{Maximum value.} \item{\code{IQR}}{Interquartile range (\code{Q3 - Q1}).} \item{\code{SD}}{Standard deviation.} -\item{\code{CV}}{Coefficient of variation (\code{SD / mean}).} +\item{\code{CV}}{Coefficient of variation (\code{SD / mean}); \code{NA} when the mean is zero or unavailable.} \item{\code{variance}}{Variance.} \item{\code{sem}}{Standard error of the mean (\code{SD / sqrt(n_nonNA)}).} \item{\code{skewness}}{Sample skewness (via \pkg{e1071}).} @@ -56,7 +56,9 @@ including measures of location, dispersion, shape, normality, and outliers. \details{ Missing values are dropped when \code{na.rm = TRUE}. The Shapiro-Wilk test for normality is only run for sample sizes between 4 and 4999; otherwise -its p-value is reported as \code{NA}. For skewness: negative/longer left tail, +its p-value is reported as \code{NA}. Empty and all-missing inputs return one +row with unavailable summaries reported as \code{NA}. For skewness: +negative/longer left tail, positive/longer right tail, values above 1 usually means non-normality. For kurtosis consider lower values, broader shape and longer tails (platy ~<3), normal (meso ~3) and slender/no tails (lepto ~<3). diff --git a/tests/testthat/test-epi_stats_numeric.R b/tests/testthat/test-epi_stats_numeric.R index 86b1cab..924605c 100644 --- a/tests/testthat/test-epi_stats_numeric.R +++ b/tests/testthat/test-epi_stats_numeric.R @@ -82,3 +82,53 @@ test_that("Shapiro–Wilk skipped for too few or too many observations", { res_large <- epi_stats_numeric(large) expect_true(is.na(res_large$Shapiro_Wilk_p_value)) }) + +test_that("all-missing numeric vectors return deterministic missing summaries", { + res <- expect_silent(epi_stats_numeric(c(NA_real_, NA_real_, NA_real_))) + + expect_s3_class(res, "data.frame") + expect_equal(nrow(res), 1L) + expect_equal(res$n, 3L) + expect_equal(res$n_nonNA, 0L) + expect_equal(res$NA_count, 3L) + expect_equal(res$NA_percentage, 100) + expect_true(is.na(res$min)) + expect_true(is.na(res$mean)) + expect_true(is.na(res$max)) + expect_true(is.na(res$CV)) + expect_true(is.na(res$skewness)) + expect_true(is.na(res$kurtosis)) + expect_true(is.na(res$Shapiro_Wilk_p_value)) + expect_true(is.na(res$outlier_percentage)) +}) + +test_that("zero-length numeric vectors return one stable summary row", { + res <- expect_silent(epi_stats_numeric(numeric())) + + expect_s3_class(res, "data.frame") + expect_equal(nrow(res), 1L) + expect_equal(res$n, 0L) + expect_equal(res$n_nonNA, 0L) + expect_equal(res$NA_count, 0L) + expect_true(is.na(res$NA_percentage)) + expect_true(is.na(res$mean)) + expect_true(is.na(res$CV)) + expect_true(is.na(res$Shapiro_Wilk_p_value)) + expect_true(is.na(res$outlier_percentage)) +}) + +test_that("coefficient of variation is missing when the mean is zero", { + res <- epi_stats_numeric(c(-1, 0, 1)) + + expect_true(is.na(res$CV)) + expect_false(is.infinite(res$CV)) +}) + +test_that("shape and normality statistics are skipped when usable values are insufficient", { + res <- epi_stats_numeric(c(1, NA_real_)) + + expect_equal(res$n_nonNA, 1L) + expect_true(is.na(res$skewness)) + expect_true(is.na(res$kurtosis)) + expect_true(is.na(res$Shapiro_Wilk_p_value)) +}) diff --git a/tests/testthat/test-stats.R b/tests/testthat/test-stats.R index e8ab678..7ee5266 100644 --- a/tests/testthat/test-stats.R +++ b/tests/testthat/test-stats.R @@ -95,6 +95,37 @@ test_that("epi_stats_na_perc", { # summary(na_rows) expect_identical(length(which(na_rows$na_counts == 0)), length(which(complete.cases(df2)))) }) + +test_that("epi_stats_na_perc rejects invalid margins", { + expect_error( + epi_stats_na_perc(df2, margin = 3), + "margin must be either 1 \\(rows\\) or 2 \\(columns\\)" + ) +}) + +test_that("epi_stats_na_perc counts mixed-type data without coercion errors", { + mixed_df <- data.frame( + num = c(1, NA_real_, 3), + chr = c("a", NA_character_, "NA"), + fct = factor(c("x", NA, "z")), + stringsAsFactors = FALSE + ) + + na_cols <- epi_stats_na_perc(mixed_df, margin = 2) + na_rows <- epi_stats_na_perc(mixed_df, margin = 1) + + expect_s3_class(na_cols, "data.frame") + expect_named(na_cols, c("na_counts", "na_perc")) + expect_equal(rownames(na_cols), names(mixed_df)) + expect_equal(na_cols$na_counts, c(1, 1, 1)) + expect_equal(na_cols$na_perc, rep(100 / 3, 3)) + + expect_s3_class(na_rows, "data.frame") + expect_named(na_rows, c("na_counts", "na_perc")) + expect_equal(nrow(na_rows), nrow(mixed_df)) + expect_equal(na_rows$na_counts, c(0, 3, 0)) + expect_equal(na_rows$na_perc, c(0, 100, 0)) +}) ###################### ###################### From 4e2cf3921eeb09830fb54ac8ce5761eeec2a8c56 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Mon, 15 Jun 2026 16:29:19 -0600 Subject: [PATCH 28/33] bot review --- R/epi_stats_numeric.R | 2 +- future/TODOs.md | 16 +++- .../2026-06-15-senior-r-package-review.md | 82 +++++++++++++++++++ man/epi_clean_cond_numeric.Rd | 2 +- man/epi_clean_make_names.Rd | 2 +- man/epi_clean_merge_nested_dfs.Rd | 2 +- man/epi_output_name.Rd | 2 +- man/epi_plot_heatmap_triangle.Rd | 2 +- man/epi_plot_list.Rd | 2 +- man/epi_read.Rd | 2 +- man/epi_stats_corr.Rd | 2 +- man/epi_stats_numeric.Rd | 2 +- man/epi_utils_log.Rd | 2 +- 13 files changed, 106 insertions(+), 14 deletions(-) create mode 100644 future/reviews/2026-06-15-senior-r-package-review.md diff --git a/R/epi_stats_numeric.R b/R/epi_stats_numeric.R index 77805b1..cc15f5f 100644 --- a/R/epi_stats_numeric.R +++ b/R/epi_stats_numeric.R @@ -49,7 +49,7 @@ #' negative/longer left tail, #' positive/longer right tail, values above 1 usually means non-normality. #' For kurtosis consider lower values, broader shape and longer tails (platy ~<3), -#' normal (meso ~3) and slender/no tails (lepto ~<3). +#' normal (meso ~3) and slender/no tails (lepto ~>3). #' Outliers are detected with the Tukey method (above and below 1.5 * IQR) or #' using the multiplier \code{coef}. Coefficient of variation (\code{CV}) is #' calculated as \code{SD / mean}. You may pass further arguments (e.g., diff --git a/future/TODOs.md b/future/TODOs.md index 447fa8a..ea3fae0 100644 --- a/future/TODOs.md +++ b/future/TODOs.md @@ -31,9 +31,15 @@ For simple tasks only use e.g. `- [ ] xxx` ### Priority 1 -#TODO: - [ ] thorough code review, use prompt in `future/prompts/senior-r-package-review.md` - - correct small/important aspects - - plan improvements and specify in `future` +- Short title: Address senior R package review follow-ups + - Problem: The review in `future/reviews/2026-06-15-senior-r-package-review.md` found API-surface, test-design, EDA edge-case and dependency-guard risks. + - Goal: Convert the review findings into a scoped SDD/TDD implementation plan before package-code changes. + - User need: Keep the package maintainable and CRAN-ready without widening the current branch ad hoc. + - Proposed scope: Create a numbered spec for the highest-value findings: EDA summary edge cases, missing-code semantics, non-circular fixture expectations, public API cleanup and optional dependency guards. + - Out of scope: Large-data backend implementation and unrelated helper rewrites. + - Candidate files: `future/reviews/2026-06-15-senior-r-package-review.md`, `future/specs/`, `R/eda_*`, `R/epi_plot_parallel.R`, `R/utils-tidy-eval.R`, `tests/testthat/`. + - Risks: API cleanup may be breaking if current users rely on accidental exports. + - Suggested spec ID: `004-senior-review-followups` - [ ] get instructions to download, install, episcout2 branch @@ -60,6 +66,10 @@ For simple tasks only use e.g. `- [ ] xxx` ## Done +- [x] thorough code review, use prompt in `future/prompts/senior-r-package-review.md` + - Review written to `future/reviews/2026-06-15-senior-r-package-review.md`. + - Corrected the confirmed `epi_stats_numeric()` kurtosis documentation issue. + - Planned follow-up work as `004-senior-review-followups` candidate. - [x] Completed spec: `001-phase-1-helper-stabilization` - Baseline recorded before package-code changes. - TDD edge-case tests added before implementation. diff --git a/future/reviews/2026-06-15-senior-r-package-review.md b/future/reviews/2026-06-15-senior-r-package-review.md new file mode 100644 index 0000000..0329197 --- /dev/null +++ b/future/reviews/2026-06-15-senior-r-package-review.md @@ -0,0 +1,82 @@ +# Senior R Package Review + +Date: 2026-06-15 + +Prompt: `future/prompts/senior-r-package-review.md` + +## A. Executive Summary + +Overall judgement: mostly ready, with focused revision needed before treating the current development branch as a long-term stable base. + +Main risks: + +- The public API is broad and includes low-level tidy-eval re-exports that make the namespace harder to maintain. +- EDA summary tests currently duplicate implementation logic, so they may not catch real analytical mistakes. +- The EDA MVP is clear, but several edge cases around missing data, zero-row data, categorical denominators and large data are documented only lightly or not tested. +- Optional dependency handling is inconsistent: several functions guard suggested packages at runtime, while others call suggested packages directly. + +Highest-value fixes: + +- Tighten the exported API and avoid exporting implementation-adjacent helpers unless they are intentional user-facing functions. +- Replace circular EDA expected-output tests with committed fixture outputs or hand-computed expectations. +- Add edge-case tests for all-missing numeric EDA columns, zero-row EDA data, categorical percentages and missing-code semantics. +- Run `scripts/check-local.sh` and `scripts/check-cran.sh` after the review follow-up changes. + +Confirmed small correction made during this review: + +- Fixed the `epi_stats_numeric()` roxygen kurtosis wording from `lepto ~<3` to `lepto ~>3` in `R/epi_stats_numeric.R`. + +## B. Findings Table + +| Area | Severity | Location | Issue | Why it matters | Suggested fix | +| --- | --- | --- | --- | --- | --- | +| Public API stability | major | `R/utils-tidy-eval.R:33-42`, `NAMESPACE:3-11`, `NAMESPACE:96-104` | The package re-exports many `rlang` helpers and symbols (`quo`, `enquo`, `.data`, `:=`, etc.). | These exports widen the public API beyond episcout's domain and increase compatibility surface with little package-specific value. | Decide which, if any, tidy-eval helpers are intentionally public. Prefer importing internally and removing unnecessary re-exports in a scoped API cleanup. | +| Test quality | major | `tests/testthat/test-eda_summaries-fixtures.R:10-49`, `R/eda_summaries.R:51-98` | Fixture tests compute expected numeric and categorical summaries with nearly the same logic as implementation. | Duplicated implementation logic can let the same bug exist in both code and tests. | Store reviewed expected CSV outputs or use simpler hand-computed expectations, following the anti-circularity rule in `future/specs/002-penguins-raw-fixture/sdd.md:42-46`. | +| Statistical correctness | major | `R/eda_summaries.R:51-63` | All-missing numeric variables use `mean(..., na.rm = TRUE)`, `min(..., na.rm = TRUE)` and `max(..., na.rm = TRUE)` directly. | All-missing columns can produce `NaN`, `Inf`, `-Inf` and warnings in machine-readable summaries. | Reuse the stable empty/all-missing handling pattern from `epi_stats_numeric()` or add local safe scalar helpers with explicit tests. | +| User-facing semantics | major | `R/eda_missing.R:17-35`, `README.md:124-131` | The specification includes `missing_codes`, but missingness profiling only counts `NA`. | Epidemiological dictionaries often encode missing values as sentinel codes; ignoring them can understate missingness. | Clarify that `missing_codes` is reserved/not implemented or implement missing-code parsing under a numbered spec with tests. | +| Data semantics | minor | `R/eda_schema.R:87-96` | Logical variables are reported as `binary`, while integer variables are always reported as `numeric`. | A spec declaring integer or binary can look type-compatible or incompatible in surprising ways. | Define and test schema compatibility rules for `integer`, `numeric`, `binary` and labelled/factor variables. | +| EDA output interpretation | minor | `R/eda_summaries.R:82-94` | Categorical percentages divide by total row count, including missing values, without documenting whether the denominator is all rows or observed rows. | Both denominators are defensible, but users need predictable interpretation. | Document the denominator and add tests for categories with missing values. Consider including both `p_total` and `p_observed` if needed. | +| Dependency policy | minor | `R/epi_plot_parallel.R:67-74`, `DESCRIPTION:51-53` | `epi_plot_parallel()` calls `future::plan()` while `future` is only in `Suggests`; only `foreach` is explicitly checked. | If `future` is absent, the error comes from namespace lookup rather than the package's dependency guard. | Add `check_suggests(c("future", "doFuture", "foreach"))` or equivalent runtime checks before using suggested parallel packages. | +| CRAN check risk | minor | `R/epi_plot_parallel.R:66-71`, `R/epi_plot_parallel.R:123-128` | Parallel functions change the global future plan and then reset it. | Global plan mutation can surprise users and CRAN examples/tests if not tightly controlled. | Keep examples under `dontrun`, document the side effect, and test that the plan is restored. | +| Documentation accuracy | minor | `R/epi_stats_numeric.R:45-57` | Kurtosis interpretation text incorrectly described leptokurtic values as `<3`. | This is a statistical guidance error in a public helper. | Fixed in this review; regenerate Rd documentation before release. | +| Maintainability | note | `R/check_suggests.R:9-14`, `tests/testthat/test-check-suggests.R:1-6` | `check_suggests()` is internal but has roxygen docs and tests through `:::`. | This is acceptable, but it creates generated internal documentation and exposes internal naming to tests. | Keep if useful, but consider moving tests to user-facing functions that exercise missing suggested dependency paths. | + +## C. Test Gaps + +- `epi_eda_profile_summaries()`: all-missing numeric columns, zero-row data, non-numeric data under numeric spec, categorical levels with no observations, and missing categorical values. +- `epi_eda_profile_missing()`: zero-row data and specification `missing_codes` behaviour. +- `epi_eda_check_schema()`: integer versus numeric compatibility, binary/logical compatibility, factor versus character categorical variables, and date/datetime coercion expectations. +- `epi_eda_profile_plots()`: date histograms, high-cardinality text variables, non-syntactic column names, and missing `ggplot2` behaviour. +- `epi_plot_parallel()` and `epi_plot_save_parallel()`: missing suggested package errors and restoration of the caller's future plan. +- `epi_stats_numeric()`: non-numeric input rejection or coercion policy, infinite values, and `na.rm = FALSE` behaviour. + +## D. Statistical Concerns + +Confirmed issues: + +- `epi_stats_numeric()` contained incorrect kurtosis guidance in roxygen text; this review corrected the source comment. +- `epi_eda_profile_summaries()` can emit non-finite numeric summaries for all-missing numeric variables. +- `epi_eda_profile_missing()` does not use dictionary `missing_codes`, despite the README quickstart showing that column as part of the expected specification. + +Assumptions needing clarification: + +- Whether categorical percentages should use all rows or observed non-missing rows as the denominator. +- Whether schema validation should treat integer columns as compatible with numeric specs and `0/1` integer columns as compatible with binary specs. +- Whether synthetic data should intentionally ignore distributional assumptions and generate only structurally valid placeholder values. + +## E. Maintainability Concerns + +- The namespace is broader than the package story. Re-exporting general tidy-eval helpers makes `episcout` look like a programming helper package as well as an epidemiology EDA package. +- The EDA code is readable and simple, but tests should assert independent contracts rather than restating the implementation. +- Optional dependency checks should use one consistent helper pattern so missing package errors remain predictable. +- The current `future/` workflow is useful, but review-derived work should be promoted to a numbered SDD/TDD spec before package-code changes. + +## F. Suggested Next Commits + +1. Regenerate documentation after the kurtosis wording fix. +2. Add `004-senior-review-followups` with SDD/TDD scope for API cleanup, EDA edge cases and dependency guards. +3. Replace circular EDA summary fixture tests with committed expected outputs. +4. Harden `epi_eda_profile_summaries()` for all-missing numeric columns and document categorical percentage denominators. +5. Clarify or implement `missing_codes` semantics in the EDA missingness workflow. +6. Tighten optional dependency checks for plotting, reporting and parallel helpers. +7. Run `scripts/check-local.sh` and `scripts/check-cran.sh`. diff --git a/man/epi_clean_cond_numeric.Rd b/man/epi_clean_cond_numeric.Rd index 874668f..0606bed 100644 --- a/man/epi_clean_cond_numeric.Rd +++ b/man/epi_clean_cond_numeric.Rd @@ -42,5 +42,5 @@ epi_clean_cond_numeric(df[, "x"]) # should be TRUE \code{\link[base]{is.integer}}, \code{\link[base]{is.numeric}} } \author{ -Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> +Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} } diff --git a/man/epi_clean_make_names.Rd b/man/epi_clean_make_names.Rd index 572c032..2bd5986 100644 --- a/man/epi_clean_make_names.Rd +++ b/man/epi_clean_make_names.Rd @@ -43,5 +43,5 @@ epi_clean_make_names(valid_names) # There shouldn't be anything to correct \href{https://www.r-bloggers.com/2011/07/testing-for-valid-variable-names/}{r-bloggers test valid names}. } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_clean_merge_nested_dfs.Rd b/man/epi_clean_merge_nested_dfs.Rd index 2e0a5e5..ea25897 100644 --- a/man/epi_clean_merge_nested_dfs.Rd +++ b/man/epi_clean_merge_nested_dfs.Rd @@ -143,5 +143,5 @@ View(all_merged) \code{\link{epi_clean_transpose}}, \code{\link[base]{merge}}. } \author{ -Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> +Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} } diff --git a/man/epi_output_name.Rd b/man/epi_output_name.Rd index 2241d87..a9ef06c 100644 --- a/man/epi_output_name.Rd +++ b/man/epi_output_name.Rd @@ -30,5 +30,5 @@ dev.off() } \author{ -Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> +Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} } diff --git a/man/epi_plot_heatmap_triangle.Rd b/man/epi_plot_heatmap_triangle.Rd index d8e65d3..1975d80 100644 --- a/man/epi_plot_heatmap_triangle.Rd +++ b/man/epi_plot_heatmap_triangle.Rd @@ -84,5 +84,5 @@ ggsave("epi_heatmap_triangle.svg", \code{\link{epi_stats_corr_rename}}, \code{\link{epi_plot_heatmap}} } \author{ -Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> +Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} } diff --git a/man/epi_plot_list.Rd b/man/epi_plot_list.Rd index 4185b13..5d73556 100644 --- a/man/epi_plot_list.Rd +++ b/man/epi_plot_list.Rd @@ -24,5 +24,5 @@ See example in \code{\link{epi_plot_cow_save}} and ggplot2 wrappers epi_plot_*() \code{\link{epi_plot_cow_save}}. } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_read.Rd b/man/epi_read.Rd index 4662ca3..a7a3f88 100644 --- a/man/epi_read.Rd +++ b/man/epi_read.Rd @@ -53,5 +53,5 @@ super_data <- epi_read("super_data.tsv") \code{\link[data.table]{fread}}, \code{\link[tibble]{as.tibble}}. } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_stats_corr.Rd b/man/epi_stats_corr.Rd index 2a806f4..8149c97 100644 --- a/man/epi_stats_corr.Rd +++ b/man/epi_stats_corr.Rd @@ -54,5 +54,5 @@ class(cormat_all) \code{\link[data.table]{melt}}. } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_stats_numeric.Rd b/man/epi_stats_numeric.Rd index 68bbce2..cee11bc 100644 --- a/man/epi_stats_numeric.Rd +++ b/man/epi_stats_numeric.Rd @@ -61,7 +61,7 @@ row with unavailable summaries reported as \code{NA}. For skewness: negative/longer left tail, positive/longer right tail, values above 1 usually means non-normality. For kurtosis consider lower values, broader shape and longer tails (platy ~<3), -normal (meso ~3) and slender/no tails (lepto ~<3). +normal (meso ~3) and slender/no tails (lepto ~>3). Outliers are detected with the Tukey method (above and below 1.5 * IQR) or using the multiplier \code{coef}. Coefficient of variation (\code{CV}) is calculated as \code{SD / mean}. You may pass further arguments (e.g., diff --git a/man/epi_utils_log.Rd b/man/epi_utils_log.Rd index b2061a7..40d759a 100644 --- a/man/epi_utils_log.Rd +++ b/man/epi_utils_log.Rd @@ -39,5 +39,5 @@ epi_utils_log("xy") \code{\link[episcout]{epi_utils_session}}. } \author{ -Antonio J Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> +Antonio J Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} } From 59e777f3d72ebe1bb32837de10de87eee190d56a Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Mon, 15 Jun 2026 17:40:51 -0600 Subject: [PATCH 29/33] added todos in future, sdd tdd for review findings, started work on review findings for epi_eda* edge cases --- AGENTS.MD | 26 ++- NEWS.md | 3 + R/eda_missing.R | 37 ++- R/eda_summaries.R | 68 +++++- R/epi_plot_parallel.R | 11 +- README.md | 8 +- future/README.md | 9 +- future/TODOs.md | 27 --- future/changelog.md | 26 +++ .../ideas/eda-stats-helper-boundary-idea.md | 212 ++++++++++++++++++ .../manifest.yml | 6 +- .../001-phase-1-helper-stabilization/tdd.md | 8 +- .../002-penguins-raw-fixture/manifest.yml | 4 +- future/specs/002-penguins-raw-fixture/tdd.md | 4 +- .../manifest.yml | 4 +- .../003-large-data-backend-strategy/tdd.md | 4 +- .../004-senior-review-followups/acceptance.md | 22 ++ .../004-senior-review-followups/brief.md | 55 +++++ .../004-senior-review-followups/manifest.yml | 41 ++++ .../004-senior-review-followups/review.md | 24 ++ .../specs/004-senior-review-followups/sdd.md | 99 ++++++++ .../specs/004-senior-review-followups/tdd.md | 60 +++++ future/specs/template/manifest.yml | 4 +- future/specs/template/tdd.md | 6 +- man/epi_clean_add_colname_suffix.Rd | 2 +- man/epi_clean_add_rep_num.Rd | 2 +- man/epi_clean_cond_chr_fct.Rd | 2 +- man/epi_clean_cond_numeric.Rd | 2 +- man/epi_clean_get_dups.Rd | 2 +- man/epi_clean_make_names.Rd | 2 +- man/epi_clean_spread_repeated.Rd | 2 +- man/epi_clean_transpose.Rd | 2 +- man/epi_eda_profile_missing.Rd | 3 +- man/epi_eda_profile_summaries.Rd | 7 +- man/epi_plot_box.Rd | 2 +- man/epi_plot_list.Rd | 2 +- man/epi_read.Rd | 2 +- man/epi_stats_summary.Rd | 2 +- man/epi_stats_tidy.Rd | 2 +- man/epi_utils_log.Rd | 2 +- scripts/check-local.sh | 6 +- tests/testthat/test-eda_missing-fixtures.R | 47 ++++ tests/testthat/test-eda_schema-fixtures.R | 21 ++ tests/testthat/test-eda_summaries-fixtures.R | 97 ++++---- tests/testthat/test-plot-parallel.R | 14 ++ vignettes/specification-first-eda.Rmd | 7 +- 46 files changed, 847 insertions(+), 151 deletions(-) create mode 100644 future/ideas/eda-stats-helper-boundary-idea.md create mode 100644 future/specs/004-senior-review-followups/acceptance.md create mode 100644 future/specs/004-senior-review-followups/brief.md create mode 100644 future/specs/004-senior-review-followups/manifest.yml create mode 100644 future/specs/004-senior-review-followups/review.md create mode 100644 future/specs/004-senior-review-followups/sdd.md create mode 100644 future/specs/004-senior-review-followups/tdd.md diff --git a/AGENTS.MD b/AGENTS.MD index c3d1123..bd34b47 100644 --- a/AGENTS.MD +++ b/AGENTS.MD @@ -47,6 +47,14 @@ Do not use bare `Rscript` unless you have verified it resolves to the `episcout` mamba environment. To use a different R binary, set `EPISCOUT_RSCRIPT`; see `scripts/rscript_env_caller.R`. +For `devtools` checks, set a real CRAN mirror inside the R expression. Do not +leave `repos` as the placeholder `@CRAN@`, because package checks may spend a +long time probing package repositories. + +```bash +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" +``` + ## Repo Script Discovery and Naming Before adding or changing helper commands, inspect existing project entry @@ -126,13 +134,13 @@ All new functions must include unit tests in `/tests/testthat/`. Run tests locally before committing: ```bash -Rscript -e "devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" ``` Use `covr` to monitor test coverage: ```bash -Rscript -e "covr::report()" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); covr::report()" ``` Ensure test coverage does not decrease for existing functions. @@ -145,19 +153,19 @@ Always update documentation and verify build integrity before PRs. ```bash # Regenerate documentation -Rscript -e "devtools::document()" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::document()" # Build the package tarball R CMD build . --no-resave-data --compact-vignettes=gs+qpdf # Run CRAN-like checks -R CMD check --no-manual --as-cran +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" ``` Optional: rebuild vignettes manually ```bash -Rscript -e "devtools::build_vignettes()" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::build_vignettes()" ``` --- @@ -201,10 +209,10 @@ Examples: Run all checks **before requesting a review**: ```bash -Rscript -e "styler::style_pkg()" -Rscript -e "lintr::lint_package()" -Rscript -e "devtools::check(manual = FALSE)" -Rscript -e "covr::report()" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); styler::style_pkg()" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); lintr::lint_package()" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); covr::report()" ``` All of the above must pass before merging. diff --git a/NEWS.md b/NEWS.md index 03fa482..1a6c306 100644 --- a/NEWS.md +++ b/NEWS.md @@ -12,6 +12,9 @@ - Added fixture-backed tests for the EDA workflow using a pinned `blood_storage` fixture. - Added an HTML EDA report template. +- EDA missingness and summaries now apply specification `missing_codes`. +- Categorical EDA summaries now include `p_observed` alongside the existing + total-row denominator `p`. - Documented current MVP limits: basic summaries and plots, synthetic data for pipeline preparation only, and no Arrow, DuckDB or data.table EDA backends yet. - Cleaned up Rd documentation and CRAN-check issues on the development branch. diff --git a/R/eda_missing.R b/R/eda_missing.R index e7e56fc..db59668 100644 --- a/R/eda_missing.R +++ b/R/eda_missing.R @@ -1,6 +1,7 @@ #' Profile missingness using an EDA specification #' -#' Count standard missing values for variables listed in an EDA specification. +#' Count standard missing values and configured missing codes for variables +#' listed in an EDA specification. #' #' @param data A data frame to profile. #' @param spec An EDA specification data frame or CSV path. @@ -22,7 +23,7 @@ epi_eda_profile_missing <- function(data, spec) { if (!name %in% names(data)) { return(NA_integer_) } - sum(is.na(data[[name]])) + sum(eda_missing_mask(data[[name]], eda_missing_codes(spec, name))) }, integer(1) )) @@ -31,7 +32,37 @@ epi_eda_profile_missing <- function(data, spec) { name = spec$name, n = n, n_missing = n_missing, - p_missing = n_missing / n, + p_missing = if (n > 0) n_missing / n else rep(NA_real_, length(n_missing)), stringsAsFactors = FALSE ) } + +eda_missing_codes <- function(spec, name) { + if (!("missing_codes" %in% names(spec))) { + return(character()) + } + + row_index <- match(name, spec$name) + if (is.na(row_index)) { + return(character()) + } + + value <- spec$missing_codes[[row_index]] + if (is.na(value) || !nzchar(trimws(as.character(value)))) { + return(character()) + } + + codes <- trimws(strsplit(as.character(value), ";", fixed = TRUE)[[1]]) + codes[nzchar(codes)] +} + +eda_missing_mask <- function(values, missing_codes = character()) { + missing <- is.na(values) + + if (length(missing_codes) == 0) { + return(missing) + } + + values_chr <- as.character(values) + missing | (!is.na(values_chr) & values_chr %in% missing_codes) +} diff --git a/R/eda_summaries.R b/R/eda_summaries.R index 3e18506..135d513 100644 --- a/R/eda_summaries.R +++ b/R/eda_summaries.R @@ -1,12 +1,15 @@ #' Profile summaries using an EDA specification #' #' Produce machine-readable descriptive summaries for variables listed in a -#' specification-first EDA data dictionary. +#' specification-first EDA data dictionary. Standard `NA` values and configured +#' `missing_codes` are excluded from observed summaries. #' #' @param data A data frame containing observed data. #' @param spec An EDA specification data frame or CSV path. #' -#' @return A named list with `numeric` and `categorical` data frames. +#' @return A named list with `numeric` and `categorical` data frames. In +#' categorical summaries, `p` uses total rows as the denominator and +#' `p_observed` uses observed non-missing rows. #' #' @export epi_eda_profile_summaries <- function(data, spec) { @@ -50,15 +53,17 @@ profile_summaries_numeric <- function(data, spec) { rows <- lapply(numeric_spec$name, function(name) { values <- data[[name]] + missing <- eda_missing_mask(values, eda_missing_codes(spec, name)) + observed <- values[!missing] data.frame( name = name, n = length(values), - n_missing = sum(is.na(values)), - mean = mean(values, na.rm = TRUE), - sd = stats::sd(values, na.rm = TRUE), - median = stats::median(values, na.rm = TRUE), - min = min(values, na.rm = TRUE), - max = max(values, na.rm = TRUE), + n_missing = sum(missing), + mean = eda_safe_numeric_summary(observed, mean), + sd = eda_safe_numeric_summary(observed, stats::sd), + median = eda_safe_numeric_summary(observed, stats::median), + min = eda_safe_numeric_summary(observed, min), + max = eda_safe_numeric_summary(observed, max), stringsAsFactors = FALSE ) }) @@ -75,25 +80,66 @@ profile_summaries_categorical <- function(data, spec) { level = character(), n = integer(), p = numeric(), + p_observed = numeric(), stringsAsFactors = FALSE )) } rows <- lapply(categorical_spec$name, function(name) { values <- data[[name]] - observed <- as.character(values[!is.na(values)]) + missing <- eda_missing_mask(values, eda_missing_codes(spec, name)) + observed <- as.character(values[!missing]) levels_value <- categorical_spec$levels[categorical_spec$name == name] - levels <- strsplit(levels_value, ";", fixed = TRUE)[[1]] + levels <- eda_spec_levels(levels_value) + if (length(levels) == 0) { + levels <- sort(unique(observed)) + } counts <- vapply(levels, function(level) sum(observed == level), integer(1)) + total_n <- length(values) + observed_n <- length(observed) data.frame( name = name, level = levels, n = as.integer(counts), - p = as.numeric(counts) / length(values), + p = eda_safe_proportion(counts, total_n), + p_observed = eda_safe_proportion(counts, observed_n), stringsAsFactors = FALSE ) }) do.call(rbind, rows) } + +eda_safe_numeric_summary <- function(values, fun) { + if (length(values) == 0) { + return(NA_real_) + } + + value <- tryCatch( + suppressWarnings(fun(values)), + error = function(e) NA_real_ + ) + if (length(value) == 0 || is.nan(value)) { + return(NA_real_) + } + + as.numeric(value) +} + +eda_safe_proportion <- function(counts, denominator) { + if (denominator == 0) { + return(rep(NA_real_, length(counts))) + } + + as.numeric(counts) / denominator +} + +eda_spec_levels <- function(levels_value) { + if (length(levels_value) == 0 || is.na(levels_value[[1]])) { + return(character()) + } + + levels <- trimws(strsplit(as.character(levels_value[[1]]), ";", fixed = TRUE)[[1]]) + levels[nzchar(levels)] +} diff --git a/R/epi_plot_parallel.R b/R/epi_plot_parallel.R index a953045..d75d51a 100644 --- a/R/epi_plot_parallel.R +++ b/R/epi_plot_parallel.R @@ -43,6 +43,8 @@ epi_plot_parallel <- function(df, num_cores = NULL, future_plan = "multisession", ...) { + check_suggests(c("future", "doFuture", "foreach", "iterators", "parallel")) + if (is.null(vars_to_plot)) { if (!requireNamespace("dplyr", quietly = TRUE)) { stop("Package dplyr needed for this function to work. Please install it.", @@ -57,12 +59,7 @@ epi_plot_parallel <- function(df, ) vars_to_plot <- names(cond) } - if (!requireNamespace("foreach", quietly = TRUE)) { - stop("Package foreach needed for this function to work. Please install it.", - call. = FALSE - ) - } - + # Capture existing plan for cleanup prev_plan <- future::plan() on.exit({ @@ -116,6 +113,8 @@ epi_plot_save_parallel <- function(plot_list, num_cores = NULL, future_plan = "multisession", ...) { + check_suggests(c("future", "doFuture", "foreach", "iterators", "parallel")) + if (!length(plot_list)) { stop("plot_list must contain at least one plot") } diff --git a/README.md b/README.md index dd90566..e4dc5fa 100644 --- a/README.md +++ b/README.md @@ -126,10 +126,16 @@ Start from a data dictionary with at least these columns: ``` csv name,label,type,role,units,levels,min,max,missing_codes,required,group,description age,Age at baseline,numeric,covariate,years,,18,110,,TRUE,demographics,Age in years -sex,Sex at birth,categorical,covariate,,"Female;Male;Unknown",,,,TRUE,demographics,Recorded sex +sex,Sex at birth,categorical,covariate,,"Female;Male;Unknown",,,Unknown,TRUE,demographics,Recorded sex death,Death during follow-up,binary,outcome,,"0;1",0,1,,TRUE,outcomes,Outcome indicator ``` +The optional `missing_codes` column accepts semicolon-separated sentinel values +such as `Unknown;Refused`. These values are counted as missing in +`epi_eda_profile_missing()` and excluded from observed EDA summaries. In +categorical summaries, `p` uses all rows as the denominator and `p_observed` +uses only observed non-missing rows. + You can prepare the workflow before real data arrive by generating synthetic data from the same specification: diff --git a/future/README.md b/future/README.md index 2ffead4..997f4fb 100644 --- a/future/README.md +++ b/future/README.md @@ -38,14 +38,17 @@ future/ Use the repo-local wrapper for R commands: ```bash -scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" -scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" ``` -Do not use bare `Rscript` in future specs or check instructions. +Do not use bare `Rscript` in future specs or check instructions. For +`devtools::check()`, always set an explicit CRAN mirror instead of relying on +`@CRAN@`; otherwise checks can spend a long time probing repository indexes. ## Active Specs - `001-phase-1-helper-stabilization` - `002-penguins-raw-fixture` - `003-large-data-backend-strategy` +- `004-senior-review-followups` diff --git a/future/TODOs.md b/future/TODOs.md index ea3fae0..b16a83f 100644 --- a/future/TODOs.md +++ b/future/TODOs.md @@ -31,17 +31,6 @@ For simple tasks only use e.g. `- [ ] xxx` ### Priority 1 -- Short title: Address senior R package review follow-ups - - Problem: The review in `future/reviews/2026-06-15-senior-r-package-review.md` found API-surface, test-design, EDA edge-case and dependency-guard risks. - - Goal: Convert the review findings into a scoped SDD/TDD implementation plan before package-code changes. - - User need: Keep the package maintainable and CRAN-ready without widening the current branch ad hoc. - - Proposed scope: Create a numbered spec for the highest-value findings: EDA summary edge cases, missing-code semantics, non-circular fixture expectations, public API cleanup and optional dependency guards. - - Out of scope: Large-data backend implementation and unrelated helper rewrites. - - Candidate files: `future/reviews/2026-06-15-senior-r-package-review.md`, `future/specs/`, `R/eda_*`, `R/epi_plot_parallel.R`, `R/utils-tidy-eval.R`, `tests/testthat/`. - - Risks: API cleanup may be breaking if current users rely on accidental exports. - - Suggested spec ID: `004-senior-review-followups` - - - [ ] get instructions to download, install, episcout2 branch - [ ] human live walkthrough, no agent needed here - [ ] carry out changes needed from human review @@ -63,19 +52,3 @@ For simple tasks only use e.g. `- [ ] xxx` - [ ] Consider visual-regression strategy for EDA plots only after plot contracts are stable. - [ ] Add biomedical EDA extensions as separate numbered specs. - -## Done - -- [x] thorough code review, use prompt in `future/prompts/senior-r-package-review.md` - - Review written to `future/reviews/2026-06-15-senior-r-package-review.md`. - - Corrected the confirmed `epi_stats_numeric()` kurtosis documentation issue. - - Planned follow-up work as `004-senior-review-followups` candidate. -- [x] Completed spec: `001-phase-1-helper-stabilization` - - Baseline recorded before package-code changes. - - TDD edge-case tests added before implementation. - - Implementation scoped to `epi_stats_numeric()` and `epi_stats_na_perc()`. - - Final `devtools::test(reporter = 'summary')` passed. -- [x] check future work has been setup properly for SDD-TDD: - - PR14 - - add penguins_raw - - ? diff --git a/future/changelog.md b/future/changelog.md index aa2c30a..9f33a8f 100644 --- a/future/changelog.md +++ b/future/changelog.md @@ -12,3 +12,29 @@ - Added templates, prompts, todos, references and review folders. - Implemented `specs/001-phase-1-helper-stabilization/` with TDD coverage for numeric summary and missingness helper edge cases. +- Implemented `specs/004-senior-review-followups/` with EDA `missing_codes`, + non-circular summary tests, stable all-missing summaries, categorical + `p_observed`, and parallel dependency guards. +- Standardised future R check commands to set an explicit CRAN mirror + (`https://cloud.r-project.org`) before `devtools` checks. +- Completed a thorough code review using + `future/prompts/senior-r-package-review.md`. + - Wrote the review to + `future/reviews/2026-06-15-senior-r-package-review.md`. + - Corrected the confirmed `epi_stats_numeric()` kurtosis documentation issue. + - Planned follow-up work as `004-senior-review-followups` candidate. +- Completed spec `004-senior-review-followups`. + - Implemented EDA `missing_codes` handling in missingness and summaries. + - Replaced circular summary expectations with hand-computed tests. + - Added stable all-missing numeric summary behavior and categorical + `p_observed`. + - Deferred tidy-eval public API cleanup. + - Ran final `devtools::test(reporter = 'summary')`, which passed. + - `devtools::check(manual = FALSE)` passed with one existing `.gitkeep` NOTE. +- Completed spec `001-phase-1-helper-stabilization`. + - Recorded the baseline before package-code changes. + - Added TDD edge-case tests before implementation. + - Scoped implementation to `epi_stats_numeric()` and `epi_stats_na_perc()`. + - Ran final `devtools::test(reporter = 'summary')`, which passed. +- Reviewed future work setup for SDD/TDD, including PR14 and `penguins_raw` + follow-ups. diff --git a/future/ideas/eda-stats-helper-boundary-idea.md b/future/ideas/eda-stats-helper-boundary-idea.md new file mode 100644 index 0000000..5df97f0 --- /dev/null +++ b/future/ideas/eda-stats-helper-boundary-idea.md @@ -0,0 +1,212 @@ +# Idea: EDA and Stats Helper Boundaries + +Status: Idea to consider +Suggested spec ID: `005-eda-stats-helper-boundaries` +Date captured: 2026-06-15 + +## Summary + +The current `epi_eda_*` workflow mostly does not call `epi_stats_*` helpers. +That is not automatically a problem: the EDA workflow is specification-first +and returns smaller, machine-readable outputs, while the older `epi_stats_*` +helpers expose broader descriptive-statistics APIs. + +The risk is duplicated edge-case logic. Numeric summaries and missingness are +computed directly in the EDA layer, even though similar behaviours exist in +`epi_stats_numeric()` and `epi_stats_na_perc()`. Future work should decide +whether to share small internal helper functions rather than make EDA depend +directly on the full exported `epi_stats_*` APIs. + +## Checks Performed + +Commands used for inspection: + +```bash +rg --files R tests DESCRIPTION NAMESPACE future | sort +rg -n "epi_eda_|epi_stats_|@import|::|library\\(|require\\(|Imports|Suggests" R DESCRIPTION NAMESPACE tests future +sed -n '1,220p' R/eda_summaries.R +sed -n '1,220p' R/eda_missing.R +sed -n '1,240p' R/eda_schema.R +sed -n '1,220p' R/eda_plots.R +sed -n '1,260p' R/eda_spec.R +sed -n '1,260p' R/run_eda.R +sed -n '1,280p' R/eda_report.R +sed -n '1,260p' R/eda_synthetic.R +sed -n '1,140p' DESCRIPTION +sed -n '1,230p' R/epi_stats_numeric.R +sed -n '1,180p' R/epi_stats_na_perc.R +sed -n '1,160p' R/epi_stats_summary.R +sed -n '1,150p' tests/testthat/test-eda_summaries-fixtures.R +``` + +Observed call structure: + +- `epi_eda_run()` orchestrates `epi_eda_check_schema()`, + `epi_eda_profile_missing()`, `epi_eda_profile_summaries()` and + `epi_eda_profile_plots()`. +- `epi_eda_profile_summaries()` calls `epi_eda_spec()` and internal helpers + `profile_summaries_numeric()` and `profile_summaries_categorical()`. +- `profile_summaries_numeric()` computes `n`, `n_missing`, `mean`, `sd`, + `median`, `min` and `max` directly using base R and `stats`. +- `epi_eda_profile_missing()` computes missingness directly with + `sum(is.na(...))`. +- `epi_eda_profile_plots()` uses `requireNamespace("ggplot2")` and then + `ggplot2::` calls. +- `epi_eda_render_report()` uses `requireNamespace("rmarkdown")` and + `rmarkdown::render()`. +- No inspected `epi_eda_*` function directly calls `epi_stats_numeric()`, + `epi_stats_na_perc()`, `epi_stats_summary()` or another exported + `epi_stats_*` summary helper. + +## Why This Exists + +Direct reuse of `epi_stats_numeric()` inside EDA is not obviously correct. +`epi_stats_numeric()` returns a wide, rich statistics row with quantiles, IQR, +CV, variance, SEM, skewness, kurtosis, Shapiro-Wilk p-value and Tukey outlier +fields. The EDA summary contract is much smaller and tied to a data dictionary: +`name`, `n`, `n_missing`, `mean`, `sd`, `median`, `min`, `max`. + +Direct reuse of `epi_stats_na_perc()` also is not a perfect fit. Its output is +row- or column-indexed and uses `na_counts` and `na_perc`, while +`epi_eda_profile_missing()` returns one row per spec variable with `name`, `n`, +`n_missing` and `p_missing`. + +So the recommended direction is not "make EDA call every matching stats +function". The better direction is to share narrow internal primitives for +behaviour that must stay consistent, especially safe scalar summaries and +missing-proportion handling. + +## Where To Look + +Primary EDA files: + +- `R/run_eda.R`: EDA workflow orchestration and output writing. +- `R/eda_summaries.R`: numeric and categorical EDA summary logic. +- `R/eda_missing.R`: EDA missingness summary logic. +- `R/eda_plots.R`: EDA plot generation and `ggplot2` dependency guard. +- `R/eda_report.R`: report rendering and `rmarkdown` dependency guard. +- `R/eda_spec.R`: spec parsing and validation. +- `R/eda_synthetic.R`: synthetic data generation from specs. + +Related stats files: + +- `R/epi_stats_numeric.R`: richer numeric summaries and safer empty/all-missing + handling. +- `R/epi_stats_na_perc.R`: row/column missingness summaries. +- `R/epi_stats_summary.R`: older multi-column summary wrapper that already + delegates to `epi_stats_numeric()` for numeric columns in one path. + +Related tests: + +- `tests/testthat/test-eda_summaries-fixtures.R` +- `tests/testthat/test-eda_missing-fixtures.R` +- `tests/testthat/test-run_eda-fixtures.R` +- `tests/testthat/test-epi_stats_numeric.R` +- `tests/testthat/test-stats.R` + +## Future SDD Construction + +Create a numbered spec only if this becomes active work. The SDD should make +these decisions explicit: + +- Goal: align EDA and stats edge-case behaviour without changing public + `epi_eda_*` or `epi_stats_*` return contracts unless a specific bug requires + it. +- Preferred implementation: extract small unexported helper(s) for safe numeric + scalar summaries and missing proportions, then call them from both relevant + EDA and stats code where it reduces duplication. +- Avoid direct dependency from EDA summaries to the full `epi_stats_numeric()` + output unless the EDA output schema is intentionally expanded. +- Preserve existing EDA CSV output column names written by `epi_eda_run()`. +- Preserve existing `epi_stats_numeric()` and `epi_stats_na_perc()` public + argument names and documented return shapes. +- Do not add new package dependencies. +- Keep changes scoped to summary/missingness behaviour; plotting, schema + validation, synthetic data and report rendering should remain out of scope + unless tests reveal a directly related issue. + +Suggested SDD sections: + +- Scope: EDA numeric summaries, EDA missingness and shared internal helper + behaviour. +- Desired behaviour: empty, all-missing, mixed missing/non-missing, zero-row and + non-numeric edge cases. +- Compatibility: explicit list of preserved public output columns and any + intentional bug-fix changes. +- Out of scope: broad EDA redesign, richer statistical summaries, plotting + redesign and dependency changes. + +## Future TDD Construction + +The TDD should start from executable tests before package-code changes. + +Baseline commands: + +```bash +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" +``` + +Focused test files: + +- Add or extend `tests/testthat/test-eda_summaries-fixtures.R` for EDA summary + edge cases. +- Add or extend `tests/testthat/test-eda_missing-fixtures.R` for missingness + edge cases. +- Extend `tests/testthat/test-epi_stats_numeric.R` and + `tests/testthat/test-stats.R` only if shared helper changes affect stats + behaviour. + +Required test scenarios: + +- All-missing numeric EDA variable returns finite-compatible missing summary + values, preferably `NA_real_` for unavailable statistics rather than `NaN`, + `Inf` or `-Inf`. +- Zero-row EDA data returns stable summary and missingness shapes. +- Mixed missing/non-missing numeric variables keep existing valid calculations. +- EDA missingness keeps `name`, `n`, `n_missing` and `p_missing` columns. +- Stats helper outputs keep their current public column names. +- Shared helper changes do not introduce `ggplot2`, `rmarkdown` or tidyverse + requirements into non-plot/non-report EDA paths. + +Acceptance commands: + +```bash +scripts/rscript_env_caller.R -e "devtools::load_all(); testthat::test_file('tests/testthat/test-eda_summaries-fixtures.R')" +scripts/rscript_env_caller.R -e "devtools::load_all(); testthat::test_file('tests/testthat/test-eda_missing-fixtures.R')" +scripts/rscript_env_caller.R -e "devtools::load_all(); testthat::test_file('tests/testthat/test-epi_stats_numeric.R')" +scripts/rscript_env_caller.R -e "devtools::load_all(); testthat::test_file('tests/testthat/test-stats.R')" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" +``` + +## Machine-Readable Handoff + +```yaml +idea_id: eda-stats-helper-boundaries +status: idea +suggested_spec_id: 005-eda-stats-helper-boundaries +primary_question: Should EDA and stats share narrow internal helpers for duplicated summary and missingness edge-case behaviour? +recommendation: Extract small unexported primitives if future tests show duplicated behaviour must stay aligned; do not make EDA directly depend on full exported epi_stats_* output schemas. +primary_files: + - R/eda_summaries.R + - R/eda_missing.R + - R/run_eda.R + - R/epi_stats_numeric.R + - R/epi_stats_na_perc.R +primary_tests: + - tests/testthat/test-eda_summaries-fixtures.R + - tests/testthat/test-eda_missing-fixtures.R + - tests/testthat/test-epi_stats_numeric.R + - tests/testthat/test-stats.R +public_contracts_to_preserve: + - epi_eda_profile_summaries return list names: numeric, categorical + - EDA numeric summary columns: name, n, n_missing, mean, sd, median, min, max + - EDA missingness columns: name, n, n_missing, p_missing + - epi_stats_numeric public arguments and return columns + - epi_stats_na_perc public arguments and return columns +out_of_scope: + - plotting redesign + - report rendering redesign + - synthetic data redesign + - new dependencies +``` diff --git a/future/specs/001-phase-1-helper-stabilization/manifest.yml b/future/specs/001-phase-1-helper-stabilization/manifest.yml index 0f024ca..df56513 100644 --- a/future/specs/001-phase-1-helper-stabilization/manifest.yml +++ b/future/specs/001-phase-1-helper-stabilization/manifest.yml @@ -23,9 +23,9 @@ documents: review_required: true check_commands: - - scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" - - scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" - - scripts/rscript_env_caller.R -e "covr::package_coverage()" + - scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" + - scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" + - scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); covr::package_coverage()" notes: - Add tests before implementation. diff --git a/future/specs/001-phase-1-helper-stabilization/tdd.md b/future/specs/001-phase-1-helper-stabilization/tdd.md index 9153595..92aae16 100644 --- a/future/specs/001-phase-1-helper-stabilization/tdd.md +++ b/future/specs/001-phase-1-helper-stabilization/tdd.md @@ -8,9 +8,9 @@ Status: Implemented Run before implementation where practical: ```bash -scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" -scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" -scripts/rscript_env_caller.R -e "covr::package_coverage()" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); covr::package_coverage()" ``` ## Test Files @@ -38,5 +38,5 @@ Use a narrower new test file only if it makes the edge-case contracts clearer. ```bash scripts/rscript_env_caller.R -e "devtools::load_all(); testthat::test_file('tests/testthat/test-epi_stats_numeric.R')" scripts/rscript_env_caller.R -e "devtools::load_all(); testthat::test_file('tests/testthat/test-stats.R')" -scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" ``` diff --git a/future/specs/002-penguins-raw-fixture/manifest.yml b/future/specs/002-penguins-raw-fixture/manifest.yml index 80b2f6e..03c6810 100644 --- a/future/specs/002-penguins-raw-fixture/manifest.yml +++ b/future/specs/002-penguins-raw-fixture/manifest.yml @@ -23,8 +23,8 @@ documents: review_required: true check_commands: - - scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" - - scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" + - scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" + - scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" notes: - Do not add palmerpenguins to Imports. diff --git a/future/specs/002-penguins-raw-fixture/tdd.md b/future/specs/002-penguins-raw-fixture/tdd.md index fba6fa8..57308e3 100644 --- a/future/specs/002-penguins-raw-fixture/tdd.md +++ b/future/specs/002-penguins-raw-fixture/tdd.md @@ -43,6 +43,6 @@ Do not run regeneration scripts from routine tests. ## Acceptance Commands ```bash -scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" -scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" ``` diff --git a/future/specs/003-large-data-backend-strategy/manifest.yml b/future/specs/003-large-data-backend-strategy/manifest.yml index e4b497c..86c7482 100644 --- a/future/specs/003-large-data-backend-strategy/manifest.yml +++ b/future/specs/003-large-data-backend-strategy/manifest.yml @@ -21,8 +21,8 @@ documents: review_required: true check_commands: - - scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" - - scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" + - scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" + - scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" notes: - Design-only first. diff --git a/future/specs/003-large-data-backend-strategy/tdd.md b/future/specs/003-large-data-backend-strategy/tdd.md index 5c4293f..3f5b20c 100644 --- a/future/specs/003-large-data-backend-strategy/tdd.md +++ b/future/specs/003-large-data-backend-strategy/tdd.md @@ -19,6 +19,6 @@ required unless package-facing behaviour changes. ## Future Acceptance Commands ```bash -scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" -scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" ``` diff --git a/future/specs/004-senior-review-followups/acceptance.md b/future/specs/004-senior-review-followups/acceptance.md new file mode 100644 index 0000000..ed443eb --- /dev/null +++ b/future/specs/004-senior-review-followups/acceptance.md @@ -0,0 +1,22 @@ +# Acceptance + +Spec ID: `004-senior-review-followups` +Status: Implemented + +- [x] SDD is complete before package-code implementation. +- [x] TDD plan is complete before package-code implementation. +- [x] Executable tests are under `tests/testthat/`. +- [x] Package code changes are scoped to this spec. +- [x] Documentation is updated for missing-code and denominator behavior. +- [x] Roxygen documentation is regenerated. +- [x] Tidy-eval re-export cleanup is deferred, not implemented. +- [x] Checks listed in `manifest.yml` are run or explicitly deferred. +- [x] Review notes are recorded in `review.md`. + +## Check Results + +- `scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')"`: + passed with existing skips and one existing parallel-worker warning. +- `scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)"`: + passed with 0 errors, 0 warnings and 1 NOTE for bundled `.gitkeep` files in + `inst/project-template/`. diff --git a/future/specs/004-senior-review-followups/brief.md b/future/specs/004-senior-review-followups/brief.md new file mode 100644 index 0000000..9ac2112 --- /dev/null +++ b/future/specs/004-senior-review-followups/brief.md @@ -0,0 +1,55 @@ +# Brief + +Spec ID: `004-senior-review-followups` +Status: Implemented +Owner: TBD + +## Problem + +The senior package review in +`future/reviews/2026-06-15-senior-r-package-review.md` identified focused +maintenance risks before the development branch can be treated as stable: + +- EDA missingness ignores data-dictionary `missing_codes`. +- EDA summary tests duplicate implementation logic. +- Numeric EDA summaries are unstable for all-missing variables. +- Categorical summary percentage denominators are not explicit. +- Some optional dependency failures are indirect. +- Tidy-eval re-exports broaden the public API, but changing them may be + compatibility-breaking. + +## Goal + +Lock and implement small, reviewable follow-ups that improve EDA correctness, +test quality and user-facing error clarity without broad package redesign. + +## Non-goals + +- Removing or deprecating tidy-eval re-exports. +- Implementing large-data backends. +- Redesigning the full EDA schema API. +- Adding new package dependencies. +- Rewriting unrelated helper functions. + +## Candidate Files + +- `R/eda_missing.R` +- `R/eda_summaries.R` +- `R/eda_schema.R` +- `R/epi_plot_parallel.R` +- `R/eda_report.R` +- `tests/testthat/test-eda_missing-fixtures.R` +- `tests/testthat/test-eda_summaries-fixtures.R` +- `tests/testthat/test-eda_schema-fixtures.R` +- `tests/testthat/test-plot-parallel.R` +- `README.md` +- `vignettes/specification-first-eda.Rmd` + +## Risks + +- Applying `missing_codes` changes summary and missingness output for data sets + that already use sentinel values. +- Adding `p_observed` changes categorical summary output shape. +- Tightening dependency guards can change exact error messages. +- Public API cleanup is intentionally deferred because removing accidental + exports may break existing users. diff --git a/future/specs/004-senior-review-followups/manifest.yml b/future/specs/004-senior-review-followups/manifest.yml new file mode 100644 index 0000000..944ef76 --- /dev/null +++ b/future/specs/004-senior-review-followups/manifest.yml @@ -0,0 +1,41 @@ +id: "004" +slug: "senior-review-followups" +status: implemented +owner: TBD +base_branch: episcout2 +working_branch: TBD +created: 2026-06-15 +updated: 2026-06-15 + +related_files: + - future/reviews/2026-06-15-senior-r-package-review.md + - R/eda_missing.R + - R/eda_summaries.R + - R/eda_schema.R + - R/epi_plot_parallel.R + - R/eda_report.R + - tests/testthat/test-eda_missing-fixtures.R + - tests/testthat/test-eda_summaries-fixtures.R + - tests/testthat/test-eda_schema-fixtures.R + - tests/testthat/test-plot-parallel.R + - tests/testthat/test-eda_report.R + - README.md + - vignettes/specification-first-eda.Rmd + +documents: + brief: brief.md + sdd: sdd.md + tdd: tdd.md + acceptance: acceptance.md + review: review.md + +review_required: true + +check_commands: + - scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" + - scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" + +notes: + - Add tests before package-code implementation. + - Do not implement tidy-eval export cleanup in this spec. + - Keep large-data backend work in spec 003. diff --git a/future/specs/004-senior-review-followups/review.md b/future/specs/004-senior-review-followups/review.md new file mode 100644 index 0000000..dcfdb32 --- /dev/null +++ b/future/specs/004-senior-review-followups/review.md @@ -0,0 +1,24 @@ +# Review + +Spec ID: `004-senior-review-followups` +Status: Implemented + +## Pre-implementation Review + +- Source review: `future/reviews/2026-06-15-senior-r-package-review.md` +- Tidy-eval public API cleanup is intentionally deferred for a separate + compatibility decision. + +## Post-implementation Review + +- Implemented missing-code handling for EDA missingness and summaries. +- Added stable all-missing numeric summary behavior. +- Added categorical `p_observed` while preserving `p` as total-row denominator. +- Added tests that use hand-computed expectations instead of implementation-copy + fixture calculations. +- Added descriptive schema tests for integer/numeric and logical/binary current + behavior. +- Added explicit optional dependency guards in parallel plotting functions. +- Deferred tidy-eval re-export cleanup as planned. +- Full test suite passed. `devtools::check(manual = FALSE)` passed with one + existing NOTE about `.gitkeep` files in `inst/project-template/`. diff --git a/future/specs/004-senior-review-followups/sdd.md b/future/specs/004-senior-review-followups/sdd.md new file mode 100644 index 0000000..2676776 --- /dev/null +++ b/future/specs/004-senior-review-followups/sdd.md @@ -0,0 +1,99 @@ +# Software Design + +Spec ID: `004-senior-review-followups` +Status: Implemented + +## Scope + +Implement the highest-value issues from the senior package review while keeping +changes small and easy to inspect: + +- apply EDA `missing_codes` in missingness and summaries; +- make numeric summaries stable for all-missing and zero-row data; +- make categorical percentage denominators explicit; +- document descriptive schema behavior without redesigning it; +- add direct optional dependency guards where review found indirect errors. + +## Public API + +No new exported functions. + +Existing outputs change as follows: + +- `epi_eda_profile_missing()` counts standard `NA` and specification + `missing_codes`. +- `epi_eda_profile_summaries()` excludes standard `NA` and specification + `missing_codes` from numeric calculations and observed categorical counts. +- Categorical summaries keep `p` as the total-row denominator and add + `p_observed` as the observed non-missing denominator. + +Tidy-eval re-export cleanup is deferred to a later compatibility review and must +not be implemented in this spec. + +## Inputs And Outputs + +`missing_codes` rules: + +- The column is optional. +- Values are semicolon-separated. +- Codes are trimmed with `trimws()`. +- Empty values mean no sentinel codes. +- Matching is by character representation of the source value. +- Standard `NA` is always missing. + +Numeric summaries: + +- `n` is total row count for the variable. +- `n_missing` counts standard `NA` and sentinel missing codes. +- `mean`, `sd`, `median`, `min` and `max` use only observed non-missing values. +- If no observed values remain, unavailable statistics are `NA_real_`. + +Categorical summaries: + +- Declared levels are retained even when the observed count is zero. +- `n` counts observed non-missing values matching the declared level. +- `p` is `n / total_rows`; for zero-row data it is `NA_real_`. +- `p_observed` is `n / observed_non_missing_rows`; for no observed values it is + `NA_real_`. + +## Data Flow + +1. Validate `data` and `spec` with existing entry points. +2. For each specified variable, parse its `missing_codes`. +3. Build a missing mask using `is.na()` or character-code matching. +4. Compute missingness and summaries from the mask. +5. Return stable data frames with documented columns. + +## Edge Cases + +- Zero-row data returns `n = 0` and missing proportions as `NA_real_`. +- All-missing numeric data returns one stable row with summary statistics as + `NA_real_`. +- Sentinel codes in numeric columns may be numeric or character values; matching + is by character representation. +- Categorical levels with no observations remain in output. +- Missing variables keep existing error behavior in summaries and existing + schema reporting behavior. +- Non-syntactic names continue to work through existing name-based indexing. + +## Errors And Warnings + +Errors should be clear, actionable and covered by tests. + +- Missing suggested packages should fail through `check_suggests()` or an + equivalent explicit guard. +- No warnings should be emitted for all-missing numeric summaries. + +## Dependencies + +No new dependencies. + +Optional dependency guards should be explicit for reviewed plotting, reporting +and parallel paths. In particular, parallel plotting should check the optional +packages it uses before calling their namespaces. + +## Deferred Work + +- Public API cleanup for tidy-eval re-exports. +- Large-data backend support. +- Full schema compatibility pass/fail redesign. diff --git a/future/specs/004-senior-review-followups/tdd.md b/future/specs/004-senior-review-followups/tdd.md new file mode 100644 index 0000000..b196347 --- /dev/null +++ b/future/specs/004-senior-review-followups/tdd.md @@ -0,0 +1,60 @@ +# Test Design + +Spec ID: `004-senior-review-followups` +Status: Implemented + +## Baseline Commands + +Run before package-code implementation where practical: + +```bash +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" +``` + +## Test Files + +- `tests/testthat/test-eda_missing-fixtures.R` +- `tests/testthat/test-eda_summaries-fixtures.R` +- `tests/testthat/test-eda_schema-fixtures.R` +- `tests/testthat/test-plot-parallel.R` +- `tests/testthat/test-eda_report.R` + +## Required Tests + +- [x] `epi_eda_profile_missing()` counts standard `NA` plus one sentinel missing + code. +- [x] `epi_eda_profile_missing()` counts multiple semicolon-separated sentinel + codes. +- [x] `epi_eda_profile_missing()` treats absent or empty `missing_codes` as + standard `NA` only. +- [x] `epi_eda_profile_missing()` returns `NA_real_` proportions for zero-row + data. +- [x] `epi_eda_profile_summaries()` returns stable `NA_real_` numeric statistics + for all-missing variables. +- [x] `epi_eda_profile_summaries()` excludes sentinel codes from numeric + summaries. +- [x] `epi_eda_profile_summaries()` keeps declared categorical levels with zero + counts. +- [x] `epi_eda_profile_summaries()` keeps `p` as total-row denominator and adds + `p_observed` as observed-row denominator. +- [x] EDA schema tests document current descriptive behavior for integer/numeric + and logical/binary cases. +- [x] Parallel plotting missing-dependency checks fail with clear suggested + package messages. + +## Anti-circularity + +Expected summary outputs must not be calculated by copying implementation logic. +Use small hand-computed data frames or committed expected fixture CSVs with +transparent values. + +## Acceptance Commands + +```bash +scripts/rscript_env_caller.R -e "devtools::load_all(); testthat::test_file('tests/testthat/test-eda_missing-fixtures.R')" +scripts/rscript_env_caller.R -e "devtools::load_all(); testthat::test_file('tests/testthat/test-eda_summaries-fixtures.R')" +scripts/rscript_env_caller.R -e "devtools::load_all(); testthat::test_file('tests/testthat/test-eda_schema-fixtures.R')" +scripts/rscript_env_caller.R -e "devtools::load_all(); testthat::test_file('tests/testthat/test-plot-parallel.R')" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" +``` diff --git a/future/specs/template/manifest.yml b/future/specs/template/manifest.yml index 5e25b36..2571768 100644 --- a/future/specs/template/manifest.yml +++ b/future/specs/template/manifest.yml @@ -21,8 +21,8 @@ documents: review_required: true check_commands: - - scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" - - scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" + - scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" + - scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" notes: - Keep executable tests under tests/testthat/. diff --git a/future/specs/template/tdd.md b/future/specs/template/tdd.md index 59f2c63..20cc3b7 100644 --- a/future/specs/template/tdd.md +++ b/future/specs/template/tdd.md @@ -10,7 +10,7 @@ Status: Draft ## Baseline Commands ```bash -scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" ``` ## Behaviour Tests @@ -28,6 +28,6 @@ scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" ## Acceptance Commands ```bash -scripts/rscript_env_caller.R -e "devtools::test(reporter = 'summary')" -scripts/rscript_env_caller.R -e "devtools::check(manual = FALSE)" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" ``` diff --git a/man/epi_clean_add_colname_suffix.Rd b/man/epi_clean_add_colname_suffix.Rd index c9c4040..402b2ce 100644 --- a/man/epi_clean_add_colname_suffix.Rd +++ b/man/epi_clean_add_colname_suffix.Rd @@ -48,5 +48,5 @@ names(df2) \code{\link{epi_clean_spread_repeated}} } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_clean_add_rep_num.Rd b/man/epi_clean_add_rep_num.Rd index 5fdf154..8f26dce 100644 --- a/man/epi_clean_add_rep_num.Rd +++ b/man/epi_clean_add_rep_num.Rd @@ -63,5 +63,5 @@ df2 } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_clean_cond_chr_fct.Rd b/man/epi_clean_cond_chr_fct.Rd index b217f47..ac2ea40 100644 --- a/man/epi_clean_cond_chr_fct.Rd +++ b/man/epi_clean_cond_chr_fct.Rd @@ -43,5 +43,5 @@ epi_clean_cond_chr_fct(df_cont_chr[, "x"]) # should be 'FALSE' \code{\link[base]{is.factor}}. } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_clean_cond_numeric.Rd b/man/epi_clean_cond_numeric.Rd index 0606bed..874668f 100644 --- a/man/epi_clean_cond_numeric.Rd +++ b/man/epi_clean_cond_numeric.Rd @@ -42,5 +42,5 @@ epi_clean_cond_numeric(df[, "x"]) # should be TRUE \code{\link[base]{is.integer}}, \code{\link[base]{is.numeric}} } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_clean_get_dups.Rd b/man/epi_clean_get_dups.Rd index 1e28df6..ef9a6d4 100644 --- a/man/epi_clean_get_dups.Rd +++ b/man/epi_clean_get_dups.Rd @@ -41,5 +41,5 @@ check_dups \code{\link[base]{duplicated}} } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_clean_make_names.Rd b/man/epi_clean_make_names.Rd index 2bd5986..572c032 100644 --- a/man/epi_clean_make_names.Rd +++ b/man/epi_clean_make_names.Rd @@ -43,5 +43,5 @@ epi_clean_make_names(valid_names) # There shouldn't be anything to correct \href{https://www.r-bloggers.com/2011/07/testing-for-valid-variable-names/}{r-bloggers test valid names}. } \author{ -Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> +Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} } diff --git a/man/epi_clean_spread_repeated.Rd b/man/epi_clean_spread_repeated.Rd index b6bd8ca..81e8622 100644 --- a/man/epi_clean_spread_repeated.Rd +++ b/man/epi_clean_spread_repeated.Rd @@ -47,5 +47,5 @@ df_spread \code{\link{epi_clean_transpose}} } \author{ -Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> +Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} } diff --git a/man/epi_clean_transpose.Rd b/man/epi_clean_transpose.Rd index 88f6d0f..9dee83a 100644 --- a/man/epi_clean_transpose.Rd +++ b/man/epi_clean_transpose.Rd @@ -52,5 +52,5 @@ names(df_t) \code{\link[data.table]{as.data.table}}. } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_eda_profile_missing.Rd b/man/epi_eda_profile_missing.Rd index d3fc728..aa2c5c9 100644 --- a/man/epi_eda_profile_missing.Rd +++ b/man/epi_eda_profile_missing.Rd @@ -16,5 +16,6 @@ A data frame with variable names, row counts, missing counts and missing proportions. } \description{ -Count standard missing values for variables listed in an EDA specification. +Count standard missing values and configured missing codes for variables +listed in an EDA specification. } diff --git a/man/epi_eda_profile_summaries.Rd b/man/epi_eda_profile_summaries.Rd index 8cc413a..be929fa 100644 --- a/man/epi_eda_profile_summaries.Rd +++ b/man/epi_eda_profile_summaries.Rd @@ -12,9 +12,12 @@ epi_eda_profile_summaries(data, spec) \item{spec}{An EDA specification data frame or CSV path.} } \value{ -A named list with \code{numeric} and \code{categorical} data frames. +A named list with \code{numeric} and \code{categorical} data frames. In +categorical summaries, \code{p} uses total rows as the denominator and +\code{p_observed} uses observed non-missing rows. } \description{ Produce machine-readable descriptive summaries for variables listed in a -specification-first EDA data dictionary. +specification-first EDA data dictionary. Standard \code{NA} values and configured +\code{missing_codes} are excluded from observed summaries. } diff --git a/man/epi_plot_box.Rd b/man/epi_plot_box.Rd index 892a5a9..6276b40 100644 --- a/man/epi_plot_box.Rd +++ b/man/epi_plot_box.Rd @@ -122,5 +122,5 @@ my_boxplot + \code{\link[ggplot2]{geom_boxplot}}. } \author{ -Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_plot_list.Rd b/man/epi_plot_list.Rd index 5d73556..4185b13 100644 --- a/man/epi_plot_list.Rd +++ b/man/epi_plot_list.Rd @@ -24,5 +24,5 @@ See example in \code{\link{epi_plot_cow_save}} and ggplot2 wrappers epi_plot_*() \code{\link{epi_plot_cow_save}}. } \author{ -Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> +Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} } diff --git a/man/epi_read.Rd b/man/epi_read.Rd index a7a3f88..4662ca3 100644 --- a/man/epi_read.Rd +++ b/man/epi_read.Rd @@ -53,5 +53,5 @@ super_data <- epi_read("super_data.tsv") \code{\link[data.table]{fread}}, \code{\link[tibble]{as.tibble}}. } \author{ -Antonio Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> +Antonio Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} } diff --git a/man/epi_stats_summary.Rd b/man/epi_stats_summary.Rd index 344d621..8efffda 100644 --- a/man/epi_stats_summary.Rd +++ b/man/epi_stats_summary.Rd @@ -189,5 +189,5 @@ stat_sum_zero \code{\link{epi_clean_cond_numeric}}. } \author{ -Antonio J Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio J Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_stats_tidy.Rd b/man/epi_stats_tidy.Rd index 33024fc..5715e95 100644 --- a/man/epi_stats_tidy.Rd +++ b/man/epi_stats_tidy.Rd @@ -187,5 +187,5 @@ stat_sum_zero \code{\link{epi_stats_numeric}}. } \author{ -Antonio J Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio J Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/man/epi_utils_log.Rd b/man/epi_utils_log.Rd index 40d759a..b2061a7 100644 --- a/man/epi_utils_log.Rd +++ b/man/epi_utils_log.Rd @@ -39,5 +39,5 @@ epi_utils_log("xy") \code{\link[episcout]{epi_utils_session}}. } \author{ -Antonio J Berlanga-Taylor \if{html}{\out{<\url{https://github.com/AntonioJBT/episcout}>}} +Antonio J Berlanga-Taylor <\url{https://github.com/AntonioJBT/episcout}> } diff --git a/scripts/check-local.sh b/scripts/check-local.sh index f09f3d3..f899419 100755 --- a/scripts/check-local.sh +++ b/scripts/check-local.sh @@ -7,6 +7,6 @@ rscript="${script_dir}/rscript_env_caller.R" cd "$repo_root" -"$rscript" -e 'devtools::document()' -"$rscript" -e 'devtools::test(reporter = "summary")' -"$rscript" -e 'devtools::check(manual = FALSE)' +"$rscript" -e 'options(repos = c(CRAN = "https://cloud.r-project.org")); devtools::document()' +"$rscript" -e 'options(repos = c(CRAN = "https://cloud.r-project.org")); devtools::test(reporter = "summary")' +"$rscript" -e 'options(repos = c(CRAN = "https://cloud.r-project.org")); devtools::check(manual = FALSE)' diff --git a/tests/testthat/test-eda_missing-fixtures.R b/tests/testthat/test-eda_missing-fixtures.R index 2f625bf..7195d00 100644 --- a/tests/testthat/test-eda_missing-fixtures.R +++ b/tests/testthat/test-eda_missing-fixtures.R @@ -54,3 +54,50 @@ test_that("epi_eda_profile_missing reports zero missingness for complete variabl expect_true(all(complete_rows$n_missing == 0)) expect_true(all(complete_rows$p_missing == 0)) }) + +test_that("epi_eda_profile_missing counts sentinel missing codes", { + data <- data.frame( + age = c(45, 999, NA, 50), + status = c("A", "UNK", "B", "REF"), + complete = c(1, 2, 3, 4), + stringsAsFactors = FALSE + ) + spec <- data.frame( + name = c("age", "status", "complete"), + label = c("Age", "Status", "Complete"), + type = c("numeric", "categorical", "integer"), + role = c("covariate", "covariate", "covariate"), + levels = c("", "A;B;UNK;REF", ""), + missing_codes = c("999", "UNK; REF", ""), + stringsAsFactors = FALSE + ) + + observed <- epi_eda_profile_missing(data, spec) + + expect_equal(observed$n, c(4L, 4L, 4L)) + expect_equal(observed$n_missing, c(2L, 2L, 0L)) + expect_equal(observed$p_missing, c(0.5, 0.5, 0)) +}) + +test_that("epi_eda_profile_missing uses NA proportions for zero-row data", { + data <- data.frame( + age = numeric(), + status = character(), + stringsAsFactors = FALSE + ) + spec <- data.frame( + name = c("age", "status"), + label = c("Age", "Status"), + type = c("numeric", "categorical"), + role = c("covariate", "covariate"), + levels = c("", "A;B"), + missing_codes = c("999", ""), + stringsAsFactors = FALSE + ) + + observed <- epi_eda_profile_missing(data, spec) + + expect_equal(observed$n, c(0L, 0L)) + expect_equal(observed$n_missing, c(0L, 0L)) + expect_true(all(is.na(observed$p_missing))) +}) diff --git a/tests/testthat/test-eda_schema-fixtures.R b/tests/testthat/test-eda_schema-fixtures.R index 38c0f41..a54e933 100644 --- a/tests/testthat/test-eda_schema-fixtures.R +++ b/tests/testthat/test-eda_schema-fixtures.R @@ -56,3 +56,24 @@ test_that("all expected variables are present in the unmodified fixture", { expect_true(all(expected_rows$observed_present)) expect_false(any(expected_rows$status == "missing")) }) + +test_that("epi_eda_check_schema remains descriptive for numeric and binary types", { + data <- data.frame( + integer_count = c(1L, 2L), + logical_flag = c(TRUE, FALSE), + stringsAsFactors = FALSE + ) + spec <- data.frame( + name = c("integer_count", "logical_flag"), + label = c("Integer count", "Logical flag"), + type = c("integer", "binary"), + role = c("covariate", "covariate"), + stringsAsFactors = FALSE + ) + + observed <- epi_eda_check_schema(data, spec) + + expect_equal(observed$expected_type, c("integer", "binary")) + expect_equal(observed$observed_type, c("numeric", "binary")) + expect_equal(observed$status, c("present", "present")) +}) diff --git a/tests/testthat/test-eda_summaries-fixtures.R b/tests/testthat/test-eda_summaries-fixtures.R index 9bef13d..0079f9f 100644 --- a/tests/testthat/test-eda_summaries-fixtures.R +++ b/tests/testthat/test-eda_summaries-fixtures.R @@ -7,48 +7,6 @@ fixture_dir <- file.path("fixtures", "blood_storage") data_path <- file.path(fixture_dir, "blood_storage.csv") spec_path <- file.path(fixture_dir, "blood_storage_spec.csv") -make_expected_numeric_summary <- function(data, spec) { - numeric_spec <- spec[spec$type %in% c("numeric", "integer"), , drop = FALSE] - - rows <- lapply(numeric_spec$name, function(name) { - values <- data[[name]] - data.frame( - name = name, - n = length(values), - n_missing = sum(is.na(values)), - mean = mean(values, na.rm = TRUE), - sd = stats::sd(values, na.rm = TRUE), - median = stats::median(values, na.rm = TRUE), - min = min(values, na.rm = TRUE), - max = max(values, na.rm = TRUE), - stringsAsFactors = FALSE - ) - }) - - do.call(rbind, rows) -} - -make_expected_categorical_summary <- function(data, spec) { - categorical_spec <- spec[spec$type %in% c("categorical", "binary"), , drop = FALSE] - - rows <- lapply(categorical_spec$name, function(name) { - values <- data[[name]] - observed <- as.character(values[!is.na(values)]) - levels <- strsplit(categorical_spec$levels[categorical_spec$name == name], ";", fixed = TRUE)[[1]] - counts <- vapply(levels, function(level) sum(observed == level), integer(1)) - - data.frame( - name = name, - level = levels, - n = as.integer(counts), - p = as.numeric(counts) / length(values), - stringsAsFactors = FALSE - ) - }) - - do.call(rbind, rows) -} - test_that("epi_eda_profile_summaries returns numeric and categorical summary components", { data <- read.csv(data_path, check.names = FALSE) spec <- epi_eda_spec(spec_path) @@ -61,10 +19,31 @@ test_that("epi_eda_profile_summaries returns numeric and categorical summary com expect_s3_class(observed$categorical, "data.frame") }) -test_that("epi_eda_profile_summaries numeric output matches independent fixture calculations", { - data <- read.csv(data_path, check.names = FALSE) - spec <- epi_eda_spec(spec_path) - expected <- make_expected_numeric_summary(data, spec) +test_that("epi_eda_profile_summaries numeric output matches hand-computed values", { + data <- data.frame( + age = c(10, 20, 999, NA), + all_missing = c(999, NA, 999, NA), + stringsAsFactors = FALSE + ) + spec <- data.frame( + name = c("age", "all_missing"), + label = c("Age", "All missing"), + type = c("numeric", "numeric"), + role = c("covariate", "covariate"), + missing_codes = c("999", "999"), + stringsAsFactors = FALSE + ) + expected <- data.frame( + name = c("age", "all_missing"), + n = c(4L, 4L), + n_missing = c(2L, 4L), + mean = c(15, NA_real_), + sd = c(sqrt(50), NA_real_), + median = c(15, NA_real_), + min = c(10, NA_real_), + max = c(20, NA_real_), + stringsAsFactors = FALSE + ) observed <- epi_eda_profile_summaries(data, spec) @@ -76,10 +55,28 @@ test_that("epi_eda_profile_summaries numeric output matches independent fixture ) }) -test_that("epi_eda_profile_summaries categorical output matches independent fixture calculations", { - data <- read.csv(data_path, check.names = FALSE) - spec <- epi_eda_spec(spec_path) - expected <- make_expected_categorical_summary(data, spec) +test_that("epi_eda_profile_summaries categorical output documents denominators", { + data <- data.frame( + status = c("A", "B", "UNK", NA, "A"), + stringsAsFactors = FALSE + ) + spec <- data.frame( + name = "status", + label = "Status", + type = "categorical", + role = "covariate", + levels = "A;B;C", + missing_codes = "UNK", + stringsAsFactors = FALSE + ) + expected <- data.frame( + name = c("status", "status", "status"), + level = c("A", "B", "C"), + n = c(2L, 1L, 0L), + p = c(2 / 5, 1 / 5, 0), + p_observed = c(2 / 3, 1 / 3, 0), + stringsAsFactors = FALSE + ) observed <- epi_eda_profile_summaries(data, spec) diff --git a/tests/testthat/test-plot-parallel.R b/tests/testthat/test-plot-parallel.R index 5aff377..0446936 100644 --- a/tests/testthat/test-plot-parallel.R +++ b/tests/testthat/test-plot-parallel.R @@ -14,6 +14,20 @@ skip_if_not_installed("withr") skip_if_not_installed("ggplot2") skip_if_not_installed("cowplot") +test_that("parallel plotting functions declare explicit suggested package guards", { + parallel_body <- paste(deparse(body(epi_plot_parallel)), collapse = "\n") + save_body <- paste(deparse(body(epi_plot_save_parallel)), collapse = "\n") + + expect_match(parallel_body, "check_suggests") + expect_match(parallel_body, "future") + expect_match(parallel_body, "doFuture") + expect_match(parallel_body, "foreach") + expect_match(save_body, "check_suggests") + expect_match(save_body, "future") + expect_match(save_body, "doFuture") + expect_match(save_body, "foreach") +}) + # Multisession futures start fresh R workers that must be able to attach # the installed package. devtools::test()/pkgload exposes the source tree to # the main process, but not as an installed package to worker processes. diff --git a/vignettes/specification-first-eda.Rmd b/vignettes/specification-first-eda.Rmd index 7bcbcd1..138b837 100644 --- a/vignettes/specification-first-eda.Rmd +++ b/vignettes/specification-first-eda.Rmd @@ -61,7 +61,7 @@ spec_data <- data.frame( levels = c("", "Female;Male;Unknown", "0;1", ""), min = c(18, NA, 0, 0), max = c(110, NA, 1, 3650), - missing_codes = c("", "", "", ""), + missing_codes = c("", "Unknown", "", ""), required = c(TRUE, TRUE, TRUE, TRUE), group = c("demographics", "demographics", "outcomes", "outcomes"), description = c( @@ -77,6 +77,11 @@ spec <- epi_eda_spec(spec_data) spec ``` +The optional `missing_codes` column accepts semicolon-separated sentinel values. +These values are counted as missing in the missingness profile and excluded from +observed summaries. Categorical summaries report `p` over all rows and +`p_observed` over observed non-missing rows. + ## Prepare the workflow before data access Use the specification to generate deterministic synthetic data, then run the From 7ff73f7cd7b244f8c0ec7f7a1855a0fe53693b61 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Mon, 15 Jun 2026 17:51:20 -0600 Subject: [PATCH 30/33] added todos and ideas for future work --- future/TODOs.md | 9 +- .../codex_reviews_from_PRs_to_check_minor.md | 82 +++++++++++++++++++ ...> eda-stats-helper-boundary-idea_major.md} | 0 3 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 future/ideas/codex_reviews_from_PRs_to_check_minor.md rename future/ideas/{eda-stats-helper-boundary-idea.md => eda-stats-helper-boundary-idea_major.md} (100%) diff --git a/future/TODOs.md b/future/TODOs.md index b16a83f..cee6877 100644 --- a/future/TODOs.md +++ b/future/TODOs.md @@ -9,9 +9,8 @@ - For the activated spec, record baseline package test/check status before package-code changes. - Keep implementation work scoped to one numbered spec at a time. - Review fixture anti-circularity guardrails before adding new expected outputs. -- Keep README and `NEWS.md` aligned with user-facing workflow changes. -Use this template and place the task under the appropriate priority heading: +- Use this template and place the task under the appropriate priority heading: ```markdown - Short title: @@ -25,7 +24,10 @@ Use this template and place the task under the appropriate priority heading: - Suggested spec ID: ``` -For simple tasks only use e.g. `- [ ] xxx` +- For simple tasks only use e.g. `- [ ] xxx` + +- Keep README and `NEWS.md` aligned with user-facing workflow changes. +- Move completed items to `changelog.md` ## Task list @@ -46,6 +48,7 @@ For simple tasks only use e.g. `- [ ] xxx` - [ ] Sanitise dictionaries so that R, QGIS, SQL/MariaDB/postgreSQL can easily use them as input - [ ] Revisit large-data backend implementation after the design spec is accepted. +- [ ] review `future/ideas/codex_reviews_from_PRs_to_check_minor.md` and determine whether they are still valid and need addressing. ### Priority 3 diff --git a/future/ideas/codex_reviews_from_PRs_to_check_minor.md b/future/ideas/codex_reviews_from_PRs_to_check_minor.md new file mode 100644 index 0000000..3b19782 --- /dev/null +++ b/future/ideas/codex_reviews_from_PRs_to_check_minor.md @@ -0,0 +1,82 @@ +# Notes from codex checks from PRs that were not addressed + +Review if still valid, if correct/useful/needed/etc. + + +In R/eda_synthetic.R: + +> + binary = generate_synthetic_binary(row, n), ++ date = generate_synthetic_date(row, n), ++ datetime = generate_synthetic_datetime(row, n), ++ text = generate_synthetic_text(row, n), ++ stop("Unsupported EDA specification type: ", type, call. = FALSE) ++ ) ++} ++ ++generate_synthetic_numeric <- function(row, n) { ++ bounds <- synthetic_numeric_bounds(row, default_min = 0, default_max = 100) ++ stats::runif(n, min = bounds[["min"]], max = bounds[["max"]]) ++} ++ ++generate_synthetic_integer <- function(row, n) { ++ bounds <- synthetic_numeric_bounds(row, default_min = 0, default_max = 100) ++ sample(seq.int(ceiling(bounds[["min"]]), floor(bounds[["max"]])), n, replace = TRUE) +P2 Badge Sample singleton integer ranges without expansion + +When an integer variable has a single positive allowed value, e.g. min = max = 5, seq.int(...) returns the length-one numeric vector 5, and base sample(5, n, replace = TRUE) samples from 1:5 rather than from the literal value 5. This makes generated integer values fall outside the specification bounds, violating the min/max guarantee for exact-value integer fields. + + + +In tests/testthat/test-eda_plots-fixtures.R: + +> + expect_s3_class(observed$RBC.Age.Group, "ggplot") ++ expect_s3_class(observed$Recurrence, "ggplot") +P2 Badge Assert plot dispatch for spec-coded categories + +These assertions only verify that the categorical/binary fixtures return some ggplot, but the fixture columns such as RBC.Age.Group and Recurrence are read from CSV as numeric values. In that scenario, an implementation that ignores spec$type and dispatches purely from the observed column class would still pass by returning numeric histograms for these variables, so the PR would not actually drive the spec-based plot dispatch it is meant to test. Please assert the dispatched plot kind through a non-visual property, such as the expected geom class or an explicit plot-type marker, for at least one spec-coded categorical/binary variable. + + +In tests/testthat/test-eda_synthetic-fixtures.R: + +> + expect_true( ++ all(observed_values %in% allowed_levels), +P2 Badge Require generated values before level/range assertions + +Here, and in the numeric range check below, the assertion runs after na.omit(), so a generator that returns only NA for every fixture column passes: observed_values is empty and all(logical(0)) is TRUE. Since these fixture variables are required and this PR is meant to drive the implementation, add an assertion that each generated column has at least one non-missing value (or no unexpected missingness) before checking levels/ranges. + +In R/eda_summaries.R: + +> + levels_value <- categorical_spec$levels[categorical_spec$name == name] ++ levels <- strsplit(levels_value, ";", fixed = TRUE)[[1]] +P2 Badge Do not assume categorical levels are present + +eda_spec() accepts a minimal valid specification with only name, label, type, and role, but this code unconditionally reads categorical_spec$levels. For any categorical/binary variable in such a spec, levels_value is NULL and strsplit() errors; with a blank levels value it builds a zero-row level vector and errors in the data.frame() below. This breaks summaries for valid data dictionaries that do not enumerate levels, so either derive observed non-missing levels here or reject missing levels during validation. + + + +In tests/testthat/test-run_eda-fixtures.R: + +> + data = NULL, ++ spec = spec, ++ synthetic = TRUE, +P2 Badge Exercise the documented synthetic mode argument + +Instruction 9 requires run_eda() to support mode = "real" and mode = "synthetic", and the SDD's synthetic workflow generates synthetic data first and then calls run_eda(data = data, spec = spec, mode = "synthetic"). This test instead drives an undocumented synthetic = TRUE API with data = NULL, so an implementation that follows the documented contract will fail these new tests, while an implementation that omits mode could pass them. + + +In R/run_eda.R: + +> +run_eda <- function(data, ++ spec, ++ output_dir = NULL, ++ synthetic = FALSE, ++ n = 100, ++ seed = NULL) { +P2 Badge Add the planned mode argument to run_eda + +The implementation exposes synthetic instead of the planned mode argument, but Instruction 9 requires mode = "real"/"synthetic" and the SDD workflows call run_eda(..., mode = "real") or mode = "synthetic" (spec_driven_EDA_plan/docs/codex/tdd-first-codex-instructions.md:264, spec_driven_EDA_plan/docs/sdd/0001-spec-first-eda.md:83-88). Any caller following that contract, including the upcoming report/project-template work, currently gets unused argument (mode = "real") before the workflow runs. + + +P2 Badge Derive schema types from the committed fixture + +This records classes from the in-memory medicaldata::blood_storage object before CSV serialization, but the fixture-backed tests are meant to consume tests/testthat/fixtures/blood_storage/blood_storage.csv. Reading the committed CSV with base R and preserved names makes columns such as RBC.Age.Group integer while expected_schema.csv records numeric, so any schema test comparing observed types from the committed fixture to this expected file will fail even when the fixture is unchanged. Please compute observed_type after reading back the written CSV using the intended fixture-loading options. diff --git a/future/ideas/eda-stats-helper-boundary-idea.md b/future/ideas/eda-stats-helper-boundary-idea_major.md similarity index 100% rename from future/ideas/eda-stats-helper-boundary-idea.md rename to future/ideas/eda-stats-helper-boundary-idea_major.md From 764cc4985bd66ea31a1f2e64a1296d399c9b908f Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Tue, 16 Jun 2026 11:09:30 -0600 Subject: [PATCH 31/33] todos and agent, minor --- AGENTS.MD | 11 +++++++++-- future/TODOs.md | 14 ++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/AGENTS.MD b/AGENTS.MD index bd34b47..6844db8 100644 --- a/AGENTS.MD +++ b/AGENTS.MD @@ -1,18 +1,25 @@ # AGENTS.md This file defines conventions and instructions to guide OpenAI Codex and other agents in contributing to this R package. -It standardises formatting, documentation, testing, and pull request workflows. +It standardises code writing, formatting, documentation, testing, and pull request workflows. -It should be respected by automated agents when: +It should be respected by automated agents at all times, particularly when: * Writing new functions (must include roxygen2 and tests) * Refactoring existing code (preserve naming and indentation) * Editing documentation (run `devtools::document()`) * Opening a PR (follow title/description/branch rules) * Running checks (use scripts provided above) +* Debugging, identifying errors and making improvements Do not overcomplicate and over-engineer. Prefer the simpler solution. Make it easy for a human to review, supervise and monitor work. Do not bloat file content or create numerous files when one would suffice. e.g. use one `TODOs.md`, not a dir with `now.md`, `next.md`, `later.md`. +When fixing code, identify the cause; do not simply fix the symptom. + +Never hide behaviour behind 'clean up' when refactoring. + +When writing tests, never mirror the implementation. Always test against the intended behaviour. If this is unclear, undocumented or may be wrong, always stop and report. + --- ## 📁 Project Structure diff --git a/future/TODOs.md b/future/TODOs.md index cee6877..81396bb 100644 --- a/future/TODOs.md +++ b/future/TODOs.md @@ -25,14 +25,19 @@ ``` - For simple tasks only use e.g. `- [ ] xxx` - - Keep README and `NEWS.md` aligned with user-facing workflow changes. -- Move completed items to `changelog.md` +- Move completed items to `changelog.md`. +- Check this `TODOs.md` file and `future/specs/` specs are aligned. ## Task list ### Priority 1 +- [ ] run task from codex cloud thread locally (clone, install, follow vignette) +- [ ] check findings in `future/reviews/2026-06-15-senior-r-package-review.md` were implemented +- [ ] move 'implemented' specs in `future/specs` to a `future/done` dir +- why aren't specs in `future/specs` mirrored here, in `TODOs.md`? + - [ ] get instructions to download, install, episcout2 branch - [ ] human live walkthrough, no agent needed here - [ ] carry out changes needed from human review @@ -46,9 +51,10 @@ - [ ] Add functions to create a data dictionary when real data already exists but does not have a dictionary. - [ ] Sanitise dictionaries so that R, QGIS, SQL/MariaDB/postgreSQL can easily use them as input -- [ ] Revisit large-data backend implementation after the design spec is - accepted. +- [ ] Revisit large-data backend implementation after the design spec is accepted. - [ ] review `future/ideas/codex_reviews_from_PRs_to_check_minor.md` and determine whether they are still valid and need addressing. +- [ ] add functions to pseudonimise PII: identify columns, create toke ID, bridge file, remove columns, pass for check, write read-only raw data file. +- [ ] add functions to load, connect, etc data into db. ### Priority 3 From 79db47b1e771a16a8cd62f9b35cc3d64f4c8b643 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:20:46 -0600 Subject: [PATCH 32/33] epi_sec function for pseudonym v1 --- DESCRIPTION | 1 + NAMESPACE | 2 + NEWS.md | 2 + R/epi_sec_pseudonym.R | 121 ++++++++++++++++++ future/TODOs.md | 2 +- .../005-pseudonymisation-bridge/acceptance.md | 23 ++++ .../005-pseudonymisation-bridge/brief.md | 39 ++++++ .../005-pseudonymisation-bridge/manifest.yml | 33 +++++ .../005-pseudonymisation-bridge/review.md | 14 ++ .../specs/005-pseudonymisation-bridge/sdd.md | 33 +++++ .../specs/005-pseudonymisation-bridge/tdd.md | 44 +++++++ man/epi_sec_pseudonym.Rd | 53 ++++++++ tests/testthat/test-epi_sec_pseudonym.R | 97 ++++++++++++++ 13 files changed, 463 insertions(+), 1 deletion(-) create mode 100644 R/epi_sec_pseudonym.R create mode 100644 future/specs/005-pseudonymisation-bridge/acceptance.md create mode 100644 future/specs/005-pseudonymisation-bridge/brief.md create mode 100644 future/specs/005-pseudonymisation-bridge/manifest.yml create mode 100644 future/specs/005-pseudonymisation-bridge/review.md create mode 100644 future/specs/005-pseudonymisation-bridge/sdd.md create mode 100644 future/specs/005-pseudonymisation-bridge/tdd.md create mode 100644 man/epi_sec_pseudonym.Rd create mode 100644 tests/testthat/test-epi_sec_pseudonym.R diff --git a/DESCRIPTION b/DESCRIPTION index c48de38..675fec1 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -22,6 +22,7 @@ Imports: utils, stats, grDevices, + openssl, magrittr, rlang (>= 0.1.2), broom, diff --git a/NAMESPACE b/NAMESPACE index 1329b2b..1c48f8e 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -61,6 +61,7 @@ export(epi_plot_theme_imss) export(epi_plot_volcano) export(epi_plots_to_grid) export(epi_read) +export(epi_sec_pseudonym) export(epi_stats_2x2_all) export(epi_stats_2x2_cols) export(epi_stats_2x2_test) @@ -119,6 +120,7 @@ importFrom(graphics,abline) importFrom(graphics,plot) importFrom(magrittr,"%>%") importFrom(magrittr,`%>%`) +importFrom(openssl,rand_bytes) importFrom(rlang,":=") importFrom(rlang,.data) importFrom(rlang,enexpr) diff --git a/NEWS.md b/NEWS.md index 1a6c306..7378423 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,8 @@ ## Development version +- Added `epi_sec_pseudonym()` to create secure participant pseudonymisation + bridge tables using cryptographic random tokens. - Added a specification-first EDA workflow driven by data dictionaries: `epi_eda_spec()`, `epi_eda_check_schema()`, `epi_eda_profile_missing()`, `epi_eda_profile_summaries()`, `epi_eda_profile_plots()`, diff --git a/R/epi_sec_pseudonym.R b/R/epi_sec_pseudonym.R new file mode 100644 index 0000000..4e17762 --- /dev/null +++ b/R/epi_sec_pseudonym.R @@ -0,0 +1,121 @@ +#' Generate a Secure Pseudonymisation Bridge Table +#' +#' Generate random, non-derivable token identifiers for participant +#' pseudonymisation. The returned bridge table maps each original participant +#' identifier to a cryptographically random token. +#' +#' @param participant_id Character, numeric or factor vector of participant +#' identifiers. Values must be unique and non-missing. +#' @param n_bytes Number of random bytes per token. Must be at least 16. The +#' default of 24 gives 192-bit tokens. +#' @param prefix Character prefix added before each token. +#' @param bridge_path Optional file path for writing the bridge table as CSV. +#' If `NULL`, no file is written. +#' @param overwrite Logical. If `FALSE`, an existing `bridge_path` is not +#' overwritten. +#' +#' @return A tibble with columns `participant_id` and `token_id`. +#' +#' @details +#' Tokens are generated from cryptographic random bytes using +#' `openssl::rand_bytes()`. The function does not accept a seed and does not +#' provide deterministic output. +#' +#' The bridge table remains re-identifying information. Store it separately +#' from pseudonymised analysis datasets and protect it with appropriate access +#' controls. Pseudonymisation reduces risk but does not by itself anonymise a +#' dataset. +#' +#' @examples +#' participant_ids <- sprintf("study_%04d", 1:10) +#' bridge <- epi_sec_pseudonym(participant_ids, prefix = "MXSTUDY") +#' head(bridge) +#' +#' @importFrom openssl rand_bytes +#' @export +epi_sec_pseudonym <- function(participant_id, + n_bytes = 24, + prefix = "P", + bridge_path = NULL, + overwrite = FALSE) { + if (!is.character(participant_id) && + !is.numeric(participant_id) && + !is.factor(participant_id)) { + stop( + "participant_id must be a character, numeric or factor vector.", + call. = FALSE + ) + } + + if (length(participant_id) == 0) { + stop("participant_id must contain at least one value.", call. = FALSE) + } + + if (anyNA(participant_id)) { + stop("participant_id must not contain missing values.", call. = FALSE) + } + + if (anyDuplicated(participant_id)) { + stop("participant_id values must be unique.", call. = FALSE) + } + + if (!is.numeric(n_bytes) || + length(n_bytes) != 1 || + is.na(n_bytes) || + n_bytes != as.integer(n_bytes) || + n_bytes < 16) { + stop("n_bytes must be a whole number greater than or equal to 16.", + call. = FALSE + ) + } + + if (!is.character(prefix) || length(prefix) != 1 || is.na(prefix)) { + stop("prefix must be a single non-missing character value.", call. = FALSE) + } + + if (!is.logical(overwrite) || length(overwrite) != 1 || is.na(overwrite)) { + stop("overwrite must be TRUE or FALSE.", call. = FALSE) + } + + if (!is.null(bridge_path)) { + if (!is.character(bridge_path) || + length(bridge_path) != 1 || + is.na(bridge_path)) { + stop("bridge_path must be NULL or a single file path.", call. = FALSE) + } + + if (file.exists(bridge_path) && !overwrite) { + stop( + "bridge_path already exists. Set overwrite = TRUE to replace it.", + call. = FALSE + ) + } + } + + n_bytes <- as.integer(n_bytes) + n <- length(participant_id) + random_bytes <- openssl::rand_bytes(n * n_bytes) + random_hex <- format(random_bytes) + token_hex <- apply( + matrix(random_hex, nrow = n, ncol = n_bytes, byrow = TRUE), + 1, + paste0, + collapse = "" + ) + token_id <- paste0(prefix, "_", token_hex) + + if (anyDuplicated(token_id)) { + stop("Duplicate token generated. Re-run with more bytes.", call. = FALSE) + } + + bridge <- tibble::tibble( + participant_id = participant_id, + token_id = token_id + ) + + if (!is.null(bridge_path)) { + utils::write.csv(bridge, file = bridge_path, row.names = FALSE) + } + + bridge +} diff --git a/future/TODOs.md b/future/TODOs.md index 81396bb..d5ce251 100644 --- a/future/TODOs.md +++ b/future/TODOs.md @@ -53,7 +53,7 @@ - [ ] Sanitise dictionaries so that R, QGIS, SQL/MariaDB/postgreSQL can easily use them as input - [ ] Revisit large-data backend implementation after the design spec is accepted. - [ ] review `future/ideas/codex_reviews_from_PRs_to_check_minor.md` and determine whether they are still valid and need addressing. -- [ ] add functions to pseudonimise PII: identify columns, create toke ID, bridge file, remove columns, pass for check, write read-only raw data file. +- [ ] Add pseudonymisation helpers. Activated v1 as spec `005-pseudonymisation-bridge`: create secure token bridge table only. Later scope remains: identify PII columns, pseudonymise datasets, remove identifier columns, validate outputs, and write read-only raw data files. - [ ] add functions to load, connect, etc data into db. ### Priority 3 diff --git a/future/specs/005-pseudonymisation-bridge/acceptance.md b/future/specs/005-pseudonymisation-bridge/acceptance.md new file mode 100644 index 0000000..886fed7 --- /dev/null +++ b/future/specs/005-pseudonymisation-bridge/acceptance.md @@ -0,0 +1,23 @@ +# Acceptance + +Spec ID: `005` +Status: Implemented + +- [x] SDD is complete before implementation. +- [x] TDD plan is complete before implementation. +- [x] Executable tests are under `tests/testthat/`. +- [x] Package code changes are scoped to this spec. +- [x] Documentation is updated where user-facing behaviour changes. +- [x] `openssl` is declared in `Imports`. +- [x] Checks listed in `manifest.yml` are run or explicitly deferred. +- [x] Review notes are recorded in `review.md`. + +## Check Results + +- `scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::document()"`: + passed and generated `man/epi_sec_pseudonym.Rd`. +- `scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')"`: + passed with existing skips and one existing parallel-worker warning. +- `scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)"`: + passed with 0 errors, 0 warnings and 1 NOTE for bundled `.gitkeep` files in + `inst/project-template/`. diff --git a/future/specs/005-pseudonymisation-bridge/brief.md b/future/specs/005-pseudonymisation-bridge/brief.md new file mode 100644 index 0000000..9037008 --- /dev/null +++ b/future/specs/005-pseudonymisation-bridge/brief.md @@ -0,0 +1,39 @@ +# Brief + +Spec ID: `005` +Status: Implemented +Owner: Antonio Berlanga-Taylor + +## Problem + +Study datasets often need participant identifiers replaced by non-derivable +tokens before analysis. Hashing names, CURP, email, study IDs or other direct +identifiers is unsafe without carefully managed keyed HMAC because identifiers +can be guessed. + +## Goal + +Add `epi_sec_pseudonym()` to generate a secure bridge table that maps each +participant identifier to a random cryptographic token. + +## Non-goals + +- Detecting PII columns. +- Rewriting full study datasets. +- Removing identifier columns from analysis datasets. +- Chunked bridge generation. +- Database-backed bridge storage. + +## Candidate Files + +- `R/epi_sec_pseudonym.R` +- `tests/testthat/test-epi_sec_pseudonym.R` +- `DESCRIPTION` +- `NEWS.md` +- `future/TODOs.md` + +## Risks + +- Bridge files remain re-identifying data and must not be treated as anonymous. +- Non-cryptographic or deterministic token generation would weaken the helper. +- Overwriting an existing bridge file could break linkage or audit trails. diff --git a/future/specs/005-pseudonymisation-bridge/manifest.yml b/future/specs/005-pseudonymisation-bridge/manifest.yml new file mode 100644 index 0000000..83fe814 --- /dev/null +++ b/future/specs/005-pseudonymisation-bridge/manifest.yml @@ -0,0 +1,33 @@ +id: "005" +slug: "pseudonymisation-bridge" +status: implemented +owner: Antonio Berlanga-Taylor +base_branch: episcout2 +working_branch: feature/pseudonymisation-bridge +created: 2026-06-18 +updated: 2026-06-18 + +related_files: + - R/epi_sec_pseudonym.R + - tests/testthat/test-epi_sec_pseudonym.R + - DESCRIPTION + - NEWS.md + - future/TODOs.md + +documents: + brief: brief.md + sdd: sdd.md + tdd: tdd.md + acceptance: acceptance.md + review: review.md + +review_required: true + +check_commands: + - scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::document()" + - scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" + - scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" + +notes: + - V1 creates a secure bridge table only. + - Full dataset pseudonymisation, PII detection and database-backed bridges are out of scope. diff --git a/future/specs/005-pseudonymisation-bridge/review.md b/future/specs/005-pseudonymisation-bridge/review.md new file mode 100644 index 0000000..a5b893c --- /dev/null +++ b/future/specs/005-pseudonymisation-bridge/review.md @@ -0,0 +1,14 @@ +# Review + +Spec ID: `005` +Status: Implemented + +## Notes + +- Implemented narrow v1 bridge-table generation only. +- Used `openssl::rand_bytes()` for non-deterministic cryptographic tokens. +- Added overwrite protection for optional bridge CSV output. +- Documented that bridge files remain re-identifying data and must be stored + separately from pseudonymised analysis data. +- Full test suite passed. `devtools::check(manual = FALSE)` passed with one + existing NOTE about `.gitkeep` files in `inst/project-template/`. diff --git a/future/specs/005-pseudonymisation-bridge/sdd.md b/future/specs/005-pseudonymisation-bridge/sdd.md new file mode 100644 index 0000000..63404c3 --- /dev/null +++ b/future/specs/005-pseudonymisation-bridge/sdd.md @@ -0,0 +1,33 @@ +# Software Design + +Spec ID: `005` +Status: Implemented + +## Public API + +```r +epi_sec_pseudonym( + participant_id, + n_bytes = 24, + prefix = "P", + bridge_path = NULL, + overwrite = FALSE +) +``` + +## Behaviour + +- Accept character, numeric or factor participant IDs. +- Require participant IDs to be non-missing and unique. +- Require `n_bytes` to be a scalar whole number of at least 16. +- Generate lowercase hexadecimal token IDs from `openssl::rand_bytes()`. +- Return a tibble with `participant_id` and `token_id`, preserving input order. +- Write a CSV bridge file only when `bridge_path` is supplied. +- Refuse to overwrite an existing bridge file unless `overwrite = TRUE`. +- Do not accept a seed and do not provide deterministic output. + +## Security Notes + +The bridge table remains re-identifying information. Documentation must state +that it should be stored separately from pseudonymised analysis data and +protected with appropriate access controls. diff --git a/future/specs/005-pseudonymisation-bridge/tdd.md b/future/specs/005-pseudonymisation-bridge/tdd.md new file mode 100644 index 0000000..1ae9acc --- /dev/null +++ b/future/specs/005-pseudonymisation-bridge/tdd.md @@ -0,0 +1,44 @@ +# Test Design + +Spec ID: `005` +Status: Implemented + +## Test Files + +- `tests/testthat/test-epi_sec_pseudonym.R` + +## Baseline Commands + +```bash +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" +``` + +## Behaviour Tests + +- [x] Returns a tibble with columns `participant_id` and `token_id`. +- [x] Preserves participant ID order and row count. +- [x] Generates unique tokens for representative input. +- [x] Uses requested prefix and expected hex length. +- [x] Writes CSV output matching the returned bridge when `bridge_path` is supplied. + +## Edge-case Tests + +- [x] Accepts numeric participant IDs. +- [x] Accepts factor participant IDs. +- [x] Uses 192-bit tokens by default. +- [x] Allows overwriting only when `overwrite = TRUE`. + +## Failure Tests + +- [x] Errors on duplicate participant IDs. +- [x] Errors on missing participant IDs. +- [x] Errors when `n_bytes < 16`. +- [x] Errors when output file exists and `overwrite = FALSE`. + +## Acceptance Commands + +```bash +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::document()" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::test(reporter = 'summary')" +scripts/rscript_env_caller.R -e "options(repos = c(CRAN = 'https://cloud.r-project.org')); devtools::check(manual = FALSE)" +``` diff --git a/man/epi_sec_pseudonym.Rd b/man/epi_sec_pseudonym.Rd new file mode 100644 index 0000000..1467e8a --- /dev/null +++ b/man/epi_sec_pseudonym.Rd @@ -0,0 +1,53 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/epi_sec_pseudonym.R +\name{epi_sec_pseudonym} +\alias{epi_sec_pseudonym} +\title{Generate a Secure Pseudonymisation Bridge Table} +\usage{ +epi_sec_pseudonym( + participant_id, + n_bytes = 24, + prefix = "P", + bridge_path = NULL, + overwrite = FALSE +) +} +\arguments{ +\item{participant_id}{Character, numeric or factor vector of participant +identifiers. Values must be unique and non-missing.} + +\item{n_bytes}{Number of random bytes per token. Must be at least 16. The +default of 24 gives 192-bit tokens.} + +\item{prefix}{Character prefix added before each token.} + +\item{bridge_path}{Optional file path for writing the bridge table as CSV. +If \code{NULL}, no file is written.} + +\item{overwrite}{Logical. If \code{FALSE}, an existing \code{bridge_path} is not +overwritten.} +} +\value{ +A tibble with columns \code{participant_id} and \code{token_id}. +} +\description{ +Generate random, non-derivable token identifiers for participant +pseudonymisation. The returned bridge table maps each original participant +identifier to a cryptographically random token. +} +\details{ +Tokens are generated from cryptographic random bytes using +\code{openssl::rand_bytes()}. The function does not accept a seed and does not +provide deterministic output. + +The bridge table remains re-identifying information. Store it separately +from pseudonymised analysis datasets and protect it with appropriate access +controls. Pseudonymisation reduces risk but does not by itself anonymise a +dataset. +} +\examples{ +participant_ids <- sprintf("study_\%04d", 1:10) +bridge <- epi_sec_pseudonym(participant_ids, prefix = "MXSTUDY") +head(bridge) + +} diff --git a/tests/testthat/test-epi_sec_pseudonym.R b/tests/testthat/test-epi_sec_pseudonym.R new file mode 100644 index 0000000..aa01b5f --- /dev/null +++ b/tests/testthat/test-epi_sec_pseudonym.R @@ -0,0 +1,97 @@ +context("secure pseudonymisation bridge helper") + +library(episcout) +library(testthat) + +test_that("epi_sec_pseudonym returns bridge table in input order", { + participant_id <- c("study_003", "study_001", "study_002") + bridge <- epi_sec_pseudonym(participant_id, prefix = "MX", n_bytes = 16) + + expect_true(tibble::is_tibble(bridge)) + expect_named(bridge, c("participant_id", "token_id")) + expect_equal(nrow(bridge), length(participant_id)) + expect_equal(bridge$participant_id, participant_id) +}) + +test_that("epi_sec_pseudonym generates unique prefixed hex tokens", { + participant_id <- sprintf("study_%03d", 1:100) + bridge <- epi_sec_pseudonym(participant_id, prefix = "MXSTUDY", n_bytes = 16) + + expect_equal(length(unique(bridge$token_id)), length(participant_id)) + expect_true(all(grepl("^MXSTUDY_[0-9a-f]{32}$", bridge$token_id))) +}) + +test_that("epi_sec_pseudonym uses 192-bit tokens by default", { + bridge <- epi_sec_pseudonym(c("a", "b"), prefix = "P") + + expect_true(all(grepl("^P_[0-9a-f]{48}$", bridge$token_id))) +}) + +test_that("epi_sec_pseudonym accepts numeric and factor participant IDs", { + numeric_bridge <- epi_sec_pseudonym(1:3, n_bytes = 16) + factor_bridge <- epi_sec_pseudonym(factor(c("a", "b", "c")), n_bytes = 16) + + expect_equal(numeric_bridge$participant_id, 1:3) + expect_true(is.factor(factor_bridge$participant_id)) + expect_equal(as.character(factor_bridge$participant_id), c("a", "b", "c")) +}) + +test_that("epi_sec_pseudonym errors on invalid participant IDs", { + expect_error( + epi_sec_pseudonym(c("a", "a"), n_bytes = 16), + "participant_id values must be unique" + ) + expect_error( + epi_sec_pseudonym(c("a", NA), n_bytes = 16), + "participant_id must not contain missing values" + ) + expect_error( + epi_sec_pseudonym(list("a", "b"), n_bytes = 16), + "participant_id must be a character, numeric or factor vector" + ) +}) + +test_that("epi_sec_pseudonym errors on invalid token options", { + expect_error( + epi_sec_pseudonym(c("a", "b"), n_bytes = 15), + "n_bytes must be a whole number greater than or equal to 16" + ) + expect_error( + epi_sec_pseudonym(c("a", "b"), prefix = NA_character_, n_bytes = 16), + "prefix must be a single non-missing character value" + ) +}) + +test_that("epi_sec_pseudonym optionally writes bridge CSV", { + bridge_path <- tempfile(fileext = ".csv") + bridge <- epi_sec_pseudonym( + c("study_1", "study_2"), + prefix = "TEST", + n_bytes = 16, + bridge_path = bridge_path + ) + written <- utils::read.csv(bridge_path, stringsAsFactors = FALSE) + + expect_true(file.exists(bridge_path)) + expect_equal(written, as.data.frame(bridge)) +}) + +test_that("epi_sec_pseudonym refuses overwrite unless requested", { + bridge_path <- tempfile(fileext = ".csv") + writeLines("existing", bridge_path) + + expect_error( + epi_sec_pseudonym(c("study_1", "study_2"), bridge_path = bridge_path), + "bridge_path already exists" + ) + + bridge <- epi_sec_pseudonym( + c("study_1", "study_2"), + n_bytes = 16, + bridge_path = bridge_path, + overwrite = TRUE + ) + written <- utils::read.csv(bridge_path, stringsAsFactors = FALSE) + + expect_equal(written, as.data.frame(bridge)) +}) From 167ea24750c371ee46b97ccff2916b55dcedac60 Mon Sep 17 00:00:00 2001 From: antoniojbt <5346783+antoniojbt@users.noreply.github.com> Date: Thu, 18 Jun 2026 17:52:14 -0600 Subject: [PATCH 33/33] Fix CodeFactor lint findings --- .lintr | 2 +- R/eda_synthetic.R | 4 +- R/epi_stats_numeric.R | 18 +- R/use_episcout_project.R | 4 +- tests/testthat/test-eda_synthetic-fixtures.R | 18 +- tests/testthat/test-epi_stats_numeric.R | 200 +++++++++---------- tests/testthat/test-project-template.R | 12 +- 7 files changed, 133 insertions(+), 125 deletions(-) diff --git a/.lintr b/.lintr index 2ebeac5..ae21df4 100644 --- a/.lintr +++ b/.lintr @@ -1,2 +1,2 @@ linters: linters_with_defaults(line_length_linter = line_length_linter(150), object_name_linter = object_name_linter(styles = "snake_case"), assignment_linter = assignment_linter(), spaces_left_parentheses_linter = spaces_left_parentheses_linter(), object_usage_linter = object_usage_linter(), commented_code_linter = NULL, indentation_linter(indent = 2L, hanging_indent_style = "always")) -exclusions: list("blurbs_and_old_code/*") +exclusions: list("blurbs_and_old_code/*", "scripts/rscript_env_caller.R") diff --git a/R/eda_synthetic.R b/R/eda_synthetic.R index 9a98ddb..3c09b01 100644 --- a/R/eda_synthetic.R +++ b/R/eda_synthetic.R @@ -14,7 +14,7 @@ #' @return A data frame with one column per specification variable and `n` rows. #' #' @export -epi_eda_generate_synthetic_data <- function(spec, n = 100, seed = NULL) { +epi_eda_generate_synthetic_data <- function(spec, n = 100, seed = NULL) { # nolint: object_length_linter spec <- if (is.data.frame(spec)) { epi_eda_validate_spec(spec) } else { @@ -41,7 +41,7 @@ epi_eda_generate_synthetic_data <- function(spec, n = 100, seed = NULL) { rm(".Random.seed", envir = .GlobalEnv) } } else { - assign(".Random.seed", old_seed, envir = .GlobalEnv) + assign(".Random.seed", old_seed, envir = .GlobalEnv) # nolint: object_name_linter } }, add = TRUE) set.seed(seed) diff --git a/R/epi_stats_numeric.R b/R/epi_stats_numeric.R index cc15f5f..52f49dc 100644 --- a/R/epi_stats_numeric.R +++ b/R/epi_stats_numeric.R @@ -77,8 +77,8 @@ epi_stats_numeric <- function(num_vec = NULL, # Basic counts n <- length(num_vec) - n_nonNA <- sum(!is.na(num_vec)) - NA_count <- n - n_nonNA + n_non_na <- sum(!is.na(num_vec)) + NA_count <- n - n_non_na NA_percentage <- if (n > 0) (NA_count / n) * 100 else NA_real_ safe_scalar <- function(expr) { @@ -94,7 +94,7 @@ epi_stats_numeric <- function(num_vec = NULL, } # Summaries - has_non_missing <- n_nonNA > 0 + has_non_missing <- n_non_na > 0 sum_val <- sum(num_vec, na.rm = na.rm) min_val <- if (has_non_missing) safe_scalar(min(num_vec, na.rm = na.rm)) else NA_real_ q1 <- if (has_non_missing) { @@ -114,13 +114,13 @@ epi_stats_numeric <- function(num_vec = NULL, sd_val <- if (has_non_missing) safe_scalar(sd(num_vec, na.rm = na.rm)) else NA_real_ variance_val <- if (has_non_missing) safe_scalar(var(num_vec, na.rm = na.rm)) else NA_real_ cv_val <- if (!is.na(mean_val) && mean_val != 0) sd_val / mean_val else NA_real_ - sem_val <- if (n_nonNA > 0) sd_val / sqrt(n_nonNA) else NA_real_ - skewness_val <- if (n_nonNA >= 3) { + sem_val <- if (n_non_na > 0) sd_val / sqrt(n_non_na) else NA_real_ + skewness_val <- if (n_non_na >= 3) { safe_scalar(e1071::skewness(num_vec, na.rm = na.rm, ...)) } else { NA_real_ } - kurtosis_val <- if (n_nonNA >= 3) { + kurtosis_val <- if (n_non_na >= 3) { safe_scalar(e1071::kurtosis(num_vec, na.rm = na.rm, ...)) } else { NA_real_ @@ -132,11 +132,11 @@ epi_stats_numeric <- function(num_vec = NULL, n_below_lower <- sum(num_vec < lower_fence, na.rm = TRUE) n_above_upper <- sum(num_vec > upper_fence, na.rm = TRUE) outlier_count <- epi_stats_count_outliers(num_vec, coef = coef) - outlier_percentage <- if (n_nonNA > 0) outlier_count / n_nonNA * 100 else NA_real_ + outlier_percentage <- if (n_non_na > 0) outlier_count / n_non_na * 100 else NA_real_ # Shapiro-Wilk normality normality <- NA_real_ - if (n_nonNA > 3 && n_nonNA < 5000) { + if (n_non_na > 3 && n_non_na < 5000) { normality <- tryCatch( shapiro.test(num_vec[!is.na(num_vec)])$p.value, error = function(e) NA_real_ @@ -147,7 +147,7 @@ epi_stats_numeric <- function(num_vec = NULL, desc_stats <- data.frame( n = n, - n_nonNA = n_nonNA, + n_nonNA = n_non_na, # nolint: object_name_linter NA_count = NA_count, NA_percentage = NA_percentage, sum = sum_val, diff --git a/R/use_episcout_project.R b/R/use_episcout_project.R index d63eb02..de4b07e 100644 --- a/R/use_episcout_project.R +++ b/R/use_episcout_project.R @@ -14,7 +14,7 @@ #' @export epi_eda_create_project <- function(path, overwrite = FALSE) { path <- validate_episcout_project_path(path) - overwrite <- validate_episcout_project_overwrite(overwrite) + overwrite <- validate_project_overwrite(overwrite) template_path <- system.file("project-template", package = "episcout") if (!nzchar(template_path) || !dir.exists(template_path)) { @@ -94,7 +94,7 @@ validate_episcout_project_path <- function(path) { path.expand(path) } -validate_episcout_project_overwrite <- function(overwrite) { +validate_project_overwrite <- function(overwrite) { if (!is.logical(overwrite) || length(overwrite) != 1 || is.na(overwrite)) { stop("overwrite must be TRUE or FALSE.", call. = FALSE) } diff --git a/tests/testthat/test-eda_synthetic-fixtures.R b/tests/testthat/test-eda_synthetic-fixtures.R index 7052794..f4ee420 100644 --- a/tests/testthat/test-eda_synthetic-fixtures.R +++ b/tests/testthat/test-eda_synthetic-fixtures.R @@ -28,8 +28,12 @@ test_that("synthetic data has the requested row count", { test_that("synthetic categorical and binary values respect specification levels", { synthetic <- epi_eda_generate_synthetic_data(spec, n = 100, seed = 1) - categorical_spec <- spec[spec$type %in% c("categorical", "binary") & - !is.na(spec$levels) & spec$levels != "", , drop = FALSE] + categorical_spec <- spec[ + spec$type %in% c("categorical", "binary") & + !is.na(spec$levels) & spec$levels != "", + , + drop = FALSE + ] for (i in seq_len(nrow(categorical_spec))) { name <- categorical_spec$name[i] @@ -46,9 +50,13 @@ test_that("synthetic categorical and binary values respect specification levels" test_that("synthetic numeric and integer values respect specification min and max", { synthetic <- epi_eda_generate_synthetic_data(spec, n = 100, seed = 1) - ranged_spec <- spec[spec$type %in% c("numeric", "integer") & - !is.na(spec$min) & spec$min != "" & - !is.na(spec$max) & spec$max != "", , drop = FALSE] + ranged_spec <- spec[ + spec$type %in% c("numeric", "integer") & + !is.na(spec$min) & spec$min != "" & + !is.na(spec$max) & spec$max != "", + , + drop = FALSE + ] for (i in seq_len(nrow(ranged_spec))) { name <- ranged_spec$name[i] diff --git a/tests/testthat/test-epi_stats_numeric.R b/tests/testthat/test-epi_stats_numeric.R index 924605c..9ffe04f 100644 --- a/tests/testthat/test-epi_stats_numeric.R +++ b/tests/testthat/test-epi_stats_numeric.R @@ -7,128 +7,128 @@ library(testthat) context("epi_stats_numeric") test_that("basic summary for a simple vector without NAs", { - v <- c(1, 2, 3, 4) - res <- epi_stats_numeric(v) - - expect_s3_class(res, "data.frame") - expect_equal(nrow(res), 1L) - - # Counts - expect_equal(res$n, 4L) - expect_equal(res$n_nonNA, 4L) - expect_equal(res$NA_count, 0L) - expect_equal(res$NA_percentage, 0) - - # Location & dispersion - expect_equal(res$sum, sum(v)) - expect_equal(res$min, min(v)) - expect_equal(res$quantile_25, quantile(v, .25, names = FALSE)) - expect_equal(res$mean, mean(v)) - expect_equal(res$median, median(v)) - expect_equal(res$quantile_75, quantile(v, .75, names = FALSE)) - expect_equal(res$max, max(v)) - - # IQR, SD, variance, CV, sem - expect_equal(res$IQR, IQR(v)) - expect_equal(res$SD, sd(v)) - expect_equal(res$variance, var(v)) - expect_equal(res$CV, sd(v) / mean(v)) - expect_equal(res$sem, sd(v) / sqrt(length(v))) - - # Outlier fences & counts (no outliers here) - q1 <- quantile(v, .25, names = FALSE) - q3 <- quantile(v, .75, names = FALSE) - expect_equal(res$lower_fence, q1 - 1.5 * IQR(v)) - expect_equal(res$upper_fence, q3 + 1.5 * IQR(v)) - expect_equal(res$n_below_lower, 0L) - expect_equal(res$n_above_upper, 0L) - expect_equal(res$outlier_count, 0L) - expect_equal(res$outlier_percentage, 0) - - # Shapiro–Wilk p–value should be numeric for n=4 - expect_true(is.numeric(res$Shapiro_Wilk_p_value)) - expect_true(!is.na(res$Shapiro_Wilk_p_value)) + v <- c(1, 2, 3, 4) + res <- epi_stats_numeric(v) + + expect_s3_class(res, "data.frame") + expect_equal(nrow(res), 1L) + + # Counts + expect_equal(res$n, 4L) + expect_equal(res$n_nonNA, 4L) + expect_equal(res$NA_count, 0L) + expect_equal(res$NA_percentage, 0) + + # Location & dispersion + expect_equal(res$sum, sum(v)) + expect_equal(res$min, min(v)) + expect_equal(res$quantile_25, quantile(v, .25, names = FALSE)) + expect_equal(res$mean, mean(v)) + expect_equal(res$median, median(v)) + expect_equal(res$quantile_75, quantile(v, .75, names = FALSE)) + expect_equal(res$max, max(v)) + + # IQR, SD, variance, CV, sem + expect_equal(res$IQR, IQR(v)) + expect_equal(res$SD, sd(v)) + expect_equal(res$variance, var(v)) + expect_equal(res$CV, sd(v) / mean(v)) + expect_equal(res$sem, sd(v) / sqrt(length(v))) + + # Outlier fences & counts (no outliers here) + q1 <- quantile(v, .25, names = FALSE) + q3 <- quantile(v, .75, names = FALSE) + expect_equal(res$lower_fence, q1 - 1.5 * IQR(v)) + expect_equal(res$upper_fence, q3 + 1.5 * IQR(v)) + expect_equal(res$n_below_lower, 0L) + expect_equal(res$n_above_upper, 0L) + expect_equal(res$outlier_count, 0L) + expect_equal(res$outlier_percentage, 0) + + # Shapiro–Wilk p–value should be numeric for n=4 + expect_true(is.numeric(res$Shapiro_Wilk_p_value)) + expect_true(!is.na(res$Shapiro_Wilk_p_value)) }) test_that("NAs are counted and percentages correct", { - v <- c(NA, 1, 2, NA) - res <- epi_stats_numeric(v) + v <- c(NA, 1, 2, NA) + res <- epi_stats_numeric(v) - expect_equal(res$n, 4L) - expect_equal(res$n_nonNA, 2L) - expect_equal(res$NA_count, 2L) - expect_equal(res$NA_percentage, 50) + expect_equal(res$n, 4L) + expect_equal(res$n_nonNA, 2L) + expect_equal(res$NA_count, 2L) + expect_equal(res$NA_percentage, 50) }) test_that("outlier detection works", { - v <- c(1, 2, 3, 100) # 100 is a Tukey outlier - res <- epi_stats_numeric(v, coef = 1.5) - - # Only one value above the upper fence - expect_equal(res$outlier_count, 1L) - expect_equal(res$n_above_upper, 1L) - expect_equal(res$n_below_lower, 0L) - expect_equal(res$outlier_percentage, 1 / res$n_nonNA * 100) + v <- c(1, 2, 3, 100) # 100 is a Tukey outlier + res <- epi_stats_numeric(v, coef = 1.5) + + # Only one value above the upper fence + expect_equal(res$outlier_count, 1L) + expect_equal(res$n_above_upper, 1L) + expect_equal(res$n_below_lower, 0L) + expect_equal(res$outlier_percentage, 1 / res$n_nonNA * 100) }) test_that("Shapiro–Wilk skipped for too few or too many observations", { - # Too few (n_nonNA = 3) - small <- c(1, 2, NA, 3) - res_small <- epi_stats_numeric(small) - expect_true(is.na(res_small$Shapiro_Wilk_p_value)) - - # Too many (n_nonNA > 4999) - large <- rep(1, 5000) - res_large <- epi_stats_numeric(large) - expect_true(is.na(res_large$Shapiro_Wilk_p_value)) + # Too few (n_nonNA = 3) + small <- c(1, 2, NA, 3) + res_small <- epi_stats_numeric(small) + expect_true(is.na(res_small$Shapiro_Wilk_p_value)) + + # Too many (n_nonNA > 4999) + large <- rep(1, 5000) + res_large <- epi_stats_numeric(large) + expect_true(is.na(res_large$Shapiro_Wilk_p_value)) }) test_that("all-missing numeric vectors return deterministic missing summaries", { - res <- expect_silent(epi_stats_numeric(c(NA_real_, NA_real_, NA_real_))) - - expect_s3_class(res, "data.frame") - expect_equal(nrow(res), 1L) - expect_equal(res$n, 3L) - expect_equal(res$n_nonNA, 0L) - expect_equal(res$NA_count, 3L) - expect_equal(res$NA_percentage, 100) - expect_true(is.na(res$min)) - expect_true(is.na(res$mean)) - expect_true(is.na(res$max)) - expect_true(is.na(res$CV)) - expect_true(is.na(res$skewness)) - expect_true(is.na(res$kurtosis)) - expect_true(is.na(res$Shapiro_Wilk_p_value)) - expect_true(is.na(res$outlier_percentage)) + res <- expect_silent(epi_stats_numeric(c(NA_real_, NA_real_, NA_real_))) + + expect_s3_class(res, "data.frame") + expect_equal(nrow(res), 1L) + expect_equal(res$n, 3L) + expect_equal(res$n_nonNA, 0L) + expect_equal(res$NA_count, 3L) + expect_equal(res$NA_percentage, 100) + expect_true(is.na(res$min)) + expect_true(is.na(res$mean)) + expect_true(is.na(res$max)) + expect_true(is.na(res$CV)) + expect_true(is.na(res$skewness)) + expect_true(is.na(res$kurtosis)) + expect_true(is.na(res$Shapiro_Wilk_p_value)) + expect_true(is.na(res$outlier_percentage)) }) test_that("zero-length numeric vectors return one stable summary row", { - res <- expect_silent(epi_stats_numeric(numeric())) - - expect_s3_class(res, "data.frame") - expect_equal(nrow(res), 1L) - expect_equal(res$n, 0L) - expect_equal(res$n_nonNA, 0L) - expect_equal(res$NA_count, 0L) - expect_true(is.na(res$NA_percentage)) - expect_true(is.na(res$mean)) - expect_true(is.na(res$CV)) - expect_true(is.na(res$Shapiro_Wilk_p_value)) - expect_true(is.na(res$outlier_percentage)) + res <- expect_silent(epi_stats_numeric(numeric())) + + expect_s3_class(res, "data.frame") + expect_equal(nrow(res), 1L) + expect_equal(res$n, 0L) + expect_equal(res$n_nonNA, 0L) + expect_equal(res$NA_count, 0L) + expect_true(is.na(res$NA_percentage)) + expect_true(is.na(res$mean)) + expect_true(is.na(res$CV)) + expect_true(is.na(res$Shapiro_Wilk_p_value)) + expect_true(is.na(res$outlier_percentage)) }) test_that("coefficient of variation is missing when the mean is zero", { - res <- epi_stats_numeric(c(-1, 0, 1)) + res <- epi_stats_numeric(c(-1, 0, 1)) - expect_true(is.na(res$CV)) - expect_false(is.infinite(res$CV)) + expect_true(is.na(res$CV)) + expect_false(is.infinite(res$CV)) }) test_that("shape and normality statistics are skipped when usable values are insufficient", { - res <- epi_stats_numeric(c(1, NA_real_)) + res <- epi_stats_numeric(c(1, NA_real_)) - expect_equal(res$n_nonNA, 1L) - expect_true(is.na(res$skewness)) - expect_true(is.na(res$kurtosis)) - expect_true(is.na(res$Shapiro_Wilk_p_value)) + expect_equal(res$n_nonNA, 1L) + expect_true(is.na(res$skewness)) + expect_true(is.na(res$kurtosis)) + expect_true(is.na(res$Shapiro_Wilk_p_value)) }) diff --git a/tests/testthat/test-project-template.R b/tests/testthat/test-project-template.R index dd4953a..9ca1aca 100644 --- a/tests/testthat/test-project-template.R +++ b/tests/testthat/test-project-template.R @@ -3,7 +3,7 @@ context("project template contract tests") library(testthat) library(episcout) -expected_project_template_entries <- c( +template_entries <- c( "README.md", "metadata/data_dictionary.csv", "config/eda.yml", @@ -14,8 +14,8 @@ expected_project_template_entries <- c( "outputs/.gitkeep" ) -expect_project_template_entries <- function(root) { - for (entry in expected_project_template_entries) { +expect_template_entries <- function(root) { + for (entry in template_entries) { expect_true( file.exists(file.path(root, entry)), info = paste("Missing project template entry:", entry) @@ -46,7 +46,7 @@ test_that("project template is bundled with the expected scaffold", { } expect_true(dir.exists(template_path)) - expect_project_template_entries(template_path) + expect_template_entries(template_path) expect_match(read_project_report_template(template_path), "epi_eda_render_report") expect_match(read_project_targets(template_path), "epi_eda_render_report") expect_no_match(read_project_report_template(template_path), "\\brender_eda_report\\b") @@ -62,7 +62,7 @@ test_that("epi_eda_create_project creates the expected scaffold", { normalizePath(returned_path, winslash = "/", mustWork = TRUE), normalizePath(project_path, winslash = "/", mustWork = TRUE) ) - expect_project_template_entries(project_path) + expect_template_entries(project_path) expect_match(read_project_report_template(project_path), "epi_eda_render_report") expect_match(read_project_targets(project_path), "epi_eda_render_report") }) @@ -92,6 +92,6 @@ test_that("epi_eda_create_project can overwrite existing files when requested", normalizePath(returned_path, winslash = "/", mustWork = TRUE), normalizePath(project_path, winslash = "/", mustWork = TRUE) ) - expect_project_template_entries(project_path) + expect_template_entries(project_path) expect_false(identical(readLines(existing_file, warn = FALSE), "existing")) })