From 71f07e04c8bdc36b0c68284f3bfc041f3a832c8b Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 26 Sep 2026 01:19:08 +0000 Subject: [PATCH 1/6] feat(proofs): Agda foundation for ILR bases and compositions (#20) Adds the Agda proof suite (Agda 2.6.4.3, stdlib 2.1, --safe --without-K, no postulates) and the verification plan that fixes its scope: - Composition.{Tree,Node,Sum}: shared vocabulary for #20 and #21 - ILR.SBP: SBP rows of a tree, Egozcue-Pawlowsky-Glahn conditions - ILR.Contrast: centred, orthogonal, norm^2 = r s (r+s), from SBP row - ILR.Kernel: injectivity on centred vectors under Cancellable - ILR.Invariance / Orthonormal: via named LogHom / Normaliser seams - ILR.Comb: comb tree = MetaManifold's Helmert default - ILR.Integer: positive weights discharge the hypothesis; signed-weight counterexample; philr's known-answer SBP - scripts/check-proofs.sh + CI 'proofs' job with negative controls Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/ci.yml | 34 +++ .gitignore | 7 + Justfile | 5 + README.md | 1 + docs/formal/verification-plan.md | 167 +++++++++++++ proofs/agda/MetaManifold/All.agda | 22 ++ .../agda/MetaManifold/Composition/Node.agda | 58 +++++ proofs/agda/MetaManifold/Composition/Sum.agda | 197 +++++++++++++++ .../agda/MetaManifold/Composition/Tree.agda | 113 +++++++++ proofs/agda/MetaManifold/ILR/Comb.agda | 116 +++++++++ proofs/agda/MetaManifold/ILR/Contrast.agda | 236 ++++++++++++++++++ proofs/agda/MetaManifold/ILR/Integer.agda | 122 +++++++++ proofs/agda/MetaManifold/ILR/Invariance.agda | 105 ++++++++ proofs/agda/MetaManifold/ILR/Kernel.agda | 105 ++++++++ proofs/agda/MetaManifold/ILR/Orthonormal.agda | 60 +++++ proofs/agda/MetaManifold/ILR/SBP.agda | 77 ++++++ proofs/agda/README.md | 34 +++ proofs/agda/metamanifold-proofs.agda-lib | 4 + .../agda/reject/KernelWithoutHypothesis.agda | 21 ++ proofs/agda/reject/OrthogonalSelf.agda | 21 ++ proofs/agda/reject/Postulate.agda | 11 + scripts/check-proofs.sh | 120 +++++++++ 22 files changed, 1636 insertions(+) create mode 100644 docs/formal/verification-plan.md create mode 100644 proofs/agda/MetaManifold/All.agda create mode 100644 proofs/agda/MetaManifold/Composition/Node.agda create mode 100644 proofs/agda/MetaManifold/Composition/Sum.agda create mode 100644 proofs/agda/MetaManifold/Composition/Tree.agda create mode 100644 proofs/agda/MetaManifold/ILR/Comb.agda create mode 100644 proofs/agda/MetaManifold/ILR/Contrast.agda create mode 100644 proofs/agda/MetaManifold/ILR/Integer.agda create mode 100644 proofs/agda/MetaManifold/ILR/Invariance.agda create mode 100644 proofs/agda/MetaManifold/ILR/Kernel.agda create mode 100644 proofs/agda/MetaManifold/ILR/Orthonormal.agda create mode 100644 proofs/agda/MetaManifold/ILR/SBP.agda create mode 100644 proofs/agda/README.md create mode 100644 proofs/agda/metamanifold-proofs.agda-lib create mode 100644 proofs/agda/reject/KernelWithoutHypothesis.agda create mode 100644 proofs/agda/reject/OrthogonalSelf.agda create mode 100644 proofs/agda/reject/Postulate.agda create mode 100755 scripts/check-proofs.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6acff72..fd06f929 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -162,6 +162,40 @@ jobs: fi echo "commit convention ok: $count non-merge commit(s) graded in $range" + # Agda proof gate for the compositional transforms (issues #20, #21); scope, + # layering and residue are in docs/formal/verification-plan.md. The toolchain + # is the estate's pin (epistemic-types, residual-evidence-types): Debian 13 + # packages agda-bin 2.6.4.3-1+b2 and agda-stdlib 2.1-4, installed from the + # signed Debian archive rather than a downloaded binary. The same script runs + # locally with AGDA=... AGDA_STDLIB_LIB=... overrides. + proofs: + name: Proofs (Agda) + runs-on: ubuntu-24.04 + timeout-minutes: 30 + container: debian:13-slim@sha256:d7e12182ce18b85b93007c1dedf31f2d29e01ccf3182cc4017c709b6259bc132 + permissions: + contents: read + env: + LANG: C.UTF-8 + LC_ALL: C.UTF-8 + steps: + - name: Install the pinned Agda toolchain (Debian archive) + run: | + apt-get update + apt-get install --no-install-recommends -y ca-certificates git \ + agda-bin=2.6.4.3-1+b2 agda-stdlib=2.1-4 + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Guard, type-check, negative controls + run: | + test "$(agda --version)" = 'Agda version 2.6.4.3' + AGDA_STDLIB_LIB=$(dpkg -L agda-stdlib | grep 'standard-library\.agda-lib$' | head -1) + export AGDA_STDLIB_LIB + bash scripts/check-proofs.sh + test: # The name is a fixed string, it carries no version, and the job has NO matrix. # Both are required, and the first without the second is a trap that has already diff --git a/.gitignore b/.gitignore index cdfeec6c..fc809529 100644 --- a/.gitignore +++ b/.gitignore @@ -265,6 +265,8 @@ docs/* !docs/owner-review-2026-09-25.md !docs/integration/ !docs/integration/** +!docs/formal/ +!docs/formal/** # Test + benchmark run artifacts (not the committed fixtures/baselines) frontend/tests/results/ @@ -357,3 +359,8 @@ archive/ # Track the Stipple/Vue migration recon and implementation plan. !docs/migration/ !docs/migration/** + +### Agda ### +# Interface files (regenerated by scripts/check-proofs.sh) +proofs/agda/_build/ +*.agdai diff --git a/Justfile b/Justfile index 94daf40f..abbae4f4 100644 --- a/Justfile +++ b/Justfile @@ -274,6 +274,11 @@ lint: hygiene: spdx format lint @echo "hygiene: OK" +# Agda proof gate (docs/formal/verification-plan.md): guard, type-check, +# negative controls. Honours AGDA=... and AGDA_STDLIB_LIB=... overrides. +proofs: + ./scripts/check-proofs.sh + # Lint a commit message against the canonical format (default: HEAD). commit-check msg="": #!/usr/bin/env bash diff --git a/README.md b/README.md index f76644dc..915fd8b7 100644 --- a/README.md +++ b/README.md @@ -541,6 +541,7 @@ clean checkout (`frontend/`): | Licence headers | `scripts/check-spdx.sh` | gated | | Formatting | `scripts/check-format.sh` | gated | | Lint (tsc semantics + shell) | `scripts/check-lint.sh` | gated | +| Agda proofs (ILR bases, compositions) | `scripts/check-proofs.sh` | gated (`proofs` job) | CI runs the same gates (see `.github/workflows/ci.yml`: repo-hygiene job, then the pinned Julia and frontend jobs). Contributor setup, commit and diff --git a/docs/formal/verification-plan.md b/docs/formal/verification-plan.md new file mode 100644 index 00000000..26bb0e2d --- /dev/null +++ b/docs/formal/verification-plan.md @@ -0,0 +1,167 @@ + +# Formal verification plan — compositional transforms (issues #20, #21) + +**Status: adopted 2026-09-26.** This document fixes *which* prover is used, *what* is +proved, *what is deliberately not proved*, and *how* the proofs are tied to the Julia code +that the statistics actually run. It is written before the ILR-basis implementation +(issue #20) lands, so that the implementation is built against it rather than the other +way round. + +## 1. Decision: Agda + +| Candidate | Fit | Decision | +|-----------|-----|----------| +| **Agda 2.6.4.3 + stdlib 2.1** | Same pin as the estate's existing proof repositories (`epistemic-types`, `residual-evidence-types`: Debian 13 `agda-bin 2.6.4.3-1+b2`, `agda-stdlib 2.1-4`); constructive; `--safe --without-K`; generic over `CommutativeRing` so one proof covers ℤ, ℚ and any exact model. | **Chosen.** | +| Lean 4 + Mathlib | Best real-analysis library (`Real.log`, `Real.sqrt`, inner-product spaces). | Not chosen *now*. Adopted only if a genuine rationale appears (see §7) and then only as a labelled mirror of an Agda statement, never as a second source of truth. | +| Isabelle/HOL | Strong automation; nothing in the estate uses it. | Not chosen: a third proof culture with no shared definitions. | + +The deciding consideration is **coherence**: issue #21 (zero handling) is being done in +Agda, and the objects it reasons about — compositions, closure, log-ratios, the +geometric mean — are the same objects the ILR bases act on. Two provers would mean two +definitions of "composition" that nobody checks agree. + +### Guardrails (enforced in CI, mirroring the estate repositories) + +- Every module starts with `{-# OPTIONS --safe --without-K #-}`; the library file sets the + same flags so a file that forgets the pragma is still checked safely. +- No `postulate`, no `{-# TERMINATING #-}`, no `{-# NON_TERMINATING #-}`, no + `{-# NO_POSITIVITY_CHECK #-}`, no `{-# NO_UNIVERSE_CHECK #-}`, no holes (`?`, `{! !}`), + no `trustMe`. `--safe` already rejects most of these; the grep guard makes the policy + visible and catches the ones `--safe` permits. +- Assumptions that cannot be proved (transcendental functions, floating point) are + **record parameters with names**, never postulates, so every theorem that depends on + one says so in its type. +- Negative controls: `proofs/agda/reject/` holds files that must **fail** to type-check + (e.g. claiming non-degeneracy without its hypothesis). CI checks they still fail. A + proof suite that cannot fail has not been shown to check anything. + +## 2. Layering — what is proved at which level of abstraction + +The Julia code computes in `Float64` with `log` and `sqrt`. No prover in reach verifies +that. The honest arrangement is four layers with explicit seams: + +| Layer | Content | Status | +|-------|---------|--------| +| **L0 combinatorics** | Rooted binary trees; `D` leaves ⇒ `D − 1` internal nodes; preorder enumeration of nodes is a bijection with `Fin (D − 1)`; the sequential binary partition (SBP) code of a tree; every SBP row has a non-empty `+` and `−` part. | Proved (ℕ, `Fin`). | +| **L1 exact algebra** | Balance contrasts over an arbitrary commutative ring, with arbitrary part weights (philr's `p`): each contrast is weighted-sum-zero; distinct contrasts are orthogonal under the weighted inner product; the norm of each contrast is `r·s·(r+s)`; the balance map is linear; under a stated cancellation hypothesis it has trivial kernel; it ignores constant shifts (= scale invariance after `log`). | Proved generically; instantiated at ℤ. | +| **L2 transcendental seam** | `log` turns products into sums (`LogHom` record); the normalisation `sqrt(rs/(r+s))` is a positive scalar per balance and so preserves orthogonality and kernel. | Assumed via named records; the consequences (perturbation ↦ translation, powering ↦ scaling) are proved *relative to* the record. | +| **L3 floating point** | The Julia implementation (`src/analysis/ilr_basis.jl`). | Not proved. Each L1 theorem has a named Julia property test at tolerance (§5), and cross-implementation fixtures against an independent port of R `philr`. | + +The exact-vs-approximate distinction in `docs/statistics/numeric-contracts.md` is the +same line as L1/L3: an L1 theorem is a statement about exact arithmetic; the matching +L3 test is the claim that `Float64` stays within the stated tolerance of it. + +### Why unnormalised contrasts in L1 + +The normalised philr/Egozcue contrast has entries `±c/n±` with `c = sqrt(n₊n₋/(n₊+n₋))`: +neither the division nor the square root exists in a general ring. Every structural +property (orthogonality, sum-zero, kernel, invariance) is invariant under multiplying +each contrast by a non-zero scalar, so L1 proves them for the **integer-valued** +contrast `(Σw₋)·𝟙₊ − (Σw₊)·𝟙₋` and records the scalar separately. Over a field of +characteristic 0 the normalised basis is this one scaled by `1/sqrt(r·s·(r+s))`, which is +where the Julia constant comes from (Key identity: `norm² = r·s·(r+s)`, proved). + +## 3. Module map + +``` +proofs/agda/ + metamanifold-proofs.agda-lib depend: standard-library; flags --safe --without-K + MetaManifold/All.agda imports everything (the CI entry point) + MetaManifold/Composition/ + Tree.agda binary trees, leaves/internal counts, tip vectors + Node.agda internal-node positions; preorder bijection with Fin + Sum.agda weighted sums / inner product over a CommutativeRing + MetaManifold/ILR/ + SBP.agda sign codes; each balance has a + and a − part + Contrast.agda contrasts; sum-zero; orthogonality; norm identity + Kernel.agda trivial kernel under a named cancellation hypothesis + Invariance.agda linearity; shift (scale) invariance; LogHom seam + Comb.agda the comb tree reproduces the Helmert default + Integer.agda ℤ instance; uniform weights discharge the hypothesis; + a counterexample showing the hypothesis is needed + reject/ files that MUST fail to type-check +scripts/check-proofs.sh guard + type-check + expected-rejection check +``` + +### Shared foundation for issue #21 + +`MetaManifold.Composition.*` is deliberately prover-level *shared* vocabulary, not ILR +code. #21's modules (multiplicative replacement with δ, Bayesian-multiplicative) should +live under `MetaManifold.ZeroReplacement.*` and reuse `Composition.Sum` (weighted sums, +closure-as-sum) rather than redefining them. At the time of writing no #21 Agda file +exists on any branch of this repository; when one appears the first task is to reconcile +it with this namespace (merge definitions, keep one), and this section must be updated. +Natural #21 statements that compose with the ILR results: + +- multiplicative replacement preserves the ratios of non-zero parts (so every ILR balance + whose parts are all non-zero is unchanged by replacement) — this is the formal content + of "replacement touches only the zero cells", and it composes directly with the + shift-invariance theorem here; +- replacement output is strictly positive, which is the precondition of the L2 `log`. + +## 4. Mapping the proof objects to the implementation + +| Proof object | Julia | Notes | +|---|---|---| +| `Tree` (`leaf`, `node l r`) | `ILRBasis.PhyloTree` after validation | Julia refuses multifurcations and unrooted (trifurcating-root) trees instead of resolving them: the proof object *is* bifurcating, and silently resolving a polytomy would invent a hypothesis. | +| `code` (SBP of a tree) | `sbp_from_tree` | Row = balance, `+` = first child, `−` = second child (philr's `phylo2sbp` convention). | +| SBP validity | `validate_sbp` | Julia checks the Egozcue & Pawlowsky-Glahn (2005) conditions by reconstructing the tree; a matrix that reconstructs is exactly one in the image of `code`. | +| `contrast` / weights `w` | `balance_coefficients`, `part_weights` | Unnormalised in Agda, normalised in Julia by `1/sqrt(rs(r+s))`. | +| preorder `toFin` | balance order | Julia emits balances in preorder (root first), the same order as philr's node numbering. | +| `Comb` theorem | `helmert_balance_matrix` equivalence test | The comb tree's root contrast is Helmert's *last* balance, i.e. the order is reversed; the test maps indices explicitly. | + +## 5. Theorem ↔ test traceability + +Every L1 theorem has a Julia property test with the same name in its description +(`test/unit/test_ilr_basis.jl`). If a theorem is added, a test is added. + +| Agda theorem | Julia test (tolerance) | +|---|---| +| `internal-count` | "D taxa give D-1 balances" (exact) | +| `code-has-plus`, `code-has-minus` | "every SBP row has a + and a - part" (exact) | +| `contrast-sum-zero` | "balances of a constant composition are zero" (1e-12) | +| `contrast-orthogonal` | "basis columns are orthonormal under part weights" (1e-12) | +| `contrast-norm` | "normalisation constant equals sqrt(rs/(r+s))" (1e-12) | +| `balance-kernel` | "distinct CLR vectors give distinct balances" (sampled; 1e-9) | +| `balance-shift-invariant` | "scaling a sample leaves balances unchanged" (1e-12) | +| `balance-linear` | "perturbation adds balance vectors" (1e-12) | +| `comb-is-helmert` | "comb tree reproduces the Helmert default" (1e-12) | +| `nondegeneracy-needs-hypothesis` (counterexample) | "signed part weights are refused" (exact) | + +## 6. Residue — stated, not hidden + +Not proved, by design, and listed so no one mistakes the suite's scope: + +1. Properties of IEEE-754 `log`, `sqrt`, summation order (L3; covered by tolerance tests). +2. That `D − 1` orthogonal non-zero contrasts **span** the sum-zero hyperplane. Over a + field this is dimension counting (`D − 1` independent vectors in a `D − 1`-dimensional + space); stdlib 2.1 has no linear-algebra dimension theory, and building one to restate + a textbook fact is poor value. The kernel theorem (injectivity) is the half the + statistics depends on: no two distinct CLR vectors share balances. +3. Correctness of the Newick/CSV parsers (tested; parsers are not in scope for proof). +4. Hierarchical clustering optimality (the dendrogram basis is *defined* by the Lance– + Williams recurrence; tests pin it to scipy and to hand-computed cases). +5. Statistical validity of any downstream test. Proofs here are about the transform. + +## 7. When Lean would be justified + +Recorded so the question is answered by criteria rather than taste. A Lean mirror is +justified if a *required* statement needs real analysis that Agda's stdlib cannot +express without building a real-number library, for example: + +- concavity/uniqueness of the negative-binomial likelihood maximiser (#21 glmGamPoi); +- convergence or error bounds of an iterative estimator; +- measure-theoretic properties of a Bayesian-multiplicative posterior. + +In that case: the Agda statement remains canonical where one exists; the Lean file says +which Agda theorem it mirrors; and CI builds both. Nothing in #20 meets the criterion. + +## 8. CI + +Job `proofs` in `.github/workflows/ci.yml`: Debian 13 container, `apt-get install +agda-bin=2.6.4.3-1+b2 agda-stdlib=2.1-4` (the estate pin), then +`scripts/check-proofs.sh`, which (a) runs the guard grep, (b) type-checks +`MetaManifold/All.agda`, (c) asserts every file in `reject/` fails. Locally the same +script runs with `AGDA=... AGDA_STDLIB=...` overrides. diff --git a/proofs/agda/MetaManifold/All.agda b/proofs/agda/MetaManifold/All.agda new file mode 100644 index 00000000..e45a897c --- /dev/null +++ b/proofs/agda/MetaManifold/All.agda @@ -0,0 +1,22 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +------------------------------------------------------------------------ +-- Entry point: type-checking this module checks the whole suite. +-- See docs/formal/verification-plan.md for scope and residue. +------------------------------------------------------------------------ +{-# OPTIONS --safe --without-K #-} + +module MetaManifold.All where + +-- Shared compositional vocabulary (also for issue #21). +import MetaManifold.Composition.Tree +import MetaManifold.Composition.Node +import MetaManifold.Composition.Sum + +-- Issue #20: ILR bases from trees, SBPs and dendrograms. +import MetaManifold.ILR.SBP +import MetaManifold.ILR.Contrast +import MetaManifold.ILR.Kernel +import MetaManifold.ILR.Invariance +import MetaManifold.ILR.Orthonormal +import MetaManifold.ILR.Comb +import MetaManifold.ILR.Integer diff --git a/proofs/agda/MetaManifold/Composition/Node.agda b/proofs/agda/MetaManifold/Composition/Node.agda new file mode 100644 index 00000000..681755c6 --- /dev/null +++ b/proofs/agda/MetaManifold/Composition/Node.agda @@ -0,0 +1,58 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +------------------------------------------------------------------------ +-- Positions of internal nodes, and their preorder enumeration. +-- +-- `Node t` names one internal node of `t`: the root (`here`) or a node +-- inside the left / right subtree. Each internal node carries exactly +-- one balance, so `Node t` indexes the balances. +-- +-- `toFin`/`fromFin` are the preorder numbering (root first, then the +-- left subtree, then the right subtree) and are proved mutually +-- inverse: the balances of a D-tip tree are in bijection with +-- `Fin (D - 1)`. Preorder is the order in which R `philr` numbers +-- balances (ape's internal-node order) and the order in which +-- src/analysis/ilr_basis.jl emits them. +------------------------------------------------------------------------ +{-# OPTIONS --safe --without-K #-} + +module MetaManifold.Composition.Node where + +open import Data.Fin.Base using (Fin; zero; suc; _↑ˡ_; _↑ʳ_; splitAt) +open import Data.Fin.Properties + using (splitAt-↑ˡ; splitAt-↑ʳ; splitAt⁻¹-↑ˡ; splitAt⁻¹-↑ʳ) +open import Data.Sum.Base using (inj₁; inj₂) +open import Relation.Binary.PropositionalEquality + using (_≡_; refl; cong; trans) + +open import MetaManifold.Composition.Tree + +data Node : Tree → Set where + here : ∀ {l r} → Node (node l r) + inl : ∀ {l r} → Node l → Node (node l r) + inr : ∀ {l r} → Node r → Node (node l r) + +toFin : ∀ {t} → Node t → Fin (internal t) +toFin here = zero +toFin {node l r} (inl n) = suc (toFin n ↑ˡ internal r) +toFin {node l r} (inr n) = suc (internal l ↑ʳ toFin n) + +fromFin : ∀ {t} → Fin (internal t) → Node t +fromFin {node l r} zero = here +fromFin {node l r} (suc i) with splitAt (internal l) i +... | inj₁ j = inl (fromFin j) +... | inj₂ k = inr (fromFin k) + +fromFin-toFin : ∀ {t} (n : Node t) → fromFin (toFin n) ≡ n +fromFin-toFin here = refl +fromFin-toFin {node l r} (inl n) + rewrite splitAt-↑ˡ (internal l) (toFin n) (internal r) + = cong inl (fromFin-toFin n) +fromFin-toFin {node l r} (inr n) + rewrite splitAt-↑ʳ (internal l) (internal r) (toFin n) + = cong inr (fromFin-toFin n) + +toFin-fromFin : ∀ {t} (i : Fin (internal t)) → toFin (fromFin {t} i) ≡ i +toFin-fromFin {node l r} zero = refl +toFin-fromFin {node l r} (suc i) with splitAt (internal l) i in eq +... | inj₁ j = cong suc (trans (cong (_↑ˡ internal r) (toFin-fromFin j)) (splitAt⁻¹-↑ˡ eq)) +... | inj₂ k = cong suc (trans (cong (internal l ↑ʳ_) (toFin-fromFin k)) (splitAt⁻¹-↑ʳ eq)) diff --git a/proofs/agda/MetaManifold/Composition/Sum.agda b/proofs/agda/MetaManifold/Composition/Sum.agda new file mode 100644 index 00000000..22687a39 --- /dev/null +++ b/proofs/agda/MetaManifold/Composition/Sum.agda @@ -0,0 +1,197 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +------------------------------------------------------------------------ +-- Weighted sums and the weighted inner product on tip vectors, over an +-- arbitrary commutative ring. +-- +-- With part weights p (philr's `p`; all ones for the standard Aitchison +-- geometry) the inner product of two log-scale vectors is +-- +-- ⟪ x , y ⟫ₚ = Σᵢ pᵢ · xᵢ · yᵢ (wdot p x y) +-- +-- and the weighted sum is Σᵢ pᵢ · xᵢ (wsum p x). A vector is centred +-- (lies in the image of philr's clrp) exactly when wsum p x ≈ 0. +-- +-- This module is shared vocabulary for every compositional transform in +-- the proof suite (ILR here, zero replacement in #21): nothing in it is +-- specific to balances. +------------------------------------------------------------------------ +{-# OPTIONS --safe --without-K #-} + +open import Algebra.Bundles using (CommutativeRing) + +module MetaManifold.Composition.Sum {c ℓ} (R : CommutativeRing c ℓ) where + +open CommutativeRing R +open import Algebra.Properties.Ring ring + using (-‿distribˡ-*; -‿distribʳ-*; -‿involutive; -‿+-comm; + +-inverseˡ-unique; x[y-z]≈xy-xz) +open import Data.Product.Base using (_×_; _,_) +open import Relation.Binary.Reasoning.Setoid setoid +open import Relation.Binary.PropositionalEquality.Core as ≡ using (_≡_) + +open import MetaManifold.Composition.Tree + +import Algebra.Solver.Ring.NaturalCoefficients.Default commutativeSemiring as Solver +open Solver using (solve; _:+_; _:*_; _:=_) + +private + variable t : Tree + +------------------------------------------------------------------------ +-- Ring facts used below (negation is outside the ℕ-coefficient solver, +-- so it is handled by hand and the solver sees negation-free goals). + +neg*neg : ∀ a b → (- a) * (- b) ≈ a * b +neg*neg a b = begin + (- a) * (- b) ≈⟨ -‿distribˡ-* a (- b) ⟨ + - (a * (- b)) ≈⟨ -‿cong (-‿distribʳ-* a b) ⟨ + - (- (a * b)) ≈⟨ -‿involutive (a * b) ⟩ + a * b ∎ + +interchange : ∀ a b c d → (a + b) + (c + d) ≈ (a + c) + (b + d) +interchange = solve 4 (λ a b c d → (a :+ b) :+ (c :+ d) := (a :+ c) :+ (b :+ d)) refl + +sum-zero : ∀ {a b} → a ≈ 0# → b ≈ 0# → a + b ≈ 0# +sum-zero a≈0 b≈0 = trans (+-cong a≈0 b≈0) (+-identityʳ 0#) + +-- a - b ≈ 0 ⇒ a ≈ b +difference-zero : ∀ {a b} → a - b ≈ 0# → a ≈ b +difference-zero {a} {b} eq = trans (+-inverseˡ-unique a (- b) eq) (-‿involutive b) + +------------------------------------------------------------------------ +-- Definitions + +-- Σᵢ xᵢ +total : TVec Carrier t → Carrier +total (tip x) = x +total (u ⊗ v) = total u + total v + +-- Σᵢ pᵢ xᵢ +wsum : TVec Carrier t → TVec Carrier t → Carrier +wsum (tip p) (tip x) = p * x +wsum (w ⊗ w′) (x ⊗ x′) = wsum w x + wsum w′ x′ + +-- Σᵢ pᵢ xᵢ yᵢ +wdot : TVec Carrier t → TVec Carrier t → TVec Carrier t → Carrier +wdot (tip p) (tip x) (tip y) = p * (x * y) +wdot (w ⊗ w′) (x ⊗ x′) (y ⊗ y′) = wdot w x y + wdot w′ x′ y′ + +-- Pointwise equality of tip vectors. +infix 4 _≋_ +_≋_ : TVec Carrier t → TVec Carrier t → Set ℓ +tip x ≋ tip y = x ≈ y +(u ⊗ v) ≋ (u′ ⊗ v′) = u ≋ u′ × v ≋ v′ + +≋-refl : (x : TVec Carrier t) → x ≋ x +≋-refl (tip x) = refl +≋-refl (u ⊗ v) = ≋-refl u , ≋-refl v + +≡⇒≋ : {x y : TVec Carrier t} → x ≡ y → x ≋ y +≡⇒≋ {x = x} ≡.refl = ≋-refl x + +-- Vector operations. +infixl 6 _+ᵥ_ _-ᵥ_ +_+ᵥ_ _-ᵥ_ : TVec Carrier t → TVec Carrier t → TVec Carrier t +_+ᵥ_ = zipWith _+_ +_-ᵥ_ = zipWith _-_ + +infixl 7 _·ᵥ_ +_·ᵥ_ : Carrier → TVec Carrier t → TVec Carrier t +k ·ᵥ x = map (k *_) x + +------------------------------------------------------------------------ +-- Constants + +wsum-pure : ∀ (w : TVec Carrier t) k → wsum w (pure k) ≈ total w * k +wsum-pure (tip p) k = refl +wsum-pure (w ⊗ w′) k = trans (+-cong (wsum-pure w k) (wsum-pure w′ k)) + (sym (distribʳ k (total w) (total w′))) + +wsum-zero : ∀ (w : TVec Carrier t) → wsum w (pure 0#) ≈ 0# +wsum-zero w = trans (wsum-pure w 0#) (zeroʳ (total w)) + +wdot-pureˡ : ∀ (w : TVec Carrier t) k y → wdot w (pure k) y ≈ k * wsum w y +wdot-pureˡ (tip p) k (tip y) = + solve 3 (λ p k y → p :* (k :* y) := k :* (p :* y)) refl p k y +wdot-pureˡ (w ⊗ w′) k (y ⊗ y′) = + trans (+-cong (wdot-pureˡ w k y) (wdot-pureˡ w′ k y′)) + (sym (distribˡ k (wsum w y) (wsum w′ y′))) + +wdot-comm : ∀ (w x y : TVec Carrier t) → wdot w x y ≈ wdot w y x +wdot-comm (tip p) (tip x) (tip y) = *-congˡ (*-comm x y) +wdot-comm (w ⊗ w′) (x ⊗ x′) (y ⊗ y′) = +-cong (wdot-comm w x y) (wdot-comm w′ x′ y′) + +wdot-pureʳ : ∀ (w x : TVec Carrier t) k → wdot w x (pure k) ≈ k * wsum w x +wdot-pureʳ w x k = trans (wdot-comm w x (pure k)) (wdot-pureˡ w k x) + +wdot-zeroˡ : ∀ (w y : TVec Carrier t) → wdot w (pure 0#) y ≈ 0# +wdot-zeroˡ w y = trans (wdot-pureˡ w 0# y) (zeroˡ (wsum w y)) + +wdot-zeroʳ : ∀ (w x : TVec Carrier t) → wdot w x (pure 0#) ≈ 0# +wdot-zeroʳ w x = trans (wdot-pureʳ w x 0#) (zeroˡ (wsum w x)) + +------------------------------------------------------------------------ +-- Linearity in the last argument (the data argument of a balance). + +wdot-+ʳ : ∀ (w x y z : TVec Carrier t) → + wdot w x (y +ᵥ z) ≈ wdot w x y + wdot w x z +wdot-+ʳ (tip p) (tip x) (tip y) (tip z) = + solve 4 (λ p x y z → p :* (x :* (y :+ z)) := p :* (x :* y) :+ p :* (x :* z)) + refl p x y z +wdot-+ʳ (w ⊗ w′) (x ⊗ x′) (y ⊗ y′) (z ⊗ z′) = + trans (+-cong (wdot-+ʳ w x y z) (wdot-+ʳ w′ x′ y′ z′)) + (interchange _ _ _ _) + +wdot-·ʳ : ∀ (w x : TVec Carrier t) k y → wdot w x (k ·ᵥ y) ≈ k * wdot w x y +wdot-·ʳ (tip p) (tip x) k (tip y) = + solve 4 (λ p x k y → p :* (x :* (k :* y)) := k :* (p :* (x :* y))) refl p x k y +wdot-·ʳ (w ⊗ w′) (x ⊗ x′) k (y ⊗ y′) = + trans (+-cong (wdot-·ʳ w x k y) (wdot-·ʳ w′ x′ k y′)) + (sym (distribˡ k _ _)) + +-- (a - b) + (c - d) ≈ (a + c) - (b + d) +private + sub-interchange : ∀ a b c d → (a - b) + (c - d) ≈ (a + c) - (b + d) + sub-interchange a b c d = begin + (a + - b) + (c + - d) ≈⟨ interchange a (- b) c (- d) ⟩ + (a + c) + (- b + - d) ≈⟨ +-congˡ (-‿+-comm b d) ⟩ + (a + c) + - (b + d) ∎ + +wdot--ʳ : ∀ (w x y z : TVec Carrier t) → + wdot w x (y -ᵥ z) ≈ wdot w x y - wdot w x z +wdot--ʳ (tip p) (tip x) (tip y) (tip z) = begin + p * (x * (y - z)) ≈⟨ *-congˡ (x[y-z]≈xy-xz x y z) ⟩ + p * (x * y - x * z) ≈⟨ x[y-z]≈xy-xz p (x * y) (x * z) ⟩ + p * (x * y) - p * (x * z) ∎ +wdot--ʳ (w ⊗ w′) (x ⊗ x′) (y ⊗ y′) (z ⊗ z′) = + trans (+-cong (wdot--ʳ w x y z) (wdot--ʳ w′ x′ y′ z′)) + (sub-interchange _ _ _ _) + +wsum--ʳ : ∀ (w y z : TVec Carrier t) → wsum w (y -ᵥ z) ≈ wsum w y - wsum w z +wsum--ʳ (tip p) (tip y) (tip z) = x[y-z]≈xy-xz p y z +wsum--ʳ (w ⊗ w′) (y ⊗ y′) (z ⊗ z′) = + trans (+-cong (wsum--ʳ w y z) (wsum--ʳ w′ y′ z′)) (sub-interchange _ _ _ _) + +------------------------------------------------------------------------ +-- All entries of x - y are zero ⇒ x ≋ y + +AllZero : TVec Carrier t → Set ℓ +AllZero (tip x) = x ≈ 0# +AllZero (u ⊗ v) = AllZero u × AllZero v + +allZero-difference : ∀ (x y : TVec Carrier t) → AllZero (x -ᵥ y) → x ≋ y +allZero-difference (tip x) (tip y) z = difference-zero z +allZero-difference (u ⊗ v) (u′ ⊗ v′) (zu , zv) = + allZero-difference u u′ zu , allZero-difference v v′ zv + +------------------------------------------------------------------------ +-- Congruence + +wdot-congʳ : ∀ (w x : TVec Carrier t) {y z} → y ≋ z → wdot w x y ≈ wdot w x z +wdot-congʳ (tip p) (tip x) {tip y} {tip z} y≈z = *-congˡ (*-congˡ y≈z) +wdot-congʳ (w ⊗ w′) (x ⊗ x′) {y ⊗ y′} {z ⊗ z′} (e , e′) = + +-cong (wdot-congʳ w x e) (wdot-congʳ w′ x′ e′) + +wdot-·ˡ : ∀ (w : TVec Carrier t) k x y → wdot w (k ·ᵥ x) y ≈ k * wdot w x y +wdot-·ˡ w k x y = trans (wdot-comm w (k ·ᵥ x) y) + (trans (wdot-·ʳ w y k x) (*-congˡ (wdot-comm w y x))) diff --git a/proofs/agda/MetaManifold/Composition/Tree.agda b/proofs/agda/MetaManifold/Composition/Tree.agda new file mode 100644 index 00000000..51fb155a --- /dev/null +++ b/proofs/agda/MetaManifold/Composition/Tree.agda @@ -0,0 +1,113 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +------------------------------------------------------------------------ +-- Rooted binary trees and tip-indexed vectors. +-- +-- A `Tree` is the shape of a phylogeny, a sequential binary partition +-- (SBP) or a balance dendrogram once validation has succeeded: rooted +-- and strictly bifurcating. The Julia side (src/analysis/ilr_basis.jl) +-- refuses multifurcating and unrooted input instead of resolving it, so +-- every tree the analysis uses is one of these. +-- +-- `TVec A t` is a vector with one entry per tip of `t`, structured like +-- the tree. It is the proof-side form of "one value per taxon"; the +-- flat `Vec` form is recovered by `flatten`, in left-to-right tip order. +------------------------------------------------------------------------ +{-# OPTIONS --safe --without-K #-} + +module MetaManifold.Composition.Tree where + +open import Level using (Level) +open import Data.Nat.Base using (ℕ; zero; suc; _+_) +open import Data.Nat.Properties using (+-suc; +-comm) +open import Data.Vec.Base using (Vec; []; _∷_; _++_) +open import Relation.Binary.PropositionalEquality + using (_≡_; refl; sym; trans; cong; cong₂) + +private + variable + a b c : Level + A B C : Set a + +data Tree : Set where + leaf : Tree + node : Tree → Tree → Tree + +-- Number of tips (taxa, parts of the composition): D. +leaves : Tree → ℕ +leaves leaf = 1 +leaves (node l r) = leaves l + leaves r + +-- Number of internal nodes: one balance each. +internal : Tree → ℕ +internal leaf = 0 +internal (node l r) = suc (internal l + internal r) + +-- D parts give exactly D - 1 balances. +internal-count : ∀ t → suc (internal t) ≡ leaves t +internal-count leaf = refl +internal-count (node l r) = + trans (cong suc (sym (+-suc (internal l) (internal r)))) + (cong₂ _+_ (internal-count l) (internal-count r)) + +------------------------------------------------------------------------ +-- Tip-indexed vectors + +infixr 5 _⊗_ + +data TVec (A : Set a) : Tree → Set a where + tip : A → TVec A leaf + _⊗_ : ∀ {l r} → TVec A l → TVec A r → TVec A (node l r) + +pure : ∀ {t} → A → TVec A t +pure {t = leaf} x = tip x +pure {t = node l r} x = pure x ⊗ pure x + +map : ∀ {t} → (A → B) → TVec A t → TVec B t +map f (tip x) = tip (f x) +map f (u ⊗ v) = map f u ⊗ map f v + +zipWith : ∀ {t} → (A → B → C) → TVec A t → TVec B t → TVec C t +zipWith f (tip x) (tip y) = tip (f x y) +zipWith f (u ⊗ v) (u′ ⊗ v′) = zipWith f u u′ ⊗ zipWith f v v′ + +flatten : ∀ {t} → TVec A t → Vec A (leaves t) +flatten (tip x) = x ∷ [] +flatten (u ⊗ v) = flatten u ++ flatten v + +map-pure : ∀ {t} (f : A → B) (x : A) → map {t = t} f (pure x) ≡ pure (f x) +map-pure {t = leaf} f x = refl +map-pure {t = node l r} f x = cong₂ _⊗_ (map-pure f x) (map-pure f x) + +------------------------------------------------------------------------ +-- Predicates on tip vectors + +-- Some tip satisfies P. +data Any {p} {A : Set a} (P : A → Set p) : ∀ {t} → TVec A t → Set (a Level.⊔ p) where + at : ∀ {x} → P x → Any P (tip x) + left : ∀ {l r} {u : TVec A l} {v : TVec A r} → Any P u → Any P (u ⊗ v) + right : ∀ {l r} {u : TVec A l} {v : TVec A r} → Any P v → Any P (u ⊗ v) + +-- Every tip satisfies P. +data All {p} {A : Set a} (P : A → Set p) : ∀ {t} → TVec A t → Set (a Level.⊔ p) where + at : ∀ {x} → P x → All P (tip x) + both : ∀ {l r} {u : TVec A l} {v : TVec A r} → All P u → All P v → All P (u ⊗ v) + +any-pure : ∀ {p} {P : A → Set p} {t} {x} → P x → Any P (pure {t = t} x) +any-pure {t = leaf} px = at px +any-pure {t = node l r} px = left (any-pure px) + +all-pure : ∀ {p} {P : A → Set p} {t} {x} → P x → All P (pure {t = t} x) +all-pure {t = leaf} px = at px +all-pure {t = node l r} px = both (all-pure px) (all-pure px) + +------------------------------------------------------------------------ +-- The comb (caterpillar) tree: ((((1,2),3),4),...). It is the tree whose +-- balances are the Helmert contrasts that MetaManifold uses by default. + +comb : ℕ → Tree +comb zero = leaf +comb (suc n) = node (comb n) leaf + +comb-leaves : ∀ n → leaves (comb n) ≡ suc n +comb-leaves zero = refl +comb-leaves (suc n) = trans (cong (_+ 1) (comb-leaves n)) (+-comm (suc n) 1) diff --git a/proofs/agda/MetaManifold/ILR/Comb.agda b/proofs/agda/MetaManifold/ILR/Comb.agda new file mode 100644 index 00000000..a6da70bb --- /dev/null +++ b/proofs/agda/MetaManifold/ILR/Comb.agda @@ -0,0 +1,116 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +------------------------------------------------------------------------ +-- The comb tree reproduces MetaManifold's default (Helmert) ILR basis. +-- +-- src/analysis/Execution.jl's default basis is, for i = 1 … D-1, +-- +-- ilr_i = sqrt(i/(i+1)) · ( mean(log x₁ … log xᵢ) - log xᵢ₊₁ ) +-- +-- whose unnormalised contrast is (1, …, 1, -i, 0, …, 0) with i ones. +-- `helmert` below is that row, defined without reference to trees. The +-- theorem says the balances of the comb tree ((…((1,2),3)…),D) under +-- uniform weights are exactly these rows, and its masses are r = i, +-- s = 1, so the normalising constant 1/sqrt(r·s·(r+s)) turns +-- i·(mean - log xᵢ₊₁) into sqrt(i/(i+1))·(mean - log xᵢ₊₁). +-- +-- Order: the tree's preorder visits the root first, and the comb's root +-- is Helmert's *last* balance (all D-1 first parts against part D). +-- `index` numbers the comb's balances in preorder; row `k` of the +-- preorder is Helmert balance i = D-1-k. The Julia equivalence test maps +-- indices explicitly rather than assuming the two orders agree. +------------------------------------------------------------------------ +{-# OPTIONS --safe --without-K #-} + +open import Algebra.Bundles using (CommutativeRing) + +module MetaManifold.ILR.Comb {c ℓ} (R : CommutativeRing c ℓ) where + +open CommutativeRing R hiding (zero) +open import Data.Fin.Base using (Fin; zero; suc; toℕ) +open import Data.Fin.Properties using (toℕ-↑ˡ) +open import Data.Nat.Base using (ℕ; zero; suc) renaming (_+_ to _+ℕ_) +import Data.Nat.Properties as ℕ +open import Data.Product.Base using (_×_; _,_) +open import Data.Vec.Base using (Vec; []; _∷_; _++_; _∷ʳ_; replicate) +open import Data.Vec.Relation.Binary.Pointwise.Inductive using (Pointwise; []; _∷_) +open import Relation.Binary.PropositionalEquality.Core as ≡ using (_≡_; cong; subst) + +open import MetaManifold.Composition.Tree +open import MetaManifold.Composition.Node +open import MetaManifold.Composition.Sum R +open import MetaManifold.ILR.Contrast R + +-- n as a ring element: 1 + 1 + … + 1. +count : ℕ → Carrier +count zero = 0# +count (suc n) = 1# + count n + +-- Helmert rows, in the comb tree's preorder: row zero of `helmert n` +-- is Julia's balance_{n} (first n parts against part n+1). +helmert : ∀ n → Fin n → Vec Carrier (suc n) +helmert (suc n) zero = replicate (suc n) 1# ∷ʳ - count (suc n) +helmert (suc n) (suc k) = helmert n k ∷ʳ 0# + +uniform : ∀ {t} → TVec Carrier t +uniform = pure 1# + +-- Preorder number of a comb balance, as a Fin n. +index : ∀ {n} → Node (comb n) → Fin n +index {suc n} here = zero +index {suc n} (inl i) = suc (index i) + +-- `index` is the preorder numbering `toFin` of Node.agda. +index-is-preorder : ∀ {n} (i : Node (comb n)) → toℕ (toFin i) ≡ toℕ (index i) +index-is-preorder {suc n} here = ≡.refl +index-is-preorder {suc n} (inl i) = + cong suc (≡.trans (toℕ-↑ˡ (toFin i) 0) (index-is-preorder i)) + +total-uniform-comb : ∀ n → total (uniform {comb n}) ≈ count (suc n) +total-uniform-comb zero = sym (+-identityʳ 1#) +total-uniform-comb (suc n) = trans (+-congʳ (total-uniform-comb n)) (+-comm _ 1#) + +-- Julia's balance number i for a comb balance: the size of its +-- numerator clade. +julia-index : ∀ {n} → Node (comb n) → ℕ +julia-index {suc n} here = suc n +julia-index {suc n} (inl i) = julia-index i + +-- i = D - 1 - k, i.e. i + k = n (= D - 1) with k the preorder number. +julia-index-reverses : ∀ {n} (i : Node (comb n)) → julia-index i +ℕ toℕ (index i) ≡ n +julia-index-reverses {suc n} here = ℕ.+-identityʳ (suc n) +julia-index-reverses {suc n} (inl i) = + ≡.trans (ℕ.+-suc (julia-index i) (toℕ (index i))) (cong suc (julia-index-reverses i)) + +-- Masses: r = i, s = 1 (so 1/sqrt(r·s·(r+s)) = 1/sqrt(i(i+1))). +comb-masses : ∀ n (i : Node (comb n)) → + plus-mass uniform i ≈ count (julia-index i) × minus-mass uniform i ≈ 1# +comb-masses (suc n) here = total-uniform-comb n , refl +comb-masses (suc n) (inl i) = comb-masses n i + +------------------------------------------------------------------------ +-- The theorem + +private + -- flatten appends with _++_, Helmert rows with _∷ʳ_. + snoc⁺ : ∀ {m n} {xs : Vec Carrier m} {ys : Vec Carrier n} {x y} → + Pointwise _≈_ xs ys → x ≈ y → Pointwise _≈_ (xs ++ x ∷ []) (ys ∷ʳ y) + snoc⁺ [] x≈y = x≈y ∷ [] + snoc⁺ (e ∷ es) x≈y = e ∷ snoc⁺ es x≈y + + replicate-snoc : ∀ n (k : Carrier) → replicate (suc n) k ≡ replicate n k ∷ʳ k + replicate-snoc zero k = ≡.refl + replicate-snoc (suc n) k = cong (k ∷_) (replicate-snoc n k) + + flatten-pure-comb : ∀ n k → Pointwise _≈_ (flatten (pure {t = comb n} k)) (replicate (suc n) k) + flatten-pure-comb zero k = refl ∷ [] + flatten-pure-comb (suc n) k = + subst (Pointwise _≈_ (flatten (pure {t = comb n} k) ++ k ∷ [])) + (≡.sym (replicate-snoc (suc n) k)) + (snoc⁺ (flatten-pure-comb n k) refl) + +comb-is-helmert : ∀ n (i : Node (comb n)) → + Pointwise _≈_ (flatten (contrast uniform i)) (helmert n (index i)) +comb-is-helmert (suc n) here = + snoc⁺ (flatten-pure-comb n 1#) (-‿cong (total-uniform-comb n)) +comb-is-helmert (suc n) (inl i) = + snoc⁺ (comb-is-helmert n i) refl diff --git a/proofs/agda/MetaManifold/ILR/Contrast.agda b/proofs/agda/MetaManifold/ILR/Contrast.agda new file mode 100644 index 00000000..edf4fa01 --- /dev/null +++ b/proofs/agda/MetaManifold/ILR/Contrast.agda @@ -0,0 +1,236 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +------------------------------------------------------------------------ +-- Balance contrasts of a tree under part weights, over any commutative +-- ring. +-- +-- For internal node n with numerator parts N₊ and denominator parts N₋ +-- and part weights p, write r = Σ_{N₊} p and s = Σ_{N₋} p. The contrast +-- +-- contrast p n = s on N₊, -r on N₋, 0 elsewhere +-- +-- is philr's `buildilrBasep` column (entries +c/r and -c/s with +-- c = sqrt(rs/(r+s))) multiplied by the non-zero scalar rs/c. Every +-- property proved here is invariant under that rescaling, so it holds +-- for the normalised basis the Julia code uses; the scalar itself is +-- pinned by `contrast-norm` (norm² = r·s·(r+s), hence the normalised +-- column has norm 1 exactly when multiplied by 1/sqrt(r·s·(r+s))). +-- +-- Uniform weights (p = 1) give the classical Egozcue et al. (2003) +-- balance with coefficients ±sqrt(rs/(r+s))·(1/r or 1/s). +------------------------------------------------------------------------ +{-# OPTIONS --safe --without-K #-} + +open import Algebra.Bundles using (CommutativeRing) + +module MetaManifold.ILR.Contrast {c ℓ} (R : CommutativeRing c ℓ) where + +open CommutativeRing R +open import Algebra.Properties.Ring ring using (-‿distribʳ-*) +open import Data.Empty using (⊥-elim) +open import Data.Product.Base using (_,_) +open import Relation.Binary.PropositionalEquality.Core as ≡ + using (_≡_; _≢_; cong) +open import Relation.Binary.Reasoning.Setoid setoid + +open import MetaManifold.Composition.Tree +open import MetaManifold.Composition.Node +open import MetaManifold.Composition.Sum R +open import MetaManifold.ILR.SBP + +import Algebra.Solver.Ring.NaturalCoefficients.Default commutativeSemiring as Solver +open Solver using (solve; _:+_; _:*_; _:=_) + +private + variable t : Tree + +------------------------------------------------------------------------ +-- Masses and contrasts + +-- r: total weight of the numerator of balance n. +plus-mass : TVec Carrier t → Node t → Carrier +plus-mass (wl ⊗ wr) here = total wl +plus-mass (wl ⊗ wr) (inl n) = plus-mass wl n +plus-mass (wl ⊗ wr) (inr n) = plus-mass wr n + +-- s: total weight of the denominator of balance n. +minus-mass : TVec Carrier t → Node t → Carrier +minus-mass (wl ⊗ wr) here = total wr +minus-mass (wl ⊗ wr) (inl n) = minus-mass wl n +minus-mass (wl ⊗ wr) (inr n) = minus-mass wr n + +contrast : TVec Carrier t → Node t → TVec Carrier t +contrast (wl ⊗ wr) here = pure (total wr) ⊗ pure (- total wl) +contrast (wl ⊗ wr) (inl n) = contrast wl n ⊗ pure 0# +contrast (wl ⊗ wr) (inr n) = pure 0# ⊗ contrast wr n + +-- The (unnormalised) balance of a log-scale vector x at node n. +balance : TVec Carrier t → Node t → TVec Carrier t → Carrier +balance w n x = wdot w (contrast w n) x + +------------------------------------------------------------------------ +-- The contrast is determined by the SBP row and the two masses: this is +-- what lets the Julia code compute a balance from a sign matrix alone. + +interpret : Carrier → Carrier → Sign → Carrier +interpret r s plus = s +interpret r s minus = - r +interpret r s off = 0# + +private + -- `w` only fixes the tree index, which `_≋_` cannot infer. + pure≋map : ∀ {t} (w : TVec Carrier t) (f : Sign → Carrier) σ → + pure {t = t} (f σ) ≋ map f (pure σ) + pure≋map {t} w f σ = ≡⇒≋ {t = t} (≡.sym (map-pure f σ)) + +contrast-from-code : ∀ (w : TVec Carrier t) n → + contrast w n ≋ map (interpret (plus-mass w n) (minus-mass w n)) (code n) +contrast-from-code (wl ⊗ wr) here = + pure≋map wl (interpret (total wl) (total wr)) plus , + pure≋map wr (interpret (total wl) (total wr)) minus +contrast-from-code (wl ⊗ wr) (inl n) = + contrast-from-code wl n , + pure≋map wr (interpret (plus-mass wl n) (minus-mass wl n)) off +contrast-from-code (wl ⊗ wr) (inr n) = + pure≋map wl (interpret (plus-mass wr n) (minus-mass wr n)) off , + contrast-from-code wr n + +-- Masses from the SBP row: r = Σ p over the +1 entries, s over the -1. +indicator : Sign → Sign → Carrier +indicator plus plus = 1# +indicator minus minus = 1# +indicator _ _ = 0# + +private + wsum-one : ∀ (w : TVec Carrier t) → wsum w (pure 1#) ≈ total w + wsum-one w = trans (wsum-pure w 1#) (*-identityʳ (total w)) + + wsum-map-pure : ∀ (w : TVec Carrier t) (f : Sign → Carrier) σ → + wsum w (map f (pure σ)) ≈ total w * f σ + wsum-map-pure w f σ = trans (≡⇒≈ (cong (wsum w) (map-pure f σ))) (wsum-pure w (f σ)) + where + ≡⇒≈ : ∀ {a b} → a ≡ b → a ≈ b + ≡⇒≈ ≡.refl = refl + + plus-side : ∀ a b → a * 1# + b * 0# ≈ a + plus-side a b = trans (+-cong (*-identityʳ a) (zeroʳ b)) (+-identityʳ a) + + minus-side : ∀ a b → a * 0# + b * 1# ≈ b + minus-side a b = trans (+-cong (zeroʳ a) (*-identityʳ b)) (+-identityˡ b) + + plus-zero : ∀ a b → a + b * 0# ≈ a + plus-zero a b = trans (+-congˡ (zeroʳ b)) (+-identityʳ a) + + zero-plus : ∀ a b → a * 0# + b ≈ b + zero-plus a b = trans (+-congʳ (zeroʳ a)) (+-identityˡ b) + +plus-mass-from-code : ∀ (w : TVec Carrier t) n → + plus-mass w n ≈ wsum w (map (indicator plus) (code n)) +plus-mass-from-code (wl ⊗ wr) here = sym (trans + (+-cong (wsum-map-pure wl (indicator plus) plus) (wsum-map-pure wr (indicator plus) minus)) + (plus-side (total wl) (total wr))) +plus-mass-from-code (wl ⊗ wr) (inl n) = sym (trans + (+-congˡ (wsum-map-pure wr (indicator plus) off)) + (trans (plus-zero _ (total wr)) (sym (plus-mass-from-code wl n)))) +plus-mass-from-code (wl ⊗ wr) (inr n) = sym (trans + (+-congʳ (wsum-map-pure wl (indicator plus) off)) + (trans (zero-plus (total wl) _) (sym (plus-mass-from-code wr n)))) + +minus-mass-from-code : ∀ (w : TVec Carrier t) n → + minus-mass w n ≈ wsum w (map (indicator minus) (code n)) +minus-mass-from-code (wl ⊗ wr) here = sym (trans + (+-cong (wsum-map-pure wl (indicator minus) plus) (wsum-map-pure wr (indicator minus) minus)) + (minus-side (total wl) (total wr))) +minus-mass-from-code (wl ⊗ wr) (inl n) = sym (trans + (+-congˡ (wsum-map-pure wr (indicator minus) off)) + (trans (plus-zero _ (total wr)) (sym (minus-mass-from-code wl n)))) +minus-mass-from-code (wl ⊗ wr) (inr n) = sym (trans + (+-congʳ (wsum-map-pure wl (indicator minus) off)) + (trans (zero-plus (total wl) _) (sym (minus-mass-from-code wr n)))) + +------------------------------------------------------------------------ +-- Each contrast is centred: Σᵢ pᵢ cᵢ = 0. Consequently a balance does not +-- depend on which centring (clr, clrp, none) is applied to log x first. + +private + rs-cancel : ∀ r s → r * s + s * (- r) ≈ 0# + rs-cancel r s = begin + r * s + s * (- r) ≈⟨ +-congˡ (-‿distribʳ-* s r) ⟨ + r * s + - (s * r) ≈⟨ +-congˡ (-‿cong (*-comm s r)) ⟩ + r * s + - (r * s) ≈⟨ -‿inverseʳ (r * s) ⟩ + 0# ∎ + +contrast-sum-zero : ∀ (w : TVec Carrier t) n → wsum w (contrast w n) ≈ 0# +contrast-sum-zero (wl ⊗ wr) here = + trans (+-cong (wsum-pure wl (total wr)) (wsum-pure wr (- total wl))) + (rs-cancel (total wl) (total wr)) +contrast-sum-zero (wl ⊗ wr) (inl n) = sum-zero (contrast-sum-zero wl n) (wsum-zero wr) +contrast-sum-zero (wl ⊗ wr) (inr n) = sum-zero (wsum-zero wl) (contrast-sum-zero wr n) + +------------------------------------------------------------------------ +-- Distinct contrasts are orthogonal in the weighted inner product. + +private + -- The root contrast against any contrast strictly inside the left or + -- right subtree. + root-inl : ∀ {l r} (wl : TVec Carrier l) (wr : TVec Carrier r) (m : Node l) → + wdot (wl ⊗ wr) (contrast (wl ⊗ wr) here) (contrast (wl ⊗ wr) (inl m)) ≈ 0# + root-inl wl wr m = sum-zero + (trans (wdot-pureˡ wl (total wr) (contrast wl m)) + (trans (*-congˡ (contrast-sum-zero wl m)) (zeroʳ (total wr)))) + (wdot-zeroʳ wr (pure (- total wl))) + + root-inr : ∀ {l r} (wl : TVec Carrier l) (wr : TVec Carrier r) (m : Node r) → + wdot (wl ⊗ wr) (contrast (wl ⊗ wr) here) (contrast (wl ⊗ wr) (inr m)) ≈ 0# + root-inr wl wr m = sum-zero + (wdot-zeroʳ wl (pure (total wr))) + (trans (wdot-pureˡ wr (- total wl) (contrast wr m)) + (trans (*-congˡ (contrast-sum-zero wr m)) (zeroʳ (- total wl)))) + + flip : ∀ (w x y : TVec Carrier t) → wdot w x y ≈ 0# → wdot w y x ≈ 0# + flip w x y eq = trans (wdot-comm w y x) eq + +contrast-orthogonal : ∀ (w : TVec Carrier t) {n m : Node t} → n ≢ m → + wdot w (contrast w n) (contrast w m) ≈ 0# +contrast-orthogonal (wl ⊗ wr) {here} {here} n≢m = ⊥-elim (n≢m ≡.refl) +contrast-orthogonal (wl ⊗ wr) {here} {inl m} _ = root-inl wl wr m +contrast-orthogonal (wl ⊗ wr) {here} {inr m} _ = root-inr wl wr m +contrast-orthogonal (wl ⊗ wr) {inl n} {here} _ = + flip (wl ⊗ wr) (contrast (wl ⊗ wr) here) (contrast (wl ⊗ wr) (inl n)) (root-inl wl wr n) +contrast-orthogonal (wl ⊗ wr) {inr n} {here} _ = + flip (wl ⊗ wr) (contrast (wl ⊗ wr) here) (contrast (wl ⊗ wr) (inr n)) (root-inr wl wr n) +contrast-orthogonal (wl ⊗ wr) {inl n} {inl m} n≢m = + sum-zero (contrast-orthogonal wl (λ eq → n≢m (cong inl eq))) (wdot-zeroʳ wr (pure 0#)) +contrast-orthogonal (wl ⊗ wr) {inl n} {inr m} _ = + sum-zero (wdot-zeroʳ wl (contrast wl n)) (wdot-zeroˡ wr (contrast wr m)) +contrast-orthogonal (wl ⊗ wr) {inr n} {inl m} _ = + sum-zero (wdot-zeroˡ wl (contrast wl m)) (wdot-zeroʳ wr (contrast wr n)) +contrast-orthogonal (wl ⊗ wr) {inr n} {inr m} n≢m = + sum-zero (wdot-zeroʳ wl (pure 0#)) (contrast-orthogonal wr (λ eq → n≢m (cong inr eq))) + +------------------------------------------------------------------------ +-- Norm: ⟪ c , c ⟫ₚ = r·s·(r+s). The normalised philr/Egozcue column is +-- c / sqrt(r·s·(r+s)); for uniform weights this is the textbook constant +-- sqrt(rs/(r+s)) on the ±1/r, ±1/s scale. + +contrast-norm : ∀ (w : TVec Carrier t) n → + wdot w (contrast w n) (contrast w n) + ≈ (plus-mass w n * minus-mass w n) * (plus-mass w n + minus-mass w n) +contrast-norm (wl ⊗ wr) here = begin + wdot wl (pure s) (pure s) + wdot wr (pure (- r)) (pure (- r)) + ≈⟨ +-cong (wdot-pureˡ wl s (pure s)) (wdot-pureˡ wr (- r) (pure (- r))) ⟩ + s * wsum wl (pure s) + (- r) * wsum wr (pure (- r)) + ≈⟨ +-cong (*-congˡ (wsum-pure wl s)) (*-congˡ (wsum-pure wr (- r))) ⟩ + s * (r * s) + (- r) * (s * (- r)) + ≈⟨ +-congˡ (*-congˡ (-‿distribʳ-* s r)) ⟨ + s * (r * s) + (- r) * (- (s * r)) + ≈⟨ +-congˡ (neg*neg r (s * r)) ⟩ + s * (r * s) + r * (s * r) + ≈⟨ solve 2 (λ r s → s :* (r :* s) :+ r :* (s :* r) := (r :* s) :* (r :+ s)) refl r s ⟩ + (r * s) * (r + s) ∎ + where r = total wl; s = total wr +contrast-norm (wl ⊗ wr) (inl n) = + trans (+-congˡ (wdot-zeroʳ wr (pure 0#))) + (trans (+-identityʳ _) (contrast-norm wl n)) +contrast-norm (wl ⊗ wr) (inr n) = + trans (+-congʳ (wdot-zeroʳ wl (pure 0#))) + (trans (+-identityˡ _) (contrast-norm wr n)) diff --git a/proofs/agda/MetaManifold/ILR/Integer.agda b/proofs/agda/MetaManifold/ILR/Integer.agda new file mode 100644 index 00000000..5eb1ddf7 --- /dev/null +++ b/proofs/agda/MetaManifold/ILR/Integer.agda @@ -0,0 +1,122 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +------------------------------------------------------------------------ +-- The integer instance: the generic results specialised to ℤ, the +-- side-condition of the kernel theorem discharged for positive weights, +-- a counterexample proving that side-condition is needed, and philr's +-- own known-answer SBP reproduced by `code`. +------------------------------------------------------------------------ +{-# OPTIONS --safe --without-K #-} + +module MetaManifold.ILR.Integer where + +open import Data.Integer.Base + using (ℤ; +_; -[1+_]; 0ℤ; 1ℤ; -1ℤ; _*_; _<_; +<+; >-nonZero) +open import Data.Integer.Properties + using (+-*-commutativeRing; *-cancelˡ-≡; *-zeroʳ; +-mono-<) +open import Data.Nat.Base using (s≤s; z≤n) +open import Data.Product.Base using (_×_; _,_) +open import Data.Vec.Base using (_∷_; []) +open import Relation.Binary.PropositionalEquality using (_≡_; refl; sym; trans) +open import Relation.Nullary.Negation using (¬_) + +open import MetaManifold.Composition.Tree +open import MetaManifold.Composition.Node +open import MetaManifold.Composition.Sum +-*-commutativeRing +open import MetaManifold.ILR.SBP +open import MetaManifold.ILR.Contrast +-*-commutativeRing +open import MetaManifold.ILR.Kernel +-*-commutativeRing + +------------------------------------------------------------------------ +-- Positive weights are cancellable. + +Positive : ∀ {t} → TVec ℤ t → Set +Positive = All (0ℤ <_) + +private + total-positive : ∀ {t} {w : TVec ℤ t} → Positive w → 0ℤ < total w + total-positive (at p) = p + total-positive (both l r) = +-mono-< (total-positive l) (total-positive r) + + cancel : ∀ {k} → 0ℤ < k → ∀ {a} → k * a ≡ 0ℤ → a ≡ 0ℤ + cancel {k} k>0 {a} eq = *-cancelˡ-≡ k a 0ℤ {{>-nonZero k>0}} (trans eq (sym (*-zeroʳ k))) + +positive-cancellable : ∀ {t} {w : TVec ℤ t} → Positive w → Cancellable w +positive-cancellable (at p) = tipC (cancel p) +positive-cancellable (both l r) = + nodeC (cancel (+-mono-< (total-positive l) (total-positive r))) + (positive-cancellable l) (positive-cancellable r) + +-- Uniform weights (the standard Aitchison geometry) in particular. +uniform-cancellable : ∀ t → Cancellable (pure {t = t} 1ℤ) +uniform-cancellable t = positive-cancellable (all-pure {t = t} (+<+ (s≤s z≤n))) + +-- Hence, over ℤ with positive weights, balances are injective on +-- centred vectors. +balance-injective-positive : ∀ {t} {w : TVec ℤ t} → Positive w → ∀ x y → + wsum w x ≡ 0ℤ → wsum w y ≡ 0ℤ → (∀ n → balance w n x ≡ balance w n y) → x ≋ y +balance-injective-positive pw = balance-injective (positive-cancellable pw) + +------------------------------------------------------------------------ +-- Negative control: without the hypothesis the kernel theorem is false. +-- +-- Weights (1, -1) have clade total 0. The vector x = (1, 1) is centred +-- (1·1 + (-1)·1 = 0) and its only balance is 1·(-1·1) + (-1)·(-1·1) = 0, +-- yet x ≠ 0. This is why the Julia code refuses non-positive part +-- weights instead of computing with them. + +private + w₀ x₀ : TVec ℤ (node leaf leaf) + w₀ = tip 1ℤ ⊗ tip -1ℤ + x₀ = tip 1ℤ ⊗ tip 1ℤ + + x₀-centred : wsum w₀ x₀ ≡ 0ℤ + x₀-centred = refl + + x₀-balances : ∀ n → balance w₀ n x₀ ≡ 0ℤ + x₀-balances here = refl + + x₀-nonzero : ¬ AllZero x₀ + x₀-nonzero (() , _) + +kernel-needs-hypothesis : + ¬ (∀ {t} (w x : TVec ℤ t) → wsum w x ≡ 0ℤ → (∀ n → balance w n x ≡ 0ℤ) → AllZero x) +kernel-needs-hypothesis claim = x₀-nonzero (claim w₀ x₀ x₀-centred x₀-balances) + +signed-weights-not-cancellable : ¬ Cancellable w₀ +signed-weights-not-cancellable c = x₀-nonzero (balance-kernel c x₀ x₀-centred x₀-balances) + +------------------------------------------------------------------------ +-- Known answer shared with R philr's test suite (tests/testthat): the +-- tree (otu1,(otu2,otu3)) has SBP columns (1,-1,-1) and (0,1,-1), and +-- with part weights p = (1, 1, 0.5) philr's basis is +-- [[ 0.7745967, 0], [-0.5163978, 0.5773503], [-0.5163978, -1.1547005]]. +-- Over ℤ we use p = (2, 2, 1) (= 2 × philr's p): the unnormalised +-- contrasts below are proportional to philr's columns (3 : -2 : -2 and +-- 1 : -2), which is all that the scale of p can change. + +t₃ : Tree +t₃ = node leaf (node leaf leaf) + +sbp-column-1 : flatten (code (here {leaf} {node leaf leaf})) ≡ plus ∷ minus ∷ minus ∷ [] +sbp-column-1 = refl + +sbp-column-2 : flatten (code (inr {leaf} (here {leaf} {leaf}))) ≡ off ∷ plus ∷ minus ∷ [] +sbp-column-2 = refl + +p₃ : TVec ℤ t₃ +p₃ = tip (+ 2) ⊗ (tip (+ 2) ⊗ tip (+ 1)) + +contrast-column-1 : flatten (contrast p₃ here) ≡ + 3 ∷ -[1+ 1 ] ∷ -[1+ 1 ] ∷ [] +contrast-column-1 = refl + +contrast-column-2 : flatten (contrast p₃ (inr here)) ≡ 0ℤ ∷ + 1 ∷ -[1+ 1 ] ∷ [] +contrast-column-2 = refl + +-- The two columns are orthogonal under p (2·3·0 + 2·(-2)·1 + 1·(-2)·(-2) = 0) +-- and have squared norms r·s·(r+s) = 2·3·5 = 30 and 2·1·3 = 6. +columns-orthogonal : wdot p₃ (contrast p₃ here) (contrast p₃ (inr here)) ≡ 0ℤ +columns-orthogonal = refl + +column-norms : wdot p₃ (contrast p₃ here) (contrast p₃ here) ≡ + 30 + × wdot p₃ (contrast p₃ (inr here)) (contrast p₃ (inr here)) ≡ + 6 +column-norms = refl , refl diff --git a/proofs/agda/MetaManifold/ILR/Invariance.agda b/proofs/agda/MetaManifold/ILR/Invariance.agda new file mode 100644 index 00000000..ea6229d9 --- /dev/null +++ b/proofs/agda/MetaManifold/ILR/Invariance.agda @@ -0,0 +1,105 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +------------------------------------------------------------------------ +-- Invariance of balances. +-- +-- Exact part (any commutative ring): a balance is linear in the +-- log-scale data and blind to adding a constant to every part. +-- +-- Transcendental seam: `LogHom` is a *named assumption*, not a +-- postulate — a map from a commutative monoid (think: positive reals +-- under ×) into the ring that turns products into sums. Every theorem +-- that uses it takes it as an argument. From it: +-- +-- * `balance-perturb`: Aitchison perturbation x ⊕ y (partwise +-- product) becomes addition of balance vectors; +-- * `balance-scale-invariant`: multiplying every part of a sample by +-- the same positive factor — library size, closure to proportions, +-- any per-sample scaling — leaves every balance unchanged. +-- +-- The second is why ILR balances are computed on counts or proportions +-- interchangeably, and why sample-level scaling factors (issue #16) do +-- not interact with the choice of ILR basis (issue #20). +------------------------------------------------------------------------ +{-# OPTIONS --safe --without-K #-} + +open import Algebra.Bundles using (CommutativeRing; CommutativeMonoid) + +module MetaManifold.ILR.Invariance {c ℓ} (R : CommutativeRing c ℓ) where + +open CommutativeRing R +open import Data.Product.Base using (_,_) +open import Level using (_⊔_; suc) +open import Relation.Binary.PropositionalEquality.Core as ≡ using (_≡_) +open import Relation.Binary.Reasoning.Setoid setoid + +open import MetaManifold.Composition.Tree +open import MetaManifold.Composition.Node +open import MetaManifold.Composition.Sum R +open import MetaManifold.ILR.Contrast R + +private + variable t : Tree + +------------------------------------------------------------------------ +-- Exact invariance + +balance-+ : ∀ (w : TVec Carrier t) n x y → + balance w n (x +ᵥ y) ≈ balance w n x + balance w n y +balance-+ w n = wdot-+ʳ w (contrast w n) + +balance-· : ∀ (w : TVec Carrier t) n k x → balance w n (k ·ᵥ x) ≈ k * balance w n x +balance-· w n = wdot-·ʳ w (contrast w n) + +balance-constant : ∀ (w : TVec Carrier t) n k → balance w n (pure k) ≈ 0# +balance-constant w n k = + trans (wdot-pureʳ w (contrast w n) k) + (trans (*-congˡ (contrast-sum-zero w n)) (zeroʳ k)) + +balance-shift-invariant : ∀ (w : TVec Carrier t) n x k → + balance w n (x +ᵥ pure k) ≈ balance w n x +balance-shift-invariant w n x k = + trans (balance-+ w n x (pure k)) + (trans (+-congˡ (balance-constant w n k)) (+-identityʳ _)) + +balance-cong : ∀ (w : TVec Carrier t) n {x y} → x ≋ y → balance w n x ≈ balance w n y +balance-cong w n = wdot-congʳ w (contrast w n) + +------------------------------------------------------------------------ +-- The log seam + +record LogHom {m ℓm} (M : CommutativeMonoid m ℓm) : Set (c ⊔ ℓ ⊔ m ⊔ ℓm) where + open CommutativeMonoid M using () renaming (Carrier to Pos; _∙_ to _×ₘ_) + field + log : Pos → Carrier + log-hom : ∀ a b → log (a ×ₘ b) ≈ log a + log b + +module _ {m ℓm} {M : CommutativeMonoid m ℓm} (L : LogHom M) where + open CommutativeMonoid M using () renaming (Carrier to Pos; _∙_ to _×ₘ_) + open LogHom L + + -- Aitchison perturbation: partwise product. + perturb : TVec Pos t → TVec Pos t → TVec Pos t + perturb = zipWith _×ₘ_ + + logs : TVec Pos t → TVec Carrier t + logs = map log + + logs-perturb : ∀ (x y : TVec Pos t) → logs (perturb x y) ≋ logs x +ᵥ logs y + logs-perturb (tip a) (tip b) = log-hom a b + logs-perturb (u ⊗ v) (u′ ⊗ v′) = logs-perturb u u′ , logs-perturb v v′ + + balance-perturb : ∀ (w : TVec Carrier t) n x y → + balance w n (logs (perturb x y)) ≈ balance w n (logs x) + balance w n (logs y) + balance-perturb w n x y = + trans (balance-cong w n (logs-perturb x y)) (balance-+ w n (logs x) (logs y)) + + balance-scale-invariant : ∀ (w : TVec Carrier t) n x (λ′ : Pos) → + balance w n (logs (perturb x (pure λ′))) ≈ balance w n (logs x) + balance-scale-invariant {t} w n x λ′ = begin + balance w n (logs (perturb x (pure λ′))) ≈⟨ balance-perturb w n x (pure λ′) ⟩ + balance w n (logs x) + balance w n (logs (pure λ′)) + ≈⟨ +-congˡ (balance-cong w n (≡⇒≋ {t = t} (map-pure log λ′))) ⟩ + balance w n (logs x) + balance w n (pure (log λ′)) + ≈⟨ +-congˡ (balance-constant w n (log λ′)) ⟩ + balance w n (logs x) + 0# ≈⟨ +-identityʳ _ ⟩ + balance w n (logs x) ∎ diff --git a/proofs/agda/MetaManifold/ILR/Kernel.agda b/proofs/agda/MetaManifold/ILR/Kernel.agda new file mode 100644 index 00000000..3260d47e --- /dev/null +++ b/proofs/agda/MetaManifold/ILR/Kernel.agda @@ -0,0 +1,105 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +------------------------------------------------------------------------ +-- The balance map has trivial kernel on centred vectors, hence is +-- injective there: two samples with different (weighted) CLR vectors +-- never share all their balances. No information is lost by moving from +-- CLR to any tree-based ILR basis. +-- +-- The hypothesis `Cancellable w` is necessary, not a proof convenience: +-- MetaManifold.ILR.Integer exhibits weights with a zero clade total for +-- which the conclusion is false. It holds for the uniform weights and +-- for all strictly positive weights over an ordered field — the only +-- weights the Julia code accepts (it refuses non-positive part weights). +------------------------------------------------------------------------ +{-# OPTIONS --safe --without-K #-} + +open import Algebra.Bundles using (CommutativeRing) + +module MetaManifold.ILR.Kernel {c ℓ} (R : CommutativeRing c ℓ) where + +open CommutativeRing R +open import Algebra.Properties.Ring ring using (+-inverseʳ-unique; -0#≈0#) +open import Data.Product.Base using (_,_) +open import Level using (_⊔_) +open import Relation.Binary.Reasoning.Setoid setoid + +open import MetaManifold.Composition.Tree +open import MetaManifold.Composition.Node +open import MetaManifold.Composition.Sum R +open import MetaManifold.ILR.Contrast R + +import Algebra.Solver.Ring.NaturalCoefficients.Default commutativeSemiring as Solver +open Solver using (solve; _:+_; _:*_; _:=_) + +-- Multiplying by every tip weight, and by every clade's total weight, +-- is injective. Over a field this says: no tip weight is zero and no +-- clade has zero total weight. +data Cancellable : ∀ {t} → TVec Carrier t → Set (c ⊔ ℓ) where + tipC : ∀ {p} → (∀ {a} → p * a ≈ 0# → a ≈ 0#) → Cancellable (tip p) + nodeC : ∀ {l r} {wl : TVec Carrier l} {wr : TVec Carrier r} → + (∀ {a} → (total wl + total wr) * a ≈ 0# → a ≈ 0#) → + Cancellable wl → Cancellable wr → Cancellable (wl ⊗ wr) + +private + left-zero : ∀ {a b} → a + b ≈ 0# → b ≈ 0# → a ≈ 0# + left-zero {a} {b} eq b≈0 = begin + a ≈⟨ +-identityʳ a ⟨ + a + 0# ≈⟨ +-congˡ b≈0 ⟨ + a + b ≈⟨ eq ⟩ + 0# ∎ + + right-zero : ∀ {a b} → a + b ≈ 0# → a ≈ 0# → b ≈ 0# + right-zero {a} {b} eq a≈0 = left-zero (trans (+-comm b a) eq) a≈0 + +balance-kernel : ∀ {t} {w : TVec Carrier t} → Cancellable w → ∀ x → + wsum w x ≈ 0# → (∀ n → balance w n x ≈ 0#) → AllZero x +balance-kernel (tipC cancel) (tip a) centred _ = cancel centred +balance-kernel {w = wl ⊗ wr} (nodeC cancel cl cr) (xl ⊗ xr) centred bal = + balance-kernel cl xl A≈0 left-balances , balance-kernel cr xr B≈0 right-balances + where + r = total wl + s = total wr + A = wsum wl xl + B = wsum wr xr + + B≈-A : B ≈ - A + B≈-A = +-inverseʳ-unique A B centred + + root : s * A + (- r) * B ≈ 0# + root = trans (sym (+-cong (wdot-pureˡ wl s xl) (wdot-pureˡ wr (- r) xr))) (bal here) + + scaled : (r + s) * A ≈ 0# + scaled = begin + (r + s) * A ≈⟨ solve 3 (λ r s A → (r :+ s) :* A := s :* A :+ r :* A) refl r s A ⟩ + s * A + r * A ≈⟨ +-congˡ (neg*neg r A) ⟨ + s * A + (- r) * (- A) ≈⟨ +-congˡ (*-congˡ B≈-A) ⟨ + s * A + (- r) * B ≈⟨ root ⟩ + 0# ∎ + + A≈0 : A ≈ 0# + A≈0 = cancel scaled + + B≈0 : B ≈ 0# + B≈0 = trans B≈-A (trans (-‿cong A≈0) -0#≈0#) + + left-balances : ∀ n → balance wl n xl ≈ 0# + left-balances n = left-zero (bal (inl n)) (wdot-zeroˡ wr xr) + + right-balances : ∀ n → balance wr n xr ≈ 0# + right-balances n = right-zero (bal (inr n)) (wdot-zeroˡ wl xl) + +-- Injectivity on centred vectors. +balance-injective : ∀ {t} {w : TVec Carrier t} → Cancellable w → ∀ x y → + wsum w x ≈ 0# → wsum w y ≈ 0# → + (∀ n → balance w n x ≈ balance w n y) → x ≋ y +balance-injective {w = w} cw x y x₀ y₀ same = + allZero-difference x y (balance-kernel cw (x -ᵥ y) centred balances) + where + sub-self : ∀ {a b} → a ≈ b → a - b ≈ 0# + sub-self {a} a≈b = trans (+-congˡ (-‿cong (sym a≈b))) (-‿inverseʳ a) + + centred : wsum w (x -ᵥ y) ≈ 0# + centred = trans (wsum--ʳ w x y) (sub-self (trans x₀ (sym y₀))) + + balances : ∀ n → balance w n (x -ᵥ y) ≈ 0# + balances n = trans (wdot--ʳ w (contrast w n) x y) (sub-self (same n)) diff --git a/proofs/agda/MetaManifold/ILR/Orthonormal.agda b/proofs/agda/MetaManifold/ILR/Orthonormal.agda new file mode 100644 index 00000000..a11330c0 --- /dev/null +++ b/proofs/agda/MetaManifold/ILR/Orthonormal.agda @@ -0,0 +1,60 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +------------------------------------------------------------------------ +-- Orthonormality of the normalised basis, relative to a named +-- normaliser. +-- +-- A `Normaliser` supplies, for every balance, a scalar κ with +-- κ · κ · (r · s · (r + s)) ≈ 1. Over the reals κ = 1/sqrt(r·s·(r+s)), +-- which is the constant in src/analysis/ilr_basis.jl; its existence is +-- the only analytic fact the normalised basis needs, and it is taken as +-- an argument rather than postulated. Given it, the scaled contrasts +-- are orthonormal in the weighted inner product: the PhILR / SBP / +-- dendrogram bases are isometries of the (weighted) Aitchison simplex +-- onto R^{D-1}. +------------------------------------------------------------------------ +{-# OPTIONS --safe --without-K #-} + +open import Algebra.Bundles using (CommutativeRing) + +module MetaManifold.ILR.Orthonormal {c ℓ} (R : CommutativeRing c ℓ) where + +open CommutativeRing R +open import Level using (_⊔_) +open import Relation.Binary.PropositionalEquality.Core using (_≢_) +open import Relation.Binary.Reasoning.Setoid setoid + +open import MetaManifold.Composition.Tree +open import MetaManifold.Composition.Node +open import MetaManifold.Composition.Sum R +open import MetaManifold.ILR.Contrast R + + +record Normaliser {t} (w : TVec Carrier t) : Set (c ⊔ ℓ) where + field + κ : Node t → Carrier + κ-norm : ∀ n → κ n * κ n * + ((plus-mass w n * minus-mass w n) * (plus-mass w n + minus-mass w n)) ≈ 1# + +module _ {t} {w : TVec Carrier t} (N : Normaliser w) where + open Normaliser N + + basis : Node t → TVec Carrier t + basis n = κ n ·ᵥ contrast w n + + private + scaled : ∀ n m → wdot w (basis n) (basis m) ≈ κ n * κ m * wdot w (contrast w n) (contrast w m) + scaled n m = begin + wdot w (κ n ·ᵥ contrast w n) (κ m ·ᵥ contrast w m) + ≈⟨ wdot-·ˡ w (κ n) (contrast w n) (κ m ·ᵥ contrast w m) ⟩ + κ n * wdot w (contrast w n) (κ m ·ᵥ contrast w m) + ≈⟨ *-congˡ (wdot-·ʳ w (contrast w n) (κ m) (contrast w m)) ⟩ + κ n * (κ m * wdot w (contrast w n) (contrast w m)) + ≈⟨ *-assoc (κ n) (κ m) _ ⟨ + κ n * κ m * wdot w (contrast w n) (contrast w m) ∎ + + basis-unit : ∀ n → wdot w (basis n) (basis n) ≈ 1# + basis-unit n = trans (scaled n n) (trans (*-congˡ (contrast-norm w n)) (κ-norm n)) + + basis-orthogonal : ∀ {n m} → n ≢ m → wdot w (basis n) (basis m) ≈ 0# + basis-orthogonal {n} {m} n≢m = + trans (scaled n m) (trans (*-congˡ (contrast-orthogonal w n≢m)) (zeroʳ _)) diff --git a/proofs/agda/MetaManifold/ILR/SBP.agda b/proofs/agda/MetaManifold/ILR/SBP.agda new file mode 100644 index 00000000..669836c4 --- /dev/null +++ b/proofs/agda/MetaManifold/ILR/SBP.agda @@ -0,0 +1,77 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +------------------------------------------------------------------------ +-- Sequential binary partitions (Egozcue & Pawlowsky-Glahn 2005). +-- +-- An SBP is a (D-1) × D sign matrix: row n says which parts are in the +-- numerator (+1), which in the denominator (-1) and which are not +-- involved (0) in balance n. Egozcue & Pawlowsky-Glahn's conditions +-- are that the first row splits all parts into two non-empty groups +-- and that every later row splits one group produced by an earlier row +-- into two non-empty groups. That is precisely "the rows are the +-- internal nodes of a rooted binary tree", which is how it is stated +-- here: `code n` is row n of the SBP of a tree. +-- +-- Proved: every row has a non-empty numerator and denominator +-- (`code-has-plus`, `code-has-minus`), a child row is supported inside +-- one side of its parent row (`code-nested-inl`, `code-nested-inr`), +-- and there are exactly D - 1 rows (`Node.toFin` bijection together +-- with `Tree.internal-count`). +-- +-- The Julia validator (`validate_sbp` in src/analysis/ilr_basis.jl) +-- decides the converse — that a user-supplied matrix *is* the code of +-- some tree — by reconstructing that tree, and refuses otherwise. +------------------------------------------------------------------------ +{-# OPTIONS --safe --without-K #-} + +module MetaManifold.ILR.SBP where + +open import Data.Product.Base using (_×_; _,_) +open import Data.Sum.Base using (_⊎_; inj₁; inj₂) +open import Relation.Binary.PropositionalEquality using (_≡_; refl) + +open import MetaManifold.Composition.Tree +open import MetaManifold.Composition.Node + +data Sign : Set where + plus minus off : Sign + +-- Row n of the SBP of t. philr's convention (phylo2sbp): first child +-- in the numerator, second child in the denominator. +code : ∀ {t} → Node t → TVec Sign t +code here = pure plus ⊗ pure minus +code (inl n) = code n ⊗ pure off +code (inr n) = pure off ⊗ code n + +code-has-plus : ∀ {t} (n : Node t) → Any (_≡ plus) (code n) +code-has-plus here = left (any-pure refl) +code-has-plus (inl n) = left (code-has-plus n) +code-has-plus (inr n) = right (code-has-plus n) + +code-has-minus : ∀ {t} (n : Node t) → Any (_≡ minus) (code n) +code-has-minus here = right (any-pure refl) +code-has-minus (inl n) = left (code-has-minus n) +code-has-minus (inr n) = right (code-has-minus n) + +------------------------------------------------------------------------ +-- Nesting: a row below the root only involves parts on one side of the +-- root row. + +-- `child ⊑[ σ ] parent`: every part the child row involves has sign σ +-- in the parent row. +_⊑[_]_ : ∀ {t} → TVec Sign t → Sign → TVec Sign t → Set +child ⊑[ σ ] parent = All (λ { (c , p) → c ≡ off ⊎ p ≡ σ }) (zipWith _,_ child parent) + +private + zip-pureʳ : ∀ {t} (x : TVec Sign t) σ → x ⊑[ σ ] pure σ + zip-pureʳ (tip x) σ = at (inj₂ refl) + zip-pureʳ (u ⊗ v) σ = both (zip-pureʳ u σ) (zip-pureʳ v σ) + + zip-offˡ : ∀ {t} (y : TVec Sign t) σ → pure off ⊑[ σ ] y + zip-offˡ (tip y) σ = at (inj₁ refl) + zip-offˡ (u ⊗ v) σ = both (zip-offˡ u σ) (zip-offˡ v σ) + +code-nested-inl : ∀ {l r} (m : Node l) → code (inl {r = r} m) ⊑[ plus ] code (here {l} {r}) +code-nested-inl m = both (zip-pureʳ (code m) plus) (zip-offˡ (pure minus) plus) + +code-nested-inr : ∀ {l r} (m : Node r) → code (inr {l = l} m) ⊑[ minus ] code (here {l} {r}) +code-nested-inr m = both (zip-offˡ (pure plus) minus) (zip-pureʳ (code m) minus) diff --git a/proofs/agda/README.md b/proofs/agda/README.md new file mode 100644 index 00000000..d105ee0a --- /dev/null +++ b/proofs/agda/README.md @@ -0,0 +1,34 @@ + +# Agda proofs + +Machine-checked statements about MetaManifold's compositional transforms. +Scope, layering, the theorem-to-test map and what is deliberately **not** +proved: [`docs/formal/verification-plan.md`](../../docs/formal/verification-plan.md). + +```sh +just proofs # or: scripts/check-proofs.sh +# with a non-system toolchain: +AGDA=/path/to/agda AGDA_STDLIB_LIB=/path/to/standard-library.agda-lib scripts/check-proofs.sh +``` + +Toolchain: Agda 2.6.4.3, agda-stdlib 2.1 (the estate pin). Every module is +`--safe --without-K`; there are no postulates. Assumptions that cannot be proved +in exact arithmetic (`log` turns products into sums; `1/sqrt` exists) are record +arguments (`LogHom`, `Normaliser`) so each theorem that relies on one says so in +its type. + +| Module | Content | +|---|---| +| `Composition.Tree` | rooted binary trees, tip vectors, D tips ⇒ D−1 internal nodes, comb tree | +| `Composition.Node` | balance positions; preorder numbering is a bijection with `Fin (D−1)` | +| `Composition.Sum` | weighted sums / inner product over any commutative ring (shared with #21) | +| `ILR.SBP` | SBP rows of a tree; non-empty ± parts; nesting (Egozcue & Pawlowsky-Glahn 2005) | +| `ILR.Contrast` | contrasts from masses and SBP row; centred; orthogonal; norm² = r·s·(r+s) | +| `ILR.Kernel` | trivial kernel / injectivity on centred vectors, under `Cancellable` | +| `ILR.Invariance` | linearity; constant-shift and sample-scale invariance (via `LogHom`) | +| `ILR.Orthonormal` | normalised basis orthonormal (via `Normaliser`) | +| `ILR.Comb` | comb tree under uniform weights = MetaManifold's Helmert default | +| `ILR.Integer` | ℤ instance; positive weights ⇒ `Cancellable`; counterexample for signed weights; philr known answer | +| `reject/` | must **fail** to type-check, each for the reason in its `-- EXPECT:` line | diff --git a/proofs/agda/metamanifold-proofs.agda-lib b/proofs/agda/metamanifold-proofs.agda-lib new file mode 100644 index 00000000..c4e79ded --- /dev/null +++ b/proofs/agda/metamanifold-proofs.agda-lib @@ -0,0 +1,4 @@ +name: metamanifold-proofs +depend: standard-library +include: . +flags: --safe --without-K diff --git a/proofs/agda/reject/KernelWithoutHypothesis.agda b/proofs/agda/reject/KernelWithoutHypothesis.agda new file mode 100644 index 00000000..92027195 --- /dev/null +++ b/proofs/agda/reject/KernelWithoutHypothesis.agda @@ -0,0 +1,21 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +-- MUST FAIL: the kernel theorem without `Cancellable` is false +-- (MetaManifold.ILR.Integer.kernel-needs-hypothesis). If this ever +-- type-checks, a definition has been weakened. The only Cancellable +-- witnesses available are for specific weights, so using one for an +-- arbitrary w is a type error. +-- EXPECT: != +{-# OPTIONS --safe --without-K #-} +module reject.KernelWithoutHypothesis where + +open import Data.Integer.Base using (0ℤ) +open import Data.Integer.Properties using (+-*-commutativeRing) +open import Relation.Binary.PropositionalEquality using (_≡_) +open import MetaManifold.Composition.Tree +open import MetaManifold.Composition.Sum +-*-commutativeRing +open import MetaManifold.ILR.Contrast +-*-commutativeRing +open import MetaManifold.ILR.Kernel +-*-commutativeRing +open import MetaManifold.ILR.Integer using (uniform-cancellable) + +bad : ∀ {t} (w x : TVec _ t) → wsum w x ≡ 0ℤ → (∀ n → balance w n x ≡ 0ℤ) → AllZero x +bad {t} w x = balance-kernel {w = w} (uniform-cancellable t) x diff --git a/proofs/agda/reject/OrthogonalSelf.agda b/proofs/agda/reject/OrthogonalSelf.agda new file mode 100644 index 00000000..c6dba0c1 --- /dev/null +++ b/proofs/agda/reject/OrthogonalSelf.agda @@ -0,0 +1,21 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +-- MUST FAIL: a contrast is not orthogonal to itself (its norm is +-- r·s·(r+s) = 1·1·2 = 2 here, not 0). Checks that orthogonality is not +-- vacuously provable, e.g. by a definition collapsing to zero. +-- EXPECT: 2 != 0 +{-# OPTIONS --safe --without-K #-} +module reject.OrthogonalSelf where + +open import Data.Integer.Base using (0ℤ; 1ℤ) +open import Data.Integer.Properties using (+-*-commutativeRing) +open import Relation.Binary.PropositionalEquality using (_≡_; refl) +open import MetaManifold.Composition.Tree +open import MetaManifold.Composition.Node +open import MetaManifold.Composition.Sum +-*-commutativeRing +open import MetaManifold.ILR.Contrast +-*-commutativeRing + +w : TVec _ (node leaf leaf) +w = tip 1ℤ ⊗ tip 1ℤ + +bad : wdot w (contrast w here) (contrast w here) ≡ 0ℤ +bad = refl diff --git a/proofs/agda/reject/Postulate.agda b/proofs/agda/reject/Postulate.agda new file mode 100644 index 00000000..197a3f97 --- /dev/null +++ b/proofs/agda/reject/Postulate.agda @@ -0,0 +1,11 @@ +-- SPDX-License-Identifier: AGPL-3.0-only +-- MUST FAIL: `--safe` (set in metamanifold-proofs.agda-lib, deliberately +-- NOT repeated in a pragma here) forbids postulates. Guards against the +-- library flags being dropped. +-- EXPECT: [Pp]ostulate +module reject.Postulate where + +open import Relation.Binary.PropositionalEquality using (_≡_) +open import MetaManifold.Composition.Tree + +postulate every-tree-is-a-leaf : (t : Tree) → t ≡ leaf diff --git a/scripts/check-proofs.sh b/scripts/check-proofs.sh new file mode 100755 index 00000000..a54757df --- /dev/null +++ b/scripts/check-proofs.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# +# check-proofs.sh — the Agda proof gate (docs/formal/verification-plan.md). +# +# 1. guard : no postulate / termination or positivity escape hatches / +# holes / unsafe flags in proofs/agda (comments ignored), and +# every module outside reject/ opens with +# {-# OPTIONS --safe --without-K #-} +# 2. check : proofs/agda/MetaManifold/All.agda type-checks +# 3. reject : every proofs/agda/reject/*.agda FAILS to type-check, with an +# error matching its "-- EXPECT: " line. A negative +# control that fails for some other reason (a parse error, a +# missing import) proves nothing, so the reason is checked. +# +# Toolchain pin (the estate's, see epistemic-types): Agda 2.6.4.3 with +# agda-stdlib 2.1. Overrides: +# AGDA=/path/to/agda +# AGDA_STDLIB_LIB=/path/to/standard-library.agda-lib +set -u +cd "$(dirname "$0")/.." +root=$(pwd) +proofs="$root/proofs/agda" +AGDA=${AGDA:-agda} +fail=0 + +if [ -z "${AGDA_STDLIB_LIB:-}" ]; then + for cand in /usr/share/agda-stdlib/standard-library.agda-lib \ + /usr/share/agda/lib/stdlib/standard-library.agda-lib; do + [ -f "$cand" ] && AGDA_STDLIB_LIB=$cand && break + done +fi +if [ -z "${AGDA_STDLIB_LIB:-}" ] || [ ! -f "$AGDA_STDLIB_LIB" ]; then + printf 'check-proofs: standard library not found; set AGDA_STDLIB_LIB\n' >&2 + exit 2 +fi +command -v "$AGDA" >/dev/null 2>&1 || { printf 'check-proofs: %s not found\n' "$AGDA" >&2; exit 2; } + +version=$("$AGDA" --numeric-version 2>/dev/null || true) +printf 'check-proofs: Agda %s, stdlib %s\n' "$version" "$AGDA_STDLIB_LIB" +case "$version" in + 2.6.4.3*) ;; + *) printf 'check-proofs: WARNING: expected Agda 2.6.4.3 (estate pin), got %s\n' "$version" >&2 ;; +esac + +# A private library registry: only the stdlib and this suite are visible. +agda_dir=$(mktemp -d) +trap 'rm -rf "$agda_dir"' EXIT +printf '%s\n%s\n' "$AGDA_STDLIB_LIB" "$proofs/metamanifold-proofs.agda-lib" > "$agda_dir/libraries" +export AGDA_DIR="$agda_dir" + +# ---------------------------------------------------------------- 1. guard +printf '%s\n' '[1/3] guard' +mapfile -t modules < <(cd "$proofs" && find MetaManifold -name '*.agda' | sort) +if [ "${#modules[@]}" -eq 0 ]; then + printf 'check-proofs: no modules found under %s\n' "$proofs" >&2; exit 1 +fi +forbidden='postulate|TERMINATING|NON_TERMINATING|NO_POSITIVITY_CHECK|NO_UNIVERSE_CHECK|NON_COVERING|INJECTIVE|trustMe|--type-in-type|--allow-unsolved-metas|--allow-incomplete-matches|--no-positivity-check|--no-termination-check|--cumulativity|--sized-types|--guardedness|--rewriting|\{!|(^|[[:space:](])\?([[:space:])]|$)' +for m in "${modules[@]}"; do + # Strip line comments (-- to end of line) and block comments on one line. + code=$(sed -e 's/{-[^#].*-}//g' -e 's/--.*$//' "$proofs/$m") + if hits=$(printf '%s\n' "$code" | grep -nE "$forbidden"); then + printf 'FORBIDDEN %s\n%s\n' "$m" "$hits" >&2; fail=1 + fi + if ! grep -qE '^\{-# OPTIONS --safe --without-K #-\}' "$proofs/$m"; then + printf 'MISSING-SAFE-PRAGMA %s\n' "$m" >&2; fail=1 + fi +done +printf ' %d modules scanned\n' "${#modules[@]}" + +# ---------------------------------------------------------------- 2. check +printf '%s\n' '[2/3] type-check MetaManifold/All.agda' +# Fresh interfaces for this suite (the stdlib's may be reused). +rm -rf "$proofs/_build" +if ! (cd "$proofs" && "$AGDA" MetaManifold/All.agda > "$agda_dir/check.log" 2>&1); then + grep -v '^ *Checking ' "$agda_dir/check.log" >&2 + printf 'check-proofs: type-check FAILED\n' >&2 + fail=1 +else + # Every module under MetaManifold/ must be reachable from All.agda, or it + # is not being checked at all. + unreachable=0 + for m in "${modules[@]}"; do + mod=${m%.agda}; mod=${mod//\//.} + [ "$mod" = MetaManifold.All ] && continue + if ! grep -q "^import $mod\$" "$proofs/MetaManifold/All.agda"; then + printf 'NOT-IN-ALL %s\n' "$mod" >&2; unreachable=1; fail=1 + fi + done + [ "$unreachable" -eq 0 ] && printf ' OK\n' +fi + +# ---------------------------------------------------------------- 3. reject +printf '%s\n' '[3/3] negative controls (must fail, for the stated reason)' +mapfile -t rejects < <(cd "$proofs" && find reject -name '*.agda' | sort) +if [ "${#rejects[@]}" -eq 0 ]; then + printf 'check-proofs: no negative controls found\n' >&2; fail=1 +fi +for r in "${rejects[@]}"; do + expect=$(sed -n 's/^-- EXPECT: //p' "$proofs/$r" | head -1) + if [ -z "$expect" ]; then + printf 'NO-EXPECT %s\n' "$r" >&2; fail=1; continue + fi + if (cd "$proofs" && "$AGDA" "$r" > "$agda_dir/reject.log" 2>&1); then + printf 'ACCEPTED (must be rejected) %s\n' "$r" >&2; fail=1 + elif grep -v '^ *Checking ' "$agda_dir/reject.log" | grep -qE -- "$expect"; then + printf ' rejected as expected %s\n' "$r" + else + printf 'REJECTED FOR THE WRONG REASON %s (expected /%s/)\n' "$r" "$expect" >&2 + grep -v '^ *Checking ' "$agda_dir/reject.log" | head -8 >&2 + fail=1 + fi +done + +if [ "$fail" -eq 0 ]; then + printf 'check-proofs: OK\n' +else + printf 'check-proofs: FAILED\n' >&2 +fi +exit "$fail" From 41f40ff07f80acaf0aeb51ccbe8b14e5ef6e2ba4 Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:11:43 +0000 Subject: [PATCH 2/6] feat(analysis): implement PhILR, SBP and balance-dendrogram ILR bases (#20) One engine (src/analysis/ilr_basis.jl) for the three bases that were refused as deferred: each basis is a rooted binary tree and balances are clade sums, O(D) per sample, no dense basis. Validation refuses non-bifurcating/unrooted trees, invalid SBPs (Egozcue & Pawlowsky-Glahn 2005) and taxa mismatches; provenance records tree/SBP SHA-256, dendrogram method and weights; >3 distinct SBPs raise a DANGER. The default Helmert basis and config hashes are unchanged. Tests: known answers, 17 expectations recomputed every run by an independent Julia reference (test/fixtures/ilr/ilr_reference.jl), R-gated philr/compositions/robCompositions cross-checks, negative controls, and scaling tests at 100/1000/10000 taxa. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- src/MetaManifold.jl | 2 + src/analysis/AnalysisConfig.jl | 265 +++- src/analysis/Execution.jl | 116 +- src/analysis/ilr_basis.jl | 1318 +++++++++++++++++ src/server/routes/analysis_config.jl | 8 + test/fixtures/ilr/dendrogram_d30/counts.csv | 31 + .../ilr/dendrogram_d30/expected_average.csv | 30 + .../ilr/dendrogram_d30/expected_complete.csv | 30 + .../ilr/dendrogram_d30/expected_ward.csv | 30 + .../ilr/dendrogram_d30/merge_average.csv | 30 + .../ilr/dendrogram_d30/merge_complete.csv | 30 + .../ilr/dendrogram_d30/merge_ward.csv | 30 + .../fixtures/ilr/dendrogram_d30/variation.csv | 31 + test/fixtures/ilr/ilr_reference.jl | 557 +++++++ test/fixtures/ilr/manifest.json | 56 + test/fixtures/ilr/philr_d25/counts.csv | 26 + .../ilr/philr_d25/expected_anorm__blw.csv | 25 + ...ed_anorm_x_gm_counts__mean_descendants.csv | 25 + .../philr_d25/expected_uniform__uniform.csv | 25 + test/fixtures/ilr/philr_d25/tree.nwk | 1 + test/fixtures/ilr/philr_d60_pruned/counts.csv | 61 + .../expected_enorm__uniform.csv | 60 + .../expected_gm_counts__mean_descendants.csv | 60 + .../expected_uniform__uniform.csv | 60 + test/fixtures/ilr/philr_d60_pruned/tree.nwk | 1 + test/fixtures/ilr/philr_d8/counts.csv | 9 + .../expected_enorm_x_gm_counts__blw_sqrt.csv | 8 + .../ilr/philr_d8/expected_gm_counts__blw.csv | 8 + .../philr_d8/expected_uniform__uniform.csv | 8 + test/fixtures/ilr/philr_d8/tree.nwk | 1 + test/fixtures/ilr/sbp_d12/counts.csv | 13 + test/fixtures/ilr/sbp_d12/expected.csv | 12 + test/fixtures/ilr/sbp_d12/sbp.csv | 13 + test/runtests.jl | 1 + test/unit/test_ilr_basis.jl | 694 +++++++++ test/unit/test_scaling.jl | 12 +- 36 files changed, 3630 insertions(+), 57 deletions(-) create mode 100644 src/analysis/ilr_basis.jl create mode 100644 test/fixtures/ilr/dendrogram_d30/counts.csv create mode 100644 test/fixtures/ilr/dendrogram_d30/expected_average.csv create mode 100644 test/fixtures/ilr/dendrogram_d30/expected_complete.csv create mode 100644 test/fixtures/ilr/dendrogram_d30/expected_ward.csv create mode 100644 test/fixtures/ilr/dendrogram_d30/merge_average.csv create mode 100644 test/fixtures/ilr/dendrogram_d30/merge_complete.csv create mode 100644 test/fixtures/ilr/dendrogram_d30/merge_ward.csv create mode 100644 test/fixtures/ilr/dendrogram_d30/variation.csv create mode 100644 test/fixtures/ilr/ilr_reference.jl create mode 100644 test/fixtures/ilr/manifest.json create mode 100644 test/fixtures/ilr/philr_d25/counts.csv create mode 100644 test/fixtures/ilr/philr_d25/expected_anorm__blw.csv create mode 100644 test/fixtures/ilr/philr_d25/expected_anorm_x_gm_counts__mean_descendants.csv create mode 100644 test/fixtures/ilr/philr_d25/expected_uniform__uniform.csv create mode 100644 test/fixtures/ilr/philr_d25/tree.nwk create mode 100644 test/fixtures/ilr/philr_d60_pruned/counts.csv create mode 100644 test/fixtures/ilr/philr_d60_pruned/expected_enorm__uniform.csv create mode 100644 test/fixtures/ilr/philr_d60_pruned/expected_gm_counts__mean_descendants.csv create mode 100644 test/fixtures/ilr/philr_d60_pruned/expected_uniform__uniform.csv create mode 100644 test/fixtures/ilr/philr_d60_pruned/tree.nwk create mode 100644 test/fixtures/ilr/philr_d8/counts.csv create mode 100644 test/fixtures/ilr/philr_d8/expected_enorm_x_gm_counts__blw_sqrt.csv create mode 100644 test/fixtures/ilr/philr_d8/expected_gm_counts__blw.csv create mode 100644 test/fixtures/ilr/philr_d8/expected_uniform__uniform.csv create mode 100644 test/fixtures/ilr/philr_d8/tree.nwk create mode 100644 test/fixtures/ilr/sbp_d12/counts.csv create mode 100644 test/fixtures/ilr/sbp_d12/expected.csv create mode 100644 test/fixtures/ilr/sbp_d12/sbp.csv create mode 100644 test/unit/test_ilr_basis.jl diff --git a/src/MetaManifold.jl b/src/MetaManifold.jl index e8c4b006..74a67886 100644 --- a/src/MetaManifold.jl +++ b/src/MetaManifold.jl @@ -42,6 +42,8 @@ include("analysis/estimation.jl") # response is and which offset it gets, and Scaling computes both under the conditions # published in docs/statistics/method-conditions/scaling-and-offsets.md. include("analysis/scaling.jl") +# ILR bases (issue #20): phylogenetic, SBP, balance dendrogram. Before Execution, which uses it. +include("analysis/ilr_basis.jl") include("analysis/Execution.jl") end diff --git a/src/analysis/AnalysisConfig.jl b/src/analysis/AnalysisConfig.jl index 852d3293..d45b2d9d 100644 --- a/src/analysis/AnalysisConfig.jl +++ b/src/analysis/AnalysisConfig.jl @@ -98,7 +98,17 @@ const ZERO_POLICY_STRINGS = Dict{String,ZeroPolicy}( const VALID_DISPERSION_METHODS = ("parametric", "local", "mean", "pooled", "glmGamPoi") const VALID_ZERO_HANDLING = ("pseudocount", "multiplicative_replacement", "bayesian_multiplicative", "refuse") const VALID_ILR_BASIS = ("default", "phylogenetic", "sequential_binary_partition", "balance_dendrogram") -const DEFERRED_ILR_BASIS = ("phylogenetic", "sequential_binary_partition", "balance_dendrogram") +# Issue #20 implemented the three bases that used to be listed here (src/analysis/ilr_basis.jl, +# held to docs/statistics/method-conditions/ilr-bases.md). The name is kept, empty, so code +# that iterates it keeps compiling and so a future deferral has an obvious home. +const DEFERRED_ILR_BASIS = () +# The inputs each non-default basis needs. Held here, not in ILRBasis, because the +# configuration is validated before any data or module that computes balances exists; the +# test suite asserts both copies agree. +const VALID_ILR_PART_WEIGHTS = ("uniform", "gm_counts", "anorm", "enorm", "anorm_x_gm_counts", "enorm_x_gm_counts") +const VALID_ILR_BALANCE_WEIGHTS = ("uniform", "blw", "blw_sqrt", "mean_descendants") +const VALID_ILR_DENDROGRAM_METHODS = ("ward", "complete", "average") +const ILR_SBP_ATTEMPT_DANGER_THRESHOLD = 3 # Method names are canonicalised to lower case by `NormalizationConfig`, so the allowed # names are held in the same case and compared in it. They were not always: the entries # for TSS/CSS/RSS arrived upper case while the constructor stored `"tss"`, so every one of @@ -174,7 +184,7 @@ struct NormalizationConfig end if method_clean == "ilr" if !isnothing(ilr_basis) && (ilr_basis in DEFERRED_ILR_BASIS) - throw(ArgumentError("ilr_basis '$ilr_basis' is not implemented (deferred, see GitHub issue #20). Only 'default' (Helmert sequential binary partition) is implemented. Refusing meaningless substitution.")) + throw(ArgumentError("ilr_basis '$ilr_basis' is not implemented (deferred). Refusing meaningless substitution.")) elseif !isnothing(ilr_basis) && !(ilr_basis in VALID_ILR_BASIS) throw(ArgumentError("ilr_basis must be one of $(join(VALID_ILR_BASIS, ", ")) (got '$ilr_basis')")) end @@ -296,6 +306,22 @@ struct CorrectionConfig end end +""" + _ilr_path(value, field) -> Union{String,Nothing} + +Check an ILR input path on its own: `nothing` stays `nothing`; otherwise the path must be +non-empty and free of characters that would corrupt the JSON/Nickel/DEED renderings. Whether +the file exists is checked when the run reads it, not here: a configuration is written on +one machine and may be run on another. +""" +function _ilr_path(value::Union{AbstractString,Nothing}, field::String) + isnothing(value) && return nothing + p = String(strip(value)) + isempty(p) && throw(ArgumentError("$field is empty — give the path of the file, or leave the field unset. See context_help('$field')")) + occursin(r"[\"\n\r\0\[\]{}`;\$]", p) && throw(ArgumentError("$field contains a character that is not allowed in a path here (quote, newline, NUL, brackets, backtick, ';' or '\$': they would corrupt the JSON/Nickel/DEED renderings or look like injection) — got '$p'. Rename the file.")) + return p +end + """ AdvancedConfig(; ...) @@ -316,6 +342,15 @@ struct AdvancedConfig min_samples_per_group::Int robust::Bool acknowledgment_token::Union{String,Nothing} + # ILR bases (issue #20; docs/statistics/method-conditions/ilr-bases.md). Which of these + # a basis requires, and which it forbids, depends on `normalization.ilr_basis`, so that + # cross-check is made by `AnalysisConfig`; here each value is checked on its own. + ilr_phylo_tree_path::Union{String,Nothing} + ilr_sbp_matrix_path::Union{String,Nothing} + ilr_balance_dendrogram_method::Union{String,Nothing} + ilr_part_weights::String + ilr_balance_weights::String + ilr_sbp_history::Vector{String} function AdvancedConfig(; dispersion_method::String="parametric", @@ -328,7 +363,13 @@ struct AdvancedConfig max_features::Union{Int,Nothing}=nothing, min_samples_per_group::Int=3, robust::Bool=false, - acknowledgment_token::Union{String,Nothing}=nothing + acknowledgment_token::Union{String,Nothing}=nothing, + ilr_phylo_tree_path::Union{AbstractString,Nothing}=nothing, + ilr_sbp_matrix_path::Union{AbstractString,Nothing}=nothing, + ilr_balance_dendrogram_method::Union{AbstractString,Nothing}=nothing, + ilr_part_weights::AbstractString="uniform", + ilr_balance_weights::AbstractString="uniform", + ilr_sbp_history::AbstractVector{<:AbstractString}=String[] ) dispersion_method_clean = lowercase(strip(dispersion_method)) zero_handling_clean = lowercase(strip(zero_handling)) @@ -411,7 +452,29 @@ struct AdvancedConfig end end - new(dispersion_method_clean, zero_handling_clean, zp, pseudocount, epsilon, min_prevalence, min_abundance, max_features, min_samples_per_group, robust, acknowledgment_token) + # ILR basis inputs. Paths are carried into the JSON, Nickel and DEED renderings as + # quoted strings; a quote, a newline or a bracket would break those documents (DEED + # allows only round brackets anywhere), so such paths are refused rather than escaped + # into something the user did not write. + tree_path = _ilr_path(ilr_phylo_tree_path, "advanced.ilr_phylo_tree_path") + sbp_path = _ilr_path(ilr_sbp_matrix_path, "advanced.ilr_sbp_matrix_path") + dendro = isnothing(ilr_balance_dendrogram_method) ? nothing : lowercase(strip(ilr_balance_dendrogram_method)) + if !isnothing(dendro) && !(dendro in VALID_ILR_DENDROGRAM_METHODS) + throw(ArgumentError("advanced.ilr_balance_dendrogram_method must be one of $(join(VALID_ILR_DENDROGRAM_METHODS, ", ")) (ward is R's ward.D2) — got '$ilr_balance_dendrogram_method'. See context_help('advanced.ilr_balance_dendrogram_method')")) + end + part_w = lowercase(strip(ilr_part_weights)) + part_w in VALID_ILR_PART_WEIGHTS || throw(ArgumentError("advanced.ilr_part_weights must be one of $(join(VALID_ILR_PART_WEIGHTS, ", ")) — got '$ilr_part_weights'. See context_help('advanced.ilr_part_weights')")) + bal_w = lowercase(strip(ilr_balance_weights)) + bal_w in VALID_ILR_BALANCE_WEIGHTS || throw(ArgumentError("advanced.ilr_balance_weights must be one of $(join(VALID_ILR_BALANCE_WEIGHTS, ", ")) — got '$ilr_balance_weights'. See context_help('advanced.ilr_balance_weights')")) + history = String[] + for h in ilr_sbp_history + hl = lowercase(strip(h)) + occursin(r"^[0-9a-f]{64}$", hl) || throw(ArgumentError("advanced.ilr_sbp_history entries must be SHA-256 hex digests of SBP files tried earlier (64 hex characters) — got '$h'. See context_help('advanced.ilr_sbp_history')")) + hl in history || push!(history, hl) + end + + new(dispersion_method_clean, zero_handling_clean, zp, pseudocount, epsilon, min_prevalence, min_abundance, max_features, min_samples_per_group, robust, acknowledgment_token, + tree_path, sbp_path, dendro, part_w, bal_w, history) end end @@ -419,6 +482,65 @@ end # Main immutable AnalysisConfig struct — exactly user's answers # -------------------------------------------------------------------------- +""" + _ilr_inputs_set(advanced) -> Bool + +True when any ILR basis input differs from its default. +""" +_ilr_inputs_set(a::AdvancedConfig) = + !isnothing(a.ilr_phylo_tree_path) || !isnothing(a.ilr_sbp_matrix_path) || + !isnothing(a.ilr_balance_dendrogram_method) || a.ilr_part_weights != "uniform" || + a.ilr_balance_weights != "uniform" || !isempty(a.ilr_sbp_history) + +"""The ILR basis inputs as an ordered dictionary (hash, JSON, provenance).""" +_ilr_inputs_dict(a::AdvancedConfig) = OrderedDict{String,Any}( + "ilr_phylo_tree_path" => a.ilr_phylo_tree_path, + "ilr_sbp_matrix_path" => a.ilr_sbp_matrix_path, + "ilr_balance_dendrogram_method" => a.ilr_balance_dendrogram_method, + "ilr_part_weights" => a.ilr_part_weights, + "ilr_balance_weights" => a.ilr_balance_weights, + "ilr_sbp_history" => a.ilr_sbp_history +) + +""" + _validate_ilr_inputs(norm_method, ilr_basis, advanced) + +The configuration contract of docs/statistics/method-conditions/ilr-bases.md: the ILR basis +inputs are meaningless outside ILR; `phylogenetic` requires a tree path, `sequential_binary_partition` +an SBP path and `balance_dendrogram` a clustering method, and each forbids the others' inputs; +part weights need a non-default basis (the Helmert loop is unweighted and stays unchanged); +balance weights need branch lengths, so the phylogenetic basis; an SBP history needs an SBP. +Refuses rather than ignores: an input that would be silently unused is a misunderstanding. +""" +function _validate_ilr_inputs(norm_method::AbstractString, ilr_basis::Union{String,Nothing}, a::AdvancedConfig) + if norm_method != "ilr" + if _ilr_inputs_set(a) + set = [k for (k, v) in _ilr_inputs_dict(a) if !(isnothing(v) || v == "uniform" || (v isa Vector && isempty(v)))] + throw(ArgumentError("$(join(("advanced." * k for k in set), ", ")) only meaningful for the ILR transform (normalization.method = 'ilr'), not for '$norm_method'. Refusing. See context_help('normalization.ilr_basis')")) + end + return nothing + end + basis = something(ilr_basis, "default") + need(field, value, wanted) = if basis == wanted && isnothing(value) + throw(ArgumentError("normalization.ilr_basis = '$basis' requires advanced.$field. See context_help('advanced.$field')")) + elseif basis != wanted && !isnothing(value) + throw(ArgumentError("advanced.$field is only used by ilr_basis = '$wanted', but ilr_basis is '$basis'. Refusing an input that would be silently ignored. See context_help('advanced.$field')")) + end + need("ilr_phylo_tree_path", a.ilr_phylo_tree_path, "phylogenetic") + need("ilr_sbp_matrix_path", a.ilr_sbp_matrix_path, "sequential_binary_partition") + need("ilr_balance_dendrogram_method", a.ilr_balance_dendrogram_method, "balance_dendrogram") + if basis == "default" && a.ilr_part_weights != "uniform" + throw(ArgumentError("advanced.ilr_part_weights = '$(a.ilr_part_weights)' needs a phylogenetic, sequential_binary_partition or balance_dendrogram basis: the default Helmert basis is unweighted. See context_help('advanced.ilr_part_weights')")) + end + if basis != "phylogenetic" && a.ilr_balance_weights != "uniform" + throw(ArgumentError("advanced.ilr_balance_weights = '$(a.ilr_balance_weights)' needs branch lengths, which only ilr_basis = 'phylogenetic' has. See context_help('advanced.ilr_balance_weights')")) + end + if basis != "sequential_binary_partition" && !isempty(a.ilr_sbp_history) + throw(ArgumentError("advanced.ilr_sbp_history records SBP matrices tried earlier and is only used by ilr_basis = 'sequential_binary_partition'. See context_help('advanced.ilr_sbp_history')")) + end + return nothing +end + """ AnalysisConfig(; method, formula, metadata_columns, ...) @@ -539,11 +661,19 @@ struct AnalysisConfig throw(ArgumentError("normalization.method '$norm_method' incompatible with method '$(METHOD_TO_STRING[method_enum])'. Allowed for $(METHOD_TO_STRING[method_enum]): $(join(allowed_norms, ", ")). See context_help('normalization.method') and MethodNormalizationCompatibility contract in Nickel. Refusing meaningless combination.")) end + # ILR basis inputs: each basis requires its own input and forbids the others'. + _validate_ilr_inputs(norm_method, normalization.ilr_basis, advanced) + # Dangerous computed is_dang = false if correction.allow_no_correction is_dang = true end + # SBP p-hacking guard, the part knowable before the run: more distinct SBPs already + # recorded than the threshold. The run adds the current SBP's digest and re-counts. + if length(advanced.ilr_sbp_history) > ILR_SBP_ATTEMPT_DANGER_THRESHOLD + is_dang = true + end if advanced.zero_handling == "refuse" || advanced.zero_policy == REFUSE is_dang = true end @@ -614,6 +744,14 @@ struct AnalysisConfig "robust" => advanced.robust ) ) + # The ILR basis inputs enter the hash whenever any is set. When none is, the + # canonical form is exactly what it was before issue #20, so every configuration + # hashed before these fields existed keeps its hash. + if _ilr_inputs_set(advanced) + adv_canonical = OrderedDict{String,Any}(canonical["advanced"]) + adv_canonical["ilr"] = _ilr_inputs_dict(advanced) + canonical["advanced"] = adv_canonical + end bytes2hex(sha256(JSON3.write(canonical))) else hash @@ -799,7 +937,10 @@ function canonical_json(config::AnalysisConfig) "zero_handling" => config.advanced.zero_handling, "pseudocount" => config.advanced.pseudocount, "epsilon" => config.advanced.epsilon, - "min_prevalence" => config.advanced.min_prevalence + "min_prevalence" => config.advanced.min_prevalence, + # Only when set, as in the hash: canonical JSON of a configuration without ILR + # basis inputs is unchanged by issue #20. + (_ilr_inputs_set(config.advanced) ? ("ilr" => _ilr_inputs_dict(config.advanced),) : ())... ), "hash" => config.hash, "dangerous" => config.dangerous @@ -964,14 +1105,72 @@ function context_help(field_path::String) See context_help('advanced.zero_policy') and Nickel ZeroHandlingContract. """, "normalization.ilr_basis" => """ - ILR basis (only for ILR, meaningless otherwise) - - - default: Helmert-style sequential binary partition (first taxon vs rest, second vs rest, etc., creating n-1 balances) - - phylogenetic: NOT IMPLEMENTED (deferred, see GitHub issue #20) - - sequential_binary_partition: NOT IMPLEMENTED (deferred, see GitHub issue #20) - - balance_dendrogram: NOT IMPLEMENTED (deferred, see GitHub issue #20) - - Refuses unimplemented bases and meaningless use for non-ILR methods. See JSON schema enum and Nickel. + ILR basis (only for ILR, meaningless otherwise). Conditions, refusals and evidence: + docs/statistics/method-conditions/ilr-bases.md. Every basis gives D-1 orthonormal balances; + the model reports one test per balance and Benjamini-Hochberg across balances is mandatory. + + - default: Helmert sequential binary partition (balance i = taxa 1..i against taxon i+1). + Unchanged; the new engine reproduces it exactly (Agda: comb-is-helmert). + - phylogenetic: PhILR (Silverman et al. 2017, eLife 6:e21887). Needs + advanced.ilr_phylo_tree_path, a ROOTED, strictly BIFURCATING Newick tree whose tips are the + taxon ids. Unrooted trees and polytomies are refused, never resolved; tips that are not + retained taxa are pruned (ape::keep.tip). Optional advanced.ilr_part_weights and + advanced.ilr_balance_weights reproduce philr's part.weights and ilr.weights. + - sequential_binary_partition: a user-defined SBP (Egozcue & Pawlowsky-Glahn 2005, Math. Geol. + 37:795-828). Needs advanced.ilr_sbp_matrix_path, a CSV with taxa as rows, balances as + columns and entries 1 / -1 / 0; validity is checked by reconstructing the partition tree. + Trying several SBPs is a forking-paths risk: more than 3 distinct SBPs (see + advanced.ilr_sbp_history) raises the DANGER banner. + - balance_dendrogram: clusters parts on the variation matrix Var(log(x_i/x_j)) and uses the + dendrogram as the SBP (Pawlowsky-Glahn, Egozcue & Tolosana-Delgado 2015, Modeling and + Analysis of Compositional Data, Wiley). Needs advanced.ilr_balance_dendrogram_method. + Data-derived basis: recorded as such. + + Refuses meaningless use for non-ILR methods. See JSON schema enum and Nickel IlrBasisInputsContract. + """, + "advanced.ilr_phylo_tree_path" => """ + Newick tree for ilr_basis = 'phylogenetic' (required then, refused otherwise). + + Tip labels must equal taxon ids exactly (no case folding, no underscore/space rewriting; + quote labels with spaces). The tree must be rooted (a basal trifurcation is how FastTree, + IQ-TREE and RAxML write UNROOTED trees — root it in a phylogenetics tool first) and strictly + bifurcating. Extra tips are pruned; retained taxa missing from the tree are refused. The + file's SHA-256 is recorded in the provenance. Relative paths resolve against the working + directory of the run. Silverman et al. (2017) eLife 6:e21887. + """, + "advanced.ilr_sbp_matrix_path" => """ + SBP CSV for ilr_basis = 'sequential_binary_partition' (required then, refused otherwise). + + First column: taxon ids (exactly the retained taxa). Other columns: one per balance, header = + balance id, entries 1 (numerator), -1 (denominator), 0 (not involved). D taxa need exactly D-1 + columns; the first partition involves every taxon and each later one splits a group made by an + earlier one (Egozcue & Pawlowsky-Glahn 2005). Each failure names the column and taxa. The file's + SHA-256 is recorded; add it to advanced.ilr_sbp_history if you try another SBP. + """, + "advanced.ilr_balance_dendrogram_method" => """ + Clustering for ilr_basis = 'balance_dendrogram' (required then, refused otherwise): ward (R's + ward.D2), complete or average, applied to the variation matrix by R's hclust algorithm — the same + as robCompositions::clustCoDa_qmode with the classical variation. Needs at least 2 samples. The + basis is chosen from the data it is then used to test; this is recorded. Pawlowsky-Glahn, + Egozcue & Tolosana-Delgado (2015). + """, + "advanced.ilr_part_weights" => """ + Part weights for a non-default ILR basis (philr's part.weights): uniform (default), gm_counts, + anorm, enorm, anorm_x_gm_counts, enorm_x_gm_counts. Non-uniform weights give the weighted ILR of + Silverman et al. (2017) — orthonormal in the weighted Aitchison geometry — and are computed from + the zero-handled table itself, which is recorded. Refused for the default basis. + """, + "advanced.ilr_balance_weights" => """ + Balance weights for ilr_basis = 'phylogenetic' only (philr's ilr.weights): uniform (default), + blw, blw_sqrt, mean_descendants. They need branch lengths. A balance weight multiplies a balance by + a constant: effect sizes change, per-balance test statistics do not, and the coordinates stop being + isometric (recorded). Zero-length tip edges are replaced by the smallest non-zero edge, as in philr. + """, + "advanced.ilr_sbp_history" => """ + SHA-256 digests of SBP files tried earlier in this project (ilr_basis = + 'sequential_binary_partition' only). The run counts distinct SBPs including the current one; + more than 3 raises the DANGER banner. Trying partitions until one 'works' is a forking-paths + problem BH cannot correct. The count is disclosed, not refused: pre-register the SBP. """, "correction.method" => """ Multiple testing correction — BH mandatory in v1, hard-stop DANGER banner on overrides @@ -1116,6 +1315,9 @@ function danger_banner(config::AnalysisConfig) if config.advanced.min_samples_per_group < 3 push!(reasons, "min_samples_per_group=$(config.advanced.min_samples_per_group) <3 — statistical power very low, results unreliable") end + if length(config.advanced.ilr_sbp_history) > ILR_SBP_ATTEMPT_DANGER_THRESHOLD + push!(reasons, "SBP p-hacking guard: $(length(config.advanced.ilr_sbp_history)) distinct SBP matrices already tried in this project (more than $ILR_SBP_ATTEMPT_DANGER_THRESHOLD) — trying partitions until one 'works' inflates false discoveries that BH cannot correct; pre-register the SBP and report every partition tried") + end if config.normalization.method == "rarefy" && config.method == NB_GLM push!(reasons, "rarefy + NB_GLM — rarefy discards data and NB_GLM already handles library size via size_factors — combining is questionable") end @@ -1215,7 +1417,13 @@ function to_json(config::AnalysisConfig) "max_features" => config.advanced.max_features, "min_samples_per_group" => config.advanced.min_samples_per_group, "robust" => config.advanced.robust, - "acknowledgment_token" => config.advanced.acknowledgment_token + "acknowledgment_token" => config.advanced.acknowledgment_token, + "ilr_phylo_tree_path" => config.advanced.ilr_phylo_tree_path, + "ilr_sbp_matrix_path" => config.advanced.ilr_sbp_matrix_path, + "ilr_balance_dendrogram_method" => config.advanced.ilr_balance_dendrogram_method, + "ilr_part_weights" => config.advanced.ilr_part_weights, + "ilr_balance_weights" => config.advanced.ilr_balance_weights, + "ilr_sbp_history" => config.advanced.ilr_sbp_history ), "provenance" => config.provenance, "hash" => config.hash, @@ -1232,6 +1440,7 @@ defaults. JSON parsing and constructor validation errors are propagated. """ function from_json(json_str::String) data = JSON3.read(json_str) + _str_or_nothing(v) = isnothing(v) ? nothing : String(v) norm_data = data.normalization norm = NormalizationConfig( @@ -1271,7 +1480,14 @@ function from_json(json_str::String) max_features=_f64(get(adv_data, :max_features, nothing)), min_samples_per_group=get(adv_data, :min_samples_per_group, 3), robust=get(adv_data, :robust, false), - acknowledgment_token=get(adv_data, :acknowledgment_token, nothing) + acknowledgment_token=get(adv_data, :acknowledgment_token, nothing), + # Absent from JSON written before issue #20: the defaults apply. + ilr_phylo_tree_path=_str_or_nothing(get(adv_data, :ilr_phylo_tree_path, nothing)), + ilr_sbp_matrix_path=_str_or_nothing(get(adv_data, :ilr_sbp_matrix_path, nothing)), + ilr_balance_dendrogram_method=_str_or_nothing(get(adv_data, :ilr_balance_dendrogram_method, nothing)), + ilr_part_weights=String(get(adv_data, :ilr_part_weights, "uniform")), + ilr_balance_weights=String(get(adv_data, :ilr_balance_weights, "uniform")), + ilr_sbp_history=String[String(h) for h in get(adv_data, :ilr_sbp_history, String[])] ) prov = OrderedDict{String,Any}() @@ -1301,6 +1517,8 @@ function from_json(json_str::String) return cfg end +_nickel_str(v::Union{String,Nothing}) = isnothing(v) ? "null" : "\"$v\"" + """Render `config` as the module's generated Nickel source text.""" function to_nickel(config::AnalysisConfig) # Nickel contract from hyperpolymath/standards 1-formats/k9/*.ncl style @@ -1352,6 +1570,12 @@ function to_nickel(config::AnalysisConfig) max_features = $(isnothing(config.advanced.max_features) ? "null" : string(config.advanced.max_features)), min_samples_per_group = $(config.advanced.min_samples_per_group), robust = $(config.advanced.robust ? "true" : "false"), + ilr_phylo_tree_path = $(_nickel_str(config.advanced.ilr_phylo_tree_path)), + ilr_sbp_matrix_path = $(_nickel_str(config.advanced.ilr_sbp_matrix_path)), + ilr_balance_dendrogram_method = $(isnothing(config.advanced.ilr_balance_dendrogram_method) ? "null" : "'$(config.advanced.ilr_balance_dendrogram_method)'"), + ilr_part_weights = '$(config.advanced.ilr_part_weights)', + ilr_balance_weights = '$(config.advanced.ilr_balance_weights)', + ilr_sbp_history = [$(join(["\"$h\"" for h in config.advanced.ilr_sbp_history], ", "))], } | ZeroHandlingContract, provenance = { @@ -1361,7 +1585,7 @@ function to_nickel(config::AnalysisConfig) hash = "$(config.hash)", dangerous = $(config.dangerous ? "true" : "false"), - } + } | IlrBasisInputsContract """ end @@ -1477,7 +1701,13 @@ function to_deed(config::AnalysisConfig) :max-features $(isnothing(config.advanced.max_features) ? "0" : string(config.advanced.max_features)) :min-samples-per-group $(config.advanced.min_samples_per_group) :robust $(robust_bool) - :acknowledgment-token "$(isnothing(config.advanced.acknowledgment_token) ? "" : config.advanced.acknowledgment_token)") + :acknowledgment-token "$(isnothing(config.advanced.acknowledgment_token) ? "" : config.advanced.acknowledgment_token)" + :ilr-phylo-tree-path "$(something(config.advanced.ilr_phylo_tree_path, ""))" + :ilr-sbp-matrix-path "$(something(config.advanced.ilr_sbp_matrix_path, ""))" + :ilr-balance-dendrogram-method "$(something(config.advanced.ilr_balance_dendrogram_method, ""))" + :ilr-part-weights "$(config.advanced.ilr_part_weights)" + :ilr-balance-weights "$(config.advanced.ilr_balance_weights)" + :ilr-sbp-history ($(join(["\"$h\"" for h in config.advanced.ilr_sbp_history], " ")))) (provenance :id "$(config.id)" @@ -1504,6 +1734,7 @@ function to_deed(config::AnalysisConfig) :formula "R-style formula e.g. ~ group must reference only metadata_columns forbids ; backtick dollar" :correction "BH mandatory in v1 any override triggers DANGER banner requires acknowledgment token $(DANGER_ACK_TOKEN)" :normalization "Normalization must be compatible with method: nb_glm allows none/rarefy/size_factors/relative/tss/css/rss (tss/css/rss are exact offsets, see docs/statistics/method-conditions/scaling-and-offsets.md), clr_lm requires clr, ilr_lm requires ilr" + :ilr-basis "ILR basis default (Helmert) phylogenetic (PhILR, needs a rooted bifurcating Newick tree) sequential_binary_partition (needs a valid SBP CSV, more than 3 distinct SBPs is DANGER) balance_dendrogram (ward complete average on the variation matrix) see docs/statistics/method-conditions/ilr-bases.md" :advanced "All advanced options behind Advanced Analysis expander hidden unless Evidence Mode heavy validation refusal meaningless")) """ end diff --git a/src/analysis/Execution.jl b/src/analysis/Execution.jl index c512d044..744b7d88 100644 --- a/src/analysis/Execution.jl +++ b/src/analysis/Execution.jl @@ -44,6 +44,7 @@ using JSON3 using OrderedCollections using Logging import ..Scaling +import ..ILRBasis using Statistics # Use AnalysisConfig from parent module @@ -1057,6 +1058,9 @@ function prepare_analysis_table( transform_method = lowercase(strip(config.normalization.method)) prepared = copy(counts_after_zero) filtered_taxa_ids_before_ilr = copy(filtered_taxa_ids) + # Set by the non-default ILR bases; its checks, warnings, DANGER reasons and provenance + # are recorded below. + ilr_outcome = nothing offset = nothing scaling = nothing count_response = config.method == AnalysisConfig.NB_GLM @@ -1143,10 +1147,9 @@ function prepare_analysis_table( prepared[:, j] = log_col .- mean_log end elseif transform_method == "ilr" - # Isometric Log-Ratio with default basis (phylogenetic, SBP, balance_dendrogram deferred) - # For stub, use CLR then transform via default ILR basis (e.g., Helmert matrix or sequential binary partition default) - # Default ILR basis: for n taxa, n-1 balances, each balance is log ratio of geometric mean of first k vs k+1? - # Simplified: use CLR then multiply by Helmert submatrix + # Isometric log-ratio. The basis is `normalization.ilr_basis`: `default` (Helmert) + # below, or phylogenetic / sequential_binary_partition / balance_dendrogram through + # ILRBasis (issue #20, docs/statistics/method-conditions/ilr-bases.md). # First compute CLR clr_table = similar(counts_after_zero) for j in 1:size(counts_after_zero, 2) @@ -1158,44 +1161,58 @@ function prepare_analysis_table( clr_table[:, j] = log_col .- mean(log_col) end - # Default ILR basis: create (n-1) x n matrix where each row is a balance n_taxa = size(clr_table, 1) if n_taxa < 2 throw(ArgumentError("ILR requires at least 2 taxa — got $n_taxa")) end - # For default basis, we use sequential binary partition: first balance is first taxon vs rest, second is second vs rest, etc. - # Or use Helmert: ilr_basis[i,j] = sqrt(j/(j+1)) * (1/j for k<=j, -1 for k=j+1, 0 otherwise) — but need orthonormal - # For stub, we use simple: ilr = clr transformed via basis where basis is identity minus 1/n? Actually CLR already centered, ILR is orthonormal version - # For simplicity, we return CLR for now with warning that exact ILR basis deferred, and set prepared to clr_table - # Real implementation would use `compositions::ilr` or `philr` or custom - - if !isnothing(config.normalization.ilr_basis) && (config.normalization.ilr_basis in AnalysisConfig.DEFERRED_ILR_BASIS) - throw(ArgumentError("ILR basis '$(config.normalization.ilr_basis)' is not implemented (deferred, see GitHub issue #20). Refusing to substitute default Helmert basis.")) + ilr_basis_name = something(config.normalization.ilr_basis, "default") + if ilr_basis_name in AnalysisConfig.DEFERRED_ILR_BASIS + throw(ArgumentError("ILR basis '$(ilr_basis_name)' is not implemented (deferred). Refusing to substitute the default Helmert basis.")) end - # For default basis, we can compute ILR as: ilr = V^T * clr where V is n x (n-1) orthonormal basis - # For stub, we use simple basis: first n-1 rows of clr_table (drop last taxon) — not orthonormal but works for testing - # Better: use Helmert matrix - # Create Helmert matrix of size n x n, then take first n-1 columns as basis for CLR space? Actually Helmert for ILR is n x (n-1) - # Let's create simple ILR: for i=1..n-1, ilr_i = sqrt(i/(i+1)) * (mean(log(x_1..x_i)) - log(x_{i+1})) - - ilr_table = zeros(n_taxa - 1, size(clr_table, 2)) - for j in 1:size(clr_table, 2) - # For each sample, compute ILR balances - # Use log counts, not CLR, for ILR formula - log_col = log.(counts_after_zero[:, j]) - for i in 1:(n_taxa-1) - # Balance i: first i taxa vs taxon i+1 - # geo_mean_first_i = exp(mean(log_col[1:i])) - # ilr_i = sqrt(i/(i+1)) * (mean(log_col[1:i]) - log_col[i+1]) - mean_first_i = mean(log_col[1:i]) - ilr_table[i, j] = sqrt(i/(i+1)) * (mean_first_i - log_col[i+1]) + if ilr_basis_name == "default" + # Helmert basis, unchanged byte for byte by issue #20: + # ilr_i = sqrt(i/(i+1)) * (mean(log(x_1..x_i)) - log(x_{i+1})), i = 1..n-1. + # ILRBasis.comb_tree reproduces it (Agda: comb-is-helmert; tested to 1e-12), but + # this loop is left exactly as it was so that no default-basis result moves. + ilr_table = zeros(n_taxa - 1, size(clr_table, 2)) + for j in 1:size(clr_table, 2) + # For each sample, compute ILR balances + # Use log counts, not CLR, for ILR formula + log_col = log.(counts_after_zero[:, j]) + for i in 1:(n_taxa-1) + # Balance i: first i taxa vs taxon i+1 + # geo_mean_first_i = exp(mean(log_col[1:i])) + # ilr_i = sqrt(i/(i+1)) * (mean(log_col[1:i]) - log_col[i+1]) + mean_first_i = mean(log_col[1:i]) + ilr_table[i, j] = sqrt(i/(i+1)) * (mean_first_i - log_col[i+1]) + end end - end - prepared = ilr_table - filtered_taxa_ids = ["balance_$i" for i in 1:(n_taxa - 1)] + prepared = ilr_table + filtered_taxa_ids = ["balance_$i" for i in 1:(n_taxa - 1)] + else + # Phylogenetic (PhILR), sequential binary partition, balance dendrogram: one + # engine, held to docs/statistics/method-conditions/ilr-bases.md. It refuses -- + # never substitutes -- when a tree, SBP or clustering condition fails. The basis is + # built on the retained taxa, after filtering and zero handling. + adv = config.advanced + retained_set = Set(filtered_taxa_ids_before_ilr) + ilr_outcome = ILRBasis.ilr_transform( + counts_after_zero, filtered_taxa_ids_before_ilr; + basis = ilr_basis_name, + tree_path = adv.ilr_phylo_tree_path, + sbp_path = adv.ilr_sbp_matrix_path, + dendrogram_method = adv.ilr_balance_dendrogram_method, + part_weights_kind = adv.ilr_part_weights, + balance_weights_kind = adv.ilr_balance_weights, + sbp_history = adv.ilr_sbp_history, + removed_by_filtering = String[t for t in taxa_ids if !(t in retained_set)] + ) + prepared = ilr_outcome.balances + filtered_taxa_ids = copy(ilr_outcome.balance_ids) + end elseif transform_method == "presence_absence" prepared = Float64.(counts_after_zero .> 0) @@ -1235,13 +1252,16 @@ function prepare_analysis_table( checks["scaling"] = Scaling.factor_checks(scaling) append!(warnings, scaling.warnings) end - if transform_method == "ilr" + if transform_method == "ilr" && isnothing(ilr_outcome) checks["ilr"] = OrderedDict{String,Any}( "basis" => something(config.normalization.ilr_basis, "default"), "definition" => "Helmert-style sequential binary partition (balance_i = sqrt(i/(i+1)) * (mean(log(x_1..x_i)) - log(x_{i+1})))", "taxa_in" => length(filtered_taxa_ids_before_ilr), "taxa_order" => filtered_taxa_ids_before_ilr ) + elseif !isnothing(ilr_outcome) + checks["ilr"] = ilr_outcome.checks + append!(warnings, ilr_outcome.warnings) end checks["all_zero_taxa"] = OrderedDict{String,Any}( "all_zero_taxa_indices" => all_zero_taxa_indices, @@ -1342,6 +1362,26 @@ function prepare_analysis_table( @warn "Safe self-healing performed" healings=config.id banner=banner end + # ILR basis selection DANGER (the SBP p-hacking guard, counted with the current SBP's + # digest). Its own banner: it is neither a healing nor a failure, and it must not be + # folded into either. + if !isnothing(ilr_outcome) && !isempty(ilr_outcome.dangers) + append!(warnings, ilr_outcome.dangers) + ilr_banner = """ + ╔════════════════════════════════════════════════════════════════════════════╗ + ║ ⚠️ DANGER — ILR BASIS SELECTION ⚠️ ║ + ╠════════════════════════════════════════════════════════════════════════════╣ + $(join(["║ - $d" for d in ilr_outcome.dangers], "\n")) + ║ ║ + ║ Config ID: $(config.id) ║ + ║ Recorded in the manifest and DOI bundle (see checks["ilr"]). ║ + ╚════════════════════════════════════════════════════════════════════════════╝ + """ + banner = isnothing(banner) ? ilr_banner : banner * "\n" * ilr_banner + is_dangerous_diag = true + @warn "ILR basis selection flagged DANGER" config_id=config.id banner=ilr_banner + end + # Also include config dangerous banner if present if AnalysisConfig.is_dangerous(config) config_banner = AnalysisConfig.danger_banner(config) @@ -1410,7 +1450,13 @@ function prepare_analysis_table( "zero_policy" => string(zero_policy), "pseudocount" => effective_pseudocount, "epsilon" => effective_epsilon, - "scaling" => isnothing(scaling) ? nothing : Scaling.factor_provenance(scaling) + "scaling" => isnothing(scaling) ? nothing : Scaling.factor_provenance(scaling), + # ILR basis provenance (issue #20): basis, SHA-256 of the tree or SBP file, + # dendrogram method, weights, SBP attempt count, pruned tips, balance-id rule. + "ilr" => transform_method != "ilr" ? nothing : + !isnothing(ilr_outcome) ? ilr_outcome.provenance : + OrderedDict{String,Any}("basis" => "default", "source_sha256" => nothing, + "balance_id_rule" => "balance_1..balance_(D-1), Helmert order") ) ) diff --git a/src/analysis/ilr_basis.jl b/src/analysis/ilr_basis.jl new file mode 100644 index 00000000..9e3a82ef --- /dev/null +++ b/src/analysis/ilr_basis.jl @@ -0,0 +1,1318 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# ILR bases beyond the Helmert default: phylogenetic (PhILR), sequential binary partition +# (SBP) and balance dendrogram. Issue #20. +# +# The conditions this code is held to were published before it existed +# (docs/statistics/method-conditions/ilr-bases.md). Where this file and that document +# disagree, the document is right and this file is the bug. +# +# Every basis is reduced to one object, a validated rooted binary tree over the retained +# taxa (`BalanceTree`). Balances are then computed from clade sums in one post-order pass +# per sample -- O(D) time and memory per sample -- so no dense D x (D-1) basis matrix is +# ever built. For 10 000 taxa a dense basis is ~800 MB; this needs a few hundred kB. +# +# The exact-arithmetic facts the engine relies on are machine-checked in Agda +# (proofs/agda/MetaManifold/ILR/*.agda, plan in docs/formal/verification-plan.md): +# internal-count a tree over D tips has D-1 internal nodes, so D-1 balances +# contrast-sum-zero each balance is p-centred: CLR centring does not matter +# contrast-orthogonal, with c = sqrt(r*s/(r+s)) the basis is orthonormal +# contrast-norm +# balance-scale-invariant multiplying a sample by a positive factor changes nothing, so +# counts and proportions give identical balances +# kernel-needs-hypothesis non-positive part weights break injectivity: refused here +# comb-is-helmert the comb tree reproduces the Helmert default exactly +# The Julia tests that check this floating-point code against those facts carry the +# theorem names (test/unit/test_ilr_basis.jl). +# +# What this module does NOT do, on purpose: +# - root, resolve or otherwise edit a tree (refuses unrooted trees and polytomies); +# - drop or add taxa to make an SBP fit (refuses, naming the taxa); +# - substitute any basis for another; +# - call R. `philr`, `compositions` and `robCompositions` are not in renv.lock. +module ILRBasis + +using Statistics +using SHA +using OrderedCollections + +export BalanceTree, ILROutcome, + parse_newick, phylo_balance_tree, + parse_sbp, sbp_balance_tree, sbp_matrix, + variation_condensed, hclust_r, dendrogram_balance_tree, + comb_tree, part_weights, balance_weights, tree_balances, ilr_transform, + VALID_BASES, VALID_PART_WEIGHTS, VALID_BALANCE_WEIGHTS, VALID_DENDROGRAM_METHODS, + SBP_ATTEMPT_DANGER_THRESHOLD, DENDROGRAM_WARN_BYTES, DENDROGRAM_REFUSE_BYTES + +"The three bases implemented here. `default` (Helmert) stays in Execution, unchanged." +const VALID_BASES = ["phylogenetic", "sequential_binary_partition", "balance_dendrogram"] + +"`philr`'s `part.weights`, with `.` written `_`." +const VALID_PART_WEIGHTS = ["uniform", "gm_counts", "anorm", "enorm", "anorm_x_gm_counts", "enorm_x_gm_counts"] + +"`philr`'s `ilr.weights`, with `.` written `_`. Phylogenetic basis only." +const VALID_BALANCE_WEIGHTS = ["uniform", "blw", "blw_sqrt", "mean_descendants"] + +"`ward` is R's `ward.D2`." +const VALID_DENDROGRAM_METHODS = ["ward", "complete", "average"] + +"More distinct SBP matrices than this in one project raises the DANGER flag." +const SBP_ATTEMPT_DANGER_THRESHOLD = 3 + +"Condensed variation matrix size above which a warning is recorded (1 GiB)." +const DENDROGRAM_WARN_BYTES = Int64(1) << 30 + +"Condensed variation matrix size above which the dendrogram basis is refused (2 GiB)." +const DENDROGRAM_REFUSE_BYTES = Int64(2) << 30 + +const CONDITIONS_DOC = "docs/statistics/method-conditions/ilr-bases.md" + +_refuse(msg::AbstractString) = throw(ArgumentError("ILR basis refused: " * msg * " See " * CONDITIONS_DOC * ".")) + +function _name_list(names, limit::Int=10) + v = collect(names) + shown = join(("'" * string(x) * "'" for x in first(v, limit)), ", ") + return length(v) > limit ? shown * ", ... ($(length(v)) in total)" : shown +end + +# ========================================================================================== +# The balance tree +# ========================================================================================== + +""" + BalanceTree + +A rooted, strictly bifurcating tree over `taxa`, stored as arrays with nodes numbered in +preorder (node 1 is the root, and every child has a larger number than its parent, so +iterating `N:-1:1` is a valid post-order). + +- `left[k]`, `right[k]`: children of internal node `k` (0 for a tip). The left child is the + numerator group, the right child the denominator group. +- `tip[k]`: for a tip, the index of its taxon in `taxa` (= its row in the count table); + 0 for an internal node. +- `edge_length[k]`: length of the edge above `k` (`NaN` when absent; the root's is unused). +- `node_label[k]`: label from the source (Newick label, merge id, ...), possibly empty. +- `balance_row[k]`: for an internal node, the output row of its balance; 0 for a tip. +- `balance_ids[row]`: the id of the balance in output row `row`. +""" +struct BalanceTree + taxa::Vector{String} + left::Vector{Int} + right::Vector{Int} + tip::Vector{Int} + edge_length::Vector{Float64} + node_label::Vector{String} + balance_row::Vector{Int} + balance_ids::Vector{String} + + function BalanceTree(taxa, left, right, tip, edge_length, node_label, balance_row, balance_ids) + D = length(taxa) + N = length(left) + D >= 2 || throw(ArgumentError("INTERNAL: a balance tree needs at least 2 taxa, got $D")) + N == 2D - 1 || throw(ArgumentError("INTERNAL: a binary tree over $D tips has $(2D - 1) nodes, got $N")) + all(length(v) == N for v in (right, tip, edge_length, node_label, balance_row)) || + throw(ArgumentError("INTERNAL: balance tree arrays have inconsistent lengths")) + length(balance_ids) == D - 1 || throw(ArgumentError("INTERNAL: $(D - 1) balance ids expected, got $(length(balance_ids))")) + seen_tip = falses(D) + seen_row = falses(D - 1) + for k in 1:N + if tip[k] > 0 + (left[k] == 0 && right[k] == 0) || throw(ArgumentError("INTERNAL: tip node $k has children")) + seen_tip[tip[k]] && throw(ArgumentError("INTERNAL: taxon $(taxa[tip[k]]) appears twice as a tip")) + seen_tip[tip[k]] = true + balance_row[k] == 0 || throw(ArgumentError("INTERNAL: tip node $k has a balance row")) + else + (k < left[k] <= N && k < right[k] <= N) || throw(ArgumentError("INTERNAL: node $k is not in preorder")) + seen_row[balance_row[k]] && throw(ArgumentError("INTERNAL: balance row $(balance_row[k]) used twice")) + seen_row[balance_row[k]] = true + end + end + all(seen_tip) || throw(ArgumentError("INTERNAL: not every taxon is a tip of the balance tree")) + return new(taxa, left, right, tip, edge_length, node_label, balance_row, balance_ids) + end +end + +n_nodes(bt::BalanceTree) = length(bt.left) +n_balances(bt::BalanceTree) = length(bt.balance_ids) +is_tip(bt::BalanceTree, k::Int) = bt.tip[k] > 0 + +"Preorder rows 1..D-1 for the internal nodes of a preorder-numbered tree." +function _preorder_rows(tip::Vector{Int}) + rows = zeros(Int, length(tip)) + r = 0 + for k in eachindex(tip) + if tip[k] == 0 + r += 1 + rows[k] = r + end + end + return rows +end + +""" + comb_tree(taxa) -> BalanceTree + +The comb (caterpillar) tree whose balances are the Helmert default used by Execution: +balance `i` (row `i`) contrasts taxa `1..i` (numerator) with taxon `i+1` (denominator). +Agda: `comb-is-helmert`, `comb-masses`. Used by the tests to show the engine reproduces the +default basis, not by the default path itself (which is left byte-for-byte unchanged). +""" +function comb_tree(taxa::Vector{String}) + D = length(taxa) + D >= 2 || _refuse("ILR needs at least 2 taxa, got $D.") + left = Int[]; right = Int[]; tip = Int[]; row = Int[] + # Stack items: (m, parent, side). m > 0 is the group {1..m}; m < 0 is the tip -m. + stack = Tuple{Int,Int,Int}[(D, 0, 0)] + while !isempty(stack) + (m, parent, side) = pop!(stack) + push!(left, 0); push!(right, 0) + id = length(left) + if parent > 0 + side == 1 ? (left[parent] = id) : (right[parent] = id) + end + if m < 0 || m == 1 + push!(tip, m < 0 ? -m : 1); push!(row, 0) + else + push!(tip, 0); push!(row, m - 1) # group {1..m} is Helmert balance m-1 + push!(stack, (-m, id, 2)) # denominator: taxon m + push!(stack, (m - 1, id, 1)) # numerator: taxa 1..m-1 (popped first) + end + end + N = length(left) + return BalanceTree(copy(taxa), left, right, tip, fill(NaN, N), fill("", N), row, + ["balance_$i" for i in 1:(D - 1)]) +end + +""" + sbp_matrix(bt) -> Matrix{Int8} + +The SBP of a balance tree: `D x (D-1)`, rows in `bt.taxa` order, columns in balance-row +order, entries `1` (numerator), `-1` (denominator), `0`. Dense, so O(D^2): for export and +tests, never used to compute balances. +""" +function sbp_matrix(bt::BalanceTree) + D = length(bt.taxa) + W = zeros(Int8, D, D - 1) + N = n_nodes(bt) + # Tips below each node, collected post-order. + below = Vector{Vector{Int}}(undef, N) + for k in N:-1:1 + if is_tip(bt, k) + below[k] = [bt.tip[k]] + else + l, r = bt.left[k], bt.right[k] + row = bt.balance_row[k] + for t in below[l]; W[t, row] = 1; end + for t in below[r]; W[t, row] = -1; end + below[k] = vcat(below[l], below[r]) + end + end + return W +end + +# ========================================================================================== +# Newick (phylogenetic basis) +# ========================================================================================== + +mutable struct NewickNode + label::String + length::Float64 + children::Vector{NewickNode} + has_label::Bool + has_length::Bool +end +NewickNode() = NewickNode("", NaN, NewickNode[], false, false) + +const _NEWICK_DELIMS = ('(', ')', ',', ':', ';', '[', '\'') + +_newick_error(msg) = _refuse("the phylogenetic tree is not valid Newick: " * msg * ".") + +function _set_label!(nd::NewickNode, label::String, at::Int) + nd.has_label && _newick_error("a second label '$label' at character $at for a node that already has label '$(nd.label)' (labels containing spaces must be quoted)") + nd.label = label + nd.has_label = true + return nd +end + +""" + parse_newick(text) -> NewickNode + +Parses one Newick tree. Quoted labels (`'...'`, with `''` for a quote) and bracketed +comments are supported; labels are kept exactly as written -- underscores are not turned +into spaces and case is not folded, because tips are matched to taxon ids exactly. +Iterative, so a 10 000-taxon caterpillar does not exhaust the stack. +""" +function parse_newick(text::AbstractString) + chars = collect(text) + if !isempty(chars) && chars[1] == '\ufeff' + chars = chars[2:end] + end + n = length(chars) + root = NewickNode() + cur = root + stack = NewickNode[] + finished = false + i = 1 + while i <= n + c = chars[i] + if isspace(c) + i += 1 + elseif c == '[' + j = findnext(==(']'), chars, i) + j === nothing && _newick_error("unterminated comment '[' at character $i") + i = j + 1 + elseif c == '(' + child = NewickNode() + push!(cur.children, child) + push!(stack, cur) + cur = child + i += 1 + elseif c == ',' + isempty(stack) && _newick_error("',' outside any parentheses at character $i") + child = NewickNode() + push!(stack[end].children, child) + cur = child + i += 1 + elseif c == ')' + isempty(stack) && _newick_error("unbalanced ')' at character $i") + cur = pop!(stack) + i += 1 + elseif c == ':' + cur.has_length && _newick_error("a second branch length at character $i") + j = i + 1 + while j <= n && isspace(chars[j]); j += 1; end + k = j + while k <= n && !(chars[k] in _NEWICK_DELIMS) && !isspace(chars[k]); k += 1; end + tok = String(chars[j:(k - 1)]) + v = tryparse(Float64, tok) + (v === nothing || !isfinite(v)) && _newick_error("branch length '$tok' at character $i is not a finite number") + cur.length = v + cur.has_length = true + i = k + elseif c == ';' + isempty(stack) || _newick_error("$(length(stack)) '(' not closed before ';'") + rest = strip(String(chars[(i + 1):end])) + isempty(rest) || _newick_error("text after the terminating ';' (one tree per file)") + finished = true + break + elseif c == '\'' + buf = IOBuffer() + j = i + 1 + closed = false + while j <= n + if chars[j] == '\'' + if j < n && chars[j + 1] == '\'' + write(buf, '\'') + j += 2 + else + closed = true + j += 1 + break + end + else + write(buf, chars[j]) + j += 1 + end + end + closed || _newick_error("unterminated quoted label starting at character $i") + _set_label!(cur, String(take!(buf)), i) + i = j + else + j = i + while j <= n && !(chars[j] in _NEWICK_DELIMS) && !isspace(chars[j]); j += 1; end + _set_label!(cur, String(chars[i:(j - 1)]), i) + i = j + end + end + finished || _newick_error("no terminating ';'") + return root +end + +"All nodes in preorder (children in Newick order). Iterative." +function _newick_preorder(root::NewickNode) + out = NewickNode[] + stack = NewickNode[root] + while !isempty(stack) + nd = pop!(stack) + push!(out, nd) + for c in Iterators.reverse(nd.children) + push!(stack, c) + end + end + return out +end + +function _clade_description(nd::NewickNode) + nd.has_label && !isempty(nd.label) && return "node '$(nd.label)'" + tips = [x.label for x in _newick_preorder(nd) if isempty(x.children)] + return "the node whose clade contains " * _name_list(tips, 5) +end + +""" + phylo_balance_tree(root, taxa) -> (BalanceTree, info) + +Validates a parsed tree against the retained `taxa` (in count-table row order), prunes the +tips that are not retained exactly as `ape::keep.tip` does (collapsing unary nodes and +summing branch lengths), and returns the balance tree plus a record of what was done. + +Refused, with the reason named: duplicate or empty tip labels; a single-tip tree; an +unrooted tree (basal trifurcation, checked *before* pruning, because pruning one basal +branch of an unrooted tree would silently invent a root); retained taxa missing from the +tree; and any multifurcation that survives pruning to the retained taxa. +""" +function phylo_balance_tree(root::NewickNode, taxa::Vector{String}) + D = length(taxa) + D >= 2 || _refuse("ILR needs at least 2 taxa, got $D.") + all_nodes = _newick_preorder(root) + tips = [nd for nd in all_nodes if isempty(nd.children)] + tip_labels = [nd.label for nd in tips] + any(isempty, tip_labels) && _refuse("the phylogenetic tree has $(count(isempty, tip_labels)) tip(s) without a label; every tip must be named by a taxon id.") + if !allunique(tip_labels) + seen = Set{String}(); dups = String[] + for l in tip_labels + (l in seen && !(l in dups)) ? push!(dups, l) : push!(seen, l) + end + _refuse("the phylogenetic tree has duplicate tip labels: $(_name_list(dups)). Tips are matched to taxon ids exactly, so each id may appear once.") + end + length(tips) >= 2 || _refuse("the phylogenetic tree has a single tip.") + + # Unary nodes above the root carry no split: step down to the first real node. + input_unary = count(nd -> length(nd.children) == 1, all_nodes) + top = root + while length(top.children) == 1 + top = top.children[1] + end + nbasal = length(top.children) + if nbasal == 3 + _refuse("the phylogenetic tree is unrooted: its root has 3 children (a basal trifurcation, which is how FastTree, IQ-TREE and RAxML write unrooted trees). The root decides every balance, so MetaManifold does not choose one: root the tree in a phylogenetics tool (outgroup or midpoint) and record that decision there.") + elseif nbasal > 3 + _refuse("the root of the phylogenetic tree has $nbasal children: the tree is unrooted or its root is unresolved. Supply a rooted, strictly bifurcating tree.") + end + + tip_set = Set(tip_labels) + missing_taxa = [t for t in taxa if !(t in tip_set)] + isempty(missing_taxa) || _refuse("$(length(missing_taxa)) retained taxa are not tips of the phylogenetic tree: $(_name_list(missing_taxa)). Tips are matched to taxon ids exactly (no case folding, no underscore/space rewriting).") + + # Prune (ape::keep.tip): children before parents, i.e. reverse preorder. + keep = Set(taxa) + kept = IdDict{NewickNode,Union{Nothing,NewickNode}}() + collapsed = 0 + for nd in Iterators.reverse(all_nodes) + if isempty(nd.children) + kept[nd] = nd.label in keep ? NewickNode(nd.label, nd.length, NewickNode[], true, nd.has_length) : nothing + continue + end + kids = NewickNode[] + for c in nd.children + k = kept[c] + k === nothing || push!(kids, k) + end + if isempty(kids) + kept[nd] = nothing + elseif length(kids) == 1 + k = kids[1] + collapsed += 1 + kept[nd] = NewickNode(k.label, k.length + nd.length, k.children, k.has_label, k.has_length && nd.has_length) + else + kept[nd] = NewickNode(nd.label, nd.length, kids, nd.has_label, nd.has_length) + end + end + pruned_root = kept[root] + pruned_root === nothing && _refuse("INTERNAL: pruning removed every tip.") + pruned_tips = [l for l in tip_labels if !(l in keep)] + + for nd in _newick_preorder(pruned_root) + nc = length(nd.children) + if nc > 2 + _refuse("the phylogenetic tree has a multifurcation (polytomy) with $nc retained children at $(_clade_description(nd)). Resolving it arbitrarily (as ape::multi2di does, randomly) would invent balances no data supports; supply a strictly bifurcating tree.") + end + end + + # Preorder numbering, left child = first Newick child = numerator (philr's convention). + taxon_index = Dict(t => i for (i, t) in enumerate(taxa)) + left = Int[]; right = Int[]; tip = Int[]; len = Float64[]; lab = String[] + stack = Tuple{NewickNode,Int,Int}[(pruned_root, 0, 0)] + while !isempty(stack) + (nd, parent, side) = pop!(stack) + push!(left, 0); push!(right, 0) + push!(len, nd.has_length ? nd.length : NaN) + push!(lab, nd.label) + id = length(left) + if parent > 0 + side == 1 ? (left[parent] = id) : (right[parent] = id) + end + if isempty(nd.children) + push!(tip, taxon_index[nd.label]) + else + push!(tip, 0) + push!(stack, (nd.children[2], id, 2)) + push!(stack, (nd.children[1], id, 1)) + end + end + rows = _preorder_rows(tip) + internal_labels = [lab[k] for k in eachindex(tip) if tip[k] == 0] + use_labels = all(!isempty, internal_labels) && allunique(internal_labels) && + all(l -> tryparse(Float64, l) === nothing, internal_labels) && + !any(l -> haskey(taxon_index, l), internal_labels) + ids = use_labels ? internal_labels : ["n$i" for i in 1:(D - 1)] + rule = use_labels ? "internal node labels (all unique, non-empty and non-numeric)" : + "n1..n$(D - 1) in preorder (internal node labels missing, repeated, numeric support values, or equal to a taxon id)" + bt = BalanceTree(copy(taxa), left, right, tip, len, lab, rows, ids) + info = OrderedDict{String,Any}( + "tree_tips" => length(tips), + "retained_taxa" => D, + "pruned_tips_count" => length(pruned_tips), + "pruned_tips_sample" => first(pruned_tips, 20), + "unary_nodes_collapsed" => collapsed, + "unary_nodes_in_input" => input_unary, + "balance_id_rule" => rule, + "order" => "preorder of the pruned tree, children in Newick order (philr column order)" + ) + return bt, info +end + +phylo_balance_tree(text::AbstractString, taxa::Vector{String}) = phylo_balance_tree(parse_newick(text), taxa) + +# ========================================================================================== +# Sequential binary partition +# ========================================================================================== + +""" + _csv_foreach_row(f, text) + +Minimal RFC 4180 reader: quoted fields, doubled quotes, CRLF, BOM, blank lines skipped. +Calls `f(row)` once per non-blank row, in order. The reader never holds more than one row: +an SBP over 10,000 taxa is 10^8 cells, and materialising them all (as the first version of +this reader did, with `collect(text)` plus one `String` per cell) costs several GB for a +matrix that is 100 MB as `Int8`. `row` is reused for the next row, so `f` copies what it keeps. +""" +function _csv_foreach_row(f, text::AbstractString) + row = String[] + field = IOBuffer() + inq = false + started = false + first_char = true + prev_quote = false # the previous character closed a quote (for doubled quotes) + emit() = (length(row) == 1 && isempty(strip(row[1]))) || f(row) + for c in text + if first_char + first_char = false + c == '\ufeff' && continue + end + if inq + if c == '"' + inq = false + prev_quote = true + continue + end + write(field, c) + elseif c == '"' + if prev_quote + # `""` inside a quoted field: a literal quote, and the field stays quoted + write(field, '"') + inq = true + else + inq = true + started = true + end + elseif c == ',' + push!(row, String(take!(field))) + started = true + elseif c == '\r' + # part of CRLF; ignored + elseif c == '\n' + push!(row, String(take!(field))) + emit() + empty!(row) + started = false + else + write(field, c) + started = true + end + prev_quote = false + end + inq && _refuse("the SBP file has an unterminated quoted field.") + if started + push!(row, String(take!(field))) + emit() + end + return nothing +end + +"All non-blank rows of a small CSV (tests and error paths; `parse_sbp` streams instead)." +function _csv_rows(text::AbstractString) + rows = Vector{String}[] + _csv_foreach_row(r -> push!(rows, copy(r)), text) + return rows +end + +const _SBP_VALUE = r"^[+-]?[01](\.0+)?$" + +function _sbp_value(s::AbstractString) + # the three canonical spellings first: no regex, no Float64 parse (10^8 calls at 10,000 taxa) + s == "0" && return Int8(0) + s == "1" && return Int8(1) + s == "-1" && return Int8(-1) + occursin(_SBP_VALUE, s) || return nothing + v = parse(Float64, s) + return v == 1.0 ? Int8(1) : v == -1.0 ? Int8(-1) : Int8(0) +end + +""" + parse_sbp(text) -> (taxa, balance_ids, W::Matrix{Int8}) + +Reads an SBP CSV: first column taxon ids, every other column a balance whose header is its +id; entries `1`, `-1`, `0` (also written `+1`, `1.0`, `-1.0`). Shape and entries only -- +validity is decided by `sbp_balance_tree`. +""" +function parse_sbp(text::AbstractString) + # Pass 1: shape only (header, row count, first ragged row). Refusals come in the order + # they always have -- too few rows, too few columns, a ragged row, then entries -- so a + # file with several problems reports the same one whichever reader version reads it. + # (Counters are Refs: a variable reassigned inside a closure is boxed and untyped, and + # the entry loop below runs 10^8 times at 10,000 taxa.) + header = String[] + rows_seen = Ref(0) + ragged = Ref{Union{Nothing,Tuple{Int,Int}}}(nothing) + _csv_foreach_row(text) do row + rows_seen[] += 1 + if rows_seen[] == 1 + append!(header, String(strip(h)) for h in row) + elseif ragged[] === nothing && length(row) != length(header) + ragged[] = (rows_seen[], length(row)) + end + end + nrows = rows_seen[] + nrows >= 3 || _refuse("the SBP file needs a header row and at least 2 taxon rows; it has $nrows non-blank row(s).") + ncol = length(header) + ncol >= 2 || _refuse("the SBP header has $ncol column(s); it needs a taxon-id column and at least one balance column.") + bad = ragged[] + bad === nothing || _refuse("SBP row $(bad[1]) has $(bad[2]) fields; the header has $ncol.") + ids = header[2:end] + # Pass 2: taxa and entries, straight into the Int8 matrix. + D = nrows - 1 + taxa = Vector{String}(undef, D) + W = Matrix{Int8}(undef, D, ncol - 1) + seen = Ref(0) + _csv_foreach_row(text) do row + seen[] += 1 + seen[] == 1 && return + i = seen[] - 1 + taxa[i] = String(strip(row[1])) + for j in 1:(ncol - 1) + cell = strip(row[j + 1]) + v = _sbp_value(cell) + v === nothing && _refuse("the SBP entry for taxon '$(taxa[i])' in balance column '$(ids[j])' is '$cell'; entries must be 1 (numerator), -1 (denominator) or 0 (not involved).") + W[i, j] = v + end + end + return taxa, ids, W +end + +function _group_text(names::Vector{String}, G::Vector{Int}) + shown = join((names[g] for g in first(G, 6)), ", ") + return length(G) > 6 ? "{" * shown * ", ... ($(length(G)) taxa)}" : "{" * shown * "}" +end + +""" + sbp_balance_tree(sbp_taxa, ids, W, taxa; removed_by_filtering=String[]) -> BalanceTree + +Validates an SBP by Egozcue & Pawlowsky-Glahn's (2005) definition -- decided by +reconstructing the tree it encodes -- and returns that tree with balance rows in the SBP's +column order and ids from its header. `taxa` are the retained taxa in count-table order; +the SBP must be over exactly those (see the conditions document for why an extra taxon is +refused rather than dropped). `removed_by_filtering` lets the message say which extra taxa +were removed by prevalence/abundance filtering. +""" +function sbp_balance_tree(sbp_taxa::Vector{String}, ids::Vector{String}, W::AbstractMatrix{<:Integer}, + taxa::Vector{String}; removed_by_filtering::Vector{String}=String[]) + D = length(sbp_taxa) + D >= 2 || _refuse("an SBP needs at least 2 taxa, got $D.") + size(W) == (D, length(ids)) || _refuse("INTERNAL: SBP matrix shape $(size(W)) does not match $D taxa and $(length(ids)) ids.") + any(isempty, sbp_taxa) && _refuse("the SBP has a row with an empty taxon id.") + allunique(sbp_taxa) || _refuse("the SBP lists a taxon more than once: $(_name_list(unique(t for t in sbp_taxa if count(==(t), sbp_taxa) > 1))).") + length(ids) == D - 1 || _refuse("an SBP over $D taxa has exactly $(D - 1) balance columns (Egozcue & Pawlowsky-Glahn 2005); this one has $(length(ids)).") + any(isempty, ids) && _refuse("the SBP has a balance column with an empty header; every balance needs an id.") + allunique(ids) || _refuse("the SBP has repeated balance ids: $(_name_list(unique(x for x in ids if count(==(x), ids) > 1))).") + + sbp_set = Set(sbp_taxa) + retained_set = Set(taxa) + missing_taxa = [t for t in taxa if !(t in sbp_set)] + isempty(missing_taxa) || _refuse("$(length(missing_taxa)) retained taxa are not rows of the SBP: $(_name_list(missing_taxa)). Every retained taxon must be assigned in every partition.") + extra = [t for t in sbp_taxa if !(t in retained_set)] + if !isempty(extra) + filtered = [t for t in extra if t in removed_by_filtering] + absent = [t for t in extra if !(t in removed_by_filtering)] + parts = String[] + isempty(filtered) || push!(parts, "$(length(filtered)) removed by prevalence/abundance filtering ($(_name_list(filtered)); lower advanced.min_prevalence / advanced.min_abundance to keep them)") + isempty(absent) || push!(parts, "$(length(absent)) not in the count table ($(_name_list(absent)))") + _refuse("the SBP has rows for taxa that are not in the analysis: " * join(parts, "; ") * ". Removing a taxon always removes a balance that isolates it, so a partition you wrote down would silently disappear; delete those rows and re-balance the SBP, or keep the taxa.") + end + + # Columns must be genuine splits. + for j in eachindex(ids) + col = view(W, :, j) + any(==(1), col) || _refuse("SBP column '$(ids[j])' has no +1 entry: every partition needs a numerator group.") + any(==(-1), col) || _refuse("SBP column '$(ids[j])' has no -1 entry: every partition needs a denominator group.") + end + + # Support set of each column -> column, rejecting repeats. Keyed by (size, hash of the + # ascending row indices) rather than by the index vector itself: a comb-shaped SBP over + # 10 000 taxa has supports totalling ~5e7 indices (~400 MB), while the keys are O(D). + # A key match is only a candidate; it is confirmed exactly before it is trusted. + support_of(j) = [i for i in 1:D if W[i, j] != 0] + function support_key(j) + h = zero(UInt) + c = 0 + @inbounds for i in 1:D + if W[i, j] != 0 + h = hash(i, h) + c += 1 + end + end + return (c, h) + end + same_support(j, k) = all(i -> (W[i, j] != 0) == (W[i, k] != 0), 1:D) + support = Dict{Tuple{Int,UInt},Vector{Int}}() + for j in eachindex(ids) + bucket = get!(() -> Int[], support, support_key(j)) + for k in bucket + same_support(k, j) && _refuse("SBP columns '$(ids[k])' and '$(ids[j])' involve the same taxa $(_group_text(sbp_taxa, support_of(j))); each group is split exactly once.") + end + push!(bucket, j) + end + # The column whose support is exactly `G` (ascending), or 0. The key fixes |support| = |G|, + # so support ⊇ G means support == G. + function column_splitting(G::Vector{Int}) + h = zero(UInt) + for i in G + h = hash(i, h) + end + for k in get(support, (length(G), h), Int[]) + all(i -> W[i, k] != 0, G) && return k + end + return 0 + end + + # Reconstruct the tree top-down in preorder (numerator first). + taxon_index = Dict(t => i for (i, t) in enumerate(taxa)) + used = falses(length(ids)) + unsplit = String[] + left = Int[]; right = Int[]; tip = Int[]; lab = String[]; row = Int[] + # (group of SBP rows, parent node, side, description of where the group came from) + stack = Tuple{Vector{Int},Int,Int,String}[(collect(1:D), 0, 0, "all taxa")] + while !isempty(stack) + (G, parent, side, origin) = pop!(stack) + push!(left, 0); push!(right, 0) + id = length(left) + if parent > 0 + side == 1 ? (left[parent] = id) : (right[parent] = id) + end + if length(G) == 1 + push!(tip, taxon_index[sbp_taxa[G[1]]]); push!(lab, sbp_taxa[G[1]]); push!(row, 0) + continue + end + j = column_splitting(G) + if j == 0 + push!(unsplit, origin == "all taxa" ? + "no column involves every taxon: the first partition of an SBP must split all $D parts" : + "the group $(_group_text(sbp_taxa, G)) ($origin) is never split") + push!(tip, 0); push!(lab, ""); push!(row, 0) + continue + end + used[j] = true + push!(tip, 0); push!(lab, ids[j]); push!(row, j) + P = [i for i in G if W[i, j] == 1] + M = [i for i in G if W[i, j] == -1] + push!(stack, (M, id, 2, "the denominator of '$(ids[j])'")) + push!(stack, (P, id, 1, "the numerator of '$(ids[j])'")) + end + if !isempty(unsplit) || !all(used) + problems = copy(unsplit) + for j in eachindex(ids) + if !used[j] + push!(problems, "column '$(ids[j])' involves $(_group_text(sbp_taxa, support_of(j))), which is not one side of any earlier partition") + end + end + _refuse("the matrix is not a sequential binary partition (Egozcue & Pawlowsky-Glahn 2005: each partition splits one group produced by an earlier partition, and every group of two or more parts is split exactly once): " * join(problems, "; ") * ".") + end + N = length(left) + return BalanceTree(copy(taxa), left, right, tip, fill(NaN, N), lab, row, copy(ids)) +end + +# ========================================================================================== +# Balance dendrogram +# ========================================================================================== + +"Condensed index of (i, j), i < j, as R's hclust.f IOFFST." +@inline _ioffst(n::Int, i::Int, j::Int) = j + (i - 1) * n - (i * (i + 1)) ÷ 2 + +""" + variation_condensed(X) -> Vector{Float64} + +The variation matrix `tau_ij = Var(log(x_i / x_j))` across samples (denominator `n - 1`) of +a positive taxa x samples table, condensed in R's `dist` order (upper triangle by rows). +This is `robCompositions::variation(x, method = "Pairwise")`. +""" +function variation_condensed(X::AbstractMatrix{<:Real}) + D, n = size(X) + n >= 2 || _refuse("the balance dendrogram needs at least 2 samples to estimate a variance of log-ratios; got $n.") + C = Matrix{Float64}(undef, n, D) + for i in 1:D + m = 0.0 + for s in 1:n + C[s, i] = log(Float64(X[i, s])) + m += C[s, i] + end + m /= n + for s in 1:n + C[s, i] -= m + end + end + tau = Vector{Float64}(undef, (D * (D - 1)) ÷ 2) + idx = 0 + for i in 1:(D - 1), j in (i + 1):D + acc = 0.0 + @inbounds for s in 1:n + d = C[s, i] - C[s, j] + acc += d * d + end + idx += 1 + tau[idx] = acc / (n - 1) + end + return tau +end + +""" + hclust_r(diss, n, method) -> (merge::Matrix{Int}, height::Vector{Float64}) + +R's `hclust` for `method` in `ward` (R's `ward.D2`), `complete`, `average`: a line-by-line +port of Murtagh's nearest-neighbour-list algorithm in R's `src/library/stats/src/hclust.f` +(subroutines HCLUST and HCASS2), including its tie-breaking and its `INF = 1e300`. `diss` is +condensed in `dist` order and is not modified. `merge` follows R's convention: singletons +negative, clusters by the step that formed them. +""" +function hclust_r(diss_in::AbstractVector{<:Real}, n::Int, method::AbstractString) + method in VALID_DENDROGRAM_METHODS || _refuse("balance dendrogram method must be one of $(join(VALID_DENDROGRAM_METHODS, ", ")); got '$method'.") + n >= 2 || _refuse("clustering needs at least 2 parts; got $n.") + length(diss_in) == (n * (n - 1)) ÷ 2 || throw(ArgumentError("INTERNAL: condensed dissimilarity has $(length(diss_in)) entries, expected $((n * (n - 1)) ÷ 2)")) + iopt = method == "ward" ? 8 : method == "complete" ? 3 : 4 + is_ward = iopt == 8 + INF = 1.0e300 + diss = Float64.(diss_in) + if is_ward + diss .= diss .* diss + end + membr = ones(Float64, n) + flag = trues(n) + nn = zeros(Int, n) + disnn = zeros(Float64, n) + ia = zeros(Int, n) + ib = zeros(Int, n) + crit = zeros(Float64, n) + im = 0; jj = 0; jm = 0 + for i in 1:(n - 1) + dmin = INF + for j in (i + 1):n + ind = _ioffst(n, i, j) + if dmin > diss[ind] + dmin = diss[ind] + jm = j + end + end + nn[i] = jm + disnn[i] = dmin + end + ncl = n + while ncl > 1 + dmin = INF + for i in 1:(n - 1) + if flag[i] && disnn[i] < dmin + dmin = disnn[i] + im = i + jm = nn[i] + end + end + ncl -= 1 + i2 = min(im, jm) + j2 = max(im, jm) + ia[n - ncl] = i2 + ib[n - ncl] = j2 + if iopt == 8 + dmin = sqrt(dmin) + end + crit[n - ncl] = dmin + flag[j2] = false + dmin = INF + for k in 1:n + if flag[k] && k != i2 + ind1 = i2 < k ? _ioffst(n, i2, k) : _ioffst(n, k, i2) + ind2 = j2 < k ? _ioffst(n, j2, k) : _ioffst(n, k, j2) + d12 = diss[_ioffst(n, i2, j2)] + if is_ward + diss[ind1] = (membr[i2] + membr[k]) * diss[ind1] + + (membr[j2] + membr[k]) * diss[ind2] - membr[k] * d12 + diss[ind1] = diss[ind1] / (membr[i2] + membr[j2] + membr[k]) + elseif iopt == 3 + diss[ind1] = max(diss[ind1], diss[ind2]) + else # iopt == 4, average + diss[ind1] = (membr[i2] * diss[ind1] + membr[j2] * diss[ind2]) / + (membr[i2] + membr[j2]) + end + if i2 < k + if diss[ind1] < dmin + dmin = diss[ind1] + jj = k + end + else + if diss[ind1] < disnn[k] + disnn[k] = diss[ind1] + nn[k] = i2 + end + end + end + end + membr[i2] = membr[i2] + membr[j2] + disnn[i2] = dmin + nn[i2] = jj + for i in 1:(n - 1) + if flag[i] && (nn[i] == i2 || nn[i] == j2) + dmin = INF + for j in (i + 1):n + if flag[j] + ind = _ioffst(n, i, j) + if diss[ind] < dmin + dmin = diss[ind] + jj = j + end + end + end + nn[i] = jj + disnn[i] = dmin + end + end + end + # HCASS2: convert to R's merge convention. + iia = copy(ia) + iib = copy(ib) + for i in 1:(n - 2) + k = min(ia[i], ib[i]) + for j in (i + 1):(n - 1) + ia[j] == k && (iia[j] = -i) + ib[j] == k && (iib[j] = -i) + end + end + for i in 1:(n - 1) + iia[i] = -iia[i] + iib[i] = -iib[i] + end + for i in 1:(n - 1) + if iia[i] > 0 && iib[i] < 0 + k = iia[i]; iia[i] = iib[i]; iib[i] = k + end + if iia[i] > 0 && iib[i] > 0 + k1 = min(iia[i], iib[i]); k2 = max(iia[i], iib[i]) + iia[i] = k1; iib[i] = k2 + end + end + merge = Matrix{Int}(undef, n - 1, 2) + for i in 1:(n - 1) + merge[i, 1] = iia[i] + merge[i, 2] = iib[i] + end + return merge, crit[1:(n - 1)] +end + +""" + dendrogram_balance_tree(merge, taxa) -> BalanceTree + +The balance tree of an R-convention merge matrix, as `compositions::gsi.merge2signary` +reads it: merge step `k` is balance `m`, its second cluster the numerator and its first +the denominator. Balances are emitted in preorder from the root (the last merge). +""" +function dendrogram_balance_tree(merge::AbstractMatrix{<:Integer}, taxa::Vector{String}) + D = length(taxa) + size(merge) == (D - 1, 2) || throw(ArgumentError("INTERNAL: merge matrix for $D taxa must be $(D - 1) x 2, got $(size(merge))")) + left = Int[]; right = Int[]; tip = Int[]; lab = String[] + # code < 0: singleton taxon -code; code > 0: the cluster formed at merge step `code`. + stack = Tuple{Int,Int,Int}[(D - 1, 0, 0)] + while !isempty(stack) + (code, parent, side) = pop!(stack) + push!(left, 0); push!(right, 0) + id = length(left) + if parent > 0 + side == 1 ? (left[parent] = id) : (right[parent] = id) + end + if code < 0 + push!(tip, -code); push!(lab, taxa[-code]) + else + push!(tip, 0); push!(lab, "m$code") + push!(stack, (Int(merge[code, 1]), id, 2)) # denominator: first cluster + push!(stack, (Int(merge[code, 2]), id, 1)) # numerator: second cluster + end + end + rows = _preorder_rows(tip) + ids = Vector{String}(undef, D - 1) + for k in eachindex(tip) + tip[k] == 0 && (ids[rows[k]] = lab[k]) + end + return BalanceTree(copy(taxa), left, right, tip, fill(NaN, length(left)), lab, rows, ids) +end + +# ========================================================================================== +# Weights +# ========================================================================================== + +""" + part_weights(X, kind) -> Vector{Float64} + +`philr`'s part weights computed on the zero-handled taxa x samples table `X`: +`gm_counts` is each taxon's geometric mean across samples, `anorm` the Aitchison norm of its +profile across samples, `enorm` the Euclidean norm of its closed profile; `*_x_gm_counts` +multiply by `gm_counts`. Weights must be finite and positive (Agda: +`kernel-needs-hypothesis` -- with a non-positive weight balances stop being injective). +""" +function part_weights(X::AbstractMatrix{<:Real}, kind::AbstractString) + D, n = size(X) + kind in VALID_PART_WEIGHTS || _refuse("part weights must be one of $(join(VALID_PART_WEIGHTS, ", ")); got '$kind'.") + kind == "uniform" && return ones(Float64, D) + gm = Vector{Float64}(undef, D) + an = Vector{Float64}(undef, D) + en = Vector{Float64}(undef, D) + for i in 1:D + row = Float64.(view(X, i, :)) + L = log.(row) + mL = mean(L) + gm[i] = exp(mL) + an[i] = sqrt(sum(abs2, L .- mL)) + t = sum(row) + en[i] = sqrt(sum(abs2, row ./ t)) + end + p = kind == "gm_counts" ? gm : + kind == "anorm" ? an : + kind == "enorm" ? en : + kind == "anorm_x_gm_counts" ? gm .* an : + gm .* en + for i in 1:D + (isfinite(p[i]) && p[i] > 0) || _refuse("part weight '$kind' is $(p[i]) for taxon $i; part weights must be finite and positive (with one sample, or a taxon that is constant across samples, 'anorm' is 0).") + end + return p +end + +""" + balance_weights(bt, kind) -> (weights by balance row, info) + +`philr`'s `ilr.weights`. Needs branch lengths on every non-root edge. As in `philr`, +zero-length tip edges are first replaced by the smallest non-zero edge length of the tree. +""" +function balance_weights(bt::BalanceTree, kind::AbstractString) + nb = n_balances(bt) + kind in VALID_BALANCE_WEIGHTS || _refuse("balance weights must be one of $(join(VALID_BALANCE_WEIGHTS, ", ")); got '$kind'.") + info = OrderedDict{String,Any}("balance_weights" => kind) + kind == "uniform" && return ones(Float64, nb), info + N = n_nodes(bt) + len = copy(bt.edge_length) + for k in 2:N + isnan(len[k]) && _refuse("balance weights '$kind' need a branch length on every edge; $(is_tip(bt, k) ? "tip '$(bt.taxa[bt.tip[k]])'" : "an internal node") has none.") + len[k] < 0 && _refuse("balance weights '$kind' need non-negative branch lengths; $(is_tip(bt, k) ? "tip '$(bt.taxa[bt.tip[k]])'" : "an internal node") has length $(len[k]).") + end + positive = [len[k] for k in 2:N if len[k] > 0] + isempty(positive) && _refuse("balance weights '$kind' need at least one non-zero branch length.") + min_nonzero = minimum(positive) + replaced = 0 + for k in 2:N + if is_tip(bt, k) && len[k] == 0 + len[k] = min_nonzero + replaced += 1 + end + end + w = zeros(Float64, nb) + if kind == "blw" || kind == "blw_sqrt" + for k in 1:N + is_tip(bt, k) && continue + w[bt.balance_row[k]] = len[bt.left[k]] + len[bt.right[k]] + end + kind == "blw_sqrt" && (w .= sqrt.(w)) + else # mean_descendants + ntips = zeros(Int, N) + sumd = zeros(Float64, N) # sum over descendant tips of the path length from the node + for k in N:-1:1 + if is_tip(bt, k) + ntips[k] = 1 + else + l, r = bt.left[k], bt.right[k] + ntips[k] = ntips[l] + ntips[r] + sumd[k] = sumd[l] + ntips[l] * len[l] + sumd[r] + ntips[r] * len[r] + end + end + for k in 1:N + is_tip(bt, k) && continue + l, r = bt.left[k], bt.right[k] + w[bt.balance_row[k]] = (len[l] + sumd[l] / ntips[l]) + (len[r] + sumd[r] / ntips[r]) + end + end + for row in 1:nb + w[row] > 0 || _refuse("balance weight '$kind' is $(w[row]) for balance '$(bt.balance_ids[row])' (both child edges have length 0), which would erase that balance.") + end + info["zero_length_tip_edges_replaced"] = replaced + info["min_nonzero_edge_length"] = min_nonzero + info["isometric"] = false + return w, info +end + +# ========================================================================================== +# Balances +# ========================================================================================== + +""" + tree_balances(bt, X, p, bw) -> Matrix{Float64} + +Balances `(D-1) x n` of the positive taxa x samples table `X` (rows of `X` indexed by +`bt.tip`), part weights `p`, balance weights `bw` (by balance row): + + b_n = bw_n * sqrt(r s / (r + s)) * (mean_p(log(x/p)) over numerator - over denominator) + +computed from clade sums of `p * log(x/p)` and `p` in one post-order pass per sample. +""" +function tree_balances(bt::BalanceTree, X::AbstractMatrix{<:Real}, p::AbstractVector{<:Real}, + bw::AbstractVector{<:Real}) + D, n = size(X) + D == length(bt.taxa) || throw(ArgumentError("INTERNAL: table has $D taxa, balance tree $(length(bt.taxa))")) + N = n_nodes(bt) + nb = n_balances(bt) + mass = zeros(Float64, N) + for k in N:-1:1 + t = bt.tip[k] + mass[k] = t > 0 ? Float64(p[t]) : mass[bt.left[k]] + mass[bt.right[k]] + end + coef = zeros(Float64, N) + for k in 1:N + if bt.tip[k] == 0 + r = mass[bt.left[k]] + s = mass[bt.right[k]] + coef[k] = sqrt(r * s / (r + s)) * Float64(bw[bt.balance_row[k]]) + end + end + logp = log.(Float64.(p)) + out = Matrix{Float64}(undef, nb, n) + spy = zeros(Float64, N) + @inbounds for j in 1:n + for k in N:-1:1 + t = bt.tip[k] + if t > 0 + spy[k] = p[t] * (log(Float64(X[t, j])) - logp[t]) + else + l = bt.left[k] + r = bt.right[k] + spy[k] = spy[l] + spy[r] + out[bt.balance_row[k], j] = coef[k] * (spy[l] / mass[l] - spy[r] / mass[r]) + end + end + end + return out +end + +# ========================================================================================== +# Orchestration +# ========================================================================================== + +""" + ILROutcome + +- `balances`: `(D-1) x n`, rows in `balance_ids` order. +- `checks`: goes to `diagnostics.checks["ilr"]`. +- `provenance`: goes to the manifest's provenance under `"ilr"`. +- `warnings`: recorded, not fatal. +- `dangers`: reasons the run must carry a DANGER banner (the SBP p-hacking guard). +""" +struct ILROutcome + balances::Matrix{Float64} + balance_ids::Vector{String} + checks::OrderedDict{String,Any} + provenance::OrderedDict{String,Any} + warnings::Vector{String} + dangers::Vector{String} +end + +function _read_source(path, what::AbstractString) + path === nothing && _refuse("the $what is required for this basis and was not given.") + p = String(path) + isempty(strip(p)) && _refuse("the $what is required for this basis and is empty.") + isfile(p) || _refuse("the $what '$p' does not exist or is not a file (relative paths resolve against the working directory, $(pwd())).") + return read(p) +end + +const _SHA256_HEX = r"^[0-9a-f]{64}$" + +function _balance_records(bt::BalanceTree, p::AbstractVector{<:Real}, bw::AbstractVector{<:Real}) + N = n_nodes(bt) + mass = zeros(Float64, N) + for k in N:-1:1 + t = bt.tip[k] + mass[k] = t > 0 ? Float64(p[t]) : mass[bt.left[k]] + mass[bt.right[k]] + end + name(c) = is_tip(bt, c) ? (bt.taxa[bt.tip[c]], "taxon") : (bt.balance_ids[bt.balance_row[c]], "balance") + recs = Vector{OrderedDict{String,Any}}(undef, n_balances(bt)) + for k in 1:N + is_tip(bt, k) && continue + l, r = bt.left[k], bt.right[k] + (nl, kl) = name(l) + (nr, kr) = name(r) + rr, ss = mass[l], mass[r] + row = bt.balance_row[k] + recs[row] = OrderedDict{String,Any}( + "id" => bt.balance_ids[row], + "numerator" => nl, "numerator_kind" => kl, + "denominator" => nr, "denominator_kind" => kr, + "r" => rr, "s" => ss, + "coefficient" => sqrt(rr * ss / (rr + ss)), + "balance_weight" => Float64(bw[row]) + ) + end + return recs +end + +""" + ilr_transform(X, taxa; basis, tree_path, sbp_path, dendrogram_method, part_weights_kind, + balance_weights_kind, sbp_history, removed_by_filtering) -> ILROutcome + +Computes the balances of the zero-handled, filtered, strictly positive taxa x samples table +`X` (rows named by `taxa`) in one of the three bases of `VALID_BASES`. Refuses -- never +substitutes -- when a condition of the conditions document fails. +""" +function ilr_transform(X::AbstractMatrix{<:Real}, taxa::Vector{String}; + basis::AbstractString, + tree_path::Union{AbstractString,Nothing}=nothing, + sbp_path::Union{AbstractString,Nothing}=nothing, + dendrogram_method::Union{AbstractString,Nothing}=nothing, + part_weights_kind::AbstractString="uniform", + balance_weights_kind::AbstractString="uniform", + sbp_history::Vector{String}=String[], + removed_by_filtering::Vector{String}=String[]) + D, n = size(X) + basis in VALID_BASES || _refuse("basis '$basis' is not one of $(join(VALID_BASES, ", ")) (the Helmert 'default' basis is computed by Execution).") + D == length(taxa) || throw(ArgumentError("INTERNAL: $(length(taxa)) taxon ids for a table with $D rows")) + D >= 2 || _refuse("ILR needs at least 2 taxa; got $D.") + n >= 1 || _refuse("ILR needs at least 1 sample; got $n.") + allunique(taxa) || _refuse("taxon ids must be unique.") + for j in 1:n, i in 1:D + v = X[i, j] + (isfinite(v) && v > 0) || _refuse("ILR needs a strictly positive, finite table after zero handling; taxon '$(taxa[i])' has $v in sample column $j. Zero handling (advanced.zero_policy, issue #21) must run first.") + end + part_weights_kind in VALID_PART_WEIGHTS || _refuse("advanced.ilr_part_weights must be one of $(join(VALID_PART_WEIGHTS, ", ")); got '$part_weights_kind'.") + balance_weights_kind in VALID_BALANCE_WEIGHTS || _refuse("advanced.ilr_balance_weights must be one of $(join(VALID_BALANCE_WEIGHTS, ", ")); got '$balance_weights_kind'.") + (balance_weights_kind != "uniform" && basis != "phylogenetic") && + _refuse("advanced.ilr_balance_weights = '$balance_weights_kind' needs branch lengths, which only the phylogenetic basis has.") + + warnings = String[] + dangers = String[] + basis_checks = OrderedDict{String,Any}() + source_sha = nothing + source_path = nothing + sbp_attempts = nothing + method = nothing + rule = "" + + if basis == "phylogenetic" + bytes = _read_source(tree_path, "phylogenetic tree (advanced.ilr_phylo_tree_path)") + source_sha = bytes2hex(sha256(bytes)) + source_path = String(tree_path) + bt, info = phylo_balance_tree(String(copy(bytes)), taxa) + rule = info["balance_id_rule"] + merge!(basis_checks, info) + if info["pruned_tips_count"] > 0 + push!(warnings, "Phylogenetic ILR: $(info["pruned_tips_count"]) tree tips that are not retained taxa were pruned (ape::keep.tip semantics) before building the basis.") + end + elseif basis == "sequential_binary_partition" + bytes = _read_source(sbp_path, "SBP matrix (advanced.ilr_sbp_matrix_path)") + source_sha = bytes2hex(sha256(bytes)) + source_path = String(sbp_path) + (sbp_taxa, ids, W) = parse_sbp(String(copy(bytes))) + bt = sbp_balance_tree(sbp_taxa, ids, W, taxa; removed_by_filtering=removed_by_filtering) + rule = "SBP column headers, in SBP column order" + history = String[] + for h in sbp_history + hl = lowercase(strip(h)) + occursin(_SHA256_HEX, hl) || _refuse("advanced.ilr_sbp_history entries must be SHA-256 hex digests (64 hex characters); got '$h'.") + push!(history, hl) + end + sbp_attempts = length(unique(vcat(history, [source_sha]))) + basis_checks["sbp_attempts"] = sbp_attempts + basis_checks["sbp_attempt_threshold"] = SBP_ATTEMPT_DANGER_THRESHOLD + basis_checks["sbp_history"] = unique(history) + if sbp_attempts > SBP_ATTEMPT_DANGER_THRESHOLD + push!(dangers, "SBP p-hacking guard: $sbp_attempts distinct SBP matrices have been tried in this project (more than $SBP_ATTEMPT_DANGER_THRESHOLD). Trying partitions until one 'works' inflates false discoveries that BH cannot correct; pre-register the SBP and report every partition tried.") + end + else # balance_dendrogram + method = dendrogram_method === nothing ? "" : String(dendrogram_method) + method in VALID_DENDROGRAM_METHODS || _refuse("advanced.ilr_balance_dendrogram_method must be one of $(join(VALID_DENDROGRAM_METHODS, ", ")); got '$(something(dendrogram_method, "nothing"))'.") + need = Int64(8) * ((Int64(D) * (D - 1)) ÷ 2) + need > DENDROGRAM_REFUSE_BYTES && _refuse("the balance dendrogram for $D taxa needs a $(round(need / 2^30, digits=2)) GiB variation matrix, above the $(DENDROGRAM_REFUSE_BYTES ÷ 2^30) GiB limit. Filter taxa or use a phylogenetic or SBP basis.") + need > DENDROGRAM_WARN_BYTES && push!(warnings, "Balance dendrogram: the variation matrix for $D taxa takes $(round(need / 2^30, digits=2)) GiB.") + tau = variation_condensed(X) + all(isfinite, tau) || _refuse("the variation matrix has non-finite entries.") + merge_m, height = hclust_r(tau, D, method) + bt = dendrogram_balance_tree(merge_m, taxa) + rule = "m = the k-th merge (rows of the merge matrix R's hclust() returns), emitted in preorder from the root" + basis_checks["method"] = method + basis_checks["r_equivalent"] = method == "ward" ? "hclust(as.dist(variation(x, method = \"Pairwise\")), \"ward.D2\")" : + "hclust(as.dist(variation(x, method = \"Pairwise\")), \"$method\")" + basis_checks["dissimilarity"] = "variation matrix Var(log(x_i/x_j)), sample variance (n - 1)" + basis_checks["variation_matrix_bytes"] = need + basis_checks["merge"] = [[merge_m[k, 1], merge_m[k, 2]] for k in 1:(D - 1)] + basis_checks["height"] = height + basis_checks["data_derived"] = "the basis is chosen from the same table that is then tested; it uses no sample metadata, so it cannot see group labels, but it is not independent of the data" + end + + p = part_weights(X, part_weights_kind) + bw, bw_info = balance_weights(bt, balance_weights_kind) + if get(bw_info, "zero_length_tip_edges_replaced", 0) > 0 + push!(warnings, "Phylogenetic ILR balance weights: $(bw_info["zero_length_tip_edges_replaced"]) zero-length tip edge(s) replaced by the smallest non-zero edge length $(bw_info["min_nonzero_edge_length"]), as philr does.") + end + B = tree_balances(bt, X, p, bw) + all(isfinite, B) || _refuse("INTERNAL: non-finite balances from a positive table; please report this.") + + data_derived_weights = part_weights_kind != "uniform" + checks = OrderedDict{String,Any}( + "basis" => basis, + "definition" => "b_n = sqrt(r*s/(r+s)) * (mean_p(log(x/p)) over the numerator group - over the denominator group); r, s = part-weight masses; see $CONDITIONS_DOC", + "taxa_in" => D, + "taxa_order" => copy(taxa), + "n_balances" => D - 1, + "balance_ids" => copy(bt.balance_ids), + "balance_id_rule" => rule, + "part_weights" => part_weights_kind, + "part_weight_values" => data_derived_weights ? p : nothing, + "balance_weights" => balance_weights_kind, + "isometric" => balance_weights_kind == "uniform", + "isometric_note" => balance_weights_kind == "uniform" ? + (data_derived_weights ? "orthonormal in the p-weighted Aitchison geometry (weighted ILR, Silverman et al. 2017)" : "orthonormal ILR (Egozcue et al. 2003)") : + "balance weights rescale each balance: effect sizes change, per-balance test statistics do not, and the coordinates are not isometric", + "uses_data_twice" => data_derived_weights || basis == "balance_dendrogram", + "source_path" => source_path, + "source_sha256" => source_sha, + "basis_details" => basis_checks, + "balance_weight_details" => bw_info, + "balances" => _balance_records(bt, p, bw), + "multiple_testing" => "one test per balance; Benjamini-Hochberg across balances is mandatory" + ) + provenance = OrderedDict{String,Any}( + "basis" => basis, + "source_path" => source_path, + "source_sha256" => source_sha, + "tree_sha256" => basis == "phylogenetic" ? source_sha : nothing, + "sbp_sha256" => basis == "sequential_binary_partition" ? source_sha : nothing, + "dendrogram_method" => method, + "part_weights" => part_weights_kind, + "balance_weights" => balance_weights_kind, + "sbp_attempts" => sbp_attempts, + "pruned_tips_count" => basis == "phylogenetic" ? basis_checks["pruned_tips_count"] : nothing, + "balance_id_rule" => rule, + "engine" => "MetaManifold.ILRBasis (clade-sum post-order, no dense basis)", + "conditions" => CONDITIONS_DOC + ) + return ILROutcome(B, copy(bt.balance_ids), checks, provenance, warnings, dangers) +end + +end # module ILRBasis diff --git a/src/server/routes/analysis_config.jl b/src/server/routes/analysis_config.jl index 7f0f1cf4..7672fe0f 100644 --- a/src/server/routes/analysis_config.jl +++ b/src/server/routes/analysis_config.jl @@ -98,6 +98,14 @@ end min_samples_per_group=Int(get(adv_body, "min_samples_per_group", 3)), robust=Bool(get(adv_body, "robust", false)), acknowledgment_token=get(adv_body, "acknowledgment_token", nothing) isa Nothing ? nothing : String(get(adv_body, "acknowledgment_token", nothing)), + # ILR basis inputs (issue #20). Validated, and cross-checked against + # normalization.ilr_basis, by the same constructors as everything else. + ilr_phylo_tree_path=get(adv_body, "ilr_phylo_tree_path", nothing) isa Nothing ? nothing : String(get(adv_body, "ilr_phylo_tree_path", nothing)), + ilr_sbp_matrix_path=get(adv_body, "ilr_sbp_matrix_path", nothing) isa Nothing ? nothing : String(get(adv_body, "ilr_sbp_matrix_path", nothing)), + ilr_balance_dendrogram_method=get(adv_body, "ilr_balance_dendrogram_method", nothing) isa Nothing ? nothing : String(get(adv_body, "ilr_balance_dendrogram_method", nothing)), + ilr_part_weights=String(get(adv_body, "ilr_part_weights", "uniform")), + ilr_balance_weights=String(get(adv_body, "ilr_balance_weights", "uniform")), + ilr_sbp_history=String[String(h) for h in get(adv_body, "ilr_sbp_history", String[])], ) cfg = AnalysisConfig.AnalysisConfigStruct( diff --git a/test/fixtures/ilr/dendrogram_d30/counts.csv b/test/fixtures/ilr/dendrogram_d30/counts.csv new file mode 100644 index 00000000..daab3af7 --- /dev/null +++ b/test/fixtures/ilr/dendrogram_d30/counts.csv @@ -0,0 +1,31 @@ +taxon,S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15 +P01,149.5,115.5,496.5,1769.5,68.5,276.5,164.5,1628.5,430.5,45.5,634.5,626.5,930.5,589.5,65.5 +P02,414.5,988.5,139.5,815.5,480.5,605.5,109.5,339.5,32.5,349.5,0.5,479.5,1001.5,1684.5,60.5 +P03,3.5,266.5,47.5,65.5,1.5,413.5,0.5,689.5,127.5,1.5,130.5,214.5,5.5,1448.5,25.5 +P04,195.5,275.5,127.5,25.5,20.5,947.5,129.5,1158.5,309.5,77.5,66.5,45.5,24.5,1538.5,212.5 +P05,33.5,0.5,2.5,43.5,50.5,0.5,4.5,10.5,26.5,0.5,0.5,2.5,2.5,24.5,0.5 +P06,36.5,601.5,391.5,1059.5,247.5,932.5,593.5,331.5,200.5,2.5,482.5,104.5,13.5,183.5,132.5 +P07,0.5,0.5,19.5,0.5,29.5,0.5,27.5,0.5,95.5,0.5,0.5,0.5,0.5,0.5,0.5 +P08,1.5,166.5,26.5,743.5,3.5,0.5,75.5,24.5,670.5,372.5,79.5,149.5,191.5,2873.5,0.5 +P09,592.5,3244.5,1305.5,3172.5,399.5,58.5,28.5,1667.5,838.5,216.5,88.5,388.5,513.5,614.5,289.5 +P10,0.5,0.5,0.5,0.5,0.5,0.5,22.5,121.5,8.5,0.5,0.5,0.5,0.5,2.5,692.5 +P11,6.5,704.5,410.5,1261.5,118.5,37.5,177.5,1.5,115.5,58.5,158.5,134.5,436.5,309.5,25.5 +P12,8.5,3.5,205.5,513.5,22.5,79.5,9.5,54.5,237.5,335.5,63.5,0.5,697.5,91.5,5.5 +P13,3.5,3.5,0.5,68.5,0.5,4.5,0.5,793.5,509.5,61.5,0.5,1.5,0.5,209.5,9.5 +P14,32.5,58.5,442.5,29.5,240.5,9.5,0.5,5.5,0.5,13.5,12.5,25.5,0.5,0.5,0.5 +P15,99.5,104.5,9.5,83.5,375.5,121.5,0.5,1513.5,3.5,88.5,73.5,14.5,131.5,127.5,489.5 +P16,121.5,2.5,0.5,198.5,1.5,0.5,3.5,0.5,96.5,520.5,17.5,43.5,143.5,0.5,0.5 +P17,0.5,1.5,158.5,207.5,0.5,104.5,55.5,13.5,43.5,8.5,37.5,0.5,0.5,81.5,9.5 +P18,7.5,48.5,233.5,5.5,0.5,0.5,0.5,23.5,0.5,1.5,8.5,15.5,4.5,42.5,0.5 +P19,2.5,206.5,35.5,8.5,0.5,1.5,0.5,3.5,98.5,0.5,0.5,0.5,1.5,0.5,3.5 +P20,269.5,2.5,46.5,772.5,0.5,526.5,52.5,24.5,120.5,77.5,53.5,616.5,61.5,1319.5,5.5 +P21,4.5,3.5,112.5,10.5,118.5,222.5,0.5,0.5,2.5,22.5,61.5,0.5,459.5,60.5,0.5 +P22,280.5,34.5,0.5,81.5,260.5,3.5,1.5,0.5,0.5,0.5,8.5,124.5,180.5,13.5,51.5 +P23,51.5,198.5,228.5,70.5,273.5,75.5,48.5,25.5,67.5,924.5,23.5,21.5,559.5,215.5,110.5 +P24,0.5,2.5,0.5,6.5,69.5,34.5,68.5,0.5,0.5,0.5,0.5,1.5,0.5,0.5,0.5 +P25,47.5,146.5,177.5,1067.5,182.5,0.5,10.5,222.5,2.5,427.5,238.5,184.5,9.5,28.5,184.5 +P26,1.5,859.5,380.5,0.5,0.5,3.5,132.5,2.5,5.5,22.5,26.5,0.5,0.5,1.5,0.5 +P27,1181.5,1574.5,3147.5,3300.5,2128.5,1713.5,476.5,5231.5,701.5,2726.5,1372.5,334.5,1975.5,3543.5,2330.5 +P28,29.5,237.5,314.5,89.5,71.5,982.5,257.5,488.5,215.5,6.5,236.5,564.5,283.5,1435.5,767.5 +P29,1848.5,2313.5,3335.5,1726.5,785.5,1410.5,297.5,2437.5,245.5,908.5,154.5,292.5,585.5,1416.5,527.5 +P30,32.5,35.5,14.5,58.5,41.5,416.5,86.5,238.5,35.5,236.5,0.5,40.5,339.5,101.5,1097.5 diff --git a/test/fixtures/ilr/dendrogram_d30/expected_average.csv b/test/fixtures/ilr/dendrogram_d30/expected_average.csv new file mode 100644 index 00000000..37ded911 --- /dev/null +++ b/test/fixtures/ilr/dendrogram_d30/expected_average.csv @@ -0,0 +1,30 @@ +balance,S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15 +m29,0.6876445908811251,1.2697950482912597,3.9728046982911054,-2.102465051278951,3.493209957438494,6.003143926610474,0.11876666003653327,3.769059937952705,-2.7379817321461126,-4.13418963327627,-0.9276489172256375,-1.9038627295146189,-2.472282331480628,0.9051403578581182,5.302090420080821 +m28,-2.5770162636568834,0.41149340980640514,4.982727480561335,0.011842460220920116,-2.216664098969111,2.4928613987322823,2.5152342466727964,4.790909311650858,4.305516332446043,3.82364379618071,0.8146223864598615,-1.861301595726095,-1.9294117601089895,1.725975881123437,-0.7678321135426 +m27,2.656461572738023,-2.849615664549561,-1.5901980068643737,5.20307223480351,4.074576845184521,2.586964531677597,-1.9562978941027653,3.3316649273074304,2.02447188211224,0.08880582898821608,-0.3133655530120926,3.690992731172352,3.998811933956419,4.030210054437085,3.9262004323075095 +m26,3.881044946960254,4.6356196713571425,5.1134427462637575,5.407191279678371,4.23442446912198,4.672899168092792,-0.22486341913982,-0.4981666767729741,1.659953826995128,3.9743389880896287,3.723260152877065,3.8357921644547583,4.155687263715167,3.6673751829507797,-3.2964788883775387 +m25,-3.0823731244332624,-4.351925034175138,-3.2806099219805542,0.6050837745930033,-8.823923043238432,1.1756482300474862,-2.69820586018131,1.007214826704525,2.3536887206476127,-1.3228098669348176,-0.34540814259658065,-0.7472591058671092,-5.3146229218921865,3.046017174460131,-1.7829354525668657 +m23,-1.1989476363991853,2.7581906860594736,-2.8688121877149975,-1.787939730266634,0.549306144334055,-1.6933517458694378,-4.681745275734929,3.7055287171023776,1.2585677631179566,-0.9829209084674675,-1.7128982703377615,0.021431852215891167,-1.2071445412870234,0.5186981901907006,0.7669651799629775 +m20,-3.705846693481541e-17,-3.063618812875951,-3.220077230739459,0.0316667891034963,-0.7768361992120936,-3.1965328561138926,1.1341058170346905e-16,0.09933919916406833,0.9795645614794548,2.6258919907915583,-3.9347101140688068,-3.509261092032025,-1.695568007970894,-1.3672334428773534,-0.6981877983838729 +m18,4.44750053192437,0.36120826256878,-0.867126658484487,0.9294923369956756,1.8433965934619456e-16,1.1434371039800237,-0.03929381858109356,0.42142392631719555,0.7204631902457105,1.5628557324497905,0.25126383339820524,5.032624274385722,3.4027281900036517,1.9688717530216882,-0.38646476098769156 +m22,2.5449835890967685,4.700288897199972,-0.32156582445891735,0.7017603635044065,0.6707423762374944,-1.368833488122139,3.4800334468001184,3.7934372891618873,0.6455877030335577,-1.470734187001784,-1.4883313156991516,5.912912655063326,-3.4643305912774323,0.12031785335916068,4.205179558695074 +m21,0.06429959259067079,0.32726588260999717,-1.271656803208018,-0.5393340757737842,-1.3484164694062335,4.62534494420209,3.7930240392276393,1.3720439100077861,5.324885790415214,-1.43391883984756,-1.4523662269326711,-0.7859680952113256,4.461936100829528,4.5907576365504195,2.0477439103610955 +m19,5.746178793850574,9.408271458461618,6.910309200721982,5.964619860070998,0.9304614456158601,6.715512125791123,1.2397254360767391,8.128477082979408,3.2658942094158587,7.542035727850675,6.885347286441076,6.548654355825994,8.152200712417516,7.932349611932879,8.562442593839222 +m17,-0.24487753271849974,1.224307394285705,3.330788923650045,1.0588882447179175,-1.543195505791975,0.19797835306543324,4.666657881526748,-1.8981220107170116,3.3270956818172372,-0.39210169410168944,-0.6329362215576352,1.7659380417937502,0.15652246243727014,0.8329411579048766,-1.4933504640555524 +m15,-0.6274491604988673,1.0559719419720661,2.9237094319180357,0.21107563997907564,-0.9585019858878588,-1.8658102510750538,-0.6585688392322032,-0.4589418430832659,1.8142493398039845,-2.43699285226531,6.662535216373833,-0.6615081581340381,-2.0400325119358524,-0.479889661801853,-1.3960553363321502 +m13,2.5171730655527904,-0.5379294151041221,0.03797109087674841,-1.669061337851317,-0.5888631359750213,1.2170061451601786,-1.1612223002052937,5.131470468408682,0.17907406764141104,-0.294853914386902,-0.6684795190330841,-0.49613549215026964,-1.3410172616483662,-0.08307696579126155,1.3003992429234499 +m12,3.7900370291575833,0.6988663024297779,2.6222319435888926,3.425236126729667,5.112828784622534,4.738443954296629,4.905976078520345,4.779923690363316,0.36580010327688806,4.693798815286055,5.331383560655397,5.3488451793879825,4.497657444137468,6.683972858923421,3.530984178476102 +m10,3.072135918400715,2.3179735060628235,0.9338325636886742,4.291254949417017,5.716316359866809,6.462995693508224,5.485048003779669,3.324368325814664,6.012660723022809,4.082572313025659,2.955591475679623,2.337897252917869,3.863279593481532,2.760765591291332,6.011709718239514 +m8,-2.7075271597034485,-2.2083875207471952,-2.119888380516257,-2.014068834436654,-3.147559252639356,1.1005604293977296,0.9935905817098727,-0.20282562733564108,-0.18617395124975272,-5.647424743465323,0.8451277035664471,0.25510323309844635,-2.915061687687819,-0.3617782071607481,-1.1753686401020105 +m7,-1.1078434318268948,-0.06838661888335222,0.9769355397868602,2.6622221201541763,1.427979796241366,-0.34967233332668807,0.7069437138444846,-0.5121853579695567,-0.13456643251419256,-1.8605194911665306,1.589491644476799,1.7240081022341778,1.5847123271596495,-0.9107503323419858,-0.10538221395392383 +m5,0.4886985701926185,-1.0530417304226478,0.00759200107646842,-0.7995190982613288,-1.0313535610804019,-0.47496714030254245,-0.8647287232089138,0.8081268691895523,0.3414097741712263,1.5745863196503251,-0.17929299265921694,1.4197712096416588,2.9710493174355515,1.316238981628054,0.42948672651091296 +m4,-1.1475679293880336,0.5097512233606075,-0.3228645372192186,-2.110157725751904,0.030309216070488823,0.8965332793538396,0.31686101860487276,-0.8514097059096042,-0.4893082860574517,-1.3759662619767377,-0.6978359601971453,-0.07368645975332153,-0.8404034472135459,0.629320636757475,1.7402522520646468 +m3,-0.17747774305115827,-1.1128861336866374,0.021723867353062955,-1.263107323950292,0.5687343467632617,1.9673341092297174,1.643639279898302,-0.767120209147264,-1.1333958224484952,1.56434507781793,0.569433652353038,-0.9606314703666572,0.4515850302137136,0.44437168162724333,0.4972534880484955 +m2,2.7407774535263805,1.8479896321143026,2.165213891189182,2.875886853201647,1.2683825308489256,2.4697909338346067,1.6733094573141376,4.035054504916632,1.4828650177947038,0.43440181926990473,2.4293151032590337,2.18616783015734,0.5335625271491273,1.911782946726057,1.8828231278924115 +m1,0.3164936991543585,0.27211149577736166,0.0410221679204651,-0.45818944998056027,-0.7048810406681306,-0.1375986032449572,-0.3330851228142158,-0.5400352112785182,-0.7424083464071262,-0.7770955927517544,-1.5444591504134193,-0.09487408158582987,-0.859920070892972,-0.6483645210141443,-1.0505409437380453 +m6,-1.800175696920676,-2.35230097818258,-1.6008303215788489,-1.8630670345163587,-1.7317991188431054,-0.2645768017939128,-0.16672173255295,-0.24968268305801988,0.0624323012413489,-0.2761643390595918,-2.954984446706888e-17,-1.7475733410899006,-0.7649341209059407,-1.9863798079060264,2.0492992409501074 +m16,1.7165585737923843,-1.3141005527179002,2.152693055400354,-0.2710730258106921,-0.4802183430161542,-3.4571334842875077,-1.4841475045153334,1.2429210804794604,3.765096534398963,6.083128144566138e-17,-1.7817482484141146e-16,-0.23996290110374535,0.65705027635895,1.5888289834962253,-1.18111229759657e-16 +m11,-2.9731666639763796,-7.009018231905719e-17,1.4524848214923503,-3.157873914856713,-0.3801286363238227,-1.4868193115801323e-16,1.279940071327581,-2.1528024611888306,0.9064978222548669,-6.988538706904063e-17,-2.954984446706888e-17,-1.1380444617808734,-1.1380444617808732,-2.7519325239534753,-1.2597016845906013e-16 +m14,0.2683396848974549,0.6491230876607353,-0.6459207292565362,2.53758313258931,-0.1951401993770895,-2.082032768958586,2.152802461188831,2.6164216691281426,1.1380444617808732,2.4432410315273656,2.0850038102252593,1.3993438218732859,2.0820327689585856,2.8588689681706794,4.179564389215744 +m9,-0.4497119697061789,-2.6687686857104166e-17,-0.426026753187696,-2.7505568293927287,1.174785545527462,0.727733142524124,-2.082032768958586,-3.317283900429376,-3.220077230739459,-1.9106779337253295,-0.02262934816752259,1.5917959333053335e-16,-0.2951206119105731,-0.29252696922979987,-1.6955680079708944 +m24,3.107344796848373,-2.9689324789820177,-2.8074203353628198,-0.9337908367651274,-0.5991300627646442,-1.4868193115801323e-16,-2.1717863337456302,-2.751932523953475,-1.3707128441930374,0.2365645890667456,-1.0702458108173016,-0.8729483916591168,-0.2040376228161916,-6.121022657979586,-1.2597016845906013e-16 diff --git a/test/fixtures/ilr/dendrogram_d30/expected_complete.csv b/test/fixtures/ilr/dendrogram_d30/expected_complete.csv new file mode 100644 index 00000000..84ce029d --- /dev/null +++ b/test/fixtures/ilr/dendrogram_d30/expected_complete.csv @@ -0,0 +1,30 @@ +balance,S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15 +m29,1.3940513202663147,-0.7810704101089591,-3.2240408953072706,1.025368752642196,-2.1177725976400925,6.685124718474491,1.1686490183308982,6.791073503946649,1.540468059739026,-2.2228333407535317,-0.13758813367500408,3.4133100312238667,0.13180230397005707,5.721896579918465,7.4498618004139345 +m28,-3.7804650199748524,-2.2690019317790124,2.500404784673517,-0.41023838587881645,-6.1884880409961065,0.43472636711025175,2.154861421437487,1.8331896813998108,5.0175090723365265,-2.3528364498507512,4.071349874021827,-1.3005127603448816,-5.4170462766895815,1.3773262259111188,-1.9838608206929502 +m25,3.823989567219657,4.843829767231967,7.197280446416321,4.628222128044791,3.7237629413291913,7.1814605574416435,3.6911287884063855,-0.24671679179454434,1.1722027417770262,1.0443281845972614,7.059686897246073,5.941947331889905,5.036442843497496,4.124672221049667,-0.44635630158454326 +m19,-1.4961320110249285,-5.861027462412358,-1.0456772756409163,0.8612665501546168,-4.9959522413439785,-1.1809023028532764,-0.30299301720303906,-4.417044523087562,-1.4129443314756573,1.1627929663592453,-1.9497250650021398,-2.936442533159374,-2.730934847258893,-1.0789512622533193,-3.4468700603943856 +m16,4.44750053192437,0.36120826256878,-0.867126658484487,0.9294923369956756,1.8433965934619456e-16,1.1434371039800237,-0.03929381858109356,0.42142392631719555,0.7204631902457105,1.5628557324497905,0.25126383339820524,5.032624274385722,3.4027281900036517,1.9688717530216882,-0.38646476098769156 +m12,2.739967555885734,-0.02326509257014718,1.6398764861551194,1.2183710958703955,3.444999522759374,0.47077665270253916,5.517774360180566,0.06739429744462595,0.6889806447955421,2.087195656076643,0.6282178101798228,-0.05148555421873329,2.563720054278304,-0.6515486596029989,1.8147926469369935 +m7,-1.1078434318268948,-0.06838661888335222,0.9769355397868602,2.6622221201541763,1.427979796241366,-0.34967233332668807,0.7069437138444846,-0.5121853579695567,-0.13456643251419256,-1.8605194911665306,1.589491644476799,1.7240081022341778,1.5847123271596495,-0.9107503323419858,-0.10538221395392383 +m6,0.4886985701926185,-1.0530417304226478,0.00759200107646842,-0.7995190982613288,-1.0313535610804019,-0.47496714030254245,-0.8647287232089138,0.8081268691895523,0.3414097741712263,1.5745863196503251,-0.17929299265921694,1.4197712096416588,2.9710493174355515,1.316238981628054,0.42948672651091296 +m3,-1.1475679293880336,0.5097512233606075,-0.3228645372192186,-2.110157725751904,0.030309216070488823,0.8965332793538396,0.31686101860487276,-0.8514097059096042,-0.4893082860574517,-1.3759662619767377,-0.6978359601971453,-0.07368645975332153,-0.8404034472135459,0.629320636757475,1.7402522520646468 +m18,1.3759662619767379,1.3759662619767377,2.1205676181609066e-16,3.478951875961536,1.8433965934619456e-16,1.5536723984241865,-2.6917168602050596,1.3269136480610249,2.8944452224724215,3.4027281900036517,-2.954984446706888e-17,0.7768361992120933,-1.896709993021837e-17,3.1313750096251285,-3.032792608959211 +m22,-0.8086513555409286,1.2967374914047758,3.447631714608162,0.5761568132682706,-0.24798044404217498,3.891215639573122,2.0918920006974977,5.976575524516031,2.9973073684154534,5.162924720552216,-1.0130209316190524,-0.5556195642301422,0.5855898139763338,2.623863610084178,1.579651279580034 +m17,0.12573956534800276,0.4765674972600369,1.2695009900068865,0.7851165403943026,-0.7985321402239968,1.1586545758767282,4.303901804600233,-1.364576850931631,1.8555892557696079,0.962015322791524,-4.074671144455146,1.8476291166015042,1.2160550611183565,0.9606286938184464,-0.5239194888388659 +m4,-1.800175696920676,-2.35230097818258,-1.6008303215788489,-1.8630670345163587,-1.7317991188431054,-0.2645768017939128,-0.16672173255295,-0.24968268305801988,0.0624323012413489,-0.2761643390595918,-2.954984446706888e-17,-1.7475733410899006,-0.7649341209059407,-1.9863798079060264,2.0492992409501074 +m27,-4.422594736265097,-9.850026206101253,-7.795592709080357,-2.469746685826575,-1.8630416144889004,-2.8750985920437753,-1.281713851054079,-6.124247338540175,0.32035734695551227,-3.1060418296985897,-3.7562085013612854,-4.824085831831518,-0.6780435994691745,-2.797912685058586,-7.209887653784872 +m26,-1.7316600672909541,-3.117799914386236,1.6211993345681566,-3.8752546864840576,3.655140187303261,3.6815634787247657,-0.7981730191997757,-0.47658077644344154,-3.3859235945753716,-5.638579791712718,-2.8456622379241314,-5.42820938226959,-3.1901653861939057,-1.8511667592870626,0.5732066064472621 +m23,-1.2199476603320927,-1.5439541340022165,-4.337612429400516,-2.8983732317020685,-0.10404977796731264,-4.570328919441808,2.449371539562303,-1.4578546495531675,-0.8897072796135034,-5.649994592368805,-5.289012462339528,0.9888401213333876,-7.1154512610333125,-4.059023897127615,-1.3133813314466716 +m15,1.7165585737923843,-1.3141005527179002,2.152693055400354,-0.2710730258106921,-0.4802183430161542,-3.4571334842875077,-1.4841475045153334,1.2429210804794604,3.765096534398963,6.083128144566138e-17,-1.7817482484141146e-16,-0.23996290110374535,0.65705027635895,1.5888289834962253,-1.18111229759657e-16 +m10,-2.9731666639763796,-7.009018231905719e-17,1.4524848214923503,-3.157873914856713,-0.3801286363238227,-1.4868193115801323e-16,1.279940071327581,-2.1528024611888306,0.9064978222548669,-6.988538706904063e-17,-2.954984446706888e-17,-1.1380444617808734,-1.1380444617808732,-2.7519325239534753,-1.2597016845906013e-16 +m8,-0.4497119697061789,-2.6687686857104166e-17,-0.426026753187696,-2.7505568293927287,1.174785545527462,0.727733142524124,-2.082032768958586,-3.317283900429376,-3.220077230739459,-1.9106779337253295,-0.02262934816752259,1.5917959333053335e-16,-0.2951206119105731,-0.29252696922979987,-1.6955680079708944 +m20,3.107344796848373,-2.9689324789820177,-2.8074203353628198,-0.9337908367651274,-0.5991300627646442,-1.4868193115801323e-16,-2.1717863337456302,-2.751932523953475,-1.3707128441930374,0.2365645890667456,-1.0702458108173016,-0.8729483916591168,-0.2040376228161916,-6.121022657979586,-1.2597016845906013e-16 +m24,1.3007992478165693,3.506930651279408,1.092435200713919,1.7214473143071134,1.2410566938805754,5.456084897790868,5.426792969799323,1.1671526925044253,7.274880285017294,2.539003658589609,0.8061253629367189,-0.4326727004271224,5.211773579388382,3.6687855984902082,3.9858787341488884 +m21,3.9004653931022832,-0.025707049529373534,0.41825632659100925,6.3746686705814435,5.272801224989946,3.025021586988981,-0.8857510087256525,3.576662841440394,3.411816577703883,1.7936530344044517,0.712005845076231,4.457630554501325,5.735837128291455,4.603116530540297,5.099028620489578 +m14,4.0555114571858635,1.531742966845175,3.0932324131761457,4.438979971668608,6.238862545493181,4.507422112964955,5.064253053837673,3.8633776774985216,0.8762187624425537,6.292268719859057,5.191693558966546,5.2743396008024845,5.583264549855646,6.649950394028022,3.9016142599932735 +m9,3.8986725858780162,0.5039565389782239,1.0516486010926798,-0.15354148484487987,1.5275378155922243,1.9216295382828654,-0.3628093194595314,5.678959803273019,0.8951377924001024,2.3821515246223823,-0.08014786088583707,0.2046428964197322,0.4951063811948428,0.9576524619705068,2.5581610455484083 +m5,2.643189011832654,2.113263378469488,2.034141944454555,3.1324479024155982,1.0062644026813536,1.672763186826769,1.029731525754444,4.059992606894258,1.7758571537054257,-0.1118903963852801,2.100569026753468,2.3813492868103996,0.3525192314410012,1.6543223538399787,1.6093915059939552 +m2,0.7462648636805832,-0.4332425650529663,0.7422194222883244,-0.2322400544416435,0.9590020613363138,2.678084030606108,2.1074077932922632,0.6217702991411794,-0.5742874898308793,1.6196792899444428,1.3466388971058862,-0.17696942583750988,0.6036126252449745,1.0562186215032103,1.0964227937878106 +m1,0.3164936991543585,0.27211149577736166,0.0410221679204651,-0.45818944998056027,-0.7048810406681306,-0.1375986032449572,-0.3330851228142158,-0.5400352112785182,-0.7424083464071262,-0.7770955927517544,-1.5444591504134193,-0.09487408158582987,-0.859920070892972,-0.6483645210141443,-1.0505409437380453 +m13,-0.4437036122689036,0.29823995796564673,-0.6338986507072663,0.7793632597459053,-2.6339515017924815,-2.404124359239723,1.2429210804794601,2.1034712855850635,0.6570502763589497,0.5135926897847679,1.0463314626074036,0.6046698449915198,2.0990753570825564,3.4642735489405143,2.4130726252090837 +m11,1.3051965697464925,0.7816794152777643,-0.19389678863873006,3.7252695017462116,4.171857427022746,-1.4868193115801323e-16,2.152802461188831,1.5895241993607312,1.1380444617808732,3.996913429951552,2.3577083656566407,1.7513687504164621,0.5283603705343993,-0.2825598617405599,4.179564389215744 diff --git a/test/fixtures/ilr/dendrogram_d30/expected_ward.csv b/test/fixtures/ilr/dendrogram_d30/expected_ward.csv new file mode 100644 index 00000000..6fdab514 --- /dev/null +++ b/test/fixtures/ilr/dendrogram_d30/expected_ward.csv @@ -0,0 +1,30 @@ +balance,S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15 +m29,1.4628020743150392,2.089062556747034,0.7558457509576904,-0.4956313973489551,5.6883453702183076,-5.29287302543694,-2.30353921480509,-6.299361096056621,-4.479809198520199,3.223984238479126,-2.5962767502009028,-1.6930580450848256,3.4927961949500843,-5.196713872047898,-4.2618667187051225 +m28,2.8475070282806034,-2.2340518191041725,-2.9481515002049754,4.49266329740699,5.567456042876,5.8164674594555335,4.002848398150435,1.5049121557878822,5.041463761286923,3.4618520450500156,0.3333521116457987,2.3693741564931243,8.413469173237091,6.170089035914847,3.9483689989976183 +m27,2.374210719284835,-0.6456667509062047,1.6708151022107893,-2.582111598744462,4.669524884826716,5.262481808341172,-0.9660261531481683,4.396197187571772,-2.935476996444168,-2.582004500304917,-2.662507877997457,-2.6602207621548954,-1.091066223401436,0.09591045027257743,4.388158238433809 +m26,3.1635341482387362,3.889556785723607,-1.6710087144849268,0.6106865531991174,1.831143763246481,-1.1894044460465203,3.36303467068346,3.280258624590658,0.3108235318612328,-1.0818624232343488,-2.6738226504877884,5.457479386217359,-2.466509389997251,0.13371337614948262,4.492828367058234 +m25,7.200909604126662,8.935136163558292,5.4332871648157,6.327161869591189,3.0517857355494518,6.616659508261804,0.7092798202313736,7.82423699780483,2.1348345288390456,8.786313170095815,5.631587280265994,6.68712830785394,9.341563168378624,7.734391662843893,9.449500200670178 +m22,-0.0567853717022758,2.513847030972008,5.811349096723562,1.6392498130848694,0.377047847633404,4.217337870340573,3.3030838080670932,6.747677457568156,4.831525485464671,6.35906660698307,1.047011168260107,-0.09360796164360682,1.0389722061144229,3.461187533557021,1.8843124396219146 +m16,1.3749856232280036,1.6658138277788384,2.866965679742481,1.2828017828712865,1.6991287348970638,-0.7658513410140845,-1.0733080613036956,1.4549548317743797,1.583938713391295,0.7425442903826376,6.613823219574397,-0.35606299410582815,-0.10362078738950592,0.4655400104187515,0.676489469172429 +m10,1.4583572692447477,2.210793074482475,4.420136650813737,2.2750162902727213,0.4959542302679616,0.8701647870916432,4.8893725002016,-0.507538183328785,4.022509228912531,2.0118803146755804,0.6071918755007027,2.1969039296517363,1.568223599217299,1.7508678171832515,-0.08460835103310685 +m4,-0.17747774305115827,-1.1128861336866374,0.021723867353062955,-1.263107323950292,0.5687343467632617,1.9673341092297174,1.643639279898302,-0.767120209147264,-1.1333958224484952,1.56434507781793,0.569433652353038,-0.9606314703666572,0.4515850302137136,0.44437168162724333,0.4972534880484955 +m2,2.7407774535263805,1.8479896321143026,2.165213891189182,2.875886853201647,1.2683825308489256,2.4697909338346067,1.6733094573141376,4.035054504916632,1.4828650177947038,0.43440181926990473,2.4293151032590337,2.18616783015734,0.5335625271491273,1.911782946726057,1.8828231278924115 +m1,0.3164936991543585,0.27211149577736166,0.0410221679204651,-0.45818944998056027,-0.7048810406681306,-0.1375986032449572,-0.3330851228142158,-0.5400352112785182,-0.7424083464071262,-0.7770955927517544,-1.5444591504134193,-0.09487408158582987,-0.859920070892972,-0.6483645210141443,-1.0505409437380453 +m6,-1.800175696920676,-2.35230097818258,-1.6008303215788489,-1.8630670345163587,-1.7317991188431054,-0.2645768017939128,-0.16672173255295,-0.24968268305801988,0.0624323012413489,-0.2761643390595918,-2.954984446706888e-17,-1.7475733410899006,-0.7649341209059407,-1.9863798079060264,2.0492992409501074 +m15,1.7165585737923843,-1.3141005527179002,2.152693055400354,-0.2710730258106921,-0.4802183430161542,-3.4571334842875077,-1.4841475045153334,1.2429210804794604,3.765096534398963,6.083128144566138e-17,-1.7817482484141146e-16,-0.23996290110374535,0.65705027635895,1.5888289834962253,-1.18111229759657e-16 +m9,-2.9731666639763796,-7.009018231905719e-17,1.4524848214923503,-3.157873914856713,-0.3801286363238227,-1.4868193115801323e-16,1.279940071327581,-2.1528024611888306,0.9064978222548669,-6.988538706904063e-17,-2.954984446706888e-17,-1.1380444617808734,-1.1380444617808732,-2.7519325239534753,-1.2597016845906013e-16 +m8,-0.4497119697061789,-2.6687686857104166e-17,-0.426026753187696,-2.7505568293927287,1.174785545527462,0.727733142524124,-2.082032768958586,-3.317283900429376,-3.220077230739459,-1.9106779337253295,-0.02262934816752259,1.5917959333053335e-16,-0.2951206119105731,-0.29252696922979987,-1.6955680079708944 +m21,-2.9894231082720735,4.017123242096353,4.360424473902354,1.2940847550254129,2.1297275190574236,1.762607141449248,2.856748981618054,2.0373355722275726,0.8647548830259373,-1.0289146919182317,1.5175037430020835,0.9648289904647926,0.571958350766206,6.158237067795543,1.6051610929617024 +m12,1.0368568848490372,1.0199965976126606,1.9376360606400298,0.37383887147408446,2.4905361437557842,3.0529251227738397,0.6044617131418586,-1.9750963247413822,-1.2436262948918702,-1.309003344922965,0.48790196192095003,-0.07476395121782449,0.5825856777079872,-1.5756658294833068,2.7802205673424587 +m23,-2.6926913854102197,1.8959821187020007,-0.8975583737116962,-3.61198730661707,-4.409445789011293,0.5923965172471074,1.9444414798292047,-2.5608202431481564,3.619427556542418,-1.9847847471217037,-2.2873506681986497,-4.847932333378391,-1.2757353428765958,-2.4968057267782724,-1.0924988863287182 +m18,-0.36120826256877997,1.0083699169373388,1.6772243909471196,-2.0033843681303654,1.8433965934619456e-16,0.5991300627646442,3.9454647971436927,-0.23792180019586426,-2.040221175009791,2.6917168602050596,2.8074203353628193,1.5917959333053335e-16,-0.7768361992120929,0.7768361992120937,-1.3759662619767379 +m14,-0.4437036122689036,0.29823995796564673,-0.6338986507072663,0.7793632597459053,-2.6339515017924815,-2.404124359239723,1.2429210804794601,2.1034712855850635,0.6570502763589497,0.5135926897847679,1.0463314626074036,0.6046698449915198,2.0990753570825564,3.4642735489405143,2.4130726252090837 +m11,1.3051965697464925,0.7816794152777643,-0.19389678863873006,3.7252695017462116,4.171857427022746,-1.4868193115801323e-16,2.152802461188831,1.5895241993607312,1.1380444617808732,3.996913429951552,2.3577083656566407,1.7513687504164621,0.5283603705343993,-0.2825598617405599,4.179564389215744 +m24,-3.823989567219657,-4.843829767231967,-7.197280446416321,-4.628222128044791,-3.7237629413291913,-7.1814605574416435,-3.6911287884063855,0.24671679179454434,-1.1722027417770262,-1.0443281845972614,-7.059686897246073,-5.941947331889905,-5.036442843497496,-4.124672221049667,0.44635630158454326 +m20,1.3759662619767379,1.3759662619767377,2.1205676181609066e-16,3.478951875961536,1.8433965934619456e-16,1.5536723984241865,-2.6917168602050596,1.3269136480610249,2.8944452224724215,3.4027281900036517,-2.954984446706888e-17,0.7768361992120933,-1.896709993021837e-17,3.1313750096251285,-3.032792608959211 +m19,-1.4961320110249285,-5.861027462412358,-1.0456772756409163,0.8612665501546168,-4.9959522413439785,-1.1809023028532764,-0.30299301720303906,-4.417044523087562,-1.4129443314756573,1.1627929663592453,-1.9497250650021398,-2.936442533159374,-2.730934847258893,-1.0789512622533193,-3.4468700603943856 +m17,4.44750053192437,0.36120826256878,-0.867126658484487,0.9294923369956756,1.8433965934619456e-16,1.1434371039800237,-0.03929381858109356,0.42142392631719555,0.7204631902457105,1.5628557324497905,0.25126383339820524,5.032624274385722,3.4027281900036517,1.9688717530216882,-0.38646476098769156 +m13,2.739967555885734,-0.02326509257014718,1.6398764861551194,1.2183710958703955,3.444999522759374,0.47077665270253916,5.517774360180566,0.06739429744462595,0.6889806447955421,2.087195656076643,0.6282178101798228,-0.05148555421873329,2.563720054278304,-0.6515486596029989,1.8147926469369935 +m7,-1.1078434318268948,-0.06838661888335222,0.9769355397868602,2.6622221201541763,1.427979796241366,-0.34967233332668807,0.7069437138444846,-0.5121853579695567,-0.13456643251419256,-1.8605194911665306,1.589491644476799,1.7240081022341778,1.5847123271596495,-0.9107503323419858,-0.10538221395392383 +m5,0.4886985701926185,-1.0530417304226478,0.00759200107646842,-0.7995190982613288,-1.0313535610804019,-0.47496714030254245,-0.8647287232089138,0.8081268691895523,0.3414097741712263,1.5745863196503251,-0.17929299265921694,1.4197712096416588,2.9710493174355515,1.316238981628054,0.42948672651091296 +m3,-1.1475679293880336,0.5097512233606075,-0.3228645372192186,-2.110157725751904,0.030309216070488823,0.8965332793538396,0.31686101860487276,-0.8514097059096042,-0.4893082860574517,-1.3759662619767377,-0.6978359601971453,-0.07368645975332153,-0.8404034472135459,0.629320636757475,1.7402522520646468 diff --git a/test/fixtures/ilr/dendrogram_d30/merge_average.csv b/test/fixtures/ilr/dendrogram_d30/merge_average.csv new file mode 100644 index 00000000..fc1e3d11 --- /dev/null +++ b/test/fixtures/ilr/dendrogram_d30/merge_average.csv @@ -0,0 +1,30 @@ +step,a,b,height +1,-27,-29,0.5144970470095177 +2,-23,1,1.3585877047938317 +3,-9,2,1.8467304450480162 +4,-1,-28,1.937713254293549 +5,-6,4,2.458942479843958 +6,-2,-30,2.7273716958974044 +7,-4,5,2.8969399425545475 +8,3,7,3.316098644079717 +9,-12,-21,4.019094119325435 +10,-18,8,4.290971734722198 +11,-5,-7,4.512349852811605 +12,-19,10,4.658149562575898 +13,-11,12,4.867126048293894 +14,-14,-25,5.138513642639855 +15,6,13,5.53897431887146 +16,-24,11,5.574380297845472 +17,-15,15,5.943128712608813 +18,-17,-20,6.147426310190485 +19,16,17,6.4258815723889375 +20,-3,-13,7.217701442607791 +21,14,19,7.412984015847898 +22,9,21,7.742044764089445 +23,18,20,7.928799950774673 +24,-8,-16,8.33391316925383 +25,22,23,8.590777001441964 +26,-10,25,9.612281643983211 +27,-26,26,10.213862222469832 +28,-22,27,10.744133270568975 +29,24,28,11.038719082351966 diff --git a/test/fixtures/ilr/dendrogram_d30/merge_complete.csv b/test/fixtures/ilr/dendrogram_d30/merge_complete.csv new file mode 100644 index 00000000..1a766ba2 --- /dev/null +++ b/test/fixtures/ilr/dendrogram_d30/merge_complete.csv @@ -0,0 +1,30 @@ +step,a,b,height +1,-27,-29,0.5144970470095177 +2,-9,1,1.5499282223973405 +3,-1,-28,1.937713254293549 +4,-2,-30,2.7273716958974044 +5,-23,2,2.735003808577754 +6,-6,3,2.8073165202004815 +7,-4,6,3.5687585405925675 +8,-12,-21,4.019094119325435 +9,-11,5,4.4627440596940575 +10,-5,-7,4.512349852811605 +11,-18,-25,5.0271889738889115 +12,-3,7,5.208858911508971 +13,-14,11,5.697885400604592 +14,-19,9,5.8008975882726235 +15,-24,10,6.112502540543297 +16,-17,-20,6.147426310190485 +17,-15,4,6.318262626284885 +18,-10,-13,7.793715214264042 +19,12,16,7.977653786891582 +20,-8,-16,8.33391316925383 +21,-26,14,8.674050430419621 +22,-22,17,9.291481498635909 +23,8,15,10.034133780623344 +24,13,21,10.84381356896416 +25,18,19,12.91829811108009 +26,20,23,13.762831831199474 +27,24,26,15.907316324587825 +28,22,25,18.556368103507282 +29,27,28,19.54022983596864 diff --git a/test/fixtures/ilr/dendrogram_d30/merge_ward.csv b/test/fixtures/ilr/dendrogram_d30/merge_ward.csv new file mode 100644 index 00000000..a0d1732a --- /dev/null +++ b/test/fixtures/ilr/dendrogram_d30/merge_ward.csv @@ -0,0 +1,30 @@ +step,a,b,height +1,-27,-29,0.5144970470095177 +2,-23,1,1.557044002677498 +3,-1,-28,1.937713254293549 +4,-9,2,2.2487388275868754 +5,-6,3,2.640474560177711 +6,-2,-30,2.7273716958974044 +7,-4,5,3.245119605455134 +8,-12,-21,4.019094119325435 +9,-5,-7,4.512349852811605 +10,-15,4,4.580911871172746 +11,-18,-25,5.0271889738889115 +12,-8,-11,5.054887324808968 +13,-3,7,5.059454661873795 +14,-14,11,5.551810876006959 +15,-24,9,5.918667187191351 +16,6,10,5.930546190232309 +17,-17,-20,6.147426310190485 +18,-19,-26,6.1919551011201674 +19,13,17,7.648877830662127 +20,-10,-13,7.793715214264042 +21,-16,12,9.183511497111741 +22,-22,16,9.458592408532372 +23,14,18,11.100169855549918 +24,19,20,11.773109534405972 +25,15,22,12.054651737296032 +26,8,25,12.354192881300301 +27,21,26,16.118121806877912 +28,23,27,16.56912430803839 +29,24,28,20.62013586605127 diff --git a/test/fixtures/ilr/dendrogram_d30/variation.csv b/test/fixtures/ilr/dendrogram_d30/variation.csv new file mode 100644 index 00000000..dc6deeb0 --- /dev/null +++ b/test/fixtures/ilr/dendrogram_d30/variation.csv @@ -0,0 +1,31 @@ +taxon,P01,P02,P03,P04,P05,P06,P07,P08,P09,P10,P11,P12,P13,P14,P15,P16,P17,P18,P19,P20,P21,P22,P23,P24,P25,P26,P27,P28,P29,P30 +P01,0.0,5.449078787348151,4.3293776465594975,3.095459474866409,3.405748458326588,2.8073165202004815,5.983109210334959,6.041871966728427,2.175558718205337,7.3025076796830195,4.007336249447027,4.030502239542097,7.04289692203041,7.36170951973566,5.457377848654711,7.893024052733892,4.858531553648281,3.291154574785732,5.043708632663745,3.5498161064483673,6.870056586888523,7.855117122506905,3.808796460738838,6.106356692757034,5.802812088892122,9.128649320021818,1.6965137476265484,1.937713254293549,2.129642841666309,5.478550723209699 +P02,5.449078787348151,0.0,10.056066221873932,5.3953076800448745,5.2115108890689275,7.942148531636151,9.546705638910348,10.420100920130137,3.7453856457917993,11.312294308280292,6.6506738465839925,8.452670924426736,9.186458574513065,8.614743591331644,6.318262626284885,11.390746719323175,11.62416146928434,6.812445924606638,7.747783064025442,7.722336948696537,9.145590838764672,7.320269481348028,3.397825334174495,6.126368511006954,9.008816717286246,12.989604295693594,3.9031752761800176,6.05506430714256,2.5162068312777106,2.7273716958974044 +P03,4.3293776465594975,10.056066221873932,0.0,4.018278802347694,10.220772309307533,5.208858911508971,13.466710268356005,10.773645473490522,5.549122040278812,11.010610242623065,9.482525412189016,10.809809487186056,7.217701442607791,12.124400536200838,7.564101261486156,18.001636613970206,7.613088106917381,5.459362852635317,7.2158424428729635,7.3439211074666995,12.513766988783077,14.235367795724596,9.576347178787296,12.9134675946448,11.40706945595164,13.079578333120248,5.944763318901607,3.549892546996152,6.104606225155459,10.284037386091429 +P04,3.095459474866409,5.3953076800448745,4.018278802347694,0.0,5.616028534739361,3.5687585405925675,6.539732048281273,9.976899159437528,3.8167888325014068,4.730786865109841,7.378745413598557,6.613481642231034,5.013102856683553,9.438404104258007,5.235217843513211,12.932233314568803,4.744730419087267,5.080836684836883,4.959722320277746,5.446915486628133,9.199224983143276,11.300433055358189,3.881265601245199,6.651379437070284,8.749078674959263,7.104912816798761,2.04242539152787,2.0266018122046665,1.9569166891311482,3.8754882657797487 +P05,3.405748458326588,5.2115108890689275,10.220772309307533,5.616028534739361,0.0,5.348143520094849,4.512349852811605,8.545567751842507,3.34898687111718,9.079384816039648,6.737948859397146,6.41069318844014,7.428076502030012,8.476844436694577,7.703162173850878,8.95384138783845,8.758606174111359,7.383874228688633,7.358999852565296,6.957326896095803,9.682152304508687,7.479945932027127,5.157599104773391,6.112502540543297,7.515584955321435,13.811443649199337,3.7373361207105775,5.956768980788133,3.684329741059505,6.981112825964044 +P06,2.8073165202004815,7.942148531636151,5.208858911508971,3.5687585405925675,5.348143520094849,0.0,5.185700335433206,11.117942902790107,4.385883065572255,7.360089156993071,4.789609795854885,8.225259483593067,10.428740170590041,6.932916401073304,7.789352031772062,14.56250259957835,4.1606691149469865,6.130619758918092,4.826980422565819,7.977653786891582,9.637010512671665,9.463136375600987,6.1464400959001635,3.44437946255034,7.86982433818023,7.441795914982377,3.427643568889235,2.1105684394874342,3.370749062181391,7.746619706236672 +P07,5.983109210334959,9.546705638910348,13.466710268356005,6.539732048281273,4.512349852811605,5.185700335433206,0.0,10.729305723338497,6.357006813440022,8.569540461875397,5.85052165821763,7.30498892147966,12.388755688998046,8.910359912174526,12.817338661983067,11.876171857361653,7.480667701924342,10.16492791591094,6.029300926953671,11.405278862506822,10.034133780623344,13.351473825086535,4.977459315908158,5.0362580551476475,10.50602808655584,7.683751289493166,5.394615682457515,6.20244831556191,5.655706819974975,8.525287939897305 +P08,6.041871966728427,10.420100920130137,10.773645473490522,9.976899159437528,8.545567751842507,11.117942902790107,10.729305723338497,0.0,6.828957332825344,15.815124996561078,5.054887324808968,7.793700083304274,9.406879960119932,15.422155435056155,14.9391349561301,8.33391316925383,9.950171037371913,7.612033990311942,10.2775112277026,7.938560227144168,12.912220870550753,16.037845189524724,7.580281004179733,13.762831831199474,9.77487546134069,11.600095889987358,7.9982586818278705,9.797365270282919,8.911084691710602,12.884898184574215 +P09,2.175558718205337,3.7453856457917993,5.549122040278812,3.8167888325014068,3.34898687111718,4.385883065572255,6.357006813440022,6.828957332825344,0.0,8.032449465701907,4.307498331868901,5.405715925281661,6.076285426909265,5.213831900183359,4.1119704324511845,8.19738294220933,7.798263510396433,2.734398736280311,2.3227361694781297,7.308349496883814,8.29934271697853,6.938254561059978,2.735003808577754,7.655459916013813,3.7276286554355176,8.674050430419621,1.5499282223973405,4.095570625276486,1.2552593041689544,5.156036349758628 +P10,7.3025076796830195,11.312294308280292,11.010610242623065,4.730786865109841,9.079384816039648,7.360089156993071,8.569540461875397,15.815124996561078,8.032449465701907,0.0,12.918574590238421,11.75465544652529,7.793715214264042,17.46202035421924,8.648605660803565,17.684870306282363,8.813553228468304,12.317521559803676,9.239736470276101,12.91829811108009,18.95298130666303,14.967474633436508,8.357005227631303,10.488045607809948,10.271627688678189,13.679975000251655,5.827172730768859,5.075916016111983,7.118191900644553,4.951756989287439 +P11,4.007336249447027,6.6506738465839925,9.482525412189016,7.378745413598557,6.737948859397146,4.789609795854885,5.85052165821763,5.054887324808968,4.307498331868901,12.918574590238421,0.0,5.88969867420151,13.445154703148955,7.757123331597513,10.20237465212224,8.356264676694371,6.83528852446169,5.886656645763805,5.8008975882726235,7.524406613316716,5.825660616241423,7.629057949894725,3.050204953989652,5.595992766851384,7.016704629238356,7.392965395549406,4.110461217091791,4.877106227357637,4.4627440596940575,8.27756282680359 +P12,4.030502239542097,8.452670924426736,10.809809487186056,6.613481642231034,6.41069318844014,8.225259483593067,7.30498892147966,7.793700083304274,5.405715925281661,11.75465544652529,5.88969867420151,0.0,8.192666361351375,10.936854354004469,7.8540802045160385,8.432133743624782,5.003541881611453,8.548620940686778,7.626713817932303,7.1364007203845485,4.019094119325435,14.162275484771905,3.462501195374847,9.490133537646352,9.794371449591706,11.116800852565918,3.1313678683764787,7.551340724503426,4.536692697682499,7.2232709908645765 +P13,7.04289692203041,9.186458574513065,7.217701442607791,5.013102856683553,7.428076502030012,10.428740170590041,12.388755688998046,9.406879960119932,6.076285426909265,7.793715214264042,13.445154703148955,8.192666361351375,0.0,16.906858122034183,8.547491511037737,13.300179019372832,8.294038252498884,11.236098165249071,8.740262657682157,8.46415233621573,17.178675701164472,18.55289872867802,8.878364777048393,14.555472931071561,11.421287720630334,16.696371513133425,6.239926866582798,9.056407547716082,6.944822722826686,7.26569563194709 +P14,7.36170951973566,8.614743591331644,12.124400536200838,9.438404104258007,8.476844436694577,6.932916401073304,8.910359912174526,15.422155435056155,5.213831900183359,17.46202035421924,7.757123331597513,10.936854354004469,16.906858122034183,0.0,8.102910634601463,12.771386840451408,12.349335549392103,5.697885400604592,7.933461445351519,12.923628568191868,8.584259061714,9.899378004314197,6.455784771395696,7.474954651415058,5.138513642639855,8.923271949833792,5.435383752572678,9.769722144082527,4.2792788008869955,12.201609677115986 +P15,5.457377848654711,6.318262626284885,7.564101261486156,5.235217843513211,7.703162173850878,7.789352031772062,12.817338661983067,14.9391349561301,4.1119704324511845,8.648605660803565,10.20237465212224,7.8540802045160385,8.547491511037737,8.102910634601463,0.0,13.321334327442923,11.927543344329639,7.2802405893353574,9.139239276981145,11.424002533269263,8.164087772032868,6.924438958096647,4.659531846251049,9.496244975824938,5.708108100540149,15.23128975841527,2.4844837237893316,5.979211243981739,3.3974470514564348,5.205964097321206 +P16,7.893024052733892,11.390746719323175,18.001636613970206,12.932233314568803,8.95384138783845,14.56250259957835,11.876171857361653,8.33391316925383,8.19738294220933,17.684870306282363,8.356264676694371,8.432133743624782,13.300179019372832,12.771386840451408,13.321334327442923,0.0,15.399255647429204,12.684862654816262,11.514181868949505,8.067745341667568,12.64593682572926,10.552633484226869,7.539629243011198,12.935316512049281,9.760146019554798,15.907316324587825,8.700310448029011,13.86756414863813,9.366292840178653,11.649161307117529 +P17,4.858531553648281,11.62416146928434,7.613088106917381,4.744730419087267,8.758606174111359,4.1606691149469865,7.480667701924342,9.950171037371913,7.798263510396433,8.813553228468304,6.83528852446169,5.003541881611453,8.294038252498884,12.349335549392103,11.927543344329639,15.399255647429204,0.0,8.908031839348844,7.906160697709647,6.147426310190485,10.196102714505347,18.556368103507282,7.281643643000181,8.387750325639376,10.918219015482096,8.335124653718415,4.849807033519204,5.413246734929403,5.706622136752058,9.175106038446854 +P18,3.291154574785732,6.812445924606638,5.459362852635317,5.080836684836883,7.383874228688633,6.130619758918092,10.16492791591094,7.612033990311942,2.734398736280311,12.317521559803676,5.886656645763805,8.548620940686778,11.236098165249071,5.697885400604592,7.2802405893353574,12.684862654816262,8.908031839348844,0.0,5.836676724362969,7.452809657539886,9.07483946894124,10.51068551977068,5.4481620904904675,11.486994621194345,5.0271889738889115,7.051428249241682,3.6895060175329486,4.989442547106331,2.9636534678268154,9.898687991963172 +P19,5.043708632663745,7.747783064025442,7.2158424428729635,4.959722320277746,7.358999852565296,4.826980422565819,6.029300926953671,10.2775112277026,2.3227361694781297,9.239736470276101,5.8008975882726235,7.626713817932303,8.740262657682157,7.933461445351519,9.139239276981145,11.514181868949505,7.906160697709647,5.836676724362969,0.0,10.781474178329324,11.520455903026123,12.25387253601451,5.183015566385302,9.540999389453594,8.814635610493179,6.1919551011201674,4.4169245279338964,5.721380433603036,3.6122012659124434,7.837742726255408 +P20,3.5498161064483673,7.722336948696537,7.3439211074666995,5.446915486628133,6.957326896095803,7.977653786891582,11.405278862506822,7.938560227144168,7.308349496883814,12.91829811108009,7.524406613316716,7.1364007203845485,8.46415233621573,12.923628568191868,11.424002533269263,8.067745341667568,6.147426310190485,7.452809657539886,10.781474178329324,0.0,10.190454210420375,12.115994876790833,7.54565732397005,10.350597947029796,11.737488089718427,13.995042605543217,5.974534358892801,5.963553924687049,5.850006618762535,8.14586592057479 +P21,6.870056586888523,9.145590838764672,12.513766988783077,9.199224983143276,9.682152304508687,9.637010512671665,10.034133780623344,12.912220870550753,8.29934271697853,18.95298130666303,5.825660616241423,4.019094119325435,17.178675701164472,8.584259061714,8.164087772032868,12.64593682572926,10.196102714505347,9.07483946894124,11.520455903026123,10.190454210420375,0.0,10.776839145777314,4.141206766279535,9.00742335653675,12.52527702426097,12.673162638290973,5.0879210376185835,8.399741812787076,5.995080050720828,10.515277661285394 +P22,7.855117122506905,7.320269481348028,14.235367795724596,11.300433055358189,7.479945932027127,9.463136375600987,13.351473825086535,16.037845189524724,6.938254561059978,14.967474633436508,7.629057949894725,14.162275484771905,18.55289872867802,9.899378004314197,6.924438958096647,10.552633484226869,18.556368103507282,10.51068551977068,12.25387253601451,12.115994876790833,10.776839145777314,0.0,7.322810993543617,8.436482086814651,8.555701736961403,19.54022983596864,7.213758469928032,8.236247040676938,7.201604553068073,9.291481498635909 +P23,3.808796460738838,3.397825334174495,9.576347178787296,3.881265601245199,5.157599104773391,6.1464400959001635,4.977459315908158,7.580281004179733,2.735003808577754,8.357005227631303,3.050204953989652,3.462501195374847,8.878364777048393,6.455784771395696,4.659531846251049,7.539629243011198,7.281643643000181,5.4481620904904675,5.183015566385302,7.54565732397005,4.141206766279535,7.322810993543617,0.0,5.2980452543523695,5.584407686031219,7.185981936128839,1.161849881958768,4.4378058586656595,1.5553255276288953,3.088293186227666 +P24,6.106356692757034,6.126368511006954,12.9134675946448,6.651379437070284,6.112502540543297,3.44437946255034,5.0362580551476475,13.762831831199474,7.655459916013813,10.488045607809948,5.595992766851384,9.490133537646352,14.555472931071561,7.474954651415058,9.496244975824938,12.935316512049281,8.387750325639376,11.486994621194345,9.540999389453594,10.350597947029796,9.00742335653675,8.436482086814651,5.2980452543523695,0.0,10.064618819030215,9.955603399373791,5.030511735924239,5.560624963032837,4.774438166811803,6.257818703081955 +P25,5.802812088892122,9.008816717286246,11.40706945595164,8.749078674959263,7.515584955321435,7.86982433818023,10.50602808655584,9.77487546134069,3.7276286554355176,10.271627688678189,7.016704629238356,9.794371449591706,11.421287720630334,5.138513642639855,5.708108100540149,9.760146019554798,10.918219015482096,5.0271889738889115,8.814635610493179,11.737488089718427,12.52527702426097,8.555701736961403,5.584407686031219,10.064618819030215,0.0,10.84381356896416,3.8929783092822485,8.574105189344333,4.554710344821258,9.567300884452928 +P26,9.128649320021818,12.989604295693594,13.079578333120248,7.104912816798761,13.811443649199337,7.441795914982377,7.683751289493166,11.600095889987358,8.674050430419621,13.679975000251655,7.392965395549406,11.116800852565918,16.696371513133425,8.923271949833792,15.23128975841527,15.907316324587825,8.335124653718415,7.051428249241682,6.1919551011201674,13.995042605543217,12.673162638290973,19.54022983596864,7.185981936128839,9.955603399373791,10.84381356896416,0.0,7.405329258821618,9.126076965377697,6.650807612657165,13.191631275499576 +P27,1.6965137476265484,3.9031752761800176,5.944763318901607,2.04242539152787,3.7373361207105775,3.427643568889235,5.394615682457515,7.9982586818278705,1.5499282223973405,5.827172730768859,4.110461217091791,3.1313678683764787,6.239926866582798,5.435383752572678,2.4844837237893316,8.700310448029011,4.849807033519204,3.6895060175329486,4.4169245279338964,5.974534358892801,5.0879210376185835,7.213758469928032,1.161849881958768,5.030511735924239,3.8929783092822485,7.405329258821618,0.0,2.6418546271930445,0.5144970470095177,3.1538052819076845 +P28,1.937713254293549,6.05506430714256,3.549892546996152,2.0266018122046665,5.956768980788133,2.1105684394874342,6.20244831556191,9.797365270282919,4.095570625276486,5.075916016111983,4.877106227357637,7.551340724503426,9.056407547716082,9.769722144082527,5.979211243981739,13.86756414863813,5.413246734929403,4.989442547106331,5.721380433603036,5.963553924687049,8.399741812787076,8.236247040676938,4.4378058586656595,5.560624963032837,8.574105189344333,9.126076965377697,2.6418546271930445,0.0,3.043723118954573,4.536368650499382 +P29,2.129642841666309,2.5162068312777106,6.104606225155459,1.9569166891311482,3.684329741059505,3.370749062181391,5.655706819974975,8.911084691710602,1.2552593041689544,7.118191900644553,4.4627440596940575,4.536692697682499,6.944822722826686,4.2792788008869955,3.3974470514564348,9.366292840178653,5.706622136752058,2.9636534678268154,3.6122012659124434,5.850006618762535,5.995080050720828,7.201604553068073,1.5553255276288953,4.774438166811803,4.554710344821258,6.650807612657165,0.5144970470095177,3.043723118954573,0.0,3.193184077718637 +P30,5.478550723209699,2.7273716958974044,10.284037386091429,3.8754882657797487,6.981112825964044,7.746619706236672,8.525287939897305,12.884898184574215,5.156036349758628,4.951756989287439,8.27756282680359,7.2232709908645765,7.26569563194709,12.201609677115986,5.205964097321206,11.649161307117529,9.175106038446854,9.898687991963172,7.837742726255408,8.14586592057479,10.515277661285394,9.291481498635909,3.088293186227666,6.257818703081955,9.567300884452928,13.191631275499576,3.1538052819076845,4.536368650499382,3.193184077718637,0.0 diff --git a/test/fixtures/ilr/ilr_reference.jl b/test/fixtures/ilr/ilr_reference.jl new file mode 100644 index 00000000..2bb345d6 --- /dev/null +++ b/test/fixtures/ilr/ilr_reference.jl @@ -0,0 +1,557 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Independent reference for the ILR-basis fixtures (issue #20). +# +# This is NOT a copy of src/analysis/ilr_basis.jl and must never call it. It ports the R +# sources the engine is judged against, by their own algorithms, so that agreement is +# evidence rather than tautology: +# +# * philr 1.x (Silverman et al. 2017): phylo2sbp, buildilrBasep, shiftp, miniclo, clrp, +# ilrp, the six part.weights and four ilr.weights, calculate.blw (with its zero-length +# tip-edge replacement) and mean_dist_to_tips. philr builds the DENSE D x (D-1) basis +# and multiplies; the engine walks the tree in O(D) and never forms it. +# * compositions: gsi.merge2signary, gsi.buildilrBase, ilr = clr %*% V. +# * robCompositions: clustCoDa_qmode = hclust(as.dist(variation(x)), method) with the +# classical ("Pairwise") variation matrix. The clustering here is the textbook +# generic agglomeration -- global minimum over all active pairs, Lance-Williams +# update -- not the nearest-neighbour-list algorithm of R's hclust.f that the engine +# ports. The two agree on tie-free data (asserted below); ties are covered by the +# hand-computed cases in test_ilr_basis.jl. +# +# Base Julia only (no LinearAlgebra, no Statistics): nothing here can drift with a +# dependency. philr's own known-answer test is re-checked before anything is computed. +# +# The committed inputs (counts.csv, tree.nwk, sbp.csv) are seeded synthetic data and are +# data, not code. The committed expectations (expected_*.csv, merge_*.csv, variation.csv) +# are recomputed from them by this module on every test run (test/unit/test_ilr_basis.jl) +# and must agree to 1e-10. To rewrite them after a deliberate change: +# +# julia --project=. test/fixtures/ilr/ilr_reference.jl --write + +module ILRReference + +export REFERENCE_DATASETS, reference_expectations, philr_known_answers, read_matrix + +const HERE = @__DIR__ + +# ------------------------------------------------------------------------------------------ +# Newick: an independent recursive-descent parser (quotes, labels, branch lengths) +# ------------------------------------------------------------------------------------------ + +mutable struct RNode + label::Union{String,Nothing} + len::Union{Float64,Nothing} + children::Vector{RNode} +end + +is_tip(n::RNode) = isempty(n.children) + +function parse_newick(text::AbstractString) + s = collect(strip(text)) + (isempty(s) || s[end] != ';') && error("Newick must end with ';'") + s = s[1:end-1] + n = length(s) + pos = Ref(1) + + function read_label() + p = pos[] + if p <= n && s[p] == '\'' + p += 1 + out = Char[] + while true + p <= n || error("unterminated quoted label") + if s[p] == '\'' + if p + 1 <= n && s[p+1] == '\'' + push!(out, '\'') + p += 2 + continue + end + p += 1 + break + end + push!(out, s[p]) + p += 1 + end + pos[] = p + return String(out) + end + q = p + while q <= n && !(s[q] in "():,;[]'") + q += 1 + end + tok = strip(String(s[p:q-1])) + pos[] = q + return isempty(tok) ? nothing : String(tok) + end + + function read_length() + p = pos[] + (p <= n && s[p] == ':') || return nothing + p += 1 + while p <= n && isspace(s[p]) + p += 1 + end + q = p + while q <= n && (isdigit(s[q]) || s[q] in "+-.eE") + q += 1 + end + pos[] = q + return parse(Float64, String(s[p:q-1])) + end + + function subtree() + node = RNode(nothing, nothing, RNode[]) + if s[pos[]] == '(' + pos[] += 1 + push!(node.children, subtree()) + while s[pos[]] == ',' + pos[] += 1 + push!(node.children, subtree()) + end + s[pos[]] == ')' || error("expected ')' at character $(pos[])") + pos[] += 1 + end + node.label = read_label() + node.len = read_length() + return node + end + + root = subtree() + pos[] == n + 1 || error("trailing input at character $(pos[])") + return root +end + +tips(n::RNode) = is_tip(n) ? String[something(n.label)] : reduce(vcat, (tips(c) for c in n.children)) + +"ape node numbering for a read.tree'd tree: internal nodes in preorder, Newick child order." +function internal_preorder(n::RNode) + is_tip(n) && return RNode[] + out = RNode[n] + for c in n.children + append!(out, internal_preorder(c)) + end + return out +end + +"ape::keep.tip semantics: prune, collapse unary nodes, summing their lengths." +function keep_tips(n::RNode, keep::Set{String}) + if is_tip(n) + return n.label in keep ? RNode(n.label, n.len, RNode[]) : nothing + end + kids = RNode[] + for c in n.children + k = keep_tips(c, keep) + k === nothing || push!(kids, k) + end + isempty(kids) && return nothing + if length(kids) == 1 + k = kids[1] + if n.len !== nothing || k.len !== nothing + k.len = something(k.len, 0.0) + something(n.len, 0.0) + end + return k + end + return RNode(n.label, n.len, kids) +end + +# ------------------------------------------------------------------------------------------ +# philr port (R/phylo2sbp.R, R/weighted_ILR.R, R/philr.R, R/branch_length_calculations.R). +# x is samples x taxa, as in philr. +# ------------------------------------------------------------------------------------------ + +function phylo2sbp(tree::RNode) + tl = tips(tree) + idx = Dict(t => i for (i, t) in enumerate(tl)) + nodes = internal_preorder(tree) + sbp = zeros(Float64, length(tl), length(nodes)) + for (j, nd) in enumerate(nodes) + length(nd.children) == 2 || error("phylo2sbp needs a bifurcating tree") + for t in tips(nd.children[1]) + sbp[idx[t], j] = 1.0 + end + for t in tips(nd.children[2]) + sbp[idx[t], j] = -1.0 + end + end + return sbp, tl, [nd.label for nd in nodes] +end + +miniclo(c::AbstractMatrix) = c ./ sum(c; dims=2) +shiftp(x::AbstractMatrix, p::AbstractVector) = x ./ reshape(p, 1, :) + +"g.rowMeans: the p-weighted geometric mean of each row." +function g_rowmeans(y::AbstractMatrix, p::AbstractVector) + return [exp(sum(log(y[i, j]) * p[j] for j in axes(y, 2)) / sum(p)) for i in axes(y, 1)] +end + +function clrp(y::AbstractMatrix, p::AbstractVector) + y = miniclo(y) + g = g_rowmeans(y, p) + return log.(y ./ g) +end + +function normp(y::AbstractVector, p::AbstractVector) + c = clrp(reshape(collect(y), 1, :), p) + return sqrt(sum(p[j] * c[1, j]^2 for j in eachindex(p))) +end + +"philr's buildilrBasep: the dense basis, +c/r on the numerator and -c/s on the denominator." +function build_ilr_base(W::AbstractMatrix, p::AbstractVector) + D, K = size(W) + V = zeros(Float64, D, K) + for k in 1:K + npos = sum(p[i] for i in 1:D if W[i, k] > 0) + nneg = sum(p[i] for i in 1:D if W[i, k] < 0) + c = sqrt(npos * nneg / (npos + nneg)) + for i in 1:D + V[i, k] = W[i, k] > 0 ? c / npos : W[i, k] < 0 ? -c / nneg : 0.0 + end + end + return V +end + +"ilrp = clrp(y, p) %*% diag(p) %*% V." +function ilrp(y::AbstractMatrix, p::AbstractVector, V::AbstractMatrix) + c = clrp(y, p) + n, D = size(c) + K = size(V, 2) + return [sum(c[s, i] * p[i] * V[i, k] for i in 1:D) for s in 1:n, k in 1:K] +end + +function part_weights(x::AbstractMatrix, kind::AbstractString) + n, D = size(x) + gm = [exp(sum(log(x[s, j]) for s in 1:n) / n) for j in 1:D] + closed_t = miniclo(permutedims(x)) # taxa x samples, each taxon closed over samples + anorm = [normp(closed_t[j, :], ones(n)) for j in 1:D] + enorm = [sqrt(sum(closed_t[j, s]^2 for s in 1:n)) for j in 1:D] + kind == "uniform" && return ones(D) + kind == "gm_counts" && return gm + kind == "anorm" && return anorm + kind == "anorm_x_gm_counts" && return gm .* anorm + kind == "enorm" && return enorm + kind == "enorm_x_gm_counts" && return gm .* enorm + error("unknown part weights '$kind'") +end + +"calculate.blw's rule: zero-length tip edges become the smallest non-zero edge length." +function replace_zero_tip_edges(tree::RNode) + lengths = Float64[] + function collect_lengths(m::RNode, is_root::Bool) + is_root || m.len === nothing || push!(lengths, m.len) + for c in m.children + collect_lengths(c, false) + end + end + collect_lengths(tree, true) + min_nonzero = minimum(l for l in lengths if l > 0) + function rebuild(m::RNode, is_root::Bool) + out = RNode(m.label, m.len, RNode[rebuild(c, false) for c in m.children]) + if !is_root && is_tip(m) && m.len == 0.0 + out.len = min_nonzero + end + return out + end + return rebuild(tree, true) +end + +"Mean path length from a node to its descendant tips (0 for a tip)." +function mean_dist_to_tips(n::RNode) + is_tip(n) && return 0.0 + dists = Float64[] + function walk(m::RNode, acc::Float64) + if is_tip(m) + push!(dists, acc) + return + end + for c in m.children + walk(c, acc + something(c.len)) + end + end + walk(n, 0.0) + return sum(dists) / length(dists) +end + +function ilr_weights(tree::RNode, kind::AbstractString) + nodes = internal_preorder(tree) + kind == "uniform" && return ones(length(nodes)) + nodes2 = internal_preorder(replace_zero_tip_edges(tree)) + if kind == "blw" || kind == "blw_sqrt" + w = [sum(something(c.len) for c in nd.children) for nd in nodes2] + return kind == "blw_sqrt" ? sqrt.(w) : w + end + if kind == "mean_descendants" + return [sum(something(c.len) + mean_dist_to_tips(c) for c in nd.children) for nd in nodes2] + end + error("unknown ilr weights '$kind'") +end + +"philr.data.frame(x, tree, part.weights, ilr.weights, pseudocount = 0); x is samples x taxa." +function philr(x::AbstractMatrix, taxa::Vector{String}, tree::RNode, part::AbstractString, ilrw::AbstractString) + all(>(0), x) || error("philr reference needs a strictly positive table") + sbp, tl, labels = phylo2sbp(tree) + order = [findfirst(==(t), tl) for t in taxa] # sbp <- sbp[colnames(x), ] + sbp = sbp[order, :] + p = part_weights(x, part) + y = shiftp(miniclo(x), p) + V = build_ilr_base(sbp, p) + out = ilrp(y, p, V) + w = ilr_weights(tree, ilrw) + return out .* reshape(w, 1, :), labels +end + +# ------------------------------------------------------------------------------------------ +# compositions / robCompositions +# ------------------------------------------------------------------------------------------ + +"robCompositions::variation(x, method = \"Pairwise\"): var(log(x_i / x_j)), denominator n - 1." +function variation(x::AbstractMatrix) + n, D = size(x) + L = log.(x) + T = zeros(Float64, D, D) + for i in 1:D, j in (i+1):D + d = [L[s, i] - L[s, j] for s in 1:n] + m = sum(d) / n + T[i, j] = T[j, i] = sum((v - m)^2 for v in d) / (n - 1) + end + return T +end + +""" + generic_hclust(diss, method) -> (merge, height) + +Textbook agglomerative clustering on a full dissimilarity matrix: at every step merge the +active pair with the globally smallest dissimilarity, then update by Lance-Williams; ward +is R's ward.D2 (Lance-Williams on squared dissimilarities, height = square root). Output in +the conventions of the merge matrix R's hclust() returns (singletons negative, clusters by merge step; a singleton +before a cluster, the lower-numbered singleton or the earlier cluster first). Refuses data +whose minimum is not unique: this reference is for tie-free fixtures only. +""" +function generic_hclust(diss::AbstractMatrix, method::AbstractString) + D = size(diss, 1) + ward = method == "ward" + d = ward ? diss .^ 2 : copy(float.(diss)) + code = [-i for i in 1:D] # R code of the cluster held at each position + sz = ones(Int, D) + active = trues(D) + merge = zeros(Int, D - 1, 2) + height = zeros(Float64, D - 1) + for step in 1:(D-1) + best = Inf + second = Inf + bi = bj = 0 + for i in 1:D, j in (i+1):D + (active[i] && active[j]) || continue + v = d[i, j] + if v < best + second = best + best = v + bi, bj = i, j + elseif v < second + second = v + end + end + second - best > 1e-12 * max(1.0, abs(best)) || + error("tied minimum at step $step: the generic reference is for tie-free data") + a, b = code[bi], code[bj] + if a > 0 && b < 0 + a, b = b, a + elseif a < 0 && b < 0 + a, b = max(a, b), min(a, b) + elseif a > 0 && b > 0 + a, b = min(a, b), max(a, b) + end + merge[step, 1] = a + merge[step, 2] = b + height[step] = ward ? sqrt(best) : best + ni, nj = sz[bi], sz[bj] + for k in 1:D + (active[k] && k != bi && k != bj) || continue + nk = sz[k] + dki, dkj = d[k, bi], d[k, bj] + new = if ward + ((ni + nk) * dki + (nj + nk) * dkj - nk * best) / (ni + nj + nk) + elseif method == "complete" + max(dki, dkj) + elseif method == "average" + (ni * dki + nj * dkj) / (ni + nj) + else + error("unknown method '$method'") + end + d[k, bi] = d[bi, k] = new + end + active[bj] = false + sz[bi] = ni + nj + code[bi] = step + end + return merge, height +end + +"compositions::gsi.merge2signary, returned as D x (D-1): column k is merge step k." +function merge2signary(M::AbstractMatrix{<:Integer}) + nm = size(M, 1) + V = zeros(Float64, nm, nm + 1) + for i in 1:nm, j in 1:2 + weight = j == 1 ? -1.0 : 1.0 + k = M[i, j] + if k < 0 + V[i, -k] = weight + else + take = V[k, :] .!= 0 + V[i, take] .= weight + end + end + return permutedims(V) +end + +"Merge steps in preorder from the root, the numerator (second) child first." +function signary_preorder(M::AbstractMatrix{<:Integer}) + order = Int[] + function rec(step::Int) + push!(order, step) + for c in (M[step, 2], M[step, 1]) + c > 0 && rec(c) + end + end + rec(size(M, 1)) + return order +end + +"compositions::ilr(x, gsi.buildilrBase(W)) = clr(x) %*% V; x is samples x taxa." +function ilr_uniform(x::AbstractMatrix, W::AbstractMatrix) + n, D = size(x) + L = log.(x) + clr = L .- sum(L; dims=2) ./ D + V = build_ilr_base(W, ones(D)) + return [sum(clr[s, i] * V[i, k] for i in 1:D) for s in 1:n, k in 1:size(V, 2)] +end + +# ------------------------------------------------------------------------------------------ +# philr's own known-answer test, re-checked before anything is computed +# ------------------------------------------------------------------------------------------ + +function philr_known_answers() + sbp = [1.0 0.0; -1.0 1.0; -1.0 -1.0] + p = [1.0, 1.0, 0.5] + V = build_ilr_base(sbp, p) + expected = [0.7745967 0.0; -0.5163978 0.5773503; -0.5163978 -1.1547005] + maximum(abs.(V .- expected)) < 1e-7 || error("philr known answer (buildilrBasep) failed: $V") + shiftp([1.0 2.0 3.0], [1.0, 0.1, 0.5]) ≈ [1.0 20.0 6.0] || error("philr known answer (shiftp) failed") + for a in 1:2, b in 1:2 # orthonormal under the p-weighted inner product + g = sum(V[i, a] * p[i] * V[i, b] for i in 1:3) + abs(g - (a == b ? 1.0 : 0.0)) < 1e-12 || error("basis not p-orthonormal: G[$a,$b] = $g") + end + return true +end + +# ------------------------------------------------------------------------------------------ +# Fixture I/O and the expectations +# ------------------------------------------------------------------------------------------ + +"rows x cols matrix from a fixture CSV (first column row ids; header corner + column ids)." +function read_matrix(path::AbstractString) + lines = [String(strip(l)) for l in readlines(path) if !isempty(strip(l))] + cols = map(String, split(lines[1], ',')[2:end]) + rows = String[] + M = zeros(Float64, length(lines) - 1, length(cols)) + for (r, l) in enumerate(lines[2:end]) + f = split(l, ',') + push!(rows, String(f[1])) + for c in eachindex(cols) + M[r, c] = parse(Float64, f[c+1]) + end + end + return rows, cols, M +end + +"samples x taxa table, taxa and sample ids from a fixture counts.csv (taxa x samples)." +function read_counts(dir::AbstractString) + taxa, samples, M = read_matrix(joinpath(dir, "counts.csv")) + return permutedims(M), taxa, samples +end + +# (dataset, [(part.weights, ilr.weights), ...]) -- the combinations committed as fixtures +const REFERENCE_DATASETS = [ + ("philr_d8", [("uniform", "uniform"), ("gm_counts", "blw"), ("enorm_x_gm_counts", "blw_sqrt")]), + ("philr_d25", [("uniform", "uniform"), ("anorm", "blw"), ("anorm_x_gm_counts", "mean_descendants")]), + ("philr_d60_pruned", [("uniform", "uniform"), ("enorm", "uniform"), ("gm_counts", "mean_descendants")]), +] + +const DENDROGRAM_METHODS = ("ward", "complete", "average") + +"Balance ids as the fixtures name them: node labels when all present, unique and non-numeric, else n1..n(D-1)." +function balance_names(labels) + usable = all(l -> l !== nothing && !isempty(l) && tryparse(Float64, l) === nothing, labels) && + allunique(labels) + return usable ? String[l for l in labels] : ["n$k" for k in eachindex(labels)] +end + +""" + reference_expectations(root = HERE) -> Vector{NamedTuple} + +Every committed expectation recomputed from the committed inputs: +`(file, corner, rows, cols, values)` with `file` relative to `root`. merge_*.csv entries +have rows = steps, cols = ["a", "b", "height"]. +""" +function reference_expectations(root::AbstractString=HERE) + philr_known_answers() + out = NamedTuple[] + for (name, combos) in REFERENCE_DATASETS + dir = joinpath(root, name) + x, taxa, samples = read_counts(dir) + full = parse_newick(read(joinpath(dir, "tree.nwk"), String)) + tree = keep_tips(full, Set(taxa)) + for (part, ilrw) in combos + vals, labels = philr(x, taxa, tree, part, ilrw) + push!(out, (file=joinpath(name, "expected_$(part)__$(ilrw).csv"), corner="balance", + rows=balance_names(labels), cols=samples, values=permutedims(vals))) + end + end + + # SBP: rows of sbp.csv are deliberately in a different order from the table + dir = joinpath(root, "sbp_d12") + x, taxa, samples = read_counts(dir) + sbp_taxa, bal, Wf = read_matrix(joinpath(dir, "sbp.csv")) + W = Wf[[findfirst(==(t), sbp_taxa) for t in taxa], :] + push!(out, (file=joinpath("sbp_d12", "expected.csv"), corner="balance", rows=bal, cols=samples, + values=permutedims(ilr_uniform(x, W)))) + + # Balance dendrogram + dir = joinpath(root, "dendrogram_d30") + x, taxa, samples = read_counts(dir) + T = variation(x) + push!(out, (file=joinpath("dendrogram_d30", "variation.csv"), corner="taxon", rows=taxa, cols=taxa, values=T)) + for method in DENDROGRAM_METHODS + M, h = generic_hclust(T, method) + all(diff(h) .>= 0) || error("$method heights are not monotone") + steps = [string(k) for k in 1:size(M, 1)] + push!(out, (file=joinpath("dendrogram_d30", "merge_$method.csv"), corner="step", rows=steps, + cols=["a", "b", "height"], values=hcat(float(M), h))) + order = signary_preorder(M) + vals = ilr_uniform(x, merge2signary(M)[:, order]) + push!(out, (file=joinpath("dendrogram_d30", "expected_$method.csv"), corner="balance", + rows=["m$k" for k in order], cols=samples, values=permutedims(vals))) + end + return out +end + +function write_expectation(root::AbstractString, e) + open(joinpath(root, e.file), "w") do io + println(io, join(vcat(e.corner, e.cols), ",")) + integer_cols = e.corner == "step" ? (1, 2) : () + for (r, id) in enumerate(e.rows) + cells = [c in integer_cols ? string(Int(e.values[r, c])) : repr(e.values[r, c]) for c in eachindex(e.cols)] + println(io, join(vcat(id, cells), ",")) + end + end +end + +if abspath(PROGRAM_FILE) == @__FILE__ + "--write" in ARGS || (println("usage: julia --project=. test/fixtures/ilr/ilr_reference.jl --write"); exit(2)) + for e in reference_expectations() + write_expectation(HERE, e) + println("wrote ", e.file) + end +end + +end # module ILRReference diff --git a/test/fixtures/ilr/manifest.json b/test/fixtures/ilr/manifest.json new file mode 100644 index 00000000..127ce8d2 --- /dev/null +++ b/test/fixtures/ilr/manifest.json @@ -0,0 +1,56 @@ +{ + "reference": "test/fixtures/ilr/ilr_reference.jl", + "inputs": "counts.csv, tree.nwk and sbp.csv are seeded synthetic data, committed as data", + "expectations": "expected_*.csv, merge_*.csv and variation.csv are recomputed from the inputs by the reference on every test run (test/unit/test_ilr_basis.jl) and must agree to 1e-10; rewrite with: julia --project=. test/fixtures/ilr/ilr_reference.jl --write", + "datasets": [ + { + "name": "philr_d8", + "kind": "phylogenetic", + "taxa": 8, + "samples": 6, + "unseen_tree_tips": 0, + "combos": [ + "uniform__uniform", + "gm_counts__blw", + "enorm_x_gm_counts__blw_sqrt" + ] + }, + { + "name": "philr_d25", + "kind": "phylogenetic", + "taxa": 25, + "samples": 12, + "unseen_tree_tips": 0, + "combos": [ + "uniform__uniform", + "anorm__blw", + "anorm_x_gm_counts__mean_descendants" + ] + }, + { + "name": "philr_d60_pruned", + "kind": "phylogenetic", + "taxa": 60, + "samples": 20, + "unseen_tree_tips": 15, + "combos": [ + "uniform__uniform", + "enorm__uniform", + "gm_counts__mean_descendants" + ] + }, + { + "name": "sbp_d12", + "kind": "sequential_binary_partition" + }, + { + "name": "dendrogram_d30", + "kind": "balance_dendrogram", + "methods": [ + "ward", + "complete", + "average" + ] + } + ] +} diff --git a/test/fixtures/ilr/philr_d25/counts.csv b/test/fixtures/ilr/philr_d25/counts.csv new file mode 100644 index 00000000..bc42ec4b --- /dev/null +++ b/test/fixtures/ilr/philr_d25/counts.csv @@ -0,0 +1,26 @@ +taxon,S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12 +OTU_001,2457.5,195.5,849.5,321.5,999.5,4771.5,1325.5,2780.5,955.5,687.5,2580.5,3333.5 +OTU_002,0.5,0.5,5.5,0.5,3.5,2.5,0.5,4.5,188.5,39.5,202.5,1.5 +OTU_003,5.5,3.5,116.5,301.5,0.5,44.5,441.5,0.5,144.5,101.5,0.5,25.5 +OTU_004,1390.5,159.5,1041.5,2139.5,734.5,923.5,2304.5,928.5,1075.5,958.5,391.5,3640.5 +OTU_005,463.5,125.5,287.5,82.5,108.5,493.5,218.5,239.5,268.5,359.5,2553.5,878.5 +OTU_006,2.5,8.5,3.5,277.5,11.5,41.5,567.5,375.5,44.5,156.5,0.5,98.5 +OTU_007,139.5,310.5,1233.5,1535.5,227.5,746.5,218.5,1040.5,1131.5,489.5,334.5,791.5 +OTU_008,1474.5,44.5,2431.5,341.5,140.5,1155.5,3157.5,568.5,684.5,155.5,1577.5,40.5 +OTU_009,15.5,0.5,0.5,0.5,0.5,56.5,0.5,110.5,0.5,0.5,30.5,10.5 +OTU_010,126.5,174.5,17.5,7.5,168.5,1779.5,19.5,269.5,263.5,52.5,2820.5,751.5 +OTU_011,31.5,38.5,1615.5,0.5,5.5,546.5,7.5,21.5,615.5,90.5,1252.5,25.5 +OTU_012,1.5,123.5,769.5,1680.5,99.5,399.5,956.5,336.5,376.5,168.5,864.5,439.5 +OTU_013,371.5,105.5,1817.5,735.5,195.5,193.5,2577.5,413.5,444.5,75.5,350.5,1567.5 +OTU_014,309.5,122.5,317.5,250.5,104.5,903.5,382.5,171.5,184.5,312.5,47.5,1757.5 +OTU_015,0.5,0.5,0.5,0.5,0.5,0.5,0.5,2.5,0.5,0.5,1086.5,1.5 +OTU_016,259.5,33.5,1409.5,462.5,87.5,273.5,1848.5,63.5,406.5,76.5,387.5,261.5 +OTU_017,1404.5,102.5,367.5,27.5,84.5,1880.5,808.5,832.5,616.5,558.5,2443.5,676.5 +OTU_018,376.5,160.5,769.5,226.5,177.5,191.5,185.5,724.5,641.5,95.5,100.5,24.5 +OTU_019,564.5,157.5,944.5,550.5,262.5,212.5,651.5,121.5,408.5,215.5,72.5,643.5 +OTU_020,596.5,24.5,7.5,19.5,19.5,176.5,58.5,1065.5,117.5,265.5,189.5,141.5 +OTU_021,100.5,89.5,1001.5,77.5,39.5,321.5,2329.5,209.5,973.5,348.5,263.5,1283.5 +OTU_022,269.5,115.5,99.5,255.5,79.5,379.5,7.5,150.5,40.5,1.5,5.5,109.5 +OTU_023,205.5,0.5,2080.5,599.5,1.5,115.5,312.5,17.5,47.5,357.5,43.5,521.5 +OTU_024,791.5,7.5,432.5,1208.5,122.5,608.5,187.5,816.5,565.5,298.5,1078.5,1720.5 +OTU_025,237.5,259.5,778.5,322.5,309.5,73.5,320.5,320.5,1052.5,518.5,464.5,1196.5 diff --git a/test/fixtures/ilr/philr_d25/expected_anorm__blw.csv b/test/fixtures/ilr/philr_d25/expected_anorm__blw.csv new file mode 100644 index 00000000..8c0064a2 --- /dev/null +++ b/test/fixtures/ilr/philr_d25/expected_anorm__blw.csv @@ -0,0 +1,25 @@ +balance,S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12 +n1,0.5398821958020339,1.4254194881002507,0.08676871966903625,-1.0016745096012334,1.6734286277085133,1.1869710886094649,0.4508630098099419,2.215094609727898,1.4397427586722529,0.9698486603941319,1.8524631700088465,0.5137278105473314 +n2,-1.9725610210700686,-0.15754112285350405,-0.10813071215667235,-3.11187254783234,-0.6616584539257883,-0.2197179027378093,-2.7128550222280725,-1.679723235707775,0.8936257026882988,0.0030083683174895516,2.671918030134564,-2.2060486772610712 +n3,-0.699948502041665,1.982616329784942,-0.45190433813666037,6.853878222209873,1.4035641510740857,0.69958695300327,4.773282591186363,4.163753948371451,-0.6223993141469728,1.733888090366355,-1.3093654833173067,3.4065228692881773 +n4,-5.2753061816031295,1.2145068472046414,3.7865296892945364,3.637662979217358,1.2248991251897754,0.5310782641783963,2.218885268735075,-1.1500137804044817,0.827766342059733,-0.5542915473615715,1.1293161081919305,0.8011848536540109 +n5,7.215246661699962,3.1328623717208326,4.862395831231948,8.364038675276674,4.185070089284006,3.5984059878015953,9.016155506385967,4.809976063319737,1.3679802317801506,2.6969228281953157,1.9830608284080034,3.228016038628118 +n6,-0.37307323378000795,0.7940984637550523,-0.8932122401375877,-0.5131292694663934,0.10506069901009485,-0.21002536947587913,-1.1781896371367637,-0.21730386021511597,-0.2552806097739157,0.6883718181585063,0.4987563809676125,1.8793547510170396 +n7,1.3120668710845005,1.3617888145000914,4.485289569257821,-1.2712550809942365,-1.3393911802905591,2.0119145779556047,0.780804982336425,-0.42577197267709826,0.7637954105326832,0.4649126922447416,0.4020105360688197,-0.14778303450312613 +n8,-6.766596543192879,-7.146846322956594,-1.5939517441767752,-3.4268598026067782,-4.805293338770543,-7.98933417875657,-4.55633713404997,-5.063359815200406,-0.6217057240202541,-0.5620789439925509,-3.3392443408974084,-7.574203485149368 +n9,-3.38826705219419,-0.39045135658982766,-0.9290010626575356,1.2259720289197686,-0.8457589364869837,-1.3538159550270277,2.8426221524268414,-0.9194234779702642,0.42091677198045513,1.0079749896154648,-3.688266569261927,-0.39456443411587966 +n10,1.564435385408087,1.4440215010533324,2.533300983975174,-0.7275899413237262,0.5608438672315363,1.1250749668445792,0.8728747390984729,0.25194903980823136,1.72763145807974,0.9109406951368934,2.781101816795146,1.674300564886529 +n11,-4.241646660849423,-1.5976530584711497,-5.3375313971537715,-0.9019722255856193,-2.129373960351943,-2.013232117632407,-1.44359575440506,0.9474924042696282,-2.2771366874554655,0.08950183884974411,-5.866684441869899,-1.2782858334254152 +n12,-7.745726933656395,-10.454332327936868,-14.265916967120726,-12.907177367573015,-11.680443856588047,-4.897527128405765,-13.785739866565347,-3.8149175639243627,-13.87094484422494,-12.531426832638909,-5.75795071343773,-8.952513791680836 +n13,-0.07956674675970346,0.2554820776168808,0.09411003711369662,-0.05192470443100194,0.07990445017581417,-0.2035826427192114,-0.2610375646111777,0.18089345766159567,0.10485419966284248,-0.25332402259320075,-0.35709231998784347,-1.0007037203506506 +n14,0.0399930037895989,-1.935687860811122,-0.47646576375647354,0.8438428704537921,-0.3586088769419737,-0.0018005948516116592,-1.0589309921984962,0.3172907807174119,-0.0956678977917361,-0.15997455724696713,0.4943342325664933,0.07144326085294948 +n15,-0.12374042513440424,0.2578836081689673,0.12064345648572437,0.19767111538007615,0.2613631458609591,-0.38111672067548086,-0.14815220988659863,0.06833421499706688,0.29627861702080627,0.24996236577508776,0.17028418174251303,0.11087050165153549 +n16,0.37153707853665724,-0.0024382720043996833,-0.18771744041096702,-0.6432566751725138,-0.07290923812235552,0.2537978804841442,-0.15575809243001953,0.12132514737246958,0.18916772249963457,0.340319738713678,0.5519092432178524,-0.3595667152797208 +n17,0.2859789567801591,-0.7414024975056259,-1.135459678992095,-2.708033928958734,-1.2806865598904313,1.25804498033324,-0.2469128616431958,1.0620527841056824,-0.0970158933638205,0.3172337649814459,2.282507790278753,-0.37400291474518244 +n18,0.0050758920475084636,-0.04812032034075694,0.25538516998514765,0.1484053157136907,0.07618493045159937,-0.2711016785789734,0.2815204799290083,0.11683643971817784,0.11671247368281863,-0.2517891931548065,0.296067230185376,-0.04251377758532845 +n19,1.893534742196469,2.5125224062801434,0.9473224197505166,1.0726192376769186,2.6677887478721836,2.033468026162111,0.229261181120955,2.53430942713041,1.2656819753463082,0.09098135088531177,0.6975650952132183,1.3382032597267874 +n20,-2.156724380918285,-1.9106140727682455,-4.0435440838307235,-2.1808054869258457,-3.4373064109709657,-3.208886568917183,-6.813808002677359,-4.260691060186366,-5.293356748269549,-8.993793589192387,-7.987206864709202,-4.658583918751369 +n21,0.7313352281976958,-0.18063313736639347,-0.1560890652570921,-0.48202311977887413,0.3511779996627396,0.453809029878505,0.4395027691773931,0.2150789594151193,-0.10027303806949879,0.03899228225416714,0.5052952141088332,0.3396104588504871 +n22,-8.57263368453738,-3.049855162480808,-12.695456026662015,-12.677894837877545,-2.7366009919799192,-9.68486280964024,-12.499592057187439,-1.7613993762023579,-9.976074945045037,-11.21454564254897,8.774651963252714,-8.754959489736246 +n23,3.1220299332240953,3.777220075705019,1.2145296594059618,1.7576217702251935,5.042577561343833,2.360837652028932,1.8861139879714728,4.4193284479759525,2.354718758902162,1.7606089599639498,3.5797807711647387,2.9458369571535656 +n24,-4.363181605193782,2.2834601798414766,-3.481725319207026,-0.8606765827387519,-1.3517607159252185,-1.1788296573270278,0.37262616202997373,-4.285152536619319,1.2884111929354531,-1.5433582808564477,-5.372379520281329,-3.6435849687667963 diff --git a/test/fixtures/ilr/philr_d25/expected_anorm_x_gm_counts__mean_descendants.csv b/test/fixtures/ilr/philr_d25/expected_anorm_x_gm_counts__mean_descendants.csv new file mode 100644 index 00000000..1556614f --- /dev/null +++ b/test/fixtures/ilr/philr_d25/expected_anorm_x_gm_counts__mean_descendants.csv @@ -0,0 +1,25 @@ +balance,S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12 +n1,-102.15373602629734,5.396098107734966,15.34634562057067,-109.97415959230383,-100.81739908031975,-73.93434491890272,23.283844773483896,-60.80900148397947,26.56670540773384,-57.02898349797601,95.42761917239436,-133.05938688393803 +n2,-151.00528530662874,13.446626819981955,-49.06447829484697,-95.51813581164276,-35.6729856481222,72.85862483480601,-77.2151069306161,-13.507594131367247,-35.50987011603287,-56.88591152043178,164.28516327673273,-198.76807202115285 +n3,-238.71958995886223,19.439556365135186,-33.342423347604175,136.20494777165726,-12.957864483794918,-51.21338659886145,27.48805378451364,30.940596890859513,-18.580070711109396,23.592400194213376,-55.96965439041143,62.7572167419656 +n4,-69.91291606308194,3.972895700806951,33.25509454375075,31.560263325987226,4.091210649419373,-3.8078639937161567,15.407634215383592,-22.946913117699903,-0.4301026656740874,-16.164680628290572,3.003008451547309,-0.7327300048641665 +n5,66.16418078599253,-54.853237536727335,73.22118503233371,129.9579446074401,-1.6915765173419843,-43.896232468969636,143.6306275397576,16.30598164740568,-21.880684504656323,7.373560037710838,-42.330604310021286,-67.87327414911341 +n6,-2.9818912316024178,19.095876511350287,-12.820639207298516,-5.631136953900989,6.062305042568591,0.10225901526636927,-18.211161266928947,-0.03541809633552283,-0.7537715812273784,17.09599223498237,13.50930006996503,39.62416408236517 +n7,-7.738931445937754,-9.137153284754824,65.31341409850789,-25.58759224790939,-34.14246552677716,-4.656340001206786,-3.0116445227880324,-22.74174513079632,18.43904041969699,14.572697574154978,-1.6191414712892327,-32.68085167461201 +n8,-7.61217557415775,-8.752406544796216,7.898707308181177,2.402481811736744,-1.730939672102879,-11.278721605824936,-0.9844104429961569,-2.5047872969239178,10.81411997180277,10.992919026753656,2.665208756667229,-10.033895444432565 +n9,-88.65862521770828,-3.22311024530459,27.398956809370294,-6.949493461682805,-49.656858269599056,-15.278937205207747,101.8148560532368,-58.209295562240385,21.29046586467863,-2.031197205465358,-42.97673808211541,-8.610229573421716 +n10,-9.294586361106136,11.196117341480242,4.0166818656408125,-34.186958435427975,-14.454588397536284,-0.04136640780503473,-0.9952000611493749,5.386471480441479,11.925209872105317,13.44601960783842,5.223618146027978,19.48017042025864 +n11,-21.555177469635385,0.7083839780481397,-30.782997515925974,6.566315312296033,-3.768934333687737,-2.790970147349416,2.005612629605647,22.13960164516084,-5.013159944365256,14.914952668107297,-35.23869336771225,3.3975931473843293 +n12,3.205292012355513,-4.289739541754273,-16.420190490959502,-11.86828628886454,-8.312631019184424,11.977225648096022,-15.550176663830548,15.353570887136978,-15.389400469322103,-11.399455742634833,8.158956490348292,-1.152090073780372 +n13,4.3978275222164696,25.655544723295442,15.307016073931948,8.306774782988654,15.037669675730111,-2.450690917231891,-6.414335836319616,19.22145675368167,15.350820692555212,-5.060771463493794,-14.6504726461753,-49.13769047004885 +n14,0.5187496302505792,-69.03012435138366,-17.252511844918775,33.823271470497716,-11.933217260629336,-0.05216321521679548,-39.50131404153998,9.402748995256651,-4.4575836445357835,-5.183458552073138,12.526713967714198,3.9522579210623445 +n15,-7.459487091703083,15.041206252439583,6.7027229081392665,13.125893119302157,15.194332038989073,-21.07609152979305,-8.940265497020924,2.393472594792799,15.752631551059869,14.262619751138354,5.833774069731131,7.063687927019287 +n16,18.267366019350305,-3.694112704327752,-19.324608316972242,-46.15220018103279,-10.642667593041145,18.18158526206111,-15.98871006356793,4.602874657860785,5.733027760740561,20.526505828419165,28.18176288569268,-22.513363816192303 +n17,-0.21677647170603484,-19.534677879320565,-26.94415344378852,-56.513340419358016,-29.674861259494367,18.06102631162785,-10.236766190503532,14.375775903815235,-7.418246503933638,0.37090914479762593,37.32404831357073,-12.62644638715397 +n18,-1.352782736350518,-2.2997374854780084,3.1030149389367456,1.198648509163071,-0.08695875002669218,-6.269066252898971,3.5682539993913016,0.6366856229640127,0.6344788826597014,-5.925281442499491,3.8272031542502587,-2.1999344722542604 +n19,-2.332495433601527,72.74713747283197,-40.17201850311721,-101.00480710897993,55.988407498439926,76.82208547465447,-78.04647153763614,87.73493395386521,2.333064210575634,-54.64778350625799,85.90058387601786,-27.800682416513926 +n20,12.133754804592181,25.162299855362342,-0.15107540029686006,25.34110551056525,1.3263539690293247,2.866944867329562,-39.29560246307568,-6.826999274244727,-15.444375765957703,-60.43567106864188,-53.802725824803254,-12.882777656733706 +n21,13.277795088025323,-11.05226930505463,-10.39746714586095,-19.092940403321546,3.135721964656334,5.873776998323205,5.492105596961656,-0.49521364462087575,-8.908372143316571,-5.192964527064578,7.247357733114412,2.827115851265978 +n22,-7.292096038504911,2.932170146981323,-8.468281698806503,-9.834943156085563,-1.8478100423825161,-6.061714983048964,-9.67170676827805,1.7462082486175412,-6.083889839018807,-7.042242286215057,25.83480216454928,-6.859509250397652 +n23,14.970916553762232,64.3790423499175,-33.06594322840369,-5.7889834029700165,81.79932202366851,9.215282527210732,3.5682374575644515,50.84927711291039,20.73122988445591,-8.940006734888776,22.72121257432407,13.555865113951674 +n24,-13.612225149935796,21.75054949493026,-8.922527553951257,5.022492923851854,2.4097302543423025,3.3297921821679415,11.584153449975707,-13.197079508424007,16.456493605866978,1.3903550976394403,-18.981558957522704,-9.783685187625176 diff --git a/test/fixtures/ilr/philr_d25/expected_uniform__uniform.csv b/test/fixtures/ilr/philr_d25/expected_uniform__uniform.csv new file mode 100644 index 00000000..885cc5a4 --- /dev/null +++ b/test/fixtures/ilr/philr_d25/expected_uniform__uniform.csv @@ -0,0 +1,25 @@ +balance,S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12 +n1,0.3611624846608233,1.5952447886460575,-0.15626238825525063,-2.530704857414675,1.6739805095610665,0.9213218826485957,0.8375179567531881,2.495302486838606,1.9243876030135085,1.2437109334058833,2.5722529637194773,0.04458337477867885 +n2,-2.520749315783496,-0.4416386193406722,-1.2359206151556872,-4.143543143611394,-1.0932267462514487,0.2630609547374998,-3.586609890161942,-1.359169677810238,0.6468714006644632,0.04887932657867384,3.950055249739599,-2.870843265243334 +n3,-0.9658887204934617,0.7864735566621405,-0.8988657745578452,3.6139883992527655,0.2892514913298093,-0.011686330774588478,2.293470795593753,2.386689989389972,-0.6248021524199248,0.8838841961919655,-1.3340597165118302,1.6372506674399223 +n4,-4.232468343314257,1.1437933290662412,3.274496904813992,3.1511734155041093,1.1524024531769343,0.5776305011968672,1.97583597625358,-0.8150121969695923,0.8234115020801391,-0.3215066684317546,1.073220000842031,0.8013909852060551 +n5,4.5845248016224796,1.7608391522908167,3.0058579344203027,5.382509848141798,2.323617895560287,1.890301619864226,5.796479359876899,2.7618484099888687,0.34569436827082445,1.5535515609725168,0.885455256440149,1.9911767293215639 +n6,-0.8183119396107963,0.733140027069061,-1.5097018062984557,-1.0044801036177893,-0.18275692064470567,-0.6015821043416439,-1.8885053224969777,-0.6112569694841917,-0.6617372055244747,0.5926038809410844,0.3405593500773325,2.175706707515589 +n7,1.1238585391778968,1.1563791723767465,4.167337555683238,-1.1055568650902978,-1.2125801067597481,1.7173740738465089,0.7154709511391912,-0.39377177214147263,0.8294456110157208,0.5607640791124114,0.41248767387912755,-0.22460980479440715 +n8,-3.912697230425593,-4.1401610605242904,-0.8184427157867163,-1.914880660992966,-2.7394538451434904,-4.6441337771185855,-2.5905292832732245,-2.893828133500183,-0.23684920722389738,-0.201180716449276,-1.8624694577342478,-4.395804330042773 +n9,-3.0448882022188855,-0.7495009099720673,-0.6649277957432558,0.6675490258246537,-1.240739817062369,-0.8629733527833293,2.7945610979636863,-0.9837968805478742,0.3087399661400873,0.5664642599494352,-2.593915119854399,-0.31943761956501826 +n10,1.120750976241653,1.3622696963992544,2.1697396557061874,-1.2500201833633064,0.17906403420289976,0.9018204892528893,0.6709345648919223,0.24909232766505943,1.6161417058911376,0.9458825930644229,2.4011278614486504,1.6975785186243344 +n11,-3.2827191693322355,-0.9697822958460138,-4.241387228668338,-0.36120826256878025,-1.4349259630837854,-1.333326341080532,-0.835014640249638,1.256683299178177,-1.564187191150752,0.5061224755521656,-4.704284562361029,-0.6904033176492393 +n12,-3.22384131885688,-4.8989111481678576,-6.714644257087243,-5.9996187953133235,-5.412817887248322,-1.6962101198583333,-6.421553772178843,-1.1784711503390621,-6.462860981791569,-5.763197250213475,-2.086671900742655,-3.8872228018065593 +n13,-0.276893227067919,0.5770935555940353,0.1735018296549003,-0.3218445522368957,0.09242839546187347,-0.6401651059296946,-1.024743016775467,0.670354411364666,0.2677273430039713,-0.9816632735559947,-1.121508434515826,-3.573391974131928 +n14,0.4863041973013929,-2.6729503174609683,-0.42586001996590395,1.4533910159442551,-0.2968935617391471,0.5090005019769228,-1.200744775604049,0.9241291874197075,0.17864485285072662,0.0868974586654156,1.2726894262959647,0.4294064240686571 +n15,-0.7502485448294052,0.6884666749088152,0.13559559411055355,0.3030744452270938,0.672627650756157,-1.7398055520211817,-0.8712241484834813,0.08629899529474233,0.913331917452302,0.7237721504363657,0.6169272501309173,0.11982736302005892 +n16,0.9654350369257754,0.11123313697162385,-0.25415801107298275,-1.2494982498655758,0.04112758278226036,0.41334067188637147,-0.3134228967878222,0.17755335178273074,0.5609753592349578,0.8147392013108112,1.1823414305277993,-0.922515210772436 +n17,0.6445251885613813,-0.30374666960936586,-0.6674609785983983,-2.118945137580035,-0.8015052166283162,1.541740970074719,0.15266667510242876,1.3608404064832982,0.2910213982205912,0.6733733392907602,2.487318922996988,0.03536270751833423 +n18,0.12911117308673653,-0.10564180760504265,1.2337170548323693,0.7616187881193693,0.44291281365558444,-1.0896505581353657,1.3490512418952465,0.6223064686078662,0.6217594109321672,-1.0044252514011254,1.413245529961036,-0.0809003351557173 +n19,3.561614328879664,5.263035993855734,1.951859896167702,1.895351827372426,5.4538725332110465,4.370829584347412,0.1868134205234541,5.434768783232465,2.7293750847124794,0.2742339830435757,1.458955224058193,2.577627099853742 +n20,-0.633530994427194,-0.6185791246276628,-1.9032305003470944,-0.8259532705665702,-1.4627092681514589,-1.3097023298901629,-3.489104220321983,-1.9799701298495078,-2.6499111905460597,-4.864489067733847,-4.188169876157061,-2.2020336597684294 +n21,2.028572859919336,-0.3271242295059144,-0.2637246627953284,-1.1056419184521253,1.046592222652558,1.3116974919723448,1.2747431216600333,0.6950360447290176,-0.11954679681445617,0.24018817002445672,1.4446909804712307,1.0167122144722562 +n22,-4.7189881122694795,-2.2260031917491876,-6.1851862284234915,-6.308312340688226,-2.4222584661415336,-5.038065853550071,-6.251792080354333,-1.8053026672807724,-5.16555046502847,-5.613000836873032,3.441744051047355,-4.7570396445948875 +n23,3.039262868157396,3.912844332070659,0.611792608802915,1.3193722858405954,5.505661782586771,2.08679456413507,1.4902922316191887,4.694074386346093,2.093118656213259,1.3192727072552357,3.6172257097348113,2.8186951675982095 +n24,-2.5602200672027347,1.375966261976738,-2.038215678387613,-0.48601284599970473,-0.7768361992120933,-0.6744252538415617,0.24435740121716484,-2.5140107237576106,0.7866915055212719,-0.8903015761915408,-3.157873914856713,-2.1340700602616876 diff --git a/test/fixtures/ilr/philr_d25/tree.nwk b/test/fixtures/ilr/philr_d25/tree.nwk new file mode 100644 index 00000000..0e671b50 --- /dev/null +++ b/test/fixtures/ilr/philr_d25/tree.nwk @@ -0,0 +1 @@ +((((OTU_012:0.33930903569812565,OTU_020:0.1768143885259625):0.30712930692589635,((OTU_005:0.3190500637782131,OTU_008:0.06507751084019771):0.2805817925751958,(OTU_011:0.10662444061302333,(OTU_002:0.30686504448977436,OTU_010:0.34123127061163633):0.29432715747688315):0.3176187989883342):0.29160536677249715):0.21658512745602176,((OTU_021:0.2262833045195322,(OTU_006:0.49059724408998184,OTU_016:0.0):0.14530665506432186):0.01887397067681794,(OTU_009:0.4795505582292628,(OTU_018:0.100538148805078,(OTU_024:0.010129606950421411,(OTU_025:0.1375253764853738,((OTU_017:0.11862076351262471,OTU_019:0.4775369457684515):0.1036039470826524,(OTU_013:0.0,OTU_014:0.11264788226318265):0.09317859275941813):0.015882036633170046):0.2998405182466171):0.05713935293305851):0.28667623559983485):0.3823609542062937):0.06929130823393305):0.01740059943751749,((OTU_022:0.33148280235475786,(OTU_001:0.0,OTU_007:0.21746853445628048):0.4613883650301439):0.13896065662756624,(OTU_015:0.42839130127114133,(OTU_004:0.05365795773188238,(OTU_003:0.3578025833001283,OTU_023:0.2257479328137567):0.36465174145317614):0.3571063293129358):0.03172775020640483):0.16792165774471626); diff --git a/test/fixtures/ilr/philr_d60_pruned/counts.csv b/test/fixtures/ilr/philr_d60_pruned/counts.csv new file mode 100644 index 00000000..1ea80d78 --- /dev/null +++ b/test/fixtures/ilr/philr_d60_pruned/counts.csv @@ -0,0 +1,61 @@ +taxon,S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15,S16,S17,S18,S19,S20 +OTU_001,50.5,352.5,148.5,143.5,159.5,303.5,695.5,101.5,20.5,3.5,83.5,489.5,393.5,103.5,213.5,353.5,95.5,99.5,208.5,611.5 +OTU_002,9.5,79.5,210.5,232.5,417.5,542.5,448.5,779.5,35.5,157.5,228.5,795.5,482.5,167.5,832.5,233.5,107.5,833.5,537.5,474.5 +OTU_003,279.5,639.5,91.5,148.5,137.5,671.5,1043.5,277.5,90.5,102.5,392.5,1077.5,499.5,271.5,458.5,385.5,176.5,124.5,927.5,1499.5 +OTU_004,164.5,344.5,105.5,135.5,897.5,77.5,86.5,124.5,20.5,128.5,138.5,5.5,51.5,147.5,27.5,144.5,10.5,604.5,147.5,168.5 +OTU_005,75.5,17.5,7.5,33.5,0.5,181.5,61.5,4.5,0.5,17.5,222.5,0.5,181.5,6.5,3.5,457.5,12.5,75.5,83.5,54.5 +OTU_006,234.5,2.5,176.5,67.5,0.5,259.5,4.5,23.5,55.5,5.5,121.5,58.5,145.5,23.5,215.5,0.5,4.5,100.5,3.5,71.5 +OTU_007,38.5,349.5,12.5,151.5,151.5,167.5,165.5,4.5,10.5,10.5,149.5,258.5,1551.5,100.5,8.5,58.5,105.5,422.5,588.5,39.5 +OTU_008,667.5,1189.5,85.5,251.5,77.5,74.5,128.5,421.5,63.5,210.5,1084.5,808.5,226.5,586.5,600.5,1103.5,13.5,100.5,83.5,286.5 +OTU_009,217.5,818.5,349.5,777.5,760.5,503.5,887.5,922.5,48.5,130.5,905.5,977.5,375.5,557.5,191.5,229.5,54.5,775.5,939.5,627.5 +OTU_010,0.5,0.5,11.5,1.5,0.5,0.5,10.5,0.5,0.5,34.5,0.5,0.5,0.5,292.5,71.5,2.5,8.5,0.5,0.5,0.5 +OTU_011,55.5,30.5,0.5,48.5,82.5,79.5,30.5,0.5,5.5,0.5,22.5,659.5,7.5,194.5,196.5,909.5,21.5,0.5,98.5,93.5 +OTU_012,11.5,0.5,5.5,0.5,1.5,20.5,0.5,4.5,3.5,5.5,142.5,0.5,0.5,11.5,0.5,0.5,0.5,7.5,0.5,22.5 +OTU_013,167.5,187.5,277.5,1047.5,468.5,656.5,775.5,174.5,185.5,137.5,813.5,650.5,1696.5,313.5,2241.5,758.5,158.5,795.5,150.5,351.5 +OTU_014,541.5,951.5,156.5,22.5,75.5,452.5,55.5,28.5,12.5,136.5,375.5,184.5,115.5,1232.5,653.5,167.5,165.5,505.5,95.5,132.5 +OTU_015,219.5,119.5,0.5,1.5,293.5,273.5,961.5,14.5,53.5,3.5,128.5,510.5,217.5,347.5,333.5,306.5,35.5,32.5,4.5,170.5 +OTU_016,287.5,3.5,51.5,224.5,175.5,2.5,0.5,20.5,11.5,0.5,207.5,446.5,3.5,0.5,82.5,151.5,2.5,0.5,3.5,0.5 +OTU_017,102.5,372.5,158.5,101.5,44.5,1.5,631.5,323.5,165.5,155.5,1223.5,281.5,345.5,9.5,65.5,246.5,56.5,187.5,2.5,553.5 +OTU_018,104.5,18.5,47.5,188.5,1265.5,410.5,13.5,25.5,9.5,130.5,273.5,385.5,1.5,4.5,372.5,126.5,42.5,56.5,0.5,166.5 +OTU_019,35.5,41.5,9.5,0.5,2.5,0.5,18.5,2.5,20.5,5.5,56.5,9.5,1.5,0.5,0.5,0.5,1.5,110.5,31.5,1.5 +OTU_020,116.5,1583.5,790.5,376.5,913.5,287.5,1308.5,544.5,138.5,216.5,528.5,1046.5,560.5,1735.5,778.5,977.5,279.5,1479.5,493.5,622.5 +OTU_021,163.5,315.5,96.5,22.5,137.5,33.5,247.5,13.5,7.5,40.5,321.5,1032.5,221.5,157.5,114.5,570.5,36.5,128.5,426.5,54.5 +OTU_022,92.5,34.5,254.5,103.5,7.5,599.5,162.5,123.5,74.5,26.5,244.5,129.5,1428.5,1.5,16.5,217.5,0.5,126.5,280.5,322.5 +OTU_023,62.5,943.5,39.5,267.5,487.5,1211.5,195.5,216.5,34.5,103.5,445.5,343.5,647.5,1516.5,291.5,525.5,147.5,762.5,82.5,95.5 +OTU_024,91.5,174.5,41.5,63.5,450.5,3440.5,300.5,204.5,7.5,4.5,223.5,26.5,32.5,966.5,91.5,36.5,47.5,3.5,365.5,199.5 +OTU_025,0.5,0.5,4.5,0.5,0.5,0.5,73.5,132.5,0.5,85.5,3.5,6.5,19.5,3.5,0.5,0.5,0.5,0.5,0.5,5.5 +OTU_026,69.5,257.5,426.5,85.5,13.5,68.5,95.5,286.5,53.5,310.5,994.5,282.5,474.5,509.5,1007.5,518.5,224.5,601.5,251.5,391.5 +OTU_027,111.5,75.5,12.5,114.5,160.5,144.5,7.5,29.5,67.5,27.5,297.5,244.5,293.5,92.5,254.5,103.5,1.5,1135.5,278.5,1.5 +OTU_028,0.5,0.5,7.5,3.5,0.5,0.5,0.5,0.5,0.5,0.5,3.5,63.5,0.5,0.5,26.5,0.5,0.5,0.5,0.5,0.5 +OTU_029,26.5,428.5,11.5,190.5,119.5,152.5,249.5,20.5,29.5,388.5,1603.5,501.5,469.5,363.5,115.5,211.5,83.5,32.5,178.5,405.5 +OTU_030,0.5,415.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,254.5,0.5,0.5,0.5,1.5,0.5,0.5,0.5,10.5,0.5 +OTU_031,0.5,22.5,0.5,1.5,438.5,0.5,59.5,128.5,149.5,0.5,6.5,133.5,0.5,0.5,7.5,115.5,0.5,40.5,286.5,127.5 +OTU_032,10.5,490.5,344.5,226.5,49.5,200.5,41.5,53.5,6.5,14.5,350.5,55.5,356.5,558.5,397.5,66.5,16.5,144.5,917.5,11.5 +OTU_033,16.5,222.5,190.5,5.5,584.5,63.5,258.5,248.5,1.5,22.5,201.5,103.5,129.5,190.5,337.5,229.5,98.5,304.5,143.5,164.5 +OTU_034,244.5,2.5,34.5,401.5,235.5,32.5,4.5,133.5,1.5,46.5,4.5,188.5,550.5,161.5,1157.5,225.5,104.5,122.5,72.5,101.5 +OTU_035,0.5,0.5,0.5,61.5,0.5,0.5,4.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,3.5,51.5,45.5,0.5 +OTU_036,95.5,49.5,36.5,154.5,2.5,19.5,275.5,208.5,3.5,0.5,181.5,35.5,94.5,1095.5,222.5,0.5,161.5,5.5,8.5,0.5 +OTU_037,0.5,33.5,0.5,237.5,20.5,192.5,100.5,40.5,24.5,38.5,239.5,3.5,207.5,1083.5,31.5,24.5,502.5,37.5,217.5,1.5 +OTU_038,13.5,216.5,50.5,2.5,11.5,55.5,150.5,10.5,49.5,4.5,9.5,29.5,4.5,22.5,0.5,0.5,44.5,47.5,68.5,34.5 +OTU_039,106.5,89.5,77.5,179.5,149.5,253.5,588.5,413.5,15.5,7.5,169.5,32.5,177.5,115.5,263.5,540.5,14.5,184.5,382.5,88.5 +OTU_040,1.5,281.5,204.5,447.5,382.5,177.5,245.5,165.5,52.5,252.5,35.5,583.5,1910.5,124.5,84.5,329.5,54.5,291.5,178.5,42.5 +OTU_041,20.5,368.5,135.5,319.5,61.5,98.5,127.5,259.5,22.5,26.5,302.5,53.5,201.5,173.5,901.5,166.5,12.5,15.5,664.5,101.5 +OTU_042,46.5,0.5,15.5,47.5,0.5,428.5,483.5,117.5,0.5,9.5,2.5,35.5,0.5,0.5,4.5,0.5,0.5,2.5,33.5,110.5 +OTU_043,0.5,0.5,0.5,0.5,0.5,17.5,8.5,0.5,24.5,0.5,0.5,72.5,79.5,100.5,2.5,1.5,0.5,0.5,7.5,2.5 +OTU_044,4.5,164.5,0.5,80.5,0.5,124.5,181.5,3.5,0.5,0.5,0.5,5.5,6.5,0.5,1.5,111.5,35.5,0.5,0.5,1.5 +OTU_045,16.5,8.5,0.5,1.5,0.5,0.5,13.5,30.5,2.5,181.5,31.5,3.5,440.5,0.5,0.5,6.5,0.5,28.5,2481.5,0.5 +OTU_046,45.5,0.5,0.5,0.5,0.5,21.5,0.5,1.5,0.5,1.5,0.5,1.5,0.5,0.5,15.5,0.5,4.5,0.5,0.5,26.5 +OTU_047,1.5,38.5,22.5,52.5,17.5,47.5,610.5,0.5,9.5,263.5,12.5,0.5,28.5,0.5,22.5,15.5,6.5,91.5,350.5,73.5 +OTU_048,24.5,58.5,2.5,527.5,3.5,6.5,2.5,0.5,97.5,5.5,1.5,96.5,0.5,22.5,187.5,10.5,2.5,37.5,6.5,307.5 +OTU_049,51.5,135.5,1.5,6.5,176.5,0.5,13.5,670.5,72.5,45.5,117.5,2.5,7.5,143.5,85.5,663.5,11.5,352.5,15.5,311.5 +OTU_050,343.5,253.5,217.5,156.5,9.5,591.5,246.5,285.5,52.5,16.5,114.5,390.5,1014.5,420.5,44.5,673.5,97.5,1476.5,89.5,464.5 +OTU_051,2.5,0.5,0.5,0.5,0.5,826.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,9.5,0.5,0.5,8.5,0.5,0.5 +OTU_052,21.5,0.5,0.5,0.5,121.5,175.5,125.5,0.5,25.5,0.5,46.5,31.5,0.5,49.5,90.5,94.5,0.5,1.5,9.5,0.5 +OTU_053,0.5,5.5,0.5,0.5,4.5,1.5,0.5,2.5,1.5,0.5,0.5,0.5,36.5,172.5,89.5,0.5,7.5,0.5,0.5,5.5 +OTU_054,0.5,18.5,58.5,6.5,71.5,44.5,462.5,0.5,60.5,0.5,14.5,12.5,0.5,101.5,0.5,0.5,25.5,0.5,2.5,290.5 +OTU_055,309.5,1395.5,447.5,682.5,383.5,635.5,2427.5,583.5,430.5,74.5,485.5,94.5,626.5,811.5,1095.5,630.5,145.5,2340.5,625.5,1477.5 +OTU_056,4.5,284.5,0.5,1.5,36.5,0.5,0.5,0.5,0.5,38.5,0.5,4.5,2.5,425.5,0.5,301.5,0.5,9.5,0.5,0.5 +OTU_057,691.5,1230.5,715.5,487.5,1121.5,1216.5,2229.5,949.5,98.5,416.5,1949.5,919.5,833.5,1485.5,1853.5,2740.5,507.5,2353.5,749.5,1943.5 +OTU_058,0.5,0.5,19.5,55.5,0.5,0.5,1.5,75.5,0.5,0.5,0.5,0.5,0.5,4.5,34.5,239.5,0.5,0.5,241.5,9.5 +OTU_059,4.5,192.5,754.5,68.5,2.5,478.5,49.5,155.5,84.5,63.5,0.5,190.5,89.5,483.5,5.5,68.5,63.5,1015.5,13.5,189.5 +OTU_060,0.5,0.5,48.5,1.5,5.5,86.5,161.5,4.5,0.5,1.5,0.5,10.5,345.5,0.5,269.5,290.5,0.5,321.5,8.5,7.5 diff --git a/test/fixtures/ilr/philr_d60_pruned/expected_enorm__uniform.csv b/test/fixtures/ilr/philr_d60_pruned/expected_enorm__uniform.csv new file mode 100644 index 00000000..e93fd519 --- /dev/null +++ b/test/fixtures/ilr/philr_d60_pruned/expected_enorm__uniform.csv @@ -0,0 +1,60 @@ +balance,S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15,S16,S17,S18,S19,S20 +n1,1.2097362577850574,-2.3111885105275625,-0.9958212905840197,-2.154931760205941,-2.4437930971358024,1.7494065128179594,-1.3925329460808886,-2.0336646411092656,-0.7638654189279311,-1.8222228209417153,0.24610198640970277,0.910634011412563,-1.3198898998098962,-5.091784969956099,-2.813658599939731,-1.6967751870649919,-1.6632962912620575,-0.21413002428017605,-1.6459115565641522,-0.6391054038983589 +n2,-1.112031422139522,-0.6190523040127639,-1.1464779386231987,-3.6015716556817043,-0.6365132544271184,-1.5772037526693365,-2.035706710973634,-1.415653858753482,-0.685239560361541,-1.6101940805747583,-0.6212176629144538,-2.5477386033929497,-1.9407552751453072,-2.090058633586651,0.45737190330959515,-1.9059897908337415,-1.69070990254376,0.9025160919060363,-1.7172157824959564,-2.3338746988408055 +n3,-3.20534653950306,2.647872662637786,-2.7794044198765553,-2.236029308853042,-2.8747235465572896,-3.879829757140158,-3.3037188041112953,-2.78267082799934,-2.5102734498855135,-2.3280891318774914,1.7866313353111156,-3.215340008407127,-3.0464905999235876,-2.631285962013628,-2.8937794484669817,-3.0711213885775286,-2.170884439892736,-4.120948822732043,-0.4376726341198131,-2.9409429913996354 +n4,-3.4020239293812384,-4.287086506199246,-4.568222193131472,-3.998799293416708,-2.053266183142704,1.19823480703464,-2.7526169197687453,-3.3076714928340194,-2.906337905943938,-3.5360865195155853,-3.99949075327332,-4.6650923283547465,-3.9136951164366685,-2.742386431381115,-0.4779222324006089,-0.6316393346616129,-4.052802005843001,-2.3545543742491057,-3.4903278265868867,-3.7894453820731266 +n5,-1.3273866602204207,-0.46558482302543197,2.7484318787005795,0.5571384966282031,0.20612705672013565,0.09378819710291487,2.4795775109045337,0.5954849018668815,-0.8693832027221167,0.30750843985900433,-1.0052835754920726,1.2910544135494277,3.6734388714781696,-1.0386237579954363,1.6215212037115188,2.353185140047223,-0.14915518702140743,2.082323326229722,1.0827860303865364,0.988206401345581 +n6,1.6868866636149735,3.4127595621702627,0.9594164334419566,1.7582224185988888,2.7344711589716426,-3.4439837129115247,1.3292378251406594,4.283863747700597,2.4834896104776303,2.81828200975554,2.65659514651353,0.6174794962922503,1.8361784444089952,2.7561323228303296,1.2204733794627058,3.493471565191876,2.0690340712835344,2.6498314290797667,1.7908676251916102,3.8662352612100963 +n7,-1.6092724823893552,-0.5001915840873783,-0.5001915840873785,-0.5001915840873783,-3.331475408470773,0.29849721335507184,-3.3481709286333037,-0.5001915840873776,-2.5267690089083374,-0.5001915840873782,-2.836425350948155,-2.6356838690489233,-0.500191584087378,-2.8686501635124726,-1.6619987417093145,-3.201940633925602,-0.5001915840873781,0.3938723109802229,-2.017841175228328,-0.5001915840873781 +n8,0.20676187419600464,0.38243110192841384,0.5166303559934262,-0.6867737901059232,-1.191387107487582,1.7597091966677947,0.0648794717763765,-1.2144228237555075,0.3198823820566244,-1.0963940423675387,-0.08474957819984173,0.045915498902554655,-0.40620749234569425,0.8264330591104092,-0.7565640830889307,-1.5354677998733781,-0.1840286683827722,-0.6182746042586722,0.22698897739738016,-0.11776976364583683 +n9,0.8122756423523161,-2.1551153550107522,0.47872481084933904,1.0884631092309127,-2.0117337924570906,0.8801442638648087,-1.7338190905948636,0.35918016257189317,0.522994611873964,-0.3722230400353198,0.43232587278773216,-0.45504483936618545,0.7841754966450064,-0.3888135201862287,1.6541718859423777,-1.5765806081139235,-0.9993868079765558,0.16219630406234986,-1.7860324634538294,0.27570658403698245 +n10,-1.0699928243068817,-0.18210384931954082,-0.2957345498281517,-0.9455034553516095,-1.0646063306525582,0.1874687734352327,-0.23278845742957097,-0.12958508314070424,0.7670374337674903,-0.9455034553516095,-1.5008572156097297,-1.5149634085717716,-1.6579431984496735,-0.8401275420106885,-2.302563910042484,-2.9759324116799752,0.058886237795633255,-0.4414952948620351,-0.7910143981122109,-0.21593006746096016 +n11,-0.924383028935617,0.5891800391376754,1.0009284681894146,1.2224977160574775,1.4233042382598606,2.6091586709981507,1.37050361226952,1.5273837842214295,0.5259577625422023,0.9133699708435342,0.7716343613986111,1.274724218941547,1.1559546777348428,1.5454235931315479,1.7143030639391483,1.1912765881615581,0.9692941551279337,1.0044495861995013,1.7013328539579409,1.349587480277612 +n12,0.13949413020048465,1.796821834975885,1.1394985686998793,1.089377909213147,0.4100764599855816,-0.3664905569256233,2.11829062661286,1.5044655549083712,1.6999942194391147,1.1977424672125145,2.1585466261968813,1.0211478364209645,1.3335921369988195,0.15375738837592262,0.6295971131919406,1.6543711274251116,0.8439903182360593,0.30861664013297135,-1.3468413593672128,1.957174935050041 +n13,-0.8861933731232313,1.2118469374585872,-0.6028523136779984,-2.992904952548013,-2.0442064717584185,-1.8799465028588047,0.5939524896769571,-1.7929201160073895,1.125765119520724,-0.9769584828484434,1.0882248473889045,-1.35463523763555,-2.646266205016419,-2.589727925213389,-3.461650806838145,-2.737511160784372,-1.9106416628378753,-0.0775450528594826,-0.40094623793426326,-1.897746286029697 +n14,-1.8301512418335186,-1.4783610460656424,-2.4179435695089904,0.9979192177213106,-0.5191940277762,-3.0702853446635814,-2.7884370786275228,-3.0909449298826135,-2.2498757679189416,-2.931645424802236,-2.622310876706665,0.1500339430851602,-3.0500204303988014,-2.953872164195101,0.8225391394149887,-0.35946822573482984,0.35702195522371966,-3.5467518551540596,-2.367238701528375,-3.028447491311889 +n15,-2.760146535238518,0.8167353582177622,-0.7910948736684656,1.5919619196510462,-1.3965151884421922,-0.02332716158929072,1.9230690700248818,0.18814125483897734,-0.6625690566712851,-0.04919213304025579,-2.6609313127145957,-2.506555090283417,0.6196174997842211,-1.1641993205037753,-3.0462440987397628,-0.35931671777163704,0.6663382594742305,2.064547662834271,0.6028635189614805,-1.728521394025243 +n16,0.9797406584924884,1.7453984711729773,-1.307133234503466,-0.2582465922415288,-1.447588675820872,1.6986679830796312,1.1376101826882703,-0.21077734884680202,-1.002123724771775,-1.3544279402258104,-0.9143547646944037,-0.24979627399013493,-0.42580132650179076,-1.195815585294123,-0.5167041512269359,1.5004630381789033,0.6747754179530329,-2.8452495273111618,-2.6962521355749574,-0.36254620344571564 +n17,-0.6756962420468728,-3.1367097929638166,-2.9864705427459697,-1.0922508184433344,-3.2808521744988415,-2.91990030718031,-2.0393805631740203,-2.8869909558346554,-2.3471987827288543,-3.0855958775370413,-2.1632431492376556,-3.47939867905692,-4.037017428106047,-2.753159031922507,-2.570955267932269,-3.210730081540035,-1.4499306818102444,-0.9741611271268704,-0.801817208942766,-2.2478544482712097 +n18,2.053747210048173,2.9974256156121797,2.8282622461932667,1.9705403292986923,2.6278424633511643,2.6284675908476727,3.484782068852956,3.2390514129364263,2.044717915086185,2.930435662626632,2.562198785639301,1.411264836355553,2.938312838432395,3.0820614471080834,1.9946932894273177,2.7968038329776914,2.5336642363794573,4.058932505551343,2.7098033019068577,2.8784346788686985 +n19,2.440965621356747,0.8363492197020409,0.8573774644468226,0.6800061661670832,1.190562097088883,1.327367980973361,0.24258363774657937,2.2590709930992037,1.0961983955708594,1.9960763007730318,1.3761441005071577,-0.7203098414498907,1.3110114218960536,-0.27547760414464173,0.006752549581029464,0.5017768265228718,1.157784146759111,1.1286256608307008,0.2646538629026531,1.1574821630013359 +n20,1.1054171183671049,-0.44968896623729604,1.7396969438375984,1.8828332518264583,1.8581936193906943,-1.5136485224103617,-1.3376591080988354,0.8090222323921291,0.7559976837179125,-1.7617238902240622,1.3724098671031812,1.6988804039155854,-0.46983496386010726,-1.4206264084529423,0.4546440554343543,1.656210312905598,-0.9161479448612508,-0.8882932972134139,-0.2566230422974865,-2.6244348317908224 +n21,-0.10920040414803138,3.1801021933230085,1.5485553567011903,2.8145368722627615,2.604238205487948,1.0180542229006302,2.936208832279726,1.3138152984546987,1.9733788959487697,2.6404793613666238,3.7752126519574616,2.755614081817426,3.221309791396545,3.1059141618568162,1.0402950349084539,2.86169499340926,1.4517230419836267,2.017054265221285,2.785195813189248,1.364781124327438 +n22,-2.5744714586433877,-2.266354480691822,1.2432154437070255,-1.50355980830034,-2.7784988249742173,-2.7594343902032183,-2.2663544806918225,-0.15056625096661225,-1.3847171303005792,-0.15056625096661255,-1.1082599265691255,-1.3551470626767694,-1.5443479456402502,-3.219906590553435,-1.1817385282502908,-4.013782739201845,-2.0863840837100303,-0.15056625096661205,-2.8697302617923452,-2.842917903108777 +n23,-0.8123239930412135,-1.286475321997016,0.4691456027275137,-0.7448879040747971,-2.179628916769039,0.5518488441394765,-1.6579004541926319,-0.6746958006142211,0.6838027080360212,-0.5683794813252107,-0.6266202211286314,-0.9259861017903314,0.5889738236596836,0.325651236343071,-1.5490013437114136,-0.0665517191752198,-1.1653309239591563,0.006058043550339012,-0.9796602483709111,0.6868610159859425 +n24,0.008953183402845543,-2.697751176843514,-2.032054857672032,-2.318052293668174,-2.0725705922089426,-1.5094680249521817,-3.0161047064898803,-2.1530766356248394,-3.106223238889405,-0.8449699374606009,1.5635196409564003,-5.187526436470569,-3.435239613574393,-2.0362753018040025,-2.601798882734939,-1.8387644270225143,-1.3808027457024408,-1.209493273066028,-2.762924166680172,-1.2994488941661275 +n25,-1.1344768697547511,-1.9546831634233617,-0.36422692024375036,-2.2729243477548002,0.326204287842895,-1.2810377063526541,-2.570653630579727,-0.2122212238710539,0.7414615981413186,-0.7794842305421736,-0.4306007094966692,-0.212221223871054,-3.101044601347379,0.06740054128197637,-1.1659040458834262,-3.554148778345813,-1.7897794589509248,-1.3439654807835568,-2.7205296287771263,-0.6458020148869453 +n26,1.333845522928305,1.9966388538335382,3.187784635037053,2.3896501743116714,0.9384027585153933,1.4016128708111029,0.7984648244261483,2.501970875082663,0.4359756451286776,1.8227287698337353,1.2021206635242836,0.19901543642233974,0.62841962032192,-0.5900932034790476,0.1396258193438694,1.6872965259789416,0.5278733464132639,3.1347991414516927,0.7149686526796032,1.6465186543424668 +n27,1.2005583051670987,1.2039685318500646,0.08566531600848615,1.1536657473560323,2.445141784875552,0.14977944475827454,1.1792598120014808,1.013728908950658,-0.03737160185932205,0.6929347241747478,2.0776911441870785,0.9850895365146557,0.18259466090878662,1.5474870991918352,1.491611041781591,0.44890678102779863,1.2069154648984202,0.530919793192149,1.358490263034693,0.5848264140942826 +n28,1.3041900750212607,-0.780839901490126,-0.5028007411199922,0.15648008487470616,0.4580380535593787,0.0741308471848664,0.4976492044605562,-0.12628994223092402,-0.0803655248716443,-0.40921304214458526,2.697603950747711,-0.19468787050556455,1.1929582766068114,-2.5643364080862283,0.45803805355937877,0.4829905281800825,-2.154828751777469,-0.9407728162436984,1.308917748113253,0.20879029420026482 +n29,2.8816644117037464,3.2323555761381426,2.4085323894655573,2.6137081827693316,2.581099667414798,1.7466372008659887,2.239383686345517,2.8786216583172677,0.7549058401241783,2.456632614959686,3.0255256103813464,1.3447558738086107,0.9799650410991394,0.622341340711642,2.409457502470946,2.5524171829942004,2.686896665610643,2.5390183878340404,2.242485271337162,2.911512829845155 +n30,2.1981903739890587,2.1273900765220595,-0.4472208241554743,-1.0731779089447526,1.8995924419776613,1.4636969149304542,0.8989380109315733,-0.2642710165554376,1.128368670375934,0.19991557973537724,1.2908725392369278,1.7889605662134058,1.42453171543096,2.958459461323507,2.875443236787315,2.0990450819002113,1.9485132435467234,1.2377176132363807,-0.740910897617621,1.32477186794661 +n31,0.3787575686029218,0.855798294930123,2.3505468438469856,1.1136410383554878,-0.5416428412433646,0.21610951388837954,-1.1500357304707305,0.2862445247901681,-0.5808114634378502,1.5026529339633692,0.44769947593940473,-0.4032190401490458,-0.24655238963887355,0.5265617562832454,0.2850004466153995,-0.23487039996627124,0.6378708567542802,1.128402447402084,1.2549126088426261,-0.09152937481808097 +n32,-1.9720507132145093,-2.6639139119432946,0.30281233783449346,-0.8182392327306706,-1.306351541936375,0.7673960537193466,1.7718396768926468,-1.9759296002495674,-0.5619892081161505,-2.806708601351089,-2.667714633892289,-0.7936218075973608,-4.99754426264225,-1.1534259559818762,-2.656441469204429,-4.353432737412539,-1.0222021985675647,-4.054357305227756,-2.5492368479088654,1.819421867741265 +n33,2.2347939288653933,-1.8077837518702253,-0.6745979712094765,0.9720728003634194,-2.4789026925300104,1.1090212563014976,0.006779305792587503,2.6949652298002826,-2.3959742624078793,1.446404596916764,-0.8878939052679048,0.5028977616983819,-0.015263955666205555,-2.6528279017730987,1.0754749220550623,-0.015263955666205687,-1.9670876892386173,0.7836878312734175,1.2730637080898555,-0.4950943660706462 +n34,0.6115842035902399,-0.8184872379688188,0.7090424344822119,1.098819801781727,2.824240034757539,1.5250925730473956,-0.8875233931784177,-0.7146968002319148,0.0660363189529731,0.6346409074059224,0.582409518177795,1.5002093005617565,-3.29308086487487,-1.1348682613173917,1.765202392670798,0.3663840304200308,1.0327797574347863,-0.5242352400898436,-4.070216390933116,1.6074044624639623 +n35,1.8454987298428276,0.9484921193780258,2.375978503108257,1.92334608748487,1.4505884895932382,2.6832977989688085,1.966684276761289,1.5600064124525013,1.4354596108197653,-1.0063672652512805,0.565468639049194,1.5754017045783444,0.3219373908400667,2.131152281438799,1.9101202153946624,2.332383871781975,1.8537219838237833,1.5606025757021416,-0.7381998852731453,3.0717194666987973 +n36,-0.42533918966724965,-0.37441994854976085,-0.37081244997787705,0.08737753274371397,1.1138154128743076,-0.2984518586297661,0.37470481843278475,0.17881496144592768,-0.4445170626853185,-0.2753900941878232,0.18738372882176837,-0.9407933676675871,-0.6495813622113584,-0.4728152234524113,0.7307204254498244,-0.052588063386674225,-0.7132801963674792,-0.7815072074699967,0.6029199265763299,-0.6161798925622826 +n37,1.049642107899611,2.047898677793476,1.5158167648656733,2.139588639222765,3.037235570890213,3.2791947243046846,1.4775472396872436,1.2093914340572733,1.4784787484357313,0.42360145654026166,1.2766730867844438,2.177375641696158,0.548693337267033,3.2389327566400117,2.477841714741453,2.1464841664939014,2.6108654332486343,1.7058481278463418,-1.1996899047530754,2.450166382731354 +n38,0.19273624259657016,2.3665730190324434,3.5625897296968914,2.830495570538547,2.6268929930842804,3.301537107989625,0.9523340315920986,0.6817470987708334,0.8715545536666429,-0.8080695579035583,1.572734205696444,1.743583131372528,0.30868251163482247,3.7956121191702032,3.6316055093074375,1.5322348709485853,2.097045777210807,1.1936571475650273,-0.0691367252740024,1.9229334738994888 +n39,-0.28596302556558456,1.456471484886637,-2.1273610435656423,-1.3420988295614582,2.041235247921728,-1.7404027142029015,-2.188985776316738,-0.5112501264221913,0.6606668866329227,-0.3475815491094088,-0.6439926510005464,1.2396123149051423,-1.2420591017127502,-0.9478055019623481,-1.655727633060642,2.183581580467072,-3.2263185704693034,0.994679680875562,0.21440745125139193,-1.6635846234632323 +n40,2.3132654056784214,0.4847630673394706,2.9283221558625616,3.2381302472810223,0.1561807621591435,3.6695217066179233,1.437366822404802,1.9059758433208878,1.423853918888574,1.1265777534655639,3.5511377023925093,1.347040994746794,3.5348336977608334,1.2183566600701614,3.7820884656534663,-0.10831029003173345,1.4992560543882065,1.5556258192421728,2.732782353857303,0.6133510663310063 +n41,0.655736660948395,-0.7308581783319246,-1.0682914188220114,-0.49441338192530726,0.3451954072160138,0.10160041632692882,-1.2585762966141192,-0.9799610427255107,0.40412650874601275,-0.04479398593308432,-0.06750633119021476,0.5821270616781323,0.09858327129151993,-0.3264384878826975,-0.5953034717374975,-0.26150678558220464,-0.9570818367028903,1.7553971949519682,-0.4282044144052796,-1.8427387675983535 +n42,-0.908069091051079,-1.0705222204437144,0.14179259212081602,0.1417925921208161,1.3296575438852778,0.14179259212081605,2.4253182209153152,2.793211303994048,2.8655366037783843,-1.9337325583793923,1.3673578244831253,1.7615889590693061,-0.6272171910107742,-3.08172841967402,1.4357332168383539,-0.31666839360664284,0.14179259212081596,0.834625786858997,3.1763260835793066,2.789478373731698 +n43,-2.026885508763923,-1.6478823170046575,-1.5185347547425903,-1.5470381985388406,-1.7766708981509882,-2.3680950510270873,-3.759383418946794,-0.9934765856595288,-1.6038508771465614,-2.8577893268837036,-2.4234373151497017,-1.644784795380078,-0.10435191523954229,-1.0227776991503847,-0.6170500574510357,-0.14684981843552253,-1.1070006398802161,-1.7089423521176625,-0.03827901481220108,-0.8210875778055031 +n44,1.9782165778857406,2.4971035458597535,0.3644089684867696,1.7449204979605129,2.41334654224,2.5475071426579543,2.537684151672752,-0.8401877846721066,1.0071691275866428,1.0689013471478748,2.438689699127599,2.8945982518576763,2.813633529085313,0.4202396771709477,-1.3256782364852642,0.6912876825574975,1.7595442730495903,3.0854195593312035,2.0810711786285223,0.5974877359604659 +n45,-1.2357179133198495,-0.10567655720707268,-2.993840344198284,-3.0528831995734476,0.391403701695761,-0.5898993116910962,-1.7157784157526668,-2.4791509244967225,-0.5383877285706996,-1.5476937538384845,-1.7903070257433267,-1.52799689347417,0.8742671904046002,0.9404950657924764,-0.3527752344158153,-3.9793147613079123,0.10437870128344905,-1.6855103295262552,-3.8316555729226542,-1.3021069691739928 +n46,2.371554565914412,2.933594747034078,1.5779583096916423,0.4394251457996343,1.6683516803313363,2.3653350529054613,2.0364774334108735,0.8262820608340661,2.259272479738236,3.013914175287636,3.5134559549026023,2.9733582543276467,3.195902414209467,2.283528963670391,1.7020095168367166,0.5854410488857545,2.8747409181041896,3.297679102651182,0.27139159278993646,1.8498159982395612 +n47,-2.1054113796832947,-2.2974055361968877,0.28369263730213384,-2.9753018037153347,-1.4738266872083146,-2.456024498291632,-1.2031851014298791,-2.1581516258841456,-2.619080906077712,-0.6658310688261767,-2.6091661673712907,-1.3410855214526307,-2.0913897761968028,0.415827239447925,0.3687395716724787,-0.38405155788683043,-0.09598990086921871,-1.8656141870444372,-2.2109589279980724,-1.8381690294712232 +n48,1.9908186864046646,2.553148806062627,1.1131446989046743,2.26571322933813,2.0463252658580253,2.6539283864431273,1.8315483493693807,1.949735456828448,1.8927857685737968,-0.7345071160923011,1.886167396014734,2.8129797281315465,2.6791716092681788,-0.1439852903869049,0.592932944796441,2.009875154138857,1.0962512609082105,1.9125346075843555,2.3650065342748188,2.8547582244980987 +n49,-1.966242176581951,-3.1302583251331337,-1.6578731005639742,-0.9831403903917538,-3.562398440747621,-2.5692329504417333,-1.8351512682714837,-3.1797062634007016,-0.8933545208999607,-0.2105528203966709,-3.085901243553049,-2.787816156967917,-2.8880891214941533,-0.20994053305544244,-1.096154254525679,-2.424008808155816,-1.498000142341162,-3.27063645296581,-2.934019557216232,-2.9951273992634078 +n50,0.11705105624194413,2.176698459241382,0.6755864318056982,1.0158394958245838,2.570111878360429,3.3992004854763103,1.931742325356082,0.4868414995312109,1.162866252958094,1.3334082326960712,2.097551296691177,-0.27298823701787,1.6146835388847445,1.2888643159209046,1.127908770221268,2.760093726247421,1.9759020637406142,1.4232491415192754,3.6689468239981546,1.5088123288561115 +n51,1.1526343310967746,0.27415841798724405,-0.9772861581369696,0.32976396419887016,0.3593552590897952,1.1982441147140337,-0.8007968621611771,2.306515710924556,0.14349975661750947,-1.555425932463211,1.3477329024453262,1.0217153312378648,0.28519707520760806,3.107002124093903,0.40220228452272394,0.24960810050481969,0.8079139338899379,-1.2976272363078738,-0.3416095088507729,-0.5279263284757744 +n52,1.5448227374406478,0.45556669693980445,1.8387550282302378,-0.4185380875764249,1.4493092039919686,1.5811344446429414,0.9766772880869925,0.5313266393428537,-0.2024965024553188,-1.2681043542755042,-0.36484060207850394,0.02569057262049028,-0.6329581321024961,-0.0469437943914478,0.34089712775257897,-0.21974344052885178,-0.3840222849276754,-0.4179820364148702,0.5926127541818981,2.0223918573932513 +n53,0.8000590412584031,1.1735985097947477,1.303355068199083,1.301939644455737,0.20113918841657566,-0.46216147779627287,1.177977041163135,0.7261977541833534,2.0292200786353125,1.4894764630710755,0.6078111583491784,0.8229958754761095,1.5555429366916018,0.1951577693430861,1.3487825165741454,1.5079172597083503,0.7570067741738677,3.0951054096292157,0.5042824492817786,1.1402625565960651 +n54,-3.1801503616972706,-1.6485721354856777,-2.3123595215644825,-0.16533466171106698,-0.7027161167901804,0.25971863341290874,-0.24493228025341396,-1.1303420804652697,-0.5433120040188893,-0.8585258420434966,-0.7789237103460669,-2.439361359582616,-0.17814583019377372,0.11803684347966814,-1.3859288697954875,-1.7490008953681144,1.3861909770422323,-0.5574372029867056,0.26312016241516634,-2.3590676278410134 +n55,2.0120082848110292,1.7510424488862493,1.6863505188563208,1.0340112110069648,1.8488896707759368,0.9643817386977427,0.5382038236674698,3.0960301117130236,1.325266280902563,0.14458589278679326,1.7623644758350387,3.3642404668591914,1.4481548803602622,3.571901054626136,1.6800708898003214,1.92650957609904,1.769296544303321,1.3677726156692847,0.36570369508702216,1.1023030425861375 +n56,1.406150248259862,1.7605005435419836,1.2368612597727344,0.6187170602747568,3.0213642163589007,1.2910211889318457,0.3037776198827719,0.8170932237939847,0.542793869815641,1.4317184202886957,1.1707838403821316,-1.0168031552486154,0.5477623488987398,0.6465773158777351,-0.21230897338251978,2.06165313415372,0.18615809441961706,2.304256941320816,1.8917118612278325,1.3444843711084897 +n57,0.3264409864085098,0.9589171080783377,-1.3985230535609046,1.7696611953915606,-0.1886988714768607,-0.3050634944352115,-2.6280690719810527,-3.4456587565098933,1.8171377127580357,-0.634739079930355,-2.1243043373820765,0.54210957445278,-3.139125235456773,-0.784825344883587,0.9819689215115219,0.7268975918686911,-1.1132649578443576,0.9612159756673115,0.07321150761731032,2.24265854027539 +n58,-1.8348659878421658,-2.037071698946572,-1.7357665779927685,-2.6051325094265434,-3.149074724659207,-2.8477466766117017,-1.0634988441957232,-0.2617112100733394,-2.6291284228680087,-1.61872158753043,-1.9535496295247552,-2.057016089836798,-2.0168424038016877,-1.0912307938431225,-2.8875633778108893,-3.7428296808436077,-1.6899594302534788,-3.2251149307404012,-2.328882891653149,-2.7652989511467627 +n59,-2.6457363249610424,-2.3220415011237563,-1.0896854018358297,-2.882696417428936,-0.8513810324874789,-1.863182904176545,-0.7094537348662443,-0.281928753077635,-1.0171175133041739,2.4740160651566123,-2.003531683519393,-0.8948714141528444,-0.8359869345109139,-2.889031823657595,-3.0623533345580274,-0.05861851995933942,-2.9045227557709725,-1.2397522975077926,-1.454177374270985,1.1225152575891144 diff --git a/test/fixtures/ilr/philr_d60_pruned/expected_gm_counts__mean_descendants.csv b/test/fixtures/ilr/philr_d60_pruned/expected_gm_counts__mean_descendants.csv new file mode 100644 index 00000000..9991aab8 --- /dev/null +++ b/test/fixtures/ilr/philr_d60_pruned/expected_gm_counts__mean_descendants.csv @@ -0,0 +1,60 @@ +balance,S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15,S16,S17,S18,S19,S20 +n1,23.51308127643658,-32.38968570950653,-29.456882052526282,-35.92391853118829,-1.2846171667072406,58.050349849059735,-9.067634215766091,10.039388026357866,-67.71669122296937,43.25252726725598,2.1927844126112586,81.61031162774529,1.4347566234692462,-31.36423004563588,-58.75214423167043,15.437220033717749,15.031278287015107,-1.3907582552464812,-11.000264925194632,27.78512895274247 +n2,-24.6458302795433,-53.210744454152724,18.403595207982757,-3.6953014173929293,17.759123289335644,-80.43134048858263,-10.493420711644937,31.598392479177527,19.56574142275215,27.28565768766906,-5.499538074459712,29.109737026241582,18.6221103293121,-46.53834129305089,35.97949779261691,-6.3243573180340515,10.085162926393375,22.180241362152366,-9.776188122158082,10.025802635385844 +n3,0.2614767251460201,10.81256423523603,-1.1393732317917797,-0.9428518786515105,-1.7606752887036137,-0.11353833485546704,-2.1276313565808316,-2.2791707406574533,1.0683470020410064,-0.6697037985325697,8.596328845725177,-3.2072043009746403,-2.750482227977715,-0.8319510127799168,-0.9538270330791083,-2.3363792114348674,-0.1358064012529238,-3.073664221938428,4.090673310734671,-2.507131079672088 +n4,13.939689615291789,7.853092422360358,-32.521955163660756,-23.3051246253449,17.58068688462176,-20.87209333540232,-4.932128927320964,16.493751640641044,24.432792324092944,-0.39560860349593246,2.809254847227777,-40.95360297795076,-25.06941039016445,16.321745122230713,9.706632212184221,35.1634339884805,-12.775924965159199,15.418698333760977,-6.484782462212109,7.590854059819297 +n5,-9.141178591318182,-9.755316040327239,13.66201409530933,0.44513727598131886,-5.885224808173241,14.981926725625032,9.631484374741008,-7.312372023405028,-9.910351312855743,-4.108857399807431,-11.287685710547976,6.45344337830775,15.422516915054624,-11.781239832490794,6.702153036968671,2.2859063051493504,-3.9826983665011095,5.578514588577183,2.0821010356095875,-4.080273645897141 +n6,-1.3006156995259115,5.68055175788071,-1.3959388534517578,0.9081591701986567,-1.3892511137188241,-12.654787158969938,-5.472697203243908,8.19318870548662,-0.6598807908565402,3.9658243033108707,-0.7198130959831543,-6.238953260472617,1.1330176819503162,-0.49090367688884384,-2.741175237585618,1.0339717979905438,1.8046728768874192,5.094629826359237,-1.738569605524438,6.988569576157193 +n7,-0.2176970831583033,1.4837735272211632,1.4837735272211627,1.4837735272211634,-2.8597745621982553,2.7090633304881906,-2.885387605327061,1.483773527221163,-1.6252529948932866,1.4837735272211632,-2.100305054761066,-1.7923421890598412,1.4837735272211632,-2.149741999881226,-0.2985858449839806,-2.661051806943725,1.483773527221163,2.855380813567845,-0.8444932206185932,1.4837735272211627 +n8,52.42804339850013,34.317143239351765,10.79561344309079,-30.45469102640307,-17.275234534837406,10.258045731155796,4.438663627835149,-57.729203134456704,-1.0357574250869268,-19.22949408342934,18.78918506758667,17.061623154954667,-7.014960990666084,23.18417731206437,-21.751522699023845,-3.746737275872508,-7.815342705897956,-10.683691244231214,29.088777197490703,-23.624637052125262 +n9,9.163738330739685,-17.177373702396963,9.138204925625889,12.516878438671247,-19.558690723247995,15.327282557807777,-13.10954593425269,8.644460956324837,14.214209619372012,-2.3515907130306597,3.4209913373040655,-5.673035862449371,6.3188366268194125,-2.061851612818955,12.369108935226238,-23.447663899748946,-4.3642652987917305,5.281852454196528,-16.07053239886041,7.418985963510037 +n10,-1.0885128152696832,2.554037571783373,2.0878694316455224,-0.5777971606705457,-1.0664148107914833,4.070203374020986,2.346104773776218,2.769494974072886,6.447874577841471,-0.5777971606705455,-2.856127122373728,-2.9139975544674064,-3.500569866583278,-0.14549415349387346,-6.145116075244433,-8.90759963497548,3.54269556825173,1.4898882952676515,0.055991737598920455,2.4152660502816854 +n11,-27.011024848378998,-5.0396667333048315,-1.178411422343808,-4.599994177074166,4.2619944073330425,26.299553531394547,3.6612300131483226,1.752804102829124,-5.830820530533917,-3.8741273430236007,-4.214449141791616,1.317783668966297,-5.677146405436793,5.657026886448002,3.5695676578849036,-6.824004395025168,-4.201396644560346,4.093882256014132,21.111158347696623,-3.273959230241554 +n12,-7.932918016426963,23.95263843282766,5.546388347999658,-9.056491031454472,-11.523390986661894,-19.73819972915817,24.129743832650863,2.883476480896863,22.30368695374977,4.041714232231499,27.502936258050433,-0.25928356226126187,-4.1382898130507,-17.750505470289593,-17.23811760953531,-0.8821064227555716,-5.612305965403807,-1.1938637325494539,-22.654351994506488,7.619239795646869 +n13,0.8470561672427297,6.307677138327271,1.5845148916401102,-4.636133727923476,-2.166932925008934,-1.7394094754551033,4.69946810513792,-1.5129037640625667,6.08362987961172,0.6108195901338584,5.985922892653788,-0.37216896212599065,-3.733928558359037,-3.586774990906436,-5.856150058942303,-3.971414042531641,-1.81930052113574,2.9517449826068005,2.11002078675319,-1.7857374076561667 +n14,-39.261909605708844,-21.281349433326056,-1.993031160335405,-8.236807649920179,41.56789555152458,-50.38345711048221,16.461572434387776,-21.410292209410805,-23.63423136026061,42.40612110102754,15.75694484277399,-15.90225065443187,-15.932971567729066,-8.275305289008312,32.639236595782585,42.01275166277106,31.924229397559454,-7.62694312621514,-15.517968462947614,6.687766043949334 +n15,-97.97100437964181,9.937629978972371,22.064990083157376,41.87079990925916,17.56482048942107,1.6747633874107364,-3.0306605916607623,10.990461805698915,27.269465081157136,28.642064819778845,-57.84240180036221,27.570773740288885,61.72052217054723,-17.40403888465484,-28.35076787603607,-3.70682336673173,-6.140977293866815,4.645101010697157,8.345917347142466,-47.85063563057719 +n16,10.534253977499688,6.880656587893292,-5.691504899423229,4.094239346195098,-7.116309389811259,7.290741069025853,7.37574588051226,-0.7450452381860279,-2.597435242878635,-6.17127064187315,-1.7070916060498835,-2.5751929479168894,-4.8907495592857995,-4.562279253444241,-1.1595987856162402,5.6300824797895626,7.061267931694079,-6.586541612633348,-5.468174415376862,0.4042063198856913 +n17,2.897646437172527,-1.2213815325413833,-0.9699242930777541,2.2004540342903613,-1.462634369585993,-0.8585048894165136,0.6152315928941606,-0.8034241123897528,0.10003253782943329,-1.1358315577328197,0.4079212933604102,-1.7949440414522635,-2.7282372461623554,-0.5794280122344445,-0.2744713817505065,-1.3452701795007385,1.6018009464750498,2.3981021704764314,2.6865562584865996,0.2663063448595444 +n18,11.195064271684378,-9.696911163717417,35.441045480189544,-20.03020397995963,-0.5195916894339492,1.6234407080116604,9.547965471663998,33.578218685446714,-13.948459941268794,-15.63599835311685,-23.340292018373347,-29.1860715126115,-14.36678641306995,-6.39820423891028,6.408153647379842,1.744506306716313,-1.9532605102769265,46.72996184368193,-8.951874898420078,-2.240701695615907 +n19,-15.772767611200276,5.397023635927494,10.645875032096635,-2.0323376945755753,-8.68553110793079,-8.779575945633486,0.5313068777097001,15.160820594614592,-0.2070611137739271,33.76473798000278,18.48901817526529,-14.812691425464173,16.29248869723212,-10.370319443370056,-16.74631683503589,-22.303646540865845,-3.556670099414505,16.276114509958692,-9.318033149722842,-3.9724345358198083 +n20,20.26612718209868,-9.303020095981186,16.74133474907717,11.311559482768011,12.206606719597422,-6.6053938875523635,-15.160362215883405,10.493789562307827,6.641152947046459,-17.026200525670568,2.2243478631183757,10.14178130512039,-9.678782934319212,-16.70786702820416,9.07168093241197,9.248204259567206,-4.064743285303949,-6.778798010833488,-5.701957446999632,-17.319459572365645 +n21,-5.509464034259641,2.287926548374101,-1.5797042219849173,1.421342977144076,0.9228236724406732,-2.837273090216546,1.7097700120834254,-2.136162579128875,-0.5726472650612773,1.00873442074623,3.6986537612947994,1.2816647329855122,2.385610397041168,2.1120615919716785,-2.784550577476834,1.5331327194454143,-1.8092481318258253,-0.469113490661684,1.3517891155199688,-2.0153465584314216 +n22,-1.6343591542716471,-1.0249026517933675,5.917039416920927,0.483907983165728,-2.037926026569704,-2.0002165052881966,-1.024902651793368,3.160134158076099,0.7189792322751242,3.1601341580760987,1.2658125862540246,0.7774689335303612,0.4032288992312711,-2.9110321915242072,1.1204716450137227,-4.481322099509232,-0.668920552321417,3.1601341580760987,-2.2183821402473622,-2.1653471973009557 +n23,-14.133661629836633,-10.597096546071537,18.86096543988144,19.41419192707751,-8.647687967355441,-21.50048034193897,32.306955724626555,29.060268823725174,14.968385739180103,16.42456018078561,-10.944383753291875,24.879180670617043,-5.978994393471334,-52.98743447379225,-41.37741049486425,-32.84527909290728,-34.51638286652923,-31.123130267870167,58.69014516703511,50.04728815500066 +n24,8.716424413403924,-3.9802061712088443,-2.2235918139045485,1.1845636754643356,-13.10949828138266,7.005086563299778,-0.0733678839561124,-7.617965933400908,-8.07614830142904,4.8295668538255985,11.099310407405996,-20.309501989149563,6.395078912131955,-3.2421642012063785,-5.41958783812886,12.532402061303747,4.696133343794054,5.2669779275944615,1.4602871245846185,0.8662011309583999 +n25,0.07701778795039192,-0.6622071438492079,0.7712187112926916,-0.9490274469841662,1.3934816305728226,-0.055072669383200094,-1.2173610485304176,0.9082164414998889,1.767739374330959,0.3969609675345557,0.711398200023128,0.9082164414998886,-1.6953849703405726,1.160230333456742,0.04869350866881894,-2.1037528491599837,-0.5135848635361646,-0.11178740093435705,-1.3524393518805662,0.5174443477687511 +n26,-10.138898674902928,-1.9415342542672853,28.130999883498728,23.411466120562245,11.722427388348269,-1.1703439794595922,-8.818106728623006,16.803338296655635,-5.5982313387275395,2.866900558753989,0.7920882433176307,-6.267769782426183,-1.3567637113410451,1.5998846060232061,-24.36075548910804,-7.7926308968457,-27.568096691260106,32.24967921232206,-7.264723774728881,-15.298928987791564 +n27,2.592086742592261,3.528442871882751,-12.090392152727176,2.4315320982326756,20.203485097720865,-11.4477487980612,2.6407996744024533,0.6125108518645406,-13.975881820323028,-3.7132237345749903,14.15543954113641,0.24471990068912713,-11.469997600860934,9.050302272384995,6.987043956334994,-7.476029486728644,4.155178725549907,-5.732171295366866,4.779091376142288,-5.475188220290291 +n28,6.616635173039722,-4.420888019895127,-2.949032115806774,0.5410027947459776,2.1373602144798043,0.1050706349393778,2.3470497641246055,-0.9558968985943141,-0.7127868102812495,-2.453606913458937,13.992950211167395,-1.3179750117076745,6.0278074014383085,-13.862183482231734,2.137360214479804,2.2694511294545356,-11.69437280079732,-5.2675248912075565,6.64166205610103,0.8179173500101545 +n29,1.2374990750903265,1.9304605550173097,0.30259594783359567,0.708020822458783,0.6435867923207745,-1.0053009026281436,-0.03163983432771969,1.2314866313689472,-2.9649500216415356,0.3976414089788489,1.521767055723607,-1.7994135219233955,-2.520235775047028,-3.2268958617850947,0.3044239598918629,0.5869105516058537,0.8526403752927512,0.5604346951684721,-0.025511140069352647,1.2964791866711505 +n30,27.367240297346694,12.566573775331957,-13.986681314664338,-42.1760292372615,3.9996505229157355,-1.8304472894193746,-2.1714061991916997,-28.0241449095253,-1.0541574421783373,9.182752210516364,5.566733010534675,7.460616854690687,-9.721226221184724,16.276391151032392,21.2523990443115,-5.206187476300358,12.33220917802332,-6.43016291150535,-11.678471350562585,6.27434830709055 +n31,-0.038650217547474636,2.2605233411709613,9.464701414313476,3.5032374015725947,-4.474666198918262,-0.8225583539780936,-7.406912372098117,-0.48453152795146304,-4.66344559554199,5.378142027210587,0.2936262563209992,-3.807510700846462,-3.052430892128186,0.6737155437551998,-0.49052756033727085,-2.996127687210528,1.21018743498782,3.5743823573306943,4.184118176602842,-2.3052728467073185 +n32,-0.05084928969835496,-6.250879808446048,8.09008895373499,2.2013926798941212,1.0516379008513457,6.687738193995587,16.97069054704688,-1.2564458609409552,6.836737861388548,0.9133266163894742,-2.7219626555175824,3.351672896769629,-14.817264271242243,-3.4925442610722186,-7.570895803962873,-15.115759637223324,1.222459932908768,-12.839715529590379,2.0699592892274272,14.720612245487219 +n33,2.4944130223314636,-1.8910737522480554,-0.6617662047614277,1.1245823730859408,-2.6191199324015404,1.2731473951453909,0.0774084686236501,2.9936180626966467,-2.5291571506669253,1.6391490641504185,-0.8931548314642324,0.6156098586347291,0.05349540145939866,-2.8077982374682087,1.236755514322808,0.05349540145939899,-2.0638905453048064,0.9202177681926988,1.4511046441095057,-0.46703631989686867 +n34,9.261852292028545,-12.376363881521247,3.2772168056774147,7.5975169872485235,26.148704643571183,6.111897202307288,-10.284821094520959,-6.398091466113978,1.116428424291488,18.1006839395294,8.857737065347338,13.34972657069965,-31.89808324978743,-24.43184575586654,13.516845693344017,0.03941947215438331,5.568129917281501,-7.471490894539973,-31.16871925695423,11.083256585823591 +n35,27.604279280901686,-19.07890831081549,14.037471612986332,-2.9690084031439232,-29.408797719569147,-4.1304097558595645,16.933475747109114,14.050700833145052,8.530862972621208,-21.60073030353008,-13.897147884797505,-1.6403853456139594,1.3696964263617049,-17.841733282608747,-14.952196113976825,13.552940232986288,-4.624323158534409,7.420544304487613,1.2350143752642053,25.408654492586308 +n36,-2.5967451028088577,-2.1040001182837114,-2.0690903908611076,2.3648094668526904,12.297638516562573,-1.368857675652605,5.145272722654965,3.24964854540323,-2.7823291872556286,-1.1456892129324359,3.3325684211780118,-7.584790002695713,-4.766734352750407,-3.056170204585366,8.590432173589328,1.010363799836776,-5.383147330667852,-6.043379431384692,7.353708030461885,-4.4435086666610255 +n37,2.794895785679306,-1.2751547120487228,-14.07692803192243,-3.329125126906752,6.73380389290394,4.807930326153026,2.1452503707583923,1.2462336471657902,2.6679454327157264,3.042707751041751,-3.7614389673796738,3.9505756784887467,-2.835674598182525,1.2733134651821791,-5.118446896097528,4.992572667017789,5.937689493732863,2.841014321131172,-17.51211420985267,5.474949710419601 +n38,-7.332035515530445,3.1357991195270705,8.895065539026898,5.369759187403908,4.3893368598558835,7.637999827055537,-3.6742889309882614,-4.977265906023342,-4.0632722373689445,-12.151288614476604,-0.6868307646337206,0.1358705167783509,-6.7737109973326,10.017155231184862,9.227402251701923,-0.8818501623943461,1.8379249517812133,-2.5122281456503863,-8.593051500159842,0.9995092902428738 +n39,-2.8504334573023957,6.2348197708258235,-14.228309987425519,20.372220869197672,10.318576197219913,2.91388870961179,-39.5033564676018,12.557147670609833,13.336875257257917,-0.12292133915762046,19.20898827701757,10.129585260177796,8.429792228907258,-8.91335220348843,28.111317005311914,4.6011893654898355,-55.11855958501763,-19.228308234346777,51.843070580829284,-48.09222991811617 +n40,4.823778157469908,-3.1743147357362784,7.848894984059964,8.849025281437829,-12.08177159001587,10.49389779056869,-5.632780154862104,-4.269879465166769,-8.828084526545096,1.5915948757626988,7.111235191837434,-6.827281168903554,11.596570547497713,5.201063830741632,8.879811609135169,-8.843684648248031,-0.19056926479800965,-4.9090172509576595,-1.3022549928632006,-10.336234470414364 +n41,10.544183271820339,-5.497947502116947,-9.401862343041193,-2.7624127681634874,6.951393754687155,4.133135249017967,-11.603352502508418,-8.379929039592145,7.633193809479727,2.4394334703163865,2.176664273088733,9.692561165449622,4.098228559730787,-0.8190367071589507,-3.929654970940955,-0.06781304025167144,-8.115229356150929,23.266643787091795,-1.99641185335053,-18.361787257407222 +n42,-4.096114008375683,-4.517677625268529,-1.3717375063664825,-1.3717375063664827,1.71075556297771,-1.3717375063664827,4.553979818631764,5.508657265058119,5.696340414554445,-6.757696475997777,1.8085872680739608,2.831611592746991,-3.3673072243212867,-9.736729808718064,1.986020462592924,-2.561437403623671,-1.371737506366482,0.42615502158437973,6.502834802732497,5.49897036281814 +n43,-12.541410473275574,-9.551712186985991,8.494847549131961,-26.345317613541642,-88.93033145204635,-31.21245272442296,-57.9680941582123,-21.567496779769183,-18.22690651007959,45.18788130151833,39.3920230745269,28.369623759742872,43.54426726869827,3.2829278645302447,-7.1726873900263675,4.4698070297836,48.89578109947888,22.7693835149591,34.176826118741445,-5.066959292752302 +n44,5.272551835377745,16.00100463645604,-29.721730539812807,18.396867733025257,40.325592955590984,22.78101338795116,18.727497674727825,-37.620015816584306,-7.236792011509782,-27.533362599214144,-9.510360915430507,11.603336163793987,26.398311901914585,-7.437698003783034,-44.931507378816455,-14.05644676518397,3.336691184119779,8.663200752284581,21.754128697415947,-15.21228289232258 +n45,-0.4353441878680994,1.4864073618612728,-3.631619446957678,-0.8995873155824506,6.19408852562013,1.499071246005364,-1.0057853250257704,-0.18310365186867888,1.922944804523357,-3.002500964688547,-4.998893711260516,-2.840421335438166,3.727680057733934,6.254318036910486,3.8974058635211315,-4.024612436512364,2.260711259294532,-4.136552741412312,-2.7839847180149078,0.6997786791593112 +n46,0.5667439594063463,1.7483329689076652,-1.1016503046232715,-3.495212755492124,-0.9116144017330815,0.5536685451162381,-0.1376958126979568,-2.6819152468626717,0.3306909449664372,1.9171902087668695,2.9673875014300104,1.831928634797081,2.2997879485586306,0.3816858663015663,-0.8408548172152578,-3.1882404213161886,1.6246033143977492,2.5137552422254243,-3.848473262718106,-0.5301181122153747 +n47,-18.021831285489977,5.346259965164715,16.496375315944235,-40.239582942379,20.617762217186804,7.589459276867324,15.033113171948143,7.0073205898615685,-53.803100079664645,-45.06885139860869,-11.194236961834163,30.213546177090503,12.101352911357568,-15.717298998946816,1.7055777537974044,13.716520643606872,26.713869860883154,-11.638058347043057,16.483528157496735,22.658273972761013 +n48,4.9099591616575475,0.8784371521144442,-4.243760536956497,18.832939457978927,-10.656046798229545,8.036995431426096,4.018191936578141,-8.09218122778226,14.521172402657944,-15.340827796619683,-8.023308876159962,8.029771009251267,5.126008128210394,-7.051119227246122,-5.788945436215794,0.48974737745431957,-2.843633675571103,-9.545133134892227,0.2487870890482735,6.492947563295899 +n49,0.20629565446698594,-1.0402468167650967,0.5365274562761337,1.2590973115191835,-1.5030247673176556,-0.4394459104699614,0.3466806353839431,-1.0932005109574885,1.3552488123186588,2.08645973251614,-0.9927449104055466,-0.6735262114796345,-0.7809083205284468,2.087115429755726,1.138070999593584,-0.2839256786991506,0.7077350745504132,-1.1905774617422116,-0.8300951285645967,-0.8955353894509751 +n50,-13.137564264392964,66.91175859088084,3.019344092342519,-46.35365800870746,-17.81786151799193,-18.558809892292214,18.860095724214148,33.42899110896362,-2.811250286946363,-13.910499393929692,-17.16521637724357,-14.64994123651672,-44.92216213741533,41.66659567484929,-33.606819624076074,5.689769913301693,-5.40150782522363,-7.17307034150254,38.10481009500681,27.826995706680126 +n51,43.90634606780136,1.1321037271204462,-30.36344334747781,18.32542963912934,-26.044221043767248,27.818895181983418,0.197679650541236,13.970160927652124,21.26142743446541,-21.486047359189286,18.394280950455922,-46.2691872777206,1.74437729491783,-3.7716086108445115,11.37030328167338,-4.4084666884390336,-24.20185471029933,-19.743385477614417,2.738726440294708,15.428483919316497 +n52,-17.53468964669939,-10.978431346167632,25.11011065164496,-4.323344237460674,20.046546779280213,29.519484483016743,38.95661403203165,-8.010784696508924,13.104451148181074,-44.62885367315414,-34.048698164785435,-52.68991953140988,-5.568497676681861,-13.242182282453804,-4.530205477618047,-28.088909531770053,18.148158632157198,32.48027798450906,20.031464671420327,26.247407882468924 +n53,-5.789262637272298,2.410658340831959,5.259068254449427,5.227996938501347,-18.936723291869708,-33.49746855657008,2.5067756570599182,-7.410662163618601,21.193223083308656,9.34479642733829,-10.009479189596544,-5.285754323117343,10.795084572559064,-19.068027124184205,6.256289480822138,9.749607934383185,-6.734343994818538,44.59148811837312,-12.28213657877208,1.6788690521924916 +n54,-21.797876779065184,-7.089452950739117,-13.464097117947938,7.154733228569151,1.994021184276751,11.23670821836395,6.39032205504251,-2.1126602806898553,3.524849457030297,0.4977114063103649,1.2621659216635186,-14.68375199980642,7.031701905353079,9.876075258564962,-4.567173382308326,-8.053914797756779,22.054720891455343,3.3891989172651966,11.26937460743459,-13.91265574301604 +n55,2.9750586040692224,0.8994523706911329,0.3849215213040806,-4.8034953028810845,1.6776856706680081,-5.357297168323372,-8.746926825516818,11.59688625849147,-2.4869819691107447,-11.87758884497291,0.9895027392618126,13.730112176838876,-1.5095806853694336,15.381752165914937,0.3349761437195179,2.2950398877513765,1.0446373298567213,-2.148905480718242,-10.11891615458478,-4.260332437089738 +n56,15.551284132805721,23.454340527990578,5.893869645768883,-8.441870143803714,28.026633841169126,-7.120933928075893,-9.463757758012079,12.150155598036763,-9.027861584358654,18.621749681686644,-3.622293438298199,-40.65984943609041,-23.03718236623995,5.496141158723757,-36.133604868027945,1.2671835155267748,-16.870740569858494,16.014403244126502,17.55940523052389,10.342927516406622 +n57,7.724360757372057,12.057629189323745,-6.874415524435894,14.871891304351115,-6.7810173182309,-5.804411059276499,-12.780685339841014,-13.862034629558348,15.714866122626201,2.211284262945968,-15.557883958794234,8.520021365961743,-25.022565010499996,3.2691206664720283,5.34427300619249,-2.7724158608493426,-4.467597533949499,3.0794065774148645,1.8882220021632565,19.24195098061228 +n58,13.315153927026858,8.468326224973453,5.09537685941905,3.9353480058379566,-16.27400638620021,-5.779860533501842,12.210249828357636,20.85630678842699,-7.929984252818613,-15.277138432412094,7.676080512902793,-0.6310161461384812,-0.42929180972320835,25.287562744585898,1.0693288846138345,-29.531551760762298,16.954238095353812,-14.944751595088384,-1.0024400338788757,-23.067930920974277 +n59,-2.847915308956949,-2.177601446090027,0.3743867740136924,-3.3386171158146967,0.8678723369346977,-1.2273876768887164,1.1617782881851755,2.0471057661398073,0.5246618420703947,7.754172672737286,-1.5180248001973442,0.7778115649208641,0.8997507499340057,-3.351736604633271,-3.710654334958181,2.5095411650279966,-3.383815543743977,0.06362517661204203,-0.3804106587367566,4.955457153443952 diff --git a/test/fixtures/ilr/philr_d60_pruned/expected_uniform__uniform.csv b/test/fixtures/ilr/philr_d60_pruned/expected_uniform__uniform.csv new file mode 100644 index 00000000..dbcebaa6 --- /dev/null +++ b/test/fixtures/ilr/philr_d60_pruned/expected_uniform__uniform.csv @@ -0,0 +1,60 @@ +balance,S01,S02,S03,S04,S05,S06,S07,S08,S09,S10,S11,S12,S13,S14,S15,S16,S17,S18,S19,S20 +n1,1.860406116780398,-3.456670878143302,-1.0707317571144956,-2.8194963850095265,-3.1744851064622077,2.411844552489454,-1.5149364945807575,-2.799111270517865,-1.0369705930706177,-2.225338311352128,-0.18523130067658936,1.6378313460056149,-1.6450468160289602,-5.913246882652579,-3.6411573908416304,-1.928730282682461,-1.7285329711845483,-0.7550563911884928,-2.975484760350867,-0.5624136115632219 +n2,-0.9917319985467654,-1.574746112132977,-0.7897752590672897,-4.751548439601538,-0.3942656959314424,-3.9000507433364504,-1.917684260609977,-1.106412440825712,0.015606318805515267,-1.2866492473799052,-0.4486519971939464,-2.518506721665056,-1.8831823828929426,-2.7184709299998953,0.6881843694677152,-2.4092342743713,-2.0920001745453463,1.863168980577813,-1.6664933956518448,-2.5247707930317933 +n3,-3.8265975019892355,3.1776797229919675,-3.516166904848712,-2.79643002454771,-3.611245689234977,-3.990981495503882,-4.217577663340878,-3.5346237476339697,-3.11583704784373,-2.874187570819616,1.985284657951662,-4.164962306797852,-3.892651603066134,-3.324901213622922,-3.3970691199450873,-3.8553970643887627,-2.6678085763492003,-4.927055143112364,-0.6132691111985245,-3.7694368049153235 +n4,-3.7491887431881272,-5.022092292232611,-5.651542739496395,-5.026513937806678,-0.8335386675773713,1.0972767208338396,-1.8938395616406876,-3.365061539158531,-2.5754415528048824,-4.010467257387752,-4.1639451575686905,-5.392560306506557,-4.397998343845752,-2.2792231415943727,0.6188694530394134,1.6859528342880532,-5.020687703180601,-2.2742181016871537,-3.4652365752137007,-4.140276804406844 +n5,-2.8883006701174443,-1.617192404633582,3.644673872056714,0.21098904990225809,-1.2025754855672943,0.6318352361956935,2.457106664347483,-0.17594667217889418,-2.571679576665836,-0.35074682428569715,-2.884495351877852,0.9760090926807833,4.8804484245848245,-2.9602486315882572,1.6121198989626027,1.9231044470671415,-0.9051392148046047,2.571636668580234,0.6123416932250438,0.48774668651051933 +n6,1.5916865787327859,4.57411086319914,0.8970131774626953,2.0942723806201125,2.5474181669510187,-5.417917490933381,0.4352828159490086,5.879731404875783,2.4583249919123427,3.6831013641163377,2.607306911144058,-0.3773271164929079,2.2111137301805956,2.744998055694993,0.873811008549512,3.7312281386490316,2.5601203068248117,3.7495849900822398,1.6017766317918205,5.253785232410358 +n7,-1.521525645425672,3.1532148268049916e-17,-7.47490760602404e-17,-3.5024186650751754e-17,-3.8841809960604663,1.0957050021178887,-3.907085242387225,-1.463903386116079e-16,-2.780220567342459,1.1171106596455916e-17,-3.2050318380113754,-2.9296386604009244,-1.8542209931013057e-16,-3.249240406395081,-1.5938597338276157,-3.7064748596130173,3.805231946603807e-17,1.2265481689182722,-2.0820327689585856,2.0543034757657799e-16 +n8,0.5213362053340161,0.36859458678863144,0.658677038700378,-1.4342691150334608,-2.392980772020723,2.605249092573028,-0.30656995013162897,-2.4331808207744987,0.3731365736728004,-2.1069270009982293,-0.3209069702236765,-0.2024907013455488,-0.947293964269652,1.0942389265517847,-1.5406736998713353,-2.9695391550380887,-0.6231905362565768,-1.21223877833171,0.1451645987584038,-0.5757364722363839 +n9,1.31268502326429,-3.7963693430736845,0.7573517764612493,1.7940263549253912,-3.5730973519616187,1.4654419175276445,-3.068891753800061,0.5551915061898435,0.8638086187630796,-0.7331662407254601,0.6432111781316814,-0.8924614141153583,1.2475464312373425,-0.7589090465280564,2.7346487963008332,-2.873929303277432,-1.790024501685047,0.20561704375627302,-3.174914366603073,0.4083441772476731 +n10,-1.76361150200519,-0.2662738815110996,-0.45790090640693887,-1.5536723984241867,-1.754527709077942,0.35697407757441857,-0.35174849899267124,-0.177706136447449,1.334359745402114,-1.5536723984241865,-2.4902220136834536,-2.514010723757611,-2.7551321076080804,-1.3759662619767377,-3.8422216757957566,-4.977791570637538,0.140132250076291,-0.7037120982100493,-1.293141764667287,-0.3233184787625844 +n11,-2.004772428674586,0.740466115901867,1.4983542112980883,1.854608275836631,2.2890098730705355,4.530758038613656,2.149988886382388,2.4209355479615704,0.6275037922893103,1.324277041498857,1.0527461715485391,1.9937031466048887,1.7250238932788549,2.5173338608138165,2.7800509302147534,1.7687157999889314,1.4270885425054538,1.5675047424968676,2.9476391211558126,2.062583604415172 +n12,0.0779662753531824,2.9387712023440837,1.7714946865058014,1.6077051376676856,0.4951956776253097,-0.8071732892063106,3.4596379836609357,2.346613630492858,2.772843779155568,1.857158647759039,3.5438497982741577,1.547210026427287,2.030501755232215,0.045407675389506526,0.8178150922182854,2.5676984470391537,1.2303905765154846,0.3896448812208089,-2.409070158929931,3.1055445813233304 +n13,-1.3644916607895403,1.9865476969745866,-0.9119326527082064,-4.729380973212767,-3.214097433739045,-2.9517375458420037,0.9996321439145183,-2.812736952749383,1.8490557915794652,-1.5094638300404817,1.789095585485991,-2.112698030747165,-4.17572140811971,-4.085417137088951,-5.478072882797381,-4.321459998762452,-3.000764577717386,-0.07289953354285243,-0.5894435170245691,-2.980167778409264 +n14,-2.479672892312083,-2.1932224121558574,-3.718642021574462,1.2872556669894681,-0.6419502427833145,-4.655373774746595,-3.8548869999186572,-4.16748843631671,-2.989325101739943,-3.340740859370857,-3.301653960979964,0.004356906810221859,-4.009578010319575,-4.584854914889639,0.5667968778789367,-0.41820210017601156,0.39143040526404704,-5.3464293971256005,-2.931571232736143,-4.412700751921746 +n15,-4.757741640275564,0.8579868162402129,-1.3475031930204806,1.8686343248221133,-2.564207982352577,-0.2775613267071545,2.321663332024217,-0.13642024548094492,-1.3166823148648885,-0.47846838145282844,-4.619070866205188,-3.6367843635595403,0.6384884079211516,-2.242531151075452,-4.640366592926078,-0.9617593609995736,0.5165121604448063,2.2096236116164643,0.0843313033799178,-3.0835829356882014 +n16,1.3455197661940435,2.1469107551419335,-2.455088931791097,-0.5904084097266458,-2.710717958052,2.1076993764374357,1.3860588611982971,-0.7798759243204273,-1.8999713568384102,-2.541165337644094,-1.7402317725968266,-0.9252543691465582,-1.2730687040888156,-2.252490712696606,-1.197259203157417,1.7651104770518917,0.7708143005542291,-4.491921718389524,-4.241126266767984,-0.9166916031266091 +n17,-0.7768361992120931,-4.478304972208192,-4.252338766962183,-1.4033519517180635,-4.695101228335425,-4.1522143088642265,-2.8278752635828814,-4.102717248476196,-3.2908469229697044,-4.4014274752324924,-3.0141698470833527,-4.993723593228717,-5.8324058451299585,-3.901428357967553,-3.6273861681222614,-4.589634626053839,-1.9413176384526387,-1.2257400467946717,-0.9665274832813281,-3.1414288299112387 +n18,3.033470096945646,4.78917527874164,4.719975494578279,2.892293506537234,4.041216144926496,4.167173214652023,5.785761172144833,5.334755111251576,3.0681759370296153,4.723397862165727,3.923233449438646,1.9630595963245028,4.673009762638226,5.0081493231000165,3.1336009200721797,4.341261234604627,4.003370454446179,6.957837031707541,4.282445100997838,4.642367877919501 +n19,2.8376505778017087,0.9249344350289006,1.354024168511809,0.828481697692179,1.3431367812569268,1.273489088085196,0.016904653225320053,3.1131660747773044,1.320445433946987,2.831379694988232,1.9982783935765824,-1.182815882556293,1.726658966959074,-0.8604511556462318,-0.3409762429643029,0.18331971539212308,1.1987295698025975,1.5242707143228307,-0.036938285957330244,1.030411110047823 +n20,1.72590584385229,-1.1682505015251479,2.5041802190037106,2.5602256220237294,2.549555030144505,-2.55671046832676,-2.536285817915932,1.0675528590150405,0.895472616047912,-3.1655787340531534,1.6262431196148293,2.2779479283644677,-1.2055552344311355,-2.6899179704451424,0.5452098898266073,2.1964139461386147,-1.6723248492685647,-1.7041864666690063,-0.8107466012135902,-4.355421368511476 +n21,-0.38223901067504884,4.775401748562854,2.2171292224546986,4.202193737383268,3.872444514491738,1.385301140533823,4.392975899146781,1.849055791579465,2.883254377127796,3.9292708043154425,5.70853753244047,4.109802546046068,4.840015441318811,4.6590745930035435,1.4201748303604427,4.276137876162492,2.0652957324064922,2.951737545842004,4.156186829319196,1.9289704279074096 +n22,-3.330140741550798,-2.906826785959495,1.9148806609929667,-1.858842892891551,-3.610448668963861,-3.5842565345749127,-2.9068267859594954,1.463903386116079e-16,-1.6955680079708941,-1.1171106596455916e-17,-1.3157505982283222,-1.6549424560257986,-1.9148806609929667,-4.216887394016624,-1.416700950707313,-5.307573324302411,-2.6595701072065454,1.19269568589775e-16,-3.735789182980685,-3.69895237610126 +n23,-1.2682710443352199,-1.7259164543827825,0.5951180965443211,-0.7703692465410688,-3.6698858525453373,0.15205790805831995,-1.7997693728666526,-0.5282124988100765,0.9435543489897311,-0.04321290506743323,-1.283994932099716,-1.1490680048258557,1.4499965326309243,-0.598215710949689,-2.9524796464239396,-0.22302880645418308,-1.9605707107578874,-0.03485740750065975,0.04660083592817306,0.8953617980357327 +n24,0.13454908979781025,-3.789695589924495,-2.9990493839211,-3.055944046231172,-3.469713221969195,-1.8181457709403341,-3.898055501618273,-3.3710851684325847,-4.526115113816528,-1.1519376802821568,2.155410251473105,-7.584933341598298,-4.06520455518612,-2.9495196068218954,-3.6832661647114624,-1.870142571153747,-1.7581366935667706,-1.6017153449803112,-3.4907269268104812,-1.891359281835198 +n25,-1.330623373267669,-2.514010723757611,-0.21931265302207267,-2.973166663976379,0.7768361992120932,-1.5420802243623235,-3.4027281900036517,-2.4015791527259845e-17,1.3759662619767377,-0.8184427157867167,-0.3150762471474844,-1.6697084109131143e-16,-4.167972215153882,0.40343613839356823,-1.3759662619767374,-4.821707446952462,-2.2760889235617467,-1.6328719347294012,-3.618968130830679,-0.6255670402243166 +n26,1.4619453476618287,2.5039026127103545,4.486604329117542,3.308730747470561,1.17557866302243,1.7274306078348556,0.783409895493845,3.3925802424431106,0.3918334166096914,2.334935580434071,1.3493953535590202,0.036278692723782396,0.6583005807700301,-0.8804460729011152,-0.29335299143043714,2.0101148284652393,0.2686881791538589,4.462167461004058,0.662202964575826,1.869000707213393 +n27,1.93228474367275,1.883613242084776,-0.18467070220787296,1.8149747077861926,4.219963974716368,-0.05045326742333013,1.8714780843074883,1.5477867891923802,-0.40188903291476474,0.944915903997542,3.5970374859707555,1.4928266259477747,0.03996673010988276,2.4741514437945877,2.450170351510507,0.5155251075962795,1.852838409048783,0.6301872790417361,2.2255378873747955,0.7605645978891847 +n28,2.1376766056953915,-1.215613310061713,-0.7684514355764751,0.2918497449173497,0.7768361992120931,0.15940998557266353,0.8405416012277966,-0.16292063778870208,-0.0890618024948524,-0.6179372028083551,4.378661507872893,-0.27292293306782545,1.9587859076543104,-4.083956172145294,0.7768361992120936,0.8169665006760507,-3.4253575381711747,-1.4728285234075273,2.145279977677073,0.3759786540450885 +n29,4.473263191267676,5.058524392577938,3.6836629851715883,4.026076605607174,3.9716569315326407,2.579039850893183,3.401374227347389,4.468185203331672,0.9239599788327261,3.7639364525724317,4.713350158088698,1.9083484502783172,1.2995566054000716,0.702725838809822,3.6852068870539036,3.9237893289495083,4.148219344671804,3.901428357967553,3.4065503981274636,4.523076596423824 +n30,3.0278895085612016,2.5627635260113575,-1.8639548463280324,-2.9340037556733556,1.9997555118869688,1.170974747773774,0.8111622262277849,-1.212679263353082,1.1589981290623574,0.14445176365979354,1.368165003180413,2.0879502260910683,1.734228095486789,3.58796553199056,3.494339586547964,2.338902702636196,2.2232294765276333,1.1218026713959193,-1.418899923458939,1.2138983312674396 +n31,0.6385109420182422,1.467050925380172,4.063179242106718,1.9148806609929665,-0.9600706846106278,0.35601846473108295,-2.0167474136583112,0.4778312525260591,-1.0281000326102985,2.5905292832732236,0.7582515083998431,-0.7196516857159374,-0.4475479074778613,0.8952221040700186,0.47567049673868683,-0.4272582445732033,1.0885474013928427,1.9405187604903171,2.1602457591935,-0.17829884962648396 +n32,-3.063394327562498,-4.380216409633087,-0.4059730707385313,-1.9804253536725582,-2.667958864078393,-0.015379378578194615,2.217098795739885,-3.1583523603631716,-1.0315431379944964,-3.5783327408406453,-4.107377217291131,-1.679075888028371,-7.165335870843173,-2.7496186414232584,-4.863069293480619,-6.968280320404486,-2.137374262440567,-6.2951223500822024,-3.167063875217493,1.8539721990109725 +n33,3.205031838011376,-2.553304542338705,-0.9391698436860347,1.4063841466783285,-3.509261092032025,1.6014565332446902,0.03139890545693563,3.8605099395191784,-3.3911360159417887,2.0820327689585856,-1.2429932538637465,0.7380809235216061,-1.8542209931013057e-16,-3.7570039776213573,1.5536723984241863,1.950857204695353e-16,-2.7802205673424587,1.138044461780873,1.835122202195506,-0.6834809685407014 +n34,0.6296437892039071,-1.9400034651851497,0.1361865010474865,0.8279610059919248,3.4797560677161425,1.1797163441654548,-1.9125438310925895,-1.5081551881525106,-0.3239315980311125,1.2823329476368766,0.5065584431862867,1.6825498017587863,-5.205401770617395,-2.962557515338115,1.691468562992695,-0.17109490820329684,0.8014053677264625,-1.333460157361212,-6.017238490188188,1.5779314209561757 +n35,2.3629101492116003,-0.05137058275745552,2.1081722924938178,1.4414294804567236,0.5439858144565103,2.2528765955282264,2.27248807006071,1.7320266859146496,1.3596035096421926,-1.9307515241834041,-0.21850620037289709,1.1190588903156802,-0.10431588862562875,1.179357944904019,1.127295574776947,2.510073591208366,1.3791508541195718,1.3922496312933825,-1.237713657774186,3.515509789101543 +n36,-0.8280518287124039,-0.7361866873935131,-0.7296782760574304,0.09695791469998018,1.9487895375572437,-0.5991300627646443,0.6153348803679545,0.2619233015599937,-0.8626512841704217,-0.5575235461900209,0.2773825073265142,-1.7580003092122432,-1.2326147879852654,-0.9137049621295144,1.2576347916114845,-0.1555587958030892,-1.3475360294094774,-1.470626709457841,1.0270655483945839,-1.1723540562416201 +n37,1.2719373431115784,2.1897218810237336,0.8996381554211087,2.1841402256953395,3.743536840224599,3.915738795226913,1.7239254251262197,1.370804987962337,1.7529470176185011,0.573074015278156,1.17944818916975,2.6165564697546797,0.4008969234551822,3.6808536320154133,2.4731813129224816,2.6371599534566434,3.2159526997090384,2.0208377833465394,-2.373014422861158,3.0084031426852142 +n38,-0.3196017459941567,2.8675719307231975,4.62111341942395,3.547752595722368,3.2492404063950806,4.238370771419072,0.7940835611191801,0.3973621702125504,0.6756486222802575,-1.786934499509262,1.7036841002995708,1.9541744795740603,-0.14960696966224252,4.962759505213697,4.722300996355786,1.6443059468741925,2.4724042071829877,1.1478997680900522,-0.7035471456158194,2.217129222454699 +n39,-1.2660722981099506,1.0177717550973577,-3.618789231639241,-2.309695393831912,2.234421657677431,-3.0265330729619966,-3.636516444387121,-0.9476476017019361,0.7646602534713793,-1.2710762242595781,-1.4344446037144718,1.1324401919521585,-2.4796435056250052,-2.1284136511141556,-2.3277631511380856,2.1293737262088803,-5.389446774456649,0.33283658870281296,0.3088050183493725,-3.038546971753011 +n40,3.4617596304141034,0.7346624025538052,4.410497322873951,4.848200940297778,-0.2416704775090703,5.47481520842521,1.7350951265745538,2.390182621088194,1.5057468474090314,1.8169098384654543,5.114420253057467,1.5403590441001547,5.382261734670339,2.161634238225905,5.510802519325814,-0.351736812001583,2.1587440567681555,1.9115373063759127,3.581930790962895,0.43527736127573546 +n41,1.1975558358483689,-1.1209820762249056,-1.685207283695076,-0.7256205697117831,0.6782969653804869,0.27097955328004586,-2.0033843681303654,-1.537509253985313,0.7768361992120932,0.026192134388948147,-0.011785385834685401,1.0744725517008482,0.2659345574725827,-0.4447482131433563,-0.8943198115923091,-0.3361753198840058,-1.4992527243496532,3.0363082718673438,-0.6149119495974263,-2.980167778409264 +n42,-1.5536723984241867,-1.7940840138023282,-7.47490760602404e-17,1.2868350135287088e-16,1.757901081822554,-1.5573477142473397e-16,3.3793506301071035,3.9237893289495083,4.030822306515829,-3.071534269947632,1.8136930840611303,2.3971090161060697,-1.1380444617808736,-4.770433764793403,1.9148806609929665,-0.6784685932408407,3.805231946603807e-17,1.0253120278897874,4.49075435682978,3.918265029123332 +n43,-2.743500619545151,-2.4539266199269663,-1.8092639688373868,-2.059634133593371,-3.1200809381733055,-3.293293133020856,-5.054938555630349,-1.586893508761546,-2.2143180062902554,-3.2066688890230304,-2.9677305850418527,-2.0188272021133336,-0.15880430834365472,-1.276504016917929,-0.976000627992844,-0.22621853547083168,-1.171919712302787,-2.229807729753591,0.16355930767942792,-1.30207654719359 +n44,2.337385917620772,3.177415467589169,-0.21814998604960725,2.104443076050821,3.3625275085137347,3.2977665352131083,3.19143911977213,-1.843459125462508,0.9705622304346155,0.7800684077592603,2.744129502373177,3.5816798324160746,3.7454171947178496,0.27209614535208765,-2.4624643559455555,0.3379617946646955,2.0901526906553536,3.7889870518636304,2.5436798561343914,0.31837305367313623 +n45,-2.0144905475390993,-0.5912971709043816,-4.250812327056555,-4.021733010656249,0.44850658873134785,-1.1115606246214502,-2.592738236780158,-3.32693360022076,-1.0106612058366231,-2.6255799196064493,-3.1008038720031066,-2.586998271707954,0.7044365418433157,1.0464576269632782,-0.5991898400966076,-5.354497949739493,-0.28206792356386384,-2.895531094535591,-5.062526536511395,-1.9644014202293507 +n46,3.489199979704967,4.415292758319915,2.1815643573555983,0.30556442583197424,2.33050859763628,3.478951875961535,2.937081958405593,0.9430017611074121,3.3041889561720454,4.547637820090979,5.370749850615683,4.480812445691051,4.8475060493030036,3.3441571921786006,2.385967762974494,0.5461598102075352,4.3183172967840004,5.015206970631166,0.028689808616375503,2.6295135443223137 +n47,-2.288373473346164,-1.896210164705379,0.9583535554379297,-3.9215903462604724,-0.5749756345222191,-2.099390515366769,-0.7699164451521859,-1.6663707035668622,-3.625345570661216,-1.6767041535994585,-2.5419010687036967,-0.42539163958916465,-1.5857101398797098,0.1778237119056841,0.6363647978406594,0.3297637382291216,0.48866314366726776,-1.603177111088721,-1.7148070945256346,-1.0360785166362276 +n48,2.3407868880076275,2.8652176822279265,0.942714985173118,3.1935046444263406,1.8234321075468902,3.2549107499431322,2.115947244476303,1.803568251307431,2.583612306761429,-1.6938012124691741,1.729768853031641,3.4457562097016576,3.1760200090880315,-0.6731906678466879,0.25965028316557026,2.1978345592831654,0.9749499014376748,1.7043514446382513,2.615520333046393,3.4382942592115486 +n49,-2.4724042071829877,-4.311989677099037,-1.9850645149285688,-0.9187318087588008,-4.994934392807417,-3.4253575381711743,-2.265231024520369,-4.390136109060091,-0.7768361992120933,0.3022485614617323,-4.241888722860413,-3.7708016208788857,-3.9292708043154425,0.3032162068829226,-1.0973364291660017,-3.195848503985772,-1.7324048148503195,-4.533840176833451,-4.0018582527682955,-4.098431739715043 +n50,0.1317827005819239,3.2999592790108214,1.04819055218582,1.263947516457773,3.2477929821379323,4.323239168505118,2.813536904333962,0.9958021790995096,1.803430384629378,2.058543569659657,2.936201529174339,0.05551798326460952,2.3432562297158355,1.9741236667226092,1.6416374966072653,4.071070935735861,2.7244067680463115,2.146329602947218,5.013770202034119,2.2330539537282474 +n51,1.655545116468729,0.2770312350118121,-1.7284778094326674,0.5458704225170374,0.02062064419593166,1.4793949153900907,-1.216961542538102,2.9506330329548565,0.30750804616603034,-2.046206549858407,1.8678654001922226,0.9392246111183439,0.38157035663038863,3.869079015066999,0.5257601883677255,0.28782762125521505,0.7660231941350666,-1.843541589722833,-0.5701056893319453,-0.8097145381027271 +n52,2.2204524273458337,0.9050495130224127,3.037000384784397,-0.1604040900283643,2.3444974376760195,2.513444450079345,2.0169891611531687,0.9722453378275318,0.36506866048036146,-1.5926336861271935,-0.4364185226664481,-0.06109954001032797,-0.4182437104610135,0.10522059826654373,0.8337774719826856,-0.08054496362702612,0.009310236137263894,0.3881920843799991,1.266386063355625,3.2653586706736064 +n53,0.8616938433246911,1.4701339080959792,1.6814879829662552,1.6791824688596606,-0.11385724230425803,-1.1942750677478284,1.4772658824488472,0.7413848262665347,2.8638137909874226,1.9846519672032223,0.5485507241042101,0.8990545242501977,2.092264396075575,-0.12360008158910944,1.7554825189118672,2.0146892736875466,0.7915681233409224,4.599981972544913,0.37991790279224297,1.415834613826235 +n54,-5.08882587631113,-2.5241894052563394,-3.6357051673827723,-0.04049979973957304,-0.9403481097885976,0.6712543661010963,-0.17378646576103557,-1.656410223784446,-0.6734250142176993,-1.201252382969664,-1.067958159644285,-3.848370469159725,-0.061952174398128085,0.4340074011544218,-2.0843917595741277,-2.692357936206529,2.5575387273330663,-0.6970777404146327,0.6769502458069075,-3.7139181576176865 +n55,3.0776300558128455,2.62812825698557,2.516699345936681,1.3930745186296136,2.796665652287933,1.2731409152769861,0.5390688144301059,4.9448084143659745,1.8947481471783374,-0.13892019265407266,2.647629934137359,5.406788568492567,2.1064181779157027,5.764474498591388,2.505882970120142,2.9303624135666553,2.6595701072065454,1.9679633780973111,0.24194517822664105,1.510704096582858 +n56,1.8393910265368798,2.4276420445167117,1.5633117059921264,0.46208996084843923,4.376658057114796,1.5127267278670293,0.07935937324357571,1.0290214032350036,0.3844195511696822,2.021399730146202,1.3780108001082945,-2.162194286583953,0.3363353757757496,0.6400227844382537,-0.9847006789762548,2.7496244432441297,-0.2164295245895394,3.1981796018068613,2.6033708671304794,1.7677139891411546 +n57,0.17581959253119772,1.086710595277549,-2.303147909999019,2.165998247767377,-0.7547096009198957,-0.9089754856751271,-3.9896372443036134,-5.042544910823239,2.29695527095907,-1.0290543499637965,-3.4464835478159332,0.5165994109061767,-4.91751695491072,-1.344932417575017,0.9453117764443324,0.5222356428334152,-1.9365021887589386,0.9186944821612051,-0.24406776427280397,2.976706743748469 +n58,-2.602986052253858,-2.9633650123223862,-2.510829950249219,-3.903374739977863,-4.929977227960032,-4.366859724148997,-1.3844283040631622,-0.03973390658466047,-4.036144264448419,-2.4869990400158697,-2.8367839675958946,-3.067637962012601,-3.002107230718089,-1.3243042156728067,-4.37561383456398,-5.980421025177467,-2.3434069244191997,-5.040372518502139,-3.5031815139146314,-4.3734898430930995 +n59,-3.7139181576176865,-3.249240406395081,-1.4801405668176861,-4.054084495751136,-1.1380444617808734,-2.5905292832732236,-0.934301761437637,-0.3205713817733673,-1.3759662619767377,3.6357051673827723,-2.792005953177144,-1.2004767630222224,-1.1159455763397934,-4.063179242106718,-4.311989677099037,0.0,-4.085417137088951,-1.6955680079708941,-2.0033843681303654,1.6955680079708941 diff --git a/test/fixtures/ilr/philr_d60_pruned/tree.nwk b/test/fixtures/ilr/philr_d60_pruned/tree.nwk new file mode 100644 index 00000000..8990117b --- /dev/null +++ b/test/fixtures/ilr/philr_d60_pruned/tree.nwk @@ -0,0 +1 @@ +(((OTU_030:0.2869246777179244,((OTU_060:0.24542262715518437,(OTU_049:0.11995769338895511,(OTU_051:0.3526826964786352,OTU_052:0.47683115237668905)0.56:0.07879231769801012)0.55:0.43784729740546374)0.54:0.1599984248354344,(UNSEEN_015:0.34881161814025324,((UNSEEN_008:0.15059598640007024,(OTU_006:0.4939610532512988,(OTU_038:0.05268266730496167,(OTU_021:0.20090260582282252,UNSEEN_004:0.1611027876915528)0.62:0.1985301988180611)0.61:0.33396814236003275)0.60:0.46263124619814106)0.59:0.1721752224432926,(OTU_002:0.10765473589379061,(OTU_017:0.21624045597240363,(UNSEEN_012:0.4910506798835085,(OTU_019:0.19084460308961115,OTU_034:0.35524924091609794)0.66:0.29315683171574103)0.65:0.08326950462807124)0.64:0.08370865942626404)0.63:0.47781287914091936)0.58:0.03380408827283761)0.57:0.023086097110183244)0.53:0.30089299451530566)0.52:0.025592079315024524,(((OTU_044:0.360163904672797,(OTU_035:0.4306127794555398,OTU_040:0.28011965885637774)0.70:0.40071610673148034)0.69:0.27127457903158125,(OTU_057:0.21419869129018318,((OTU_016:0.4005381990409556,(OTU_029:0.40074281093211883,OTU_046:0.49420521789627814)0.74:0.42978651075977914)0.73:0.3726388083456055,(OTU_028:0.2669996409419239,(OTU_011:0.30765761791280694,UNSEEN_002:0.11856501271264763)0.76:0.42638107552325516)0.75:0.04760494052863493)0.72:0.26576031535691896)0.71:0.24083301619329212)0.68:0.14367722442633896,(((OTU_012:0.03456923458581065,(OTU_005:0.23507515192453393,UNSEEN_001:0.23983233550741717)0.80:0.027045603244190537)0.79:0.10776353064636846,((OTU_009:0.035204996062877074,(OTU_022:0.101794570867443,OTU_059:0.30080506674584484)0.83:0.3686262808633575)0.82:0.31787104495164475,(OTU_003:0.053393602898026665,OTU_043:0.47203082038466493)0.84:0.2551154879549304)0.81:0.1017203791657467)0.78:0.27330361259771685,((OTU_014:0.17148626655948365,OTU_015:0.11607798407557353)0.86:0.09314682244170656,((OTU_042:0.022925166076172836,OTU_054:0.2507437777932558)0.88:0.042103934510842725,(OTU_018:0.060559723180243365,((OTU_039:0.10103667094082423,OTU_050:0.3457691434201532)0.91:0.2911068579517351,(OTU_023:0.04270071597295163,((OTU_032:0.2121233225081689,UNSEEN_003:0.10192331094651874)0.94:0.337509042214791,(OTU_045:0.36397038550138455,(UNSEEN_009:0.26275034760327126,UNSEEN_013:0.17440533472208003)0.96:0.3590506401499657)0.95:0.050425109269245316)0.93:0.04534568130430667)0.92:0.2581885069616897)0.90:0.33090852616581096)0.89:0.1237998967292362)0.87:0.21436089017682874)0.85:0.34078417494106894)0.77:0.052693530190154776)0.67:0.020560707306295833)0.51:0.047269233497693394,(((OTU_027:0.4256765078737556,OTU_041:0.3328361549598618)0.99:0.10940249269489263,(OTU_031:0.29949146212541383,OTU_056:0.4794370357669125)1.00:0.11484219771051635)0.98:0.1829811805640557,((UNSEEN_006:0.33315268872720005,(OTU_007:0.42726835422705406,(OTU_053:0.4676449662848606,(OTU_026:0.21227669629194643,OTU_058:0.3071269025070433)1.05:0.47376441601155067)1.04:0.45730109304971106)1.03:0.22178295552839453)1.02:0.28869489132183185,((OTU_001:0.1895099885617173,(UNSEEN_011:0.09354737121898732,(OTU_010:0.2821191261271549,OTU_033:0.06376287791108855)1.09:0.3643311380875367)1.08:0.13313951236677735)1.07:0.18602601630585633,((((OTU_055:0.42774246752560485,(OTU_024:0.26096406361338215,UNSEEN_007:0.4114881393740031)1.14:0.3984646507945509)1.13:0.2387407846521546,(OTU_037:0.2155873280553716,(OTU_008:0.4201110483861518,UNSEEN_014:0.3484581916460548)1.16:0.08015431781318114)1.15:0.42872640864153677)1.12:0.31797766404024774,(OTU_020:0.4582737690817804,OTU_047:0.31420380945049126)1.17:0.2048741354379073)1.11:0.262477953909287,(OTU_004:0.3300639269244508,((OTU_048:0.2512905796613398,UNSEEN_005:0.3001063698451181)1.20:0.10735332079424507,((OTU_025:0.33710243260614897,OTU_036:0.36526379834617173)1.22:0.39552156005693284,(OTU_013:0.24389069682654213,UNSEEN_010:0.2321306065742454)1.23:0.3030283581326956)1.21:0.4810006687653724)1.19:0.06457086083542982)1.18:0.36227424820540166)1.10:0.030973298970868347)1.06:0.07299540907510184)1.01:0.29202782516113535)0.97:0.3500743843594371)0.50; diff --git a/test/fixtures/ilr/philr_d8/counts.csv b/test/fixtures/ilr/philr_d8/counts.csv new file mode 100644 index 00000000..7407d3a7 --- /dev/null +++ b/test/fixtures/ilr/philr_d8/counts.csv @@ -0,0 +1,9 @@ +taxon,S01,S02,S03,S04,S05,S06 +OTU_001,5500.5,1166.5,22.5,6221.5,1281.5,1030.5 +OTU_002,196.5,800.5,398.5,2267.5,8.5,403.5 +OTU_003,82.5,0.5,0.5,0.5,14.5,5.5 +OTU_004,421.5,1468.5,153.5,5.5,38.5,82.5 +OTU_005,67.5,1886.5,3847.5,1260.5,5734.5,2441.5 +OTU_006,4751.5,120.5,3003.5,1326.5,3573.5,1246.5 +OTU_007,0.5,527.5,0.5,557.5,1009.5,15.5 +OTU_008,3.5,13.5,74.5,7027.5,156.5,1429.5 diff --git a/test/fixtures/ilr/philr_d8/expected_enorm_x_gm_counts__blw_sqrt.csv b/test/fixtures/ilr/philr_d8/expected_enorm_x_gm_counts__blw_sqrt.csv new file mode 100644 index 00000000..686a6898 --- /dev/null +++ b/test/fixtures/ilr/philr_d8/expected_enorm_x_gm_counts__blw_sqrt.csv @@ -0,0 +1,8 @@ +balance,S01,S02,S03,S04,S05,S06 +node1,28.480552875132496,19.877506518104333,-42.49205405570699,13.72548582710063,-23.72158844198629,-4.961790653430998 +node2,-19.0774146369113,5.391087522786799,33.82924867588563,7.466012571710636,-33.740675173657756,3.187163321338496 +node3,2.9456816961709946,-11.897912993281363,-20.860000689977326,26.609684620216605,7.911317948254031,2.713544292981158 +node4,-45.72561079139411,35.538012648927676,8.07414667150021,-7.389424431174819,7.735366067657756,5.773099055033226 +node5,-40.9077550539866,-3.4170494312725017,-13.137194360383573,27.549016383904352,-10.263485718063349,16.135438812556362 +node6,1.744755873045985,0.0625254356377762,-2.840788990199884,-2.2379235773578907,0.11099607648977797,0.2737855696495573 +node7,7.761452689307476,-7.136336695573849,7.118970763268367,-3.853995952609575,-3.2975575862358806,1.0770993707713195 diff --git a/test/fixtures/ilr/philr_d8/expected_gm_counts__blw.csv b/test/fixtures/ilr/philr_d8/expected_gm_counts__blw.csv new file mode 100644 index 00000000..97420952 --- /dev/null +++ b/test/fixtures/ilr/philr_d8/expected_gm_counts__blw.csv @@ -0,0 +1,8 @@ +balance,S01,S02,S03,S04,S05,S06 +node1,21.051266707954714,14.939095530572912,-31.054208412508004,12.748680583026209,-15.746295080349414,-1.9385393286964387 +node2,-19.266318023473563,6.6036519865733565,36.337122547252655,7.1974632747654645,-34.61133002033536,3.7394102352174463 +node3,1.3209977899394836,-10.155076470739644,-17.08396304368582,19.616421542937065,5.160095499938176,1.141524681610758 +node4,-46.46462568631192,33.01722465805355,5.976271670737651,-5.090776895822688,6.305665547531978,6.2562407058114085 +node5,-33.82931043228466,0.7184582360339897,-8.51542409449884,28.87697875731471,-5.619384192673193,18.368681726107972 +node6,1.5378413480019504,0.3964881475164164,-1.663693190831507,-1.2177876628927564,0.42166654238774726,0.5254848158181483 +node7,3.1148031183752174,-3.0862607564947018,2.847376095553891,-1.7200174783567224,-1.488405273138328,0.33250429406064536 diff --git a/test/fixtures/ilr/philr_d8/expected_uniform__uniform.csv b/test/fixtures/ilr/philr_d8/expected_uniform__uniform.csv new file mode 100644 index 00000000..2146a9c4 --- /dev/null +++ b/test/fixtures/ilr/philr_d8/expected_uniform__uniform.csv @@ -0,0 +1,8 @@ +balance,S01,S02,S03,S04,S05,S06 +node1,4.266129798460515,3.9855509223356798,1.1970444835485137,0.400051471761169,-2.711898571351231,0.6240822173612344 +node2,-1.671813161523924,-0.40142514583848926,1.5628557463862687,2.0462841178625633,-2.664355904871006,0.2652614829938369 +node3,1.8163978673964174,-0.1627997816615835,-1.3577760169367532,4.971680663406303,2.478499993048201,1.7854454052560824 +node4,0.7624724981302204,3.8461650556984703,4.939178362769246,1.538158557028872,2.6366165076481343,2.7643856598984846 +node5,-2.432832168409362,-0.7385991133190498,1.8218744379227005,3.969902139098499,-0.7544501097999408,2.9507093474085018 +node6,0.42969280997843,-5.081096181794673,-3.552038035874446,-6.082922405342307,-3.980508166766767,-2.637054823945729 +node7,6.4766475593980415,-1.0440425348077553,6.152310348551703,0.6129460877103605,0.8938468577447087,3.102257675761827 diff --git a/test/fixtures/ilr/philr_d8/tree.nwk b/test/fixtures/ilr/philr_d8/tree.nwk new file mode 100644 index 00000000..4d7a38ec --- /dev/null +++ b/test/fixtures/ilr/philr_d8/tree.nwk @@ -0,0 +1 @@ +((OTU_002:0.23596188780344166,(OTU_001:0.14723722168877806,OTU_004:0.2660780923422769)node3:0.4056628128936898)node2:0.012436090105114775,(OTU_005:0.2104925202993604,(OTU_008:0.493281298343706,(OTU_003:0.06964264936245652,(OTU_006:0.045103378130285816,OTU_007:0.05849367115108058)node7:0.32368663620898336)node6:0.15791824382116149)node5:0.23929398481682743)node4:0.2800135508468502)node1; diff --git a/test/fixtures/ilr/sbp_d12/counts.csv b/test/fixtures/ilr/sbp_d12/counts.csv new file mode 100644 index 00000000..96ac4821 --- /dev/null +++ b/test/fixtures/ilr/sbp_d12/counts.csv @@ -0,0 +1,13 @@ +taxon,S01,S02,S03,S04,S05,S06,S07,S08,S09,S10 +T1,947.5,0.5,0.5,0.5,0.5,0.5,1.5,94.5,40.5,1.5 +T2,1438.5,8512.5,389.5,3117.5,1374.5,4919.5,746.5,6105.5,8673.5,3558.5 +T3,508.5,743.5,27.5,367.5,310.5,687.5,224.5,1498.5,939.5,550.5 +T4,44.5,6.5,140.5,0.5,0.5,218.5,825.5,1795.5,2715.5,6.5 +T5,903.5,39.5,26.5,56.5,310.5,151.5,20.5,1056.5,1.5,0.5 +T6,2970.5,4150.5,1942.5,1382.5,642.5,1186.5,118.5,839.5,2757.5,785.5 +T7,160.5,51.5,130.5,5.5,4.5,679.5,750.5,10.5,1189.5,33.5 +T8,297.5,5150.5,14.5,19.5,2936.5,1926.5,175.5,19.5,634.5,18.5 +T9,26.5,441.5,40.5,369.5,1472.5,266.5,276.5,306.5,120.5,389.5 +T10,72.5,28.5,223.5,7516.5,68.5,1021.5,57.5,316.5,208.5,1145.5 +T11,165.5,248.5,0.5,1035.5,1887.5,1472.5,171.5,4681.5,12.5,55.5 +T12,92.5,611.5,85.5,183.5,0.5,0.5,4.5,1.5,232.5,1014.5 diff --git a/test/fixtures/ilr/sbp_d12/expected.csv b/test/fixtures/ilr/sbp_d12/expected.csv new file mode 100644 index 00000000..42ef30af --- /dev/null +++ b/test/fixtures/ilr/sbp_d12/expected.csv @@ -0,0 +1,12 @@ +balance,S01,S02,S03,S04,S05,S06,S07,S08,S09,S10 +b1,0.23410298365175186,-2.8331033557005534,-1.8307496896322148,-3.900100398265656,-6.955295054490094,-6.128506215540368,-2.699465290275083,-4.812128040099473,0.35542056443764247,-0.33980681915218064 +b2,-0.5867235097598287,4.889424546958846,1.7922381672062184,5.053668537234518,2.1170532522986294,0.005852297974764387,-0.05412582761154883,0.4088549516008535,0.7558238700127131,4.65798378424471 +b3,1.2050913782147856,0.13820671500632825,-0.8020926976310049,0.4910917768267322,4.547637820090978,5.1097013933135145,2.7646448983598138,4.8838131399749845,0.9874446342338912,-0.432271489517689 +b4,1.2554914205511583,-3.2772482965390424,-3.9347101140688063,-1.6955680079708944,-1.5536723984241865,-5.101424994146554,-4.394862776266921,1.5536723984241867,-2.3900112723843456,-2.196330464764286 +b5,-1.468064024541116,2.5797331028424115,-2.537843290555563,4.155464364535779,4.132942496631891,1.8430817659257341,0.6616676121928339,-1.011368501793726,0.30701364996418345,4.120174313774609 +b6,-1.7056748947499447,-1.2917535592467202,-3.48212488523341,-0.24022066841741624,1.0293296591534813,-1.3413042671325803,0.11515739833707024,1.394941422754813,-3.6093954051337684,-1.15627565388066 +b7,-1.295296913113376,0.4063991392805674,3.107344796848373,-0.7286657177280851,-0.17556864974537298,-1.2086877946860528,0.33773368256695296,-1.9276834613494318,1.6022481815406766,1.3777840181993461 +b8,1.8631274503527724,2.531885979759399,1.5701913749478824,1.712317032079141,0.914428485189258,1.0244732747631178,1.637633819479251,3.554272021363947,2.5896328199367025,2.6098584647417717 +b9,0.9983129144624837,3.674795043210097,-1.9341228491948825,-4.210425904491008,2.6574062289348412,0.4486116995964439,0.7890279974618386,-1.9706420940681757,0.7869378964575249,-2.9173993798263433 +b10,2.20095056624384,4.5372165201858055,2.825531302226916,4.54062077294644,3.2193222837362097,1.5309941686081685,-0.07617432524627214,-0.40422406364601887,3.0749110253934875,4.961843973773905 +b11,-2.128947848447934,-1.2759731224592978,1.1794984860579065,-3.3427679839101776,-4.547637820090978,0.25894277874957256,2.613158631440747,0.3749942658732034,5.304196246555135,1.8136930840611305 diff --git a/test/fixtures/ilr/sbp_d12/sbp.csv b/test/fixtures/ilr/sbp_d12/sbp.csv new file mode 100644 index 00000000..4369ec82 --- /dev/null +++ b/test/fixtures/ilr/sbp_d12/sbp.csv @@ -0,0 +1,13 @@ +taxon,b1,b2,b3,b4,b5,b6,b7,b8,b9,b10,b11 +T4,-1,0,0,0,-1,0,0,0,0,-1,1 +T2,-1,0,0,0,1,-1,0,1,0,0,0 +T1,1,-1,0,1,0,0,0,0,0,0,0 +T10,-1,0,0,0,1,-1,0,-1,-1,0,0 +T3,1,1,1,0,0,0,0,0,0,0,0 +T7,1,-1,0,-1,0,0,0,0,0,0,0 +T12,1,1,-1,0,0,0,0,0,0,0,0 +T9,-1,0,0,0,1,1,1,0,0,0,0 +T8,-1,0,0,0,1,-1,0,-1,1,0,0 +T5,-1,0,0,0,-1,0,0,0,0,-1,-1 +T11,-1,0,0,0,1,1,-1,0,0,0,0 +T6,-1,0,0,0,-1,0,0,0,0,1,0 diff --git a/test/runtests.jl b/test/runtests.jl index 8eef0f69..971acf22 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -66,6 +66,7 @@ using MetaManifold: AnalysisConfig include("unit/test_analysis_config.jl") include("unit/test_execution.jl") include("unit/test_scaling.jl") + include("unit/test_ilr_basis.jl") include("unit/test_estimation.jl") ## Integration tests (opt-in) diff --git a/test/unit/test_ilr_basis.jl b/test/unit/test_ilr_basis.jl new file mode 100644 index 00000000..0f82c43f --- /dev/null +++ b/test/unit/test_ilr_basis.jl @@ -0,0 +1,694 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# +# Evidence for issue #20 — ILR bases: phylogenetic (PhILR), sequential binary partition +# (SBP) and balance dendrogram — held to the conditions published before the +# implementation in docs/statistics/method-conditions/ilr-bases.md: +# +# * one test per machine-checked theorem (proofs/agda, plan in +# docs/formal/verification-plan.md §5), each description starting with the theorem's +# name: the proofs hold in exact arithmetic; these check the floating-point code +# against them; +# * reference agreement with the committed fixtures in test/fixtures/ilr/, which +# test/fixtures/ilr/ilr_reference.jl -- an independent, Base-only Julia port of philr, +# compositions::ilr / gsi.merge2signary and robCompositions::variation, with a generic +# Lance-Williams clustering rather than hclust.f's nearest-neighbour list -- recomputes +# from the committed inputs on every run. It is NOT R: philr, compositions and +# robCompositions are not in renv.lock and are not added. The R cross-check at the end +# runs only where those packages are installed and says so when it is skipped; +# * hand-computed tie cases for R's hclust.f tie-breaking; +# * a negative control for every refusal in the conditions document, asserting that it +# fires and names what it refused; +# * the integration path, from AnalysisConfig through prepare_analysis_table to the +# balances, the checks and the provenance; +# * performance at 100, 1 000 and 10 000 taxa, including the allocation bound that +# shows no dense basis is built. + +# The independent reference is a module, so it is loaded here at file scope, not inside a +# testset. It never calls MetaManifold.ILRBasis. +include(joinpath(@__DIR__, "..", "fixtures", "ilr", "ilr_reference.jl")) + +@testset "ILR bases — phylogenetic, SBP, balance dendrogram (issue #20)" begin + + ILR = MetaManifold.ILRBasis + FIX = joinpath(@__DIR__, "..", "fixtures", "ilr") + + # rows x cols matrix from a fixture CSV (first column: row ids; header: corner + col ids) + function read_matrix(path) + lines = [String(strip(l)) for l in readlines(path) if !isempty(strip(l))] + cols = map(String, split(lines[1], ',')[2:end]) + rows = String[] + M = zeros(Float64, length(lines) - 1, length(cols)) + for (r, l) in enumerate(lines[2:end]) + f = split(l, ',') + push!(rows, String(f[1])) + for c in eachindex(cols) + M[r, c] = parse(Float64, f[c + 1]) + end + end + return rows, cols, M + end + + function refusal(f) + try + f() + catch e + return sprint(showerror, e) + end + return "NO REFUSAL" + end + + # Deterministic positive table without Random: taxa x samples. + synth(D, n; k=0) = [1.5 + mod(i * 7919 + j * 104729 + k * 31337, 997) + 0.25 * mod(i * j, 7) for i in 1:D, j in 1:n] + + # SBP basis matrix V (D x (D-1)) and masses from a balance tree and part weights, the + # dense formula `buildilrBasep` uses: +c/r on the numerator, -c/s on the denominator. + function dense_basis(bt, p) + W = ILR.sbp_matrix(bt) + D = size(W, 1) + V = zeros(Float64, D, D - 1) + for n in 1:(D - 1) + r = sum(p[i] for i in 1:D if W[i, n] == 1) + s = sum(p[i] for i in 1:D if W[i, n] == -1) + c = sqrt(r * s / (r + s)) + for i in 1:D + W[i, n] == 1 && (V[i, n] = c / r) + W[i, n] == -1 && (V[i, n] = -c / s) + end + end + return V + end + + ids8, s8, X8 = read_matrix(joinpath(FIX, "philr_d8", "counts.csv")) + tree8 = read(joinpath(FIX, "philr_d8", "tree.nwk"), String) + ids25, s25, X25 = read_matrix(joinpath(FIX, "philr_d25", "counts.csv")) + tree25 = read(joinpath(FIX, "philr_d25", "tree.nwk"), String) + bt8, _ = ILR.phylo_balance_tree(tree8, ids8) + bt25, _ = ILR.phylo_balance_tree(tree25, ids25) + + # ================================================================================== + # One test per theorem (verification-plan §5) + # ================================================================================== + + @testset "internal-count: D taxa give D-1 balances with unique ids" begin + for (bt, D) in ((bt8, 8), (bt25, 25)) + @test length(bt.balance_ids) == D - 1 + @test allunique(bt.balance_ids) + @test count(==(0), bt.tip) == D - 1 + @test sort(bt.balance_row[bt.tip .== 0]) == collect(1:(D - 1)) + end + @test length(ILR.comb_tree(["a", "b"]).balance_ids) == 1 + end + + @testset "code: tree -> SBP -> tree is the identity and every column has a + and a - part" begin + for bt in (bt8, bt25) + W = ILR.sbp_matrix(bt) + D = length(bt.taxa) + @test size(W) == (D, D - 1) + @test all(any(==(1), W[:, n]) && any(==(-1), W[:, n]) for n in 1:(D - 1)) + bt2 = ILR.sbp_balance_tree(bt.taxa, bt.balance_ids, W, bt.taxa) + @test bt2.left == bt.left && bt2.right == bt.right && bt2.tip == bt.tip + @test bt2.balance_row == bt.balance_row && bt2.balance_ids == bt.balance_ids + end + end + + @testset "contrast-from-code: clade sums equal the SBP matrix formula" begin + for p in (ones(25), ILR.part_weights(X25, "gm_counts"), ILR.part_weights(X25, "enorm")) + V = dense_basis(bt25, p) + B = ILR.tree_balances(bt25, X25, p, ones(24)) + Y = log.(X25) .- log.(p) + B2 = [sum(V[i, n] * p[i] * Y[i, j] for i in 1:25) for n in 1:24, j in 1:size(X25, 2)] + @test maximum(abs.(B .- B2)) < 1e-12 + end + end + + @testset "contrast-sum-zero: balances of a constant composition are zero" begin + const_x = fill(42.0, 25, 3) + @test maximum(abs.(ILR.tree_balances(bt25, const_x, ones(25), ones(24)))) < 1e-12 + # In the p-weighted geometry the "constant" composition is x proportional to p. + p = ILR.part_weights(X25, "anorm_x_gm_counts") + prop_p = hcat(p .* 3.0, p .* 0.01) + @test maximum(abs.(ILR.tree_balances(bt25, prop_p, p, ones(24)))) < 1e-12 + end + + @testset "basis-orthogonal: basis columns are orthonormal under part weights" begin + for p in (ones(25), ILR.part_weights(X25, "gm_counts"), ILR.part_weights(X25, "anorm")) + V = dense_basis(bt25, p) + G = [sum(p[i] * V[i, a] * V[i, b] for i in 1:25) for a in 1:24, b in 1:24] + @test maximum(abs.(G .- [a == b ? 1.0 : 0.0 for a in 1:24, b in 1:24])) < 1e-12 + # contrast-sum-zero, column form: every column is p-centred. + @test maximum(abs(sum(p[i] * V[i, n] for i in 1:25)) for n in 1:24) < 1e-12 + end + end + + @testset "contrast-norm: recorded coefficient equals sqrt(rs/(r+s))" begin + dir = mktempdir() + tp = joinpath(dir, "t.nwk") + write(tp, tree25) + out = ILR.ilr_transform(X25, ids25; basis = "phylogenetic", tree_path = tp, part_weights_kind = "gm_counts") + for rec in out.checks["balances"] + r, s = rec["r"], rec["s"] + @test abs(rec["coefficient"] - sqrt(r * s / (r + s))) < 1e-12 + # the unnormalised contrast's squared norm is r*s*(r+s) under uniform weights; + # under weights it is s^2 r + r^2 s — the same expression. + @test abs((s^2 * r + r^2 * s) - r * s * (r + s)) < 1e-9 * max(1.0, r * s * (r + s)) + end + @test sum(rec["r"] + rec["s"] for rec in out.checks["balances"] if rec["id"] == out.balance_ids[1]) ≈ sum(ILR.part_weights(X25, "gm_counts")) + end + + @testset "balance-injective: balances are an isometry of centred log vectors" begin + for p in (ones(25), ILR.part_weights(X25, "enorm_x_gm_counts")) + B = ILR.tree_balances(bt25, X25, p, ones(24)) + n = size(X25, 2) + worst = 0.0 + for a in 1:n, b in (a + 1):n + d = log.(X25[:, a]) .- log.(X25[:, b]) + dbar = sum(p .* d) / sum(p) + aitchison = sum(p .* (d .- dbar) .^ 2) + worst = max(worst, abs(sum((B[:, a] .- B[:, b]) .^ 2) - aitchison)) + # distinct compositions (not proportional) give distinct balances + @test sum((B[:, a] .- B[:, b]) .^ 2) > 1e-6 + end + @test worst < 1e-9 + end + end + + @testset "balance-scale-invariant: scaling a sample leaves balances unchanged" begin + p = ILR.part_weights(X25, "gm_counts") + B = ILR.tree_balances(bt25, X25, p, ones(24)) + for λ in (1e-3, 7.0, 1e5) + Bλ = ILR.tree_balances(bt25, λ .* X25, p, ones(24)) + @test maximum(abs.(Bλ .- B) ./ max.(1.0, abs.(B))) < 1e-12 + end + # counts and proportions give the same balances + P = X25 ./ sum(X25, dims = 1) + @test maximum(abs.(ILR.tree_balances(bt25, P, p, ones(24)) .- B)) < 1e-11 + end + + @testset "balance-perturb: perturbation adds balance vectors" begin + Y = synth(25, size(X25, 2); k = 3) + Bx = ILR.tree_balances(bt25, X25, ones(25), ones(24)) + By = ILR.tree_balances(bt25, Y, ones(25), ones(24)) + Bxy = ILR.tree_balances(bt25, X25 .* Y, ones(25), ones(24)) + @test maximum(abs.(Bxy .- (Bx .+ By))) < 1e-12 + end + + @testset "comb-is-helmert: comb tree reproduces the Helmert default" begin + taxa = ["t$i" for i in 1:12] + counts = synth(12, 6) + cfg = AnalysisConfig.AnalysisConfig( + method = "ilr_lm", formula = "~ group", metadata_columns = ["group"], + normalization = AnalysisConfig.NormalizationConfig(method = "ilr", pseudocount = 0.5), + advanced = AnalysisConfig.AdvancedConfig(min_prevalence = 0.0, min_abundance = 0.0), + created_by = "test_ilr_basis") + (prepared, _, _, _, rows) = Execution.prepare_analysis_table( + cfg, counts; sample_ids = ["s$j" for j in 1:6], taxa_ids = taxa, drop_policy = "drop") + comb = ILR.comb_tree(taxa) + B = ILR.tree_balances(comb, counts .+ 0.5, ones(12), ones(11)) + @test rows == comb.balance_ids + @test maximum(abs.(B .- prepared)) < 1e-12 + end + + @testset "kernel-needs-hypothesis: signed weights lose injectivity, so non-positive weights are refused" begin + # The counterexample, in numbers: two parts with weights p = (1, -1), so r = 1 and + # s = -1. y = (1, 1) is p-centred (1*1 + (-1)*1 = 0) and non-zero, yet it is + # p-orthogonal to the balance's contrast c = (s, -r) = (-1, -1): + # 1*(-1)*1 + (-1)*(-1)*1 = 0. The balance cannot see y, so the map is not injective + # without positive weights (the hypothesis the Agda Kernel module carries). + p = (1, -1); y = (1, 1) + @test p[1] * y[1] + p[2] * y[2] == 0 + r, s = p[1], p[2] + contrast = (s, -r) + @test p[1] * contrast[1] * y[1] + p[2] * contrast[2] * y[2] == 0 + @test y != (0, 0) + # So a weight that is zero (anorm of a taxon constant across samples) is refused. + Xc = copy(X25); Xc[3, :] .= 10.0 + msg = refusal(() -> ILR.part_weights(Xc, "anorm")) + @test occursin("finite and positive", msg) + @test occursin("refused", msg) + end + + # ================================================================================== + # Reference agreement (committed fixtures, recomputed by test/fixtures/ilr/ilr_reference.jl) + # ================================================================================== + + @testset "the independent Julia reference reproduces every committed expectation (1e-10)" begin + # Three-way agreement: the committed files, the reference recomputing them from the + # committed inputs (here), and the engine (the testsets below). A change to any one + # of the three that the other two do not share fails. + @test ILRReference.philr_known_answers() + expectations = ILRReference.reference_expectations(FIX) + @test length(expectations) == 9 + 1 + 1 + 2 * 3 + for e in expectations + rows, cols, M = read_matrix(joinpath(FIX, e.file)) + @test rows == e.rows + @test cols == e.cols + @test size(M) == size(e.values) + @test maximum(abs.(M .- e.values) ./ max.(1.0, abs.(e.values))) < 1e-10 + end + end + + @testset "PhILR agrees with the philr reference on three datasets (1e-10)" begin + n_compared = 0 + for d in ("philr_d8", "philr_d25", "philr_d60_pruned") + taxa, _, X = read_matrix(joinpath(FIX, d, "counts.csv")) + bt, info = ILR.phylo_balance_tree(read(joinpath(FIX, d, "tree.nwk"), String), taxa) + for f in sort(readdir(joinpath(FIX, d))) + startswith(f, "expected_") || continue + part, ilrw = split(f[10:(end - 4)], "__") + eids, _, E = read_matrix(joinpath(FIX, d, f)) + p = ILR.part_weights(X, String(part)) + bw, _ = ILR.balance_weights(bt, String(ilrw)) + B = ILR.tree_balances(bt, X, p, bw) + @test bt.balance_ids == eids + @test maximum(abs.(B .- E) ./ max.(1.0, abs.(E))) < 1e-10 + n_compared += 1 + end + d == "philr_d60_pruned" && @test info["pruned_tips_count"] == 15 + d == "philr_d60_pruned" && @test startswith(info["balance_id_rule"], "n1") + d == "philr_d8" && @test startswith(info["balance_id_rule"], "internal node labels") + end + @test n_compared == 9 + end + + @testset "SBP agrees with compositions::ilr semantics, SBP rows in any order (1e-10)" begin + taxa, _, X = read_matrix(joinpath(FIX, "sbp_d12", "counts.csv")) + (st, ids, W) = ILR.parse_sbp(read(joinpath(FIX, "sbp_d12", "sbp.csv"), String)) + @test st != taxa # the fixture permutes the SBP rows on purpose + bt = ILR.sbp_balance_tree(st, ids, W, taxa) + eids, _, E = read_matrix(joinpath(FIX, "sbp_d12", "expected.csv")) + @test bt.balance_ids == ids == eids # the SBP's column order and names are kept + @test maximum(abs.(ILR.tree_balances(bt, X, ones(12), ones(11)) .- E)) < 1e-10 + # round trip: the reconstructed tree's SBP is the input, rows re-ordered to the table + W2 = ILR.sbp_matrix(bt) + @test all(W2[findfirst(==(t), taxa), :] == W[i, :] for (i, t) in enumerate(st)) + end + + @testset "balance dendrogram: variation matrix, R hclust merges and balances (generic reference, tie-free)" begin + taxa, _, X = read_matrix(joinpath(FIX, "dendrogram_d30", "counts.csv")) + D = length(taxa) + _, _, T = read_matrix(joinpath(FIX, "dendrogram_d30", "variation.csv")) + tau = ILR.variation_condensed(X) + @test maximum(abs(tau[k] - T[i, j]) for (k, (i, j)) in enumerate(((i, j) for i in 1:(D - 1) for j in (i + 1):D))) < 1e-12 + for m in ("ward", "complete", "average") + _, _, R = read_matrix(joinpath(FIX, "dendrogram_d30", "merge_$m.csv")) + merge, height = ILR.hclust_r(tau, D, m) + @test merge == round.(Int, R[:, 1:2]) + @test maximum(abs.(height .- R[:, 3])) < 1e-12 + bt = ILR.dendrogram_balance_tree(merge, taxa) + eids, _, E = read_matrix(joinpath(FIX, "dendrogram_d30", "expected_$m.csv")) + @test bt.balance_ids == eids + @test maximum(abs.(ILR.tree_balances(bt, X, ones(D), ones(D - 1)) .- E)) < 1e-10 + end + end + + @testset "hclust tie-breaking follows R's hclust.f (hand-computed)" begin + # All six distances equal: the first pair in scan order merges first, then the + # cluster absorbs the next singleton — R gives merge [-1 -2; -3 1; -4 2]. + for m in ("ward", "complete", "average") + merge, height = ILR.hclust_r(ones(6), 4, m) + @test merge == [-1 -2; -3 1; -4 2] + @test height ≈ [1.0, 1.0, 1.0] atol = 1e-15 + end + # Two tied pairs (d12 = d34 = 1, all else 2): (1,2) first, then (3,4), then both + # clusters, smaller step first. Ward.D2 top height = sqrt(((1+2)5 + (1+2)5 - 2)/4) + # = sqrt(7), from the squared-distance update (5 = ((1+1)4 + (1+1)4 - 1)/3). + d = [1.0, 2.0, 2.0, 2.0, 2.0, 1.0] + for m in ("ward", "complete", "average") + merge, height = ILR.hclust_r(d, 4, m) + @test merge == [-1 -2; -3 -4; 1 2] + @test height[1:2] ≈ [1.0, 1.0] atol = 1e-15 + @test height[3] ≈ (m == "ward" ? sqrt(7.0) : 2.0) atol = 1e-14 + end + # gsi.merge2signary reading: second cluster = numerator (left child). + bt = ILR.dendrogram_balance_tree([-1 -2; -3 -4; 1 2], ["a", "b", "c", "d"]) + @test bt.balance_ids == ["m3", "m2", "m1"] + @test ILR.sbp_matrix(bt) == Int8[-1 0 -1; -1 0 1; 1 -1 0; 1 1 0] + end + + # ================================================================================== + # Negative controls: every refusal fires and names what it refused + # ================================================================================== + + @testset "phylogenetic refusals" begin + abc = ["A", "B", "C"] + m = refusal(() -> ILR.phylo_balance_tree("(A:1,B:1,C:1);", abc)) + @test occursin("unrooted", m) && occursin("trifurcation", m) + # checked before pruning: pruning D would otherwise invent a root + m = refusal(() -> ILR.phylo_balance_tree("((A,B),C,D);", abc)) + @test occursin("unrooted", m) + m = refusal(() -> ILR.phylo_balance_tree("(((A,B,C),D),E);", ["A", "B", "C", "D"])) + @test occursin("polytomy", m) && occursin("'A'", m) + # a polytomy with only two retained branches is an ordinary split once pruned + bt, info = ILR.phylo_balance_tree("(((A,B,X),C),Y);", abc) + @test length(bt.balance_ids) == 2 && info["pruned_tips_count"] == 2 + m = refusal(() -> ILR.phylo_balance_tree("((A,A),B);", ["A", "B"])) + @test occursin("duplicate tip labels", m) && occursin("'A'", m) + m = refusal(() -> ILR.phylo_balance_tree("((A,B),C);", ["A", "B", "C", "Zed"])) + @test occursin("not tips", m) && occursin("'Zed'", m) + @test occursin("no terminating ';'", refusal(() -> ILR.parse_newick("((A,B),C)"))) + @test occursin("not closed", refusal(() -> ILR.parse_newick("((A,B),C;"))) + @test occursin("unbalanced ')'", refusal(() -> ILR.parse_newick("(A,B));"))) + @test occursin("not a finite number", refusal(() -> ILR.parse_newick("(A:x,B);"))) + @test occursin("second label", refusal(() -> ILR.parse_newick("(Homo sapiens,B);"))) + # quoted labels, comments and exact matching + bt, _ = ILR.phylo_balance_tree("('Homo sapiens':1,(B:1[&c],'it''s':1):1);", ["Homo sapiens", "B", "it's"]) + @test sort(bt.taxa) == sort(["Homo sapiens", "B", "it's"]) + @test occursin("not tips", refusal(() -> ILR.phylo_balance_tree("((a,B),C);", ["A", "B", "C"]))) + # labels: unique non-numeric -> used; numeric support values -> n1.. + bt, _ = ILR.phylo_balance_tree("((A,B)x,C)root;", abc) + @test bt.balance_ids == ["root", "x"] + bt, _ = ILR.phylo_balance_tree("((A,B)95,C)100;", abc) + @test bt.balance_ids == ["n1", "n2"] + end + + @testset "balance-weight refusals and philr's zero-length tip rule" begin + bt, _ = ILR.phylo_balance_tree("((A,B),C);", ["A", "B", "C"]) + @test occursin("branch length", refusal(() -> ILR.balance_weights(bt, "blw"))) + bt, _ = ILR.phylo_balance_tree("((A:0,B:2):1,C:3);", ["A", "B", "C"]) + w, info = ILR.balance_weights(bt, "blw") + @test info["zero_length_tip_edges_replaced"] == 1 && info["min_nonzero_edge_length"] == 1.0 + @test w == [1.0 + 3.0, 1.0 + 2.0] # root: edges 1 and 3; inner: A 0 -> 1, B 2 + bt, _ = ILR.phylo_balance_tree("((A:-1,B:2):1,C:3);", ["A", "B", "C"]) + @test occursin("non-negative", refusal(() -> ILR.balance_weights(bt, "blw_sqrt"))) + m = refusal(() -> ILR.ilr_transform(synth(3, 4), ["A", "B", "C"]; basis = "balance_dendrogram", + dendrogram_method = "ward", balance_weights_kind = "blw")) + @test occursin("only the phylogenetic basis", m) + end + + @testset "SBP refusals (Egozcue & Pawlowsky-Glahn 2005)" begin + t4 = ["a", "b", "c", "d"] + sbp(rows) = ILR.parse_sbp("taxon,b1,b2,b3\n" * join(rows, "\n") * "\n") + ok = ["a,1,1,0", "b,1,-1,0", "c,-1,0,1", "d,-1,0,-1"] + (st, ids, W) = sbp(ok) + @test ILR.sbp_balance_tree(st, ids, W, t4).balance_ids == ["b1", "b2", "b3"] + # spellings + (_, _, W2) = ILR.parse_sbp("taxon,b1\nx,+1\ny,-1.0\n") + @test W2 == reshape(Int8[1, -1], 2, 1) + @test occursin("'2'", refusal(() -> sbp(["a,2,1,0", "b,1,-1,0", "c,-1,0,1", "d,-1,0,-1"]))) + @test occursin("''", refusal(() -> sbp(["a,,1,0", "b,1,-1,0", "c,-1,0,1", "d,-1,0,-1"]))) + m = refusal(() -> ILR.sbp_balance_tree(["a", "b", "c", "d"], ["b1", "b2"], Int8[1 1; 1 -1; -1 0; -1 0], t4)) + @test occursin("exactly 3 balance columns", m) + (st, ids, W) = sbp(["a,1,1,0", "b,1,1,0", "c,-1,0,1", "d,-1,0,-1"]) + @test occursin("no -1 entry", refusal(() -> ILR.sbp_balance_tree(st, ids, W, t4))) + # b3 splits {b, c}, which no earlier partition produced + (st, ids, W) = sbp(["a,1,1,0", "b,1,-1,1", "c,-1,0,-1", "d,-1,0,0"]) + m = refusal(() -> ILR.sbp_balance_tree(st, ids, W, t4)) + @test occursin("not a sequential binary partition", m) + @test occursin("'b3'", m) && occursin("{c, d}", m) + # no column involves every taxon + (st, ids, W) = sbp(["a,1,1,0", "b,-1,-1,0", "c,0,1,1", "d,0,0,-1"]) + @test occursin("no column involves every taxon", refusal(() -> ILR.sbp_balance_tree(st, ids, W, t4))) + (st, ids, W) = sbp(["a,1,1,1", "b,1,-1,-1", "c,-1,0,0", "d,-1,0,0"]) + @test occursin("involve the same taxa", refusal(() -> ILR.sbp_balance_tree(st, ids, W, t4))) + # taxa must be exactly the retained taxa + (st, ids, W) = sbp(ok) + m = refusal(() -> ILR.sbp_balance_tree(st, ids, W, ["a", "b", "c", "d", "e"])) + @test occursin("not rows of the SBP", m) && occursin("'e'", m) + m = refusal(() -> ILR.sbp_balance_tree(st, ids, W, ["a", "b", "c"]; removed_by_filtering = ["d"])) + @test occursin("removed by prevalence/abundance filtering", m) && occursin("'d'", m) + m = refusal(() -> ILR.sbp_balance_tree(st, ids, W, ["a", "b", "c"])) + @test occursin("not in the count table", m) + @test occursin("more than once", refusal(() -> ILR.sbp_balance_tree(["a", "a", "c", "d"], ids, W, t4))) + @test occursin("repeated balance ids", refusal(() -> ILR.sbp_balance_tree(st, ["b1", "b1", "b3"], W, t4))) + end + + @testset "dendrogram refusals and the memory guard" begin + @test occursin("at least 2 samples", refusal(() -> ILR.variation_condensed(synth(4, 1)))) + m = refusal(() -> ILR.ilr_transform(synth(4, 3), ["a", "b", "c", "d"]; basis = "balance_dendrogram", dendrogram_method = "single")) + @test occursin("ward, complete, average", m) + # 23 171 taxa need a > 2 GiB condensed variation matrix: refused before allocating it. + D = 23_171 + m = refusal(() -> ILR.ilr_transform(ones(D, 2), ["t$i" for i in 1:D]; basis = "balance_dendrogram", dendrogram_method = "ward")) + @test occursin("GiB", m) && occursin("limit", m) + end + + @testset "engine refusals" begin + X = synth(4, 3); X[2, 2] = 0.0 + m = refusal(() -> ILR.ilr_transform(X, ["a", "b", "c", "d"]; basis = "balance_dendrogram", dendrogram_method = "ward")) + @test occursin("strictly positive", m) && occursin("'b'", m) + @test occursin("not one of", refusal(() -> ILR.ilr_transform(synth(3, 3), ["a", "b", "c"]; basis = "default"))) + @test occursin("does not exist", refusal(() -> ILR.ilr_transform(synth(3, 3), ["a", "b", "c"]; basis = "phylogenetic", tree_path = "/nonexistent/tree.nwk"))) + @test occursin("is required", refusal(() -> ILR.ilr_transform(synth(3, 3), ["a", "b", "c"]; basis = "sequential_binary_partition"))) + end + + # ================================================================================== + # Configuration contract + # ================================================================================== + + function ilr_config(; basis = nothing, norm = "ilr", method = "ilr_lm", kw...) + AnalysisConfig.AnalysisConfig( + method = method, formula = "~ group", metadata_columns = ["group"], + normalization = AnalysisConfig.NormalizationConfig(method = norm, pseudocount = 0.5, ilr_basis = basis), + advanced = AnalysisConfig.AdvancedConfig(; min_prevalence = 0.0, min_abundance = 0.0, kw...), + created_by = "test_ilr_basis") + end + digest(s) = bytes2hex(sha256(s)) + + @testset "configuration: each basis requires its input and forbids the others'" begin + # the configuration's copies of the enums agree with the engine's + @test (collect(AnalysisConfig.VALID_ILR_PART_WEIGHTS) == ILR.VALID_PART_WEIGHTS && + collect(AnalysisConfig.VALID_ILR_BALANCE_WEIGHTS) == ILR.VALID_BALANCE_WEIGHTS && + collect(AnalysisConfig.VALID_ILR_DENDROGRAM_METHODS) == ILR.VALID_DENDROGRAM_METHODS && + AnalysisConfig.ILR_SBP_ATTEMPT_DANGER_THRESHOLD == ILR.SBP_ATTEMPT_DANGER_THRESHOLD) + @test occursin("requires advanced.ilr_phylo_tree_path", refusal(() -> ilr_config(basis = "phylogenetic"))) + @test occursin("requires advanced.ilr_sbp_matrix_path", refusal(() -> ilr_config(basis = "sequential_binary_partition"))) + @test occursin("requires advanced.ilr_balance_dendrogram_method", refusal(() -> ilr_config(basis = "balance_dendrogram"))) + @test occursin("silently ignored", refusal(() -> ilr_config(basis = "default", ilr_phylo_tree_path = "t.nwk"))) + @test occursin("silently ignored", refusal(() -> ilr_config(basis = "phylogenetic", ilr_phylo_tree_path = "t.nwk", ilr_sbp_matrix_path = "s.csv"))) + @test occursin("only meaningful for the ILR transform", refusal(() -> ilr_config(norm = "clr", method = "clr_lm", ilr_balance_dendrogram_method = "ward"))) + @test occursin("default Helmert basis is unweighted", refusal(() -> ilr_config(ilr_part_weights = "gm_counts"))) + @test occursin("only ilr_basis = 'phylogenetic'", refusal(() -> ilr_config(basis = "balance_dendrogram", ilr_balance_dendrogram_method = "ward", ilr_balance_weights = "blw"))) + @test occursin("only used by ilr_basis = 'sequential_binary_partition'", refusal(() -> ilr_config(basis = "balance_dendrogram", ilr_balance_dendrogram_method = "ward", ilr_sbp_history = [digest("x")]))) + @test occursin("SHA-256", refusal(() -> AnalysisConfig.AdvancedConfig(ilr_sbp_history = ["not-a-digest"]))) + @test occursin("ward, complete, average", refusal(() -> AnalysisConfig.AdvancedConfig(ilr_balance_dendrogram_method = "single"))) + @test occursin("not allowed in a path", refusal(() -> AnalysisConfig.AdvancedConfig(ilr_phylo_tree_path = "trees/[v2].nwk"))) + @test occursin("is empty", refusal(() -> AnalysisConfig.AdvancedConfig(ilr_sbp_matrix_path = " "))) + c = ilr_config(basis = "balance_dendrogram", ilr_balance_dendrogram_method = "Ward", ilr_part_weights = "GM_COUNTS") + @test c.advanced.ilr_balance_dendrogram_method == "ward" && c.advanced.ilr_part_weights == "gm_counts" + @test !AnalysisConfig.is_dangerous(c) + end + + @testset "configuration: hash, JSON, Nickel, DEED and help carry the ILR inputs" begin + plain = ilr_config() + tree = ilr_config(basis = "phylogenetic", ilr_phylo_tree_path = "data/tree.nwk", ilr_balance_weights = "blw") + # without ILR inputs the canonical form is the pre-#20 one (no "ilr" block) + @test !occursin("\"ilr\"", AnalysisConfig.canonical_json(plain)) + @test occursin("\"ilr\"", AnalysisConfig.canonical_json(tree)) + a = ilr_config(basis = "balance_dendrogram", ilr_balance_dendrogram_method = "ward") + b = AnalysisConfig.AnalysisConfig( + id = a.id, created_at = a.created_at, method = "ilr_lm", formula = "~ group", metadata_columns = ["group"], + normalization = AnalysisConfig.NormalizationConfig(method = "ilr", pseudocount = 0.5, ilr_basis = "balance_dendrogram"), + advanced = AnalysisConfig.AdvancedConfig(min_prevalence = 0.0, min_abundance = 0.0, ilr_balance_dendrogram_method = "average"), + created_by = "test_ilr_basis") + @test a.hash != b.hash # the clustering method is part of the identity + back = AnalysisConfig.from_json(AnalysisConfig.to_json(tree)) + @test back.advanced.ilr_phylo_tree_path == "data/tree.nwk" + @test back.advanced.ilr_balance_weights == "blw" + @test back.hash == tree.hash + # JSON written before issue #20 has no ILR keys: the defaults apply + old = JSON3.read(AnalysisConfig.to_json(plain), Dict{String,Any}) + ilr_keys = ("ilr_phylo_tree_path", "ilr_sbp_matrix_path", "ilr_balance_dendrogram_method", + "ilr_part_weights", "ilr_balance_weights", "ilr_sbp_history") + old["advanced"] = Dict{String,Any}(string(k) => v for (k, v) in pairs(old["advanced"]) if !(string(k) in ilr_keys)) + @test !occursin("ilr_part_weights", JSON3.write(old)) + legacy = AnalysisConfig.from_json(JSON3.write(old)) + @test legacy.advanced.ilr_part_weights == "uniform" && isempty(legacy.advanced.ilr_sbp_history) + nk = AnalysisConfig.to_nickel(tree) + @test occursin("IlrBasisInputsContract", nk) && occursin("ilr_phylo_tree_path = \"data/tree.nwk\"", nk) + dd = AnalysisConfig.to_deed(tree) + @test occursin(":ilr-phylo-tree-path \"data/tree.nwk\"", dd) && occursin(":ilr-balance-weights \"blw\"", dd) + @test isempty(AnalysisConfig.validate_deed(dd)) + for f in ("normalization.ilr_basis", "advanced.ilr_phylo_tree_path", "advanced.ilr_sbp_matrix_path", + "advanced.ilr_balance_dendrogram_method", "advanced.ilr_part_weights", + "advanced.ilr_balance_weights", "advanced.ilr_sbp_history") + h = AnalysisConfig.context_help(f) + @test !occursin("NOT IMPLEMENTED", h) && length(h) > 200 + end + @test occursin("Silverman", AnalysisConfig.context_help("normalization.ilr_basis")) + @test occursin("Egozcue", AnalysisConfig.context_help("normalization.ilr_basis")) + @test occursin("Pawlowsky-Glahn", AnalysisConfig.context_help("normalization.ilr_basis")) + end + + @testset "configuration: more than 3 SBPs already recorded is DANGER before the run" begin + hs = [digest("sbp$i") for i in 1:4] + c = ilr_config(basis = "sequential_binary_partition", ilr_sbp_matrix_path = "s.csv", ilr_sbp_history = hs) + @test AnalysisConfig.is_dangerous(c) + @test occursin("SBP p-hacking guard", something(AnalysisConfig.danger_banner(c), "")) + c3 = ilr_config(basis = "sequential_binary_partition", ilr_sbp_matrix_path = "s.csv", ilr_sbp_history = hs[1:3]) + @test !AnalysisConfig.is_dangerous(c3) # the run decides, once it knows the current SBP + # BH stays mandatory: the ILR bases do not touch the correction contract + @test c3.correction.method == "BH" && !c3.correction.allow_no_correction + end + + # ================================================================================== + # Integration: AnalysisConfig -> prepare_analysis_table -> balances, checks, provenance + # ================================================================================== + + @testset "integration: phylogenetic basis end to end, with tree hash and pruning recorded" begin + taxa, samples, X = read_matrix(joinpath(FIX, "philr_d60_pruned", "counts.csv")) + tp = joinpath(FIX, "philr_d60_pruned", "tree.nwk") + cfg = ilr_config(basis = "phylogenetic", ilr_phylo_tree_path = tp, ilr_part_weights = "gm_counts") + (prepared, diag, manifest, _, rows) = Execution.prepare_analysis_table( + cfg, X; sample_ids = samples, taxa_ids = taxa, drop_policy = "drop") + ref = ILR.ilr_transform(X .+ 0.5, taxa; basis = "phylogenetic", tree_path = tp, part_weights_kind = "gm_counts") + @test rows == ref.balance_ids == ["n$i" for i in 1:59] + @test maximum(abs.(prepared .- ref.balances)) < 1e-12 + @test diag.checks["ilr"]["basis"] == "phylogenetic" + @test diag.checks["ilr"]["basis_details"]["pruned_tips_count"] == 15 + @test any(occursin("15 tree tips", w) for w in diag.warnings) + prov = manifest.provenance["ilr"] + @test prov["tree_sha256"] == bytes2hex(sha256(read(tp))) + @test prov["part_weights"] == "gm_counts" && prov["balance_weights"] == "uniform" + @test isnothing(prov["sbp_sha256"]) && isnothing(prov["dendrogram_method"]) + end + + @testset "integration: SBP p-hacking guard counts the current SBP" begin + taxa, samples, X = read_matrix(joinpath(FIX, "sbp_d12", "counts.csv")) + sp = joinpath(FIX, "sbp_d12", "sbp.csv") + own = bytes2hex(sha256(read(sp))) + others = [digest("earlier$i") for i in 1:3] + # own digest + 2 others = 3 distinct SBPs: disclosed, not DANGER + cfg = ilr_config(basis = "sequential_binary_partition", ilr_sbp_matrix_path = sp, ilr_sbp_history = [own, others[1], others[2]]) + (_, diag, manifest, _, rows) = Execution.prepare_analysis_table(cfg, X; sample_ids = samples, taxa_ids = taxa, drop_policy = "drop") + @test rows == ["b$i" for i in 1:11] + @test manifest.provenance["ilr"]["sbp_attempts"] == 3 + @test manifest.provenance["ilr"]["sbp_sha256"] == own + @test !diag.is_dangerous + # 3 others + the current one = 4 > 3: DANGER, with its own banner + cfg4 = ilr_config(basis = "sequential_binary_partition", ilr_sbp_matrix_path = sp, ilr_sbp_history = others) + (_, diag4, manifest4, _, _) = Execution.prepare_analysis_table(cfg4, X; sample_ids = samples, taxa_ids = taxa, drop_policy = "drop") + @test manifest4.provenance["ilr"]["sbp_attempts"] == 4 + @test diag4.is_dangerous + @test occursin("DANGER — ILR BASIS SELECTION", something(diag4.banner, "")) + @test any(occursin("SBP p-hacking guard", w) for w in diag4.warnings) + end + + @testset "integration: an SBP taxon removed by filtering is refused and named" begin + taxa, samples, X = read_matrix(joinpath(FIX, "sbp_d12", "counts.csv")) + X2 = copy(X); X2[5, 2:end] .= 0.0 # T5 present in 1 of 10 samples + cfg = AnalysisConfig.AnalysisConfig( + method = "ilr_lm", formula = "~ group", metadata_columns = ["group"], + normalization = AnalysisConfig.NormalizationConfig(method = "ilr", pseudocount = 0.5, ilr_basis = "sequential_binary_partition"), + advanced = AnalysisConfig.AdvancedConfig(min_prevalence = 0.5, ilr_sbp_matrix_path = joinpath(FIX, "sbp_d12", "sbp.csv")), + created_by = "test_ilr_basis") + m = refusal(() -> Execution.prepare_analysis_table(cfg, X2; sample_ids = samples, taxa_ids = taxa, drop_policy = "drop")) + @test occursin("removed by prevalence/abundance filtering", m) && occursin("'T5'", m) + end + + @testset "integration: balance dendrogram records its method and data-derived status" begin + taxa, samples, X = read_matrix(joinpath(FIX, "dendrogram_d30", "counts.csv")) + cfg = ilr_config(basis = "balance_dendrogram", ilr_balance_dendrogram_method = "average") + (prepared, diag, manifest, _, rows) = Execution.prepare_analysis_table(cfg, X; sample_ids = samples, taxa_ids = taxa, drop_policy = "drop") + @test all(startswith(r, "m") for r in rows) && length(rows) == 29 + @test diag.checks["ilr"]["basis_details"]["method"] == "average" + @test diag.checks["ilr"]["uses_data_twice"] === true + @test manifest.provenance["ilr"]["dendrogram_method"] == "average" + @test size(prepared) == (29, length(samples)) + end + + @testset "integration: the default basis is untouched and records itself as such" begin + taxa = ["t$i" for i in 1:6] + cfg = ilr_config() + (prepared, diag, manifest, _, rows) = Execution.prepare_analysis_table(cfg, synth(6, 4); sample_ids = ["s$j" for j in 1:4], taxa_ids = taxa, drop_policy = "drop") + @test rows == ["balance_$i" for i in 1:5] + @test manifest.provenance["ilr"]["basis"] == "default" + @test startswith(diag.checks["ilr"]["definition"], "Helmert") + # the loop's own formula, written out, for one cell + x = synth(6, 4)[:, 2] .+ 0.5 + @test prepared[3, 2] == sqrt(3 / 4) * (sum(log.(x[1:3])) / 3 - log(x[4])) || + abs(prepared[3, 2] - sqrt(3 / 4) * (sum(log.(x[1:3])) / 3 - log(x[4]))) < 1e-14 + end + + # ================================================================================== + # Performance: 100 / 1 000 / 10 000 taxa, and no dense basis + # ================================================================================== + + # Balanced Newick over t1..tD, built without recursion depth issues (depth log2 D). + function balanced_newick(lo, hi) + lo == hi && return "t$lo:0.1" + mid = (lo + hi) ÷ 2 + return "(" * balanced_newick(lo, mid) * "," * balanced_newick(mid + 1, hi) * "):0.05" + end + + @testset "performance: 100 and 1000 taxa, all three bases" begin + for D in (100, 1000) + taxa = ["t$i" for i in 1:D] + X = synth(D, 30) + dir = mktempdir() + tp = joinpath(dir, "tree.nwk"); write(tp, balanced_newick(1, D) * ";") + t = @elapsed out = ILR.ilr_transform(X, taxa; basis = "phylogenetic", tree_path = tp, balance_weights_kind = "mean_descendants") + @test size(out.balances) == (D - 1, 30) + @test t < 60 + sp = joinpath(dir, "sbp.csv") + W = ILR.sbp_matrix(ILR.phylo_balance_tree(balanced_newick(1, D) * ";", taxa)[1]) + open(sp, "w") do io + println(io, "taxon,", join(["b$k" for k in 1:(D - 1)], ",")) + for i in 1:D + println(io, taxa[i], ",", join(W[i, :], ",")) + end + end + t = @elapsed out = ILR.ilr_transform(X, taxa; basis = "sequential_binary_partition", sbp_path = sp) + @test size(out.balances) == (D - 1, 30) && t < 60 + t = @elapsed out = ILR.ilr_transform(X, taxa; basis = "balance_dendrogram", dendrogram_method = "ward") + @test size(out.balances) == (D - 1, 30) && t < 120 + end + end + + @testset "performance: 10 000 taxa balances allocate O(D·n), not a dense basis" begin + D = 10_000 + taxa = ["t$i" for i in 1:D] + X = synth(D, 5) + bt, _ = ILR.phylo_balance_tree(balanced_newick(1, D) * ";", taxa) + ILR.tree_balances(bt, X, ones(D), ones(D - 1)) # compile + bytes = @allocated ILR.tree_balances(bt, X, ones(D), ones(D - 1)) + # A dense D x (D-1) basis is 8e8 bytes; the output itself is 8·(D-1)·5 = 4e5. + @test bytes < 20_000_000 + comb = ILR.comb_tree(taxa) + @test length(comb.balance_ids) == D - 1 # iterative: no stack overflow + # A comb is the SBP reconstruction's worst case: its column supports total D^2/2 + # indices. They are hash-keyed and confirmed exactly, never stored, and the round trip + # must still be the identity. + ctaxa = taxa[1:2000] + cb = ILR.comb_tree(ctaxa) + Wc = ILR.sbp_matrix(cb) + rt = ILR.sbp_balance_tree(ctaxa, cb.balance_ids, Wc, ctaxa) + @test rt.balance_ids == cb.balance_ids + @test ILR.sbp_matrix(rt) == Wc + end + + # ================================================================================== + # R cross-check, only where the R packages are installed + # ================================================================================== + + @testset "R cross-check (philr / compositions), skipped visibly when absent" begin + # RCall is loaded by the scaling tests; reached through Main so that nothing here + # depends on a macro resolving at expansion time. + RC = isdefined(Main, :RCall) ? getfield(Main, :RCall) : nothing + have = RC !== nothing && try + RC.rcopy(RC.reval("all(sapply(c('philr', 'ape', 'compositions'), requireNamespace, quietly = TRUE))")) === true + catch + false + end + if !have + @info "ILR R cross-check skipped: philr/ape/compositions are not installed (not in renv.lock). Agreement is established against the Julia reference test/fixtures/ilr/ilr_reference.jl instead." + @test_skip false + else + taxa, samples, X = read_matrix(joinpath(FIX, "philr_d25", "counts.csv")) + tp = joinpath(FIX, "philr_d25", "tree.nwk") + RC.reval("suppressMessages(library(philr))") + RC.globalEnv[:X] = X + RC.globalEnv[:taxa] = taxa + RC.globalEnv[:samples] = samples + RC.globalEnv[:tp] = tp + Rres = RC.rcopy(RC.reval(""" + x <- t(X); colnames(x) <- taxa; rownames(x) <- samples + tr <- ape::read.tree(tp) + unname(as.matrix(philr(x, tr, part.weights = 'anorm', ilr.weights = 'blw', return.all = FALSE))) + """)) + bt, _ = ILR.phylo_balance_tree(read(tp, String), taxa) + bw, _ = ILR.balance_weights(bt, "blw") + B = ILR.tree_balances(bt, X, ILR.part_weights(X, "anorm"), bw) + @test maximum(abs.(permutedims(Rres) .- B)) < 1e-6 + end + end +end diff --git a/test/unit/test_scaling.jl b/test/unit/test_scaling.jl index ad591935..7e18068b 100644 --- a/test/unit/test_scaling.jl +++ b/test/unit/test_scaling.jl @@ -345,12 +345,14 @@ @test !haskey(diag_rel.checks, "scaling") end - @testset "deferred ILR bases are refused at construction and in prepare_analysis_table" begin - for basis in AnalysisConfig.DEFERRED_ILR_BASIS - @test_throws ArgumentError AnalysisConfig.NormalizationConfig( - method = "ilr", ilr_basis = basis) + @testset "ILR bases: the issue #20 deferral is lifted; the default basis still constructs" begin + # The three bases used to be refused here as deferred. Issue #20 implemented them + # (test/unit/test_ilr_basis.jl holds their evidence); the deferral list is kept, empty, + # so this asserts it stays empty rather than looping over nothing. + @test isempty(AnalysisConfig.DEFERRED_ILR_BASIS) + for basis in ("phylogenetic", "sequential_binary_partition", "balance_dendrogram") + @test AnalysisConfig.NormalizationConfig(method = "ilr", ilr_basis = basis).ilr_basis == basis end - # "default" basis is accepted norm_ok = AnalysisConfig.NormalizationConfig(method = "ilr", ilr_basis = "default") @test norm_ok.ilr_basis == "default" end From 3f4fc41b4dac85429d35a68ba06a21965eab83b4 Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:11:43 +0000 Subject: [PATCH 3/6] feat(schemas): Nickel, JSON Schema and DEED contracts for ILR inputs (#20) Each ILR basis requires its own input (tree path, SBP path, dendrogram method) and forbids the others'; non-ILR normalization forbids all of them; SBP history entries must be SHA-256 hex. Mirrors _validate_ilr_inputs exactly; legacy documents still validate. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- config/schemas/analysis_config.ncl | 84 +++++- config/schemas/analysis_config.schema.json | 292 +++++++++++++++++++- config/templates/analysis_config_chora.deed | 16 +- 3 files changed, 387 insertions(+), 5 deletions(-) diff --git a/config/schemas/analysis_config.ncl b/config/schemas/analysis_config.ncl index b6fffa29..56699b6f 100644 --- a/config/schemas/analysis_config.ncl +++ b/config/schemas/analysis_config.ncl @@ -12,6 +12,11 @@ let DispersionMethod = std.enum.TagOrString & [| 'parametric, 'local, 'mean, 'po let ZeroHandling = std.enum.TagOrString & [| 'pseudocount, 'multiplicative_replacement, 'bayesian_multiplicative, 'refuse |] in let ZeroPolicy = ZeroHandling in let IlrBasis = std.enum.TagOrString & [| 'default, 'phylogenetic, 'sequential_binary_partition, 'balance_dendrogram |] in +# ILR basis inputs (issue #20; docs/statistics/method-conditions/ilr-bases.md). +let IlrPartWeights = std.enum.TagOrString & [| 'uniform, 'gm_counts, 'anorm, 'enorm, 'anorm_x_gm_counts, 'enorm_x_gm_counts |] in +let IlrBalanceWeights = std.enum.TagOrString & [| 'uniform, 'blw, 'blw_sqrt, 'mean_descendants |] in +let IlrDendrogramMethod = std.enum.TagOrString & [| 'ward, 'complete, 'average |] in +let Sha256Hex = std.contract.from_predicate (fun v => std.is_string v && std.string.is_match "^[0-9a-f]{64}$" v) in let DANGER_TOKEN = "I_UNDERSTAND_THE_RISK_AND_WANT_TO_OVERRIDE_BH" in @@ -125,6 +130,52 @@ let MethodNormalizationCompatibility = fun label value => 'Ok value in +# Each non-default ILR basis requires its own input and forbids the others' (the Julia +# constructor enforces the same table, `_validate_ilr_inputs`). Whether the tree or SBP +# file exists is checked when the run reads it: a configuration may be written on one +# machine and run on another. +let IlrBasisInputsContract = fun label value => + let norm = value.normalization in + let adv = value.advanced in + let basis = + if std.record.has_field "ilr_basis" norm && norm.ilr_basis != null then norm.ilr_basis else 'default + in + let present = fun field => std.record.has_field field adv && adv."%{field}" != null in + let non_default = fun field default_value => present field && adv."%{field}" != default_value in + let pairing = fun field wanted wanted_name => + if basis == wanted && !(present field) then + "normalization.ilr_basis = '%{wanted_name}' requires advanced.%{field}" + else if basis != wanted && present field then + "advanced.%{field} is only used by ilr_basis = '%{wanted_name}' (refusing an input that would be silently ignored)" + else + "" + in + let problems = + if norm.method != 'ilr then + std.array.filter (fun m => m != "") [ + if present "ilr_phylo_tree_path" || present "ilr_sbp_matrix_path" || present "ilr_balance_dendrogram_method" + || non_default "ilr_part_weights" 'uniform || non_default "ilr_balance_weights" 'uniform + then "advanced.ilr_* inputs are only meaningful for normalization.method = 'ilr'" else "", + ] + else + std.array.filter (fun m => m != "") [ + pairing "ilr_phylo_tree_path" 'phylogenetic "phylogenetic", + pairing "ilr_sbp_matrix_path" 'sequential_binary_partition "sequential_binary_partition", + pairing "ilr_balance_dendrogram_method" 'balance_dendrogram "balance_dendrogram", + if basis == 'default && non_default "ilr_part_weights" 'uniform + then "advanced.ilr_part_weights needs a non-default ilr_basis (the Helmert default is unweighted)" else "", + if basis != 'phylogenetic && non_default "ilr_balance_weights" 'uniform + then "advanced.ilr_balance_weights needs branch lengths, so ilr_basis = 'phylogenetic'" else "", + if basis != 'sequential_binary_partition && present "ilr_sbp_history" && std.array.length adv.ilr_sbp_history > 0 + then "advanced.ilr_sbp_history is only used by ilr_basis = 'sequential_binary_partition'" else "", + ] + in + if std.array.length problems > 0 then + 'Error { message = "%{std.string.join "; " problems}. See docs/statistics/method-conditions/ilr-bases.md" } + else + 'Ok value +in + let EpsilonWarning = fun label value => if value.epsilon > 0.001 then std.contract.blame_with_message "epsilon >1e-3 large may affect zero handling and log transforms — warning" label @@ -217,7 +268,7 @@ in ilr_basis | IlrBasis | optional - | doc "ILR basis, only meaningful for ILR method. phylogenetic/sequential_binary_partition/balance_dendrogram deferred, see GitHub issue 05.", + | doc "ILR basis, only meaningful for ILR method: default (Helmert), phylogenetic (PhILR, needs advanced.ilr_phylo_tree_path), sequential_binary_partition (needs advanced.ilr_sbp_matrix_path), balance_dendrogram (needs advanced.ilr_balance_dendrogram_method). Issue #20; see docs/statistics/method-conditions/ilr-bases.md.", multiplicative_replacement_delta | std.option.Number @@ -331,6 +382,36 @@ in acknowledgment_token | std.option.String | doc "Required for dangerous zero_handling='refuse' or min_samples_per_group<3", + + ilr_phylo_tree_path + | std.option.String + | optional + | doc "Rooted, strictly bifurcating Newick tree for ilr_basis = phylogenetic (required then, refused otherwise). Tips = taxon ids exactly. SHA-256 recorded in provenance.", + + ilr_sbp_matrix_path + | std.option.String + | optional + | doc "SBP CSV (taxa rows, balance columns, entries 1/-1/0) for ilr_basis = sequential_binary_partition (required then, refused otherwise). Validated per Egozcue & Pawlowsky-Glahn (2005). SHA-256 recorded.", + + ilr_balance_dendrogram_method + | IlrDendrogramMethod + | optional + | doc "ward (R ward.D2), complete or average, for ilr_basis = balance_dendrogram (required then, refused otherwise).", + + ilr_part_weights + | IlrPartWeights + | default = 'uniform + | doc "philr part.weights for a non-default basis: uniform, gm_counts, anorm, enorm, anorm_x_gm_counts, enorm_x_gm_counts.", + + ilr_balance_weights + | IlrBalanceWeights + | default = 'uniform + | doc "philr ilr.weights, phylogenetic basis only: uniform, blw, blw_sqrt, mean_descendants. Non-uniform weights are not isometric (recorded).", + + ilr_sbp_history + | Array Sha256Hex + | default = [] + | doc "SHA-256 of SBP files tried earlier in the project. More than 3 distinct SBPs (including the current one) raises the DANGER banner.", } | ZeroHandlingContract, @@ -347,3 +428,4 @@ in | doc "Computed is_dangerous — true if BH disabled, zero_handling refuse, min_samples_per_group<3, rarefy+NB_GLM. Triggers DANGER banner.", } | MethodNormalizationCompatibility +| IlrBasisInputsContract diff --git a/config/schemas/analysis_config.schema.json b/config/schemas/analysis_config.schema.json index 45a25522..d1e2c84c 100644 --- a/config/schemas/analysis_config.schema.json +++ b/config/schemas/analysis_config.schema.json @@ -83,7 +83,7 @@ "ilr_basis": { "type": ["string", "null"], "enum": ["default", "phylogenetic", "sequential_binary_partition", "balance_dendrogram", null], - "description": "ILR basis. Only meaningful for ilr method. Refuses meaningless use for other methods. phylogenetic/sequential_binary_partition/balance_dendrogram deferred, see GitHub issues." + "description": "ILR basis. Only meaningful for ilr method. default (Helmert); phylogenetic (PhILR) requires advanced.ilr_phylo_tree_path; sequential_binary_partition requires advanced.ilr_sbp_matrix_path; balance_dendrogram requires advanced.ilr_balance_dendrogram_method. Issue #20; see docs/statistics/method-conditions/ilr-bases.md." }, "multiplicative_replacement_delta": { "type": ["number", "null"], @@ -249,6 +249,42 @@ "acknowledgment_token": { "type": ["string", "null"], "description": "Required for dangerous zero_handling='refuse' or min_samples_per_group<3" + }, + "ilr_phylo_tree_path": { + "type": ["string", "null"], + "minLength": 1, + "pattern": "^[^\"\\n\\r\\u0000\\[\\]{}`;$]+$", + "description": "Rooted, strictly bifurcating Newick tree for ilr_basis=phylogenetic (required then, refused otherwise). Tip labels = taxon ids exactly. SHA-256 recorded in provenance; existence checked at run time." + }, + "ilr_sbp_matrix_path": { + "type": ["string", "null"], + "minLength": 1, + "pattern": "^[^\"\\n\\r\\u0000\\[\\]{}`;$]+$", + "description": "SBP CSV (taxa rows, balance columns, entries 1/-1/0) for ilr_basis=sequential_binary_partition (required then, refused otherwise). Validated per Egozcue & Pawlowsky-Glahn (2005). SHA-256 recorded." + }, + "ilr_balance_dendrogram_method": { + "type": ["string", "null"], + "enum": ["ward", "complete", "average", null], + "description": "Clustering for ilr_basis=balance_dendrogram (required then, refused otherwise) on the variation matrix: ward (R ward.D2), complete, average." + }, + "ilr_part_weights": { + "type": "string", + "enum": ["uniform", "gm_counts", "anorm", "enorm", "anorm_x_gm_counts", "enorm_x_gm_counts"], + "default": "uniform", + "description": "philr part.weights for a non-default ILR basis. Must be uniform for the default (Helmert) basis." + }, + "ilr_balance_weights": { + "type": "string", + "enum": ["uniform", "blw", "blw_sqrt", "mean_descendants"], + "default": "uniform", + "description": "philr ilr.weights, phylogenetic basis only (needs branch lengths). Non-uniform weights are not isometric; recorded." + }, + "ilr_sbp_history": { + "type": "array", + "items": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "uniqueItems": true, + "default": [], + "description": "SHA-256 of SBP files tried earlier in the project (SBP basis only). More than 3 distinct SBPs including the current one raises the DANGER banner (p-hacking guard)." } } }, @@ -301,6 +337,260 @@ } } } + }, + { + "$comment": "ILR basis inputs (issue #20): each non-default basis requires its own input and forbids the others'. Same table as AnalysisConfig._validate_ilr_inputs and Nickel IlrBasisInputsContract; see docs/statistics/method-conditions/ilr-bases.md." + }, + { + "if": { + "required": [ + "normalization" + ], + "properties": { + "normalization": { + "required": [ + "ilr_basis" + ], + "properties": { + "ilr_basis": { + "enum": [ + "phylogenetic" + ] + } + } + } + } + }, + "then": { + "properties": { + "advanced": { + "properties": { + "ilr_phylo_tree_path": { + "type": "string" + } + }, + "required": [ + "ilr_phylo_tree_path" + ] + } + }, + "required": [ + "advanced" + ] + }, + "else": { + "properties": { + "advanced": { + "properties": { + "ilr_phylo_tree_path": { + "type": "null" + } + } + } + } + } + }, + { + "if": { + "required": [ + "normalization" + ], + "properties": { + "normalization": { + "required": [ + "ilr_basis" + ], + "properties": { + "ilr_basis": { + "enum": [ + "sequential_binary_partition" + ] + } + } + } + } + }, + "then": { + "properties": { + "advanced": { + "properties": { + "ilr_sbp_matrix_path": { + "type": "string" + } + }, + "required": [ + "ilr_sbp_matrix_path" + ] + } + }, + "required": [ + "advanced" + ] + }, + "else": { + "properties": { + "advanced": { + "properties": { + "ilr_sbp_matrix_path": { + "type": "null" + } + } + } + } + } + }, + { + "if": { + "required": [ + "normalization" + ], + "properties": { + "normalization": { + "required": [ + "ilr_basis" + ], + "properties": { + "ilr_basis": { + "enum": [ + "balance_dendrogram" + ] + } + } + } + } + }, + "then": { + "properties": { + "advanced": { + "properties": { + "ilr_balance_dendrogram_method": { + "type": "string" + } + }, + "required": [ + "ilr_balance_dendrogram_method" + ] + } + }, + "required": [ + "advanced" + ] + }, + "else": { + "properties": { + "advanced": { + "properties": { + "ilr_balance_dendrogram_method": { + "type": "null" + } + } + } + } + } + }, + { + "if": { + "not": { + "required": [ + "normalization" + ], + "properties": { + "normalization": { + "required": [ + "ilr_basis" + ], + "properties": { + "ilr_basis": { + "enum": [ + "phylogenetic", + "sequential_binary_partition", + "balance_dendrogram" + ] + } + } + } + } + } + }, + "then": { + "properties": { + "advanced": { + "properties": { + "ilr_part_weights": { + "const": "uniform" + } + } + } + } + } + }, + { + "if": { + "not": { + "required": [ + "normalization" + ], + "properties": { + "normalization": { + "required": [ + "ilr_basis" + ], + "properties": { + "ilr_basis": { + "enum": [ + "phylogenetic" + ] + } + } + } + } + } + }, + "then": { + "properties": { + "advanced": { + "properties": { + "ilr_balance_weights": { + "const": "uniform" + } + } + } + } + } + }, + { + "if": { + "not": { + "required": [ + "normalization" + ], + "properties": { + "normalization": { + "required": [ + "ilr_basis" + ], + "properties": { + "ilr_basis": { + "enum": [ + "sequential_binary_partition" + ] + } + } + } + } + } + }, + "then": { + "properties": { + "advanced": { + "properties": { + "ilr_sbp_history": { + "maxItems": 0 + } + } + } + } + } } ], "$defs": { diff --git a/config/templates/analysis_config_chora.deed b/config/templates/analysis_config_chora.deed index 961735e3..dc113072 100644 --- a/config/templates/analysis_config_chora.deed +++ b/config/templates/analysis_config_chora.deed @@ -47,7 +47,16 @@ :max-features 0 :min-samples-per-group 3 :robust #f - :acknowledgment-token "") + :acknowledgment-token "" + ;; ILR basis inputs (issue #20): each non-default ilr-basis requires its own input and + ;; forbids the others'; file existence is checked at run time. Provenance records the + ;; SHA-256 of the tree / SBP file and the dendrogram method. + :ilr-phylo-tree-path "" + :ilr-sbp-matrix-path "" + :ilr-balance-dendrogram-method "" + :ilr-part-weights "uniform" + :ilr-balance-weights "uniform" + :ilr-sbp-history ()) (provenance :id "00000000-0000-0000-0000-000000000000" @@ -85,5 +94,6 @@ ;; ;; ║ Config ID: 00000000-0000-0000-0000-000000000000 ║ ;; ;; ║ If you are writing a paper, you MUST disclose these overrides in Methods║ ;; ;; ╚════════════════════════════════════════════════════════════════════════════╝ -;; Deferred features: multinomial/DM, occupancy, constrained ordinations, ILR basis phylogenetic/SBP, glmGamPoi/Bayesian multiplicative — see docs/issues/milestone3/ -;; Implemented since this template was written: TSS/CSS/RSS offsets (issue #16), src/analysis/scaling.jl. +;; Deferred features: multinomial/DM, occupancy, constrained ordinations, glmGamPoi/Bayesian multiplicative — see docs/issues/milestone3/ +;; Implemented since this template was written: TSS/CSS/RSS offsets (issue #16), src/analysis/scaling.jl; +;; ILR bases phylogenetic (PhILR) / sequential binary partition / balance dendrogram (issue #20), src/analysis/ilr_basis.jl. From 11e12e49fe6213c529e18a567b1ab75cfe94c103 Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:11:43 +0000 Subject: [PATCH 4/6] feat(frontend): ILR basis inputs in Advanced Analysis (#20) Required inputs sit next to the basis select; philr weights and SBP history are Evidence-Mode only; switching basis clears the other bases' inputs. ilrInputProblems mirrors the backend contract, pinned by tests/unit/ilr-basis-contract.test.ts. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .../src/components/AnalysisConfigEditor.tsx | 30 ++- frontend/src/components/IlrBasisInputs.tsx | 210 ++++++++++++++++++ frontend/src/types/analysis_config.ts | 173 +++++++++++++++ .../tests/unit/ilr-basis-contract.test.ts | 205 +++++++++++++++++ 4 files changed, 602 insertions(+), 16 deletions(-) create mode 100644 frontend/src/components/IlrBasisInputs.tsx create mode 100644 frontend/tests/unit/ilr-basis-contract.test.ts diff --git a/frontend/src/components/AnalysisConfigEditor.tsx b/frontend/src/components/AnalysisConfigEditor.tsx index 4cb7393d..be2a44b4 100644 --- a/frontend/src/components/AnalysisConfigEditor.tsx +++ b/frontend/src/components/AnalysisConfigEditor.tsx @@ -1,9 +1,10 @@ // SPDX-License-Identifier: AGPL-3.0-only import { useState } from 'react' import type { AnalysisConfig, ValidationError } from '../types/analysis_config' -import { contextHelp, isDangerous, DANGER_ACK_TOKEN } from '../types/analysis_config' +import { contextHelp, isDangerous, DANGER_ACK_TOKEN, withoutIlrInputs } from '../types/analysis_config' import { DangerBanner } from './DangerBanner' import { AdvancedAnalysisExpander } from './AdvancedAnalysisExpander' +import { IlrBasisInputs } from './IlrBasisInputs' interface AnalysisConfigEditorProps { evidenceMode: boolean @@ -75,7 +76,9 @@ export function AnalysisConfigEditor({ evidenceMode, config, onChange, onSave, a } else if (newMethod === 'logistic' && (newNorm.method === 'clr' || newNorm.method === 'ilr')) { newNorm = { ...newNorm, method: 'presence_absence' } } - onChange({ ...config, method: newMethod, normalization: newNorm }) + // ILR basis inputs are refused outside ILR, so leaving ILR clears them (visibly: the controls disappear with it) + const newAdvanced = newNorm.method === 'ilr' ? config.advanced : withoutIlrInputs(config.advanced) + onChange({ ...config, method: newMethod, normalization: newNorm, advanced: newAdvanced }) }} style={{ width: '100%', padding: 8, marginTop: 4 }} > @@ -161,7 +164,14 @@ export function AnalysisConfigEditor({ evidenceMode, config, onChange, onSave, a onChange({ ...config, normalization: { ...config.normalization, ilr_basis: e.target.value } })} - style={{ width: '100%', padding: 8, marginTop: 4 }} - > - - - - - - + )} {helpField === 'norm' && ( diff --git a/frontend/src/components/IlrBasisInputs.tsx b/frontend/src/components/IlrBasisInputs.tsx new file mode 100644 index 00000000..138f0e82 --- /dev/null +++ b/frontend/src/components/IlrBasisInputs.tsx @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// ILR basis selection and its inputs (issue #20). The basis's REQUIRED input (tree, +// SBP file or clustering method) sits next to the basis choice, because the basis is +// unusable without it; the optional philr weights and the SBP history are Advanced +// Analysis options and appear only in Evidence Mode. Conditions and refusals: +// docs/statistics/method-conditions/ilr-bases.md. +import { useState } from 'react' +import type { AnalysisConfig, IlrBasis, IlrBalanceWeights, IlrDendrogramMethod, IlrPartWeights } from '../types/analysis_config' +import { + ILR_BALANCE_WEIGHTS, + ILR_BASES, + ILR_DENDROGRAM_METHODS, + ILR_PART_WEIGHTS, + ILR_SBP_ATTEMPT_DANGER_THRESHOLD, + contextHelp, + ilrBasisOf, + ilrInputProblems, + ilrSbpAttempts, + withIlrBasis, +} from '../types/analysis_config' + +interface IlrBasisInputsProps { + evidenceMode: boolean + config: AnalysisConfig + onChange: (config: AnalysisConfig) => void + validationErrors?: Record +} + +const BASIS_LABELS: Record = { + default: 'default — Helmert (unchanged)', + phylogenetic: 'phylogenetic — PhILR (needs a rooted, bifurcating tree)', + sequential_binary_partition: 'sequential_binary_partition — your SBP matrix (CSV)', + balance_dendrogram: 'balance_dendrogram — clustered from the data (variation matrix)', +} + +const helpBox = { fontSize: 12, background: '#fff', padding: 8, marginTop: 4, borderRadius: 4, whiteSpace: 'pre-wrap' as const } +const errorText = { color: '#c62828', fontSize: 12 } + +export function IlrBasisInputs({ evidenceMode, config, onChange, validationErrors }: IlrBasisInputsProps) { + const [helpField, setHelpField] = useState(null) + const basis = ilrBasisOf(config) + const a = config.advanced + const problems = ilrInputProblems(config) + const setAdvanced = (patch: Partial) => onChange({ ...config, advanced: { ...a, ...patch } }) + + const help = (field: string) => ( + + ) + const helpText = (field: string) => helpField === field ?
{contextHelp(field)}
: null + const fieldErrors = (field: string) => { + const messages = problems.filter(p => p.field === field).map(p => p.message) + const server = validationErrors?.[field] + if (server && !messages.includes(server)) messages.push(server) + return messages.map(m =>
{m}
) + } + const pathInput = (field: 'ilr_phylo_tree_path' | 'ilr_sbp_matrix_path', label: string, placeholder: string) => ( +
+ + setAdvanced({ [field]: e.target.value === '' ? null : e.target.value })} + placeholder={placeholder} + style={{ width: '100%', padding: 8, marginTop: 4, fontFamily: 'monospace' }} + /> + {helpText(`advanced.${field}`)} + {fieldErrors(`advanced.${field}`)} +
+ ) + + const partWeights = a.ilr_part_weights ?? 'uniform' + const balanceWeights = a.ilr_balance_weights ?? 'uniform' + const history = a.ilr_sbp_history ?? [] + const attempts = ilrSbpAttempts(a) + const shownFields = new Set([ + ...(basis === 'phylogenetic' ? ['advanced.ilr_phylo_tree_path'] : []), + ...(basis === 'sequential_binary_partition' ? ['advanced.ilr_sbp_matrix_path'] : []), + ...(basis === 'balance_dendrogram' ? ['advanced.ilr_balance_dendrogram_method'] : []), + ...(evidenceMode && basis !== 'default' ? ['advanced.ilr_part_weights'] : []), + ...(evidenceMode && basis === 'phylogenetic' ? ['advanced.ilr_balance_weights'] : []), + ...(evidenceMode && basis === 'sequential_binary_partition' ? ['advanced.ilr_sbp_history'] : []), + ]) + + return ( +
+ + + {helpText('normalization.ilr_basis')} + {basis !== 'default' && ( +
+ Switching basis clears the inputs of the other bases; nothing is chosen for you. Benjamini-Hochberg across balances stays mandatory. +
+ )} + + {basis === 'phylogenetic' && pathInput('ilr_phylo_tree_path', 'Newick tree', 'data/tree.nwk')} + {basis === 'sequential_binary_partition' && pathInput('ilr_sbp_matrix_path', 'SBP matrix CSV', 'data/sbp.csv')} + {basis === 'balance_dendrogram' && ( +
+ + + {helpText('advanced.ilr_balance_dendrogram_method')} + {fieldErrors('advanced.ilr_balance_dendrogram_method')} +
+ )} + + {basis !== 'default' && evidenceMode && ( +
+
🔬 Advanced Analysis — ILR basis options
+ + + {helpText('advanced.ilr_part_weights')} + {fieldErrors('advanced.ilr_part_weights')} + + {basis === 'phylogenetic' && ( +
+ + + {balanceWeights !== 'uniform' && ( +
+ Weighted balances are not isometric: effect sizes change, per-balance test statistics do not. Recorded in provenance. +
+ )} + {helpText('advanced.ilr_balance_weights')} + {fieldErrors('advanced.ilr_balance_weights')} +
+ )} + + {basis === 'sequential_binary_partition' && ( +
+ +