From 5322c185843c8d538359c74ce8aeea9dcf99fc2c Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Fri, 5 Jun 2026 16:25:07 +1000 Subject: [PATCH 01/17] Port The Cannon's numerical core to JAX Rewrite the training/test steps, vectorizer, continuum and model code to use jax.numpy and jax-based autodiff/optimization (jax.vmap, jax.jacfwd, jaxopt) in place of the numpy/scipy + multiprocessing implementation. - Enable 64-bit JAX globally and shim jax.tree_map for jaxopt 0.8.3 - Replace hand-written label-vector derivative with jax.jacfwd - numpy 2.0 compat (RankWarning, collections.abc.Iterable) - Add jax/jaxlib/jaxopt deps; pytest as a test extra - Add parity tests against frozen golden numpy/scipy outputs Co-Authored-By: Claude Opus 4.8 (1M context) --- conftest.py | 7 + setup.py | 4 +- thecannon/__init__.py | 24 +- thecannon/continuum.py | 28 +- thecannon/fitting.py | 800 +++++++++++++++-------------- thecannon/model.py | 167 +++--- thecannon/tests/_capture_golden.py | 234 +++++++++ thecannon/tests/conftest.py | 6 + thecannon/tests/golden/golden.pkl | Bin 0 -> 130716 bytes thecannon/tests/test_jax_parity.py | 308 +++++++++++ thecannon/utils.py | 10 +- thecannon/vectorizer/polynomial.py | 45 +- 12 files changed, 1148 insertions(+), 485 deletions(-) create mode 100644 conftest.py create mode 100644 thecannon/tests/_capture_golden.py create mode 100644 thecannon/tests/conftest.py create mode 100644 thecannon/tests/golden/golden.pkl create mode 100644 thecannon/tests/test_jax_parity.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..2bdd1ab --- /dev/null +++ b/conftest.py @@ -0,0 +1,7 @@ +import os + +# Pin the JAX CPU backend for the test session *before* anything imports jax. +# This root conftest is loaded by pytest before the `thecannon` package (and +# therefore jax) is imported, which the in-package tests/conftest.py cannot +# guarantee. Users with a working accelerator can override JAX_PLATFORMS. +os.environ.setdefault("JAX_PLATFORMS", "cpu") diff --git a/setup.py b/setup.py index 799f1e5..34b7dfb 100644 --- a/setup.py +++ b/setup.py @@ -42,9 +42,9 @@ def read(filename): ], keywords="The Cannon", packages=find_packages(exclude=["documents", "tests"]), - install_requires=["numpy", "scipy", "six"], + install_requires=["numpy", "scipy", "six", "jax", "jaxlib", "jaxopt"], extras_require={ - "test": ["coverage"] + "test": ["coverage", "pytest"] }, package_data={ "": ["LICENSE"], diff --git a/thecannon/__init__.py b/thecannon/__init__.py index 6935272..eb80a0a 100644 --- a/thecannon/__init__.py +++ b/thecannon/__init__.py @@ -4,11 +4,29 @@ __version__ = "0.2.93" import logging -from numpy import RankWarning +try: + from numpy import RankWarning +except ImportError: + # numpy >= 2.0 moved RankWarning to numpy.exceptions. + from numpy.exceptions import RankWarning from warnings import simplefilter +# The Cannon performs scientific (double-precision) linear algebra and +# optimization. JAX defaults to 32-bit; enable 64-bit globally *before* any +# submodule imports jax.numpy. NOTE: this is a process-wide setting and will +# affect any other JAX user in the same interpreter. Import thecannon early. +import jax as _jax +_jax.config.update("jax_enable_x64", True) + +# Compatibility shim: jaxopt 0.8.3 (unmaintained) still calls jax.tree_map, +# which was deprecated in JAX 0.4.x and removed in JAX 0.6. Restore the alias so +# the jaxopt-based test step (fitting.py) works on modern JAX. Harmless on older +# JAX where the attribute already exists. +if not hasattr(_jax, "tree_map"): + _jax.tree_map = _jax.tree_util.tree_map + from .model import CannonModel -from . import (censoring, fitting, plot, utils, vectorizer) +from . import (censoring, continuum, fitting, plot, utils, vectorizer) logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) # TODO: Remove this when stable. @@ -37,4 +55,4 @@ def load_model(path, **kwargs): # Clean up the top-level namespace for this module. -del handler, logger, logging, RankWarning, simplefilter +del handler, logger, logging, RankWarning, simplefilter, _jax diff --git a/thecannon/continuum.py b/thecannon/continuum.py index c475293..4941b59 100644 --- a/thecannon/continuum.py +++ b/thecannon/continuum.py @@ -12,6 +12,7 @@ import logging import numpy as np +import jax.numpy as jnp def _continuum_design_matrix(dispersion, L, order): @@ -158,18 +159,21 @@ def sines_and_cosines(dispersion, flux, ivar, continuum_pixels, L=1400, order=3, continuum_flux, continuum_ivar \ = (object_flux[continuum_mask], object_ivar[continuum_mask]) - # Solve for the amplitudes. - M = continuum_matrix - MTM = np.dot(M, continuum_ivar[:, None] * M.T) - MTy = np.dot(M, (continuum_ivar * continuum_flux).T) - - eigenvalues = np.linalg.eigvalsh(MTM) - MTM[np.diag_indices(len(MTM))] += scalar * np.max(eigenvalues) - eigenvalues = np.linalg.eigvalsh(MTM) - condition_number = max(eigenvalues)/min(eigenvalues) - - amplitudes = np.linalg.solve(MTM, MTy) - continuum[i, region_mask] = np.dot(region_matrix.T, amplitudes) + # Solve for the amplitudes (linear algebra performed in JAX). + M = jnp.asarray(continuum_matrix) + civ = jnp.asarray(continuum_ivar) + cfl = jnp.asarray(continuum_flux) + MTM = jnp.dot(M, civ[:, None] * M.T) + MTy = jnp.dot(M, civ * cfl) + + eigenvalues = jnp.linalg.eigvalsh(MTM) + MTM = MTM + jnp.eye(MTM.shape[0]) * (scalar * jnp.max(eigenvalues)) + eigenvalues = jnp.linalg.eigvalsh(MTM) + condition_number = float(jnp.max(eigenvalues) / jnp.min(eigenvalues)) + + amplitudes = np.asarray(jnp.linalg.solve(MTM, MTy)) + continuum[i, region_mask] = np.asarray( + jnp.dot(jnp.asarray(region_matrix).T, jnp.asarray(amplitudes))) object_metadata.append( (order, L, fill_value, scalar, amplitudes, condition_number)) diff --git a/thecannon/fitting.py b/thecannon/fitting.py index 9e0d4aa..1eed9d5 100644 --- a/thecannon/fitting.py +++ b/thecannon/fitting.py @@ -3,6 +3,18 @@ """ Fitting functions for use in The Cannon. + +This module has been rewritten to use JAX. The numerical core (chi-squared, +the regularized pixel objective, the linear-algebra theta estimate, the noise +scatter fit, the per-pixel training optimization, and the per-spectrum label +inference) all run on ``jax.numpy`` with pure-JAX optimizers from ``jaxopt``. +Analytic gradients and Jacobians that were previously hand-written are now +obtained by automatic differentiation. + +The public functions preserve their signatures and return numpy-compatible +results, so the rest of The Cannon (and existing user code) continues to work. +The per-pixel and per-spectrum cores are written as pure functions so they can +be ``jax.vmap``-ed and ``jax.jit``-ed by :mod:`thecannon.model`. """ from __future__ import (division, print_function, absolute_import, @@ -13,174 +25,132 @@ import logging import numpy as np -import scipy.optimize as op +import jax +import jax.numpy as jnp +from jax import lax from time import time +from jaxopt import LBFGS, LBFGSB, ProximalGradient, LevenbergMarquardt +from jaxopt.prox import prox_lasso + logger = logging.getLogger(__name__) -def fit_spectrum(flux, ivar, initial_labels, vectorizer, theta, s2, fiducials, - scales, dispersion=None, use_derivatives=True, op_kwds=None): - """ - Fit a single spectrum by least-squared fitting. +# Default optimizer settings. These are deliberately tight so that the JAX +# optimizers converge to (essentially) the same optima as the previous +# scipy-based implementation. +_TRAIN_MAXITER = 500 +_TRAIN_TOL = 1e-8 +_TEST_MAXITER = 200 +_TEST_TOL = 1e-10 - :param flux: - The normalized flux values. - :param ivar: - The inverse variance array for the normalized fluxes. - - :param initial_labels: - The point(s) to initialize optimization from. +# --------------------------------------------------------------------------- # +# Core objective pieces # +# --------------------------------------------------------------------------- # - :param vectorizer: - The vectorizer to use when fitting the data. +def chi_sq(theta, design_matrix, flux, ivar, axis=None, gradient=True): + """ + Calculate the chi-squared difference between the spectral model and flux. :param theta: - The theta coefficients (spectral derivatives) of the trained model. + The theta coefficients. - :param s2: - The pixel scatter (s^2) array for each pixel. + :param design_matrix: + The model design matrix. - :param dispersion: [optional] - The dispersion (e.g., wavelength) points for the normalized fluxes. + :param flux: + The normalized flux values. - :param use_derivatives: [optional] - Boolean `True` indicating to use analytic derivatives provided by - the vectorizer, `None` to calculate on the fly, or a callable - function to calculate your own derivatives. + :param ivar: + The inverse variances of the normalized flux values. - :param op_kwds: [optional] - Optimization keywords that get passed to `scipy.optimize.leastsq`. + :param axis: [optional] + The axis to sum the chi-squared values across. + + :param gradient: [optional] + Return the chi-squared value and its derivatives (Jacobian). :returns: - A three-length tuple containing: the optimized labels, the covariance - matrix, and metadata associated with the optimization. + The chi-squared difference between the spectral model and flux, and + optionally, the Jacobian. """ + residuals = jnp.dot(theta, design_matrix.T) - flux - adjusted_ivar = ivar/(1. + ivar * s2) - - # Exclude non-finite points (e.g., points with zero inverse variance - # or non-finite flux values, but the latter shouldn't exist anyway). - use = np.isfinite(flux * adjusted_ivar) * (adjusted_ivar > 0) - L = len(vectorizer.label_names) - - if not np.any(use): - logger.warn("No information in spectrum!") - return (np.nan * np.ones(L), None, { - "fail_message": "Pixels contained no information"}) + ivar_residuals = ivar * residuals + f = jnp.sum(ivar_residuals * residuals, axis=axis) + if not gradient: + return f - # Splice the arrays we will use most. - flux = flux[use] - weights = np.sqrt(adjusted_ivar[use]) # --> 1.0 / sigma - use_theta = theta[use] + g = 2.0 * jnp.dot(design_matrix.T, ivar_residuals) + return (f, g) - initial_labels = np.atleast_2d(initial_labels) - # Check the vectorizer whether it has a derivative built in. - if use_derivatives not in (None, False): - try: - vectorizer.get_label_vector_derivative(initial_labels[0]) +def _chi_sq_only(theta, design_matrix, flux, ivar): + """ The (smooth) chi-squared value only; used as the optimizer objective. """ + residuals = jnp.dot(theta, design_matrix.T) - flux + return jnp.sum(ivar * residuals * residuals) - except NotImplementedError: - Dfun = None - logger.warn("No label vector derivatives available in {}!".format( - vectorizer)) - except: - logger.exception("Exception raised when trying to calculate the "\ - "label vector derivative at the fiducial values:") - raise +def L1Norm_variation(theta): + """ + Return the L1 norm of theta (except the first entry) and its derivative. - else: - # Use the label vector derivative. - Dfun = lambda parameters: weights * np.dot(use_theta, - vectorizer.get_label_vector_derivative(parameters)).T + :param theta: + An array of finite values. - else: - Dfun = None - - def func(parameters): - return np.dot(use_theta, vectorizer(parameters))[:, 0] - - def residuals(parameters): - return weights * (func(parameters) - flux) - - kwds = { - "func": residuals, - "Dfun": Dfun, - "col_deriv": True, - - # These get passed through to leastsq: - "ftol": 7./3 - 4./3 - 1, # Machine precision. - "xtol": 7./3 - 4./3 - 1, # Machine precision. - "gtol": 0.0, - "maxfev": 100000, # MAGIC - "epsfcn": None, - "factor": 1.0, - } + :returns: + A two-length tuple containing: the L1 norm of theta (except the first + entry), and the derivative of the L1 norm of theta. + """ - # Only update the keywords with things that op.curve_fit/op.leastsq expects. - if op_kwds is not None: - for key in set(op_kwds).intersection(kwds): - kwds[key] = op_kwds[key] + return (jnp.sum(jnp.abs(theta[1:])), + jnp.hstack([0.0, jnp.sign(theta[1:])])) - results = [] - for x0 in initial_labels: - try: - op_labels, cov, meta, mesg, ier = op.leastsq( - x0=(x0 - fiducials)/scales, full_output=True, **kwds) +def _pixel_objective_function_fixed_scatter(theta, design_matrix, flux, ivar, + regularization, gradient=True): + """ + The objective function for a single regularized pixel with fixed scatter. - except RuntimeError: - logger.exception("Exception in fitting from {}".format(x0)) - continue + :param theta: + The spectral coefficients. - meta.update( - dict(x0=x0, chi_sq=np.sum(meta["fvec"]**2), ier=ier, mesg=mesg)) - results.append((op_labels, cov, meta)) + :param design_matrix: + The design matrix for the model. - if len(results) == 0: - logger.warn("No results found!") - return (np.nan * np.ones(L), None, dict(fail_message="No results found")) + :param flux: + The normalized flux values for a single pixel across many stars. - best_result_index = np.nanargmin([m["chi_sq"] for (o, c, m) in results]) - op_labels, cov, meta = results[best_result_index] + :param ivar: + The adjusted inverse variance of the normalized flux values. - # De-scale the optimized labels. - meta["model_flux"] = func(op_labels) - op_labels = op_labels * scales + fiducials + :param regularization: + The regularization term to scale the L1 norm of theta with. - if np.allclose(op_labels, meta["x0"]): - logger.warn( - "Discarding optimized result because it is exactly the same as the " - "initial value!") + :param gradient: [optional] + Also return the analytic derivative of the objective function. + """ - # We are in dire straits. We should not trust the result. - op_labels *= np.nan - meta["fail_message"] = "Optimized result same as initial value." + if gradient: + csq, d_csq = chi_sq(theta, design_matrix, flux, ivar, gradient=True) + L1, d_L1 = L1Norm_variation(theta) - if cov is None: - cov = np.ones((len(op_labels), len(op_labels))) + f = csq + regularization * L1 + g = d_csq + regularization * d_L1 - if not np.any(np.isfinite(cov)): - logger.warn("Non-finite covariance matrix returned!") + return (f, g) - # Save additional information. - meta.update({ - "method": "leastsq", - "label_names": vectorizer.label_names, - "best_result_index": best_result_index, - "derivatives_used": Dfun is not None, - "snr": np.nanmedian(flux * weights), - "r_chi_sq": meta["chi_sq"]/(use.sum() - L - 1), - }) - for key in ("ftol", "xtol", "gtol", "maxfev", "factor", "epsfcn"): - meta[key] = kwds[key] + else: + csq = chi_sq(theta, design_matrix, flux, ivar, gradient=False) + L1, d_L1 = L1Norm_variation(theta) - return (op_labels, cov, meta) + return csq + regularization * L1 +# --------------------------------------------------------------------------- # +# Linear algebra theta estimate # +# --------------------------------------------------------------------------- # def fit_theta_by_linalg(flux, ivar, s2, design_matrix): """ @@ -204,159 +174,325 @@ def fit_theta_by_linalg(flux, ivar, s2, design_matrix): matrix. """ - adjusted_ivar = ivar/(1. + ivar * s2) - CiA = design_matrix * np.tile(adjusted_ivar, (design_matrix.shape[1], 1)).T - try: - ATCiAinv = np.linalg.inv(np.dot(design_matrix.T, CiA)) - except np.linalg.linalg.LinAlgError: - N = design_matrix.shape[1] - return (np.hstack([1, np.zeros(N - 1)]), np.inf * np.eye(N)) + flux = jnp.asarray(flux) + ivar = jnp.asarray(ivar) + design_matrix = jnp.asarray(design_matrix) + N = design_matrix.shape[1] - ATY = np.dot(design_matrix.T, flux * adjusted_ivar) - theta = np.dot(ATCiAinv, ATY) + adjusted_ivar = ivar / (1. + ivar * s2) + CiA = design_matrix * adjusted_ivar[:, None] + ATCiAinv = jnp.linalg.inv(jnp.dot(design_matrix.T, CiA)) + ATY = jnp.dot(design_matrix.T, flux * adjusted_ivar) + theta = jnp.dot(ATCiAinv, ATY) - return (theta, ATCiAinv) + # JAX does not raise on a singular matrix (it returns inf/nan), so detect + # that case and fall back to the fiducial value, matching the original + # behaviour which caught `numpy.linalg.LinAlgError`. + ok = jnp.all(jnp.isfinite(theta)) & jnp.all(jnp.isfinite(ATCiAinv)) + theta_fallback = jnp.concatenate([jnp.ones(1), jnp.zeros(N - 1)]) + cov_fallback = jnp.inf * jnp.eye(N) + theta = jnp.where(ok, theta, theta_fallback) + cov = jnp.where(ok, ATCiAinv, cov_fallback) + return (theta, cov) -# TODO: This logic should probably go somewhere else. +# --------------------------------------------------------------------------- # +# Noise scatter fit # +# --------------------------------------------------------------------------- # -def chi_sq(theta, design_matrix, flux, ivar, axis=None, gradient=True): - """ - Calculate the chi-squared difference between the spectral model and flux. +def _scatter_objective_function(scatter, residuals_squared, ivar): + """ Legacy-compatible scalar scatter objective (kept for completeness). """ + adjusted_ivar = ivar / (1.0 + ivar * scatter ** 2) + chi_sq_value = residuals_squared * adjusted_ivar + return (jnp.median(chi_sq_value) - 1.0) ** 2 - :param theta: - The theta coefficients. - :param design_matrix: - The model design matrix. +def _fit_scatter(residuals_squared, ivar, n_iter=80): + """ + Solve for the squared scatter ``s2 = scatter**2 >= 0`` such that the median + of ``residuals_squared * ivar / (1 + ivar * s2)`` equals one. + + The median is a monotonically decreasing function of ``s2``, so we use a + bracketing bisection. If the median is already <= 1 at ``s2 = 0`` then no + positive scatter is required and ``s2 = 0`` is returned. This reproduces the + behaviour of the original Nelder-Mead minimization of ``(median - 1)**2`` + starting from zero, while being jit/vmap-safe. + """ - :param flux: - The normalized flux values. + def median_of(u): + adjusted_ivar = ivar / (1.0 + ivar * u) + return jnp.median(residuals_squared * adjusted_ivar) - :param ivar: - The inverse variances of the normalized flux values. + m0 = median_of(0.0) - :param axis: [optional] - The axis to sum the chi-squared values across. + # Each term a_i/(1 + b_i u) <= 1 once u >= residuals_squared_i, so the median + # is guaranteed <= 1 by u = max(residuals_squared); that brackets the root. + upper = jnp.max(residuals_squared) + 1.0 - :param gradient: [optional] - Return the chi-squared value and its derivatives (Jacobian). + def body(_, bounds): + lo, hi = bounds + mid = 0.5 * (lo + hi) + need_larger = median_of(mid) > 1.0 # median decreasing -> go right + lo = jnp.where(need_larger, mid, lo) + hi = jnp.where(need_larger, hi, mid) + return (lo, hi) - :returns: - The chi-squared difference between the spectral model and flux, and - optionally, the Jacobian. - """ - residuals = np.dot(theta, design_matrix.T) - flux + lo, hi = lax.fori_loop(0, n_iter, body, (0.0 * upper, upper)) + root = 0.5 * (lo + hi) - ivar_residuals = ivar * residuals - f = np.sum(ivar_residuals * residuals, axis=axis) - if not gradient: - return f + return jnp.where(m0 > 1.0, root, 0.0) - g = 2.0 * np.dot(design_matrix.T, ivar_residuals) - return (f, g) +# --------------------------------------------------------------------------- # +# Per-pixel training fit # +# --------------------------------------------------------------------------- # -def L1Norm_variation(theta): +def make_pixel_fitter(op_method="l_bfgs_b", maxiter=_TRAIN_MAXITER, + tol=_TRAIN_TOL, bounds=None): """ - Return the L1 norm of theta (except the first entry) and its derivative. + Build a pure, ``jax.vmap``-able function that fits the theta coefficients + and noise residual for a single pixel. - :param theta: - An array of finite values. + :param op_method: [optional] + The optimization method. ``"l_bfgs_b"`` (default) minimizes the + combined ``chi_sq + lambda * ||theta[1:]||_1`` objective with L-BFGS + (closest to the original scipy behaviour). ``"proximal"`` solves the + lasso with proximal gradient (``prox_lasso``), giving exact zeros. + + :param bounds: [optional] + A two-length tuple ``(lower, upper)`` of arrays of shape ``(T,)`` giving + box constraints on the theta coefficients. When provided, the bounded + L-BFGS-B solver is used. Use ``+/- inf`` for unconstrained coefficients. :returns: - A two-length tuple containing: the L1 norm of theta (except the first - entry), and the derivative of the L1 norm of theta. + A function ``fit(flux, ivar, init_stack, design_matrix, regularization, + column_mask)`` returning ``(theta, s2, fopt)``. + + - ``init_stack`` is a ``(n_init, T)`` array of initial theta guesses. + - ``column_mask`` is a ``(T,)`` boolean array; ``False`` marks censored + coefficients that are held at zero. """ - return (np.sum(np.abs(theta[1:])), np.hstack([0.0, np.sign(theta[1:])])) + op_method = (op_method or "l_bfgs_b").lower() + if op_method == "powell": + logger.warn("op_method='powell' is not supported by the JAX backend; " + "using L-BFGS instead.") + op_method = "l_bfgs_b" + if op_method not in ("l_bfgs_b", "proximal"): + raise ValueError("unknown optimization method '{}' -- 'l_bfgs_b' or " + "'proximal' are available".format(op_method)) + + if bounds is not None: + lower, upper = (jnp.asarray(bounds[0]), jnp.asarray(bounds[1])) + if op_method == "proximal": + logger.warn("theta bounds are not supported with op_method=" + "'proximal'; using bounded L-BFGS-B instead.") + + def fit(flux, ivar, init_stack, design_matrix, regularization, column_mask): + + T = design_matrix.shape[1] + mask = column_mask.astype(design_matrix.dtype) + + # No information in this pixel -> fiducial theta with infinite scatter. + no_info = jnp.sum(ivar) < ivar.size + + # Zero out censored columns so they cannot contribute. + dm = design_matrix * mask[None, :] + + def smooth_objective(theta): + return _chi_sq_only(theta, dm, flux, ivar) \ + + regularization * jnp.sum(jnp.abs(theta[1:])) + + # Choose the best starting point by objective value. + feval = jax.vmap(smooth_objective)(init_stack) + feval = jnp.where(jnp.isnan(feval), jnp.inf, feval) + best_init = init_stack[jnp.argmin(feval)] + + if bounds is not None: + # Do not constrain censored coefficients (they are masked to zero). + lower_eff = jnp.where(column_mask, lower, -jnp.inf) + upper_eff = jnp.where(column_mask, upper, jnp.inf) + best_init = jnp.clip(best_init, lower_eff, upper_eff) + solver = LBFGSB(fun=smooth_objective, maxiter=maxiter, tol=tol) + theta = solver.run(best_init, bounds=(lower_eff, upper_eff)).params + elif op_method == "proximal": + # Per-coordinate L1 weights: do not regularize theta[0] (continuum), + # and do not regularize censored coefficients (held at zero anyway). + l1reg = jnp.full((T,), regularization).at[0].set(0.0) * mask + solver = ProximalGradient( + fun=lambda th: _chi_sq_only(th, dm, flux, ivar), + prox=prox_lasso, maxiter=maxiter, tol=tol) + theta = solver.run(best_init, l1reg).params + else: + solver = LBFGS(fun=smooth_objective, maxiter=maxiter, tol=tol) + theta = solver.run(best_init).params + # Censored coefficients are exactly zero. + theta = theta * mask -def _pixel_objective_function_fixed_scatter(theta, design_matrix, flux, ivar, - regularization, gradient=True): + residuals_squared = (flux - jnp.dot(theta, dm.T)) ** 2 + s2 = _fit_scatter(residuals_squared, ivar) + fopt = smooth_objective(theta) + + fiducial = jnp.concatenate([jnp.ones(1), jnp.zeros(T - 1)]) + theta = jnp.where(no_info, fiducial, theta) + s2 = jnp.where(no_info, jnp.inf, s2) + fopt = jnp.where(no_info, jnp.nan, fopt) + + return (theta, s2, fopt) + + return fit + + +def fit_pixel_fixed_scatter(flux, ivar, initial_thetas, design_matrix, + regularization, censoring_mask, **kwargs): """ - The objective function for a single regularized pixel with fixed scatter. + Fit theta coefficients and noise residual for a single pixel, using + an initially fixed scatter value. - :param theta: - The spectral coefficients. + :param flux: + The normalized flux values. - :param normalized_flux: - The normalized flux values for a single pixel across many stars. + :param ivar: + The inverse variance array for the normalized fluxes. - :param adjusted_ivar: - The adjusted inverse variance of the normalized flux values for a single - pixel across many stars. This adjusted inverse variance array should - already have the scatter included. + :param initial_thetas: + A list of initial theta values to start from, and their source. For + example: ``[(theta_0, "guess"), (theta_1, "old_theta")]``. + + :param design_matrix: + The model design matrix. Censored coefficients may be indicated by + columns that are entirely non-finite (the historical convention). :param regularization: - The regularization term to scale the L1 norm of theta with. + The regularization strength to apply during optimization (Lambda). - :param design_matrix: - The design matrix for the model. + :param censoring_mask: + A per-label censoring mask for each pixel. (Unused directly here; the + censored coefficients are inferred from the design matrix.) - :param gradient: [optional] - Also return the analytic derivative of the objective function. + :keyword op_method: + The optimization method to use. Valid options are: ``l_bfgs_b``, + ``proximal``. (``powell`` is accepted for backwards compatibility but + falls back to ``l_bfgs_b``.) + + :returns: + The optimized theta coefficients, the noise residual ``s2``, and + metadata related to the optimization process. """ - if gradient: - csq, d_csq = chi_sq(theta, design_matrix, flux, ivar, gradient=True) - L1, d_L1 = L1Norm_variation(theta) + flux = jnp.asarray(flux) + ivar = jnp.asarray(ivar) + design_matrix = jnp.asarray(design_matrix) - f = csq + regularization * L1 - g = d_csq + regularization * d_L1 + T = design_matrix.shape[1] - return (f, g) + # Censored coefficients are those whose design-matrix column is entirely + # non-finite (the original convention used `numpy.nan` as a fill value). + column_mask = jnp.any(jnp.isfinite(design_matrix), axis=0) + # Replace non-finite entries so the masked-out columns are safe to use. + design_matrix = jnp.where(jnp.isfinite(design_matrix), design_matrix, 0.0) - else: - csq = chi_sq(theta, design_matrix, flux, ivar, gradient=False) - L1, d_L1 = L1Norm_variation(theta) + # Stack the candidate initial thetas into a fixed (n_init, T) array. + init_stack = jnp.atleast_2d( + jnp.asarray([np.asarray(theta) for theta, _ in initial_thetas])) - return csq + regularization * L1 + op_method = kwargs.get("op_method", "l_bfgs_b") + op_kwds = kwargs.get("op_kwds", {}) or {} + maxiter = op_kwds.get("maxiter", _TRAIN_MAXITER) + tol = op_kwds.get("tol", _TRAIN_TOL) + + t_init = time() + fitter = make_pixel_fitter(op_method=op_method, maxiter=maxiter, tol=tol) + theta, s2, fopt = fitter( + flux, ivar, init_stack, design_matrix, regularization, column_mask) + theta = np.asarray(theta) + s2 = float(np.asarray(s2)) -def _scatter_objective_function(scatter, residuals_squared, ivar): - adjusted_ivar = ivar/(1.0 + ivar * scatter**2) - chi_sq = residuals_squared * adjusted_ivar - return (np.median(chi_sq) - 1.0)**2 + # Determine which starting point was selected, for metadata. + metadata = dict( + op_method=("l_bfgs_b" if (op_method or "l_bfgs_b").lower() == "powell" + else (op_method or "l_bfgs_b").lower()), + op_time=time() - t_init, + fopt=float(np.asarray(fopt)), + initial_theta=np.asarray(init_stack[0]), + initial_theta_source=initial_thetas[0][1] if initial_thetas else None) + return (theta, s2, metadata) -def _remove_forbidden_op_kwds(op_method, op_kwds): - """ - Remove forbidden optimization keywords. - :param op_method: - The optimization algorithm to use. +# --------------------------------------------------------------------------- # +# Per-spectrum label inference (test step) # +# --------------------------------------------------------------------------- # - :param op_kwds: - Optimization keywords. +def make_spectrum_fitter(vectorizer, theta, s2, fiducials, scales, + maxiter=_TEST_MAXITER, tol=_TEST_TOL): + """ + Build a pure, ``jax.vmap``-able function that infers stellar labels for a + single spectrum via Levenberg-Marquardt nonlinear least squares. :returns: - `None`. The dictionary of `op_kwds` will be updated. + A function ``core(flux, ivar, initial_labels)`` returning + ``(op_labels, cov, chi_sq, model_flux, n_use)`` where ``initial_labels`` + is a ``(n_init, L)`` array of starting points (the best is selected). """ - all_allowed_keys = dict( - l_bfgs_b=("x0", "args", "bounds", "m", "factr", "pgtol", "epsilon", - "iprint", "maxfun", "maxiter", "disp", "callback", "maxls"), - powell=("x0", "args", "xtol", "ftol", "maxiter", "maxfun", - "full_output", "disp", "retall", "callback", "initial_simplex")) - forbidden_keys = set(op_kwds).difference(all_allowed_keys[op_method]) - if forbidden_keys: - logger.warn("Ignoring forbidden optimization keywords for {}: {}"\ - .format(op_method, ", ".join(forbidden_keys))) - for key in forbidden_keys: - del op_kwds[key] + theta = jnp.asarray(theta) + s2 = jnp.asarray(s2) + fiducials = jnp.asarray(fiducials) + scales = jnp.asarray(scales) - return None - + def core(flux, ivar, initial_labels): + adjusted_ivar = ivar / (1. + ivar * s2) -def fit_pixel_fixed_scatter(flux, ivar, initial_thetas, design_matrix, - regularization, censoring_mask, **kwargs): + # Exclude non-finite / zero-information pixels by zero-weighting them + # (rather than removing them) so the residual vector keeps a fixed size. + use = jnp.isfinite(flux * adjusted_ivar) & (adjusted_ivar > 0) + weights = jnp.sqrt(jnp.where(use, adjusted_ivar, 0.0)) + flux_safe = jnp.where(use, flux, 0.0) + safe_theta = jnp.where(use[:, None], jnp.nan_to_num(theta), 0.0) + + def model_flux(scaled_params): + return jnp.dot(safe_theta, vectorizer(scaled_params))[:, 0] + + def residuals(scaled_params): + return weights * (model_flux(scaled_params) - flux_safe) + + lm = LevenbergMarquardt( + residual_fun=residuals, maxiter=maxiter, tol=tol, xtol=tol, + gtol=tol) + + def solve_one(x0): + scaled_x0 = (x0 - fiducials) / scales + params = lm.run(scaled_x0).params + r = residuals(params) + return params, jnp.sum(r ** 2) + + params_all, chi_sqs = jax.vmap(solve_one)(initial_labels) + chi_sqs = jnp.where(jnp.isnan(chi_sqs), jnp.inf, chi_sqs) + best = jnp.argmin(chi_sqs) + params = params_all[best] + chi_sq_best = chi_sqs[best] + + # Covariance = inv(J^T J), where J is the Jacobian of the (weighted) + # residuals. This matches scipy.optimize.leastsq's `cov_x`. + J = jax.jacfwd(residuals)(params) + cov = jnp.linalg.inv(jnp.dot(J.T, J)) + + op_labels = params * scales + fiducials + return (op_labels, cov, chi_sq_best, model_flux(params), jnp.sum(use)) + + return core + + +def fit_spectrum(flux, ivar, initial_labels, vectorizer, theta, s2, fiducials, + scales, dispersion=None, use_derivatives=True, op_kwds=None): """ - Fit theta coefficients and noise residual for a single pixel, using - an initially fixed scatter value. + Fit a single spectrum by least-squares fitting to solve for labels. :param flux: The normalized flux values. @@ -364,170 +500,80 @@ def fit_pixel_fixed_scatter(flux, ivar, initial_thetas, design_matrix, :param ivar: The inverse variance array for the normalized fluxes. - :param initial_thetas: - A list of initial theta values to start from, and their source. For - example: `[(theta_0, "guess"), (theta_1, "old_theta")] + :param initial_labels: + The point(s) to initialize optimization from. - :param design_matrix: - The model design matrix. + :param vectorizer: + The vectorizer to use when fitting the data. - :param regularization: - The regularization strength to apply during optimization (Lambda). + :param theta: + The theta coefficients (spectral derivatives) of the trained model. - :param censoring_mask: - A per-label censoring mask for each pixel. + :param s2: + The pixel scatter (s^2) array for each pixel. - :keyword op_method: - The optimization method to use. Valid options are: `l_bfgs_b`, `powell`. + :param fiducials: + The fiducial label values used to scale the labels. + + :param scales: + The scale values used to normalize the labels. + + :param dispersion: [optional] + The dispersion (e.g., wavelength) points for the normalized fluxes. + + :param use_derivatives: [optional] + Retained for API compatibility. The Levenberg-Marquardt optimizer now + always uses Jacobians obtained by automatic differentiation. - :keyword op_kwds: - A dictionary of arguments that will be provided to the optimizer. + :param op_kwds: [optional] + Optimization keywords. ``maxiter`` and ``tol`` are honoured. :returns: - The optimized theta coefficients, the noise residual `s2`, and - metadata related to the optimization process. + A three-length tuple containing: the optimized labels, the covariance + matrix, and metadata associated with the optimization. """ - if np.sum(ivar) < 1.0 * ivar.size: # MAGIC - metadata = dict(message="No pixel information.", op_time=0.0) - fiducial = np.hstack([1.0, np.zeros(design_matrix.shape[1] - 1)]) - return (fiducial, np.inf, metadata) # MAGIC - - # Determine if any theta coefficients will be censored. - censored_theta = ~np.any(np.isfinite(design_matrix), axis=0) - # Make the design matrix safe to use. - design_matrix[:, censored_theta] = 0 - - feval = [] - for initial_theta, initial_theta_source in initial_thetas: - feval.append(_pixel_objective_function_fixed_scatter( - initial_theta, design_matrix, flux, ivar, regularization, False)) - - initial_theta, initial_theta_source = initial_thetas[np.nanargmin(feval)] - - base_op_kwds = dict(x0=initial_theta, - args=(design_matrix, flux, ivar, regularization), - disp=False, maxfun=np.inf, maxiter=np.inf) - - theta_0 = kwargs.get("__theta_0", None) - if theta_0 is not None: - logger.warn("FIXING theta_0. HIGHLY EXPERIMENTAL.") - - # Subtract from flux. - # Set design matrix entry to zero. - # Update to theta later on. - new_flux = flux - theta_0 - new_design_matrix = np.copy(design_matrix) - new_design_matrix[:, 0] = 0.0 - - base_op_kwds["args"] = (new_design_matrix, new_flux, ivar, regularization) - - if any(censored_theta): - # If the initial_theta is the same size as the censored_mask, but different - # to the design_matrix, then we need to censor the initial theta so that we - # don't bother solving for those parameters. - base_op_kwds["x0"] = np.array(base_op_kwds["x0"])[~censored_theta] - base_op_kwds["args"] = (design_matrix[:, ~censored_theta], flux, ivar, - regularization) - - # Allow either l_bfgs_b or powell - t_init = time() - default_op_method = "l_bfgs_b" - op_method = kwargs.get("op_method", default_op_method) or default_op_method - op_method = op_method.lower() - - op_strict = kwargs.get("op_strict", True) - - while True: - if op_method == "l_bfgs_b": - op_kwds = dict() - op_kwds.update(base_op_kwds) - op_kwds.update( - m=design_matrix.shape[1], maxls=20, factr=10.0, pgtol=1e-6) - op_kwds.update((kwargs.get("op_kwds", {}) or {})) - - # If op_bounds are given and we are censoring some theta terms, then we - # will need to adjust which op_bounds we provide. - if "bounds" in op_kwds and any(censored_theta): - op_kwds["bounds"] = [b for b, is_censored in \ - zip(op_kwds["bounds"], censored_theta) if not is_censored] - - # Just-in-time to remove forbidden keywords. - _remove_forbidden_op_kwds(op_method, op_kwds) - - op_params, fopt, metadata = op.fmin_l_bfgs_b( - _pixel_objective_function_fixed_scatter, - fprime=None, approx_grad=None, **op_kwds) - - metadata.update(dict(fopt=fopt)) - - warnflag = metadata.get("warnflag", -1) - if warnflag > 0: - reason = "too many function evaluations or too many iterations" \ - if warnflag == 1 else metadata["task"] - logger.warn("Optimization warning (l_bfgs_b): {}".format(reason)) - - if op_strict: - # Do optimization again. - op_method = "powell" - base_op_kwds.update(x0=op_params) - else: - break - - else: - break - - elif op_method == "powell": - op_kwds = dict() - op_kwds.update(base_op_kwds) - op_kwds.update(xtol=1e-6, ftol=1e-6) - op_kwds.update((kwargs.get("op_kwds", {}) or {})) - - # Set 'False' in args so that we don't return the gradient, - # because fmin doesn't want it. - args = list(op_kwds["args"]) - args.append(False) - op_kwds["args"] = tuple(args) - - t_init = time() - - # Just-in-time to remove forbidden keywords. - _remove_forbidden_op_kwds(op_method, op_kwds) - - op_params, fopt, direc, n_iter, n_funcs, warnflag = op.fmin_powell( - _pixel_objective_function_fixed_scatter, - full_output=True, **op_kwds) - - metadata = dict(fopt=fopt, direc=direc, n_iter=n_iter, - n_funcs=n_funcs, warnflag=warnflag) - break - - else: - raise ValueError("unknown optimization method '{}' -- " - "powell or l_bfgs_b are available".format(op_method)) + op_kwds = op_kwds or {} + maxiter = op_kwds.get("maxiter", _TEST_MAXITER) + tol = op_kwds.get("tol", _TEST_TOL) - # Additional metadata common to both optimizers. - metadata.update(dict(op_method=op_method, op_time=time() - t_init, - initial_theta=initial_theta, initial_theta_source=initial_theta_source)) + L = len(vectorizer.label_names) - # De-censor the optimized parameters. - if any(censored_theta): - theta = np.zeros(censored_theta.size) - theta[~censored_theta] = op_params + flux = jnp.asarray(flux) + ivar = jnp.asarray(ivar) + adjusted_ivar = ivar / (1. + ivar * jnp.asarray(s2)) + if not bool(jnp.any(jnp.isfinite(flux * adjusted_ivar) + & (adjusted_ivar > 0))): + logger.warn("No information in spectrum!") + return (np.nan * np.ones(L), None, + {"fail_message": "Pixels contained no information"}) - else: - theta = op_params + core = make_spectrum_fitter( + vectorizer, theta, s2, fiducials, scales, maxiter=maxiter, tol=tol) - if theta_0 is not None: - theta[0] = theta_0 + initial_labels = jnp.atleast_2d(jnp.asarray(initial_labels)) + op_labels, cov, chi_sq_value, model_flux, n_use = core( + flux, ivar, initial_labels) - # Fit the scatter. - op_fmin_kwds = dict(disp=False, maxiter=np.inf, maxfun=np.inf) - op_fmin_kwds.update( - xtol=op_kwds.get("xtol", 1e-8), ftol=op_kwds.get("ftol", 1e-8)) + op_labels = np.asarray(op_labels) + cov = np.asarray(cov) + chi_sq_value = float(np.asarray(chi_sq_value)) + n_use = int(np.asarray(n_use)) - residuals_squared = (flux - np.dot(theta, design_matrix.T))**2 - scatter = op.fmin(_scatter_objective_function, 0.0, - args=(residuals_squared, ivar), disp=False) + if cov is None or not np.any(np.isfinite(cov)): + logger.warn("Non-finite covariance matrix returned!") + if cov is None: + cov = np.ones((L, L)) + + meta = { + "chi_sq": chi_sq_value, + "r_chi_sq": chi_sq_value / max(1, (n_use - L - 1)), + "model_flux": np.asarray(model_flux), + "method": "levenberg_marquardt", + "label_names": vectorizer.label_names, + "derivatives_used": True, + "maxiter": maxiter, + "tol": tol, + } - return (theta, scatter**2, metadata) + return (op_labels, cov, meta) diff --git a/thecannon/model.py b/thecannon/model.py index 8e69301..3ee2fea 100644 --- a/thecannon/model.py +++ b/thecannon/model.py @@ -13,6 +13,8 @@ import logging import multiprocessing as mp import numpy as np +import jax +import jax.numpy as jnp import os import pickle from datetime import datetime @@ -497,16 +499,19 @@ def write(self, path, include_training_set_spectra=False, overwrite=False, attributes.remote("metadata") # Store up all the trained attributes and a hash of the training set. + # Only the vectorizer and censors are serialized via their custom + # `__getstate__` (the vectorizer to a (name, kwds) tuple and the censors + # to a dict); everything else (numpy arrays, scalars) is stored as-is. + # NB: numpy>=2.0 defines `ndarray.__getstate__` (returning None), so the + # historical broad ``value.__getstate__()`` call would corrupt arrays. state = {} for attribute in attributes: value = getattr(self, attribute) - try: - # If it's a vectorizer or censoring dict, etc, get the state. + if attribute in ("vectorizer", "censors") and value is not None \ + and hasattr(value, "__getstate__"): value = value.__getstate__() - except: - None state[attribute] = value @@ -612,54 +617,86 @@ def train(self, threads=None, op_method=None, op_strict=True, op_kwds=None, the training of each pixel. """ - kwds = dict(op_method=op_method, op_strict=op_strict, op_kwds=op_kwds) - kwds.update(kwargs) - if self.training_set_flux is None or self.training_set_ivar is None: raise TypeError( "cannot train: training set spectra not saved with the model") + if threads not in (1, None): + logger.warn("The `threads` argument is deprecated and ignored: " + "training is vectorized with jax.vmap.") + S, P = self.training_set_flux.shape T = self.design_matrix.shape[1] logger.info("Training {0}-label {1} with {2} stars and {3} pixels/star"\ .format(len(self.vectorizer.label_names), type(self).__name__, S, P)) - # Parallelise out. - if threads in (1, None): - mapper, pool = (map, None) - + op_method = op_method or "l_bfgs_b" + op_kwds = op_kwds or {} + maxiter = op_kwds.get("maxiter", fitting._TRAIN_MAXITER) + tol = op_kwds.get("tol", fitting._TRAIN_TOL) + + # Optional box constraints on theta (used by RestrictedCannonModel). The + # bounds are given as a list of (min, max) tuples per term, with None + # indicating no limit on that side. + bounds = op_kwds.get("bounds", None) + if bounds is not None: + lower = jnp.asarray( + [(-jnp.inf if lo is None else lo) for lo, _ in bounds], + dtype=float) + upper = jnp.asarray( + [(jnp.inf if hi is None else hi) for _, hi in bounds], + dtype=float) + bounds = (lower, upper) + + # Batched (pixel-major) flux and inverse variance. + flux_PN = jnp.asarray(self.training_set_flux.T) # (P, N) + ivar_PN = jnp.asarray(self.training_set_ivar.T) # (P, N) + design_matrix = jnp.asarray(self.design_matrix) # (N, T) + + fiducial = jnp.concatenate([jnp.ones(1), jnp.zeros(T - 1)]) + + # Initial theta guesses for every pixel: a linear-algebra estimate and + # the fiducial value. The regularized objective is convex, so the + # optimum is independent of the starting point; these only set where the + # optimizer begins. + linalg_theta = jax.vmap( + lambda f, i: fitting.fit_theta_by_linalg(f, i, 0.0, design_matrix)[0] + )(flux_PN, ivar_PN) # (P, T) + finite = jnp.all(jnp.isfinite(linalg_theta), axis=1, keepdims=True) + linalg_theta = jnp.where(finite, linalg_theta, fiducial[None, :]) + init_stack = jnp.stack( + [linalg_theta, jnp.broadcast_to(fiducial, (P, T))], axis=1) # (P,2,T) + + # Per-pixel column mask: True keeps the coefficient, False censors it. + if self.censors and len(set(self.censors).intersection( + self.vectorizer.label_names)) > 0: + column_mask = jnp.asarray( + censoring.design_matrix_mask(self.censors, self.vectorizer)) else: - pool = mp.Pool(threads) - mapper = pool.map + column_mask = jnp.ones((P, T), dtype=bool) - func = utils.wrapper(fitting.fit_pixel_fixed_scatter, None, kwds, P) + # Per-pixel regularization strength. + reg = 0.0 if self.regularization is None else self.regularization + reg_P = jnp.broadcast_to(jnp.asarray(reg, dtype=float), (P,)) - meta = [] - theta = np.nan * np.ones((P, T)) - s2 = np.nan * np.ones(P) + fitter = fitting.make_pixel_fitter( + op_method=op_method, maxiter=maxiter, tol=tol, bounds=bounds) + batched = jax.jit(jax.vmap( + fitter, in_axes=(0, 0, 0, None, 0, 0))) - for pixel, (flux, ivar) \ - in enumerate(zip(self.training_set_flux.T, self.training_set_ivar.T)): + theta, s2, fopt = batched( + flux_PN, ivar_PN, init_stack, design_matrix, reg_P, column_mask) - args = ( - flux, ivar, - self._initial_theta(pixel), - self._censored_design_matrix(pixel), - self._pixel_access(self.regularization, pixel, 0.0), - None - ) - (pixel_theta, pixel_s2, pixel_meta), = mapper(func, [args]) + theta = np.asarray(theta) + s2 = np.asarray(s2) + fopt = np.asarray(fopt) - meta.append(pixel_meta) - theta[pixel], s2[pixel] = (pixel_theta, pixel_s2) + meta = [dict(op_method=op_method, fopt=float(fopt[p]), + maxiter=maxiter, tol=tol) for p in range(P)] self._theta, self._s2 = (theta, s2) - if pool is not None: - pool.close() - pool.join() - return (theta, s2, meta) @@ -674,7 +711,7 @@ def __call__(self, labels): # Scale and offset the labels. scaled_labels = (np.atleast_2d(labels) - self._fiducials)/self._scales - flux = np.dot(self.theta, self.vectorizer(scaled_labels)).T + flux = np.dot(self.theta, np.asarray(self.vectorizer(scaled_labels))).T return flux[0] if flux.shape[0] == 1 else flux @@ -712,12 +749,9 @@ def test(self, flux, ivar, initial_labels=None, threads=None, if op_kwds is None: op_kwds = dict() - if threads in (1, None): - mapper, pool = (map, None) - - else: - pool = mp.Pool(threads) - mapper = pool.map + if threads not in (1, None): + logger.warn("The `threads` argument is deprecated and ignored: the " + "test step is vectorized with jax.vmap.") flux, ivar = (np.atleast_2d(flux), np.atleast_2d(ivar)) S, P = flux.shape @@ -725,28 +759,43 @@ def test(self, flux, ivar, initial_labels=None, threads=None, if ivar.shape != flux.shape: raise ValueError("flux and ivar arrays must be the same shape") + L = len(self._fiducials) + if initial_labels is None: initial_labels = self._fiducials - initial_labels = np.atleast_2d(initial_labels) - if initial_labels.shape[0] != S and len(initial_labels.shape) == 2: - initial_labels = np.tile(initial_labels.flatten(), S)\ - .reshape(S, -1, len(self._fiducials)) - - args = (self.vectorizer, self.theta, self.s2, self._fiducials, - self._scales) - kwargs = dict(use_derivatives=use_derivatives, op_kwds=op_kwds) - - func = utils.wrapper(fitting.fit_spectrum, args, kwargs, S, - message="Running test step on {} spectra".format(S)) - - labels, cov, meta = zip(*mapper(func, zip(*(flux, ivar, initial_labels)))) - - if pool is not None: - pool.close() - pool.join() - - return (np.array(labels), np.array(cov), meta) + # Coerce initial labels to shape (S, n_init, L). + initial_labels = np.atleast_2d(np.asarray(initial_labels, dtype=float)) + if initial_labels.shape == (S, L): + initial_labels = initial_labels[:, None, :] # one start per star + elif initial_labels.ndim == 2: + initial_labels = np.tile(initial_labels[None], (S, 1, 1)) # shared + elif initial_labels.ndim == 3 and initial_labels.shape[0] != S: + initial_labels = np.tile(initial_labels, (S, 1, 1)) + + maxiter = op_kwds.get("maxiter", fitting._TEST_MAXITER) + tol = op_kwds.get("tol", fitting._TEST_TOL) + + core = fitting.make_spectrum_fitter( + self.vectorizer, self.theta, self.s2, self._fiducials, + self._scales, maxiter=maxiter, tol=tol) + batched = jax.jit(jax.vmap(core, in_axes=(0, 0, 0))) + + op_labels, cov, chi_sq, model_flux, n_use = batched( + jnp.asarray(flux), jnp.asarray(ivar), jnp.asarray(initial_labels)) + + op_labels = np.asarray(op_labels) + cov = np.asarray(cov) + chi_sq = np.asarray(chi_sq) + n_use = np.asarray(n_use) + + meta = [dict(chi_sq=float(chi_sq[s]), + r_chi_sq=float(chi_sq[s]) / max(1, int(n_use[s]) - L - 1), + method="levenberg_marquardt", + label_names=self.vectorizer.label_names) + for s in range(S)] + + return (op_labels, cov, meta) def _initial_theta(self, pixel_index, **kwargs): diff --git a/thecannon/tests/_capture_golden.py b/thecannon/tests/_capture_golden.py new file mode 100644 index 0000000..d6acad8 --- /dev/null +++ b/thecannon/tests/_capture_golden.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Capture golden reference inputs+outputs from the *legacy* numpy/scipy +implementation of The Cannon, so the JAX rewrite can be verified to yield the +same results. + +Run this ONCE against the numpy/scipy code (before the JAX port), e.g.: + + python -m thecannon.tests._capture_golden + +It writes ``thecannon/tests/golden/golden.pkl`` containing a dictionary with +``inputs`` (everything needed to re-run each numerical path) and ``outputs`` +(the legacy results). The parity tests in ``test_jax_parity.py`` load this file, +re-run the same inputs through the (now JAX) code, and assert agreement. +""" + +from __future__ import (division, print_function, absolute_import, + unicode_literals) + +import os +import pickle +import numpy as np + +import thecannon as tc +from thecannon import fitting, continuum +from thecannon.vectorizer.polynomial import PolynomialVectorizer + + +HERE = os.path.dirname(os.path.abspath(__file__)) +GOLDEN_PATH = os.path.join(HERE, "golden", "golden.pkl") + +SEED = 20240601 +LABEL_NAMES = ("TEFF", "LOGG", "FEH") +ORDER = 2 +N_STARS = 40 +N_PIXELS = 60 + + +def build_synthetic(): + """Build a small, deterministic synthetic training set.""" + rng = np.random.RandomState(SEED) + + vectorizer = PolynomialVectorizer(label_names=LABEL_NAMES, order=ORDER) + T = 1 + len(vectorizer.terms) # number of theta coefficients per pixel + + # Physically-ish ranges, then we work in scaled space internally. + teff = rng.uniform(4000.0, 6500.0, size=N_STARS) + logg = rng.uniform(1.0, 4.5, size=N_STARS) + feh = rng.uniform(-1.5, 0.4, size=N_STARS) + labels = np.vstack([teff, logg, feh]).T # (N_STARS, 3) + + # Fiducials/scales the way CannonModel computes them. + scales = np.ptp(np.percentile(labels, [2.5, 97.5], axis=0), axis=0) + fiducials = np.percentile(labels, 50, axis=0) + scaled = (labels - fiducials) / scales + + design_matrix = vectorizer(scaled).T # (N_STARS, T) + + # A "true" theta with a sensible continuum (~1) and clear label structure + # (boost the linear terms so the inverse/inference problem is well posed). + true_theta = 0.05 * rng.randn(N_PIXELS, T) + true_theta[:, 0] = 1.0 + K = len(LABEL_NAMES) + true_theta[:, 1:1 + K] += 0.3 * rng.randn(N_PIXELS, K) + + clean_flux = design_matrix @ true_theta.T # (N_STARS, N_PIXELS) + sigma = 0.01 + noise = sigma * rng.randn(N_STARS, N_PIXELS) + flux = clean_flux + noise + ivar = np.ones_like(flux) / (sigma ** 2) + + dispersion = np.linspace(15000.0, 15000.0 + N_PIXELS - 1, N_PIXELS) + + return dict( + vectorizer=vectorizer, labels=labels, scaled=scaled, + scales=scales, fiducials=fiducials, design_matrix=design_matrix, + true_theta=true_theta, flux=flux, ivar=ivar, dispersion=dispersion, + T=T, sigma=sigma) + + +def capture(): + rng = np.random.RandomState(SEED + 1) + syn = build_synthetic() + vectorizer = syn["vectorizer"] + design_matrix = syn["design_matrix"] + flux, ivar = syn["flux"], syn["ivar"] + scaled = syn["scaled"] + T = syn["T"] + + inputs = {} + outputs = {} + + # ---- 1. Vectorizer ---------------------------------------------------- + # Batched label vector. + inputs["vec_labels_batch"] = scaled.copy() + outputs["get_label_vector_batch"] = vectorizer.get_label_vector(scaled) + + # Single label vector. + single = scaled[0].copy() + inputs["vec_labels_single"] = single + outputs["get_label_vector_single"] = vectorizer.get_label_vector(single) + + # Analytic derivative (single 1-D labels -> (T, L)). + outputs["get_label_vector_derivative"] = \ + vectorizer.get_label_vector_derivative(single) + + # ---- 2. fit_theta_by_linalg ------------------------------------------ + pix = 3 + f0, i0 = flux[:, pix].copy(), ivar[:, pix].copy() + inputs["linalg"] = dict(flux=f0, ivar=i0, s2=0.0, + design_matrix=design_matrix.copy()) + th, cov = fitting.fit_theta_by_linalg(f0, i0, 0.0, design_matrix.copy()) + outputs["fit_theta_by_linalg"] = dict(theta=th, cov=cov) + + # Singular case: a rank-deficient design matrix should hit the fallback. + singular_dm = np.ones((N_STARS, T)) # all columns identical -> singular + th_s, cov_s = fitting.fit_theta_by_linalg(f0, i0, 0.0, singular_dm.copy()) + inputs["linalg_singular"] = dict(flux=f0, ivar=i0, s2=0.0, + design_matrix=singular_dm) + outputs["fit_theta_by_linalg_singular"] = dict(theta=th_s, cov=cov_s) + + # ---- 3. chi_sq + L1Norm_variation + objective ------------------------ + theta_test = th.copy() + inputs["chi_sq"] = dict(theta=theta_test, design_matrix=design_matrix.copy(), + flux=f0, ivar=i0) + csq_val, csq_grad = fitting.chi_sq(theta_test, design_matrix, f0, i0, + gradient=True) + outputs["chi_sq"] = dict(value=np.asarray(csq_val), grad=np.asarray(csq_grad)) + + l1_val, l1_grad = fitting.L1Norm_variation(theta_test) + inputs["L1"] = dict(theta=theta_test) + outputs["L1Norm_variation"] = dict(value=np.asarray(l1_val), + grad=np.asarray(l1_grad)) + + reg = 10.0 + inputs["objective"] = dict(theta=theta_test, + design_matrix=design_matrix.copy(), + flux=f0, ivar=i0, regularization=reg) + obj_val, obj_grad = fitting._pixel_objective_function_fixed_scatter( + theta_test, design_matrix, f0, i0, reg, gradient=True) + outputs["objective"] = dict(value=np.asarray(obj_val), + grad=np.asarray(obj_grad)) + + # ---- 4. scatter fit --------------------------------------------------- + # Build residuals that *require* a positive scatter (median > 1). + resid_sq_hi = (rng.rand(N_STARS) * 5.0) ** 2 + ivar_sc = np.ones(N_STARS) * 50.0 + import scipy.optimize as op + sc_hi = op.fmin(fitting._scatter_objective_function, 0.0, + args=(resid_sq_hi, ivar_sc), disp=False) + inputs["scatter_hi"] = dict(residuals_squared=resid_sq_hi, ivar=ivar_sc) + outputs["scatter_hi"] = float(np.asarray(sc_hi).ravel()[0] ** 2) # s2 + + # And residuals where median already <= 1 (expect ~0 scatter). + resid_sq_lo = (rng.rand(N_STARS) * 0.01) ** 2 + sc_lo = op.fmin(fitting._scatter_objective_function, 0.0, + args=(resid_sq_lo, ivar_sc), disp=False) + inputs["scatter_lo"] = dict(residuals_squared=resid_sq_lo, ivar=ivar_sc) + outputs["scatter_lo"] = float(np.asarray(sc_lo).ravel()[0] ** 2) + + # ---- 5. Training: regularization = 0 (exact least squares) ----------- + model0 = tc.CannonModel(syn["labels"], flux, ivar, vectorizer, + dispersion=syn["dispersion"], regularization=0) + theta0, s2_0, _ = model0.train(threads=1) + outputs["train_reg0"] = dict(theta=np.asarray(theta0), + s2=np.asarray(s2_0)) + + # ---- 6. Training: regularization > 0 (lasso) ------------------------- + modelR = tc.CannonModel(syn["labels"], flux, ivar, vectorizer, + dispersion=syn["dispersion"], regularization=reg) + thetaR, s2_R, _ = modelR.train(threads=1) + outputs["train_regR"] = dict(theta=np.asarray(thetaR), + s2=np.asarray(s2_R), + regularization=reg) + # The achieved objective per pixel (for optimizer-quality comparison). + obj_per_pixel = np.array([ + fitting._pixel_objective_function_fixed_scatter( + thetaR[p], design_matrix, flux[:, p], ivar[:, p], reg, + gradient=False) + for p in range(N_PIXELS)]) + outputs["train_regR_objective"] = obj_per_pixel + + # ---- 7. Prediction (__call__) ---------------------------------------- + pred_labels = syn["labels"][:5].copy() + inputs["predict_labels"] = pred_labels + outputs["predict_reg0"] = np.asarray(model0(pred_labels)) + + # ---- 8. Test step (inference) ---------------------------------------- + n_test = 5 + test_flux = flux[:n_test].copy() + test_ivar = ivar[:n_test].copy() + inputs["test"] = dict(flux=test_flux, ivar=test_ivar, + true_labels=syn["labels"][:n_test].copy()) + rec_labels, rec_cov, _ = model0.test(test_flux, test_ivar, threads=1) + outputs["test_labels"] = np.asarray(rec_labels) + outputs["test_cov"] = np.asarray(rec_cov) + + # ---- 9. Continuum normalization -------------------------------------- + cont_disp = np.linspace(15000.0, 16000.0, 60) + cont_rng = np.random.RandomState(SEED + 2) + cont_flux = 1.0 + 0.1 * np.sin(cont_disp / 50.0) \ + + 0.02 * cont_rng.randn(3, cont_disp.size) + cont_ivar = np.ones_like(cont_flux) / (0.02 ** 2) + continuum_pixels = np.arange(0, cont_disp.size, 2) + inputs["continuum"] = dict(dispersion=cont_disp, flux=cont_flux, + ivar=cont_ivar, + continuum_pixels=continuum_pixels, + L=1400, order=3) + cont, _ = continuum.sines_and_cosines( + cont_disp, cont_flux, cont_ivar, continuum_pixels, L=1400, order=3) + outputs["continuum"] = np.asarray(cont) + + # ---- meta ------------------------------------------------------------- + meta = dict(seed=SEED, label_names=LABEL_NAMES, order=ORDER, + n_stars=N_STARS, n_pixels=N_PIXELS, T=T, + scales=syn["scales"], fiducials=syn["fiducials"], + true_theta=syn["true_theta"], labels=syn["labels"], + flux=flux, ivar=ivar, dispersion=syn["dispersion"]) + + return dict(inputs=inputs, outputs=outputs, meta=meta) + + +def main(): + golden = capture() + with open(GOLDEN_PATH, "wb") as fp: + pickle.dump(golden, fp, protocol=2) + print("Wrote golden references to {}".format(GOLDEN_PATH)) + print("Captured outputs: {}".format(sorted(golden["outputs"].keys()))) + + +if __name__ == "__main__": + main() diff --git a/thecannon/tests/conftest.py b/thecannon/tests/conftest.py new file mode 100644 index 0000000..d8e4858 --- /dev/null +++ b/thecannon/tests/conftest.py @@ -0,0 +1,6 @@ +import os + +# Run the test suite on the CPU backend for determinism and portability. This +# must be set before JAX initializes its backend. Users with a working +# accelerator can override by exporting JAX_PLATFORMS themselves. +os.environ.setdefault("JAX_PLATFORMS", "cpu") diff --git a/thecannon/tests/golden/golden.pkl b/thecannon/tests/golden/golden.pkl new file mode 100644 index 0000000000000000000000000000000000000000..72ec6f5d228c5903f74e8d9596a317bbd333dc30 GIT binary patch literal 130716 zcmeFa2UJwg_u$zg=A5%)#*8`h+8i)q!ia(>AR-SM6u+V*15p$usU!hOl2il*d-vYW zGrRxIInB=IIJ-NW!+$^3sNjEpGe39E?%6prz&YKoyWgv-d+X+^x;NP?$#(b#4R$yj z5z#U6t~h%@lI`UiG?p(C!a{?iu7rd|#RZ34afOE414E5rLaYaP0wue2khX>jt9@`@W?NPn~0|I;l20pS!e{b@tD=~&A#{_xtdZNP$Yq+; zjCA;Do0f{(J8{3J?UF3bwxn3I`Lb$?yps-Ya}U*I=}@;MORKoGRNLpx`{Ss2NFQ#N zx`pSZP~ULhBYmRn(xmwV^bx5^IIldTW7{ zY}UtCgcN97h~~@}hAq$Uozv%%BYE64M4PnkX519b5cfQ3kRr)FEm_hj?u}Y7MV?E+ zK+P9-h(2GeIadI+*cv!g-|X}c&`K*bR37Sk&6Lkt0Xmv|Cdi{>G@D}uYxcnFQXC*% zmVDHYQu2p(&{MIvcgQ#1UP-CE(kIrscu9$tk~Q)kOa+{Z-hWK9tg9R4^~wwK8nkM4 zq9$v@3R-_@f$wg8;~g%A@{;lG^^#_7ks4reFOtq%UQ15OXG>mfWJ=#HyCcn3qGpA= z25X75i#uqQ)CXwR4TITjTPBCc-DH@+`C2|a&?cv}S(+xV66LG9Z^$?BlGDH;0W=fqjspr5WuoqV}1O_scYq8=|8GeNV(y((z#)Eh$$-n47NV9k}* z5zP32>ykE+NqY^gx#?W%PHX!GrpPTt@k`9j&Zr{eJp6#0%!zcp*vwXW&(I2$Bc)X^|%j=;KgqCQ=a?RX4 zg+b_3ExWdRhJKK*45Cy@v`y0G`7T3B`d*5?uHV&W1-(lK>hseF<*gwM>0Be9PI!gO zCo3vaGWG^)vF2Ow!IOQr<9)%WY1uJRb9#`dEisT-v!>`n3$kv}XLH596(W+#U2{%_ z@=z9dS$ALB50BGl*7YHpbqFjsYpx|N^4VKkrD>O;th<`7dHQYi9W$L$GZ#+lT(d?C ztThufb0V|xRMRbO*SC>U>VHY|xT!p9proyZ;KAu`$%g@|L-dn}9EuM})&Au(HAmc` z`cAr+57#bzWI=_c(bBF3BMpMiEz^d%vw#G4(Jz`2F3ppq4FpN{5!y^!WzZq+Et7 z-2r*ESl{nN7%(S8N9@|~q(|hdu~XN!itE#5_R34mt@cP4e~bnC@TJr zz2HZW3r&xU0`0{fCHBzv?dkDvqw}O&J^@KHyxgsI)=6#JCmBrhJ8iY@O5Ra@F+@`p zyf^w@*&ibxw9OhUrDwK56ora7X+)ywYsV$6w-Eoy&ILhCVx5Hqs*eJZt@=(It>ZM= zrS*vNdTp_^K2g%pGA~gRpAh+#+IDU?T+(I@86ZX5#-bdWEv?xqwNj4&h4ZcHlQ%3F zyxR2FK_mHFrTiQn9+sx{`beLn>>4d!#KGemBY=k4$&4GeTT>FvK*}{G!CR8G!1SE0 ztz#BQw}U3Nc{a}#07X5Zo|qcXkTa2aZC2uDX&noI`rewP6^M}5k?=^1Q++rZNUA)u1mq2u1o3RBiHtg9K8xKeD&>O!SlSKOQ&4 z_B%4^{4QwglXNuPxYmw8vxtpgKh;6fO+Xr>_#rCAT+vF*}&`~I^= z|0M4-`(_|!ldU7cMru1|U#77Nr{)eu-Ivf7Dmz);?bA+#*f)P|2n8{%V52rf+_(9* zOj=@*T90}={VhjgB5EL6T9zX2o>{|o`&_|XD#j|8C;J~jVXwvscGXm@!7Tleyh#^r`x`J^tx1UkQcq({+??xKs)Va^2DYqt<|;_dm*zE!S*^WNWuk1nyCi5h>+U8S=y4Zhz0$!3b?xxpxCNEw_g0t5fn&3dVuz z{eu_kC;7ZjK3H0I51rJA>0CM6lIM)BamMs3u?*(}#!jOvZd6|+|EBW<`MeFlsrn?1 z*k}7!O@~tRhk!N|Si>ACPK3zbTT*ow^YinhtB{pG^PYBi^?AtK_mXym64kqq`}(c*mPs|0w#9FsJ&Rm4-J+<l3~rl zBr4Vy7qtno<{rBpSEcBJci*Y~4S$h&&ATL@LXa1g)&)%4cB#?UIC*4+YqFQ*4<8`a z*3_Mvqwj}ERiuTn(5*$78*SPkPcRg+kr-9Gjv>u{0{eSDD5UiIHe^zWvGmwj@oiJD zVriyns&#X|wvKb+qNNgf0O_50uj!I2-PRPzoTRn#DJThCQc?eovN%3614V2~6Xmt<_et(bkq`XvsG`7Su4b@FW>qQrgJ^+5BYn8Q5FmI7(n zC7;$ZMt77HBq5dy$3PQ5F$~B5j_-{XSF~=fkLJ6qtr*pL^78IE07AT%Xo_@gz-G1X zC8BD&@y8n^+X~v2FmlN#dK#eLyrRw@){Ndhf5(ykfPwi!n$KO7FIJ?NKeFgs%mvKg zR$Q@;nap7RUlGS)F*gqHn7SS}Q=hKT_U&Fbdpndr9~RKBNtd-G3`(iRob-a%+Y!5M zI79kmnC9KoPa9GFZmD0mP9J+o6@uUzep9;tz(}Lvx18EZyeeOcP#-^W`+K2=!ST>a zDiit8=(v^vxOa6Jw9lB;uqb)8Mw%lDi>zdtIwe+cTZta}BeH|7|vr+Gz6SZqDMDT=o z&LzV+gx%u%!umVG5R_R7)>G%v<$=~2X~rMb)FVtKoLP>8@EL|kKWb^m-ov}(%N{63 z^0hr-c|sBoxepLpt39?+(*t@Dy!bbwD`7LeHF*=(N{e-2&~=;%BjY|>!4swH7WP9q zcy;TO;2GK!Cv`v+bp49vdM{&!-4ih+-fEpI0`c^7QknGMi`L}8?CIJfFSj6+ZvLTd zvk8qLl4eYeb|CBv5gNPkuwJ0UE5;M+iafXH=3vQ}Ruod8)P;osbyaV_e>VDaVU>1m zrm?s8EG^%2C1DqsMVU+PeJ#AB&(UYW1o3M8@bb!}!y2}1g}#-#Bj`dC{X$)M?~~l# z{{Bob>0Iwr%BKgj80&{6^3*yx(W!;w!wn_Bo_0|_coEOdw6t{nwoPjwaZIvweKkhA zW}@cudOQ2g&=e{Kr}lw<)YK$==-ChC>-bosvb1V5(0-8IHH5X?c;@Md)_Qq?P|mZ? zMTPokRxGrBJoF;Uz0%p&diafS*G#9>#x9OUKkt*1Q};LBB|K0yuU155s(VA{8!3?k5TTX7~)j`eBZK3c%i~u+C`F@#z3nLxjMs zuNUa^ON4g6i2EvLOA22-*DrSnQfRT1-W=gCH5R1)6>F(-jOP9-9)2e%Czbd7VS$(Y zzp)NTHUZW*E`V&bVA`CxwRG2on%Aar0Z6@%D~K>JSrQEn-pgYX#C@tA>!tQiFw*oV z*uNF%^RXjPDTV83WUV}%?$pkUaPAfT?7&8@|ARSpOO`FIr_SEQS__ru2Tq|I(u_-B zTrYwb*FnL@-d=w%a@aUy`F$cX#2igD`t25Zw8YEEbr-Z$x@+nn;Kvsj?f~jq%`}l| zY{_TC4xlF*t;3u2%?_!Gg57$7{modU3B~7Kr0Z{hsZEl9&KEKH8hoM!iq<0_+9CmG zQn(h5xDboPkJ5^<$F&ja%}E)LVm2twf(!V+$ej z^wn5&dfU=P=+|?@G-K9LEGGiQ?=;z^`TpCDw7z|5JvLsqwEofC>2C&06D0S-PK139 z_VD%bQa#NdUV(vN>mnSeFKZGoI<@0^(2<>H4QJih#gKdLIyLJhZZAz)He&%_|3va{ z#a^rT;}=%z=hLVz{KLoB&|L()@T6nU>Dxn-F2!5=<=F6hN3{Ah`giq3c|KWR9AGm3 zgi**q|9m9Z2O}_J?n)$NjkQ%vcFNZ=SQExIeGG@ZSaJwCmS@R_@mDlmUi-n_3yx2@ zBwa8Y%sp=8$xU{AVlR}x85qWjs2Fz%xmpW~Mrrr%AyRn9n4{w{$)1_zg>{79+@mG; zA`&I*P7Dyn;(26h+EJ!h+NNt>{N=@c#qw{*ny*S%m==U#|GkE85mFiBwBaaX2Mykh zZg>d%1yHQEM(}e7;`uA;easrojewNu4A(sjNIH8M_^W#X|BVPXvy8`XqKSN!6dO}@eNB8iG!$G|1C)@O2^kV}W=tn|768c3!KNs|KK|dGtb3s2B z^m9Q!7xZ(%Kk!^&?=>#xzCe4w%RYe1X?7%shDQX)+4jLCJHR)H?Yjr5jp;#e!7H*?ohK+^TUWi z_bta%slTbO4iFr3+s6$Ifr0i3mwhs%*AYHt#(6r>KJ(KN4(i|YVfF9%_&aU%?%(r4 z(5ipWNB^FW{yiU_cv%0Q4?^=l&Rg{F`RL#C(ZA=zgvtB&e0UBJ_3!!U-}BMG=c9km zhY67&75n#m^f|rPzvrWW&&TiX`8aEWSLXul^Dg^B$d8R3@t)I7hgRDclk7|3hr*A> z1r3AGZiPgKg}NdV!tCHA`?9b3p*P%h#e};;0`1VB;jXLw;jaF0m-dId*tu`AFZ3UA z>ObPtf5fT(h*SR&Cv-yp5vTqmPW?xm`j0sMsf<4TN1XbPIQ1WK>ObN{Sg-$x)1OSI z)qlk4@AZgNm~6zVc@lGTymOJucg55-T5V~{-k}>cPd*HA9+2AW;SZ#1l=FdCF!<{R% zWuuLC_Hm$7^R^GzdT-it=luA|+BW3C1}A9++?;$Z)Ym(d_DWkRBn`g2YpPn)G@_Jf2A&sY`G=BJHFCs+lFjn1MN7M?F!-O+xPr(@n(_} z0`1%1C;3j#Bs+g(?{XY)8VYjszdJo>_@6_?Z*rCEBz^pZJpl=KTYG7`0wL`HzWaLuj&OO_n-;QudT_Lb_Tw+o_NI z=Wf)DE51^u8UBmb=$rGLFD=ws|MQ&RuAHnjn}=9rF3gLQ&eNJZ(c+BuOUb{yCrR7< zO@4*=rK|30hd<{++s{bF+HpJm{l;ijA_q|)=?8zLz}K-G&O}It{~xPif|`SxH-Xf3 zT0Z_JXJs3u!~eP#En2-SQ6Kv=z6^Pm#6jVgn&q!uLwB!;)3nhfddbj_hc#Op2U!3(WwYz^ zduMLx3r_SeyMI7Smr0BN0R&jXfR$4wkh$TRHtY-;IoVntOmAN%MQfEwDpWask+-~> zU@bb3z*&5T4P14cflZfB*P`g+5u5&6f+O#Ma?+N3WRq>BzTd-L#||w6%XyMN)w-^2 zfHC3NG|NsibdertKXloT3@hG__lWNpq;Ev1>wD^A`|%GJePSphBhbz?lrhL;70&9R zjSeVsBUyo-=#xXt#0TdpQWuoQY&k>@KuIR2nLgI!jU*Csli9{32kLT8l#fp7IL&xJ zXwB4t&N-)DM=mlgq$^n}4~KZ^n*?cfvTLdEk1?DCeuGea9U`^oI0c=}dW=>~AIiDk zCtlL-yuU>2!!={>5GQ?CvS*p>2#-m;>IMN1`RVpW`AoL2yCb!Dsl4)bl9Yo(@h|!H z0~tK*Y=eZHKs(oEKkZ2?=n6wNX$k*ZCz!N&({v=-fU+9(7t zR%XlRq+n?z&Ej||S`0uhIlSL$9o5ct3DP}D$x!#mb$0{My(LRx(%=Q8f_O@1q<2>E zXcA$x1&5l=EnWK6S~X!m53Wo)D4)E^dD3ECw0*%+3fXk92+JhJ9_!+O`6tpp44(vE z3OIA!Vl9++26{~ME zm=VV|K>e9yWhF1T_oFCN(XOE*5Xg(9>}~i&?LKT@|JIL`t>s7 zUq5)*wPB=e;z)nj5-pc1li-rXVTI&X$tRap1G%MIjsAY9ubmsoJW^|s>9#!*9_Zq} z$@HYzvmv-vlx54<)67Yi)>tRNUx~@FM8naVzI&11T80FZ=4svpmc;pBifztT-J4rHglLno}Ix%A#c7_xH?ec2ALHMKC zpetBG21k?5NI40J&KaP|{&qv3hL9r^lcU|o@*7JUQ)@0MCq7EoIzZ;05=}lyM$|6m zwZS^S@sGoPe&B)M%GaJBJSD#I*8|eX{wLV}&~Aor@jw9nI&=@5G;^!$zV z0XbW_0rsu4kV7fk0CO2BUP+zd%lAy8$+qR5MKU)?sBq$SaW8|iTC4$5c8^>Gr5+L@ zPilgG4nn=PTT@SHDdP#FH)L!!tm|G4(Y09@C3F4?&GZ2PeRpnzZ&Rnch+99MRD=6%NV0ffFlPUf5+Oq0V%Fwz1qhamxj(9rJf&}=aekJ8RXWQWN1 zri17`+H%>;oLfP@%kkq3O-g{_$!cfUP zqfhRGp}u*9L#2Afg2THQ&g%1X4r^nyfv<4AWbE?>ZOzuu)4o3QJW_{@738!pM~b}Q zo@4rYlH|q&b2&{rk}gRpq{fuD$ZPM!W%JD#zsjTe`uv!70@LBkNPxycE4Y0Ty0GI6 z61l_R_3I678FCrqAo)x5%O8>h#31(MRM20wX@VI}ofeGKdhV*WB#K^3Fos#k2%T<_$c1L!0PM^XG;jP&=Xea%;Ml#KI3* z>pt3zL!mt0gj7WyFu!CeZ$$Otz92{rMUOiIiV!=aH0S1&IB_%7 zk;5lOYdNVAtqt-u9Hz}(3S{;gt=eS_%r9h(#;HR2r$Z+1teNL#FOOwX5ZZ;3eN%H| zgUkr3rDpS;!${XMwBBjS3%f!Z6(*=b-j0_?^Dc5d8}v|wV4t+`VZ)^(h`p!G#d~x? zfn-bZJUF?05oxu(QB0sj#ze zhBK2HV`z85HRB?dqW=&OFG)o-P3z{FnKsf8%L2xG#v^C7XmE^Ugn{VsWu){guvTI^ zsnCpjG+%*sYq7i}!wz-q1$U#TjNsW$@Y-ap+~}vJYm5rE0w!UzRYNRrBAL2h9z^;U z@NkP2f>f?U=wuBueYB}t-Vc>#2Uxx&U)OGeg?K{S!{PM?9{ONN8QN-+6ip51kx?tO zdm5%g9&JfJzHO{z>}KF|hD*hou)#1CWP`v&FP9mya$Sl}uEUrz*{%c6N&W}O`N5VFpF-=3he$ zRg4~MX0%f%jL{v3!QN!u2g9{nw};V03_o;ft{3V1toTt{IayNTVW9Q5p`0oy+XS;d zn4%RIqyGReeN(%Kf$CBoEniD3l73lV%=8%sbY2(>WIC+6%-8y2v3+bAZW)$>ejd7D z9R@+WCEtdfC3ew-0>W%~wr?76S4&^=nY ze2p1e8DkAdl6FXO*1>sNIu2RHvt5JM-QrOr#7F7pm0IV0ni~(;Cw!%K96epr4Cm5Y z(-_sxz^OwJH!m41{`uWyTD=+MM{4V-Ig2;&I0H((j5Hy^T$>f4Df_bgR*$}$A1WNWZSM9VN z5rHJgTjO~kT#JGzL1WAu4!J&tWE!MnJD?<>Whz&0^6>V4EhMSvJtw0K(@cBmET{#2 zP+(ll!a%#oWf%7ed6gKS@>QVy`lnC%=DR;Vq9?@DpRyi2AUytlNLlZhk^Zst&xav| zemdyKK|dWh4GQ{sp&tkRbkJ|1^jm2?lF%+S#(r6#UGB0gLV}DXPgT#bAB$;~KNghU z8sonz(60Wum{#)-z+&1v1AJ|uUFWjvS!p&Wov3R{UDEjYn|4Ez-RSFCeZCqI7ZVm6 zXEukt|ACMugOKJxyTu@6+e5px2OWRS6e=};hcw5zPKc`}B@&HshHoPN^;O$0aNYB5 zR2Y|<7ZZppR(+Fe?@^26gA=&;eybMU=R?#Ua}ORI&sAszVYeotn@{Y37UN;h;I&9HiR$vWx#RI^Z{lz{&T&0p;``<}J%a_HZm#co zhEByxr!);U102mIK_CwVp;P!Uxu>@|Tps-1q%YU^0Za4zxRSO>=9bM|YK^p-4?DQ- zfp*(LEllDM!cPcB&`SNAedGnzM3!z*Z)NsqZD6RpB;Zn}V`5iZt*kt4U{v46V zhvw7QRa_Cc%jE;kyFps26!2kw&omOG&2^qJ^F@4@`?J2IvWf6EAKj+2Z~}Ea{lVbO z{H)1*G-x&sf3DLr41(^@%mbz~U@9>Wd`JIdtu&t?4G-9aYMo{f9{6Z9*Lm4MLd=K9 zUjVXc!&{#^^~q_*HOgGEV$jq1BtH4|bTXUEtKS=1#5L5zG>%$j?$7ps`Di289ttKW zwbO1h9P%O1{^+tlnO#%g*O%K3^?weuI}G&?3}Ah^vxoj~4rP7$t+q!K9W9I@azV*} zoSmcTr_qX>=^rtg~uBJ#T zq*$xZ&w}PZN1I>L)=beBDyp_xcNXs^fJh98ph&HDL=kSRBz|q;QofQoa_0$i`AT0# z67#4a+(>!Bovd}26P_`F=~YB%rIIM1S-_2kO<0oFjxav#k%m)?8FLl`B)0nLGA-_9 z;7Lq%%_u273N)wnA#s>)B}G0V$SUm$)u#0axM`i{M#dBVE;Eo;u%$@f&kdkaY2J4X z9<3i@04mm+>pOrpn;#a)x5?Tyh$tg#lLVArv8+pArj=~n^_y6C_&SGc@cHbHyqy

*~>L~k&hYA`@eJ3e^!fgKgj3DPGskTlF4cRvjE&HTYbAB5 z`8NIjPRWlSLB~_?K<{40QCSjY8NV$(ST zPk~xOoJ~?RaK5xmG&D(Fu{FW_X;P}!v*g`{>@*SW=>cO{zet9anek8Z$alu@Lwp^7 zf_s`JXE~2f)Ru*Zg4dWw^B%~zKc`q&sriL)+zix+jnQiD9-v!mU(Qsnch-??%$nMSZ<| z`s673WwdWX#t=Ry>c~EcNq=b5oT~=#mlM_#yYD>8^(bQf-C?U11IJ8C{CpdBK8SB6 zD^i~xaOx<}y#7g$&m6 zFw@eDgW9}0`XrYqn(iT5b8$JH$tM;I9~18{1PNv_@6e5Fe!+k;WJb(rd4G>9UzQ!H zGCO0HnW+3@bo_8I%`E2T%CewknoqY)GFF0sA zZe0bkGbLj)A8wzNkMro>EO+f#L-gI~lvc~E4Vt2(bvO8M9Wz7PyCtx z7VFk%$vWociSVuuCol^awe$7qbI7%1DmK$^95nIVBk^*9YpK4!D{t6ez_ry<;J-WuIADVO9%=9c8i7EH(1`t@xElw?^yi-gP^Ht=fERHX&8Hq}b@5`_g!W z>etABfNq?gI81YI7|@CDx|%_(7$GIlC@Pkm0mKiBJF} z|CGWtoG4#scWP>8wAo@SWS4wd{(gjVAh}!Hm?s22>?^9)uIvK zt=6uKX1Hucnhw`cbH3H`)2?u6v1vLm!5y842eV9W6t8po2FKi6Ji3wlOYfP*aDr@T%{tj!X*q+9(hbfUt(fpFyS3YBA;~z%zKr&@ zzUDd=u7V0SUgAR>%WYL!80F-8xuh)`u@0m+S@suWp?nRryIgj6pVf+Q#zJui+W*K; z3*{f}u~3E%@mMz|5+rYXES>(Very<`=%<5z9Q4yczfAmxMlaZ6l&1#T|Cr1E$3ud~ zLi|7DU%@dEcfz9L?0@2i&;L&vjrvao+W)lCsIwp1|4jH8PqLthP4-{Qc=Fi(>w)$^ z>l-jIe8Q0T~_%^%SN&!0K-Vg5$; zyg7apkTN4Qe?MxcPwe^VnEd^H>i75Q-`{6`f1myR-RJxJyzlSd4|+KkegEwH{e9{8 z_vPQ;SAKu@`~JS>`}?}@?;F0qZ~Fee<@>w+7Y4!qe4za=xa@y1B#1*PjzjjpbokC7 zPkQrPu~)-l?SDC7V0?U#u}W^myJF&9arVEGWdEzaK?bJlVc*X+1SgPx=~irT$Q4&; zxc#sFn3I3q%*nqIX#bmLPKF2gKC=I<9xLaU>he8q&vE}!LHL)#zJ2QLF78tW z_t)aIeQL(`mUG3COO@1#L3q5MC|sb8b%Bk?>hpdw1sHSeB!+9Gi<=I%{@)Xku7Qihw7W1{sCG^`P^(ut`-MS zn{!>fq(n=}8cL9Abt18gh80lUOKXvI-tt<)7C9?umDC4l)(yfNZcUR{iD1-Csqmkp zu~r4mok}n@pWkyRns{2^sheG}npz}UvDJli*uAO%yP8Ot?u^AwZF<-HVp zUB63ogKFPVk&>}DP>U(HxLdx4ZIZ0*%He}bDV!xAIrBpC&NA1}ln)?}WBzPBB&;K* zmMYs#_1Ro;Z)Nn?hiKLz`X!dmd5gKArgP02Ex_lF;+W4`A_`QK4CRz&A;l??Ay`Q+%%dJEV2y?^{g->ImUb3Os z4Zcqt*XLA%7Wc)-3#%xw+hTbq?pMykWob6Pv$oHh_s3B)gM?DAjtQXYLiO|k>nB3hB)})`MqzgypUhvON8c6qfCDf4dG z9jc0M+cG&kj)f!+@es5Akv^N4(x+=gMZeq@f*f5FSdWp`5lo*HjT0l#x`y5@K? z^8#+mNl8m!>Pjs80P%s6lfh6#7$v87Ow=3?V3oV(oC*cdEEBu=>`mmXX%`rAui*gK zL`LwGvK_lw#`&n#A*?BZ8DvyxGy#i(kw#QV_7MmmZIx-A0YX7P&_T2o7+NAVZ6&p8 z*|_~hYL`pZ1P}4QJHQzbeZLa~yG^vRa+?%g23EIJtmeBH!|4D-pWj7L@#DM)@)~-B zDBlK|1CpuMLQPMcHuoZ<-Z*nI@s~D-lv%xX8zwRt$)Y}iIwD+dX~X1c z=ipr`cAK+dK~ivBlFiojqIGk9G~Z>xNxjwT`=(j?rM7#YqU%iS&r0Zh3Y5z)HN*Q9 z^zM^+87bFW1G_hE3z4r6bAA+y{-t_zpNh)bx6JV`g@=Du15W5up;xN=l+Nv&;FQIl z-cqxD6C?aujkY#PUaLuolc&#BtBN1pkn+84JXHmaij;2R3T^Hhm_~ ztRunn(&m_bS=_^R`&>Z}6;u5f>D{(sF9c$WRt(cur-%g>jN_jDtERKsm_I}~LG&hR zm8Lvc0z+X&LfeVux5mjM3$kvTdKJ?ljdJYa5OT)m$!l2raZCADEufz- zaq`aEN<2MJUfw;&>G?=6e`MLWmEe-?#33pF;3`sU zz5Gi_m^Zqm*PeI7LE^x(tj;p-?y%O-#YJzzTUd)ztltTTcaFW>@6DJx0p9VqqoBL&%)Onr|K<8 z{cD|Q`L%L$-+Icw6gjTGy~%q4Jd|TyUOvOMYEhgOvp{lA58flC7Hf0qv%o{xet zljYq$w*9{+F~C%8mn{9VYN@FToyExv`MiD8O>C8WXx>KFpDSk*?!Jpc`#k9?Pv5mC z(;BCbI3mbub={`XJ$W6c?nJx2A0kzemN&G}iIz@h+PYa;(Y65rS~LLmJ?wg*c~MMo zo^_9-*60dzY!dV0mmx|0!gc!Ciy(WuwUU|ZWE(=t4)m;cPTsywG9WfU>ezdDmwee{ z&Xzw3CY(;P3{6IFzS9@P$tyTCj-kk@NIy)gGU~F+>uDF|gBPZ1ra8ftj@eVslEZt( zvV-G`fRv13+i6{epi@w*&-Ke4^wYQi6!6Ct*qVLH-TzX7zPBWN-$Ly_Yp3_AY2K&M zc;D*ZztrOXwMOHQT|`M0XGdx*I*q>e`IyS7LeSnfJ8AtIvbz zku(S$Ox_7z-LsgEVtsKDkygV4g>+lQa#kC&_{5qbnH;&1PbSUf1nrWP#V~w=WlMpy z?2=DwQQYpJZ3!cnj6%(a#oRc&W9oVi!jww7|G-EC(divpwdGnA>P0#yDyM%curFER z%GdUU8n$c&=cDS5U{q-7`fZ!mpgMA_b5Wr_ zniY$|vOFBilQWG89)Gk@i>37D2ogD1OO`}4@9*KhHV*neKX3{{Of%BZ*eTD3^<2|%uTg!>|(6gTF%FUiQaPceM`vyQrNw3mGwTg&-)Z4#~mod)~cVF zAa$&x*K7U(KG zr3ki^vH^#$Bu%25ONL0mZ#>@~z~!qwwo%gqtcifc;c8a|bfoVBr7kRt>8?V!7cgmQ z-uooiY;<`wM!RN0j(OJc(2K~VN+%%s4?}7_kqVc5<^VhXU7A#m(Of#ic<)#*wReK0 z)-tk@4Pe)qeGcw!(lmQc5y)Z2b)r*!m9w?Z4R**c$N2{ttTsTYvP#uzDgmkY1n2OJB#lkN4Ks zr0;9~{c0yjuLk)mlI9n>wErUm%YPhb|0gc{KMe_*@uy%Rs27nyE~$ht`#<{u℘< z`7Z+P|FQ=jl9BvZJ@8C7F~Ps<-_JPjZ@KLMHY8{Qw_J;G1$*j3 z28SdDM@2+miMnq8cS-htZ<5k~-<|fi4YL1-K>L66Ap4>HKlPyc|Gzqs|FePOe+jhz zSC{?2g#--*iqKmL_W%B4y#I$8@BbNS|78zx41W~sfGT5 ze5J*+*ImEztD=12&vVFn#FIQ*-wU3+Q}M{m+KBUm z*vGa4b1G%fal$d*;%}SN$0;tY@R!t?p^_a$5-zN1aUUT7(V%&#$8X59w_egY0h`)L z*StNGQQ9B$t3SR=lYB-#6-+~WfAb|q_9z^`xI@eR*Y^SPmo9OPv|`81kzTm_t0!Yz ze{i5PzU`84GyX6}3ou>?XNyvrro@^&P|x(~zU7j?UCa2gBviY0CK;3do4j8xrT0v~ zTI28QnCKUUV8kdN&;_3pZRO|LVe@6a4tgk+hw9I!?kmtcKcuMGj%CH@wl`POpAJZ zL3_X2gH6N6if!nTuStIL&~F(jh9i$@`r!q`tf;xtefHM6sbqTU7&FlCz+j8JNfLmR z`?Oa({W14?sl(WQeJ1OCAKYE-15TfQP_d6sqorGpIoa=2bXz*T9)M{t`4T+x*ph|f zez0e!HmsToLNJNb)`O_aB(L%^-q8;#{L+Ucp7)udfL^Rc{Q&25Zy%T{m6%>R5kuFT z4VZ1W52%|hfBw=rfSBJ6Eu4laQ*?chkesQUjBvjYJJx$I?P*ajDd;zRg6|U)`vZ4_ zeu4vC^mg>(zI?hpQ$KDb!x7x`RKBcZz8n2_k>LLwD-# zC5KTzChNve^g)e3!NOkb)17uLo4ryRqSjgZZLD!&lP(Q|XW5|hlY!UvWvAu61b(%b z>^5Na^il;oK7Y)WzUakaS|7-?vrlJ*rDp>&+`$ z?9GDQpOE0eM*qi0U5Sky#C=JzgL&;0=&?iih>VOK%6nk!F!Q5X?|K+J-25=!tS%n^ zp*?z6c7A^I_k0mM!n6-0BRw6D;?)&9n%9uNQLz&YOTy3-J={8pS6A$0UPI!;4<^M<;i{+d zc*2#a_^{ZieEx2v#ZEKV@VPPA17l~HI|BwR>0xhkZ0da6U?Od`TC~L(JJZ}tFS9)N z&gRt>JBQbvgnidzuUw7wF}L*CtFd#xxtJ`B2nqC?G zBF}IZ^XiKAufCCeF}93v3|Px~4~$)5 ze*cLvv#}#qT8>~Ark`Z{ee6neGo7yT-0a7zD|R)nA(4GCwZ_~g7Orq-Ju zJeWG~0lx3cPyeXT#SibK&sSE@Gk@1_#2vfA^u$0ndWNuxS6A$2UZJ4iq}VN7{fdRH zd}D^ZjrYLV?dJDiuz&-tkB6H`&JJg+zqy&lEziv+o2)B#2d}?kVW+u^CU<%6+Rf{C zEbK8qc(9Q80|6d-A(WI$*unEZ-+7s+^3M;^ENT4spvm4l%ySEz%RL)(7C8NgT*fjn}Z{sxkK5nrw2w6|gk(Rthc0 z3~#NwM;IlQB|xp)B4q~xiO;wlEGCRQJp=pA?Z8MZ-^S9MJhdFdr3<^vX>O71wE_BN z%>cUA+La^LYWfQE_0R<{Tj}=E#Dm8UV~s!imS*ATm`wu}8~rie8whH-Ib@W=VtP%S ze6@TCbFzh+rDOixj?253+fWGG1sw8PGmSST=~0%$>BGkl$0d{XqfQ|G7|mXYW;Sbo zyP;o~PA7WE`Lzdx0q@bCTZ(SU*MqBx=g^xbLj@m!qX-wU#yYn34138POeDZ1Yuhlb z7$X&IA(xs1`f$-wF?-wd=czmosClenlo6D#(OoN2$v~noB{F#8L@dx$O zCE{NWhztNn47zVQCQAKHeRaUBWiXJJ@+#?$NyZj)Aa3>wT(xGFXY|$HxkL1^nPBkJ zwDRuOv-;GUs)ZW>zkH;;vXGj~r7>=kiE>KvAj0Kov!;83xyd}&%)lNJ@a^<;k#RDR zjCQQ`I7R|}2EOSRQ02y8-_JyS^D>j%HRAG}C@7yOV=mhc@crCp4V<{zj|eB%54_2G zu-_`&N{%GpfHYfiHIMmk2GhKlP5ynTUAh4o@>9~aMR)G_&ZLUIyD5zS}%>hXsO zRue)XyHUA(N?IY=BSEqzU)m#C8#sp10agki!cIwJ&xr+ctI!U)vp@#R?T!sZkE;ib zFbm?ZrDeOePiL)>{!>pXhaW=l&vEbppm0toER@BV+?RLVBJNp;Tbsx7#BC_+7AMG9 z+4Q+}V9}l(W?9WKdbeaw+eIK2l1|a?rR=CMKsxEOY&Alw#!R#W#F02|BTnvX>;B5P zTLWi-^|6w_#5SHj3Fu5-w@PW6yUN4Ik7i8q`y#bB!HZ03SZBIB2C=7^qfV~~ z>N?hKm#>rdn8|d=m-(79AQ-qEQwhPoUqSdY&2lW2?8v+RQg7`B43k@o9C8L|#Ksm9 z8bXv$p*LSRrOsd7Fsvps)MQCM$A}uO%UUWe=co0o6L5g1SYDmFfRl`O?T~kg`gY+N zDX<*(F6<{bkZkxGZg@DKM%(5}vE>Mt(!t|y+TgWlv?K?>Rir8-rC{5vt6IMnb`iD- zgzKwKL>m>Uh)bBQI;#L~?sM&WDi)YB@1s zzcrG2VI;)GilRd*uT4u_Xr^HNf^8$D(C`+rrzr%YVKv-{3^G|t_x7WZ6Aut@FWY&U zpmY}vxC!!nxf695ab#Vd!h%Q{qLD>l%W-VcwNb+j0Xp_i-KwbvSrl_S0_T&9Qu4!j z&Ll5vCzT&Td?R5#8VX;mPfe0A|B*cRVJ7Iy68BhG^{r&>ACC-`nlbWh_YwH3%yPtC zx=3HoO}mopEw3k%(3cRmWArSdO>OXbx#f`Bqie|$X^4&d&eEh`*@=XFg~2{D%qD{0 z>}3-79mG%Fy)EPE{M3vUlcES40h`Yv)IyaBywx(oj`4)p3#M;EqO++;p6t|*r{F;& zGFWaKGyVQX!trj9_AKh$VwR^6ZB9nP;nnzCmX=HIG$~mbYQ9FGw{N_1+0$z68vJ>V z=72hm>a<96K%#Q?mlb{Y`nAyY ztOYTKPv@m@t0Kne;Uh~nrMKuF3_3Lbw$#nv2(Fmz@|a_R+U60aHRmCJX)sVvhORQa zX6Z{q-sgBrHY%mc?2FnNGxazz8+d`OE37lTnWIg99)d=&y#cU#0~~#3NZ0gDtWua> zI?yhUq-qj-S(xdULE}d*6PCyw(lu?oKHp<@810@ib7m}4+004A7k*Gx0gRMGu!=oC ziT8yg*v!k!D~l$bMSQxaB6~^(YUXedX!aU)FNR=9nvCE8Gf$EHSw!(H$G{mbwA2a8LCEjP4OP4Cb|#bfJ%a=_@lDVa2#6qc@K}7PC z<+wFdU+&`Iy|_oNC)&uCCQdM$O()sXHuf+`E<{g%FuG65mB=I%E@^YvoG=*4_q6nLEM5BX7q9Sku77F)4mS|@1;Gv+L+9C7g=~&=qQ7(D& zmW_9T4Rn{UNhCG#5|$cyGzU6(M3-iP*XUztfFqjlJDSa*Z^x{NGiN*vZc(|UtwgMN zY`OK*G=Ca&!yIVtlJhgrBu20o$%~u%Sso5F{def6IIRw2ve`nCd~GB1{R!#89)LV; z+kZ7=u$f{Xrs+AYH^ zS&q1cSFs==x3LT~mo%%!D2&k$6SUMD(CC7>tDu*!qt7a=S#vl9BkT)t|4Q^Gi2|>( z^R1l=x6gHiZ^Kfv(FX}Wa3yH@lwh&@^_e$k6)}6Dh0?V_OAx2<{$#d2JQoh-0yN`h z<-7YX9`PG`huB*ed&lfEdq_Lx$eXG6q}yC$l1t&4#8q%EPuxMiJXwHP;H+RddcAJT zHtcX?D{dVs8T#}jF@2D$&#)S+Xg<%1As(zR7MwBS4rBI()}SiQlCV2=SDZez&FSmW zSI%5ij$+qYsC&OnQhgAUY0Yh|kcZ|Bp;2GSzVX{Y>N8`9BQ>yjvq2BQT8>59GE2&R zAbC*!dc>4V==SG~2qxi)SDtlwV10qQc_t;wYQMM>B= z1QU@hWM%dZW=t84wxB{EsUehP%|5Y88?K`E9WI}9BPIvZi@v>#3@FAdMV&h~Y`cW* zf~>>Z<>|+8BfN=Eu%6kkjx-dCw9X8W=L4{}Ve|sXD^**@>;cyfA0Z<{vZT)@>HjO? zOmEp~!-B~-rDXZA@sK$Wk2)U^25M;tyMyzZmAmC5#*rk)48*d_f^Z=#B-W!N696>> z?gJ~`EM{O5J;J6Ks?`|Cnm^c3d*h59>wubq0^%O7U$5WP*Vu?248J(1tz$9Gm=Zk9 z^&yiX;<39-0@{KR7_Ls~NRrxE$OEf2IeLwLTLx7au8W1`UV`bburTvMTw6XbHDgDY z6_3%T!Tb*3Yzdw`_7KJ)0H!_AR76G@GYX!Mo#y$MdlDHeaFj^?4kPQ0xqWP*gKE>r2$gmV?@zR)&F({XB=&vfe-*zb| zY7o0i*m4%MNGj~jtPw#YWWbjX^Fs^@9e!-+l2q3Xx-taVjvf695IJ_6%vo=JET!{J zt4@8r*T)%P26P26^ucBJ{V@n=YLS$X$parGP7C%N#tv(?F7JUJpg$a^BGFimVLEoc4kTXMGQO99GV>chrC`H%_f`#C^>bJ$2tv%A5&rTH}DVVo-rK}&Ut?u?#!65W8AP(BGLmIY07LvS=Ih3xYh&C7^ zbBgA(2Fx_6hsL!n^whF;1-&Svs)Jq=ugZWtm%JR+`!CkW9 z1kj~Ur{f^lzJZx9gVoR}?Q2)V>d!|QphFj%eSNhfxWmnCYr!V#kcVHUu9Yu~rRAn^ zGd_5sqn@$t!*TM|EO~x|UB!~TQ(M+!31%73#j!vOc1RiKi{*@gGSi0{)>;7~YaBk& zrXR=9m9w;ru25v{QZVZ9_F(h1Gu{OkyxRG|;Cs7$TFN0`%n4BV=F&MWTqSuhUyBxw zyg538PFLYl`3w`Zs_U@S&W$kwPg~B-SR3cD8TJpQOLX95!?Rmw?%Q-N%p6`|O*--p z{UGp}&aw&mPDB7N;8#=qE=bmJ;{i+M z=3(=>!w^T;?Zfjod)R0krdqJ#6Kt?{;II#u?sp zYR3+Fl62gtkK|C7q>!f=xe3J2Q5vt|>}(`~+b5oqRiTHBop_F|#V5BK0hbpU1@>4W zhmEkq8>edJoQdr5H>G?&-xs&ZxPo;gU0ZNoiWJAT+|suPHY%IR=wUQE0;}yK&Z85? zDJ6Fb!uca0!w#(+zL(-VaL%Iv7qoqjhrYcDU!=gC*zZ-0i;Thg-UN$4aojS~73`9Tqg zj{{Db#KpOTrD}sofD?-nERi;E8ZEZu$=)$jVW3a?#5#6R-Wds5X1x1ku=tVG1{l{w z(Qc#Un$V?hFee>%@sbMsQG_{+4d;F$SS~^*x*^Gw*^o~PN{o>rK|kFZrZ0}+F)a&a zN7z_;A8XQtJk-uvc<#QMwthVieuUrLK6BY=QA+m(#)tG>C0zmXDUwEu{y4fL3IT33 z02BKN30uvsZ(MaPn}ggdF`hO2(R@@|#(G5bor^I>MVfFGXIZ?FDjwT6Iq=-Hh&>*E z@zETuxPruO1qIMJMj4v((@?`pK3aw=Mwk|mpIcha$CqOfGang(n~&`_WYsax@S^11 zT&`6!KrSpb@v=4~jx^4J4^KY9^!i{dS)2n&nR&(-B#r@>!BmB0CR!I}117aSFtoAu zJA9b+J6zNKF<;HGxv9y(OFb#(x&~u53xfxO!qkxWr?D(dOfeb0dKZMSXh3&4{n+<#^46rCt8`Z%f9Ho>rc_w{TplV*S+8K5zcem&t9xB=Nx0y zsPL~EHEJBVEX}bd@hwL?{q?sutV8e!zoX*7VNfXLQ<9`0@|m;YAn4X)f?$}ET}1i44t{6DX6rkzS%!HH2}`Tz0}6mV0m#NMOI>G$I)6A#Hp% z+nCMcP_XRbdM02-h@bYaOkl`2{SYA*^o_NkmU)O<;C*7DT1nzD)xMZyC&?oIm61WBcAQ^pRhHM_`2a2N{Ho_%m8cSuf{jvn5X3>|n+F{h*c^S}&_ z8P_gk!snw54dCyuANIq0XhZtb%R?yp^Os`fL?$V5-)`c>`(T55&XvMHut~1w9wT<) z#DRmJx3u)uDcmE+KD!RCj`M?4*Z@R0MWz#oa$C_)9@_9p{)*rnLZJ?9wK)mi%E)NT zYqep&8Hp*e(iaOiXwE)uytxj*NO)`4#yE&gc;kW3OmymZZGgoV;fPcwdoxJ1b%zw) zm66S$kz9xjMAWj>n2^7QkF)Z9Id0sb6Jg7{IW7oQKDQvrobr*X-`q`TgK*#&0I`pT zDxe4euk=gZ|6KurrC>!w|fRJG#q^b<~JpVoKAYCew-~0m&}~u)a51}I|6qhtP#P% zhdfp|HENQ4PDD%Q7{n`6n-_V-ny+o1K&Qb7lqi-u`sG|^&LDpx@OAMnx98L5<0EMp z&grvwQ&(r6(>k+xZHUvU`jq3q-T{drx1HAhH|w;- z@_$0`VIK_}*AR6!VK6ap3pei~Dy4);D`5T{GXl>f65cYZ6MkN|^E68A7cJl%z#z3+ zm;INH?Yph5yZ4*Rd^Tbu-)pNYF8yt#VjqskFV1_=`jRof<0@Fp>qsiJGLQITTt)tDBia@VW;+{VXW(D z!6F$tv=WYYpi2AEH#O&9AnH=|Y92Dw@2l0Ob+|o$p5|>s2Qvo~0Z~*LG-h!|d|*sU z0Jwk_$3PSg*C!^Dl;f-8=Qf=+24zONmaM%F{XSV6S*aiO7Wf*&T-~EPe2(rn<%dRs#(Nj(KOV#z8S`As{I0$P}e4AyD z=G>G$|(*eHs>VfJYAA?Auh`g!j$G^?pB_{J|zt5)Ky@ICVRUh*HZ{P_*xpD7hH zuKR)FjfN?frn5*KC)5~B`f)#+xm+qQ;D~D3Cds1`+dil4b)zn-3sFa1D3iKzN>^dA|1d4)x^lXD1w#flvSi-ow%iNYuPX@EJ=Zci%`@Z|r&ktEI)sulHcGd|9m` z9x=G^nRR<%#4c%b#?P8Ujs)Vm_Os7r36F2$zt#j}J*sbBjiJXs84LOZQ26=oJj_U~ z6}b;}Vs@jy_+vbSXdf*UCDXuadhmAJrko*`c;_lTX4k~3W zl3iZrz?~}T!aV5@DG!pbnjdaVEbEGao-&ec)IVB%`mFSrQU=7y*%+fAcHf0}Al|fe zFOupPcvK)=#|Dg2iT1+HGD&DiKh3#8{|Pe9WVqihLFGj+E^VQ-9T$N0{j~bk4HLBj z3qZB8s6>#DBZY;!6P{7nnk8%9r0X%;wFa#nSH+xsshtL{t&%ox7^Ix_#{#nsH!ee` zec-=J3vL)Ap5K)1w5v@!^n*;@fN`*y@j!24^+MJ&QO+ATv4)c;Scq z_i;nN1!&&GeY+&@p~^qYsfj4*3{+TpDA&gFEi@PrFY zu)M`I7AQ3-@_FtZAo?mHR;djO5eVil1l%(&q_+1k?h`3exXAM^A&yJ>-giGFVUF3= zuD@5!5Ch(5G z8wqdJk<+rp*rs=ej3hBXIovd2LdZT*YYdZry0uo{-wL}bj$pSC zm*j&GbEY)h3Y-C+>|$NZ^TO)rX9D1j_6N3u>lhmI;C}Q{gwGnteN$MkZ+AiRjN150 z9+DXWgyj}HJkWXe99=_2>5nBShB0kSfSV=vC?~#(*>&D-#ViR83<)zB9kPCjNA7wa zIF;YvF8wu2e$0Cy4s*^l0()kw(Kga}7-co$x{&rm9;C7BfP>Qby5Z?H?Ov}Fs8lUj z4>5ayIK74wiq`{U@wHM+Jmb(#{S+;O$?&`6dCS*pTLCmJ*6XAv@$|_AlN=Yda{iSo z^sdc>`#A(VRDa^5;5zI_*Z1GkLbI&DMM~oq;w$Bt@k!^!LvQkHn43jJ^XNbPUao5e0os&`5V^p^+TCn zg0Ad99tcv+)ar*)HxC^>XkFFZX!&VCBjK|EqDdVYyd!(aTE;sxWnj5Z5Mqm2w_-Zi^SB}9$a-`9Frr+P zk+RB+`>3|+sP7t-HM}mR>;(6Q249dWhy0#Tq-O-R#3AI??s=~0@@Ce@4BJsWG;EJq zNj5oNlaJTy_j68_^%1@(G^$9y;O(CejmO{I$?Q4gC-c{$U!T%96KyEhl7m`pcRDXJ zohCg=%r&(F%(mIzP$1t8hZ*dO-&oZ zuC0N^(MAV+w0M!3H~BO}2Jaq~&o_BUyS~4VLMQ^Ob3j@<967%90a*wrXuJxzBj-Kd@)qmJ1pBkh-GRkioHdh4#Lbck>CuCgW8|Y4wNaF zydGS?_(H-K`R0lybwtX$OY#;!(5UHKK#Q&ukZ208l&3(#p|I>|##sjItQTuwOIkw#!F|KUZ_f*yAWX^G^ILm>}*V9cVd= z;%}e^Naw>t7iJpKxu$I^)$u0IXbd={m!h-qMZM8t19#=~ZWM(n+vGT~X(@5mR(P(J zPL=G*NO^>Sf3v*q?Y=D&#$802O;BToO(-mbY3XKWyq75$HoDzM%VvhT%{4}}@Oc{5 zW1~#lBvj}p`4M*Zr3IG^XSgrIU?LIJK!2l^u2NtooXW!b0WH-N5zifClDm$4s+%%t z-9l-XF5;L)Tqn1*OHnXyjuq**DUrGV0I6*@y2v4O%2rhFE`#;W$no@X5Fq z52YMTcg1Sh_#2Xo`zgJ9V?kSktBe+r&XxFYn6DwBc+6lZGV$t)QT4l1rD3jiI#Gi| zs_)ph;pkaf60H+~lJ}D3^_NN4l{7~rnwedFN*XeCEYipqy#G}RWHq}d@nJ-A6P!DT zowVS9^t4A;Zg6D1Hqy0q)qNLnlz&<=IliRDYwG7XRUKrm@#}g-plq zp9A(mN!RJyr@ja?WJA7^0eFSU#>nFl11yRow1u{pMnF4I_(^TD9$iR17XgQ3Gd@e3 z-)S-=fo!J&{dV3#D}ZH)w7=HoH%Qe-%1+nO`Jc(;u~MOThFcb^Kx{Y80m$`+Zic}~ zE8^(FSORQ_Gs^ zD697NX7~>km4axMpOl)_Mqo)XLMn5Um>-lWMfA7OT6rpA?nTPJ; z2nA_5V$u`0OZyzd;-mMX(?c05xF6tMqhHPuUo9mdWNsYRuI0=cq~Z`{sX6T(en%XK zwIpuco`+XY-iu>K$b1ZEvD5~xW98f~wNFvLr3b@c3llCww3f>F=~#Zu!P+?Fds@7y znQc~ErS1BGXVQEYGH=%(K?Ksvq<$x4nEmv=)L%vBHXOsH_$k$R8V0<;(i`Avx`R1Q||*2{QhkS4&~myYsDfIQM(DvjwA=nT;>ewVj*wV!Q3afPK(BK{cI)_R%rvN}H1yU+nBxP(yq^4O zs}IWD(76rW->gIakwk2lg-Ok9Et_vfA)m<$Sh!8eCg4Mh{?y7xS${3u_umOya(uNB z_jH}u&E2hs-LZaHg+SX=6d&~I$G}pt`Z&`-)07V1bK<30Uw6-%ub^X;9QiFZYyIv& zF!bBI9!C8BoD;K+oq6k)V0IgV?h0Uxud!>i#RO<}KD^>_$MfU?r2y5XInicvb?WjN z?bxQ%uS~shYB11C6HP@`m4-v$VwktVaz_#=r=Ub83m8f%ceY7Oa5S;xIjhdD)V3F% z^2;XeiRts#!{D>;&-Is)4Q3|L2e5ia=oxUunj^bn_n^j!7dA_G4Ab8n17wT87at~Q zThrmN#YXpYRR+s(=PDb zyLCG0!lvT6^p2_uQU*^$%34dSC^~SKy_!&FnPNgQmsAztSPnJijDoP}Io5)3c~98~ zH#2HC!I`;4JuZVenxx;K2aU}$yoGMe^g#}au38)DFm8x`yFe81snpC3jYQsv!&AAz zLrdLBIpOZgFmxtLkp0udHGf?ocK6n-&TW#$yYeORk(0>%+I0gR(4%Z3Hp@m`TCn)i zE%Vcg2ozHP{*8c-E%N&~9NUK02h{A+#xy@|+a#5i7gAEnL9_Zj6j?c>?Hd6)E%FXY z8=SG5gKLepyQ4*+8C+qq8-jJaQ7rsVi&B=UwKp`^XkuwTxODxA z6*y_qm8Ats=aQAeTQuRf+xqDs=74DrldnPb+b@UKcyv(spd)j((Yns zu=r4{D|t%=bJCk)0cZ|qyXbEt2JZIli3!@CVf0!%BrcMo8B9Bj)W`Hya(E6}3ktMr z^AXw{3Abi>pW6}+B%!u7i;D zo6$p3e-z=$)32M8#c_QJpkRz%G%yFmB8rZqUp_c(*vKwAJ8@V)u>${8YwvD6XzIc^ zPQh4}EZ<9>mSYUN_GK{vW`Ba?F?+tRNa?n9EVaCn>Qxu+BtX!~lb(K9QEWH+;7^BZ z3-$7>m^)Gtm%45D$$MIS<+OAfHb)6pYt#LiGH6zwC-!CA;PQTO9xT;y=p1BeGX7sx zq|~NqZH#GATT}Fx-&Q7{gV%xN=o`l#VhSutG`)Hh%FsZhfV2@09aw>qFxYJ{NG#m`{*9~m4Y zG|uiMV>LkjGBiu_9wOb1&c-{X@WzU}FAV`7xh_Kgw_fALflH(L5&Pk*Lewx+El*cOQb{~gJLep9Oq z$E)%+u#V<|*Eg2JZzy`RaP;-+v(PkD0Jp4sb85@QlSme>o*j!ZB$ce?pj4LW?^V)t z7-B@v(kg%Hn}aJ#fgMWd==J1*3B$WbMl!H};V$>z;EVMpmDek zz*R+o4eqH%Az{%=W#IafgQ!+l$&c8a2Jz5lJJ8A$`gDc88!RBhPvzkEcP5AE$nkqV z+UJh~Bx)<8+cq)hC5SaXDrliVt5Xvcmw%Nu+nfrj-f$Ji~;rC9zt zh1&K(;w!_Cz@LUQruIjrT;XDBSk_DRd2LUAY&?gl2+ezgJ7r?1Qg3rX_4ft9W)9m* zV*SMnHz{pAUa4=_DZW6PwdG0XqNAuY<3znrXLkCeaF!vxQ0eYHcnF>9)I4=4=B6ip zUIr@F!LVOqF9AzM+}&HFb%^E|*0dI8@8 z0JAsPNs01#%Zv7v$y0e>#)`VCU3}nl^sTP zc!hw;K0VA7q~^4IdlI)5+7DVUA1&PqUw}h^ov>%)0hz{_^PK5bM>Kc8{J@YnHl86H z>$JZVg13|uaSr**W|{byw@@kMB%_Akkj>vRz3}R6q|Ff`<=gM+dp{<{Z_SIfGP2OD zyLT-0r$^H!mnt(6jaYfc&G5HbQ|Np70*OPL4{kiYbjN;U5YHdH02Ah5b_lv1R7c>p zZSw^cbAijrbTvDG$$@JQU9fvKlZBRD&4heN*2WN~-3R%0z%w_q*lfk^iZYS$&5f~^paJZZi`_;(gvKq4qzi+!j-0b|9q#Pzi_uEK8o zG9R{Uv9_FFj=(tFH=>@txS*fj;x+Cj=BYzQooewW>)whG*zTy5NXtrhf8^8fblQ&j z5SuK52C8=IZ$zuwaPVUt2$z0x?u;^BE^n_$>t1b!yB*Jv%1Kboq||3H8FP|%Xx<8F zL5NYSkD7#3=E$xEZc=yA#1L;k-l79%_dH_xVyDj{`9*AmK7W*N$kwAU5r{ghvUDx@&o1fnDj0XBKwgEkQdi^75PC06N z?!uE>p-h*tU}z(13m!^ZtvIjIhj&KEn5nT7@OrW4u3FHFdLC=LB)||Up)(*U`kBl2CJ7i)bO6IeEOg3?@k92NIDmj+U$z^T6 z|x`)68t+-v^{W2MhAkXoFCNo-WNl?~Rwb!dfW zt@Y3KS4hh;@f^gxjZfd7;hNcF%_1nvtM{GjqYrpZwa4-Lua)?R z<`R}rk{3G+je_x6JZ+ij$&**^$ZM)SiPzYKN9S#SpDvLW zVUDTzXHh=tY3yT8bD%z)J=xs;cNK`I<~=O+jK5*_6kh*kDZBZ>$x^j!%V(qJ+|ATD zvf1?G-t5ejJG}XP$de#mfsEJ)E!NH2zFoeZB7k#U09lvh$7!1>{MmSg@L25`o93l# zCKYbW8QYEXv4EfUduMFYxa0m-8H)^A7T-TYoj(4->jP~tJ$ ze*LilE!KE4*d|ry!tI#n!85kBl}Sr1`F73*Hvg()nw$OXv@HU;FF7THXKkBN6%Brj zgtk}%+`l41`%c;L@Y=Q$YMqpKXKZc<7|%B}NeQHE$-KKg^I@Qk^#vKcq22cbZP~9d zM}w%?oO8Al3o^AUK?~2=uFLQXX^zkiC~a!A|H8}-#9y&#E4m%OHdBhB=2vYzNPvGz zyH49E$!1;SFt-afM&G?=t=8-~%lN}zJ(jK)$8FjbngpUNv+qE+) z8p!tEW+TS|4co^!I|!whNOk75^Opi```!@IE4+4$S^RJ=6n)+&rcdS9La!^;rp_~w z_cOIbnzN7E&KX-Q{X3=Lj4c%Zph+FRu4im79^aDo2M>;sC@%Fb$78}EMB8h0ZWnO$ z!)aUGVl7?8n)(GB7QFH49#6N6*oT>3Z9r+Fpx5j2b7qV*nIiGein1ln0$*VI>BO^-)m(8jUfe zLCYsX2hxdm@@OP9$6bG{giVI6rQ1o7K7Yf^&cJe3Nok{puMI3pe?@SC$KE{6K4_c* z2DksDG{)fEu`j}s_p_xXY!L^@TKBXq@hSI}nWul$?=HtFsR~UO`e5fsfgF%z;7dXw^zx; zQs)$IG-P>;+EDByCCk=F~Hx$Ug%3iwH)U)b4=o@Z5%w+fSI^@8{{9 z^xl9oK3{%nt}92A)+vNnrS&Q5TP%^WkbMMO*i4_eJ^;PJdgrYRen* zkd9QBXY%cZc}Cx@%bIf&M&e{${gfV%0ZBW_xER)6_vK91h_y?ru-(1!+QRg*jr-`L zv;$BK5MR8H;jtX`#d_qL{(QT(K>|WFefzg~qMXRGX@%9=y;k^}@fAwj%024bDveLd zg5a9$D~8XDV|nlolz$IB)A|{HuKu{5CMJ6Ui`n=v*xcNgxjs7Tq#^4Eg1YQljzWH) z1R-*TE2PzYpLg&cimu(?P;BBht_j68_+I*BFdHS=(AY@fr1oFCg+)MjwR{Ql#u{r2TQJdoY`_rgVMyTF89cI`iyA&(r$$mg($A84rUyS(oNR*>;&a za22!AWv$Odwh7CaL~E@K#(I&pkJ=cvF8QZ<&RfsdbhAKPuf^^()S5;|muh9o4x`5% z87E$8*A;ZV=f?Y3knt5f(M>Dng7!?Qp)qXxS8we(d(`xt004&R5v+o8FAZh}$e_xk z?~C9I77k87;FX_QQC61qPZ^HFP_SiS3T0`7LHp->?-W8XRm(y&M$ z)A>T)lGthK&o@Z7%YDN|W*ze*mWu~KyH7;NmT2WFZg)+xJ)CdYCEYASK7DNth$lKh zpizd>%m}|r?J4ZoU?A;AY5q~Eiamz5T(I`!3$#t8dHC;5(Xjd_=rwMwbLZ(FfE+nS zZBSXA?2S?MRTr&lhfO~npu>YCyVPqV%M0kPK5MIV&DQZSC^~K(e|X-11>_vIu0(6` zmgVR;RMqJ1TS5K-bS>%SZX&M$55B$l0s`~~?crJ|Q?%!?{9cW`KcF8R{85M|Ve z8-@`k{LH~crH73ORM3VwRe*!ZxIdNkYmgH-5Q@~lVQY2ve$z%SX}Pzw3}Iz<3gGar zS!+(xW&dQbbIB3DU_N`u*1_5k!ZNM@NM<33I_!oqD$a8}*)|W_KIsAvRRIT0m&R^- z8?D-89!jz$_x_^G@A)a{xPDH8Fzj8T=lU~=W+tVIXGLf+2!hBcTl5H_L;v?Y8zPKY z9xD&90Rm?V3qLKLt2aAAopuDu&|N0a*x!}XnMPBC$OZKjcF$P>#5E_Jv*>6x)1sw!AkQXL=eXNOtR$(WP^E%2MrL@>3MP0o3?QkVZN}QZMd|{50G{L43+=xs z_4no&f|N7E-L?(a^@nx$l#j0~`x~vz`%b6A>1}idHThf6KdzSzAI$JxeoMa4t(zO~ zaflaG>1I=KWpu!uP{!s7LyoV~d?y@U^-?F}E|@9mzr2g#d;%4WT0v&h{1`6VAdh2j z&j511$&mSjqT&`ny=pW09E#4M--1fU4F&rbYoQzST20GGj=i9sbsF!q>H)KDz`C>O zMYX4XJw?%H2DKk=YBWju6U?)!nq1__B#V7aZox`$ENU?=&KvR-AqfMsgaq zaR1VCQji^Xf^W05_fkCiA&2)Srd+J#%yn_+u7&z3$qN~PyuXO$QAAwJkr(SR2EKDw z`Yl7!?{GuiZBJjW_tL2=roe!c(eX1k-NOhI+qE9Db^or%3P!ye_?Ww3E*bw;j9;3( zf5RggW%UZb3p{?1XRos{!UCKuWY)63HZCJhXqMBPe8X$1J)76q2veXlHSYP}hiN{y z=a?)1_Ca&adwkG5{)XA}dHu@=o%POMV19B&UPgF84C~`a(GPUxsLe;(lPMLj*ogBDVACvHru5Cy^2KzTXgqtD5+#3kCYfLdo_oiG zq|0X5I3`c4C@+8(cu!!t+y8`o=1?-5PI5%s;h;V4wk6E%Pe$6yhzZWVfz~G1r~C_s zH}4@w34g=vrM&)4j^J5^_A>LMlOW3pMcFBCEuSMn79#%)-~W8M3`Zzu`3A&mGT`Micb28t#9jOE zYyb5vs;ccA`C526!vRm`qp9DW52OvviP|MqZZOcVfOX;LZffTS8Te1%5~zEvbvvQL zXaHh2y)f~42;=$Z;(vWfkb6xKfQb~i02$gsIt5@9B^_`<`!>R1zm8FZJ$ar^ZPG>$ zY5z}Oqvu)zq>CbT78_=Uc^HOs`Ti~Bgoub!1I*F@$6rUD| zGk7{4`VqM@AAYXCoqTq#x#6u=QHy2aVe`IPhKVt-Ak7C5-IsO8#pH4G%M?p1UzEM- zPWS;3Z9j=7>RUv&7U3raYMw7|gN)Ux^CGao9Ww6u+}rpjOM_5Q6JF*m$+nds6%(aQe$+H$*POc@^ze=%KK-liK~F52F$1@7li8vUjPQz)U4P&KOW;eVBMle zRMOb&#)D=_&oZw~S9$lGF-@;zgqA_(C#wfNb-Mwbhu zB>}Zvg$*bDWOlUM9H!oR%>YWFj_D{fADhB-fc#S@Q^Xh_<%lay3|So3Bj1it&2-30 zUMu}tchO*H9rC^Xz>CB7tn5d`&e=c9cK?j?(SQl|YU3~F-k1}@vY@NgB6;7EG(Lnkjj*ptB9Pz-QUtn_=C z_fr3eByt|g>_zs0H}A7LX@z zjhr*0U;7s6*eTwJY;w!eALiXjDc*iiewqtrUgRt0FgCjkthiF*&99C#Li&{do zA_LnzmRKlfII1Z81+ad|IEC8wCg#)%Kl85UhfNKKxHi8qf+@W*(Q?4&2IF4YflgAr&$Y@K1GH9Xr)?j( zIL?k~#ZEkkcT&!$-8#971tU(N;llxj74rpYMjR`gAtX9o?#fPvx z`aVTEFY`9?+D>?<*67z4+VhlX!Z7iao6%R?byxReSuQ)P$q(TtCKUmwA0I2kkpq~^q0?bb16yL z45Um3sApqk9-uc9UKzZ-4x$;jmzK<1{k(z5D!YU3@L@BZDfH(!I_@pp?0-fpobvh^ z==XlxiCN}rhnMDMoAL*PXs(WVmq?p4LHhaHoE?22l6TOv;h4Xo4j2r6R_E7 zwVCVv>WJ}|w`L#2xjcxN_Rs(E z>C%<+=j|=#lQ|3|@LZ6+l@EpXHeM6{uV9x5cfL-JdQOAS?d^OS?CV77$x+W9rN<>c zw|_C;orvId#&vs#`QRDh#)s$kV-jh$lW+b5Uv-)H;Hz%_hS_^~b>geH_FnUY6J2?8 zr?1rcyJs>FB-~E!)aDJ~oe6iFeV#q*Fed$nW&difhdkf-8)on0HPznFYiz_E?tl2` zLE8OCG4yS~T=%yJF=*b?_z-`??8ChN-S~+4!P)p`Zg%B>zW7^2MaFEdGI2>NZppN( zP5M!dqp$WPYAdNPqvj5pA9FrxoYz$Q1h27wSBy`K`|B()`=q(>@Aki&_q0F7-!S_R zUQPSXLh(P%PtG1}G$+wG|G`a!*I3&~z}}EgbCR&4hc=M{N`j77?>?;^`$2&YDu}~x z6;E($NRJz#f8Fxo=v}ihb9%uE6qz|99`BArhytW;;VwcH4gw>@d029|My;s3s1Z}a z=L;KvjUqPx42O4JxFPS;2zZlsb(4TX`_)I}_L%7Buofon2dkL$9V+*jZ0TPaS!BUT zPjr0he3SoOMj?qf_F$4`Mp3?}cL|e!Bw5Prc`R7S`u10OZ_S%Y?>-^1zaRXo-Mf-| zU#Fgd*@#C4MX%U!!Q9s<&6^3&6?@=pI+p0y-9oh##%$5Qx*1wLQ+~vnP|vV5Z9)}O zt(yVFdsr932?RTY9e;L@d90}RTS2Stqjbl#hcx+r2AiiXRA6PLTB*{}=MpGYSrDZR z$Z_+~9k5f2X*crvKJ8|)SBh9!=x|3zanjO=H9H)n`3>F4-UD8K>91FBKp3r-cOYp_ zKTyln*GP>51+kOZ{~-q8pN0?0_gU!=ArnLn(JHE!L}413VgPIrUIyDFh*?z!M0G1(kbG_Vsl?KNUu1anmqSH-UehTGOX>AF0A#zl*o&32YbcuZ(1bj*$ z0kgkQnuYY zlQ|QL!y>HA#oc7K`o*ys5JD!ZR-izvKP_+2$wh}4sq?`2wJCtohS2-SyMtYf^2zr) zint0x@HFOzl#opa^X1M=4|<$xco~8F$yiF<_Myfd%TEPJOB6PcUu0~#{0hk+?n4^d z^5-v(Jq6#_{L|{AfR_Q<@seno{vtUO%ug9WKMHIgXKWt{zX|MSe@sW!w>z|wrgSYfZY|WrBQC_ z0>QRwAF!Gu>WNVw$dIYkrDMlA98Gew3}@bu&Qy%{)Z<5H(YRZGMoQjmGPmzi4jz9w zm+2-!4y(^T`IP~yncIK330O()XPFG(aXW-6% z8lQ}?uwdcrk!F~K7gfi7I}*Q)vi=5_P8w{qRqgx-b2SP z`?|t^4sBim_8fk)9~wJAE7fmiV>gUic(V$@7+$7>sMIBr4}Ch82O)IZ{2_eRv5@;@ zP+o!o`Z&368)*#Jbf-xL!o1R^?sD`RNW3Ia02u%DEz4ng(@K(&R| ziL?4H4nI?KIP@DB`f=F{*zP#$s95UnL&H%kFiWc5$hT0F7rKi5y2rI5!_Y`8L%?)p zM`0E0)gPvwp{1{9q7_OFSKE`M_@S2hGEI&D}X+M@1ZVS?qFyH;x%wwlv(JxZ82=K>aPkf=+I!&s%w!3U*LG(|{ zHxM>0dWu@)`C0xf52yTwDK7i~25hNxJ54(Ry|>uox+g%PH~7 zj`YQIfCZ?ZDuQ^=q;sbmzoEEUWR`c!;Pubg!qg?wycbM0sl^uHs#&LxFlo7_tz}F~ zmER&MePb<)x)LlMkJyF$XPnY=iE1g>;6b*8_|H6l5d-%Pl(k)RufF7goE5?n_f{X` z!`h#2_7Jr^e-@i<9=tfnI~!5SF(NN@H?>9jqNGngpyI=}=?F;`l{oEosdV*aUj8k? zGJWnE-p769?^)o%y|B_(po{NeEncA$euh~=bY%y*Cx0j8#=ys-teU7;2oiE=`yEpk zl4Bf^69PUWXhf03(|k*GTO5a&5hRnXT0qT$k&4ln#@eWhU4xw zDGOu=dZwS&v9oIlb}&)C#7Xf@A&^K{hU63MKMEsE8ou7P@R)DJd8x22rKYB8`Bg2j zpaRL$T69r!GOeViL8pdvUi>C}#iLTW9HC%)0PLFUC3T0^Iw2RX z1%o0g%g#V6^%4-Y&jVxp(R#2=N^fxhH$u7S;A-Nj3$V-WG&`(SGg&L*nTw#$REAVC z1OK$j_NK!+?Tc2z+|Q6_%|tRpvcN`9}<79<$+ zpndD82!TCrCfT$4q86osD^aww*yFzPIJbU)&$M>0?ULpPV?x8?h%L}pw$rq(fT2qf z)qW^o(&9}<#1~_@&GeLA(9iDv2QONGnx?yDa0&%>pf)chuH0Xuly;TcgCN@O`>zl( zc)s{hybE4GYSt9r{E)pR)Ic9?dA32{q7kssn2I0LwPOL;6;-|$AB?;Rq#V7%g%0?r z``8j#kph9ve>!9xh7-)rBu-v8y5q_Go40AR)rj&|9n<{X22i2!3Un}q@wKqoi*RHA z+)%AKhv$-YFdLu5Z1G`@xRn2zPH)2ogC_6ZjkzR~bCKR-D>*EQO=+eRf7+$V+$B4f z09flee(J}KWQtd?hUz$)E}yRI&+|a3ORuGDeLPiu#j$Q>rt3{K2;QW;1#Y)z?S5?L zPfL8I`t~yz03=4tf6XB{vm+IMV}!k4l5-Pg`_m-#AEe~XMTUZs=*$LG5Vc!mHpAwj zgo@o}VuF5%BYc*x_B~%~gh%^d>W9nJHHW<_(a-GTIu4tZTJigos#4o1N+FIkS5QL9 z;l;*yn<-kp#^jIP*)_2T~sm_uV3~>?vny* z?ZK=?n72CU`C5C~w0mxcOjtfIPGQ0)={jjB1zTKTCCYXg2dV|G&slnp-m**b&ro%k z4NtRg(I`{rSonZaEUeiY&M~D!N)t#WA4V2n%|2`fM@bb~zF`NC+`{=Zi7&keFM88u zL+z2YV_VN3hPT21c!k=&5K$z#j=aAZ%JkFAKW%P`Tkd*P3bBdkzwv!CawOFR%WH&* zsJ2n+!b+85|G*3Idz-cPjS;D7H&bGQ>iTAX>SrzmOR{bI;d-rmO~z~sW~A%lBklx6 z82{rp`EW4ZpC$vgKhN?`2QPVEl;Q+p$(S9U(HX9NVX!-4c0QS{a^th@pf9wsv@Jak zL~2S-1Mw%F*^JpGt;9M#Q>PLR(RG&cHJ2(V{%W>avzroT8xK3NfL$wpTBWkWOd?p7 zQfgWgLh0?Vb;N~BqEToi`YS!qh!WG)dl4t21-`E^6NPTxg_N~L^IP4bZE^IDbbAYs z(6~~}=IkO=s3T~b^gT<=dJ4uc$sg~Q^GxL8PwS6Uur?(Nw=uIi^n`A9Z`P_9_LkCK zsj#mT&g6P2Vz+>_05m3vurT z1GYsSr%xrpr_qITbp)yOC{MN zuuf}8;K{3Vz{p`769ATX*sfS`N)sfnM)h`pXW_Tz)9%{(#;Co3O!hYiLl8(QZ|?z^ z2Is-<7zD=Jp+BW0hFd@U)9~Cd&7Z%Q<;-qt(Mg$1kPpwHiDPZ@_5{kNk`IcY75Xzn zg)+Iy zJR!w%!V?GweWHZIwB0uUg@I?kwdLlqdD(d4NCy1XuqhsvLVJOOs?oNEPnag&|5&Ew z@vtub@cc(9n7mw!J|`zUMkvc38T||o{!G=rWUaHqUohavtq!qKmDkm2boS;n%^UcAyIFQIZE%in-%QTp_Y-c({HDIsR^UM>1`z)cs56hY7&lD!V{tu1$ ztwqaMyk?lpOe+EMU=;WuOs%yWr2g{lIA_Cy(zN*)(9VPrbmWTPfwjmOcs}W|EAm;D zkpv;82^j$yUi%_Sv1<|RiZxz_D)Q~Rw$9yZ5#RhrdJ3hQ66yw*A_O`jZ%u}5@jN~| zv@6?|y*n1NIc&G%lJtrOQAdwod!7>T8VZN0|(93EhW!`P1@=(_6J4@I>8vvWvgH(#9`S$-vheJ(;{iCpD@=Y}eS}2dCbcQ$!2T+>&`wdo@2u1p;N-wi*8S+M zYGSEokGW&TY2&?W^HCyqe;R#CH@gG%(_;XjW9{*+2l>p*GOC98UfO{pKpiH0*P}R{ zpY$yVo?#ig6DAzFQ7yIBRzE8FZ^rl#Yw)vdK?z*r(4yt3cjg~EX92;EBoUncw*j=R zchiYw?s#5~S%}~u3cQ^>TvDW14h{O1UkRHz&ly< z4;ULFoV_g7>#=2+(YYZ{Qf6N^c>L?r2+6<3MiQ>*l`ji8J{9`r8jaujSaY(jrBSeF zi&V}B*>&7DJ6TH4f}5#WWF-0zp_lKI{Lr<>&9l88#~r20fswWPD=XAg4I^35>_6=a zBCxP79v~<~anj@Q6n@o$WZxBMAv;g=1D zr6>5N9hO$D%QBe=pN#QYsGrkMuDxxnugRS0CHi#o-*#sSjNO;6?=0i(4ddvN_Ze<# z*)S3BW&9z!U>vS;NcV!w*)RZ}sTISlb%zoVuSE|~Na!Yo)7HRZV331rkcMHJ8n?7F z`)uwMhG7MYL# zv|AA;?WcK86Wqf2itK!9xM&Peko=fMFD5W@qzVs0fex)iBM>(tGQx+ivv$oo<2jFN&mR8_bK9R*)6Zh9^Y>u-ksBh2t*&@$ zIp&a93?HhoE%}ze;Sx%ZZ&0>RJ$yzHlqhXeV*GTf!c1k;^SgU?69)1S7MFA7`*ZY! zG5+0wo^TfNG{cAaLN@Dgkdd`Njce7R83Z4G<7OfU3$wlM#_j@%2WpYKR8#xOu-s$G zUv>G684awm0Ex*5Ty!}NQY(hBM?0@Q2Gap^k4bsrF=z`t#HHBKc*|_;}74X$pNZ!@)a%Ob9x|KM1neu?DScohX4M!AytoHYARYxYra>oJ!=isryj z?b^)y1Zh}^kRa%jHhJeK)bV=E{Ed(|xLF$?ulJ^3>>NtFLm?Fzw`r4SHV1$$eGS1DRirf#y@@oJbvyTqNIZ1d%JLYiWmD$868{-ao+%*VAT(R8elnRCgU_ zFCOYK7^4SIniv9Z12r z4YgdnT6<<rQTQ z(c0yER=e3Y>Zfc|X`yu#Z;K+m3m*?bl@}bv;GRZOMn0GXY)~VsSp zbwKJvmVp)^ML`Vo2i<6!ZM&<3QCfc~Vuo!4dVzf8n@48ZUR`}e-4f_9YQ0=%d%YVQ zfqMQJqg88l;*=XKdg7n(HX1^kVx{LcN~zSC(GSwjk>{{XA?gZO z;2oPH-qleq?zbUonUHm;0lMS zgU<>}G321B6Y|BKMUUf#|F>)1b17Q2%s@E(N(@?B4{0|t?|5}jt6tuDWnj9gJXbFT z@^J|yVcXpnojBmhXGa*jna;`QSqPD_s&!DHH7o#9I*x;`25thM4YDhaTq3Qf-YB+8 z3rL+n-Q*cM>Bj){{Makz0ef{tb7x$*c3wd@{r}mi#QzswQ|mlwX9t<}yn54XlSeqZaFDnYI)G+7dWUk;$wm=_BcB ziPUdnf@-Bh*)FA12VcmqWkxpMrDh(+R(v5FD$9*Ck)UD~!GNd*xSJzUCjX3C{%P$Y zmN;zf&blb$uCOB6l_|ckP%iAr>LLNOltro1&Q26<_FM*&d_+wRv0JNqmTY~B6J=|4>r%)QFd+zirW_~K|#5nl%H zTr`PVZD6qRBRgmS&~5hDP#7?MmI(n4xXSP6`ZJpTA99=Xa1IK35XX7Dwn|4DY>fEU zq0cxWl_~v2pKxZ`7-c)6pR~tXm!=bsQSxTb;&NKjQt~#qC`N}C-DFOtwadB&{Tj5F z<_4`~Eja75_W)hOtW;{>l{9n(VYW#vjHmLtHu0P#j39@JBB6~>=jduKR@FcTG7IH< zY%-!y!NB9RE7M&*nmxE@Wil!j-CjY{s|m4JLQw{!#ixT`LbhMLCr5O<2Fq`zENQxKYG?pzVE7BJOfu7O=B2*&;~9{PACqVPDAxjKP&Qvg?R(kKmU zdDiTk$b;$IoN(;!gfwn81b=6bhyn2#8Z8T#R?2POHiHIFKGBL;pdwkr6C16)i`ctl zd806>pmvZUb4`QhT*o?0Gq2yQJ4arH?A;3;_^tK8o!K_t(srE}AhHy~3|rSJ)BvhA zyW4nXOC~21ngu)_^lQOh8C-RTr#ws7{-98-_JDQO4VGi6^QPg!uFENhj#3nvaRkI2 z1w1>?W^BV2ec1XEZ3`9H$51cdBiYJsE!GyOH^#;ZMuo0<51{)fU(eF+>oa)XLu@1M zw-!r>$0ccGd6Yo@lB=2@jZ~0NiW6CFHlmrm;}M(W{Us#86n9`9xqJQzOkoWj-8Jt5 zp3O(;L}GC~!BZF>G*;O?xU)ZQL1o6`P^$r*LSYD zeJnQ0W_DOU2#|?Qn!6fizYjnER?g!)&IO1spqOeVCm80=E%IxLbeb%FCh?omUh8lZ zV4=FD@#5GCXW{smV-_3Wf5SqvWs(D}BQTZ|*2-)8nf|aP#3VkHBD#)!Fx`krr{3(R z?>904?Y=!%$_Y2{{>2+D*nACR2YpHWIRd}^z%qR`(W~XhZZj5U1aopliWW<+5oSUc zi&ItsB#?I~-MH-vji9aAp`~9EW?nV#^JOI1Xe`c3v1I5NOtaf;@j9k87^%x>-a+di z(%t4Ae3i+Jgd3O)JMR2fmaTu@;=T?VHA&koAuIuW<)SzM=d_gFapRG5(B4DHavT~k z!njc<;n`2zjBoBk)Uni+29nu59qnSWgVC<$?DWukbnR}Kv}%c&OcqmX@4?#BR1P+C z-!)slV%z(jyzT|+4%ebLP$*L>siosie|ng;PQNW7XXwtpI~zhMGi}CKcSwrQyFxaK zJ>2x&xXGz?Ftcw7z@SA)MkW^3v`osMMe%l(e$TompJ6qpZ{6gYv2(NOBil7MYxPx~ zVoV{?Xn?-ieq8PYCixzYM*eQ-`tu28>7pSHn!iDRaB?UkhOpOHLjDeaFVme~iiql0 zt|DGg7QIPo=|wKu;^x+^S~L&egE<_4`*O5k#|6`J6O2@O3+DdBlVHipS+}03bL;5` zZ%P?p^x0xcc(lQeJd}X&KVyz4r@t-!uwkb06f{}$Mr57`|3Fd)wn<$mBuYf8I3TB*9J~Y(P$3*H6zTim?=xr$jY^o0MT1gTF6`2pOE936>Dz!C zsONrY9HEo1Pu`Sc6Nq?6P5aazo3+xOXK(#ul7 z2VZ>=3KE=uE;*vDdk>o%>fW3a3IU$yxz8~ug@l{wUpMm$Mq&d3V6y>5aw&Q7Wk>`w z%-6D{70<;%SLb;yP1e>}26|Y2CLpxsI~ZBYe(gBJ$`s&NzwSbAt6=jVJaX;r^}Fm$ z3QGI`Z|^<3s(O|%PDDi%45&y_QGyW#F<|5%qN0GH2u92TVj!3U3KB&G0YMNXNRXU! za#r>0$vMwQnO~hd`X%P9|GM7IK704>uC98jy1J_RyOcgQWm%WC2kp{v=v55p+Jx3i zl&|-1vnpnSdZcx=6#67_y%F*d^miHqpFPe30Zp3x~%1ISh;E zaQ#L`Z_AI1)&%czCh*5ngTF=@rFYxy$P8c>q|mTE0!z$iC`@^b(#K@bm`^Kd*DqOW zPZ;8~p_Wur#M#4i`RGh58e&->con>I#nd`v*g5Hlif)0@^-%4Hex!v{eQ93LpV$18 zPnmb^PC>gBZce<*xq-Z~J0!?72CE9TL{Uo$c%7nyofaZO2+RbT&r!(F7_r=tnY{-QA>J_KsmMB&so+9b8Q$ubT=OxNSC;QGa~_xuS}#bd!c1qD ze2Sps-piyV_`U-I1=|3a=;bDbl@zJHxDR8Ba`}6~C=hmSiZxca#!L!xsC?(ki7#BKTbBnapQY3T94TGT0_KU=rLMu%s_RWjEi@ zn0^lr4&)WeCF|((S4!Se+NXeu<=KfhCO7DzPW5+|I5n5<_(7e;rW*0apeBpa;$^?x zZ;Ol4?91J?-qLtY+xYE`R>Hk*?G~U*`H{n+I1J#0-VYR=k!FOlRWq9EQq9y6ZNJ`f z;3mv5K$B6@dduSkh%;b`jz4bdUNe7V7XR12&mS|6UW;!sFx0YQmCopt{PB^wr&7cp z@1bz#kz@C1B7+}hO_{wHx1TRJzt&0aa|CEVr<1TKXgY|nesD7_Nl)HF7YAv)bQCL9 z6I0mAl15%_rsO*Y0?~ZyH!Wp!_81GZo2Bg*x6QY$nQ=Q=3t%aj2}eI7S5 z82tb~W<$kk4`!8^2in{!lw9pz;HsQfjo~ujwAPg&6aRfn>dt8Dj0~)P^;o`X3eHj} zS7N^TdT?!ushCKA+3lk<4f1{Y5mr70`T{yP({PfRY7g_-@uc|`5UgEmqOVcefz=xU5B)4KfY2GtqOZ@*fVGB5uFWae^jQy36;4fgp$$1$Zg>8FtSo=WLO>O-GogDlIee-t_s>O$|QkVy%IT za}k20MT}reLLyDgA=Vc?l<6WiG5D@4LtBDe6+OLXJ~6WB_Zg=A31UBFd63`~Z?_2U zS!sqg$b14V_L5xKyIh*g>;a|j4}`7EbnRTir*1%!NakvswjVqI3BCqfUr05zoFqD< zW)5!eB3PzJCXX?c7igJZyAN7rJJNh-SMePc;UGy#=8g?;3koqaa0~M_N*V2{_zia$ zQ$MSdBC7Hp+NZqMy9MNHu)Fb}u~lW@4U6G5)0e&H+ynEAZ!9}?cf&g%o9X)sWuqP8 zP5Q0ABo0i+b@i2>{JmqPDvDoVyy>{TuTiym%uN(t*zY%mtkz+RpAc~QWFxFUY&nq` zM|u~=Bld-s6)cp(2OtQ;EA`&btXD80emqIwL=Jk)CQsbCxnssRi0o5fp0}_|8 zHN=~FSNA=;jq&juc4UJY)u-!cGvUKg6D)qF7#4t9YpPy?Lk)bDZ7zT5JsMAI*EqU` z#oP2FI_7sS%Y4FHMt?PkiPEyWd?B3aN#Ogrm`GS7BqvVc?|1|)=?b9_GJHjHuv7r< zW@#oxZ<D1H*onwhc*cH>w;;Vw z(>G5u*4Y^1{iys^Efaf8>0TzuR2jCC3$k5t%Spv%jJ#5A{o(BZlY4AqfSQw3GcnZk zjn1T3ht^W%LNC9 zTIk(KPmZZg36J9kQuon3KXBY=@;2)sGj+xOoZHEFJI8Qy^9#_`oC?N)RMS+3H4Bhu6;Sj*&1wD|~gM`pX_LZUP~Y}o)a zJeIO242q-Jcz4^$tFI7k7$H^a*QeuN7^iF3Oy`*-Q{td7G{5tJ9aFDjRYQ5Fpbu)a z!NkfIt@2yFB=^^1+H|nVgXOsym~8TVW#ui*gHPuB@!O%u7=llv`Cf{^D@`b8ZF*Oj zt|zqh*8QS1f$ceb;w|QK>E>u&`-Nl0DFgu`s+rA+JX7wy7{gZ;HQ}#5W*|V5r&rR2 zc_=plTPT4Lgjw_REpcXW$P)RD3aLl>yDiS9XbJOM&&a{<@}~*%LvuTCX>>v(_U;(3 zZ-1NScjPFu`-TmrVx(#WbbvW?;KynZ@)x#Bm*p^9DgfzcI|goMKHSPaiV4_nNnjq? zk#E2N9k6V^Z+g7ap78YLP*aWffv?ceH_YV%Y8Yz$WQz3h|m9~;(cYBe!-oX(*-v)`mR;rJp8B2Nt zLt9{zpVyy?bs;?EzJKX)Y+2?Z_nG;=G8sjH)f?fS!mQwAv>ZEaY7a`+R$BM;BY8AU z%O356sjAI5=OEEx$?;?v>_`$gS=D9Nwnvaa0XWF)dJk!gq>rJj2J=X~wr3Q0ak5S2 z;x*dCPp113U^|4Z#;g#3gphub&mWY2T}!~WPVk4$vkV3OZc#3aUmnYl;1MQgjPIGA z1;+`KKsAq{9asfiU#YpLz)JyeIhF8+XDF8?qu!<^jh^htJc(6HeLQV-*7r&na=`Ks zR0?-o_!?Xpv}}^d4F&4;AKWRGT z#;4Z7sRTYE<==FIhwVa53ypr&W+(V>+8#4tQX3;2NUC123B^~=n@Yd43pVloE=AW2 z42Kflh`Ez$hBFUv+Z)d9VJ4b)=&Q9MrXx`kvwu@{;k$6m;Is2gk0oOdANmBUbX#7z zV@R5^VEt}2eA*vMV3qPxIl6qt%-v%UT5gbI9uL9Z)%rDeDZOC-3edPuN89#LOpHeY zmV`_97Gw@|9yPs7q{1FYgykW8SRt(*@0F&-Y;Wywa)lCz`&UQUy8Src+Ik1$rRlnQ zD)t2KeQ)~J4#0TaPqYNz1TYF^_MG35C0t^SldEiJ>gimU<*nTcm}Cmc?4vZ-&;ztu z(K236e}N2FZdjl*_DhIv*~k|Pczvhz`*2DldI$z;w+p;xzN5c-EQu*coK8PxaEBhD z0h-Lz9`{#rZ1*XWUHnH5a5E;cGRM&MTndZ^rUiem~x>-aS2oT_>*Qte>{|ajR)L zPoR}PKYyHkeO}dv_L+KMZ(2UR=Gt>g&Vzgmcrw1r*#IgNS2x2_b-{<0n4%s z2H$DQwkWft;f0)_G(z@WS^ao&_J=l&F3 zdc78op9A96%2{XHQzC$Gt%iX1{g#9kQL9>t)phA}Lj-}E6Cu*D)YY^-(E-i#y;0`8 z1~OW$Gs6F8?yL4lfFYfpj)rybUb^1aSqz-nCEdoYX~79U|T*H z583GQ4=_xO8Gav|Zne)R&|8F#zGKM&h@O$!EM^*?h9&KyAse9Uw9>LkzQmc)cxl$L zHVa!cpk%8kvxWrh+=Fn>@;MgFr$Gk_x9gOC4huSiMyu3*6EK9&rVB$!Q@Vo=oF*mR zjSmu9WXfVepauqX5#3V^<8rVM4}qku7&14J!6Azaiy%zDT9!E$CX>SvWaNZ)c=!mr zIMSb{w3=f5-W_C0FvI9_fhmd-7J18H_YAFo)W$FUOwZ1juT33@NrgX4`d%dc`(Ykt zT-~ohq*@r4BkoP~R6H7VD3mDj`3*R5qvIPS;my6jg$AE2&dv~k< z*AA)pvW2)ZKE*1U1vG=r2jn-vxQ#M@7?WfG4_C!;ABNrl)w0Ytsd^~|y0|{hR8^+J zJgkDi7VMhwv?D+sQ~Vo!s@t}@5}&{sM2E_+uho+VLo@XWJ1jjyWtMjYgnyR%Z zcL+D_vTXl{2lV)$a18<5e{W3$@2Q;WmJ9-QZe{N8^4)Ky?*asYRt+*pg^C#FtIx7~=2QHeKCKuSV$NdfnL?Kk*-S`RusTeLc88Op)o@l&iPYbLVOLFusfAo1-%7cVIgY z0!(J|lRI0*Cr#m9oe0kF+Ol7Aob;iBV<^5n>3ByE5}X0cKgZXpXwN&}M%xbpxy&;K ztEIwxUQ3NK#^<%)j`L&~>Mki-Or-B>1&KV5@YS3(LoW}4(}!6$<#T{36DB{=G-Se=>bi;bd#;B!P1DKaTd%vho#DkL3#9hwDoChh zSUM!<)uuyApfAj52@%H5G#6*SKa0CUR|hRuQqeShQf9SJr?$O7RvTOyzRa?R1~V04 zGeGKYDL!sW+#PvCy4W4{0jt%cg9htF^776_(=1Ulk=c^_fL=a+nSzM4$MgL87#Tai z)~Z>ZCFfLNENp1;a5MP_5mi`=GOE3%KHH3`ZesO0kxhkaM3aJ5DA-v#-mqMGU_u(t zFNoXEcaA+b-A@8mDeqLEVzG;ScJajP&88rI*JE124(FUaR>dt=g(XGV%g)FEL+iC< zhJ!r-vhK?Om%8Sd-Wkpiv({o*H@O3C^rA>g12eGaxWPB@4Cd&N5 zJCV#D)AJLs!5|n1d*P7&g_3hvDx&CE8na%3jf5&@SYPeU=)&W!eD*~=`y+(#z(eNG zF*6x`7*1nVrs>k0CuHi>sCw$SQV(&!5=N(TpbiGdr-OVhCX$kz$#=LilS|&2U$>dY zA)+F4@MmCQs)4!i`f}=@0}s?|vO6zhAK-s_*6E57S{=OTLHiF+t<6G;Hs?7Wm zA&MZKR#s8(b;VQWoW3h*3g4O%juzTqK2r?8NeebY3z|C*G<@lru6|mDR6= z)5a`^!%+p-el!qm{@ByMnZGW9(V-@T5`l%icce2gK064^8{?IiDSG0i6 z69wmf=Pr0TWC;Ku+CTuUqtOmaP3&Hogr(YSS>b+KYgw@=&EhNJjZ8S031xsngt>`s zOe4EY>fqKDKVh!HNF;reaB+GYd1LgcRwbKworuaGv zJ&WwraIh?Y9J1{_Ry_5qWPYK{C2ZA6s5$cL;2~JM-{QIMfS|6LdI2!RUz6t^79g-f$p_ zhe$XxO_zcDIenv(SYH`j0+=$NoJM#_x;3~2wt-L@4cTWU9sx$5&{*A8dqhSNvoH3$ zEI_2gYNWp`l_Oe4!b=bU3ss7ze?+=F)`Ypb?T{eZC zHUlv9Qgu64Cbn>y zqSds9u6$6c1UMg&PiweJIUrq!QDj!DcGl#ROogz`&`GyUg{#p(*z4TcEk~143D)HW zJ?wuDLxY50U#@^drc{MMpe{BsU#PizBEZxH?(|R!e_tv2+2*r#nLP6WBvKzQ!Hkxu z8-89}7V3-R;fEK&1-j`YWsUUhhV8zoEKH(3 ztBy$XRXTzI53C_)+^}hE@lIi0YC?4L{{0G6;g7veG?yq<;fTs)I{o9gz)kTUFu|zh zLx4r4#%m<*1X4zu?lD70$5v6rRjryblGP-Resho_Ls= zJ=~YL|PkV9MWhU>N^5xXm#yI&YbEhxjI0yVu zkTUgHW~>^bT*fc~Ki*DMIu2tQ{5I9gV4h)oaYc91OvMu}T{2*hAt!VQ%>M~-|4hyX zmc)o9Iw?8h!6P0yCS~`)q1mnvfN_u;aG_6 z?6H5lL7OMM6Hdcm>Qac};kIN`e~Av6I_FIK$*K*qFJd`h@I2Kh^L{h~%UHV>;}o`F z(ly1K&#;SP)D&!A9DGy>$QD}=s*=#(vwV6ptHs7GhO?=8YB1O|oqWxsphf*(kt zmQHhYa0*RqqcMj?;gA7P9l*g?=&+eTo=UkZVpX|P<-6LN(+~MU)Jtfs%7j>UbiH)M-$fe!mRF-YxKW$f!^;mEo)t>s=YI zi}Yn*l)eC^TCS@Mz1n3o0yH5m*-Vq1!#5YuL>}f1<%e}#y)PxlTPQdIn~{Za?fxdf zsfQYirE8g?5>Kdgz&#sVt<6}mdRBguJAiV|G zx%cXu4lA6(@~T8Pzt*hJG2EA& z+x>vkCJf#;AgF}b4V0^f z_aY31#I{J`Ev#V(YNpRFIlWjr-D|!(qQcsF3?Ax3$9LYks|sNo3x=YHP9P`S>ZwcL zVFLc7{r8JJ%!Iq5u#=43y5Rt22Q4?{^CQH%kW&7ZJAP~y{E{{ozO&cl9}S?=C2P~9 zM9hLQq#U7^q4l1x0S7++Ln<$rcBrFEk^O;DkMTte(On-uXH$OdKXz8H59jHe3Q>k4 zU&=X7v#gkV6#O+7jT`lTFLE;H414Rr3#OZd+Mh`X7{4@U!wK&rmBJqr&eOGa*q${; zu;ed_R_jNDHV1Z5zMa95dc%GXJ+np-bwAy{T`d^~0>?FAnIkAV7SnSu_yIg;g$t~e zft9)xUq=pjt~6Oa`?}vV{C+KAUPw5+{W=E1@~xp18Qyl%o=cfMv{SxaPhhNOC`tM@ zOS=!ZlQ`Qs5&rB{;{4)Zkr$@>n!Z{1nmu)3O{XK{o|kF5q0wYM6!d>$S`xVFy))7g zHwvBokaZE}6V! znxbpZd3--VO~M@H6Y&>~u(Ej-09_eX?OT4DkU=~e78kpFrtYA+nw~5Tmht;h`e9<6 z7<$bI>%2|4Zw+v@0;3<)9liiil+ud7i}YS&hZ}NGrq)zA9@F?4$vJZ?kdS4grPjd#3Fn5hmJ!5sZ7#nq@e^ME(Rl`^L8<>HV%Ot1E3$aM&NLG6aDG(ARg4x)wp7|5f&D)fH|KeQ-lbEZrFctyDpOm*{GDF zC8ovvQimGsoyTE(7$Hm!(SsoBgqJ5b1})p~C2cwi#kwp<7f5&H7Hj}0*kSTHe`Q8J zUkRyKZxW_V56<0^{0)nNjV{ZklU`_i?oEKRDpW*?Z(QqGAs*wZ8Cr^^Qr zNB7Au8=U`5kVsuOXs5iDVXSi0Dl=7KsN_OjRnyOUII>H6;?!egD*1z@){Vmq2f~7# z12@q5otBt=*G%;zD_@(|K=`CDaSylsObL7S$?1G=gG`wDv+vc*ae;-%*JbZOCZ=F2 z=(XEkAP*ORn_C8!e-A>3gtqx@lUk6VS-G%4inifbQraga()^y{IfKUVs!lR0$rpjO0d!}|0FD2t%+ zsL|D{`Lr4h(QFEL@P{;;+!u>rk{a4wA(fF{Zb!HxLzE3S$=PA;Ux-W+s5++tP>tbo ziKcWaL+(C;Vdh!w{k9xf3$uN3gt5>Q@(p_f%+`fi+(S5So-JFZ=tTf_)o;`5$_vNO zVg^#Q3A2zI6kP+LdYf6PO@ekj^iAqw@2Bt9yc-;e z&rpEfcj+4T*!)ALxfikF9LwnUOT)J+!G5Qb1;R0o$lNF3hFF~w@Ewa4XXyjFTfAv= z-=omWC-^#FWdqiAF_*0h8V&5lk~Ch#*ij}muujRvophR+s*@iV?C;RKQH#rNnTTZz zR5!fL%%f{&=p?WJNaa(5P|8+kDMic7C)eBLKJWGrC&P?T1V-_d z!3>!%kECF|H|@l3*m9PYz_6lVt{Em!=rK-udd$g|n9lXj z^=NpgKEBz^yQBQ;_^2rfyuAc`%jJiACMT2mEqJ9^+Q-V0SQ2c>7CMx%H$r~JKcx%t z=i_;1I-4Fm`M~(mOwgI7Un47l&kHt&>NGV=-=g(Wf0o@bY1$WeB1l~ibiQBu_UM@p z9Eb44BjaA0iZj~SV;@-sbN~ar^Ezw!NiRNmE?r7ev7+1Q{<5U&4o}xBQNMyFP0W7X zb-sBnE{$hA6xy0(?xK9z^o;LZ@VFU!G>GOb(NcdJA8Z-l1I;Fu#=d#3+ zar@Pkqb_bI!6z9$d@S31)l4e{fh8sJ=7=9JPOS^FBLl+P1nZEF%+)&49qDlP%qhSI zh^>S#X8dlg)eio;EZ?3H=)VY9K+KM)dWBX)ByC?C2c+q}Sb+jhF|O_EG`x@lca|cfV-533g%v zHe%7#VCd*Bi~r%LGUODdc=E^0tdWY~Lwt7w09t&Ad`jM3dfR+E%1L7|k|*;v4hL7W zS|orEuR=%m;RROenEq!=Q8c=`HuZ=)`UrOI>vOpbGw&3?DHwPnPms;p{bcnyTjE=ce9j)i~)hon|D^ zO#A4$oWjYQt>lJlHiXw5gyzq)Q(U@mP!{Y}{u-9uX4 z$>K!^eT-&8;q~4NZ%7-{N#|3z-Wr|;=vW8=s|Ij_AX^GM{MhXSMx)A1=f$KQAukoS zKD$6DYIR-Yk7Fi3XpN~4=Lwmi{b7>07*MC4%C+AR+hw(!Qp~s5WJ=M{i-8lSX?yx= z#DNBM)=WR%!$8XTr!?HkI3#1xHGz*) zMft0Wl2ol;LE#M;mKv7?Zpir2aEO%Y%sDbC5RX}4?~>G1b?aM}T&9){cZ-!P=gb#R z`6FGsXof6*GkxQ&8s72ln$^k19nafK?#`2MH9^PZyadw97BOjbv4um+w!tGRGD`xl zho9ASS_+Ly_2f8taU z%~>c|jY#)meic5Ae50OTz~|e+-KmL8Ni;7vB?v`fxs4)E4V0hwE9h4(I9up|38RYG z4i%Sg`VN~P3c^)c^5v}=Ow%4$pT2m{*Gj#`rs{&py^DPSZH6udA}?cMyeqZ3gEc6h z(iPA?uBH&Yvl#l?Au{0hzZ|Xv~`$HMB6@4>hboU0}4(Th-%L zDXv!CPT<2UczGs<&%F**rIw^Qq{}cg|3D;8p@S^f_9m=1v#jINp!f}-Be-y!kpbfSM1NKQ|Fkkw7;~65s}88 z=!C1%k9;%w;xouxMHrSoZT|740@YEqSRH)W)!Cn-(mv=;#<@nAf@Yjv30~mdr0o#% zBkZ_zy5nXw8 z>RjyRy*lbIph^RtuU_auzllLpl>dax!be|C=SlghTjRvFP;qW4`fr?ZN1N)0=BK*` z^x@`gA}-#PlB37c980d2r3s>-HqZx7p8##D`YIXJP4&(z29l zDNZ!?RB|_EN4{e0+o-^7N^U>n%_HZJkhX7)X)H~k^(xb_jVGkhOkbAKEuwBbM#>3z zr7twJejg0bC_PuLc=q^Bus>!ER+uRyumS&Io9;C}iIORqTF%Hju%>0;g?x1(ZbMhY z?QhM5J>v%Silrbq_yQUZhLiy^M6lbc_*$g*CiC?uFbTPufwQrQZ5c|*0{MiKXS>lf z4=H}{YCz)S15V0W&5D$^u#5n< z*2S(6n>9fdF1A$q-)70&1N*RP_Df$!@UY*G+~j&4F>xdVCFqoYTW*Svpc>W9N2L~K zY;Zp|2}C37k$&gOgrAl4$55gl<)`8hr^1@z)lQg&=$84rkeOhcI;BaTw76|Uv#9}c zpG1-vcuuz%W#Y_nP`3uy8D!nmNZnwQr`y`5+W)z!2|Ehm>i>uZ=p}1sQN!i43d%F*KT4tJ#vvyZ6fZ5_N->@MRP`V+nFX0)bI($)D797L_92#0W%whH_)6a=!q)ssl01#^H8b(VpwDR@f|+M+Z@+}_al{g@#BbAd{WJ^S%>)Qb#l!9N z=^R6uxU^4mm%{*x9?8FBWdix5)#}`D(BKXtPvrAk)~K||(VL_9x05qsaY@5+?}dtm zJd=3P;~lP zXD@!s21R|hw1``#18yF$c*c6uuGS*JN54!4F6)7Q9DJ3HvM4_Xj!u!(RS^>>`3cI# zKOL77-(3ceWBpCq_2vddC6$Ks~LE{q5I#Z_^ z?cY}dNL!pF=bovcwxF~oFa8vDCm$ZBPm$sotD>e(Fo(bDg%r8 zdDTIxwyjr2s6Mq4j}DaxdH1`Qo6Lq#u931Yb5U^mSc-Ot@Znmmc>ROXrb{7`NJonp zR)!2ELgWGxRWbGh=xx6l-pf;ldef7pV=e#CjbbyhexI^F{l>Lcc-G8%T;i$Jb0>+^ z0qXkD+7nU{7Y-TdU30>|m=zlFZdyW6Hot>##8{Yxj}%4O`Iu|T-EUg6nWX`^$<;{) z93Qbx?6#cTMkzj%hq5DbruM`Oyz6{YlO4aBnN_AlU;Pxr|L36THGo_iPfMwPEcEZP z+%%2vOf_ppy1e}E6;45cWuSysB`-3CKKk

EpLT7I4>p8*|$qqOOG%bnzSc58HKg z{d9_W&$-PSWn0Ju)G{?B;^X){cF!~>qSU)BXI9*VzsBC`s7j=MpA20 z>5EKe-hYqB+;Y^FGW>{(Zkt`Ft(LpTUO@{NZsnLC2|P~b&yH}Hm`(Kbl^K3zd#J!q*a!j3ee4cRWzSjE%lEMm`YPl zw{AVTA9N8d)J;M~p0Gn;s?RCe8du1lbknGAZ6<{pQo%+`M-3D1PB1XS^OFX8E%v9D zY_hi0ZF!-Edb^2Ag&b_wlEo|KMvGXiu#+A~L!ob#hNdJ~Pnyx=FbeY|9RY?kCjMwg z^m8tQMg@HHwomdEt==$VaWpO7E*bPexm9w`QK~5NKNKVkeXmq+p-(#_qRF&W**+&f z!nQn(;^E}otczw6bV!zyaS1O>Nji5mA&R)EcTgAn>RN?!dlD?;2LjW!y|+8gZ4c>O z=6Zl9#z-OheZq`mx=+$WGZnhX{CRbo56bAy*h5mwoA~7MO>v%PCZlDc*Kr7))gvRg zc`bV!%*+l`yn)LFs7TFl!R5?QhjvjhO8&g0!=`4HsdI`(ni?$xQy1f=YNyZjYANTL zxf7=KF0*%A8vG8L85Cl-CG+5J9+_Ir*V3fJ`A`a+M&~N+lI>y55iCAzs3z6|^NUV1 z8gC@66M_$e@q<{dO~mi31)$%qzkT(bX=Q*oqfP71y_+!0xLqzy+5FSx%a_-N@OGOo zU4lm4DtA#UnG8_0&&>PE+*T9>VU!Lh!PFmF(E6ZhB~imKfOic+`B&&`>nm+FS4ybx zwiwH(%knZEtC@@s1xtoCiMxu*>kDYfOh2IlZXBww!@AQps6h_{ydU%Kip)w?xQ_J) zX8%tPPe*!*3^FXO9%D{fP;hLt7DWZ=R{J zOV_NBssO18JQ=Uw9^~xDt50QEhldnl=ysh9WZ>7$yr~H%Fk2tW3zKQe-O@wUa?N4C zGa3niuW`f)BbiFVx&vqi!X;-pfTH^=2ko)_zFidj!j`~|M^TbrymS=Z&zQO2!m)I) z1WGg|`}Ad4#a^nj)|nBY8O=sciRXIrMT_T~mUqjPO?b4|vL`nC=FKGSC;2Qpg4FB4 z{EV%UZ_hQ*4b4&bK3ir}?$Kj6S6(Zn?gGqXrVg6?XD9D)NzjnANRN-zSy2Gvd^CMh zy_6RMq%irzL%@U0Ml9BLGk1VJOxcv7TJvSi6&ev79YN@E9sYR?pM=VEESz~FDWkMM&i}py@`8DqucQCV>vmp%DV46vsz%doYrn=qrG-EUxQUcJhuYLO|Wve81m;B{d$%B4|F)C8G} zGTl1>avte8ZVFDxuLzxgjdtwb0OG0@g5^39Dnlunj>KZ8LMCP8*d2}CA6WH--M(`) zDpk(u{+=)%3^slWY3S1Zs{wLDdJi+jV-#6bu%5wSeag{xmXs}NrZ)ILJ*{VLJhB#J zSIyn`QSN431{KND7C@}BJ4lM<^amIANJE%us?%NzTj9<$-+~SCz%WzlR=jE-xiX%4 z({p=gC`S66cNNA_xU*ghQOx*2R(N4=^y~UUT3uNzG2%8I?QbkN+|kz3Q8y~Q@bf~MT6UD)Xo>7HsoaO^13rEQGktX@QH^BAg`p+2Bt?^oU0<|tSjP`_A^S3? z7;;C!0w#a+3h7BhD`Vy~;>wybm1qGqKH#KUDM$-qBoMn4dB}fZdg&Uh+#my#TRL-= zUtXq%_}m5{slNxXeO+L>?6^w>n8iU%6H}${{s{Kkb&%fTgW?=L=7HB+dB9;ge8eGsFKFh97H8)K zP4&o0b;g1V(woMFKSBK#UPR5eE5=fAjmT)coYPis)mxPSny4Qd{hbVwzzs^ zAgBEl-I$J0Xg{W7FG6c+Me7OO^Ez_kJTp9u#S2eicxReEXs%9Hv~lUB;0REQ?S6+) zXf;dc!C~pX9%g3W=;uZ@OV1WFubbsTG9GU7PK5J>n+(2{l7y#RhOvSNcM2WBua$ge z>g=oII<9(G^!%9QSpl{8Z9ukiDJ3`@u9?v!KE=|;j9reGy1QJW&RWdG2Qw54sSc8l z`7hkMrXy%8*7HJH) ztaKncXg%~A%I1#{A8j$moG8bIb1V z_<2@UjlgW%O`h^KpWKPm&}ocbnf7FPvQnEyWpb0r_t&0(l^APK129gb7F4ePVb^8&ithaHlU1k{dxy<-PR%{dnEOhrgg_$BNT`auXLlm2`4bw(Y3E_sSa4=QU~I%TfmQ>KuhEP=e~gyX&a{zyPgr0Kcx7 zKk(ZOk~wTi23z`9ftpjaws~(BEcubuACTB;s&8&gcnz1aC&w7duWeG+{kZDVK2}4z z&9GD|*TKE5OgcX_n-<4QQK~$Wq&5}oY1a(Od#2ap+Rn#Gzk4(z2BBIhCHm4# zr$F2-k1)}`z${kJ_2)y27Xt0<=2;vw7dx^M%>)&u#BXM5`XR)7Q*a#e&p}0+`MAw* zT?W<$CELspTD4BT#_=hvkMXdZZ&b%zFtyuo%+x@E9+J%)B3b!SuMIzk7m!k{8ALyL zec>v0qGs1b(0yr1ga&;~fu38fb-RZoKR^d_YMuW>;93in9+5de^F2j5;$+%k7MVYg z0ygh|OU0$-W->-QQF$gh<1p|{Bc0N9;sVBxhR3;?pZAa3S+jIlz_W~Mk5f0U%iARz z&WLX6nf`NIwPzUjy%u*()uV39Lz;osrenaT7!S3ef?cC>cpsYpZx&tr>09!X(rM1raE2G{X>pGGd<5&@Hy0T z@p~M-XW%g^!kFnKRQIgzwj4&r+V-b2@6d(=2~H*^Xf*wMRtLo!SyA;^p52zhQ__Bl zhlQlz%cgp{8q#Zaz4Vd7B}n^oDl~vwKM1%a;f}b47)^>`co#Rj; zTD-zR3X;tH9Swm6xyp_QNZkW$<*fbJ<2G5*&U^|0?@Dl$ste)eN|@p&P6%zLGL|MZ zm5zmjCE0DyDOVtlI4azrEe>x1d_G!_{Z+M=c-n62O7>lcG@MpO*D`i)-i!JCnf_i) zObA+y5@J!&xJg%lEt=n&ZmQk)yxqy~^uElWLw0mWIOe4^u9Tmeyow1)Noy&{ls^5? zfc$jdZ3?%j4CTN;1gGF@(HWyQ4Jzg}9U&}4dT*yuxzULQAiL2yK7(i^(Y=? z_9|h@L9C~SHA?26GID+0Tl6Epd@*xJHpHOI2QbLyECr}!nzVvQr~0`V3mf#(&w0LZAt4@53?8H3MyvB8 zsRO2d=`AakGlYCv=rujt=-NrF=ARU^Llx;3SKo5e7?Fh_@06BHY39>o+C9v^LBz#X z`oN6oI&8*vsL@56IS}o$%)hpN`wk{*>8{f{!d^3V-Be!XZ>1H|VYaER(P3zQ5ZwWQS`&i`ZZp9^)Kq)Lq z&IbJ|eG7H$otE>&M_C0j>9swQi)qE_TJj@2*7Mn6+R0pSabJ3b&e~+8so!t%mhs8O zi!XQ+HB7(FP)OQ|*eg7^<7J}xObrulwes<0%ZC#*{vg|QNMVKs$2^i^H_3T~rpC^` zWhj4OMp!&s-T>1B$0(Dd#)o`%TB%kmpv!rxNcVu`zU1X>HrtcVT6ZGyQ+IH;TAK7j zLsIT7qgNLz8NOUQ=EGTX@MB8Mt%PKMR^0p7OIOCS%lKS2V?vF)n$h#x=fWfMI}EEv zF_3eRu)3x%AYYL66ZTor_IKc2hh^`tALv@=a}VVRA4>k%<~yW))|zfyD$h1yN1PKN1RVbyz~R^ZO7D*=f6V zK58F7!w@qMf{|&mlPzKQj!F|uq2Bn_Xyo~r=ep@`{vhv?A5Lj9brVb_MR0yhTKmjKnq3AEQ=nioi?mvB-qf5_l*>Q_8q=IK?t`SdX1CQAw?lbX@wj_zdT z2C)kaS!_c9x9*Q$aeWhri9M}v6r10-_cL)C(QUp(Yz;bM%~}|t`X$YGzTkF@YO6GPgq~gqU7w@OYc0^r|+~ zG5dW{u?(>>SxZ!LiY#9WUoJn_Jc-lW`H=3$U9C#P zX6iVnV1`h+^XqjMAL<`)TAfM+j<&X4x{{OO997jvS4s{C*PuF7T$|1^@v~R>&M}td zO%Swi9Y2`CGbYyo3s_eq%RR57AzLt_G)8kY`?h_iezVOTb9RW)kh^-!)V+|F-Fll7 zA+g#kP0?k3SVH29kQv1g_-O*x9yH^Q>M2Iaj+=^jv^jq)GQ+Rs&uOFkOF;DSrTfys zss*1lvMEDPS%afk)B>q^zKkze6M{;gz)L^PjIOQCurp6^-r%TGtO&nL)jEpp@XeM? zUe3u_s-K)dYVsa2_220lXC>XHe(f7B%}+o7HTSIBF&q_?aw}WZd3PvN`lHZCB@j@W z6HG@`3_Qe(Y^{uA4-C_}epV@3wYl!3Xu^_oZr{K8D*V9d5I1~5jrnxGd%U&~6V=_2yv2}2gh?>oQ}U;Yt5=rdqi!N(%akN1as zSaHGX!)x^L>CPx_r@xYpRq(KK@v_I#&D#@dojlqZvTNZkKI;b@)orpadkDqe{N4;^ zx+_6Gzq`hq7h-7);|XaqIjPjt*OS`OmfQ!3{`m`Uokz?3A_h27lv7~hvA)BN7I5vxTbnS)LJHu-7h zE_0nO)roM@H{A@9(GY3b?gy7)416q1`lelIy;K&rGFM12ZlC@4g#^f`s#k?~r`5dTOmo9w^=oP1=1w6-S zqh_lm=snQDnUUzwQHz78v_9Dkz1fEuV4kL*Nv_oMEniLGS}Tf;+m61|7c$ zK2OPC3$M3QC@FUrjWvnxK$>AQ<8TJd|Bk8QA|-aoru3lFY#Kw6t&qf29e&xy;|oo( zGKcvAOZ-`!h+#c!S$M&etOwja^1?zlB)Y)!EL@vtrcykl{0Vm>?AG8I7(n7#&zAy^ zxf>|@I6+l_&oTfX)T!RZCyss>XdFz`TCcn552eRS3rG*Hyk?ew85 zY(wZqMe=e8G_z4OGV1ARu775zNnrYqNUwh)Z!?rRd&pA16WLS|sdPx*W>rm`wCQ^e zvZ#)C*eKr@2Lh0NmbKBCm}KASuB~Y}yF%LC*asrEnVcdS4BCZddF&pNc7kEEsD1X> zLxvHuVGD^PlW9;#w@a70n9&PiiE3>1tc<$j{7;9;a$)xx1P zmdWmTZ8)yt{%tdHhM8gu%{B0}lvC>fUzBMNJ7EYY)9ApTM;&EQwa#4}Vb16e_bsvRoUbW*(yT+p;87CjD@#!$ReY_JlK0 zy6O*%j@-OeJt~(>+Jftn-mJaZfpW%34@eWX z6QHBpR7IQ|br3ewRDn9p)+u|%eG+kZEWD7ekN+{9KF3UhyIv2P>Rm4BFdMK$W=tlc zT&LE?=>Ri?Y(_9KFQw+xEuk)?7MGohyMIj!;DRarXv3b!Ej%o>_fQ#441=IeOwGd) z@66nFPw=yr-C^}495j<#UPvA)f`EC7zh<&vwwl9O zXIo;(<gQx~5CqH9 zfTJ=f^>)1J`q0@_5Imq-AqN!gk)ID?5?Jgzy!$!C{LxuLDVi@X3*kky8*7hW`%V7= z(7=3(U5~2mLc=iL?kncwMhX)6OuhYpzx+zzx3I#7jY`CDX<` z@9Egewr5N0&Ojc&A(i%S@VaIxvbDB{u@rk!I$>)ysy{dAcobkCBr6 zML-~+Q*=YC``+TVX?yf23T>k64bv6InrAeACvPrAd0=-mgEym8=bfYJ!sCcwn`QfL zmP^&JZ#Uh{c>Q=4m44M^n3Bl{OLf-h<9m}we2E#4=Lgm#V<7D$0|Xvx3=UbOEgv&< z)N+2k(6-zo0<;vdad9Y4%BWHbtq(gR%8S8@5m?hi>A8klHU&8GAT zGpBnunV2QHG8g}fi7>tUTrgl-G%5Ysl{KdHtX6K2UuS)#&XwA7mef^d9wS|i9mKxx z=<^Q=QG6-hu$Tn4`Y-UuR-jV7mA}sE=o+wa+IJsRj$ki^5J3L}=TT2PQk>Ee24$zC zoY{uzD&7WFWtd28Y^nV7d<={2n<2tQ`qwZ}inf-;Yl*#lHq@4J*M!#=HF*o{|*1% zN!^wD|D9O--|8crWcc4%ANlY3_uol||AYQyJ<0IDNcztyyms(yUYPaTc4cRy2Ap8y z0~-w#5Nv#4qk)YN!juFyS+LQ-#s@Y#(Pk^#kkE#NHYBtmp$!RbNN7Vs8xq=((1wII zB(x!+4GC>XXhT99655c^hJ-dGv>~Al32jJdLqZ!8+K|wOgf=9!A)yTkZAfTCLK_m= zkkE#NHYBtmp$!RbNN7Vs8xq=((1wIIB(x!+4GC>XXhT99655c^hJ-dGv>~Al32jJd zLqZ!8+K|wOgf=9!A)yTkZAfTCLK_m=kkE#NHYBtmp$!RbNN7Vs8xq=((1wIIB(x!+ z4GC>XXhT9968`@Q3A6sk1^+1q|3C5n7qb3m{(YA9zwqy!)V{yP;Qv>BgktdjoAr_Z zoqx3${LzA}|3iPUmV!TQZ-ov!I}Z8_(@V+udu12%@7CWdyL10;{k^h_{deo{m0jGw zTYs(}@!ze#S9S^iZvDNoOZ<21@0H!9f4BZ#*+hA_m4CPXUU9JUUrPMFvb*;0*550;>;G>3y|Uwwg8z?%f3NIr{JZt{%I@aBTYsTg&7UfHGmyY=_V qF7@B7zgKn-{@v26tp8IPi4zI_FDr-r-~9V5>;K{3JI|l*{(k}0=D4{4 literal 0 HcmV?d00001 diff --git a/thecannon/tests/test_jax_parity.py b/thecannon/tests/test_jax_parity.py new file mode 100644 index 0000000..1ae5167 --- /dev/null +++ b/thecannon/tests/test_jax_parity.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Parity tests for the JAX rewrite of The Cannon. + +Every numerical path is exercised against frozen "golden" reference outputs that +were captured from the original numpy/scipy implementation (see +``_capture_golden.py``). The golden file stores both the *inputs* and the +legacy *outputs*; here we re-run the same inputs through the (now JAX) code and +assert agreement. + +Tolerances reflect the nature of each computation: + +* exact-to-machine-precision for closed-form linear algebra / basis evaluation; +* tight (1e-6) for the convex, unregularized least-squares training and the + Levenberg-Marquardt test step; +* looser (1e-3) for the non-smooth lasso training, where we *additionally* + assert the JAX optimizer achieves an objective at least as good as the legacy + scipy optimizer (the strictly-convex lasso minimum is unique, so a lower + objective means a more accurate solution). +""" + +from __future__ import (division, print_function, absolute_import, + unicode_literals) + +import os +import pickle +import tempfile + +import numpy as np +import pytest + +import thecannon as tc +from thecannon import fitting, continuum +from thecannon.vectorizer.polynomial import PolynomialVectorizer + + +GOLDEN_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "golden", "golden.pkl") + + +@pytest.fixture(scope="module") +def golden(): + if not os.path.exists(GOLDEN_PATH): + pytest.skip("golden references not found; run " + "`python -m thecannon.tests._capture_golden` first") + with open(GOLDEN_PATH, "rb") as fp: + return pickle.load(fp) + + +@pytest.fixture(scope="module") +def vectorizer(golden): + m = golden["meta"] + return PolynomialVectorizer(label_names=m["label_names"], order=m["order"]) + + +@pytest.fixture(scope="module") +def trained_reg0(golden, vectorizer): + """A model trained with regularization=0 (reused across tests).""" + m = golden["meta"] + model = tc.CannonModel(m["labels"], m["flux"], m["ivar"], vectorizer, + dispersion=m["dispersion"], regularization=0) + model.train() + return model + + +# --------------------------------------------------------------------------- # +# Environment # +# --------------------------------------------------------------------------- # + +def test_float64_enabled(): + import jax + assert jax.config.jax_enable_x64 is True + + +def test_vectorizer_returns_float64(vectorizer, golden): + import numpy as np + out = np.asarray(vectorizer(golden["inputs"]["vec_labels_batch"])) + assert out.dtype == np.float64 + + +# --------------------------------------------------------------------------- # +# Vectorizer # +# --------------------------------------------------------------------------- # + +def test_get_label_vector_batch(vectorizer, golden): + out = np.asarray( + vectorizer.get_label_vector(golden["inputs"]["vec_labels_batch"])) + np.testing.assert_allclose( + out, golden["outputs"]["get_label_vector_batch"], atol=1e-12) + + +def test_get_label_vector_single(vectorizer, golden): + out = np.asarray( + vectorizer.get_label_vector(golden["inputs"]["vec_labels_single"])) + np.testing.assert_allclose( + out, golden["outputs"]["get_label_vector_single"], atol=1e-12) + + +def test_get_label_vector_derivative(vectorizer, golden): + out = np.asarray(vectorizer.get_label_vector_derivative( + golden["inputs"]["vec_labels_single"])) + expected = golden["outputs"]["get_label_vector_derivative"] + assert out.shape == expected.shape + np.testing.assert_allclose(out, expected, atol=1e-9) + + +# --------------------------------------------------------------------------- # +# Linear algebra # +# --------------------------------------------------------------------------- # + +def test_fit_theta_by_linalg(golden): + d = golden["inputs"]["linalg"] + theta, cov = fitting.fit_theta_by_linalg( + d["flux"], d["ivar"], d["s2"], d["design_matrix"]) + exp = golden["outputs"]["fit_theta_by_linalg"] + np.testing.assert_allclose(np.asarray(theta), exp["theta"], atol=1e-8) + np.testing.assert_allclose(np.asarray(cov), exp["cov"], atol=1e-8) + + +def test_fit_theta_by_linalg_singular(golden): + d = golden["inputs"]["linalg_singular"] + theta, cov = fitting.fit_theta_by_linalg( + d["flux"], d["ivar"], d["s2"], d["design_matrix"]) + exp = golden["outputs"]["fit_theta_by_linalg_singular"] + # Singular design matrix must fall back to the fiducial value [1, 0, ...]. + np.testing.assert_array_equal(np.asarray(theta), exp["theta"]) + assert np.all(~np.isfinite(np.asarray(cov))) + + +# --------------------------------------------------------------------------- # +# Objective pieces # +# --------------------------------------------------------------------------- # + +def test_chi_sq_value_and_gradient(golden): + c = golden["inputs"]["chi_sq"] + value, grad = fitting.chi_sq( + c["theta"], c["design_matrix"], c["flux"], c["ivar"], gradient=True) + exp = golden["outputs"]["chi_sq"] + np.testing.assert_allclose(float(value), exp["value"], atol=1e-9) + np.testing.assert_allclose(np.asarray(grad), exp["grad"], atol=1e-9) + + +def test_L1Norm_variation(golden): + value, grad = fitting.L1Norm_variation(golden["inputs"]["L1"]["theta"]) + exp = golden["outputs"]["L1Norm_variation"] + np.testing.assert_allclose(float(value), exp["value"], atol=1e-12) + np.testing.assert_allclose(np.asarray(grad), exp["grad"], atol=1e-12) + + +def test_pixel_objective(golden): + o = golden["inputs"]["objective"] + value, grad = fitting._pixel_objective_function_fixed_scatter( + o["theta"], o["design_matrix"], o["flux"], o["ivar"], + o["regularization"], gradient=True) + exp = golden["outputs"]["objective"] + np.testing.assert_allclose(float(value), exp["value"], atol=1e-9) + np.testing.assert_allclose(np.asarray(grad), exp["grad"], atol=1e-9) + + +# --------------------------------------------------------------------------- # +# Scatter fit # +# --------------------------------------------------------------------------- # + +def test_scatter_fit_positive(golden): + import jax.numpy as jnp + s = golden["inputs"]["scatter_hi"] + s2 = float(fitting._fit_scatter( + jnp.asarray(s["residuals_squared"]), jnp.asarray(s["ivar"]))) + np.testing.assert_allclose(s2, golden["outputs"]["scatter_hi"], atol=1e-3) + + +def test_scatter_fit_zero(golden): + import jax.numpy as jnp + s = golden["inputs"]["scatter_lo"] + s2 = float(fitting._fit_scatter( + jnp.asarray(s["residuals_squared"]), jnp.asarray(s["ivar"]))) + np.testing.assert_allclose(s2, golden["outputs"]["scatter_lo"], atol=1e-8) + + +# --------------------------------------------------------------------------- # +# Training # +# --------------------------------------------------------------------------- # + +def test_train_reg0_theta(trained_reg0, golden): + exp = golden["outputs"]["train_reg0"] + np.testing.assert_allclose(trained_reg0.theta, exp["theta"], atol=1e-6) + + +def test_train_reg0_s2(trained_reg0, golden): + exp = golden["outputs"]["train_reg0"] + np.testing.assert_allclose(trained_reg0.s2, exp["s2"], atol=1e-8) + + +def test_train_regularized_matches_within_tolerance(golden, vectorizer): + m = golden["meta"] + reg = golden["outputs"]["train_regR"]["regularization"] + model = tc.CannonModel(m["labels"], m["flux"], m["ivar"], vectorizer, + dispersion=m["dispersion"], regularization=reg) + theta, s2, _ = model.train() + np.testing.assert_allclose( + theta, golden["outputs"]["train_regR"]["theta"], atol=2e-3) + + +def test_train_regularized_objective_at_least_as_good(golden, vectorizer): + """The strictly-convex lasso minimum is unique; the JAX optimizer must + reach an objective no worse than the legacy scipy optimizer.""" + import jax.numpy as jnp + m = golden["meta"] + reg = golden["outputs"]["train_regR"]["regularization"] + model = tc.CannonModel(m["labels"], m["flux"], m["ivar"], vectorizer, + dispersion=m["dispersion"], regularization=reg) + theta, _, _ = model.train() + + scaled = (m["labels"] - m["fiducials"]) / m["scales"] + dm = np.asarray(vectorizer(scaled).T) + jax_obj = np.array([ + float(fitting._pixel_objective_function_fixed_scatter( + jnp.asarray(theta[p]), jnp.asarray(dm), + jnp.asarray(m["flux"][:, p]), jnp.asarray(m["ivar"][:, p]), reg, + gradient=False)) + for p in range(m["n_pixels"])]) + golden_obj = golden["outputs"]["train_regR_objective"] + # JAX objective must not exceed the legacy objective (allowing a tiny margin). + assert np.all(jax_obj <= golden_obj + 1e-8) + + +# --------------------------------------------------------------------------- # +# Prediction & inference # +# --------------------------------------------------------------------------- # + +def test_predict(trained_reg0, golden): + pred = np.asarray(trained_reg0(golden["inputs"]["predict_labels"])) + np.testing.assert_allclose( + pred, golden["outputs"]["predict_reg0"], atol=1e-9) + + +def test_test_step_labels(trained_reg0, golden): + t = golden["inputs"]["test"] + labels, cov, meta = trained_reg0.test(t["flux"], t["ivar"]) + np.testing.assert_allclose( + labels, golden["outputs"]["test_labels"], atol=1e-4) + + +def test_test_step_covariance(trained_reg0, golden): + t = golden["inputs"]["test"] + labels, cov, meta = trained_reg0.test(t["flux"], t["ivar"]) + np.testing.assert_allclose( + cov, golden["outputs"]["test_cov"], rtol=1e-5, atol=1e-12) + + +def test_test_step_recovers_true_labels(trained_reg0, golden): + t = golden["inputs"]["test"] + labels, cov, meta = trained_reg0.test(t["flux"], t["ivar"]) + rel = np.abs(labels - t["true_labels"]) / golden["meta"]["scales"] + assert np.all(rel < 0.05) + + +# --------------------------------------------------------------------------- # +# Continuum # +# --------------------------------------------------------------------------- # + +def test_continuum(golden): + c = golden["inputs"]["continuum"] + cont, meta = continuum.sines_and_cosines( + c["dispersion"], c["flux"], c["ivar"], c["continuum_pixels"], + L=c["L"], order=c["order"]) + np.testing.assert_allclose( + np.asarray(cont), golden["outputs"]["continuum"], atol=1e-10) + + +# --------------------------------------------------------------------------- # +# I/O round-trip & format compatibility # +# --------------------------------------------------------------------------- # + +def test_write_read_roundtrip(trained_reg0, golden): + fd, path = tempfile.mkstemp(suffix=".model") + os.close(fd) + try: + trained_reg0.write(path, include_training_set_spectra=True, + overwrite=True) + reloaded = tc.CannonModel.read(path) + # Trained attributes survive the round-trip as numpy arrays. + assert isinstance(reloaded.theta, np.ndarray) + np.testing.assert_array_equal(reloaded.theta, trained_reg0.theta) + np.testing.assert_array_equal(reloaded.s2, trained_reg0.s2) + # Predictions are identical after reload. + pred_a = np.asarray(trained_reg0(golden["inputs"]["predict_labels"])) + pred_b = np.asarray(reloaded(golden["inputs"]["predict_labels"])) + np.testing.assert_allclose(pred_a, pred_b, atol=1e-12) + finally: + os.remove(path) + + +# --------------------------------------------------------------------------- # +# Proximal-gradient lasso variant # +# --------------------------------------------------------------------------- # + +def test_proximal_matches_lbfgs_at_reg0(golden, vectorizer): + """At regularization=0 the proximal solver reduces to least squares and + must agree with the default path (and the golden least-squares result).""" + m = golden["meta"] + model = tc.CannonModel(m["labels"], m["flux"], m["ivar"], vectorizer, + dispersion=m["dispersion"], regularization=0) + theta, s2, _ = model.train(op_method="proximal") + np.testing.assert_allclose( + theta, golden["outputs"]["train_reg0"]["theta"], atol=1e-5) diff --git a/thecannon/utils.py b/thecannon/utils.py index 9b7b8bc..1177aa8 100644 --- a/thecannon/utils.py +++ b/thecannon/utils.py @@ -16,7 +16,10 @@ from six import string_types from tempfile import mkstemp from time import time -from collections import Iterable +try: + from collections.abc import Iterable +except ImportError: + from collections import Iterable from hashlib import md5 from multiprocessing.pool import Pool from multiprocessing import Lock, TimeoutError, Value @@ -37,6 +40,11 @@ class wrapper(object): A generic wrapper with a progressbar, which can be used either in serial or in parallel. + .. deprecated:: + The training and test steps are now vectorized with ``jax.vmap`` and no + longer use this multiprocessing wrapper. It is retained only for + backwards-compatible imports. + :param f: The function to apply. diff --git a/thecannon/vectorizer/polynomial.py b/thecannon/vectorizer/polynomial.py index 5c0c0c2..63e8c2c 100644 --- a/thecannon/vectorizer/polynomial.py +++ b/thecannon/vectorizer/polynomial.py @@ -11,6 +11,8 @@ __all__ = ["PolynomialVectorizer"] import numpy as np +import jax +import jax.numpy as jnp from collections import (Counter, OrderedDict) from itertools import combinations_with_replacement from six import string_types @@ -71,17 +73,17 @@ def get_label_vector(self, labels): two-dimensional array of `N` by `K` labels. """ - labels = np.atleast_2d(labels) + labels = jnp.atleast_2d(labels) if labels.ndim > 2: raise ValueError("labels must be a 1-d or 2-d array") - columns = [np.ones(labels.shape[0], dtype=float)] + columns = [jnp.ones(labels.shape[0], dtype=float)] for term in self.terms: column = 1. # This works; don't use np.multiply/np.product. for index, order in term: - column *= labels[:, index]**order + column = column * labels[:, index]**order columns.append(column) - return np.vstack(columns) + return jnp.vstack(columns) def get_label_vector_derivative(self, labels): @@ -96,33 +98,14 @@ def get_label_vector_derivative(self, labels): where `D` is the number of terms in the label vector description. """ - L, T = (len(labels), len(self.terms)) - - slicer = np.arange(L) - indices_used = np.zeros(L, dtype=bool) - - columns = np.ones((T + 1, L), dtype=float) - columns[0] = 0.0 # First theta derivative always zero. - - for t, term in enumerate(self.terms, start=1): - - indices_used[:] = False - - for index, order in term: - - dy = order * (labels[index]**(order - 1)) - y = labels[index]**order - - # If it's the index w.r.t. it, take derivative. - columns[t, index] *= dy - - # Otherwise, calculate as normal. - columns[t, slicer != index] *= y - indices_used[index] = True - - columns[t, ~indices_used] = 0 - - return columns + # The label vector derivative is the Jacobian of the (single-sample) + # label vector with respect to the labels. Rather than maintaining a + # hand-written derivative, we obtain it exactly via JAX forward-mode + # autodiff. The output shape is `(T + 1, L)` where `T = len(self.terms)` + # and `L = len(labels)`; the first row (the constant pivot term) is zero. + labels = jnp.asarray(labels, dtype=float) + single_label_vector = lambda l: self.get_label_vector(l)[:, 0] + return jax.jacfwd(single_label_vector)(labels) def get_human_readable_label_vector(self, mul="*", pow="^", bracket=False): From 8e7dad89e2a50c890a5036494d2eb6f7e833fde8 Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Mon, 8 Jun 2026 19:40:18 +1000 Subject: [PATCH 02/17] Add progress bar to train() and keep trained arrays as JAX Fit pixels in fixed-size, pad-aligned batches instead of one fused vmap. This drives a tqdm progress bar (the single fused call is opaque), compiles the optimizer once and reuses it across batches, and bounds peak memory. Pixels are independent so results are identical. Keep theta/s2 as JAX arrays end-to-end (block_until_ready instead of np.asarray, jnp.concatenate, single tolist() for metadata). Round-trip now preserves them as JAX arrays; update the test contract accordingly. Co-Authored-By: Claude Opus 4.8 (1M context) --- thecannon/model.py | 78 +++++++++++++++++++++++++++--- thecannon/tests/test_jax_parity.py | 6 ++- 2 files changed, 74 insertions(+), 10 deletions(-) diff --git a/thecannon/model.py b/thecannon/model.py index 3ee2fea..80b025c 100644 --- a/thecannon/model.py +++ b/thecannon/model.py @@ -594,7 +594,7 @@ def read(cls, path, **kwargs): def train(self, threads=None, op_method=None, op_strict=True, op_kwds=None, - **kwargs): + progressbar=True, batch_size=None, **kwargs): """ Train the model. @@ -611,6 +611,17 @@ def train(self, threads=None, op_method=None, op_strict=True, op_kwds=None, :param op_kwds: Keyword arguments to provide directly to the optimization function. + :param progressbar: [optional] + Display a progress bar while pixels are being fit (requires + ``tqdm``; silently disabled if it is not installed). + + :param batch_size: [optional] + The number of pixels to fit per vectorized batch. Pixels are fit + independently, so this does not change the result; it only controls + the granularity of the progress bar and bounds the peak memory and + compilation cost of the vmapped optimizer. Defaults to a value that + yields ~50 progress updates with reasonably large batches. + :returns: A three-length tuple containing the spectral coefficients `theta`, the squared scatter term at each pixel `s2`, and metadata related to @@ -685,14 +696,65 @@ def train(self, threads=None, op_method=None, op_strict=True, op_kwds=None, batched = jax.jit(jax.vmap( fitter, in_axes=(0, 0, 0, None, 0, 0))) - theta, s2, fopt = batched( - flux_PN, ivar_PN, init_stack, design_matrix, reg_P, column_mask) + # Pixels are fit independently, so we run them in fixed-size batches + # rather than one giant vmap. This lets us report progress (the single + # fused call is opaque), bounds peak memory, and keeps the compiled + # program small enough to compile once and reuse for every batch. + if batch_size is None: + batch_size = max(512, int(np.ceil(P / 50))) + batch_size = int(min(max(1, batch_size), P)) + + # Pad the pixel axis up to a whole number of equally-sized batches so + # every call to `batched` has identical shapes and XLA compiles it only + # once. The padding pixels are duplicates of the last real pixel and are + # discarded below. + n_batches = int(np.ceil(P / batch_size)) + pad = n_batches * batch_size - P + if pad: + edge = lambda a: jnp.broadcast_to(a[-1:], (pad,) + a.shape[1:]) + flux_PN = jnp.concatenate([flux_PN, edge(flux_PN)], axis=0) + ivar_PN = jnp.concatenate([ivar_PN, edge(ivar_PN)], axis=0) + init_stack = jnp.concatenate([init_stack, edge(init_stack)], axis=0) + reg_P = jnp.concatenate([reg_P, edge(reg_P)], axis=0) + column_mask = jnp.concatenate([column_mask, edge(column_mask)], + axis=0) - theta = np.asarray(theta) - s2 = np.asarray(s2) - fopt = np.asarray(fopt) - - meta = [dict(op_method=op_method, fopt=float(fopt[p]), + try: + from tqdm import tqdm + except ImportError: + tqdm = None + + pbar = None + if progressbar and tqdm is not None: + pbar = tqdm(total=P, desc="Training", unit="px") + + theta_batches, s2_batches, fopt_batches = [], [], [] + for b in range(n_batches): + sl = slice(b * batch_size, (b + 1) * batch_size) + th, s2b, fob = batched( + flux_PN[sl], ivar_PN[sl], init_stack[sl], design_matrix, + reg_P[sl], column_mask[sl]) + # Block on the device (keeping the results as JAX arrays) so the bar + # tracks real work rather than JAX's asynchronous dispatch. + jax.block_until_ready((th, s2b, fob)) + theta_batches.append(th) + s2_batches.append(s2b) + fopt_batches.append(fob) + if pbar is not None: + pbar.update(min((b + 1) * batch_size, P) - b * batch_size) + + if pbar is not None: + pbar.close() + + # Concatenate the batches and drop any padding pixels. The trained + # quantities are kept as JAX arrays end-to-end. + theta = jnp.concatenate(theta_batches, axis=0)[:P] + s2 = jnp.concatenate(s2_batches, axis=0)[:P] + fopt = jnp.concatenate(fopt_batches, axis=0)[:P] + + # A single host transfer for the per-pixel metadata. + fopt_values = fopt.tolist() + meta = [dict(op_method=op_method, fopt=fopt_values[p], maxiter=maxiter, tol=tol) for p in range(P)] self._theta, self._s2 = (theta, s2) diff --git a/thecannon/tests/test_jax_parity.py b/thecannon/tests/test_jax_parity.py index 1ae5167..ea5c498 100644 --- a/thecannon/tests/test_jax_parity.py +++ b/thecannon/tests/test_jax_parity.py @@ -281,8 +281,10 @@ def test_write_read_roundtrip(trained_reg0, golden): trained_reg0.write(path, include_training_set_spectra=True, overwrite=True) reloaded = tc.CannonModel.read(path) - # Trained attributes survive the round-trip as numpy arrays. - assert isinstance(reloaded.theta, np.ndarray) + # Trained attributes are JAX arrays and survive the round-trip as such. + import jax + assert isinstance(trained_reg0.theta, jax.Array) + assert isinstance(reloaded.theta, jax.Array) np.testing.assert_array_equal(reloaded.theta, trained_reg0.theta) np.testing.assert_array_equal(reloaded.s2, trained_reg0.s2) # Predictions are identical after reload. From 34dbe34e0a12a028ae5d1bbe6775da8154805b11 Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Mon, 8 Jun 2026 19:51:42 +1000 Subject: [PATCH 03/17] Skip the optimizer at regularization=0 via closed-form solve The unregularized, unbounded pixel objective is a convex quadratic, so its minimum is the normal-equations solution -- running L-BFGS on it is wasted work (verified to land on the same point to ~1e-13). Add make_pixel_closed_form() and route the default reg=0/no-bounds case through it, ~3x faster at P=4200 with identical results. Censored coefficients are pinned to zero via a unit diagonal on the Gram matrix. Also JIT the L-BFGS init computation (one fused kernel, built only when the optimizer runs) and block per batch only when the progress bar is shown so batches can overlap otherwise. Add a closed-form-vs-L-BFGS parity test covering censoring. Co-Authored-By: Claude Opus 4.8 (1M context) --- thecannon/fitting.py | 51 ++++++++++++++++++++ thecannon/model.py | 75 ++++++++++++++++++++---------- thecannon/tests/test_jax_parity.py | 41 ++++++++++++++++ 3 files changed, 143 insertions(+), 24 deletions(-) diff --git a/thecannon/fitting.py b/thecannon/fitting.py index 1eed9d5..826ded6 100644 --- a/thecannon/fitting.py +++ b/thecannon/fitting.py @@ -346,6 +346,57 @@ def smooth_objective(theta): return fit +def make_pixel_closed_form(): + """ + Build a pure, ``jax.vmap``-able function that fits a single pixel in closed + form. With no regularization and no box constraints the pixel objective is a + convex quadratic (weighted least squares), so its minimum is the + normal-equations solution and the iterative optimizer is unnecessary. + + :returns: + A function ``fit(flux, ivar, design_matrix, column_mask)`` returning + ``(theta, s2, fopt)`` with the same conventions as the optimizer built + by :func:`make_pixel_fitter` (censored coefficients are exactly zero; + information-free pixels return the fiducial theta with infinite scatter). + """ + + def fit(flux, ivar, design_matrix, column_mask): + + T = design_matrix.shape[1] + mask = column_mask.astype(design_matrix.dtype) + + # No information in this pixel -> fiducial theta with infinite scatter. + no_info = jnp.sum(ivar) < ivar.size + + # Zero out censored columns so they cannot contribute. + dm = design_matrix * mask[None, :] + + # Normal equations A^T C A theta = A^T C y (with C = diag(ivar)). A + # censored column is zeroed, leaving a zero row/column in the Gram + # matrix; adding a unit diagonal there makes it non-singular and forces + # that coefficient to exactly zero without coupling to the active ones. + CiA = dm * ivar[:, None] + gram = jnp.dot(dm.T, CiA) + jnp.diag(1.0 - mask) + rhs = jnp.dot(dm.T, flux * ivar) + theta = jnp.linalg.solve(gram, rhs) * mask + + fiducial = jnp.concatenate([jnp.ones(1), jnp.zeros(T - 1)]) + ok = jnp.all(jnp.isfinite(theta)) + theta = jnp.where(ok, theta, fiducial) + + residuals_squared = (flux - jnp.dot(theta, dm.T)) ** 2 + s2 = _fit_scatter(residuals_squared, ivar) + fopt = _chi_sq_only(theta, dm, flux, ivar) + + theta = jnp.where(no_info, fiducial, theta) + s2 = jnp.where(no_info, jnp.inf, s2) + fopt = jnp.where(no_info, jnp.nan, fopt) + + return (theta, s2, fopt) + + return fit + + def fit_pixel_fixed_scatter(flux, ivar, initial_thetas, design_matrix, regularization, censoring_mask, **kwargs): """ diff --git a/thecannon/model.py b/thecannon/model.py index 80b025c..feb754a 100644 --- a/thecannon/model.py +++ b/thecannon/model.py @@ -667,18 +667,6 @@ def train(self, threads=None, op_method=None, op_strict=True, op_kwds=None, fiducial = jnp.concatenate([jnp.ones(1), jnp.zeros(T - 1)]) - # Initial theta guesses for every pixel: a linear-algebra estimate and - # the fiducial value. The regularized objective is convex, so the - # optimum is independent of the starting point; these only set where the - # optimizer begins. - linalg_theta = jax.vmap( - lambda f, i: fitting.fit_theta_by_linalg(f, i, 0.0, design_matrix)[0] - )(flux_PN, ivar_PN) # (P, T) - finite = jnp.all(jnp.isfinite(linalg_theta), axis=1, keepdims=True) - linalg_theta = jnp.where(finite, linalg_theta, fiducial[None, :]) - init_stack = jnp.stack( - [linalg_theta, jnp.broadcast_to(fiducial, (P, T))], axis=1) # (P,2,T) - # Per-pixel column mask: True keeps the coefficient, False censors it. if self.censors and len(set(self.censors).intersection( self.vectorizer.label_names)) > 0: @@ -691,10 +679,48 @@ def train(self, threads=None, op_method=None, op_strict=True, op_kwds=None, reg = 0.0 if self.regularization is None else self.regularization reg_P = jnp.broadcast_to(jnp.asarray(reg, dtype=float), (P,)) - fitter = fitting.make_pixel_fitter( - op_method=op_method, maxiter=maxiter, tol=tol, bounds=bounds) - batched = jax.jit(jax.vmap( - fitter, in_axes=(0, 0, 0, None, 0, 0))) + # With no regularization and no box constraints the pixel objective is a + # convex quadratic, so its minimum is the closed-form least-squares + # solution -- there is no need to run the iterative optimizer at all. + closed_form = bounds is None and ( + self.regularization is None + or (np.ndim(self.regularization) == 0 + and float(self.regularization) == 0.0)) + + if closed_form: + fitter = fitting.make_pixel_closed_form() + batched = jax.jit(jax.vmap(fitter, in_axes=(0, 0, None, 0))) + + def run_batch(sl): + return batched(flux_PN[sl], ivar_PN[sl], design_matrix, + column_mask[sl]) + else: + # Initial theta guesses for every pixel: a linear-algebra estimate + # and the fiducial value. The regularized objective is convex, so + # the optimum is independent of the starting point; these only set + # where the optimizer begins. Computed in a single jitted (hence + # fused) call rather than op-by-op. + @jax.jit + def _initial_stack(flux_PN, ivar_PN): + linalg_theta = jax.vmap( + lambda f, i: fitting.fit_theta_by_linalg( + f, i, 0.0, design_matrix)[0])(flux_PN, ivar_PN) + finite = jnp.all( + jnp.isfinite(linalg_theta), axis=1, keepdims=True) + linalg_theta = jnp.where(finite, linalg_theta, fiducial[None, :]) + return jnp.stack( + [linalg_theta, jnp.broadcast_to(fiducial, (P, T))], axis=1) + + init_stack = _initial_stack(flux_PN, ivar_PN) # (P, 2, T) + + fitter = fitting.make_pixel_fitter( + op_method=op_method, maxiter=maxiter, tol=tol, bounds=bounds) + batched = jax.jit(jax.vmap( + fitter, in_axes=(0, 0, 0, None, 0, 0))) + + def run_batch(sl): + return batched(flux_PN[sl], ivar_PN[sl], init_stack[sl], + design_matrix, reg_P[sl], column_mask[sl]) # Pixels are fit independently, so we run them in fixed-size batches # rather than one giant vmap. This lets us report progress (the single @@ -714,10 +740,12 @@ def train(self, threads=None, op_method=None, op_strict=True, op_kwds=None, edge = lambda a: jnp.broadcast_to(a[-1:], (pad,) + a.shape[1:]) flux_PN = jnp.concatenate([flux_PN, edge(flux_PN)], axis=0) ivar_PN = jnp.concatenate([ivar_PN, edge(ivar_PN)], axis=0) - init_stack = jnp.concatenate([init_stack, edge(init_stack)], axis=0) - reg_P = jnp.concatenate([reg_P, edge(reg_P)], axis=0) column_mask = jnp.concatenate([column_mask, edge(column_mask)], axis=0) + if not closed_form: + init_stack = jnp.concatenate([init_stack, edge(init_stack)], + axis=0) + reg_P = jnp.concatenate([reg_P, edge(reg_P)], axis=0) try: from tqdm import tqdm @@ -731,16 +759,15 @@ def train(self, threads=None, op_method=None, op_strict=True, op_kwds=None, theta_batches, s2_batches, fopt_batches = [], [], [] for b in range(n_batches): sl = slice(b * batch_size, (b + 1) * batch_size) - th, s2b, fob = batched( - flux_PN[sl], ivar_PN[sl], init_stack[sl], design_matrix, - reg_P[sl], column_mask[sl]) - # Block on the device (keeping the results as JAX arrays) so the bar - # tracks real work rather than JAX's asynchronous dispatch. - jax.block_until_ready((th, s2b, fob)) + th, s2b, fob = run_batch(sl) theta_batches.append(th) s2_batches.append(s2b) fopt_batches.append(fob) if pbar is not None: + # Block on the device so the bar tracks real work rather than + # JAX's asynchronous dispatch. With no bar we let the batches + # dispatch asynchronously and overlap. + jax.block_until_ready((th, s2b, fob)) pbar.update(min((b + 1) * batch_size, P) - b * batch_size) if pbar is not None: diff --git a/thecannon/tests/test_jax_parity.py b/thecannon/tests/test_jax_parity.py index ea5c498..660282a 100644 --- a/thecannon/tests/test_jax_parity.py +++ b/thecannon/tests/test_jax_parity.py @@ -308,3 +308,44 @@ def test_proximal_matches_lbfgs_at_reg0(golden, vectorizer): theta, s2, _ = model.train(op_method="proximal") np.testing.assert_allclose( theta, golden["outputs"]["train_reg0"]["theta"], atol=1e-5) + + +# --------------------------------------------------------------------------- # +# Closed-form fast path (regularization == 0) # +# --------------------------------------------------------------------------- # + +def test_closed_form_matches_lbfgs_with_censoring(golden, vectorizer): + """The closed-form fast path used at regularization=0 must reproduce the + iterative L-BFGS optimum, including when coefficients are censored.""" + import jax + import jax.numpy as jnp + from thecannon import fitting + + m = golden["meta"] + model = tc.CannonModel(m["labels"], m["flux"], m["ivar"], vectorizer, + dispersion=m["dispersion"], regularization=0) + dm = jnp.asarray(model.design_matrix) + flux_PN = jnp.asarray(model.training_set_flux.T) + ivar_PN = jnp.asarray(model.training_set_ivar.T) + P, T = flux_PN.shape[0], dm.shape[1] + + # Censor three (non-continuum) coefficients on every pixel. + rng = np.random.default_rng(1) + mask = np.ones((P, T), bool) + for p in range(P): + mask[p, rng.choice(np.arange(1, T), size=3, replace=False)] = False + mask = jnp.asarray(mask) + + fiducial = jnp.concatenate([jnp.ones(1), jnp.zeros(T - 1)]) + init = jnp.broadcast_to(jnp.stack([fiducial, fiducial]), (P, 2, T)) + reg = jnp.zeros(P) + + cf = jax.vmap(fitting.make_pixel_closed_form(), in_axes=(0, 0, None, 0)) + lb = jax.vmap(fitting.make_pixel_fitter(op_method="l_bfgs_b"), + in_axes=(0, 0, 0, None, 0, 0)) + theta_cf, _, _ = cf(flux_PN, ivar_PN, dm, mask) + theta_lb, _, _ = lb(flux_PN, ivar_PN, init, dm, reg, mask) + + np.testing.assert_allclose(theta_cf, theta_lb, atol=1e-8) + # Censored coefficients are exactly zero. + assert bool(jnp.all(theta_cf[~np.asarray(mask)] == 0.0)) From 2b857fc3c5d498a7f46be68ff16db6d9fa4151c1 Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Mon, 8 Jun 2026 22:41:21 +1000 Subject: [PATCH 04/17] Add cross-validated Cannon sweep with offline W&B and PBS jobs scripts/sweep_cannon.py: k-fold cross-validation over a grid of label sets, polynomial orders, regularization strengths and censoring schemes. Reports per-label bias/scatter, median r_chi_sq and convex-hull coverage, writes a tidy CSV, and (optionally) logs one offline W&B run per grid point -- config + scalar metrics + residual histograms + a per-run one-to-one spread figure -- plus a sweep-level summary run with a results table and a spread-vs-parameters bar chart. scripts/sweep.pbs / sync_wandb.pbs: run the sweep offline on a compute node, then spawn a copyq (data-mover) job that `wandb sync`s the offline runs to the cloud. Interpreter is selected via PYTHON_BIN. .gitignore: ignore wandb/ run data and sweep_*results.csv. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 4 + scripts/sweep.pbs | 61 ++++ scripts/sweep_cannon.py | 603 ++++++++++++++++++++++++++++++++++++++++ scripts/sync_wandb.pbs | 50 ++++ 4 files changed, 718 insertions(+) create mode 100755 scripts/sweep.pbs create mode 100644 scripts/sweep_cannon.py create mode 100755 scripts/sync_wandb.pbs diff --git a/.gitignore b/.gitignore index 82e95ea..87d22f0 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ sandbox_*.py build/ dist/ *.egg-info/ + +# Cannon sweep / W&B artifacts +wandb/ +sweep_*results.csv diff --git a/scripts/sweep.pbs b/scripts/sweep.pbs new file mode 100755 index 0000000..8b25ddb --- /dev/null +++ b/scripts/sweep.pbs @@ -0,0 +1,61 @@ +#!/bin/bash +# =========================================================================== +# The Cannon hyper-parameter sweep -- main PBS job. +# +# Runs scripts/sweep_cannon.py on a compute node (no outbound network), which +# writes one *offline* Weights & Biases run per grid point under ./wandb. When +# the sweep finishes it submits a second job to the `copyq` queue -- the NCI +# Gadi data-mover queue, which has internet access -- to `wandb sync` those +# offline runs to the cloud (see scripts/sync_wandb.pbs). +# +# On non-Gadi PBS sites, point the sync job at whatever queue is allowed to +# reach the internet. +# +# Submit with: qsub scripts/sweep.pbs +# =========================================================================== +#PBS -N cannon-sweep +#PBS -P REPLACE_WITH_PROJECT +#PBS -q normal +#PBS -l ncpus=16 +#PBS -l mem=64GB +#PBS -l walltime=04:00:00 +#PBS -l storage=scratch/REPLACE_WITH_PROJECT+gdata/REPLACE_WITH_PROJECT +#PBS -l wd +#PBS -j oe + +set -uo pipefail + +# ---- user configuration --------------------------------------------------- +PROJECT_DIR="${PBS_O_WORKDIR:-$PWD}" +# Absolute path to the Python interpreter (conda/venv) that has thecannon + +# wandb installed, e.g. /scratch/REPLACE_WITH_PROJECT/envs/thecannon/bin/python +PYTHON_BIN="python" +WANDB_PROJECT="cannon-sweep" +# The command that runs the sweep. Replace `--demo` with your own driver: a +# Python script that loads your spectra/labels and calls +# scripts.sweep_cannon.sweep(...) with wandb_project=WANDB_PROJECT. +SWEEP_CMD="${PYTHON_BIN} -m scripts.sweep_cannon --demo --wandb-project ${WANDB_PROJECT} --wandb-mode offline -v" +# --------------------------------------------------------------------------- + +cd "${PROJECT_DIR}" + +# Keep W&B fully offline on the compute node; runs are uploaded later on copyq. +export WANDB_MODE=offline +export WANDB_DIR="${PROJECT_DIR}/wandb" +mkdir -p "${WANDB_DIR}" + +echo "[$(date)] starting sweep on $(hostname)" +${SWEEP_CMD} +sweep_rc=$? +echo "[$(date)] sweep finished with exit code ${sweep_rc}" + +# Spawn the copyq job to upload the offline runs (it has external network +# access). Whatever offline runs exist are synced, even if the sweep partially +# failed, so we submit regardless of sweep_rc. +echo "[$(date)] submitting copyq sync job" +sync_job=$(qsub \ + -v "PROJECT_DIR=${PROJECT_DIR},PYTHON_BIN=${PYTHON_BIN},WANDB_DIR=${WANDB_DIR}" \ + "${PROJECT_DIR}/scripts/sync_wandb.pbs") +echo "[$(date)] submitted sync job: ${sync_job}" + +exit ${sweep_rc} diff --git a/scripts/sweep_cannon.py b/scripts/sweep_cannon.py new file mode 100644 index 0000000..f0bfc49 --- /dev/null +++ b/scripts/sweep_cannon.py @@ -0,0 +1,603 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Cross-validated hyper-parameter sweep for The Cannon. + +This explores the knobs that matter for a Cannon model -- the label set, the +polynomial order of the vectorizer, the L1 regularization strength, and +(optionally) pixel censoring -- and scores each combination by k-fold +cross-validation. For every grid point it trains on the training folds, runs the +test step on the held-out fold, and reports how well the held-out labels are +recovered (bias and scatter against the reference labels) together with the +median reduced chi-squared and the fraction of held-out stars that fall inside +the convex hull of the training labels. + +The same fold assignment (fixed RNG seed) is reused for every grid point so the +comparison between hyper-parameters is not confounded by split noise. + +Usage +----- +As a module (the intended entry point -- real data comes in too many formats for +a one-size CLI):: + + from scripts.sweep_cannon import sweep + + # `labels` is a mapping {label_name: (N,) array}; a numpy structured/record + # array or an astropy Table works directly. `flux`/`ivar` are (N, P) arrays + # of pseudo-continuum-normalized flux and inverse variance. + results = sweep( + labels, flux, ivar, dispersion, + label_sets=[("TEFF", "LOGG", "FE_H"), + ("TEFF", "LOGG", "FE_H", "MG_FE")], + orders=[1, 2], + regularizations=[0.0, 1e2, 1e3, 1e4], + n_splits=5, + output="sweep_results.csv", + ) + +For a quick end-to-end smoke test on the bundled golden data:: + + python -m scripts.sweep_cannon --demo +""" + +from __future__ import (division, print_function, absolute_import, + unicode_literals) + +import argparse +import itertools +import logging + +import numpy as np + +from thecannon.model import CannonModel +from thecannon.vectorizer.polynomial import PolynomialVectorizer + +try: + import wandb +except ImportError: + wandb = None + +try: + import matplotlib + matplotlib.use("Agg") # headless: the sweep only saves figures + import matplotlib.pyplot as plt +except ImportError: + plt = None + +logger = logging.getLogger("thecannon.sweep") + + +# --------------------------------------------------------------------------- # +# Helpers # +# --------------------------------------------------------------------------- # + +def _label_matrix(labels, names): + """ + Return an ``(N, len(names))`` float array for the requested label names, + accepting a mapping (dict / structured array / astropy Table) keyed by name. + """ + try: + columns = [np.asarray(labels[name], dtype=float) for name in names] + except (KeyError, IndexError, ValueError) as exc: + raise KeyError( + "could not extract labels {0} from the provided container " + "({1})".format(list(names), exc)) + return np.vstack(columns).T + + +def _kfold_indices(n, n_splits, seed): + """ Yield ``(train_idx, test_idx)`` pairs for shuffled k-fold splits. """ + if n_splits < 2: + raise ValueError("n_splits must be at least 2") + if n_splits > n: + raise ValueError( + "n_splits ({0}) cannot exceed the number of stars ({1})" + .format(n_splits, n)) + + rng = np.random.default_rng(seed) + order = rng.permutation(n) + folds = np.array_split(order, n_splits) + for k in range(n_splits): + test_idx = folds[k] + train_idx = np.concatenate( + [folds[j] for j in range(n_splits) if j != k]) + yield np.sort(train_idx), np.sort(test_idx) + + +# --------------------------------------------------------------------------- # +# Cross-validation for a single hyper-parameter combination # +# --------------------------------------------------------------------------- # + +def cross_validate(label_array, flux, ivar, dispersion, label_names, order, + regularization, censors=None, n_splits=5, seed=0, train_kwds=None, + test_kwds=None): + """ + Run k-fold cross-validation for one hyper-parameter combination. + + :param label_array: + An ``(N, K)`` array of reference labels aligned with ``label_names``. + + :param flux, ivar: + ``(N, P)`` arrays of normalized flux and inverse variance. + + :param dispersion: + The ``(P,)`` dispersion array (may be ``None``). + + :param label_names: + The labels to fit (defines the vectorizer and the columns used). + + :param order: + The polynomial order of the vectorizer. + + :param regularization: + The L1 regularization strength (``0`` uses the closed-form fast path). + + :param censors: [optional] + A :class:`thecannon.censoring.Censors` object (or ``None``). + + :returns: + A dict of held-out arrays: ``recovered`` ``(N, K)``, ``residual`` + ``(N, K)``, ``r_chi_sq`` ``(N,)`` and ``in_hull`` ``(N,)``. + """ + + N, K = label_array.shape + recovered = np.full((N, K), np.nan, dtype=float) + r_chi_sq = np.full(N, np.nan, dtype=float) + in_hull = np.zeros(N, dtype=bool) + + for train_idx, test_idx in _kfold_indices(N, n_splits, seed): + + vectorizer = PolynomialVectorizer(label_names=label_names, order=order) + model = CannonModel( + label_array[train_idx], flux[train_idx], ivar[train_idx], + vectorizer, dispersion=dispersion, regularization=regularization, + censors=censors) + model.train(progressbar=False, **(train_kwds or {})) + + op_labels, _, meta = model.test( + flux[test_idx], ivar[test_idx], **(test_kwds or {})) + + recovered[test_idx] = op_labels + r_chi_sq[test_idx] = [m["r_chi_sq"] for m in meta] + + # Convex-hull membership flags held-out stars that require extrapolation + # (Delaunay can fail for degenerate / high-dimensional label sets). + try: + in_hull[test_idx] = model.in_convex_hull(label_array[test_idx]) + except Exception as exc: # pragma: no cover + logger.debug("convex-hull test skipped: %s", exc) + in_hull[test_idx] = False + + return dict(recovered=recovered, residual=recovered - label_array, + r_chi_sq=r_chi_sq, in_hull=in_hull) + + +def _summarize(cv, label_names): + """ Reduce the per-star cross-validation arrays to a tidy metric row. """ + residual = cv["residual"] + row = {} + scatters = [] + for i, name in enumerate(label_names): + r = residual[:, i] + row["bias_{0}".format(name)] = float(np.nanmean(r)) + row["scatter_{0}".format(name)] = float(np.nanstd(r)) + row["rmse_{0}".format(name)] = float(np.sqrt(np.nanmean(r ** 2))) + scatters.append(row["scatter_{0}".format(name)]) + row["mean_scatter"] = float(np.mean(scatters)) + row["median_r_chi_sq"] = float(np.nanmedian(cv["r_chi_sq"])) + row["frac_in_hull"] = float(np.mean(cv["in_hull"])) + row["n_test"] = int(np.sum(np.isfinite(cv["recovered"][:, 0]))) + return row + + +# --------------------------------------------------------------------------- # +# Weights & Biases logging (optional, offline by default) # +# --------------------------------------------------------------------------- # + +def _init_wandb_run(project, entity, group, mode, run_dir, name, config): + """ + Start a single (offline by default) W&B run for one grid point. Returns the + run object, or ``None`` if W&B is unavailable or initialization fails. + """ + if wandb is None: + return None + try: + return wandb.init( + project=project, entity=entity, group=group, name=name, + mode=mode, dir=run_dir, config=config, reinit=True, + settings=wandb.Settings(silent=True)) + except Exception as exc: # pragma: no cover + logger.warning("wandb.init failed (%s); continuing without logging", + exc) + return None + + +def _log_wandb_run(run, row, cv, label_names): + """ Log scalar metrics and per-label residual histograms to a W&B run. """ + if run is None: + return + # Scalar metrics only (the string/config fields already live in run.config). + metrics = {k: v for k, v in row.items() + if isinstance(v, (int, float)) and not isinstance(v, bool)} + + if cv is not None: + residual = cv["residual"] + for i, name in enumerate(label_names): + finite = residual[:, i][np.isfinite(residual[:, i])] + if finite.size: + try: + metrics["residual_hist/{0}".format(name)] = \ + wandb.Histogram(finite) + except Exception: # pragma: no cover + pass + + run.log(metrics) + run.summary["status"] = row.get("status", "ok") + + +# --------------------------------------------------------------------------- # +# Spread figures # +# --------------------------------------------------------------------------- # + +def spread_figure(recovered, reference, label_names, title=None): + """ + Build a one-to-one (recovered vs reference) figure with one panel per label, + annotated with the held-out bias and scatter (spread). Returns a matplotlib + ``Figure`` (or ``None`` if matplotlib is unavailable). + """ + if plt is None: + return None + + K = len(label_names) + ncols = min(K, 3) + nrows = int(np.ceil(K / ncols)) + fig, axes = plt.subplots(nrows, ncols, squeeze=False, + figsize=(3.4 * ncols, 3.2 * nrows)) + axes = axes.flatten() + + for i, name in enumerate(label_names): + ax = axes[i] + x, y = reference[:, i], recovered[:, i] + good = np.isfinite(x) & np.isfinite(y) + ax.scatter(x[good], y[good], s=6, c="k", alpha=0.5) + + if good.any(): + lo = float(min(x[good].min(), y[good].min())) + hi = float(max(x[good].max(), y[good].max())) + else: + lo, hi = 0.0, 1.0 + ax.plot([lo, hi], [lo, hi], ":", c="#666666", zorder=-1) + ax.set_xlim(lo, hi) + ax.set_ylim(lo, hi) + + resid = (y - x)[good] + bias = float(np.mean(resid)) if resid.size else float("nan") + scatter = float(np.std(resid)) if resid.size else float("nan") + ax.set_title("{0}\nbias={1:.3g} sigma={2:.3g}".format( + name, bias, scatter), fontsize=9) + ax.set_xlabel("reference") + ax.set_ylabel("recovered") + + for j in range(K, len(axes)): + axes[j].set_visible(False) + if title: + fig.suptitle(title, fontsize=10) + fig.tight_layout() + return fig + + +def parameter_spread_figure(rows): + """ + Build a bar chart of the mean held-out scatter for every (successful) grid + point, so the spread can be compared across the swept parameters. Returns a + matplotlib ``Figure`` (or ``None``). + """ + if plt is None: + return None + ok = [r for r in rows if r.get("status") == "ok" and "mean_scatter" in r] + if not ok: + return None + + names = ["{0}|o{1}|reg{2:g}|{3}".format( + r["label_set"], r["order"], r["regularization"], r["censor"]) + for r in ok] + values = [r["mean_scatter"] for r in ok] + + fig, ax = plt.subplots(figsize=(max(6.0, 0.55 * len(ok)), 4.0)) + ax.bar(range(len(ok)), values, color="#4C72B0") + ax.set_xticks(range(len(ok))) + ax.set_xticklabels(names, rotation=90, fontsize=7) + ax.set_ylabel("mean held-out scatter") + ax.set_title("Label spread across swept parameters") + fig.tight_layout() + return fig + + +def _log_wandb_summary(project, entity, group, mode, run_dir, rows): + """ + Log a sweep-level summary run: a table of every grid point's metrics and a + bar chart comparing the spread across parameters. + """ + run = _init_wandb_run(project, entity, group, mode, run_dir, + name="summary", config=dict(n_combos=len(rows))) + if run is None: + return + + columns = [] + for row in rows: + for key in row: + if key not in columns: + columns.append(key) + data = [[row.get(c, None) for c in columns] for row in rows] + try: + run.log({"results": wandb.Table(columns=columns, data=data)}) + except Exception as exc: # pragma: no cover + logger.warning("could not log summary table: %s", exc) + + fig = parameter_spread_figure(rows) + if fig is not None: + run.log({"spread/by_parameters": wandb.Image(fig)}) + plt.close(fig) + + run.finish() + + +# --------------------------------------------------------------------------- # +# The grid sweep # +# --------------------------------------------------------------------------- # + +def sweep(labels, flux, ivar, dispersion, label_sets, orders, regularizations, + censor_factories=None, n_splits=5, seed=0, output=None, train_kwds=None, + test_kwds=None, wandb_project=None, wandb_entity=None, wandb_group=None, + wandb_mode="offline", wandb_dir=None): + """ + Cross-validate The Cannon over a grid of label sets, polynomial orders, + regularization strengths and (optionally) censoring schemes. + + :param labels: + A mapping of ``{label_name: (N,) array}`` (dict, numpy structured array, + or astropy Table) covering every name used in ``label_sets``. + + :param flux, ivar: + ``(N, P)`` arrays of normalized flux and inverse variance. + + :param dispersion: + The ``(P,)`` dispersion array, or ``None``. + + :param label_sets: + An iterable of label-name tuples to try as the model's label set. + + :param orders: + An iterable of polynomial orders for the vectorizer. + + :param regularizations: + An iterable of L1 regularization strengths. + + :param censor_factories: [optional] + A mapping ``{name: factory}`` where ``factory(label_names, dispersion, + n_pixels)`` returns a :class:`~thecannon.censoring.Censors` object (or + ``None`` for no censoring). Defaults to a single ``{"none": None}`` + scheme. A factory is used (rather than a fixed object) because the + censor mask depends on the label set being tried. + + :param n_splits: + The number of cross-validation folds. + + :param seed: + RNG seed for the (shared) fold assignment. + + :param output: [optional] + If given, write the tidy results to this CSV path. + + :param wandb_project: [optional] + If given (and :mod:`wandb` is installed), log one W&B run per grid point + under this project. Each run records the hyper-parameters as its config + and the cross-validation metrics (plus per-label residual histograms). + + :param wandb_entity, wandb_group: [optional] + The W&B entity (team/user) and the group used to tie the sweep's runs + together in the UI. + + :param wandb_mode: [optional] + The W&B run mode. Defaults to ``"offline"`` -- runs are written locally + (no network/login needed) and can later be uploaded with ``wandb sync``. + + :param wandb_dir: [optional] + The directory under which offline run data is written (defaults to + ``./wandb``). + + :returns: + A list of result-row dicts (one per grid point). If :mod:`pandas` is + installed, a ``DataFrame`` is returned instead. + """ + + if wandb_project and wandb is None: + logger.warning("wandb_project given but wandb is not installed; " + "install it with `pip install wandb` to enable logging") + + if censor_factories is None: + censor_factories = {"none": None} + + n_pixels = np.atleast_2d(flux).shape[1] + grid = list(itertools.product( + [tuple(s) for s in label_sets], list(orders), list(regularizations), + list(censor_factories.items()))) + + logger.info("Sweeping %d combinations (%d folds each)", len(grid), n_splits) + + rows = [] + for n, (label_set, order, reg, (censor_name, factory)) in enumerate(grid, 1): + + config = dict(label_set="+".join(label_set), n_labels=len(label_set), + order=order, regularization=reg, censor=censor_name, + n_splits=n_splits, seed=seed) + row = dict(label_set=config["label_set"], n_labels=len(label_set), + order=order, regularization=reg, censor=censor_name) + + run = None + if wandb_project: + run = _init_wandb_run( + wandb_project, wandb_entity, wandb_group, wandb_mode, wandb_dir, + name="{0}|o{1}|reg{2:g}|{3}".format( + config["label_set"], order, reg, censor_name), + config=config) + + cv = None + try: + label_array = _label_matrix(labels, label_set) + censors = factory(label_set, dispersion, n_pixels) \ + if callable(factory) else factory + + cv = cross_validate( + label_array, np.atleast_2d(flux), np.atleast_2d(ivar), + dispersion, label_set, order, reg, censors=censors, + n_splits=n_splits, seed=seed, train_kwds=train_kwds, + test_kwds=test_kwds) + + row.update(_summarize(cv, label_set)) + row["status"] = "ok" + logger.info( + "[%d/%d] %s order=%d reg=%g censor=%s -> " + "mean_scatter=%.4f r_chi_sq=%.2f", n, len(grid), + row["label_set"], order, reg, censor_name, + row["mean_scatter"], row["median_r_chi_sq"]) + + # Per-run figure: the spread (recovered vs reference) for all labels. + if run is not None: + fig = spread_figure( + cv["recovered"], label_array, label_set, + title="{0} | order={1} | reg={2:g} | {3}".format( + config["label_set"], order, reg, censor_name)) + if fig is not None: + run.log({"spread/one_to_one": wandb.Image(fig)}) + plt.close(fig) + + except Exception as exc: # keep the sweep alive + row["status"] = "error: {0}".format(exc) + logger.warning("[%d/%d] %s order=%d reg=%g censor=%s FAILED: %s", + n, len(grid), row["label_set"], order, reg, + censor_name, exc) + finally: + if run is not None: + _log_wandb_run(run, row, cv, label_set) + run.finish() + rows.append(row) + + # Sweep-level summary run: a results table plus the spread-vs-parameters + # comparison, so the whole sweep can be browsed from one W&B run. + if wandb_project and wandb is not None: + _log_wandb_summary(wandb_project, wandb_entity, wandb_group, wandb_mode, + wandb_dir, rows) + + if output: + _write_csv(rows, output) + logger.info("Wrote %d rows to %s", len(rows), output) + + try: + import pandas as pd + return pd.DataFrame(rows) + except ImportError: + return rows + + +def _write_csv(rows, path): + """ Write the result rows to CSV with a stable, union-of-keys header. """ + import csv + fieldnames = [] + for row in rows: + for key in row: + if key not in fieldnames: + fieldnames.append(key) + with open(path, "w", newline="") as fp: + writer = csv.DictWriter(fp, fieldnames=fieldnames) + writer.writeheader() + for row in rows: + writer.writerow(row) + + +# --------------------------------------------------------------------------- # +# Demo / smoke test # +# --------------------------------------------------------------------------- # + +def _demo(wandb_project=None, wandb_mode="offline"): + """ Run a tiny sweep on the bundled golden data to verify the pipeline. """ + import os + import pickle + + golden_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", "thecannon", "tests", "golden", "golden.pkl") + + if os.path.exists(golden_path): + with open(golden_path, "rb") as fp: + meta = pickle.load(fp)["meta"] + names = list(meta["label_names"]) + label_array = np.atleast_2d(np.asarray(meta["labels"], dtype=float)) + labels = {name: label_array[:, i] for i, name in enumerate(names)} + flux, ivar, dispersion = meta["flux"], meta["ivar"], meta["dispersion"] + print("Loaded golden data: {0} stars, {1} pixels, labels {2}".format( + flux.shape[0], flux.shape[1], names)) + else: + print("Golden data not found; synthesizing a toy data set.") + rng = np.random.default_rng(0) + names = ["A", "B", "C"] + N, P = 80, 120 + label_array = rng.normal(size=(N, 3)) + labels = {name: label_array[:, i] for i, name in enumerate(names)} + dispersion = np.linspace(0, 1, P) + coeff = rng.normal(size=(P, 4)) + clean = (coeff[:, 0] + label_array @ coeff[:, 1:].T).T + flux = 1.0 + 0.1 * clean + rng.normal(0, 0.01, size=(N, P)) + ivar = np.full((N, P), 1.0 / 0.01 ** 2) + + results = sweep( + labels, flux, ivar, dispersion, + label_sets=[tuple(names[:2]), tuple(names)], + orders=[1, 2], + regularizations=[0.0, 1e3], + n_splits=3, + seed=0, + output="sweep_demo_results.csv", + wandb_project=wandb_project, + wandb_group="cannon-sweep-demo", + wandb_mode=wandb_mode) + + print("\n=== sweep results ===") + try: + import pandas as pd # noqa: F401 + cols = ["label_set", "order", "regularization", "censor", + "mean_scatter", "median_r_chi_sq", "frac_in_hull", "status"] + print(results[cols].to_string(index=False)) + except ImportError: + for row in results: + print({k: row[k] for k in ( + "label_set", "order", "regularization", "mean_scatter", + "median_r_chi_sq", "status")}) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--demo", action="store_true", + help="run a small sweep on the bundled golden data") + parser.add_argument("--wandb-project", default=None, + help="log each grid point to this W&B project") + parser.add_argument("--wandb-mode", default="offline", + choices=["offline", "online", "disabled"], + help="W&B run mode (default: offline)") + parser.add_argument("-v", "--verbose", action="store_true", + help="enable INFO-level logging") + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO if args.verbose else logging.WARNING, + format="%(asctime)s [%(levelname)s] %(message)s") + + if args.demo: + _demo(wandb_project=args.wandb_project, wandb_mode=args.wandb_mode) + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/scripts/sync_wandb.pbs b/scripts/sync_wandb.pbs new file mode 100755 index 0000000..bf0cc81 --- /dev/null +++ b/scripts/sync_wandb.pbs @@ -0,0 +1,50 @@ +#!/bin/bash +# =========================================================================== +# Upload offline Weights & Biases runs -- copyq (data-mover) PBS job. +# +# Submitted automatically by scripts/sweep.pbs once the sweep has produced +# the offline runs. Runs on the `copyq` queue (NCI Gadi), which -- unlike the +# compute queues -- has outbound internet access, and `wandb sync`s every +# offline run under $WANDB_DIR to the W&B cloud. +# +# Authentication: run `wandb login` once on a login node (this writes the API +# key to ~/.netrc, which copyq can read), or export WANDB_API_KEY before +# submitting. +# +# It can also be submitted by hand: +# qsub -v WANDB_DIR=$PWD/wandb scripts/sync_wandb.pbs +# =========================================================================== +#PBS -N cannon-wandb-sync +#PBS -P REPLACE_WITH_PROJECT +#PBS -q copyq +#PBS -l ncpus=1 +#PBS -l mem=8GB +#PBS -l walltime=02:00:00 +#PBS -l storage=scratch/REPLACE_WITH_PROJECT+gdata/REPLACE_WITH_PROJECT +#PBS -l wd +#PBS -j oe + +set -uo pipefail + +# These are passed through by sweep.pbs via `qsub -v`; the defaults let the job +# be submitted standalone too. +PROJECT_DIR="${PROJECT_DIR:-${PBS_O_WORKDIR:-$PWD}}" +PYTHON_BIN="${PYTHON_BIN:-python}" +WANDB_DIR="${WANDB_DIR:-${PROJECT_DIR}/wandb}" + +cd "${PROJECT_DIR}" + +export WANDB_DIR="${WANDB_DIR}" + +echo "[$(date)] syncing offline W&B runs in ${WANDB_DIR} from $(hostname)" + +# Prefer the bulk flag (scans ./wandb, i.e. ${WANDB_DIR} since we cd'd here); +# fall back to syncing each offline run directory explicitly. +if ! "${PYTHON_BIN}" -m wandb sync --sync-all; then + echo "[$(date)] --sync-all failed; syncing run directories individually" + for run_dir in "${WANDB_DIR}"/offline-run-*; do + [ -d "${run_dir}" ] && "${PYTHON_BIN}" -m wandb sync "${run_dir}" + done +fi + +echo "[$(date)] sync complete" From ff105f8039bab289ca1149e73bdbb50457b40040 Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Mon, 8 Jun 2026 22:59:31 +1000 Subject: [PATCH 05/17] Add end-to-end train/validate script (from start.ipynb) scripts/train_cannon.py loads an APOGEE parquet table, assembles the dispersion/flux/ivar arrays, continuum-normalizes, splits into train/validation (dropping non-finite labels), fits a polynomial CannonModel, runs the test step, and writes a one-to-one figure, a predictions CSV, and optionally the trained model. Parametrized via a CLI with a --demo mode on the bundled golden data. Device is left to JAX_PLATFORMS rather than forced to cpu. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/train_cannon.py | 331 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 331 insertions(+) create mode 100644 scripts/train_cannon.py diff --git a/scripts/train_cannon.py b/scripts/train_cannon.py new file mode 100644 index 0000000..7c305d9 --- /dev/null +++ b/scripts/train_cannon.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Train and validate a single CannonModel end-to-end (from notebooks/start.ipynb). + +The workflow: + + 1. load a parquet table of APOGEE spectra + ASPCAP labels, + 2. assemble the (n_pixels,) dispersion and (n_stars, n_pixels) flux / ivar + arrays (each table cell holds a per-star array), + 3. pseudo-continuum-normalize the spectra, + 4. split into training / validation sets (dropping non-finite labels), + 5. fit a polynomial CannonModel on the training set, + 6. run the test step on the validation set, and + 7. write a one-to-one (Cannon vs reference) figure, a predictions CSV, and + optionally the trained model. + +JAX device selection is via the ``JAX_PLATFORMS`` environment variable (this +script does not force one), e.g. ``JAX_PLATFORMS=cpu`` on machines where the GPU +backend is unavailable. + +Usage +----- +On the real data (defaults point at the bulge-ages-and-orbits data set):: + + python -m scripts.train_cannon \\ + --spectra /path/to/cleaned_ages.parquet \\ + --continuum-list /path/to/continuum.list \\ + --labels raw_teff,raw_logg,raw_fe_h,raw_mg_h,raw_ce_h,age_L,mass_L \\ + --order 2 --output-dir results/ + +Quick end-to-end check on the bundled golden data:: + + JAX_PLATFORMS=cpu python -m scripts.train_cannon --demo +""" + +from __future__ import (division, print_function, absolute_import, + unicode_literals) + +import argparse +import logging +import os + +import numpy as np + +import matplotlib +matplotlib.use("Agg") # headless: only saves figures +import matplotlib.pyplot as plt + +import thecannon as tc +from thecannon import continuum + +logger = logging.getLogger("thecannon.train") + + +# Defaults mirroring notebooks/start.ipynb. +DEFAULT_DATA_DIR = "/home/100/mj8805/scr_mk27/bulge-ages-and-orbits/data" +DEFAULT_LABELS = ["raw_teff", "raw_logg", "raw_fe_h", "raw_mg_h", "raw_ce_h", + "age_L", "mass_L"] +APOGEE_REGIONS = ([15090, 15822], [15823, 16451], [16452, 16971]) +CONTINUUM_L = 1400 +CONTINUUM_ORDER = 3 + + +# --------------------------------------------------------------------------- # +# Data loading # +# --------------------------------------------------------------------------- # + +def _to_array(x): + """ Coerce a table cell (array, list, or stringified list) to a 1-D array. """ + if isinstance(x, str): + return np.fromstring(x.strip("[] \n"), sep=",") + return np.asarray(x, dtype=float) + + +def load_spectra(spectra_path): + """ + Read the parquet table and assemble ``(spectra, dispersion, flux, ivar)``. + Each of the ``wavelength``/``flux``/``ivar`` columns holds one array per row. + """ + import pandas as pd + + spectra = pd.read_parquet(spectra_path) + + dispersion = _to_array(spectra["wavelength"].iloc[0]) # (n_pixels,) + flux = np.vstack([_to_array(x) for x in spectra["flux"]]) # (n_stars, P) + ivar = np.vstack([_to_array(x) for x in spectra["ivar"]]) # (n_stars, P) + + assert dispersion.ndim == 1, dispersion.shape + assert flux.shape == ivar.shape == (len(spectra), dispersion.size) + assert np.all(np.diff(dispersion) > 0), "dispersion must be sorted ascending" + logger.info("loaded %d stars x %d pixels", *flux.shape) + return spectra, dispersion, flux, ivar + + +def normalize_spectra(dispersion, flux, ivar, continuum_list_path): + """ Pseudo-continuum-normalize the spectra over the APOGEE chip regions. """ + continuum_pixels = np.loadtxt( + continuum_list_path, dtype=int, comments="#") + continuum_pixels = continuum_pixels[continuum_pixels < dispersion.size] + + normalized_flux, normalized_ivar, _, _ = continuum.normalize( + dispersion, flux, ivar, continuum_pixels, + L=CONTINUUM_L, order=CONTINUUM_ORDER, regions=APOGEE_REGIONS) + + good = ivar > 0 + logger.info("median normalized flux on good pixels: %.4f (target ~1)", + np.median(normalized_flux[good])) + return normalized_flux, normalized_ivar + + +# --------------------------------------------------------------------------- # +# Train / validate split # +# --------------------------------------------------------------------------- # + +def make_split(label_array, train_frac, validate_frac, seed): + """ + Return boolean ``(train_set, validate_set)`` masks. Stars with any + non-finite label are excluded from both. A single shuffled uniform draw + assigns each finite star to the training fold, the validation fold, or + neither (reproducible via ``seed``). + """ + finite = np.isfinite(label_array).all(axis=1) + rng = np.random.RandomState(seed) + u = rng.random_sample(len(label_array)) + + train_set = finite & (u < train_frac) + validate_set = finite & (u >= train_frac) & (u < train_frac + validate_frac) + logger.info("%d training, %d validation stars (%d dropped, non-finite)", + train_set.sum(), validate_set.sum(), (~finite).sum()) + return train_set, validate_set + + +# --------------------------------------------------------------------------- # +# One-to-one figure # +# --------------------------------------------------------------------------- # + +def one_to_one_figure(truth, predicted, label_names, title=None): + """ Cannon-vs-reference scatter per label, annotated with bias and scatter. """ + K = len(label_names) + ncols = 2 + nrows = int(np.ceil(K / ncols)) + fig, axes = plt.subplots(nrows, ncols, figsize=(5 * ncols, 3.5 * nrows), + squeeze=False) + axes = axes.ravel() + + for i, name in enumerate(label_names): + ax = axes[i] + x, y = truth[:, i], predicted[:, i] + good = np.isfinite(x) & np.isfinite(y) + ax.scatter(x[good], y[good], facecolor="k", s=10, alpha=0.5) + if good.any(): + lims = [float(min(x[good].min(), y[good].min())), + float(max(x[good].max(), y[good].max()))] + else: + lims = [0.0, 1.0] + ax.plot(lims, lims, "-", color="r", lw=1) # 1:1 line + ax.set_xlim(lims) + ax.set_ylim(lims) + d = (y - x)[good] + bias = float(np.nanmean(d)) if d.size else float("nan") + scatter = float(np.nanstd(d)) if d.size else float("nan") + ax.set_xlabel("{0} (reference)".format(name)) + ax.set_ylabel("{0} (Cannon)".format(name)) + ax.set_title("{0}: bias={1:+.3f}, scatter={2:.3f}".format( + name, bias, scatter)) + + for j in range(K, len(axes)): + axes[j].set_visible(False) + if title: + fig.suptitle(title) + fig.tight_layout() + return fig + + +# --------------------------------------------------------------------------- # +# Orchestration # +# --------------------------------------------------------------------------- # + +def run(labels, normalized_flux, normalized_ivar, dispersion, label_names, + order=2, regularization=0, train_frac=0.1, validate_frac=0.1, seed=888, + output_dir=".", save_model=None): + """ + Train on the training split, test on the validation split, and write the + one-to-one figure + predictions CSV. ``labels`` is anything CannonModel + accepts (a DataFrame, structured array, or (N, K) array aligned with + ``label_names``). + """ + import pandas as pd + + label_array = np.vstack( + [np.asarray(labels[name], dtype=float) for name in label_names]).T + + train_set, validate_set = make_split( + label_array, train_frac, validate_frac, seed) + if train_set.sum() == 0 or validate_set.sum() == 0: + raise ValueError("empty training or validation set; adjust the " + "train/validate fractions or seed") + + vectorizer = tc.vectorizer.PolynomialVectorizer( + label_names=label_names, order=order) + model = tc.CannonModel( + label_array[train_set], normalized_flux[train_set], + normalized_ivar[train_set], vectorizer, dispersion=dispersion, + regularization=regularization) + logger.info("%s", model) + + model.train() + logger.info("trained: %s | theta shape: %s", model.is_trained, + np.asarray(model.theta).shape) + + predicted, _, _ = model.test( + normalized_flux[validate_set], normalized_ivar[validate_set]) + predicted = np.asarray(predicted) + truth = label_array[validate_set] + + # Per-label metrics. + residual = predicted - truth + print("\n=== validation metrics ({0} stars) ===".format(len(truth))) + for i, name in enumerate(label_names): + d = residual[:, i][np.isfinite(residual[:, i])] + print(" {0:<12s} bias={1:+.4f} scatter={2:.4f}".format( + name, float(np.mean(d)), float(np.std(d)))) + + os.makedirs(output_dir, exist_ok=True) + + fig = one_to_one_figure( + truth, predicted, label_names, + title="order={0}, reg={1:g}".format(order, regularization)) + fig_path = os.path.join(output_dir, "one_to_one.png") + fig.savefig(fig_path, dpi=150) + plt.close(fig) + logger.info("wrote %s", fig_path) + + pred_cols = {"{0}_reference".format(n): truth[:, i] + for i, n in enumerate(label_names)} + pred_cols.update({"{0}_cannon".format(n): predicted[:, i] + for i, n in enumerate(label_names)}) + csv_path = os.path.join(output_dir, "validation_predictions.csv") + pd.DataFrame(pred_cols).to_csv(csv_path, index=False) + logger.info("wrote %s", csv_path) + + if save_model: + model.write(save_model, include_training_set_spectra=False, + overwrite=True) + logger.info("wrote model to %s", save_model) + + return model, truth, predicted + + +def _run_real(args): + spectra, dispersion, flux, ivar = load_spectra(args.spectra) + normalized_flux, normalized_ivar = normalize_spectra( + dispersion, flux, ivar, args.continuum_list) + return run( + spectra, normalized_flux, normalized_ivar, dispersion, args.labels, + order=args.order, regularization=args.regularization, + train_frac=args.train_frac, validate_frac=args.validate_frac, + seed=args.seed, output_dir=args.output_dir, save_model=args.save_model) + + +def _run_demo(args): + """ Smoke test on the bundled (already-normalized) golden data. """ + import pickle + + golden_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", "thecannon", "tests", "golden", "golden.pkl") + with open(golden_path, "rb") as fp: + meta = pickle.load(fp)["meta"] + + names = list(meta["label_names"]) + label_array = np.atleast_2d(np.asarray(meta["labels"], dtype=float)) + labels = {name: label_array[:, i] for i, name in enumerate(names)} + print("Demo on golden data: {0} stars, {1} pixels, labels {2}".format( + meta["flux"].shape[0], meta["flux"].shape[1], names)) + + # Golden flux is already normalized; use a 50/50 split given the small N. + return run( + labels, meta["flux"], meta["ivar"], meta["dispersion"], names, + order=args.order, regularization=args.regularization, + train_frac=0.5, validate_frac=0.5, seed=args.seed, + output_dir=args.output_dir, save_model=args.save_model) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--spectra", + default=os.path.join(DEFAULT_DATA_DIR, + "cleaned_ages.parquet"), + help="parquet table of spectra + labels") + parser.add_argument("--continuum-list", + default=os.path.join(DEFAULT_DATA_DIR, "continuum.list"), + help="text file of continuum pixel indices") + parser.add_argument("--labels", type=lambda s: s.split(","), + default=DEFAULT_LABELS, + help="comma-separated label column names") + parser.add_argument("--order", type=int, default=2, + help="polynomial order of the vectorizer") + parser.add_argument("--regularization", type=float, default=0.0, + help="L1 regularization strength (0 = none)") + parser.add_argument("--train-frac", type=float, default=0.1, + help="fraction of finite-label stars used for training") + parser.add_argument("--validate-frac", type=float, default=0.1, + help="fraction used for validation") + parser.add_argument("--seed", type=int, default=888, + help="RNG seed for the split") + parser.add_argument("--output-dir", default=".", + help="directory for the figure and predictions CSV") + parser.add_argument("--save-model", default=None, + help="optional path to write the trained .model file") + parser.add_argument("--demo", action="store_true", + help="run on the bundled golden data instead") + parser.add_argument("-v", "--verbose", action="store_true", + help="enable INFO-level logging") + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO if args.verbose else logging.WARNING, + format="%(asctime)s [%(levelname)s] %(message)s") + + if args.demo: + _run_demo(args) + else: + _run_real(args) + + +if __name__ == "__main__": + main() From c90bc40eb20d60f0517cca0ae4b09748d65867a7 Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Mon, 8 Jun 2026 23:06:33 +1000 Subject: [PATCH 06/17] Add run_sweep driver and point the PBS job at it scripts/run_sweep.py loads + continuum-normalizes the spectra via the train_cannon helpers, drops non-finite-label stars across the union of swept labels, then cross-validates a grid of label sets / orders / regularizations through sweep_cannon.sweep (offline W&B per grid point). CLI with nested-label-set defaults and a --demo mode. sweep.pbs now runs this driver instead of the bare sweep_cannon demo. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/run_sweep.py | 171 +++++++++++++++++++++++++++++++++++++++++++ scripts/sweep.pbs | 9 ++- 2 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 scripts/run_sweep.py diff --git a/scripts/run_sweep.py b/scripts/run_sweep.py new file mode 100644 index 0000000..04e62b0 --- /dev/null +++ b/scripts/run_sweep.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Run a cross-validated Cannon hyper-parameter sweep on real APOGEE data. + +This is the driver that ties the two pieces together: it loads and +continuum-normalizes the spectra with the helpers from +:mod:`scripts.train_cannon`, then hands the in-memory arrays to +:func:`scripts.sweep_cannon.sweep`, which does the k-fold cross-validation over +a grid of label sets, polynomial orders and regularization strengths (logging +one offline W&B run per grid point). + +JAX device selection is via the ``JAX_PLATFORMS`` environment variable. + +Usage +----- +:: + + python -m scripts.run_sweep \\ + --spectra /path/cleaned_ages.parquet \\ + --continuum-list /path/continuum.list \\ + --labels raw_teff,raw_logg,raw_fe_h,raw_mg_h,raw_ce_h,age_L,mass_L \\ + --label-set raw_teff,raw_logg,raw_fe_h \\ + --label-set raw_teff,raw_logg,raw_fe_h,raw_mg_h,raw_ce_h \\ + --orders 1,2 --regularizations 0,1e2,1e3,1e4 \\ + --n-splits 5 --wandb-project cannon-sweep + +Quick end-to-end check on the bundled golden data:: + + JAX_PLATFORMS=cpu python -m scripts.run_sweep --demo +""" + +from __future__ import (division, print_function, absolute_import, + unicode_literals) + +import argparse +import logging +import os + +import numpy as np + +# Work both as a package module (`python -m scripts.run_sweep`) and when run +# directly from the scripts/ directory. +try: + from scripts.train_cannon import (load_spectra, normalize_spectra, + DEFAULT_DATA_DIR, DEFAULT_LABELS) + from scripts.sweep_cannon import sweep +except ImportError: + from train_cannon import (load_spectra, normalize_spectra, + DEFAULT_DATA_DIR, DEFAULT_LABELS) + from sweep_cannon import sweep + +logger = logging.getLogger("thecannon.run_sweep") + + +def default_label_sets(labels): + """ Nested label sets (first 3, first 5, all) -- 'does adding labels help?' """ + sets = [] + for k in (3, 5, len(labels)): + if 1 <= k <= len(labels): + sets.append(tuple(labels[:k])) + return list(dict.fromkeys(sets)) # dedup, preserve order + + +def finite_label_mapping(label_source, label_union): + """ + Build a ``{name: (M,) array}`` mapping restricted to stars with finite + values across *every* label in ``label_union`` (the sweep does no filtering, + and a NaN label silently poisons that pixel's fit). + """ + matrix = np.vstack( + [np.asarray(label_source[name], dtype=float) for name in label_union]).T + finite = np.isfinite(matrix).all(axis=1) + logger.info("%d/%d stars retained after dropping non-finite labels", + int(finite.sum()), len(finite)) + mapping = {name: matrix[finite, i] for i, name in enumerate(label_union)} + return mapping, finite + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--spectra", + default=os.path.join(DEFAULT_DATA_DIR, + "cleaned_ages.parquet"), + help="parquet table of spectra + labels") + parser.add_argument("--continuum-list", + default=os.path.join(DEFAULT_DATA_DIR, "continuum.list"), + help="text file of continuum pixel indices") + parser.add_argument("--labels", type=lambda s: s.split(","), + default=DEFAULT_LABELS, + help="comma-separated master list of label columns") + parser.add_argument("--label-set", dest="label_sets", action="append", + type=lambda s: tuple(s.split(",")), default=None, + help="a label set to try (repeatable); defaults to " + "nested subsets of --labels") + parser.add_argument("--orders", type=lambda s: [int(x) for x in s.split(",")], + default=[1, 2], help="comma-separated polynomial orders") + parser.add_argument("--regularizations", + type=lambda s: [float(x) for x in s.split(",")], + default=[0.0, 1e2, 1e3, 1e4], + help="comma-separated L1 regularization strengths") + parser.add_argument("--n-splits", type=int, default=5, + help="number of cross-validation folds") + parser.add_argument("--seed", type=int, default=0, + help="RNG seed for the (shared) fold assignment") + parser.add_argument("--output", default="sweep_results.csv", + help="CSV path for the tidy results table") + parser.add_argument("--wandb-project", default=None, + help="log each grid point to this W&B project") + parser.add_argument("--wandb-mode", default="offline", + choices=["offline", "online", "disabled"], + help="W&B run mode (default: offline)") + parser.add_argument("--demo", action="store_true", + help="run on the bundled golden data instead") + parser.add_argument("-v", "--verbose", action="store_true", + help="enable INFO-level logging") + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO if args.verbose else logging.WARNING, + format="%(asctime)s [%(levelname)s] %(message)s") + + if args.demo: + import pickle + golden_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", "thecannon", "tests", "golden", "golden.pkl") + with open(golden_path, "rb") as fp: + meta = pickle.load(fp)["meta"] + names = list(meta["label_names"]) + arr = np.atleast_2d(np.asarray(meta["labels"], dtype=float)) + label_source = {name: arr[:, i] for i, name in enumerate(names)} + dispersion = meta["dispersion"] + norm_flux, norm_ivar = meta["flux"], meta["ivar"] # already normalized + labels_master = names + label_sets = [tuple(names[:2]), tuple(names)] + n_splits = min(args.n_splits, 3) + print("Demo on golden data: {0} stars, {1} pixels, labels {2}".format( + norm_flux.shape[0], norm_flux.shape[1], names)) + else: + label_source, dispersion, flux, ivar = load_spectra(args.spectra) + norm_flux, norm_ivar = normalize_spectra( + dispersion, flux, ivar, args.continuum_list) + labels_master = args.labels + label_sets = args.label_sets or default_label_sets(labels_master) + n_splits = args.n_splits + + # Union of every label used by any label set; drop non-finite stars once. + label_union = list(dict.fromkeys( + name for label_set in label_sets for name in label_set)) + mapping, finite = finite_label_mapping(label_source, label_union) + + logger.info("sweeping %d label sets x %d orders x %d regularizations", + len(label_sets), len(args.orders), len(args.regularizations)) + + sweep( + mapping, norm_flux[finite], norm_ivar[finite], dispersion, + label_sets=label_sets, + orders=args.orders, + regularizations=args.regularizations, + n_splits=n_splits, + seed=args.seed, + output=args.output, + wandb_project=args.wandb_project, + wandb_mode=args.wandb_mode) + + +if __name__ == "__main__": + main() diff --git a/scripts/sweep.pbs b/scripts/sweep.pbs index 8b25ddb..2f3dd86 100755 --- a/scripts/sweep.pbs +++ b/scripts/sweep.pbs @@ -31,10 +31,11 @@ PROJECT_DIR="${PBS_O_WORKDIR:-$PWD}" # wandb installed, e.g. /scratch/REPLACE_WITH_PROJECT/envs/thecannon/bin/python PYTHON_BIN="python" WANDB_PROJECT="cannon-sweep" -# The command that runs the sweep. Replace `--demo` with your own driver: a -# Python script that loads your spectra/labels and calls -# scripts.sweep_cannon.sweep(...) with wandb_project=WANDB_PROJECT. -SWEEP_CMD="${PYTHON_BIN} -m scripts.sweep_cannon --demo --wandb-project ${WANDB_PROJECT} --wandb-mode offline -v" +# The command that runs the sweep. scripts/run_sweep.py loads + normalizes the +# spectra (defaults point at the bulge-ages-and-orbits data) and cross-validates +# the grid. Override --spectra/--continuum-list/--labels/--orders/etc. as needed, +# or pass --demo to dry-run on the bundled golden data. +SWEEP_CMD="${PYTHON_BIN} -m scripts.run_sweep --wandb-project ${WANDB_PROJECT} --wandb-mode offline -v" # --------------------------------------------------------------------------- cd "${PROJECT_DIR}" From 419da9fabf32060be64ef05374ff3ca50af4ff9f Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Mon, 8 Jun 2026 23:24:24 +1000 Subject: [PATCH 07/17] Build label sets from base + age variants + abundances run_sweep now constructs the sweep grid from a fixed core (--base, default teff/logg/fe_h/mg_h), one or more age columns (--age-cols, default age_Dnu,age_L -- each a base variant so the two ages can be compared), an optional mass column, and a list of abundances, combined via --label-set-mode {one-at-a-time,cumulative,minimal}. Label sets that reference a column missing from the table are skipped with a warning, so e.g. an absent age_Dnu no longer crashes the run. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/run_sweep.py | 109 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 96 insertions(+), 13 deletions(-) diff --git a/scripts/run_sweep.py b/scripts/run_sweep.py index 04e62b0..80a06bd 100644 --- a/scripts/run_sweep.py +++ b/scripts/run_sweep.py @@ -54,13 +54,74 @@ logger = logging.getLogger("thecannon.run_sweep") -def default_label_sets(labels): - """ Nested label sets (first 3, first 5, all) -- 'does adding labels help?' """ +def build_label_sets(base_core, age_cols, mass_col, abundances, mode): + """ + Build the grid of label sets from a fixed core, one or more age columns + (each making a base variant), an optional mass column, and a list of + abundances. ``mode`` controls how the extras are added on top of each base: + + - ``one-at-a-time``: base, base+mass, base+each abundance, base+all + abundances, base+mass+all abundances. Isolates each addition's effect. + - ``cumulative``: base -> +mass -> +abund1 -> +abund2 -> ... (each set a + superset of the previous). + - ``minimal``: base, base+all abundances, base+mass, base+mass+all + abundances. + """ + def dedup(seq): + return tuple(dict.fromkeys(seq)) # drop dupes, preserve order + sets = [] - for k in (3, 5, len(labels)): - if 1 <= k <= len(labels): - sets.append(tuple(labels[:k])) - return list(dict.fromkeys(sets)) # dedup, preserve order + for age in age_cols: + base = list(base_core) + ([age] if age else []) + sets.append(dedup(base)) + + if mode == "one-at-a-time": + if mass_col: + sets.append(dedup(base + [mass_col])) + for a in abundances: + sets.append(dedup(base + [a])) + if abundances: + sets.append(dedup(base + list(abundances))) + if mass_col and abundances: + sets.append(dedup(base + [mass_col] + list(abundances))) + + elif mode == "cumulative": + current = list(base) + if mass_col: + current.append(mass_col) + sets.append(dedup(current)) + for a in abundances: + current.append(a) + sets.append(dedup(current)) + + elif mode == "minimal": + if abundances: + sets.append(dedup(base + list(abundances))) + if mass_col: + sets.append(dedup(base + [mass_col])) + if mass_col and abundances: + sets.append(dedup(base + [mass_col] + list(abundances))) + + return list(dict.fromkeys(sets)) # dedup whole sets across ages + + +def _has_column(label_source, name): + """ True if ``name`` is a column of the table / key of the mapping. """ + columns = getattr(label_source, "columns", None) + return (name in columns) if columns is not None else (name in label_source) + + +def filter_existing(label_sets, label_source): + """ Drop label sets that reference a column missing from the data. """ + kept = [] + for label_set in label_sets: + missing = [n for n in label_set if not _has_column(label_source, n)] + if missing: + logger.warning("skipping label set %s (missing columns: %s)", + "+".join(label_set), ",".join(missing)) + else: + kept.append(label_set) + return kept def finite_label_mapping(label_source, label_union): @@ -88,13 +149,28 @@ def main(): parser.add_argument("--continuum-list", default=os.path.join(DEFAULT_DATA_DIR, "continuum.list"), help="text file of continuum pixel indices") - parser.add_argument("--labels", type=lambda s: s.split(","), - default=DEFAULT_LABELS, - help="comma-separated master list of label columns") + parser.add_argument("--base", type=lambda s: s.split(","), + default=["raw_teff", "raw_logg", "raw_fe_h", "raw_mg_h"], + help="comma-separated core labels in every set (the age " + "column from --age-cols is appended to this)") + parser.add_argument("--age-cols", type=lambda s: s.split(","), + default=["age_Dnu", "age_L"], + help="age column(s); each makes a separate base variant " + "(missing columns are skipped)") + parser.add_argument("--mass-col", default="mass_L", + help="mass column added by the 'age and mass' sets " + "(empty string to disable)") + parser.add_argument("--abundances", type=lambda s: s.split(","), + default=["raw_ce_h", "raw_ca_h", "raw_si_h", "raw_ni_h", + "raw_mn_h", "raw_al_h", "raw_c_h", "raw_n_h"], + help="comma-separated abundances to test on top of base") + parser.add_argument("--label-set-mode", default="one-at-a-time", + choices=["one-at-a-time", "cumulative", "minimal"], + help="how extras are combined with each base") parser.add_argument("--label-set", dest="label_sets", action="append", type=lambda s: tuple(s.split(",")), default=None, - help="a label set to try (repeatable); defaults to " - "nested subsets of --labels") + help="explicit label set to try (repeatable); overrides " + "the builder when given") parser.add_argument("--orders", type=lambda s: [int(x) for x in s.split(",")], default=[1, 2], help="comma-separated polynomial orders") parser.add_argument("--regularizations", @@ -143,8 +219,13 @@ def main(): label_source, dispersion, flux, ivar = load_spectra(args.spectra) norm_flux, norm_ivar = normalize_spectra( dispersion, flux, ivar, args.continuum_list) - labels_master = args.labels - label_sets = args.label_sets or default_label_sets(labels_master) + label_sets = args.label_sets or build_label_sets( + args.base, args.age_cols, args.mass_col, args.abundances, + args.label_set_mode) + label_sets = filter_existing(label_sets, label_source) + if not label_sets: + raise ValueError("no usable label sets (all referenced columns " + "missing); check --base/--age-cols/--abundances") n_splits = args.n_splits # Union of every label used by any label set; drop non-finite stars once. @@ -154,6 +235,8 @@ def main(): logger.info("sweeping %d label sets x %d orders x %d regularizations", len(label_sets), len(args.orders), len(args.regularizations)) + for label_set in label_sets: + logger.info(" label set: %s", "+".join(label_set)) sweep( mapping, norm_flux[finite], norm_ivar[finite], dispersion, From be773c10d522defa660c9841c6b47700ab414563 Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Tue, 9 Jun 2026 16:34:51 +1000 Subject: [PATCH 08/17] Make continuum JAX-safe and add jax-tqdm progress bars - continuum.normalize: replace in-place masked assignment with jnp.where so JAX-array inputs no longer raise on immutable item assignment - model.train/test: drive progress with jax-tqdm via on-device lax.scan over batches (vmap within each batch); train keeps its prior batching, test gains a progressbar arg. Numerically identical to the previous vmap path. - continuum.normalize: add a plain tqdm bar over the host-side per-star loop - add end-to-end JAX test covering normalize -> build -> train -> test on a tiny synthetic sample - declare jax-tqdm and tqdm in install_requires Co-Authored-By: Claude Opus 4.8 --- setup.py | 3 +- thecannon/continuum.py | 36 +++-- thecannon/model.py | 162 +++++++++++++-------- thecannon/tests/test_end_to_end_jax.py | 192 +++++++++++++++++++++++++ 4 files changed, 324 insertions(+), 69 deletions(-) create mode 100644 thecannon/tests/test_end_to_end_jax.py diff --git a/setup.py b/setup.py index 34b7dfb..a72022c 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,8 @@ def read(filename): ], keywords="The Cannon", packages=find_packages(exclude=["documents", "tests"]), - install_requires=["numpy", "scipy", "six", "jax", "jaxlib", "jaxopt"], + install_requires=["numpy", "scipy", "six", "jax", "jaxlib", "jaxopt", + "jax-tqdm", "tqdm"], extras_require={ "test": ["coverage", "pytest"] }, diff --git a/thecannon/continuum.py b/thecannon/continuum.py index 4941b59..4a3abf6 100644 --- a/thecannon/continuum.py +++ b/thecannon/continuum.py @@ -40,8 +40,8 @@ def _continuum_design_matrix(dispersion, L, order): ]) -def sines_and_cosines(dispersion, flux, ivar, continuum_pixels, L=1400, order=3, - regions=None, fill_value=1.0, **kwargs): +def sines_and_cosines(dispersion, flux, ivar, continuum_pixels, L=1400, order=3, + regions=None, fill_value=1.0, progressbar=True, **kwargs): """ Fit the flux values of pre-defined continuum pixels using a sum of sine and cosine functions. @@ -124,7 +124,19 @@ def sines_and_cosines(dispersion, flux, ivar, continuum_pixels, L=1400, order=3, metadata = [] continuum = np.ones_like(flux) * fill_value - for i in range(flux.shape[0]): + + # The per-star fit below runs on the host (ragged per-region masks and + # Python-list metadata), so a plain tqdm bar is the right tool here -- the + # jax-tqdm bars used in CannonModel.train/test only work inside JAX loops. + star_iter = range(flux.shape[0]) + if progressbar: + try: + from tqdm.auto import tqdm + star_iter = tqdm(star_iter, desc="Normalizing", unit="star") + except ImportError: + pass + + for i in star_iter: warn_indices = np.where(warn_on_pixels[i])[0] if any(warn_indices): @@ -182,8 +194,8 @@ def sines_and_cosines(dispersion, flux, ivar, continuum_pixels, L=1400, order=3, return (continuum, metadata) -def normalize(dispersion, flux, ivar, continuum_pixels, L=1400, order=3, - regions=None, fill_value=1.0, **kwargs): +def normalize(dispersion, flux, ivar, continuum_pixels, L=1400, order=3, + regions=None, fill_value=1.0, progressbar=True, **kwargs): """ Pseudo-continuum-normalize the flux using a defined set of continuum pixels and a sum of sine and cosine functions. @@ -229,17 +241,17 @@ def normalize(dispersion, flux, ivar, continuum_pixels, L=1400, order=3, The continuum values for all pixels, and a dictionary that contains metadata about the fit. """ - continuum, metadata = sines_and_cosines(dispersion, flux, ivar, + continuum, metadata = sines_and_cosines(dispersion, flux, ivar, continuum_pixels, L=L, order=order, regions=regions, - fill_value=fill_value, **kwargs) + fill_value=fill_value, progressbar=progressbar, **kwargs) normalized_flux = flux/continuum normalized_ivar = continuum * ivar * continuum - normalized_flux[normalized_ivar == 0] = 1.0 - - non_finite_pixels = ~np.isfinite(normalized_flux) - normalized_flux[non_finite_pixels] = 1.0 - normalized_ivar[non_finite_pixels] = 0.0 + normalized_flux = jnp.where(normalized_ivar == 0, 1.0, normalized_flux) + + non_finite_pixels = ~jnp.isfinite(normalized_flux) + normalized_flux = jnp.where(non_finite_pixels, 1.0, normalized_flux) + normalized_ivar = jnp.where(non_finite_pixels, 0.0, normalized_ivar) return (normalized_flux, normalized_ivar, continuum, metadata) diff --git a/thecannon/model.py b/thecannon/model.py index feb754a..33fae1b 100644 --- a/thecannon/model.py +++ b/thecannon/model.py @@ -21,6 +21,7 @@ from functools import wraps from sys import version_info from scipy.spatial import Delaunay +from jax_tqdm import scan_tqdm from .vectorizer.base import BaseVectorizer from . import (censoring, fitting, utils, vectorizer as vectorizer_module, __version__) @@ -689,11 +690,6 @@ def train(self, threads=None, op_method=None, op_strict=True, op_kwds=None, if closed_form: fitter = fitting.make_pixel_closed_form() - batched = jax.jit(jax.vmap(fitter, in_axes=(0, 0, None, 0))) - - def run_batch(sl): - return batched(flux_PN[sl], ivar_PN[sl], design_matrix, - column_mask[sl]) else: # Initial theta guesses for every pixel: a linear-algebra estimate # and the fiducial value. The regularized objective is convex, so @@ -715,23 +711,17 @@ def _initial_stack(flux_PN, ivar_PN): fitter = fitting.make_pixel_fitter( op_method=op_method, maxiter=maxiter, tol=tol, bounds=bounds) - batched = jax.jit(jax.vmap( - fitter, in_axes=(0, 0, 0, None, 0, 0))) - - def run_batch(sl): - return batched(flux_PN[sl], ivar_PN[sl], init_stack[sl], - design_matrix, reg_P[sl], column_mask[sl]) # Pixels are fit independently, so we run them in fixed-size batches - # rather than one giant vmap. This lets us report progress (the single - # fused call is opaque), bounds peak memory, and keeps the compiled - # program small enough to compile once and reuse for every batch. + # (vmap within a batch, lax.scan across batches) rather than one giant + # vmap. The scan drives a per-batch jax-tqdm progress bar, bounds peak + # memory, and keeps the compiled body small (compiled once, reused). if batch_size is None: batch_size = max(512, int(np.ceil(P / 50))) batch_size = int(min(max(1, batch_size), P)) # Pad the pixel axis up to a whole number of equally-sized batches so - # every call to `batched` has identical shapes and XLA compiles it only + # every scan step has identical shapes and XLA compiles the body only # once. The padding pixels are duplicates of the last real pixel and are # discarded below. n_batches = int(np.ceil(P / batch_size)) @@ -747,37 +737,51 @@ def run_batch(sl): axis=0) reg_P = jnp.concatenate([reg_P, edge(reg_P)], axis=0) - try: - from tqdm import tqdm - except ImportError: - tqdm = None - - pbar = None - if progressbar and tqdm is not None: - pbar = tqdm(total=P, desc="Training", unit="px") - - theta_batches, s2_batches, fopt_batches = [], [], [] - for b in range(n_batches): - sl = slice(b * batch_size, (b + 1) * batch_size) - th, s2b, fob = run_batch(sl) - theta_batches.append(th) - s2_batches.append(s2b) - fopt_batches.append(fob) - if pbar is not None: - # Block on the device so the bar tracks real work rather than - # JAX's asynchronous dispatch. With no bar we let the batches - # dispatch asynchronously and overlap. - jax.block_until_ready((th, s2b, fob)) - pbar.update(min((b + 1) * batch_size, P) - b * batch_size) - - if pbar is not None: - pbar.close() - - # Concatenate the batches and drop any padding pixels. The trained - # quantities are kept as JAX arrays end-to-end. - theta = jnp.concatenate(theta_batches, axis=0)[:P] - s2 = jnp.concatenate(s2_batches, axis=0)[:P] - fopt = jnp.concatenate(fopt_batches, axis=0)[:P] + # Reshape the (padded) pixel axis into (n_batches, batch_size, ...) and + # fit one batch per scan step. Pixels are fit independently, so the + # batching is only a loop granularity: vmap parallelizes within a batch + # while a device-side lax.scan walks the batches, driving a jax-tqdm + # progress bar via ordered host callbacks. + reshape_batches = lambda a: a.reshape((n_batches, batch_size) + a.shape[1:]) + flux_b = reshape_batches(flux_PN) + ivar_b = reshape_batches(ivar_PN) + mask_b = reshape_batches(column_mask) + + if closed_form: + pixels = jax.vmap(fitter, in_axes=(0, 0, None, 0)) + + def fit_batch(carry, x): + _i, fb, ib, cm = x + return carry, pixels(fb, ib, design_matrix, cm) + + xs = (jnp.arange(n_batches), flux_b, ivar_b, mask_b) + else: + init_b = reshape_batches(init_stack) + reg_b = reshape_batches(reg_P) + pixels = jax.vmap(fitter, in_axes=(0, 0, 0, None, 0, 0)) + + def fit_batch(carry, x): + _i, fb, ib, st, rg, cm = x + return carry, pixels(fb, ib, st, design_matrix, rg, cm) + + xs = (jnp.arange(n_batches), flux_b, ivar_b, init_b, reg_b, mask_b) + + if progressbar: + fit_batch = scan_tqdm(n_batches, desc="Training")(fit_batch) + + @jax.jit + def run_training_scan(xs): + _, out = jax.lax.scan(fit_batch, None, xs) + return out + + theta_b, s2_b, fopt_b = run_training_scan(xs) + + # Flatten the batch axis back onto the pixel axis and drop any padding. + # The trained quantities are kept as JAX arrays end-to-end. + unbatch = lambda a: a.reshape((n_batches * batch_size,) + a.shape[2:])[:P] + theta = unbatch(theta_b) + s2 = unbatch(s2_b) + fopt = unbatch(fopt_b) # A single host transfer for the per-pixel metadata. fopt_values = fopt.tolist() @@ -805,8 +809,8 @@ def __call__(self, labels): @requires_training - def test(self, flux, ivar, initial_labels=None, threads=None, - use_derivatives=True, op_kwds=None): + def test(self, flux, ivar, initial_labels=None, threads=None, + use_derivatives=True, op_kwds=None, progressbar=True, batch_size=None): """ Run the test step on spectra. @@ -824,12 +828,20 @@ def test(self, flux, ivar, initial_labels=None, threads=None, The number of parallel threads to use. :param use_derivatives: [optional] - Boolean `True` indicating to use analytic derivatives provided by + Boolean `True` indicating to use analytic derivatives provided by the vectorizer, `None` to calculate on the fly, or a callable function to calculate your own derivatives. :param op_kwds: [optional] Optimization keywords that get passed to `scipy.optimize.leastsq`. + + :param progressbar: [optional] + Display a jax-tqdm progress bar while spectra are fit. + + :param batch_size: [optional] + The number of spectra to fit per vectorized batch. Stars are fit + independently, so this only controls progress-bar granularity and + bounds peak memory/compile cost. Defaults to ~50 progress updates. """ if flux is None or ivar is None: @@ -868,15 +880,53 @@ def test(self, flux, ivar, initial_labels=None, threads=None, core = fitting.make_spectrum_fitter( self.vectorizer, self.theta, self.s2, self._fiducials, self._scales, maxiter=maxiter, tol=tol) - batched = jax.jit(jax.vmap(core, in_axes=(0, 0, 0))) - op_labels, cov, chi_sq, model_flux, n_use = batched( - jnp.asarray(flux), jnp.asarray(ivar), jnp.asarray(initial_labels)) + # Batch over stars so a device-side lax.scan can drive a jax-tqdm + # progress bar. Stars are fit independently (vmap within each batch); + # the batching is only loop granularity and does not change results. + if batch_size is None: + batch_size = max(1, int(np.ceil(S / 50))) + batch_size = int(min(max(1, batch_size), S)) + n_batches = int(np.ceil(S / batch_size)) + + flux_j = jnp.asarray(flux) + ivar_j = jnp.asarray(ivar) + init_j = jnp.asarray(initial_labels) - op_labels = np.asarray(op_labels) - cov = np.asarray(cov) - chi_sq = np.asarray(chi_sq) - n_use = np.asarray(n_use) + # Pad the star axis up to whole batches so XLA compiles the scan body + # once; the duplicated padding stars are discarded below. + pad = n_batches * batch_size - S + if pad: + edge = lambda a: jnp.broadcast_to(a[-1:], (pad,) + a.shape[1:]) + flux_j = jnp.concatenate([flux_j, edge(flux_j)], axis=0) + ivar_j = jnp.concatenate([ivar_j, edge(ivar_j)], axis=0) + init_j = jnp.concatenate([init_j, edge(init_j)], axis=0) + + reshape_batches = lambda a: a.reshape((n_batches, batch_size) + a.shape[1:]) + spectra = jax.vmap(core, in_axes=(0, 0, 0)) + + def fit_batch(carry, x): + _i, fb, ib, lb = x + return carry, spectra(fb, ib, lb) + + if progressbar: + fit_batch = scan_tqdm(n_batches, desc="Testing")(fit_batch) + + @jax.jit + def run_test_scan(xs): + _, out = jax.lax.scan(fit_batch, None, xs) + return out + + xs = (jnp.arange(n_batches), reshape_batches(flux_j), + reshape_batches(ivar_j), reshape_batches(init_j)) + op_labels, cov, chi_sq, model_flux, n_use = run_test_scan(xs) + + # Flatten the batch axis back onto the star axis and drop padding. + unbatch = lambda a: a.reshape((n_batches * batch_size,) + a.shape[2:])[:S] + op_labels = np.asarray(unbatch(op_labels)) + cov = np.asarray(unbatch(cov)) + chi_sq = np.asarray(unbatch(chi_sq)) + n_use = np.asarray(unbatch(n_use)) meta = [dict(chi_sq=float(chi_sq[s]), r_chi_sq=float(chi_sq[s]) / max(1, int(n_use[s]) - L - 1), diff --git a/thecannon/tests/test_end_to_end_jax.py b/thecannon/tests/test_end_to_end_jax.py new file mode 100644 index 0000000..959ad02 --- /dev/null +++ b/thecannon/tests/test_end_to_end_jax.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +End-to-end smoke test of the full Cannon pipeline on a tiny synthetic data set, +driven entirely with JAX arrays. + +This mirrors ``notebooks/start.ipynb`` -- continuum-normalize, build a +``CannonModel`` with a quadratic ``PolynomialVectorizer``, ``train`` it, then run +the ``test`` step and check the labels are recovered -- but on a self-contained, +deterministic toy data set so it needs no external files and runs in seconds. + +Crucially every input array is a ``jax.numpy`` array (as in the notebook after +the ``jnp.array(...)`` conversion cell). This is exactly the path that used to +break: ``continuum.normalize`` did in-place ``x[mask] = 1.0`` assignment, which +JAX arrays do not support. The test asserts the pipeline runs and that the +quantities that should stay on-device (normalized flux/ivar, trained theta/s2) +are genuine ``jax.Array`` objects. +""" + +from __future__ import (division, print_function, absolute_import, + unicode_literals) + +import numpy as np +import jax +import jax.numpy as jnp +import pytest + +import thecannon as tc +from thecannon import continuum +from thecannon.vectorizer.polynomial import PolynomialVectorizer + + +LABEL_NAMES = ["teff", "logg", "feh"] +# Physical-ish centres and half-ranges, so the labels span realistic magnitudes +# (the model rescales them internally; this just exercises that scaling). +LABEL_CENTERS = np.array([4800.0, 2.5, -0.2]) +LABEL_RANGES = np.array([600.0, 1.0, 0.4]) + +N_TRAIN = 48 +N_VAL = 16 +N_PIXELS = 60 +SIGMA = 0.002 # per-pixel flux noise (high S/N) + + +def _make_dataset(): + """ + Build a tiny synthetic spectral library with a known label dependence. + + Even-indexed pixels are flat "continuum" pixels (flux == 1 + noise); these + are the pixels handed to the continuum fit, so the fitted continuum is ~1 + everywhere and normalization is ~identity. Odd-indexed "feature" pixels + carry a smooth quadratic dependence on the (scaled) labels, which is exactly + what an order-2 polynomial Cannon can represent -- so training fits it well + and the test step inverts it back to the labels. + + Returns plain numpy; callers convert to JAX at the boundary. + """ + rng = np.random.RandomState(20240609) + + # Scaled labels x ~ U(-1, 1); raw labels are an affine map of x. + x_all = rng.uniform(-1.0, 1.0, size=(N_TRAIN + N_VAL, len(LABEL_NAMES))) + labels_all = LABEL_CENTERS + LABEL_RANGES * x_all + + dispersion = np.linspace(15100.0, 16900.0, N_PIXELS) + + feature = np.zeros(N_PIXELS, dtype=bool) + feature[1::2] = True # odd pixels carry label info + continuum_pixels = np.where(~feature)[0].astype(int) + + # Per-feature-pixel linear (A) and quadratic (B) coefficients. + A = np.zeros((N_PIXELS, len(LABEL_NAMES))) + B = np.zeros((N_PIXELS, len(LABEL_NAMES))) + A[feature] = rng.normal(0.0, 0.06, size=(feature.sum(), len(LABEL_NAMES))) + B[feature] = rng.normal(0.0, 0.015, size=(feature.sum(), len(LABEL_NAMES))) + + # flux = 1 + A.x + B.x^2 (continuum pixels have A = B = 0 -> flux == 1) + flux = 1.0 + x_all @ A.T + (x_all ** 2) @ B.T + flux = flux + rng.normal(0.0, SIGMA, size=flux.shape) + ivar = np.full_like(flux, 1.0 / SIGMA ** 2) + + train = np.arange(N_TRAIN) + val = np.arange(N_TRAIN, N_TRAIN + N_VAL) + return dict(dispersion=dispersion, flux=flux, ivar=ivar, + continuum_pixels=continuum_pixels, labels=labels_all, + train=train, val=val) + + +@pytest.fixture(scope="module") +def pipeline(): + """Run normalize -> build -> train -> test once, all with JAX arrays.""" + data = _make_dataset() + + # --- promote every input to JAX, as the notebook does before normalize --- + dispersion = jnp.asarray(data["dispersion"]) + flux = jnp.asarray(data["flux"]) + ivar = jnp.asarray(data["ivar"]) + continuum_pixels = jnp.asarray(data["continuum_pixels"]) + + normalized_flux, normalized_ivar, cont, meta = continuum.normalize( + dispersion, flux, ivar, continuum_pixels, + L=1400, order=3, regions=[(15100, 16900)], progressbar=False) + + train, val = data["train"], data["val"] + labels = data["labels"] # numpy (the CannonModel label branch) + + vectorizer = PolynomialVectorizer(label_names=LABEL_NAMES, order=2) + model = tc.CannonModel( + labels[train], + normalized_flux[train], + normalized_ivar[train], + vectorizer, + dispersion=dispersion, + regularization=0) + theta, s2, train_meta = model.train(progressbar=False) + + val_labels, val_cov, val_meta = model.test( + normalized_flux[val], normalized_ivar[val], progressbar=False) + + return dict(model=model, theta=theta, s2=s2, + normalized_flux=normalized_flux, normalized_ivar=normalized_ivar, + continuum=cont, val_labels=np.asarray(val_labels), + val_cov=np.asarray(val_cov), truth=labels[val]) + + +# --------------------------------------------------------------------------- # +# Normalization stays on-device and is sane # +# --------------------------------------------------------------------------- # + +def test_normalize_returns_jax_arrays(pipeline): + # The exact path that previously raised on `x[mask] = 1.0`. + assert isinstance(pipeline["normalized_flux"], jax.Array) + assert isinstance(pipeline["normalized_ivar"], jax.Array) + + +def test_normalized_flux_is_finite(pipeline): + assert bool(jnp.all(jnp.isfinite(pipeline["normalized_flux"]))) + assert bool(jnp.all(jnp.isfinite(pipeline["normalized_ivar"]))) + + +def test_continuum_is_near_unity(pipeline): + # Spectra are built around a flat continuum of 1, so normalization is ~no-op. + nf = pipeline["normalized_flux"] + assert abs(float(jnp.median(nf)) - 1.0) < 0.02 + + +# --------------------------------------------------------------------------- # +# Training stays on-device and is well-formed # +# --------------------------------------------------------------------------- # + +def test_model_is_trained(pipeline): + assert pipeline["model"].is_trained + + +def test_trained_quantities_are_jax_arrays(pipeline): + assert isinstance(pipeline["theta"], jax.Array) + assert isinstance(pipeline["s2"], jax.Array) + + +def test_theta_and_scatter_well_formed(pipeline): + theta, s2 = pipeline["theta"], pipeline["s2"] + # 10 terms for an order-2 polynomial in 3 labels (1 + 3 linear + 6 quadratic). + n_terms = pipeline["model"].design_matrix.shape[1] + assert theta.shape == (N_PIXELS, n_terms) + assert s2.shape == (N_PIXELS,) + assert bool(jnp.all(jnp.isfinite(theta))) + assert bool(jnp.all(s2 >= 0.0)) + + +# --------------------------------------------------------------------------- # +# The test step inverts the model back to the true labels # +# --------------------------------------------------------------------------- # + +def test_recovers_validation_labels(pipeline): + pred = pipeline["val_labels"] + truth = pipeline["truth"] + assert pred.shape == truth.shape == (N_VAL, len(LABEL_NAMES)) + + # Compare in scaled-label units so all three labels share one tolerance. + rel = np.abs(pred - truth) / LABEL_RANGES + assert np.all(np.isfinite(rel)) + assert rel.mean() < 0.05, f"mean scaled error too high: {rel.mean():.4f}" + assert rel.max() < 0.15, f"max scaled error too high: {rel.max():.4f}" + + +def test_forward_model_shapes(pipeline): + """The trained model can predict flux for new labels (single and batch).""" + model = pipeline["model"] + one = model(pipeline["truth"][0]) + many = model(pipeline["truth"][:3]) + assert np.asarray(one).shape == (N_PIXELS,) + assert np.asarray(many).shape == (3, N_PIXELS) From 9aba2d534c062e77f46246a9d9bfd246076ef4cb Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Tue, 9 Jun 2026 17:59:42 +1000 Subject: [PATCH 09/17] JIT and vectorize the continuum fit over stars Replace the host-side per-star loop in sines_and_cosines with a single jax.jit + jax.vmap solve (_continuum_amplitudes) that fits every star at once per region; the design matrices are shared across stars so this is a clean batch (recompiled once per region shape). Aggregate the out-of-region pixel warning into one message and move the optional tqdm bar to iterate over regions. Continuum values match the previous implementation to 1e-10. Co-Authored-By: Claude Opus 4.8 --- thecannon/continuum.py | 156 ++++++++++++++++++++++++----------------- 1 file changed, 92 insertions(+), 64 deletions(-) diff --git a/thecannon/continuum.py b/thecannon/continuum.py index 4a3abf6..4ebb700 100644 --- a/thecannon/continuum.py +++ b/thecannon/continuum.py @@ -12,9 +12,56 @@ import logging import numpy as np +import jax import jax.numpy as jnp +@jax.jit +def _continuum_amplitudes(continuum_flux, continuum_ivar, M, region_matrix, + scalar): + """ + Solve the (eigenvalue-regularized) weighted normal equations for the sine- + and-cosine amplitudes and evaluate the continuum, vectorized over stars. + + All stars in a region share the same design matrices ``M`` (at the continuum + pixels) and ``region_matrix`` (at every region pixel), so the per-star solve + is mapped with ``jax.vmap`` and the whole thing JIT-compiled. JIT recompiles + once per distinct design-matrix shape (i.e. once per region). + + :param continuum_flux: + Continuum-pixel fluxes, shape ``(n_stars, n_continuum_pixels)``. + + :param continuum_ivar: + Inverse variances matching ``continuum_flux``. + + :param M: + Continuum-pixel design matrix, shape ``(n_terms, n_continuum_pixels)``. + + :param region_matrix: + Region design matrix, shape ``(n_terms, n_region_pixels)``. + + :param scalar: + The magic eigenvalue-regularization scalar. + + :returns: + A tuple of (continuum over the region pixels, amplitudes, condition + number), each with a leading ``n_stars`` axis. + """ + def _one(cfl, civ): + MTM = jnp.dot(M, civ[:, None] * M.T) + MTy = jnp.dot(M, civ * cfl) + + eigenvalues = jnp.linalg.eigvalsh(MTM) + MTM = MTM + jnp.eye(MTM.shape[0]) * (scalar * jnp.max(eigenvalues)) + eigenvalues = jnp.linalg.eigvalsh(MTM) + condition_number = jnp.max(eigenvalues) / jnp.min(eigenvalues) + + amplitudes = jnp.linalg.solve(MTM, MTy) + return jnp.dot(region_matrix.T, amplitudes), amplitudes, condition_number + + return jax.vmap(_one)(continuum_flux, continuum_ivar) + + def _continuum_design_matrix(dispersion, L, order): """ Build a design matrix for the continuum determination, using sines and @@ -121,77 +168,58 @@ def sines_and_cosines(dispersion, flux, ivar, continuum_pixels, L=1400, order=3, # Check for non-zero pixels (e.g. ivar > 0) that are not included in a # region. We should warn about this very loudly! warn_on_pixels = (pixel_included_in_regions == 0) * (ivar > 0) - - metadata = [] + if np.any(warn_on_pixels): + n_affected = int(np.any(warn_on_pixels, axis=1).sum()) + logging.warn("Some pixels have measured flux values (e.g., ivar > 0) " + "but are not included in any specified continuum region. " + "These pixels won't be continuum-normalised ({0} spectra " + "affected).".format(n_affected)) + + S = flux.shape[0] continuum = np.ones_like(flux) * fill_value - - # The per-star fit below runs on the host (ragged per-region masks and - # Python-list metadata), so a plain tqdm bar is the right tool here -- the - # jax-tqdm bars used in CannonModel.train/test only work inside JAX loops. - star_iter = range(flux.shape[0]) + metadata = [[] for _ in range(S)] + + # Each region is fit for every star at once: the design matrices are shared + # across stars, so the per-star normal-equation solve is vmapped and JIT- + # compiled (see `_continuum_amplitudes`). The host loop below only walks the + # handful of regions, so a plain tqdm bar over regions is the right tool -- + # the jax-tqdm bars used in CannonModel.train/test only work inside JAX + # loops, and here there is no per-star Python loop left to track. + regions_iter = list( + zip(region_masks, region_matrices, continuum_masks, continuum_matrices)) if progressbar: try: from tqdm.auto import tqdm - star_iter = tqdm(star_iter, desc="Normalizing", unit="star") + regions_iter = tqdm(regions_iter, desc="Normalizing", unit="region") except ImportError: pass - for i in star_iter: - - warn_indices = np.where(warn_on_pixels[i])[0] - if any(warn_indices): - # Split by deltas so that we give useful warning messages. - segment_indices = np.where(np.diff(warn_indices) > 1)[0] - segment_indices = np.sort(np.hstack( - [0, segment_indices, segment_indices + 1, len(warn_indices)])) - segment_indices = segment_indices.reshape(-1, 2) - - segments = ", ".join(["{:.1f} to {:.1f} ({:d} pixels)".format( - dispersion[s], dispersion[e], e-s) for s, e in segment_indices]) - - logging.warn("Some pixels in spectrum index {0} have measured flux " - "values (e.g., ivar > 0) but are not included in any " - "specified continuum region. These pixels won't be " - "continuum-normalised: {1}".format(i, segments)) - - # Get the flux and inverse variance for this object. - object_metadata = [] - object_flux, object_ivar = (flux[i], ivar[i]) - - # Normalize each region. - for region_mask, region_matrix, continuum_mask, continuum_matrix in \ - zip(region_masks, region_matrices, continuum_masks, continuum_matrices): - if continuum_mask.size == 0: - # Skipping.. - object_metadata.append([order, L, fill_value, scalar, [], None]) - continue - - # We will fit to continuum pixels only. - continuum_disp = dispersion[continuum_mask] - continuum_flux, continuum_ivar \ - = (object_flux[continuum_mask], object_ivar[continuum_mask]) - - # Solve for the amplitudes (linear algebra performed in JAX). - M = jnp.asarray(continuum_matrix) - civ = jnp.asarray(continuum_ivar) - cfl = jnp.asarray(continuum_flux) - MTM = jnp.dot(M, civ[:, None] * M.T) - MTy = jnp.dot(M, civ * cfl) - - eigenvalues = jnp.linalg.eigvalsh(MTM) - MTM = MTM + jnp.eye(MTM.shape[0]) * (scalar * jnp.max(eigenvalues)) - eigenvalues = jnp.linalg.eigvalsh(MTM) - condition_number = float(jnp.max(eigenvalues) / jnp.min(eigenvalues)) - - amplitudes = np.asarray(jnp.linalg.solve(MTM, MTy)) - continuum[i, region_mask] = np.asarray( - jnp.dot(jnp.asarray(region_matrix).T, jnp.asarray(amplitudes))) - object_metadata.append( - (order, L, fill_value, scalar, amplitudes, condition_number)) - - metadata.append(object_metadata) - - return (continuum, metadata) + for region_mask, region_matrix, continuum_mask, continuum_matrix \ + in regions_iter: + if continuum_mask.size == 0: + # No continuum pixels in this region; leave it at the fill value. + for s in range(S): + metadata[s].append([order, L, fill_value, scalar, [], None]) + continue + + # Solve for the amplitudes (linear algebra performed in JAX, vmapped + # over all stars and JIT-compiled). + region_continuum, amplitudes, condition_number = _continuum_amplitudes( + jnp.asarray(flux[:, continuum_mask]), + jnp.asarray(ivar[:, continuum_mask]), + jnp.asarray(continuum_matrix), + jnp.asarray(region_matrix), + float(scalar)) + + continuum[:, region_mask] = np.asarray(region_continuum) + + amplitudes = np.asarray(amplitudes) + condition_number = np.asarray(condition_number) + for s in range(S): + metadata[s].append((order, L, fill_value, scalar, + amplitudes[s], float(condition_number[s]))) + + return (continuum, metadata) def normalize(dispersion, flux, ivar, continuum_pixels, L=1400, order=3, From 6e5b46b600500c61e3a0a2a3ddea25d979a0c0bc Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Tue, 9 Jun 2026 22:34:39 +1000 Subject: [PATCH 10/17] Add label-error (errors-in-variables) training and order-aware batching Training now optionally accounts for uncertainties on the training-set labels. Passing `training_set_label_err` (per-label 1-sigma values) makes `train()` propagate the label errors into the per-pixel weights and refine them with iteratively reweighted least squares (`n_irls` passes), so stars with uncertain labels are down-weighted. Omitting it reproduces the exact -label fit bit-for-bit. - fitting.py: add make_pixel_closed_form_eiv / make_pixel_fitter_eiv and a shared _label_variance_term (first-order propagation g^T Sigma g folded into ivar_eff = ivar / (1 + ivar * v_label)). - model.py: store/validate training_set_label_err (serialized; old models read back as None), build the label Jacobian + scaled variances once in train(), and select the EIV fitters. - model.py: make the default batch_size order-aware -- cap it to a memory budget so higher polynomial orders shrink the batch automatically instead of blowing up peak memory. - tests: add test_label_errors.py (no-regression, finite, regularized, down-weighting mechanism, round-trip, validation). - notebooks/start_with_label_errors.ipynb: copy of start.ipynb showing how to assemble per-label errors and train with them. Co-Authored-By: Claude Opus 4.8 (1M context) --- notebooks/start_with_label_errors.ipynb | 784 ++++++++++++++++++++++++ thecannon/fitting.py | 179 ++++++ thecannon/model.py | 107 +++- thecannon/tests/test_label_errors.py | 166 +++++ 4 files changed, 1230 insertions(+), 6 deletions(-) create mode 100644 notebooks/start_with_label_errors.ipynb create mode 100644 thecannon/tests/test_label_errors.py diff --git a/notebooks/start_with_label_errors.ipynb b/notebooks/start_with_label_errors.ipynb new file mode 100644 index 0000000..759f824 --- /dev/null +++ b/notebooks/start_with_label_errors.ipynb @@ -0,0 +1,784 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "le-cell-00", + "metadata": {}, + "source": [ + "# Getting started, with label (feature) uncertainties\n", + "\n", + "This is a copy of `start.ipynb` that shows how to train a `CannonModel` while\n", + "accounting for **uncertainties on the training-set labels** (the \"features\"\n", + "that build the design matrix).\n", + "\n", + "The standard Cannon treats labels as exact. If you pass\n", + "`training_set_label_err` -- a `(n_stars, n_labels)` array of 1-sigma errors\n", + "aligned to `LABEL_NAMES` -- then `train()` propagates those errors into the\n", + "per-pixel weights (errors-in-variables) and refines them with a few iteratively\n", + "reweighted least-squares passes. Stars whose labels are uncertain are\n", + "automatically down-weighted. Pass `None` (or omit it) to recover the exact\n", + "behaviour of `start.ipynb`.\n", + "\n", + "The only changes from `start.ipynb` are flagged with **[label errors]** below." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "c7192144", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "thecannon: /scratch/y89/mj8805/miniforge/envs/astro/lib/python3.11/site-packages/thecannon/__init__.py\n" + ] + } + ], + "source": [ + "# # --- environment setup (added for the JAX backend) ---\n", + "# import os, sys\n", + "# os.environ[\"JAX_PLATFORMS\"] = \"cpu\" # this machine's METAL backend is broken\n", + "# repo_root = os.path.abspath(\"..\") # notebooks/ -> AnniesLasso repo root\n", + "# if repo_root not in sys.path:\n", + "# sys.path.insert(0, repo_root) # ensure the edited working tree is imported\n", + "\n", + "import jax\n", + "jax.config.update(\"jax_platform_name\", \"cuda\")\n", + "\n", + "import thecannon as tc\n", + "print(\"thecannon:\", tc.__file__)" + ] + }, + { + "cell_type": "markdown", + "id": "93694583", + "metadata": {}, + "source": [ + "Evo_state; kepler: rgb: 1, heb=2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "957b17d9", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-03T06:40:18.306498Z", + "iopub.status.busy": "2026-06-03T06:40:18.306223Z", + "iopub.status.idle": "2026-06-03T06:40:30.118311Z", + "shell.execute_reply": "2026-06-03T06:40:30.117892Z" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np\n", + "import thecannon as tc\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib\n", + "%matplotlib inline\n", + "\n", + "from thecannon import continuum" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "01457318", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-03T06:40:30.122431Z", + "iopub.status.busy": "2026-06-03T06:40:30.122206Z", + "iopub.status.idle": "2026-06-03T06:40:32.656029Z", + "shell.execute_reply": "2026-06-03T06:40:32.655223Z" + } + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from pathlib import Path\n", + "SRC = Path(\"/home/100/mj8805/scr_mk27/\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b3709e34", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-03T06:40:32.660383Z", + "iopub.status.busy": "2026-06-03T06:40:32.659855Z", + "iopub.status.idle": "2026-06-03T06:40:33.250419Z", + "shell.execute_reply": "2026-06-03T06:40:33.249984Z" + } + }, + "outputs": [], + "source": [ + "spectra = pd.read_parquet(SRC / \"bulge-ages-and-orbits/data/cleaned_ages.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "415f5106", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "EvoState\n", + "2.0 4585\n", + "1.0 4094\n", + "0.0 413\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "spectra['EvoState'].value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "6f04c647", + "metadata": {}, + "outputs": [], + "source": [ + "# spectra= spectra[spectra['EvoState'] == 2.0]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "87bda249", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "18228" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(spectra)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "01520854", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([1.1234e+04, 5.2750e+03, 1.3360e+03, 2.5900e+02, 7.6000e+01,\n", + " 2.0000e+01, 1.7000e+01, 6.0000e+00, 2.0000e+00, 3.0000e+00]),\n", + " array([ 0. , 287.97503662, 575.95007324, 863.92510986,\n", + " 1151.90014648, 1439.87524414, 1727.85021973, 2015.82519531,\n", + " 2303.80029297, 2591.77539062, 2879.75024414]),\n", + " )" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAGdCAYAAADpBYyuAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAIltJREFUeJzt3XuwVeV5P/DnyE2gQLgjFZW0lEBAG9FyiQ0kXCQFqWOnkmAZnTCgNWKIWIXaNsSZgpKIaUtj1DjBGCz5Q0mdggTSGJRyLUoVgqYdUSFyS8JVKSDs37yrs/fvnBckXs4RDufzmVk5a6397Nvr2tlf3vW+a1eVSqVSAABQcc7/XwUAQEACADgJPUgAABkBCQAgIyABAGQEJACAjIAEAJARkAAAMo2jATt+/Hi8+eab0apVq6iqqjrdLwcAeA/SNa4PHDgQXbt2jXPOqZu+ngYdkFI46tat2+l+GQDAB7B169Y4//zzoy406ICUeo7KDdy6devT/XIAgPdg//79RQdH+Xu8LjTogFQ+rZbCkYAEAPVLVR0OjzFIGwAgIyABAGQEJACAjIAEAJARkAAAMgISAEBGQAIAyAhIAAAZAQkAICMgAQBkBCQAgIyABACQEZAAADICEgBApnG+g9pz0bRF9a45X7tn1Ol+CQBw2ulBAgDICEgAABkBCQAgIyABAGQEJACAjIAEAJARkAAAMgISAEBGQAIAyAhIAAAZAQkAICMgAQBkBCQAgIyABAAgIAEAnJoeJACAjIAEAJARkAAAMgISAEBGQAIAyAhIAAAZAQkAICMgAQBkBCQAgIyABACQEZAAADICEgBARkACABCQAABOTQ8SAEBGQAIAyAhIAAAZAQkAICMgAQB82ID07LPPxlVXXRVdu3aNqqqq+NGPflTj9lKpFDNmzChub968eQwZMiQ2bdpUo+bw4cMxefLk6NChQ7Rs2TLGjBkT27Ztq1GzZ8+eGD9+fLRp06ZY0vrevXtr1LzxxhvFa0mPkR7r1ltvjSNHjrzftwQA8OEC0ltvvRWXXHJJzJ0796S3z549O+bMmVPcvm7duujSpUsMHz48Dhw4UKmZMmVKLFy4MBYsWBArVqyIgwcPxujRo+PYsWOVmnHjxsWGDRtiyZIlxZLWU0gqS7WjRo0qXk96jPRYTzzxREydOvX9viUAgBqqSqnL5wNKPUgp6Fx99dXFdnqo1HOUAtCdd95Z6S3q3Llz3HvvvXHjjTfGvn37omPHjvHYY4/F2LFji5o333wzunXrFosXL44rr7wyNm/eHL17947Vq1dH//79i5q0PnDgwHj55ZejZ8+e8fTTTxehauvWrcVzJikk3XDDDbFr165o3br1b339+/fvL3qn0mt6L/Xv10XTFkV989o9o073SwCA0/r9XetjkLZs2RI7duyIESNGVPY1a9YsBg8eHCtXriy2169fH0ePHq1RkwJOnz59KjWrVq0q3ng5HCUDBgwo9lWvSfcph6MkhasUyNJznEy6LTVq9QUAoE4DUgpHSeoxqi5tl29Lf5s2bRpt27Y9ZU2nTp1OePy0r3pN/jzpMdNjl2tys2bNqoxpSkvqtQIA+EhmsaVTb9WlU2/5vlxec7L6D1JT3fTp04vuuPKSTs8BANRpQEoDspO8ByeNCSr39qSaNNMszVI7Vc3OnTtPePzdu3fXqMmfJz1mOn2X9yxVP92XzlVWXwAA6jQgde/evQguy5Ytq+xLYWj58uUxaNCgYrtfv37RpEmTGjXbt2+PjRs3VmrSYOzUw7N27dpKzZo1a4p91WvSfdJ9y5YuXVqEoPQcAAAfVOP3e4c0Jf9//ud/agzMTlPw27VrFxdccEExg23mzJnRo0ePYknrLVq0KKbtJ2nsz4QJE4rp+O3bty/ud/vtt0ffvn1j2LBhRU2vXr1i5MiRMXHixHjwwQeLfZMmTSpmraUZbEka5J1muqWp/9/4xjfiN7/5TfE46T56hgCAjzQg/ed//md89rOfrWzfdtttxd/rr78+5s2bF3fccUccOnQobr755uKUV5qJlnp2WrVqVbnP/fffH40bN45rr722qB06dGhx30aNGlVq5s+fX1z4sTzbLV1Msvq1l1LtokWLiuf59Kc/XVyUMoWwb37zmx+8NQAAPux1kOo710E6kesgAXCmq3fXQQIAOBsISAAAGQEJACAjIAEAZAQkAICMgAQAkBGQAAAyAhIAQEZAAgDICEgAABkBCQAgIyABAGQEJACAjIAEAJARkAAAMgISAEBGQAIAyAhIAAAZAQkAICMgAQBkBCQAgIyABACQEZAAADICEgBARkACAMgISAAAGQEJACAjIAEAZAQkAICMgAQAkBGQAAAyAhIAQEZAAgDICEgAABkBCQAgIyABAGQEJACAjIAEAJARkAAAMgISAEBGQAIAyAhIAAAZAQkAICMgAQBkBCQAgIyABACQEZAAADICEgBARkACAMgISAAAGQEJAKCuA9I777wTf/M3fxPdu3eP5s2bx8c//vG4++674/jx45WaUqkUM2bMiK5duxY1Q4YMiU2bNtV4nMOHD8fkyZOjQ4cO0bJlyxgzZkxs27atRs2ePXti/Pjx0aZNm2JJ63v37q3ttwQANDC1HpDuvffe+M53vhNz586NzZs3x+zZs+Mb3/hG/NM//VOlJu2bM2dOUbNu3bro0qVLDB8+PA4cOFCpmTJlSixcuDAWLFgQK1asiIMHD8bo0aPj2LFjlZpx48bFhg0bYsmSJcWS1lNIAgD4MKpKqTunFqUQ07lz53jkkUcq+/7sz/4sWrRoEY899ljRe5R6jlIAuvPOOyu9Rek+KVzdeOONsW/fvujYsWNRP3bs2KLmzTffjG7dusXixYvjyiuvLMJX7969Y/Xq1dG/f/+iJq0PHDgwXn755ejZs+dvfa379+8vep7S87Vu3Tpq20XTFkV989o9o073SwCA0/r9XSc9SFdccUX8+7//e/ziF78otv/rv/6r6AH6kz/5k2J7y5YtsWPHjhgxYkTlPs2aNYvBgwfHypUri+3169fH0aNHa9SkUNWnT59KzapVq4rGKYejZMCAAcW+cg0AwAfROGpZ6hVKie4Tn/hENGrUqDgl9vd///fxxS9+sbg9haMk9RhVl7Zff/31Sk3Tpk2jbdu2J9SU75/+durU6YTnT/vKNbnUU5WW6gkUAKDOe5B++MMfxg9+8IN4/PHH4/nnn49HH300vvnNbxZ/q6uqqqqxnU695ftyec3J6k/1OLNmzaoM6E5LOmUHAFDnAemv/uqvYtq0afGFL3wh+vbtWwya/upXv1qEkyQNyE7yXp5du3ZVepVSzZEjR4pZaqeq2blz5wnPv3v37hN6p8qmT59e9G6Vl61bt9bSuwYAzia1HpDefvvtOOecmg+bTrWVp/mn6f8p3CxbtqxyewpDy5cvj0GDBhXb/fr1iyZNmtSo2b59e2zcuLFSkwZjp5Czdu3aSs2aNWuKfeWaXBrrlAZzVV8AAOp8DNJVV11VjDm64IIL4pOf/GS88MILxZT+L33pS8Xt6fRXmsE2c+bM6NGjR7Gk9TTLLU3bT9LprwkTJsTUqVOjffv20a5du7j99tuLHqlhw4YVNb169YqRI0fGxIkT48EHHyz2TZo0qZhF915msAEAfGQBKV3v6G//9m/j5ptvLk6Jpdlnaer+3/3d31Vq7rjjjjh06FBRk06jpZloS5cujVatWlVq7r///mjcuHFce+21Re3QoUNj3rx5RW9U2fz58+PWW2+tzHZLF5NM11YCADijroNUn7gO0olcBwmAM129vA4SAEB9JyABAGQEJACAjIAEAJARkAAAMgISAEBGQAIAyAhIAAAZAQkAICMgAQBkBCQAgIyABACQEZAAADICEgBARkACAMgISAAAGQEJACAjIAEAZAQkAICMgAQAkBGQAAAyAhIAQEZAAgDICEgAABkBCQAgIyABAGQEJACAjIAEAJARkAAAMgISAEBGQAIAyAhIAAAZAQkAICMgAQBkBCQAgIyABACQEZAAADICEgBARkACAMgISAAAGQEJACAjIAEAZAQkAICMgAQAkBGQAAAyAhIAQEZAAgDICEgAABkBCQAgIyABAGQEJACAjIAEAPBRBKRf/vKX8Rd/8RfRvn37aNGiRfzhH/5hrF+/vnJ7qVSKGTNmRNeuXaN58+YxZMiQ2LRpU43HOHz4cEyePDk6dOgQLVu2jDFjxsS2bdtq1OzZsyfGjx8fbdq0KZa0vnfv3rp4SwBAA1LrASmFlk9/+tPRpEmTePrpp+PnP/953HffffGxj32sUjN79uyYM2dOzJ07N9atWxddunSJ4cOHx4EDByo1U6ZMiYULF8aCBQtixYoVcfDgwRg9enQcO3asUjNu3LjYsGFDLFmypFjSegpJAAAfRlUpdefUomnTpsV//Md/xHPPPXfS29PTpZ6jFIDuvPPOSm9R586d4957740bb7wx9u3bFx07dozHHnssxo4dW9S8+eab0a1bt1i8eHFceeWVsXnz5ujdu3esXr06+vfvX9Sk9YEDB8bLL78cPXv2/K2vdf/+/UXPU3q+1q1bR227aNqiqG9eu2fU6X4JAHBav7/rpAfpqaeeissuuyz+/M//PDp16hSf+tSn4uGHH67cvmXLltixY0eMGDGisq9Zs2YxePDgWLlyZbGdTscdPXq0Rk0KVX369KnUrFq1qmiccjhKBgwYUOwr1+RSEEuNWn0BAKjzgPTqq6/GAw88ED169Igf//jHcdNNN8Wtt94a3//+94vbUzhKUo9RdWm7fFv627Rp02jbtu0pa1IAy6V95ZrcrFmzKuOV0pJ6pAAA6jwgHT9+PC699NKYOXNm0XuUTplNnDixCE3VVVVVnXDqLd+Xy2tOVn+qx5k+fXrRHVdetm7d+j7fHQDQENR6QDrvvPOKsUHV9erVK954441iPQ3ITvJenl27dlV6lVLNkSNHigHfp6rZuXPnCc+/e/fuE3qnqp/KS+cqqy8AAHUekNIMtldeeaXGvl/84hdx4YUXFuvdu3cvws2yZcsqt6cwtHz58hg0aFCx3a9fv2IWXPWa7du3x8aNGys1aTB26gVau3ZtpWbNmjXFvnINAMAH0Thq2Ve/+tUioKRTbNdee20RYB566KFiSdLprzSDLd2eximlJa2n6yWlaftJGh80YcKEmDp1anEtpXbt2sXtt98effv2jWHDhlV6pUaOHFmcvnvwwQeLfZMmTSouBfBeZrABAHxkAenyyy8vrl+UxvvcfffdRY/Rt771rbjuuusqNXfccUccOnQobr755uI0WpqJtnTp0mjVqlWl5v7774/GjRsXISvVDh06NObNmxeNGjWq1MyfP78YAF6e7ZYuJpmurQQAcEZdB6k+cR2kE7kOEgBnunp5HSQAgPpOQAIAyAhIAAAZAQkAICMgAQBkBCQAgIyABACQEZAAADICEgBARkACAMgISAAAGQEJACAjIAEAZAQkAICMgAQAkBGQAAAyjfMdNGwXTVsU9c1r94w63S8BgLOMHiQAgIyABACQEZAAADICEgBARkACAMgISAAAGQEJACAjIAEAZAQkAICMgAQAkBGQAAAyAhIAQEZAAgDICEgAABkBCQAgIyABAGQEJACAjIAEAJARkAAAMgISAEBGQAIAyAhIAAAZAQkAICMgAQBkBCQAgIyABACQEZAAADICEgBARkACAMgISAAAGQEJACAjIAEAZAQkAICMgAQAkBGQAAA+6oA0a9asqKqqiilTplT2lUqlmDFjRnTt2jWaN28eQ4YMiU2bNtW43+HDh2Py5MnRoUOHaNmyZYwZMya2bdtWo2bPnj0xfvz4aNOmTbGk9b1799b1WwIAznJ1GpDWrVsXDz30UFx88cU19s+ePTvmzJkTc+fOLWq6dOkSw4cPjwMHDlRqUqBauHBhLFiwIFasWBEHDx6M0aNHx7Fjxyo148aNiw0bNsSSJUuKJa2nkAQAcEYGpBRorrvuunj44Yejbdu2NXqPvvWtb8Vdd90V11xzTfTp0yceffTRePvtt+Pxxx8vavbt2xePPPJI3HfffTFs2LD41Kc+FT/4wQ/ipZdeip/85CdFzebNm4tQ9N3vfjcGDhxYLOm5/u3f/i1eeeWVunpbAEADUGcB6ctf/nKMGjWqCDjVbdmyJXbs2BEjRoyo7GvWrFkMHjw4Vq5cWWyvX78+jh49WqMmnY5LYapcs2rVquK0Wv/+/Ss1AwYMKPaVa3LptN3+/ftrLAAAucZRB9Jpseeff744fZZL4Sjp3Llzjf1p+/XXX6/UNG3atEbPU7mmfP/0t1OnTic8ftpXrjnZeKivf/3rH+KdAQANQa33IG3dujW+8pWvFKfEzj333HetSwO3q0un3vJ9ubzmZPWnepzp06cXp+/KS3qtAAB1HpDS6bFdu3ZFv379onHjxsWyfPny+Md//MdivdxzlPfypPuUb0uDto8cOVLMUjtVzc6dO094/t27d5/QO1X9VF7r1q1rLAAAdR6Qhg4dWgymTjPKystll11WDNhO6x//+MeLcLNs2bLKfVIYSiFq0KBBxXYKV02aNKlRs3379ti4cWOlJg3KTr1Aa9eurdSsWbOm2FeuAQA4I8YgtWrVqhhMXV26jlH79u0r+9MU/pkzZ0aPHj2KJa23aNGimLafpIHWEyZMiKlTpxb3a9euXdx+++3Rt2/fyqDvXr16xciRI2PixInx4IMPFvsmTZpUXAqgZ8+etf22AIAGpE4Gaf82d9xxRxw6dChuvvnm4jRamom2dOnSIlyV3X///cUpuWuvvbaoTT1T8+bNi0aNGlVq5s+fH7feemtltlu6mGS6thIAwIdRVUqjmhuoNM0/9Val03J1MR7pommLav0xOdFr94zSLAANyP46/v5O/BYbAEBGQAIAyAhIAAAZAQkAICMgAQBkBCQAgIyABACQEZAAADICEgBARkACAMgISAAAGQEJACAjIAEAZAQkAICMgAQAkBGQAAAyAhIAQEZAAgDICEgAABkBCQAgIyABAGQEJACAjIAEAJARkAAAMgISAEBGQAIAyAhIAAAZAQkAICMgAQBkBCQAgIyABACQEZAAADICEgBARkACAMgISAAAGQEJACAjIAEAZAQkAICMgAQAkBGQAAAyAhIAQEZAAgDICEgAABkBCQAgIyABAGQEJACAjIAEAJARkAAAMgISAEBGQAIAyAhIAAB1HZBmzZoVl19+ebRq1So6deoUV199dbzyyis1akqlUsyYMSO6du0azZs3jyFDhsSmTZtq1Bw+fDgmT54cHTp0iJYtW8aYMWNi27ZtNWr27NkT48ePjzZt2hRLWt+7d29tvyUAoIGp9YC0fPny+PKXvxyrV6+OZcuWxTvvvBMjRoyIt956q1Ize/bsmDNnTsydOzfWrVsXXbp0ieHDh8eBAwcqNVOmTImFCxfGggULYsWKFXHw4MEYPXp0HDt2rFIzbty42LBhQyxZsqRY0noKSQAAH0ZVKXXn1KHdu3cXPUkpOH3mM58peo9Sz1EKQHfeeWelt6hz585x7733xo033hj79u2Ljh07xmOPPRZjx44tat58883o1q1bLF68OK688srYvHlz9O7duwhi/fv3L2rS+sCBA+Pll1+Onj17/tbXtn///qLnKT1f69ata/29XzRtUa0/Jid67Z5RmgWgAdlfx9/fH8kYpPTik3bt2hV/t2zZEjt27Ch6lcqaNWsWgwcPjpUrVxbb69evj6NHj9aoSaGqT58+lZpVq1YVjVMOR8mAAQOKfeUaAIAPonHUodRbdNttt8UVV1xRhJskhaMk9RhVl7Zff/31Sk3Tpk2jbdu2J9SU75/+pp6pXNpXrsmlnqq0VE+gAAAfaQ/SLbfcEi+++GL8y7/8ywm3VVVVnRCm8n25vOZk9ad6nDSAvDygOy3plB0AwEcWkNIMtKeeeiqeeeaZOP/88yv704DsJO/l2bVrV6VXKdUcOXKkmKV2qpqdO3eedMxT3jtVNn369OKUX3nZunVrLbxTAOBsU+sBKfXgpJ6jJ598Mn76059G9+7da9yetlO4STPcylIYSoO4Bw0aVGz369cvmjRpUqNm+/btsXHjxkpNGoydQs7atWsrNWvWrCn2lWtyaaxTGsxVfQEAqPMxSGmK/+OPPx7/+q//WlwLqdxTlE5ppWsepdNfaQbbzJkzo0ePHsWS1lu0aFFM2y/XTpgwIaZOnRrt27cvBnjffvvt0bdv3xg2bFhR06tXrxg5cmRMnDgxHnzwwWLfpEmTiksBvJcZbAAAH1lAeuCBB4q/6eKP1X3ve9+LG264oVi/44474tChQ3HzzTcXp9HSTLSlS5cWgars/vvvj8aNG8e1115b1A4dOjTmzZsXjRo1qtTMnz8/br311spst3QxyXRtJQCAM/o6SGcy10E6O7gOEkDDsv9suA4SAEB9IyABAGQEJACAjIAEAJARkAAAMgISAEBGQAIAyAhIAAAZAQkAICMgAQBkBCQAgIyABACQEZAAADICEgBApnG+A+qbi6YtivrmtXtGne6XAMAp6EECAMgISAAAGQEJACAjIAEAZAQkAICMgAQAkBGQAAAyAhIAQEZAAgDICEgAABkBCQAgIyABAGQEJACAjIAEAJARkAAAMgISAEBGQAIAyAhIAAAZAQkAICMgAQBkBCQAgIyABACQEZAAADICEgBARkACAMgISAAAGQEJACAjIAEAZAQkAICMgAQAkBGQAAAyAhIAQEZAAgDINM53AHXvommL6l0zv3bPqNP9EgA+MnqQAAAyAhIAQEZAAgA42wLSt7/97ejevXuce+650a9fv3juuedO90sCAOq5eh2QfvjDH8aUKVPirrvuihdeeCH++I//OD7/+c/HG2+8cbpfGgBQj1WVSqVS1FP9+/ePSy+9NB544IHKvl69esXVV18ds2bN+q33379/f7Rp0yb27dsXrVu3rvXXVx9nKsHZxMw7ODvtr+Pv73o9zf/IkSOxfv36mDZtWo39I0aMiJUrV570PocPHy6WstSw5YauC8cPv10njwu8N3X12QbOjM92Xfbx1NuA9Ktf/SqOHTsWnTt3rrE/be/YseOk90m9Sl//+tdP2N+tW7c6e53A6dPmW1ofzmYHDhwoepLqQr0NSGVVVVU1tlOazPeVTZ8+PW677bbK9vHjx+M3v/lNtG/f/l3v82HSbQpeW7durbPuv7ORdtNujrn6wWdVu53O4y1916dw1LVr1zp7znobkDp06BCNGjU6obdo165dJ/QqlTVr1qxYqvvYxz5Wp68z/YcUkLTbR8Xxpu0+ao457Xa6jre66jmq97PYmjZtWkzrX7ZsWY39aXvQoEGn7XUBAPVfve1BStLpsvHjx8dll10WAwcOjIceeqiY4n/TTTed7pcGANRj9TogjR07Nn7961/H3XffHdu3b48+ffrE4sWL48ILLzzdL604lfe1r33thFN6aDfH25nFZ1W7Od7OfM1Ow3dqvb4OEgBAXai3Y5AAAOqKgAQAkBGQAAAyAhIAQEZAqgPf/va3o3v37nHuuecW12p67rnnoiGbMWNGcaXy6kuXLl0qt6d5AqkmXRG1efPmMWTIkNi0aVONx0i/oTd58uTiAqEtW7aMMWPGxLZt2+Js8uyzz8ZVV11VtENqox/96Ec1bq+tdtqzZ09xeYx0kbW0pPW9e/fG2dpuN9xwwwnH34ABA6Kht1v66aXLL788WrVqFZ06dSp+5PuVV16pUeOY+2Dt5pg7ufTD8hdffHHlYo/p8jxPP/30mXu8pVls1J4FCxaUmjRpUnr44YdLP//5z0tf+cpXSi1btiy9/vrrDbaZv/a1r5U++clPlrZv315Zdu3aVbn9nnvuKbVq1ar0xBNPlF566aXS2LFjS+edd15p//79lZqbbrqp9Lu/+7ulZcuWlZ5//vnSZz/72dIll1xSeuedd0pni8WLF5fuuuuuoh3SR3PhwoU1bq+tdho5cmSpT58+pZUrVxZLWh89enTpbG2366+/vnjP1Y+/X//61zVqGmK7XXnllaXvfe97pY0bN5Y2bNhQGjVqVOmCCy4oHTx4sFLjmPtg7eaYO7mnnnqqtGjRotIrr7xSLH/9139dfF+mtjwTjzcBqZb90R/9UfEfsLpPfOITpWnTppUackBKB/DJHD9+vNSlS5fig1H2v//7v6U2bdqUvvOd7xTbe/fuLT5EKXyW/fKXvyydc845pSVLlpTORvkXfW21Uwrt6bFXr15dqVm1alWx7+WXX/6I3l3debeA9Kd/+qfveh/t9n/SP1pS+y1fvrzYdsx9sHZzzL0/bdu2LX33u989I483p9hq0ZEjR2L9+vUxYsSIGvvT9sqVK6Mh++///u+i2zSdevzCF74Qr776arF/y5Ytxe/pVW+zdCGwwYMHV9ostenRo0dr1KTHShcGbSjtWlvttGrVqqLLuX///pWadLop7Tub2/JnP/tZcTrkD/7gD2LixInFbzaWabf/s2/fvuJvu3btir+OuQ/Wbo659+bYsWOxYMGCeOutt4pTbWfi8SYg1aJf/epXxX/0/Mdy03b+o7oNSTpQv//978ePf/zjePjhh4u2SL+Xl66CXm6XU7VZ+pt+e69t27bvWnO2q612Sn9TUMilfWdrW37+85+P+fPnx09/+tO47777Yt26dfG5z32uGMuQaLf/G/uRfrrpiiuuKL5syu2SOObeX7s55k7tpZdeit/5nd8pwk/6WbCFCxdG7969z8jjrV7/1MiZKg0CzT9E+b6GJH1BlfXt27f418Lv/d7vxaOPPloZLPtB2qwhtmtttNPJ6s/mtkw/SVSWvsTSbzemnyNatGhRXHPNNe96v4bUbrfccku8+OKLsWLFihNuc8y9/3ZzzL27nj17xoYNG4pB00888URcf/31sXz58jPyeNODVIvSqPpGjRqdkFJTd36eihuyNPMgBaV02q08m+1UbZZq0unLNDPh3WrOdrXVTqlm586dJzz+7t27G0xbnnfeeUVASsdf0tDbLc0Ieuqpp+KZZ56J888/v7LfMffB2u1kHHP/X+oB+v3f//3iHyppRuAll1wS//AP/3BGHm8CUi3/h0/T+pctW1Zjf9pOp5T4P+nUxubNm4v/00hjktIBXb3N0gcg/Yui3GapTZs0aVKjJv048caNGxtMu9ZWO6XeuzRmYu3atZWaNWvWFPsaSlumU7tbt24tjr+G3G7pX9SpB+TJJ58sTj+mY6w6x9wHa7eTccyduj3Td8IZeby9zwHnvMdp/o888kgxmn7KlCnFNP/XXnutwbbd1KlTSz/72c9Kr776ajGzIE23TFM5y22SZi2kmQpPPvlkMbXzi1/84kmndp5//vmln/zkJ8XUzs997nNn3TT/AwcOlF544YViSR/NOXPmFOvlS0TUVjulKbAXX3xxMbMjLX379q3X09VP1W7ptnT8pam+W7ZsKT3zzDOlgQMHFtOEG3q7/eVf/mVxPKXPZvVLILz99tuVGsfc+283x9y7mz59eunZZ58tPosvvvhiMc0/zUBbunTpGXm8CUh14J//+Z9LF154Yalp06alSy+9tMb0z4aofC2LFBy7du1auuaaa0qbNm2q3J6md6ZLAaQpns2aNSt95jOfKT4c1R06dKh0yy23lNq1a1dq3rx5cbC/8cYbpbNJ+vJOX/D5kqap12Y7pWsAXXfddUVITUta37NnT+lsbLf0pTVixIhSx44di+MvXa8m7c/bpCG228naLC3pGj9ljrn3326OuXf3pS99qfLdmD6TQ4cOrYSjM/F4q0r/8/76nAAAzm7GIAEAZAQkAICMgAQAkBGQAAAyAhIAQEZAAgDICEgAABkBCQAgIyABAGQEJACAjIAEAJARkAAAoqb/B9uICPePmDRFAAAAAElFTkSuQmCC", + "text/plain": [ + "

" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.hist(spectra['snr'])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "eaf14bd7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "10633" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "spectra= spectra[spectra['snr'] > 200]\n", + "len(spectra)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "f0fe2d8d", + "metadata": {}, + "outputs": [], + "source": [ + "spectra['mg_fe'] = spectra['raw_mg_h'] - spectra['raw_fe_h']\n", + "spectra['ce_fe'] = spectra['raw_ce_h'] - spectra['raw_fe_h']\n", + "spectra['log_age_L'] = np.log10(spectra['age_L'])" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "28f49cf5", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-03T06:40:33.298414Z", + "iopub.status.busy": "2026-06-03T06:40:33.298281Z", + "iopub.status.idle": "2026-06-03T06:40:33.381236Z", + "shell.execute_reply": "2026-06-03T06:40:33.380858Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(8575,) (10633, 8575) (10633, 8575) float64\n" + ] + } + ], + "source": [ + "labels = spectra[['raw_teff', 'raw_logg', 'raw_fe_h', 'mg_fe', 'ce_fe', 'log_age_L']]\n", + "\n", + "# The wavelength/flux/ivar columns hold one array PER ROW. We need a single\n", + "# 1-D dispersion array and 2-D (n_stars, n_pixels) flux/ivar arrays -- NOT the\n", + "# raw pandas Series (a Series-of-arrays, which is what broke searchsorted).\n", + "def to_array(x):\n", + " if isinstance(x, str): # stringified list (e.g. from CSV)\n", + " return np.fromstring(x.strip(\"[] \\n\"), sep=\",\")\n", + " return np.asarray(x, dtype=float)\n", + "\n", + "dispersion = to_array(spectra['wavelength'].iloc[0]) # (n_pixels,)\n", + "flux = np.vstack([to_array(x) for x in spectra['flux']]) # (n_stars, n_pixels)\n", + "ivar = np.vstack([to_array(x) for x in spectra['ivar']]) # (n_stars, n_pixels)\n", + "\n", + "# Sanity checks -- all must pass before normalizing.\n", + "assert dispersion.ndim == 1, dispersion.shape\n", + "assert flux.shape == ivar.shape == (len(spectra), dispersion.size)\n", + "assert np.all(np.diff(dispersion) > 0), \"dispersion must be sorted ascending\"\n", + "print(dispersion.shape, flux.shape, ivar.shape, dispersion.dtype)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "58e5e366", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-03T06:40:33.382937Z", + "iopub.status.busy": "2026-06-03T06:40:33.382796Z", + "iopub.status.idle": "2026-06-03T06:40:33.773418Z", + "shell.execute_reply": "2026-06-03T06:40:33.773040Z" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABdEAAAMWCAYAAAAeaM88AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAzYRJREFUeJzs/Ql4FGXW+H8fyMKSCSEEWTKsOoBIIipL2EZAQgISEFFRUQaUQWbYjAQRRCX4KJsKOEEQGCRIZHl0BHFwgCCbPAHZRAkyiBo2DYsYCMEYAvR7nfv/dv3SSTokkKU7/f1cV0FX1d2dqt7urlOnzl3BZrPZBAAAAAAAAAAA5FEx7yIAAAAAAAAAAEAQHQAAAAAAAACAApCJDgAAAAAAAACAEwTRAQAAAAAAAABwgiA6AAAAAAAAAABOEEQHAAAAAAAAAMAJgugAAAAAAAAAADhBEB0AAAAAAAAAACcIogMAAAAAAAAA4ARBdABl5ttvv5XY2Fg5evToTT3OypUrpUWLFlKlShWpUKGC7N+/3yyPi4uTP/3pT+Lr62uWnz9/vpi2HAAAz7BlyxbTh+r/AADAfXz++efSunVr8fPzM3356tWri/XxGzVqJFFRUcX6mIArI4gOoEyD6JMnT76pIPrZs2dl4MCBctttt8m6detkx44d0rRpUxNIHz16tHTt2lU2bdpklvv7+xfr9gMAAAAA4GpsNpv0799ffHx8ZM2aNeZ4uHPnzmW9WYBb8y7rDQBQNn777TepWrWq2z/93333nWRnZ8uTTz7p8KPg4MGD5v+hQ4dK27Zty3ALAQDw7L4aAABP4gr9988//yy//vqrPPjgg9KtW7cy3RagvCATHfAAWjJFL9/at2+fPPzwwxIYGGgyt/fs2SOPPfaYuQxLS6Ho/48//rgcO3bMum96erp4e3vLG2+8YS375ZdfpGLFihIQECBXrlyxlmvm9y233GLOel9PfHy8PPLII+a2Zovr9umky+02btxoOvxq1aqZHyEdO3Y0l6TZDR48WDp16mRuP/roo+b+Xbp0MZMG1VVYWJhZrm0BAHBVrthXF0Sz2tq3b2/6Z73Sq3v37ibLLbdPPvlE7rzzTqlUqZLceuut8vbbb1v7mpOWXBsyZIjUqFFD/vCHP0ivXr3kxx9/NO20PQAArsgV+2/dpnr16pnbL7zwgtk+/ft2R44ckQEDBkitWrVM/9y8eXN55513bvg50CvC77nnHrOft99+u7z33ns3/FiAKyOIDniQfv36mRrhH374obz77rumjEqzZs1k9uzZsn79epk+fbqkpqZKmzZtTOetNICt8xrQttNAtna2Fy9elF27dlnLtc19992X58A4P3pwPGXKFHNbO2w98NZJl6uEhASJiIgwf3/JkiXyv//7v+bAOjIy0gqkv/zyy1Znr4+l9587d66ZXnrpJbN88eLFZrm2BQDA1blSX+3MsmXL5IEHHjB/d/ny5bJo0SJJS0szJ7G3b9/ucFCt+xMUFGTGL5kxY4Zpr/16TteuXZPevXubx9WD/VWrVpmT4D169LjhbQQAwFP777/+9a/y8ccfm9ujRo0yx8Pat9pLqurfTE5Olrfeekv+/e9/m2NwDdJrqdWi+vrrryUmJkaee+4568S5nhTftm1bkR8LcHk2AOXepEmT9HS17ZVXXimw3ZUrV2wZGRk2Pz8/29tvv20tf+mll2xVqlSx/f7772b+r3/9q61Hjx62O++80zZ58mSz7KeffjJ/Y8GCBYXerg8//NDcZ/PmzQ7LL126ZKtRo4atd+/eDsuvXr1qa9mypa1t27bWMr2vPoY+Vk6LFy82y3fv3l3o7QEAoKy4al9t72ftfbX2xcHBwbbQ0FBz2+7ixYu2WrVq2Tp06GAta9Omja1+/fq2rKwsh3ZBQUHmMe3Wrl1r5ufNm+fwt6dOnWqW63MDAIArctX+OyUlxdznjTfecFgeGRlpq1evnu3ChQsOy0eOHGmrXLmy7ddffy3032jYsKG5z7Fjx6xlmZmZ5lh+2LBhhX4cwF2QiQ54kIceeshhPiMjw2R86RlzvYxMJ72E+tKlS3Lo0CGrnZZUyczMlKSkJOssuF62HR4eLomJidYypctulv4drd82aNAgcwmbfdJMNc1K2717t9lGAADKG1fvqw8fPmzqrOqg3nq5uZ1uk277zp07TS1Y3T69lL1v377i6+vr0E6zznPaunWr+V8HQMtJL3sHAMAduHr/rX7//XeT6a510rUcW85j7fvvv9+s1368KO666y5p0KCBNV+5cmVp2rSpQ9kaoLxgYFHAg9StW9dhXuugaSeqpU70ki69nEwvD9MOVDtyuw4dOphOVjvv+vXrm0vTtGM/efKkxMXFmR8Iuk5rnTZu3Pimt/P06dPmf60p54wG2f38/G76bwEA4Epcva8+d+5cvtupgoODzQlvLe2iNVt1ql27dp52uZfpY2pwQcu2FdQOAABX5er9t72/1YC5Pq5O+bGXmiksLdmWm5ajybmPQHlBEB3wIDnrp124cMHUP5s0aZKMHz/eWp6VlWUC1DlpBpkO4Kmdtw5QUqdOHQkNDTUdudqyZYv5gRAVFVUs21mzZk3zv3bs7dq1y7cNB9YAgPLI1ftq+8Gy1nXNTTPUNTtdB1XTALrui/3EeE6nTp3K85h6UK/7lDOQnrsdAACuytX7b6X9s5eXl7mabMSIEfm2KY6kOKC8IogOeHAnrwe4epY4p3/+859y9erVPO310rEJEyaIv7+/dRmZZoJrkFuD3XrgXNTLy+x/O/dZ6o4dO0r16tXNoCcjR468gb0DAMD9uUJfnZsOkvbHP/7RDAI6duxYK2igl6f/61//kvbt25uMOtW6dWtZvXq1vPnmm1ZJF82o08BCTp07dzaDjurgo3//+9+t5StWrLipbQUAoCy4Yv+ttH/u2rWrfPXVV2YA0Jzl1gBcH0F0wEPp5WT33nuvvPHGGybzu1GjRqYm6aJFi0wAOzet1aYdvp4FX7JkibVcO3M9w64/FHS08KIICQkx/y9YsMD8YND6aXrmWzPS9MeC1kTXM/Va1qVWrVpy9uxZM/q3/j9v3rxieBYAAHBdrtBX56aZ5hrwfuKJJ0xW3LBhw0xmnW7j+fPnZdq0aVbbV199VXr16iWRkZHy7LPPmm3TdloTNmcmno53oifQY2JiJD09XVq1aiU7duyQ999/3/qbAAC4C1fsv+3efvttk/n+5z//2Zy41m27ePGifP/99/Lpp5/Kpk2biuXvAOURv0gBD6ZZZHomety4cdKvXz8zAJgOXhIQEJCn7d13322VWcl5Ftx+W9fnVw+tIBownz17tgmMd+nSxdSK045bPfnkk7J582aTsaYH6Pp39AB837595kcGAACeoKz76vxonVfNMNfaqo8++qg89dRTJmCg/bYemOcMjmt2ur3dmDFjzGBmDzzwgEMQQYPk2v8/9thjJgiv67/44gtJSEgw6/MLOAAA4Mpcsf9Wd9xxhzmm1oS2l156SSIiImTIkCHy0UcfcZwNXEcFm15jAgAAAAAlLDs7W+666y5TEmbDhg3XDUBoxvv//d//mYHXAAAAgLJCORcAAAAAJUKz27p37y5169Y1A4W+++67cujQIXM5eU7Lly+Xn376yQymppnpO3fuNJfB6+XwBNABAABQ1giiAyh2eoFLfgOm5KSjguccwRwAAJS/vlrrrOoApDqeiY+Pj9xzzz3y2Wef5RkgTcdG0YFEX3vtNTNIqQbdBw8ebOYBAEDp9d/6+AUVrdDH1r8BeBrKuQAodvHx8aY+akG0bqrWQQcAAKWPvhoAAPdTGv23DjZ67Ngxp+s7d+4sW7ZsueHHB9wVQXQAxU4HEEtJSSmwTbNmzUzWGQAAKH301QAAuJ/S6L8PHDggWVlZTtfrY+vfADwNQXQAAAAAAAAAAJyo6GwFAAAAAAAAAACejoFFC+natWvy888/m8tWGAwRAFAadEAfHZQvODhYKlbkvHdR0XcDAEobfffNoe8GALhq300QvZA0gF6/fv3ien0AACi0EydOSL169XjGioi+GwBQVui7bwx9NwDAVftuguiFZB+UQZ/QatWqFc+rAwBAAdLT080JXAbhvTH03QCA0kbffXPouwEArtp3E0QvJHsJFw2gE0QHAJQmyojd3PNG3w0AKG303Tf3vNF3AwBcre+mwCoAAAAAAAAAAE4QRAcAAAAAAAAAwAmC6AAAAAAAAAAAOEEQHQAAAAAAAAAAJwiiAwAAAAAAAADgBEF0AAAAAAAAAACcIIgOAAAAAAAAAIATBNEBAAAAAAAAAHCCIDoAAAAAAAAAAE4QRAcAAAAAAAAAwAmC6AAAAAAAAAAAEEQHAAAAAAAAAKBovIvYHkA502j82iLf5+i0XiWyLQAA4ProuwEAgLvgdwvKC8q5AAAAAAAAAADgBEF0AAAAAAAAAACcIIgOAAAAAAAAAIAT1EQHypkbqTcGAAAAAAAAIH8E0QGUOAYSAQAAAAAAgLuinAsAAAAAAAAAAE4QRAcAAAAAAAAAwAnKuQAAAADlXFFLqx2d1qvEtgUAAABwN2SiAwAAAAAAAABAEB0AAAAAAAAAADfKRN+2bZv07t1bgoODpUKFCrJ69WprXXZ2trzwwgsSGhoqfn5+ps1f/vIX+fnnnx0eIysrS0aNGiU1a9Y07fr06SMnT550aJOWliYDBw6UgIAAM+nt8+fPl9p+AgAAAAAAAADcU5nWRL906ZK0bNlSnnrqKXnooYcc1v3222+yb98+efnll00bDYRHR0ebIPmePXusdrrs008/lRUrVkhQUJDExMRIVFSU7N27V7y8vEybAQMGmMD6unXrzPwzzzxjAul6PwAlX1cVAAAAAAAAcFdlGkTv2bOnmfKjGeOJiYkOy+Li4qRt27Zy/PhxadCggVy4cEEWLVokS5culfDwcNMmISFB6tevLxs3bpTIyEg5dOiQCZ7v3LlTwsLCTJuFCxdK+/bt5fDhw9KsWbNS2FMAAAAAAAAAgDtyq4FFNWiuZV+qV69u5jXbXMu+REREWG207EtISIgkJSWZ+R07dpiAvD2Artq1a2eW2dsAAAAAAAAAAOBymehF8fvvv8v48eNNaZZq1aqZZadOnRJfX18JDAx0aFu7dm2zzt6mVq1aeR5Pl9nb5Edrretkl56eXox7AwAAAAAAAABwB26Ria7Z5o899phcu3ZN5s6de932NpvNZKzb5bztrE1uU6dOtQYi1UlLxAAAAAAAAAAAPEtFdwig9+/fX1JSUkyNdHsWuqpTp45cvnzZDDqa05kzZ0w2ur3N6dOn8zzu2bNnrTb5mTBhgikfY59OnDhRrPsFAAAAAAAAAHB9Fd0hgH7kyBEzUGhQUJDD+latWomPj4/DAKSpqamSnJwsHTp0MPM6gKgGwXft2mW1+fLLL80ye5v8VKpUyQTsc04AAAAAAAAAAM9SpkH0jIwM2b9/v5mUZpvr7ePHj8uVK1fk4Ycflj179sgHH3wgV69eNTXMddLsc6VlVoYMGSIxMTHy+eefy1dffSVPPvmkhIaGSnh4uGnTvHlz6dGjhwwdOlR27txpJr0dFRUlzZo1K8vdBwDA7Wzbtk169+5tBvLWsmirV692WK/L8pveeOMNq02XLl3yrNeybTnpVWYDBw60yqrp7fPnz5fafgIAAAAA4BJBdA2Q33333WZSY8aMMbdfeeUVOXnypKxZs8b8f9ddd0ndunWtKSkpyXqMWbNmSd++fU3GeseOHaVq1ary6aefipeXl9VGg/AaWI+IiDDTnXfeKUuXLi2TfQYAwJ1dunRJWrZsKXPmzMl3vV4RlnN67733TJD8oYcecminJ7Rztps/f77Deh1IXE+sr1u3zkx6WwPpAAAAAACUNu+yfMo1E00H+HSmoHV2lStXlri4ODM5U6NGDUlISLjh7QQAAP+fnj17mskZHYskp08++US6du0qt956q8NyPemdu63doUOHTOBcrx4LCwszyxYuXGhKtB0+fJgryQAAAAAApcqla6IDAAD3pQN7r1271pRey02vEqtZs6a0aNFCxo4dKxcvXrTW7dixw5RwsQfQVbt27cyynFejAQAAAABQ7jPRAQBA+bVkyRLx9/eXfv36OSx/4oknpHHjxiYTXQcDnzBhgnz99dfWQOE6/kmtWrXyPJ4u03XOZGVlmckuPT29WPcHAAAAAOCZyEQHAAAlQuuha8BcS6/lroeuA4CHhISYAUU/+ugj2bhxo+zbt89qo3XU8yvzlt9yu6lTp1oDkepUv379Yt4jAADKnytXrshLL71kTnBXqVLFlGB79dVX5dq1aw59cGxsrBlYXNtoadaDBw86PI6eyB41apS50szPz0/69OljxjgDAKA8IIgOAACK3RdffGHql//1r3+9btt77rlHfHx85MiRI2ZeM9S1FExuZ8+eldq1azt9HM1ov3DhgjWdOHHiJvcCAIDyb/r06fLuu++aQcN1XJIZM2bIG2+84TDumC6bOXOmabN7927TV3fv3t2hHFt0dLSsWrVKVqxYIdu3b5eMjAyJioqSq1evltGeAQBQfCjnAgAAit2iRYukVatW0rJly+u21Uy27OxsqVu3rpnXAUQ1CL5r1y5p27atWfbll1+aZR06dHD6OJUqVTITAAAoPB2L5IEHHpBevXqZ+UaNGsny5ctlz549Vhb67NmzZeLEiVaJNi3Zpie2ly1bJsOGDTN9tPb9S5cuNVebqYSEBHNVmF5tFhkZyUsCAHBrZKIDAIBC06yy/fv3m0mlpKSY28ePH3eoRf7hhx/mm4X+ww8/mEvE9cD86NGj8tlnn8kjjzwid999t3Ts2NG0ad68ufTo0cOUfdm5c6eZ9LZmszVr1oxXCwCAYtSpUyf5/PPP5bvvvjPzOk6JZpLff//9Vl+vY5JERERY99GT1p07d7YG/N67d685IZ6zjZZ+0dJtDAoOACgPyEQHAACFpsHvrl27WvNjxowx/w8aNEji4+PNbb2MW7PWHn/88Tz39/X1NQfqb7/9tgnIa4aaZr5NmjRJvLy8rHYffPCBjB492joY17qqegk5AAAoXi+88ILJJL/99ttNX6zlV15//XWrH7cP6p27pJrOHzt2zGqjfXxgYGCeNgwKDgAoDwiiAwCAQtOBxDRAXpBnnnnGTPnRoPnWrVuv+3dq1KhhLgMHAAAla+XKlabP1dIsLVq0MFeYaX1zzSTXk+R2uQf3vt6A34UdFHzy5MnFsBcAAJQsyrkAAAAAAOChnn/+eRk/frw89thjEhoaKgMHDpTnnnvOBLiVDiKqcmeUnzlzxspO1zaXL1+WtLQ0p23yw6DgAAB3QRAdAAAAAAAP9dtvv0nFio6hAS3rcu3aNXO7cePGJkiemJhordeAuV5ZZh/wWwcT9/HxcWiTmpoqycnJ1x0UvFq1ag4TAACuiHIuAAAAAAB4qN69e5sa6A0aNDDlXL766iuZOXOmPP3002a9lmPR8i5TpkyRJk2amElvV61aVQYMGGDaBAQEyJAhQyQmJkaCgoJMWbaxY8eazPbw8PAy3kMAAG4eQXQAAAAAADxUXFycvPzyyzJ8+HBTfkVroQ8bNkxeeeUVq824ceMkMzPTtNGSLWFhYbJhwwbx9/e32syaNUu8vb2lf//+pm23bt3MoOM5Bw4HAMBdEUQHAAAAAMBDaSB89uzZZnJGs9FjY2PN5EzlypVNQF4nAADKG2qiAwAAAAAAAADgBEF0AAAAAAAAAACcIIgOAAAAAAAAAIAT1EQHAAAAAAAA4BIajV9bpPZHp/UqsW0B7MhEBwAAAAAAAADACYLoAAAAAAAAAAA4QRAdAAAAAAAAAAAnqIkOAAAAAAAAwCNqqCvqqKOoyEQHAAAAAAAAAMAJgugAAAAAAAAAADhBORcALonLsQAAAAAAcP9jdaA8IBMdAAAAAAAAAAAnCKIDAAAAAAAAAOAEQXQAAAAAAAAAAJwgiA4AAAAAAAAAgBME0QEAAAAAAAAAcIIgOgAAAAAAAAAAThBEBwAAAAAAAADACYLoAAAAAAAAAAAQRAcAAAAAAAAAoGjIRAcAAAAAAAAAwAmC6AAAAAAAAAAAOOHtbAUAAAAAAACA8qnR+LVlvQmA2yATHQAAFNq2bdukd+/eEhwcLBUqVJDVq1c7rB88eLBZnnNq166dQ5usrCwZNWqU1KxZU/z8/KRPnz5y8uRJhzZpaWkycOBACQgIMJPePn/+PK8UAAAAAMCzgujXOxC32WwSGxtr1lepUkW6dOkiBw8edGjDgTgAAKXn0qVL0rJlS5kzZ47TNj169JDU1FRr+uyzzxzWR0dHy6pVq2TFihWyfft2ycjIkKioKLl69arVZsCAAbJ//35Zt26dmfS2BtIBAAAAAPCoIPr1DsRnzJghM2fONOt3794tderUke7du8vFixetNhyIAwBQenr27Cmvvfaa9OvXz2mbSpUqmT7bPtWoUcNad+HCBVm0aJG89dZbEh4eLnfffbckJCTIgQMHZOPGjabNoUOHTOD8n//8p7Rv395MCxculH//+99y+PDhUtlPAAAAAABcIohe0IG4ZqHPnj1bJk6caNaHhITIkiVL5LfffpNly5aZNhyIAwDgerZs2SK1atWSpk2bytChQ+XMmTPWur1790p2drZERERYy/SKM+3nk5KSzPyOHTtMCZewsDCrjZaE0WX2NgAAAAAAiKfXRE9JSZFTp045HGRrZlvnzp2tA+iSPBDXMjHp6ekOEwAAuP4J8g8++EA2bdpkss31SrL77rvP9KtK+3ZfX18JDAx0uF/t2rXNOnsbDcLnpsvsbei7AQAAAADi6UF0+0GyHlQXdJBdUgfiU6dOtQYz06l+/frFsl8AAJRnjz76qPTq1cuc0NZxT/7zn//Id999J2vXri3wfnoFmo6PYpfztrM2udF3AwAAAAA8Kohul/tg+XoH0MV1ID5hwgRTLsY+nThx4oa2HwAAT1a3bl1p2LChHDlyxMxrjfTLly9LWlqaQzst+WI/ca5tTp8+neexzp49m+fkek703QAAAAAAjwqi6wG0yp0tnvsgu6QOxLV0TLVq1RwmAABQNOfOnTMnojWYrlq1aiU+Pj6SmJhotUlNTZXk5GTp0KGDmdeBRPUE9q5du6w2X375pVlmb0PfDQAAAAAoLd7ioho3bmwC4HqQfffdd5tlGjDfunWrTJ8+Pc+BeP/+/R0OxGfMmJHnQLxt27aFPhAHAAB5ZWRkyPfff+8whsn+/fulRo0aZoqNjZWHHnrIBM2PHj0qL774otSsWVMefPBB015LpA0ZMkRiYmIkKCjI3Gfs2LESGhoq4eHhpk3z5s2lR48eZlDS+fPnm2XPPPOMREVFSbNmzXhZAAAAgFwajS+4fCIANw6iF3Qg3qBBA4mOjpYpU6ZIkyZNzKS3q1atKgMGDDDtORAHAKB07dmzR7p27WrNjxkzxvw/aNAgmTdvnhw4cEDef/99OX/+vAmka9uVK1eKv7+/dZ9Zs2aJt7e3OQGemZkp3bp1k/j4ePHy8rLa6OCko0ePtgYP79Onj8yZM6dU9xUAAAAAgDIPohd0IK4H0+PGjTMH18OHDzclW8LCwmTDhg0ciAMAUEa6dOlixhVxZv369dd9jMqVK0tcXJyZnNET6gkJCTe8nQAAAAAAlIsg+vUOxHXgT70sXCdnOBAHAAAAAAAAAHhcTXQAAAAAAAAAKOsa8ken9eJF8HAVy3oDAAAAAAAAAABwVQTRAQAAAAAAAABwgiA6AAAAAAAAAABOEEQHAAAAAAAAAMAJBhYFAAAAAAAA3HjgSwAli0x0AAAAAAAAAACcIIgOAAAAAAAAAIATBNEBAAAAAAAAAHCCIDoAAAAAAAAAAE4wsCjgwhhIBAAAAAAAAChbZKIDAAAAAAAAAOAEQXQAAAAAAAAAAJwgiA4AAAAAAAAAgBME0QEAAAAAAAAAcIIgOgAAAAAAAAAAThBEBwAAAAAAAACAIDoAAAAAAAAAAEVDJjoAAAAAAAAAAE4QRAcAAAAAAAAAwAmC6AAAAAAAAAAAOEEQHQAAAAAAAAAAJwiiAwAAAADgwX766Sd58sknJSgoSKpWrSp33XWX7N2711pvs9kkNjZWgoODpUqVKtKlSxc5ePCgw2NkZWXJqFGjpGbNmuLn5yd9+vSRkydPlsHeAABQ/AiiAwAAAADgodLS0qRjx47i4+Mj//nPf+Tbb7+Vt956S6pXr261mTFjhsycOVPmzJkju3fvljp16kj37t3l4sWLVpvo6GhZtWqVrFixQrZv3y4ZGRkSFRUlV69eLaM9AwCg+HgX42MBAAAAAAA3Mn36dKlfv74sXrzYWtaoUSOHLPTZs2fLxIkTpV+/fmbZkiVLpHbt2rJs2TIZNmyYXLhwQRYtWiRLly6V8PBw0yYhIcE87saNGyUyMrIM9gwAgOJDJjoAAAAAAB5qzZo10rp1a3nkkUekVq1acvfdd8vChQut9SkpKXLq1CmJiIiwllWqVEk6d+4sSUlJZl5Lv2RnZzu00dIvISEhVpv8aAmY9PR0hwkAAFdEEB0AAAAAAA/1448/yrx586RJkyayfv16+dvf/iajR4+W999/36zXALrSzPOcdN6+Tv/39fWVwMBAp23yM3XqVAkICLAmzVwHAMAVEUQHAAAAAMBDXbt2Te655x6ZMmWKyULX8ixDhw41gfWcKlSo4DCvZV5yL8vtem0mTJhgSsHYpxMnTtzk3gAAUDIIogMAgELbtm2b9O7d21yirQfFq1evttbpZdwvvPCChIaGip+fn2nzl7/8RX7++WeHx+jSpYu5b87pscceyzPI2cCBA63MNL19/vx5XikAAIpZ3bp15Y477nBY1rx5czl+/Li5rYOIqtwZ5WfOnLGy07XN5cuXTf/trE1+tCxMtWrVHCYAAFwRQXQAAFBoly5dkpYtW8qcOXPyrPvtt99k37598vLLL5v/P/74Y/nuu++kT58+edpqhltqaqo1zZ8/32H9gAEDZP/+/bJu3Toz6W0NpAMAgOLVsWNHOXz4sMMy7b8bNmxobjdu3NgEyRMTE631GjDfunWrdOjQwcy3atVKfHx8HNpo/56cnGy1AQDAnXmX9QYAAAD30bNnTzPlRzPGcx48q7i4OGnbtq3JZmvQoIG1vGrVqlZmW26HDh0ygfOdO3dKWFiYWaYDnLVv394c5Ddr1qxY9wkAAE/23HPPmUC3lnPp37+/7Nq1SxYsWGAmpVeMRUdHm/VaN10nva19uZ70tv8GGDJkiMTExEhQUJDUqFFDxo4da65OCw8PL+M9BADg5hFEBwAAJUbrm+rBd/Xq1R2Wf/DBB5KQkGAu8dag/KRJk8Tf39+s27FjhzkYtwfQVbt27cyypKQkp0H0rKwsM9mlp6eX2H4B5V2j8WuLfJ+j03qVyLYAKFlt2rSRVatWmfrkr776qsk8nz17tjzxxBNWm3HjxklmZqYMHz7clGzRPnrDhg1W361mzZol3t7eJhCvbbt16ybx8fHi5eXFSwgAcHsE0QEAQIn4/fffZfz48SZLLWeNUz0ot18arpd560H7119/bWWxa83VWrVq5Xk8XZa7HmtOU6dOlcmTJ/NqAgBQRFFRUWZyRk+Ix8bGmsmZypUrmyvQdAIAoLwhiA4AAIqdDjKqg4Veu3ZN5s6dm6ceul1ISIi5LLx169amjvo999xjHaznZrPZ8l1up8H4MWPGOGSi169fv5j2CAAAAADgqQiiAwCAYg+g66XcKSkpsmnTJocs9Pxo4FwHIzty5Ii5rRnqp0+fztPu7NmzpvyLM5UqVTITAAAAAADFiSA6AAAo9gC6BsQ3b95sBhe7noMHD5r71a1b18zrAKJaS10HNtNBSdWXX35plunAZwAAAEB5H2sEgGupKC7sypUr8tJLL5m6qVWqVJFbb73VDHSil4bnvLRb67IFBwebNl26dDEH4znpIGOjRo2SmjVrip+fn/Tp00dOnjxZBnsEAIB7y8jIkP3795tJaba53j5+/Ljptx9++GHZs2ePGTj06tWrpoa5TpcvXzbtf/jhB9OXa5ujR4/KZ599Jo888ojcfffd0rFjR9OmefPm0qNHD1P2ZefOnWbS21qr1dmgogAAAAAAeGQQffr06fLuu+/KnDlz5NChQzJjxgx54403HAYq0WUzZ840bXbv3m0uAe/evbtcvHjRahMdHW1GG1+xYoVs377dBAD0QFwP7gEAQOFp8FsD3joprUGut1955RVzgnrNmjXm/7vuustkltunpKQk097X11c+//xziYyMNAHx0aNHS0REhGzcuFG8vLysv6NB+NDQULNOpzvvvFOWLl3KSwUAAAAAKHUuXc5lx44d8sADD0ivXr3MfKNGjWT58uXmAN6ehT579myZOHGi9OvXzyxbsmSJqZe6bNkyGTZsmLn0e9GiRebAOzw83LRJSEgwA43pAbsexAMAgMLRK760/3WmoHVK+9+tW7de9+/UqFHD9NcAAAAAAJQ1l85E79Spk8lW++6778z8119/bTLJ77//fusScr1EXDPU7HRAsc6dO1sZb3v37jV1VnO20dIvISEhVpv8aAmY9PR0hwkAAAAAAAAA4FlcOhP9hRdeMJnkt99+u7nEW8uvvP766/L444+b9RpAV5p5npPOHzt2zGqjl44HBgbmaWO/f36mTp0qkydPLoG9AgAAAAAAAAC4C5fORF+5cqW5lFtLs+zbt8+UannzzTfN/zlVqFAhz6XkuZfldr02EyZMMAF8+3TixImb3BsAAAAAAAAAgLtx6Uz0559/XsaPHy+PPfaYmdcBxjTDXLPEBw0aZAYRVZpRroOW2Z05c8bKTtc2ly9flrS0NIdsdG3ToUMHp39by8LoBAAAAAAAAADwXC6dif7bb79JxYqOm6hlXa5du2ZuN27c2ATJExMTrfUaMNcBy+wB8latWomPj49Dm9TUVElOTi4wiA4AAAAAAAAAgEtnovfu3dvUQG/QoIG0aNFCvvrqK5k5c6Y8/fTTZr2WY4mOjpYpU6ZIkyZNzKS3q1atKgMGDDBtAgICZMiQIRITEyNBQUFSo0YNGTt2rMlqDw8PL+M9BAAAAAAAAAC4MpcOosfFxcnLL78sw4cPN+VXgoODZdiwYfLKK69YbcaNGyeZmZmmjZZsCQsLkw0bNoi/v7/VZtasWeLt7S39+/c3bbt16ybx8fEmqx0AAAAAAAAAALcMomsgfPbs2WZyRrPRY2NjzeRM5cqVTUBeJwAAAAAAAAAAykVNdAAAAAAAAAAAyhJBdAAAAAAAAAAAnCCIDgAAAAAAAAAAQXQAAAAAAAAAAIqGTHQAAAAAAAAAAJwgiA4AAAAAAAAAgBME0QEAAAAAAAAAKM4g+q233irnzp3Ls/z8+fNmHQAAcC303QAAlD/07wAAuHAQ/ejRo3L16tU8y7OysuSnn34qju0CAADFiL4bAIDyh/4dAIDS4V2UxmvWrLFur1+/XgICAqx5Dap//vnn0qhRo+LdQgAopEbj1xbpuTo6rRfPLco9+m4AAMof+nfAvY49AXhYEL1v377m/woVKsigQYMc1vn4+JgA+ltvvVW8WwgAAG4YfTcAAOUP/TsAAC4cRL927Zr5v3HjxrJ7926pWbNmSW0XAAAoBvTdAACUP/TvAAC4cBDdLiUlpfi3BAAAlBj6bgAAyh/6dwAAXDiIrrT+uU5nzpyxzoLbvffee8WxbQAAoBjRdwMAUP7QvwMA4KJB9MmTJ8urr74qrVu3lrp165oa6QAAwHXRdwMAUP7QvwMA4MJB9HfffVfi4+Nl4MCBxb9FAACg2NF3AwBQ/tC/AwBQOireyJ0uX74sHTp0KP6tAQAAJYK+GwCA8of+HQAAFw6i//Wvf5Vly5YV/9YAAIASQd8NAED5Q/8OAIALl3P5/fffZcGCBbJx40a58847xcfHx2H9zJkzi2v7AABAMaDvBgCg/KF/BwDAhYPo33zzjdx1113mdnJyssM6BhkFAMD10HcDAFD+0L8DAODCQfTNmzcX/5YAAIASU1x997Zt2+SNN96QvXv3SmpqqqxatUr69u1rrbfZbDJ58mRzxVpaWpqEhYXJO++8Iy1atLDaZGVlydixY2X58uWSmZkp3bp1k7lz50q9evWsNnrf0aNHy5o1a8x8nz59JC4uTqpXr14s+wEAQHnAsTlw8xqNX8vTCKBkgugAAMAzXbp0SVq2bClPPfWUPPTQQ3nWz5gxw5R1i4+Pl6ZNm8prr70m3bt3l8OHD4u/v79pEx0dLZ9++qmsWLFCgoKCJCYmRqKiokxg3svLy7QZMGCAnDx5UtatW2fmn3nmGRk4cKC5HwAAAAC4+smWo9N6lci2wI2C6F27di2wbMumTZtuZpsAAEAxK66+u2fPnmbKj2ahz549WyZOnCj9+vUzy5YsWSK1a9c2A5IPGzZMLly4IIsWLZKlS5dKeHi4aZOQkCD169c3Y61ERkbKoUOHTPB8586dJpNdLVy4UNq3b2+C8c2aNbuBZwAAgPKHY3MAAFw4iG6vh26XnZ0t+/fvN/XRBw0aVFzbBgAAiklp9N0pKSly6tQpiYiIsJZVqlRJOnfuLElJSSaIrtnm+rdztgkODpaQkBDTRoPoO3bskICAACuArtq1a2eWaRuC6AAAlF7/DgAAbjCIPmvWrHyXx8bGSkZGBs8rAAAupjT6bg2gK808z0nnjx07ZrXx9fWVwMDAPG3s99f/a9WqlefxdZm9TX601rpOdunp6Te5RwAAuDaOzQEAKB0Vi/PBnnzySXnvvfeK8yEBAEAJKom+O3fZGC3zUlApmfza5Nf+eo8zdepUk61un7REDAAAnohjcwAAXDiIrpdfV65cuTgfEgAAlKDi7Lvr1Klj/s+dLX7mzBkrO13bXL58WdLS0gpsc/r06TyPf/bs2TxZ7jlNmDDB1Fy3TydOnCiW/QIAwN1wbA4AgAuUc7EPFpYzMyw1NVX27NkjL7/8cnFtGwAAKCal0Xc3btzYBMATExPl7rvvNss0YL5161aZPn26mW/VqpX4+PiYNv379zfLdDu0duuMGTPMvA4gqkHwXbt2Sdu2bc2yL7/80izr0KGD07+v9dd1AgDAU3BsDgCACwfR9RLpnCpWrGgG+Xr11VcdBgoDAACuobj6bq2f/v333zsMJqoDmNWoUUMaNGgg0dHRMmXKFGnSpImZ9HbVqlVlwIAB1nYMGTJEYmJiJCgoyNxv7NixEhoaKuHh4aZN8+bNpUePHjJ06FCZP3++WfbMM89IVFQUg4oCAFAC/TsAACiBIPrixYtv5G4AAKCMFFffrZnrXbt2tebHjBlj/h80aJDEx8fLuHHjJDMzU4YPH25KtoSFhcmGDRvE39/fYRA0b29vk4mubbt162bu6+XlZbX54IMPZPTo0VYAoE+fPjJnzpxi2QcAAMoLjs0BAHDhILrd3r175dChQ2aQrzvuuMO6dBsAALimm+27u3TpYkrBOKOPGxsbayZntAZ7XFycmZzRDPWEhIQibRsAAJ6KY3MAAFwwiK6Dfz322GOyZcsWqV69ujmY1jqlmpm2YsUKueWWW4p/SwEAwA2j7wYAoPyhfwcAoHRUvJE7jRo1StLT0+XgwYPy66+/msu1dUAwXaaXXgMAANdC3w0AQPlD/w4AgAtnoq9bt042btxoBv6y00vC33nnHQYvAQDABdF3AwBQ/tC/AwDgwpno165dEx8fnzzLdZmuAwAAroW+GwCA8of+HQAAFw6i33ffffLss8/Kzz//bC376aef5LnnnpNu3boV5/YBAIBiQN8NAED5Q/8OAEDpuKEg+pw5c+TixYvSqFEjue222+RPf/qTNG7c2CyLi4sr1g3U4PyTTz4pQUFBUrVqVbnrrrvMyON2OqhpbGysBAcHS5UqVaRLly6mVntOWVlZplZczZo1xc/PT/r06SMnT54s1u0EAMCVlWbfDQAASgf9OwAALlwTvX79+rJv3z5JTEyU//73vyaQrTXRw8PDi3XjdMDSjh07SteuXeU///mP1KpVS3744QepXr261WbGjBkyc+ZMiY+Pl6ZNm8prr70m3bt3l8OHD4u/v79pEx0dLZ9++qmsWLHCBONjYmIkKirKBOO9vLyKdZsBAHBFpdV3AwCA0kP/DgCAC2aib9q0yRxwp6enm3kNVmuG9+jRo6VNmzbSokUL+eKLL4pt46ZPn25+FCxevFjatm1rsue0XIxm0CkNAMyePVsmTpwo/fr1k5CQEFmyZIn89ttvsmzZMtPmwoULsmjRInnrrbdMoODuu++WhIQEOXDggBkcFQCA8qy0+24AAFDy6N8BAHDhILoGrIcOHSrVqlXLsy4gIECGDRtmssKLy5o1a6R169byyCOPmCx0DYAvXLjQWp+SkiKnTp2SiIgIa1mlSpWkc+fOkpSUZOY12zw7O9uhjZZ+0YC7vU1+tASMBhxyTgAAuJvS7rsBAEDJo38HAMCFg+hff/219OjRw+l6DVTnrFd+s3788UeZN2+eNGnSRNavXy9/+9vfTObc+++/b9ZrAF3Vrl3b4X46b1+n//v6+kpgYKDTNvmZOnWqCS7YJ82IBwDA3ZR23w0AAEoe/TsAAC4cRD99+rT4+Pg4Xe/t7S1nz56V4nLt2jW55557ZMqUKSYLXbPlNJtOA+s5VahQwWFey7zkXpbb9dpMmDDBlIKxTydOnLjJvQEAoPSVdt8NAABKHv07AAAuHET/4x//aGqJO/PNN99I3bp1pbjoY2kd15yaN28ux48fN7fr1Klj/s+dUX7mzBkrO13bXL582QxS6qxNfrQsjF76nnMCAMDdlHbfDQAA3Lt/16uyNeEsOjraIQktNjbWlEatUqWKdOnSRQ4ePJinJKqOu1KzZk3x8/OTPn36yMmTJ29oGwAAcOsg+v333y+vvPKK/P7773nWZWZmyqRJkyQqKqrYNq5jx45y+PBhh2XfffedNGzY0Nxu3LixCZInJiZa6zVgvnXrVunQoYOZb9WqlcnAy9kmNTVVkpOTrTYAAJRXpd13AwAA9+3fd+/eLQsWLJA777zTYfmMGTPMGCpz5swxbfQ4XAcrv3jxotVGg+6rVq2SFStWyPbt2yUjI8Nsw9WrV29wLwEAcB3eRWn80ksvyccffyxNmzaVkSNHSrNmzcwZ6kOHDsk777xjOseJEycW28Y999xzJtCt5Vz69+8vu3btMh26Tsp+dlzXa910nfR21apVZcCAAaaN1jMfMmSIxMTESFBQkNSoUUPGjh0roaGhEh4eXmzbCgCAKyrtvhtA0TUav5anDUCZ9+8a9H7iiSdk4cKF8tprrzlkoetApvp4/fr1M8uWLFliruxetmyZKbuqJVAXLVokS5cutY6zExISzNhiGzdulMjISF5hAIDnBNG1k0xKSpK///3vpma4dqZKO2vtFOfOnVtgiZSiatOmjTmTrX/r1VdfNZnn2nlrx243btw4c6Z9+PDhpmRLWFiYbNiwQfz9/a02s2bNMjVfNRCvbbt16ybx8fHi5eVVbNsKAIArKu2+GwAAuGf/PmLECOnVq5cJgucMoqekpJgSqjoYec7yp507dzbboEF0HaQ8OzvboY2WfgkJCTFtCKIDADwqiK60lMpnn31mAtbff/+96aw1AzwwMLBENlAv/yroMjT9kaC12XRypnLlyhIXF2cmAAA8TWn33QAAwL36dy3Bsm/fPlOqJTf7GGS5g/I6f+zYMauNr69vnr+tbXKPYZa7jrpOdunp6UXedgAASkORg+h22jlqpjgAAHAP9N0AAJQ/N9u/nzhxQp599llzRbcmoBWUwJaTBu1zL8vtem10ENPJkyffwFYDAOAmQXQARUO9UwAAAACuRkuxnDlzRlq1amUt05rq27ZtMwOJHj582CzTjPK6detabfQ+9ux0HWj08uXLJis+Zza6ttFxzpzRUjRjxoxxyETXOuoAALiaimW9AQAAAAAAoGzomGEHDhyQ/fv3W1Pr1q3NWGR6+9ZbbzVB8sTEROs+GjDfunWrFSDXALyPj49Dm9TUVElOTi4wiK611atVq+YwAQDgishEBwAAAADAQ/n7+5sBQHPy8/OToKAga3l0dLRMmTLF1FzXSW9XrVpVBgwYYNYHBATIkCFDJCYmxtyvRo0aMnbsWAkNDTUDlQKlhSvAAZQUgugAAAAAAMCpcePGSWZmpgwfPtyUbAkLCzM11DUAbzdr1izx9vaW/v37m7aa4R4fHy9eXl48swAAt0cQHQAAAAAAWLZs2eLwbOjgoLGxsWZyRgcljYuLMxMAAOUNNdEBAAAAAAAAAHCCIDoAAAAAAAAAAE4QRAcAAAAAAAAAwAmC6AAAAAAAAAAAOEEQHQAAAAAAAAAAgugAAAAAAAAAABQNmegAAAAAAAAAADjh7WwFAADAjWjUqJEcO3Ysz/Lhw4fLO++8I4MHD5YlS5Y4rAsLC5OdO3da81lZWTJ27FhZvny5ZGZmSrdu3WTu3LlSr149XhQAAAAALq/R+LVFvs/Rab1KZFtw88hEBwAAxWr37t2SmppqTYmJiWb5I488YrXp0aOHQ5vPPvvM4TGio6Nl1apVsmLFCtm+fbtkZGRIVFSUXL16lVcLAAAAAFCqyEQHAADF6pZbbnGYnzZtmtx2223SuXNna1mlSpWkTp06+d7/woULsmjRIlm6dKmEh4ebZQkJCVK/fn3ZuHGjREZG8ooBAAAAAEoNmegAAKDEXL582QTAn376aalQoYK1fMuWLVKrVi1p2rSpDB06VM6cOWOt27t3r2RnZ0tERIS1LDg4WEJCQiQpKcnp39ISMOnp6Q4TAAAAAAA3iyA6AAAoMatXr5bz58+bOuh2PXv2lA8++EA2bdokb731lin/ct9995kguDp16pT4+vpKYGCgw2PVrl3brHNm6tSpEhAQYE2auQ4AAAAAwM2inAsAACgxWpZFg+aaSW736KOPWrc1u7x169bSsGFDWbt2rfTr18/pY9lsNods9twmTJggY8aMseY1E51AOgAAAADgZhFEBwAAJeLYsWOmhvnHH39cYLu6deuaIPqRI0fMvNZK1zIwaWlpDtnoWvKlQ4cOTh9H66zrBAAAAABAcaKcCwAAKBGLFy82dc979epVYLtz587JiRMnTDBdtWrVSnx8fCQxMdFqk5qaKsnJyQUG0QEAAAAAKAlkogMAgGJ37do1E0QfNGiQeHv/v58bGRkZEhsbKw899JAJmh89elRefPFFqVmzpjz44IOmjdYzHzJkiMTExEhQUJDUqFFDxo4dK6GhoRIeHs6rBbioRuPXFvk+R6cVfJINAAAAcAUE0QEAQLHTMi7Hjx+Xp59+2mG5l5eXHDhwQN5//30z4KgG0rt27SorV64Uf39/q92sWbNM8L1///6SmZkp3bp1k/j4eHN/AAAAAABKE0F0AABQ7CIiIsxAoLlVqVJF1q9ff937V65cWeLi4swEAAAAAEBZoiY6AAAAAAAAAABOEEQHAAAAAAAAAMAJgugAAAAAAAAAADhBEB0AAAAAAAAAACcIogMAAAAAAAAA4ARBdAAAAAAAAAAAnCCIDgAAAAAAAACAEwTRAQAAAAAAAABwgiA6AAAAAAAAAAAE0QEAAAAAAAAAKBoy0QEAAAAAAAAAcMLb2QoAKO8ajV9b5PscndarRLYFAAAAAAAArsmtguhTp06VF198UZ599lmZPXu2WWaz2WTy5MmyYMECSUtLk7CwMHnnnXekRYsW1v2ysrJk7Nixsnz5csnMzJRu3brJ3LlzpV69emW4NwAAAAAAACjOxCcA8OhyLrt37zaB8jvvvNNh+YwZM2TmzJkyZ84c06ZOnTrSvXt3uXjxotUmOjpaVq1aJStWrJDt27dLRkaGREVFydWrV8tgTwAAAAAAAAAA7sItMtE16P3EE0/IwoUL5bXXXrOWaxa6ZqRPnDhR+vXrZ5YtWbJEateuLcuWLZNhw4bJhQsXZNGiRbJ06VIJDw83bRISEqR+/fqyceNGiYyMLLP9AgAAAAAAAIAbufqCkrOlxy0y0UeMGCG9evWyguB2KSkpcurUKYmIiLCWVapUSTp37ixJSUlmfu/evZKdne3QJjg4WEJCQqw2AAAAAAAAAAC4ZSa6lmDZt2+fKdWSmwbQlWae56Tzx44ds9r4+vpKYGBgnjb2++dH66jrZJeenn7T+wIAAAAAAAAAcC8unYl+4sQJM4ioll+pXLmy03YVKlRwmNcyL7mX5Xa9NjqIaUBAgDVp+RcAAAAAAAAAgGdx6SC6lmI5c+aMtGrVSry9vc20detW+cc//mFu2zPQc2eU633s63Sg0cuXL0taWprTNvmZMGGCqadunzSgDwAAAAAAAADwLC4dRO/WrZscOHBA9u/fb02tW7c2g4zq7VtvvdUEyRMTE637aMBcA+0dOnQw8xqA9/HxcWiTmpoqycnJVpv8aG31atWqOUwAAAAAAAAAAM/i0jXR/f39zQCgOfn5+UlQUJC1PDo6WqZMmSJNmjQxk96uWrWqDBgwwKzXUixDhgyRmJgYc78aNWrI2LFjJTQ0NM9ApQAAAAAAAAAAuE0QvTDGjRsnmZmZMnz4cFOyJSwsTDZs2GAC8HazZs0y5V/69+9v2mqGe3x8vHh5eZXptgMAAAAAAAAAXJvbBdG3bNniMK+Dg8bGxprJGR2UNC4uzkwAAAAAAAAAAJSLmugAAAAAAAAAAJQlt8tEBwAAAFA+NBq/tkjtj07rVWLbAgAAADhDJjoAAAAAAAAAAE4QRAcAAAAAAAAAwAmC6AAAAAAAAAAAOEFNdAAAUGxiY2Nl8uTJDstq164tp06dMrdtNptZv2DBAklLS5OwsDB55513pEWLFlb7rKwsGTt2rCxfvlwyMzOlW7duMnfuXKlXrx6vFODhilpDXVFHHQAAADeLTHQAAFCsNCCemppqTQcOHLDWzZgxQ2bOnClz5syR3bt3S506daR79+5y8eJFq010dLSsWrVKVqxYIdu3b5eMjAyJioqSq1ev8koBAAAAAEodmehAKWZCAYAn8Pb2NsHx3DQLffbs2TJx4kTp16+fWbZkyRKTqb5s2TIZNmyYXLhwQRYtWiRLly6V8PBw0yYhIUHq168vGzdulMjIyFLfHwAAAACAZyMTHQAAFKsjR45IcHCwNG7cWB577DH58ccfzfKUlBRT1iUiIsJqW6lSJencubMkJSWZ+b1790p2drZDG32skJAQq40zWgYmPT3dYQIAAAAA4GYRRAcAAMVGa5y///77sn79elm4cKEJmnfo0EHOnTtn1UXXzHNnNdP1f19fXwkMDHTaxpmpU6dKQECANWn2OgAAAAAAN4sgOgAAKDY9e/aUhx56SEJDQ005lrVr11plW+wqVKiQp8xL7mW5FabNhAkTTDkY+3TixImb2hcAAAAAABQ10QEAQInx8/MzAXUt8dK3b1+zTDPK69ata7U5c+aMlZ2utdQvX74saWlpDtno2kYz2guipWF0AgAAgOthXDEA7oxMdAAAUGK0TvmhQ4dM0FxrpGuQPDEx0VqvAfOtW7daAfJWrVqJj4+PQ5vU1FRJTk6+bhAdAAAAAICSQCY6AAAoNmPHjpXevXtLgwYNTPb4a6+9Zgb4HDRokCnHEh0dLVOmTJEmTZqYSW9XrVpVBgwYYO6vtcyHDBkiMTExEhQUJDVq1DCPaS8PAwAAAABAaSOIDnBZGQAUm5MnT8rjjz8uv/zyi9xyyy3Srl072blzpzRs2NCsHzdunGRmZsrw4cNNyRYdiHTDhg3i7+9vPcasWbPE29tb+vfvb9p269ZN4uPjxcvLi1cKAAAAAFDqCKIDAIBis2LFigLXazZ6bGysmZypXLmyxMXFmQkAAAAAgLJGTXQAAAAAADzU1KlTpU2bNuaqsFq1apmBwA8fPuzQxmazmRPgwcHBUqVKFenSpYscPHgwzzgoo0aNkpo1a5qBxfv06WOuUAMAoDwgiA4AAAAAgIfSAb5HjBhhyq/pwN5XrlyRiIgIuXTpktVmxowZMnPmTJkzZ47s3r3bDBTevXt3uXjxotVGxz1ZtWqVuSpt+/btkpGRIVFRUXL16tUy2jMAAIoP5VwAAAAAAPBQ69atc5hfvHixyUjfu3ev3HvvvSYLffbs2TJx4kTp16+fabNkyRKpXbu2LFu2TIYNGyYXLlyQRYsWydKlS62BwBMSEqR+/fqyceNGiYyMLJN9A4DyrtH4tUW+z9FpvUpkW8o7MtEBAAAAAIChAXFVo0YN839KSoqcOnXKZKfbVapUSTp37ixJSUlmXgPu2dnZDm209EtISIjVJj9aAiY9Pd1hAgDAFZGJDgAAAJRhNhAAuArNOh8zZox06tTJBMCVBtCVZp7npPPHjh2z2vj6+kpgYGCeNvb7O6vHPnny5BLYEwAAiheZ6AAAAAAAQEaOHCnffPONLF++PM+zUaFChTwB99zLcrtemwkTJpjMd/t04sQJXgUAgEsiiA4AAAAAgIcbNWqUrFmzRjZv3iz16tWzlusgoip3RvmZM2es7HRtc/nyZUlLS3PaJj9aFqZatWoOEwAAroggOgAAAAAAHkqzxTUD/eOPP5ZNmzZJ48aNHdbrvAbJExMTrWUaMN+6dat06NDBzLdq1Up8fHwc2qSmpkpycrLVBgAAd0ZNdAAAAAAAPNSIESNk2bJl8sknn4i/v7+VcR4QECBVqlQx5Viio6NlypQp0qRJEzPp7apVq8qAAQOstkOGDJGYmBgJCgoyg5KOHTtWQkNDJTw8vIz3EACAm0cQHQAAAAAADzVv3jzzf5cuXRyWL168WAYPHmxujxs3TjIzM2X48OGmZEtYWJhs2LDBBN3tZs2aJd7e3tK/f3/Ttlu3bhIfHy9eXl6lvEcAABQ/gugAUASNxq8t0vN1dFovnl8AAMoQfTdw/XIu16PZ6LGxsWZypnLlyhIXF2cmAADKG4LoAAAAAAAAKNGTlADgzhhYFAAAAAAAAAAAJwiiAwAAAAAAAABAEB0AAAAAAAAAgKIhEx0AAAAAAAAAACcYWBTlcrCSo9N6lci2AAAAAAAAAPAsBNFRLjFKOAAAAAAAAIDiQDkXAAAAAAAAAACcIIgOAAAAAAAAAIA7BtGnTp0qbdq0EX9/f6lVq5b07dtXDh8+7NDGZrNJbGysBAcHS5UqVaRLly5y8OBBhzZZWVkyatQoqVmzpvj5+UmfPn3k5MmTpbw3AAAAAAAAAAB349JB9K1bt8qIESNk586dkpiYKFeuXJGIiAi5dOmS1WbGjBkyc+ZMmTNnjuzevVvq1Kkj3bt3l4sXL1ptoqOjZdWqVbJixQrZvn27ZGRkSFRUlFy9erWM9gwAAAAAAAAA4A5cemDRdevWOcwvXrzYZKTv3btX7r33XpOFPnv2bJk4caL069fPtFmyZInUrl1bli1bJsOGDZMLFy7IokWLZOnSpRIeHm7aJCQkSP369WXjxo0SGRlZJvsGAAAA18ZA5QAAAABcPoiemwbEVY0aNcz/KSkpcurUKZOdblepUiXp3LmzJCUlmSC6Btyzs7Md2mjpl5CQENOGIDoAAAAAAAAAT1DURJGj03qV2La4E7cJomvW+ZgxY6RTp04mAK40gK408zwnnT927JjVxtfXVwIDA/O0sd8/P1pHXSe79PT0Yt0fAAAAAAAAAIDrc5sg+siRI+Wbb74xNc1zq1ChQp6Ae+5luV2vjQ5qOnny5JvYYgAAAACeUMaHDC0AAIDyzaUHFrUbNWqUrFmzRjZv3iz16tWzlusgoip3RvmZM2es7HRtc/nyZUlLS3PaJj8TJkww5WPs04kTJ4p5rwAAAAAAAAAArs6lg+iaLa4Z6B9//LFs2rRJGjdu7LBe5zVInpiYaC3TgPnWrVulQ4cOZr5Vq1bi4+Pj0CY1NVWSk5OtNvnR2urVqlVzmAAAAAAAAAAAnsWlg+gjRoyQhIQEWbZsmfj7+5uMc50yMzPNei3HEh0dLVOmTJFVq1aZwPjgwYOlatWqMmDAANMmICBAhgwZIjExMfL555/LV199JU8++aSEhoZKeHh4Ge8hAADli5ZDa9Omjem3a9WqJX379pXDhw87tNG+WvvwnFO7du0c2ui4JHolWs2aNcXPz0/69OkjJ0+eLOW9AQAAAADAxYPo8+bNM6VUunTpInXr1rWmlStXWm3GjRtnAunDhw+X1q1by08//SQbNmwwB+92s2bNMgfx/fv3l44dO5og+6effipeXl5ltGcAAJRPejWYngTfuXOnuQrsypUrEhERIZcuXXJo16NHD3NlmH367LPPHNZr364nyFesWGHGQ8nIyJCoqCi5evVqKe8RAAAAAMDTebt6OZfr0ey12NhYMzlTuXJliYuLMxMAACg569atc5hfvHixyUjfu3ev3HvvvQ5l0+xjm+SmJ9AXLVokS5cuta4a0yvT6tevLxs3bpTIyEheQgAAAABAqXHpTHQAAODeNCCuatSo4bB8y5YtJrjetGlTGTp0qBnw204D7tnZ2SaD3S44OFhCQkIkKSmpFLceAAAAAAAXz0QHAADuS68oGzNmjHTq1MkEwO169uwpjzzyiDRs2FBSUlLk5Zdflvvuu88EzzVDXcc/8fX1lcDAQIfHq127tlnnjNZR18kuPT29hPYMAAAAAOBJCKIDAIASMXLkSPnmm29MTfOcHn30Ueu2Btd1TBMNqK9du1b69etXYFBey7gVNKjp5MmTi2nrAQAAPEej8WvLehMAwKURRAcAAMVu1KhRsmbNGtm2bZvUq1evwLY6aLgG0Y8cOWLmtVb65cuXJS0tzSEbXUu+dOjQwenjTJgwwWS+58xE1zrqgB0BAgAAAAA3giA6ALhYwObotF4lsi1AadBscQ2gr1q1ytQ9b9y48XXvc+7cOTlx4oQJpqtWrVqJj4+PJCYmSv/+/c2y1NRUSU5OlhkzZjh9HC0FoxM8AwFxAAAAAKWFIDoAACg2I0aMkGXLlsknn3wi/v7+Vg3zgIAAqVKlimRkZEhsbKw89NBDJmh+9OhRefHFF6VmzZry4IMPWm2HDBkiMTExEhQUZAYlHTt2rISGhkp4eDivFgAAAACgVBFEBwAAxWbevHnm/y5dujgsX7x4sQwePFi8vLzkwIED8v7778v58+dNIL1r166ycuVKE3S3mzVrlnh7e5tM9MzMTOnWrZvEx8eb+6N8IrMcAAAAgKsiiA4AAIq1nEtBNBt9/fr1132cypUrS1xcnJkAAAAAAChLBNFR6sg0AwAAAAAAAOAuKpb1BgAAAAAAAAAA4KoIogMAAAAAAAAA4ATlXAAAAACgFMsVHp3Wi+cbAADAjZCJDgAAAAAAAACAEwTRAQAAAAAAAABwgnIuAAAAKNPSFgAAAADgyshEBwAAAAAAAADACYLoAAAAAAAAAAA4QRAdAAAAAAAAAAAnCKIDAAAAAAAAAOAEA4sCAAAAAACUIwzyDQDFiyA6AAAACsSBOAAAAABPRhAdAAAAAAAAAFAsCTVHp/Uqd88kNdEBAAAAAAAAAHCCTHQAAAAAAAAXRVk1ACh7ZKIDAAAAAAAAAOAEmei4KZwRBwAAAAAAAFCekYkOAAAAAAAAAIATZKIDAAAAgItfzXl0Wq8S2RYAAABcH0F0AHDzA2sOqgEAAAD3QElUAHBPlHMBAAAAAAAAAMAJMtEBAAA8CBlwAAAAAFA0ZKIDAAAAAAAAAOAEmeiwkJkGAAAAAAAAAI4IogMAAAAAANwAktEAwDMQRAcAAAAAAAAAlNkJxqPTern0s09NdAAAAAAAAAAAnCATHQAAAABcXHnM6AIAAHAXHhVEnzt3rrzxxhuSmpoqLVq0kNmzZ8uf//znst4sAADgBH13wajDCgBwNfTdAIDyyGOC6CtXrpTo6GjToXfs2FHmz58vPXv2lG+//VYaNGgg5REH1gAAd0bfDQCAe/HEvhsA4Bkq2Gw2m3iAsLAwueeee2TevHnWsubNm0vfvn1l6tSp171/enq6BAQEyIULF6RatWriDgiiA3CGy7vdgzv2PcWJvhsAbg79femj73bv426OoQHA837npBey7/GITPTLly/L3r17Zfz48Q7LIyIiJCkpKd/7ZGVlmclOn0j7E1sWQiatL5O/C6B8KqvvMlf4bkyeHCnu9jp5yPnuctd334hrWb+V9SYAKEcaPPdhif+NovarpdV3F/XvFNfvA/pu1+q7OY4GgPIrvZj6isL23R4RRP/ll1/k6tWrUrt2bYflOn/q1Kl876NnySdPnpxnef369UtsOwGgtATM9tzn2h33/eLFi+bMuCeh7wYA91Aa/ao7/g367v+H424AQHnouz0iiG5XoUIFh3k9w5B7md2ECRNkzJgx1vy1a9fk119/laCgIKf3yXkGQ4PtJ06ccPvL79kX18Tr4rp4bVyTu74u2k9pRx4cHCyeqrT67tLgru/D0sLzw/PD+4fPV3n4/qHvdt++21P6IU/ZT0/aV0/ZT0/aV0/ZT1fZ18L23R4RRK9Zs6Z4eXnlyTo/c+ZMnux0u0qVKpkpp+rVqxfp7+qLX17e7OyLa+J1cV28Nq7JHV8XT8tAL+u+uzS44/uwNPH88Pzw/uHz5e7fP/Td7t13e0o/5Cn76Un76in76Un76in76Qr7Wpi+u6J4AF9fX2nVqpUkJiY6LNf5Dh06lNl2AQCA/NF3AwDgXui7AQDlmUdkoiu9RGzgwIHSunVrad++vSxYsECOHz8uf/vb38p60wAAQD7ouwEAcC/03QCA8spjguiPPvqonDt3Tl599VVJTU2VkJAQ+eyzz6Rhw4bF/rf0crRJkybluSzNHbEvronXxXXx2rim8vS6eJLS7LtLA+9Dnh/eP3y++P5xTXw/Fx937rs95X3gKfvpSfvqKfvpSfvqKfvpbvtawabV0wEAAAAAAAAAgGfWRAcAAAAAAAAA4EYQRAcAAAAAAAAAwAmC6AAAAAAAAAAAOEEQHQAAAAAAAAAAJwii52PevHly5513SrVq1czUvn17+c9//mOtr1ChQr7TG2+8YbXp0qVLnvWPPfaYw99JS0uTgQMHSkBAgJn09vnz56UkTZ061WxLdHS0tUzHlo2NjZXg4GCpUqWK2faDBw863C8rK0tGjRolNWvWFD8/P+nTp4+cPHmyTPcn975kZ2fLCy+8IKGhoWYbdX/+8pe/yM8//+xwP1d8bfJ7XQYPHpxnO9u1a+d2r4u7fWb0s5B7O+rUqeOWn5eC9sXdPi/Xe13c9fOC8mPbtm3Su3dv81nS99/q1asLbL9ly5Z8vxf/+9//SnmkfUObNm3E399fatWqJX379pXDhw9f935bt26VVq1aSeXKleXWW2+Vd999V8qjG3l+POk9dL3f5p783rmR58eT3juF/a3q6e8hT3Yjv/0yMjJk5MiRUq9ePXM80Lx5c/M5dGU3+hv30KFD5jez3kf7KP19ffz4cXFVN/tbftiwYeb7Yfbs2eLqirqvhT3+K2tz586Vxo0bm+9e/Q7+4osvyu13dVH29eOPP5bu3bvLLbfcYvX169evF3cwt4ivqd3//d//ibe3t9x1113iMmzIY82aNba1a9faDh8+bKYXX3zR5uPjY0tOTjbrU1NTHab33nvPVqFCBdsPP/xgPUbnzp1tQ4cOdWh3/vx5h7/To0cPW0hIiC0pKclMejsqKqrEXpFdu3bZGjVqZLvzzjttzz77rLV82rRpNn9/f9u//vUv24EDB2yPPvqorW7durb09HSrzd/+9jfbH//4R1tiYqJt3759tq5du9patmxpu3LlSpnsT377os9veHi4beXKlbb//ve/th07dtjCwsJsrVq1crivq702zl6XQYMGme3IuZ3nzp1zuK87vC7u9pmZNGmSrUWLFg7bcebMGbf8vBS0L+72ebne6+KOnxeUL5999plt4sSJ5rtBf16tWrWqwPabN2827fR3Rs73bc73Y3kSGRlpW7x4sfkttX//fluvXr1sDRo0sGVkZDi9z48//mirWrWq6U++/fZb28KFC83vsY8++shW3tzI8+NJ76Hr/Tb35PfOjTw/nvTeKexvVU9/D3myG/nt99e//tV22223mc9SSkqKbf78+TYvLy/b6tWrbeVpP7///ntbjRo1bM8//7z57azHbv/+979tp0+ftrmqm/ktr7/d9NggODjYNmvWLJurK+q+Fvb4ryytWLHCfNfqd65+9+p3sJ+fn+3YsWPl7ru6qPuq66dPn276se+++842YcIEc3/9bJan/cz5fr311lttERER5nPpKgiiF1JgYKDtn//8Z77rHnjgAdt9992XJ/BU0A8zffPoj9edO3day/RLTJfpF1pxu3jxoq1JkyYmeJRz265du2arU6eOCQza/f7777aAgADbu+++a7159U2vb367n376yVaxYkXbunXrSn1/nO1LfvQLRrch5wfUlV6bgvZFg4L63nLGnV8XV/7MaLDW2Ze0u31eCtoXd/u8XG9f3O3zgvKtKEH0tLQ0myfSk2C6/1u3bnXaZty4cbbbb7/dYdmwYcNs7dq1s5V3hXl+PP09VNBvc09+7xTm+fHU905RfqvyHvIMN/rbTxM7Xn31VYdl99xzj+2ll16ylaf91GShJ5980uYubua3/MmTJ02yjZ58bNiwocsH0YvruCW/47+y1LZtW5P4lJP25+PHjy9339VF3df83HHHHbbJkyfbyuN+Pvroo+Y7tagxjZJGOZfruHr1qqxYsUIuXbpkLpfI7fTp07J27VoZMmRInnUffPCBKRvQokULGTt2rFy8eNFat2PHDnPJTVhYmLVML43SZUlJSVLcRowYIb169ZLw8HCH5SkpKXLq1CmJiIiwllWqVEk6d+5sbcfevXvNpT852+ilPyEhIVab0twfZ/uSnwsXLpjLsapXr+6Sr8319kUvt9XLups2bSpDhw6VM2fOWOvc9XVxh8/MkSNHzHOplxxp6ZIff/zRbT8vzvbFHT8v19sXd/q8AHZ333231K1bV7p16yabN2/2mCdGv29UjRo1nLbRz2TOz6yKjIyUPXv2mM+zpz8/nvoeut5vc09/7xTm+fHU905RjiE8+T3kSW70t1+nTp1kzZo18tNPP5lSj/r5+e6778x7pLzs57Vr18wxm/6u1v3S39h6/+uVq3PH11P3VUuhPP/88+Z4xx0U13GLs+O/snD58mVzzJb7u1fnne2Tu35X38i+5ve+1ePywvxWdLf9XLx4sfzwww8yadIkcTXeZb0BrurAgQPmh+fvv/8uf/jDH2TVqlVyxx135Gm3ZMkSUxusX79+DsufeOIJE+jRmr3JyckyYcIE+frrryUxMdGs10CcdkS56TJdV5z0h/S+fftk9+7dedbZ/1bt2rUdluv8sWPHrDa+vr4SGBiYp439/qW1PwXtS2762o0fP14GDBhgaka52mtzvX3p2bOnPPLII9KwYUMTvH355ZflvvvuM19CGrh119fF1T8z+kPk/fffNz8YNeD/2muvSYcOHUzdc3f7vBS0L0FBQW71ebnevrjT5wVQGrhasGCBqQuo9fqXLl1qAll6Mujee+8t10+SBhzGjBljghB6IssZ/dzl93175coV+eWXX8xz6MnPj6e9hwr729xT3ztFeX487b1T1N+qnvoe8kQ3+tvvH//4h0nY0JroWq+3YsWK8s9//tN8b5eX/dRkFK39Pm3aNPO7e/r06bJu3TpzDKcnDTSJqLy8nrpv+jqOHj1a3EVxHLc4O/4rK/rdqieC8/vudbZP7vpdfSP7mttbb71lTpj3799fXNUvN7Cfmjin70utm66fS1fjelvkIpo1ayb79+83AzP861//kkGDBpkBC3L/GH3vvfdMkEkL5OeknaqdHgA1adJEWrdubX683XPPPWa5nvHL78Apv+U36sSJE/Lss8/Khg0b8mxjTrn/ZmG2I3ebkt6fwu6L0rOOmqmqZ+d0EANXe20Ksy+PPvqow3bqNmqAUDMCcgegC9pOV9gXd/rMaDDWTgdd0QPS2267zQT/7QNVusPn5Xr7ogEad/m8FGZf3OXzAuT8naGTnb6n9fv0zTffLLdBLDsdjO2bb76R7du3X7dtft+3+S33xOfH095Dhf1t7qnvnaI8P5723inqb1VPfQ+VJzog/eTJkwtsYz+hciO//TSIvnPnTpONrr83dXDx4cOHm4BdYa50cIf91OMC9cADD8hzzz1nbuvgfpo9qgM3lmYQvST3UxNu3n77bXNs4wqf7ZJ+7xbm+K+sFfVY252/q28krqCWL19u3iuffPJJvidT3HU/r169ak7q6GdAk+dcEUF0JzRj8U9/+pO5rcEY/aLSL9f58+dbbfTMyOHDh2XlypXXfaI12OTj42POquhtzerUbMrczp49m+cszc3QTkHPImumSc43pnb0c+bMMduv9CxQzrN0eh/7dui26mUYOvpzzixObaOZoPY2Jb0/19sXzaTx8vIyHYKejdNs1E2bNl33rGpZvDaF3Zec9PXRH2m6ne74urjLZyY3Hblcg7a6HX379nWbz8v19sXOHT4vhd0Xd/i8AAXRE3UJCQnl+kkaNWqUCTpoH6EZfAXRz2TuLBX9zGpWSu6raTzx+fG091Bhfpt78nunKM+Pp713buR3tye+h8rbyUgNEBakUaNG5oRlUX/7ZWZmyosvvmiu9tDyQOrOO+80J7H0RFRpBtFLcj+1lKO+33OfiGvevHmhToK7y37q8al+ths0aODw/RATEyOzZ8+Wo0ePSnnZ1xs9/ist+p7T7+L8vnud7ZO7flffyL7aaSxFS+N++OGHpfp9Uxr7efHiRVOK56uvvjKfBaUnejTorq+pngzXq83LEjXRC0lfNP2BldOiRYvMj7GWLVte9/5ackC/rOyBN8340PpTu3btstp8+eWXZpk9oFMc9NJMvbxTO3X7pD+sNRNYb996663mi8demkFpoEkzV+zbofuoQbOcbVJTU01pB3ub0tif6+1LzgC6Bs42btxYqC/OsnhtCrMvuZ07d85k0ti3051eF3f6zOSmn/tDhw6Z7bCXNXGHz8v19kW5y+elMPviLp8XoCD6g9FVLzstjt9R+mP4448/Ngds+n16PfqZzPmZVfrjWfsY/Tx7+vPjae+hwvw298T3zo08P5723rmR3928h9ybBm5uv/32Aie9KuFGfvvp72CdtIRLTvo+smdvl4f91BNzbdq0sZLu7LT2uyaqlJf91FroGpDO+f2g4yZpffT169dLaSvJfb3R47/Sou85PWbL3X/rvLN9ctfv6hvZV3sG+uDBg2XZsmXWSTxX5lvE/dQTOrn767/97W/W1XY5xwAoM2U9sqkrmjBhgm3btm22lJQU2zfffGN78cUXbRUrVrRt2LDBanPhwgVb1apVbfPmzctz/++//96MkLt7927zGGvXrjWjz9599922K1euWO169Ohhu/POO80oyjqFhobaoqKiSnz/co9GP23aNFtAQIDt448/th04cMD2+OOP2+rWrWtLT0+32uhouvXq1bNt3LjRtm/fPtt9991nRsgt6/3JuS/Z2dm2Pn36mO3cv3+/LTU11ZqysrJc/rXJuS8XL160xcTE2JKSksx2bt682da+fXszYri7vS7u9pnR533Lli22H3/80Yx2ro/v7+9vO3r0qNt9XgraF3f7vBS0L+7+eUH5oO/Dr776ykz682rmzJnm9rFjx8x6HYF+4MCBVvtZs2bZVq1aZfvuu+9sycnJZr3e71//+petPPr73/9uvjv1c5zz++a3336z2uR+jvTzrv3Gc889Z/v2229tixYtsvn4+Ng++ugjW3lzI8+PJ72Hrvfb3JPfOzfy/HjSe6ewv1U9/T3kyQrz269Zs2bmt3/O90+LFi3Mb059ryxevNhWuXJl29y5c23laT/1tr7vFyxYYDty5IgtLi7O5uXlZfviiy9s5Wk/c2vYsKH5nnR1Rd3Xwhz/lbUVK1aY95x+5+p3b3R0tM3Pz886Fi9P39VF3ddly5bZvL29be+8847Da3f+/HlbedrP3CZNmmSO2V0FQfR8PP300+aL09fX13bLLbfYunXr5hBAV/Pnz7dVqVIl3zfs8ePHbffee6+tRo0a5jFuu+022+jRo23nzp1zaKfzTzzxhAkE6aS309LSbKX9o/HatWvmjVmnTh1bpUqVzLZrcDCnzMxM28iRI80+6X7rl7PuZ1nvT8590QMHPQDIb9IfOK7+2uTcFz1ojoiIMO8//cJp0KCBbdCgQXmec3d4XdztM/Poo4+aoLg+78HBwbZ+/frZDh486Jafl4L2xd0+LwXti7t/XlA+6Ocmv8+TvheV/q/fjXbTp083nyk96A4MDLR16tTJnKgqr5x932jgwS73c6Q0qKwn7vQ7qFGjRvmeiPXU58eT3kPX+23uye+dG3l+POm9U9jfqp7+HvJkhfntl/v7WANXgwcPNr9J9XOkgcq33nrLHCeUp/1UGvj605/+ZPZTA1mrV6+2ubIb3U93DKIXdV8Lc/znCjRIbO/T7rnnHtvWrVvL7Xd1UfZVbxd0rOHK3inia+rKQfQK+k9ZZ8MDAAAAAAAAAOCKqIkOAAAAAAAAAIATBNEBAAAAAAAAAHCCIDoAAAAAAAAAAE4QRAcAAAAAAAAAwAmC6AAAAAAAAAAAOEEQHQAAAAAAAAAAJwiiAwAAAAAAAADgBEF0AAAAAAAAAACcIIgOwK2cOnVKunfvLn5+flK9enWnywAAwM2rUKGCrF69mqcSAAA3UtzHyLGxsXLXXXcVy7YB7oogOgC3OiifNWuWpKamyv79++W7775zugwAAAAAAE/EMTJQ/LxL4DEBuKHLly+Lr6+vuLoffvhBWrVqJU2aNClwGQAA5ZG79NcAAKDs+m+OkYHiRyY64KG6dOkiI0eOlDFjxkjNmjXNpV4zZ86U0NBQc8lX/fr1Zfjw4ZKRkWHa22w2ueWWW+Rf//qX9Rh6OVetWrWs+R07doiPj491H2caNWpk/n/wwQdNRrp9Xn366acmIF65cmW59dZbZfLkyXLlyhXrfvr333//fXO/wYMH57sMAIDyoiz76/wcOHBA7rvvPqlSpYoEBQXJM8884/A42mePHj3aXDqu61944QUZNGiQ9O3b12pz8eJFeeKJJ8z2161b12TL6X5GR0ffxDMFAIDrKOvj7fyOkS9cuGD6bX3MatWqmf7866+/LtJ+LV261Dx+QECAPPbYY6ZPBzwFQXTAgy1ZskS8vb3l//7v/2T+/PlSsWJF+cc//iHJyclm3aZNm2TcuHGmrXa+9957r2zZssXMp6WlybfffivZ2dnmf6XrNAD+hz/8ocC/u3v3bvP/4sWLTRkW+/z69evlySefNAff+pi6TfHx8fL6669b9+vRo4f079/f3O/tt9/OdxkAAOVJWfXXuf3222+mzw0MDDT974cffigbN240QQK76dOnywcffGD6eN3e9PT0POXbNKCg69asWSOJiYnyxRdfyL59+4rhmQIAwHWU5fF27mNkDdL36tXL1Er/7LPPZO/evXLPPfdIt27d5Ndffy10drv26f/+97/NtHXrVpk2bdpNP0+Au6CcC+DB/vSnP8mMGTOs+dtvv9263bhxY/mf//kf+fvf/y5z5861zqYvWLDA3N62bZu0bNlSGjRoYDrzO+64w/yvba5Hz7ArzVKrU6eOtVyD5ePHjzcZa0oz0XUb9IfFpEmTzP0qVapkst9y3i+/ZQAAlBdl1V/npsHxzMxMk9mmWXRqzpw50rt3bxM8r127tsTFxcmECRPM1Wb29XqwbqcZaxo4WLZsmTlwVxpwDw4OvuHnBwAAV1SWx9u5j5E1YK9Xk505c8asU2+++aYJin/00UcmQ/16rl27ZpLc/P39zfzAgQPl888/t5LegPKOTHTAg7Vu3dphfvPmzeYysz/+8Y+mY/zLX/4i586dk0uXLpn12mEfPHhQfvnlF3PWWed10tt6+XZSUpJ07tz5hrdHz4a/+uqr5sy6fRo6dKg5e67ZbwAAeCJX6a8PHTpkDujtAXTVsWNHc1B9+PBhc5n46dOnpW3bttZ6Ly8vkzVn9+OPP5qsupxt9JLwZs2aFXl7AABwZa7Sf9uPtbUMjJZay3m8nZKSYjLMC0PLuNgD6EpLsmlQHvAUBNEBD5bzIPjYsWNy//33S0hIiKmfpp3sO++8Y9bpwa7Sddrpaidu79S1E9fbesmYZqd16tTphrdHD8K1Bvr+/futSc+WHzlyxNRIBwDAE7lKf62Xguvl5vnJuTx3G71f7tsFtQEAoDxwlf7bfqytQe+cx9o66Unw559/vlCPofXYc9K+XB8X8BSUcwFg7Nmzx5zdfuutt0ytNvW///u/Ds+OvU7bJ598Yuq4/fnPfzZnorXTf/fdd01NtZxnpq/XAV+9etVhmd5fO3G97A0AAJR9f52TXkqupVg0Y84eGNA6r7odTZs2NRnlWtJl165d5m8q7eu/+uorMziauu2228xvAG2jg6oprZuuJ8xv5mo2AABcWVn230rvq/XQtUa7ZpQDKDoy0QFYB7XaqWstU73UWkfd1o46Nz0brnVM77zzTjOit72j1zqpRamvqh231k/TjlwHTVGvvPKKqbMaGxtrLmPTy8ZXrlwpL730Eq8SAABl0F/n9MQTT5grw3TsEj2418vSR40aZWqiavBc6fzUqVNNAEBPjD/77LOmn7dnnuvBv95fs970/trfP/300yag4CzLHQAAd1eW/bcKDw+X9u3bS9++fWX9+vVy9OhRUx5Gj7U1wA/g+giiAzA0Q2zmzJlmYDC9jEw7aT0Izq1r164mqyxnB66ZY7qsKBlkegY+MTHRZKHdfffdZllkZKQZ5VuXt2nTRtq1a2e2qWHDhrxKAACUQX+dU9WqVc2B96+//mr66YcfftgMDqqDh9q98MIL8vjjj5s6r3qwrvVWtX/PWZZNt1/XRUVFmYN6ravevHlzSrcBAMqtsuy/lQbjdaBvDcjryWu9guyxxx4zwXT7iXAABatgowAhAAAAgBKgtVI1QN6/f3/5n//5n3zbaHkYHWRNT7APGTKE1wEAAAAuh5roAAAAAIqFDpy2YcMGky2XlZVlstRTUlJkwIABVhutkf7f//5X2rZtKxcuXJBXX33VLH/ggQd4FQAAAOCSKOcCoNjppWl6+XZ+U4sWLXjGAQAop/211jaPj4835V60TMuBAwdk48aNJhs9pzfffFNatmxpyrloJvoXX3whNWvWLKY9AwCg/CqJ/lvv5+wx9e8BoJwLgBJw8eJFOX36dL7rfHx8qHEOAIALoL8GAMD9lET/rVeSZWdn57tOa6brwOCAp6MmOgAAAAAAAAAATlDOBQAAAAAAAAAAJwiiAwAAAAAAAADgBEF0AAAAAAAAAACcIIgOAAAAAAAAAIATBNEBAAAAAAAAAHCCIDoAAAAAAAAAAE4QRAcAAAAAAAAAwAmC6AAAAAAAAAAAOEEQHQAAAAAAAAAAJwiiAwAAAAAAAADgBEF0AAAAAAAAAACcIIgOAAAAAAAAAIATBNEBAAAAAAAAAHCCIDoAl/L5559L69atxc/PTypUqCCrV68u600CAAC5vPTSS9KgQQPx9vaW6tWr8/wAADxCfHy8OU49evRoWW+KS9qyZYt5fj766KOy3hSg2HkX/0MCwI2x2WzSv39/adq0qaxZs8YE0ps1a8bTCQCAC/nkk0/k9ddfl4kTJ0rPnj2lUqVKZb1JAAAAQIkiiA7AZfz888/y66+/yoMPPijdunUr680BAAD5SE5ONv+PHj1aatWqxXMEAACAco9yLgAkNjbWXHL1zTffyCOPPCIBAQFSo0YNGTNmjFy5ckUOHz4sPXr0EH9/f2nUqJHMmDHD4Vk7ePCgRERESNWqVeWWW26RESNGyNq1a81j6uVchd2GevXqmdsvvPCCua/+LbsjR47IgAEDzMG6Zrw1b95c3nnnHV49AAD+//773//K448/LrVr1zZ9pZZb+ctf/iJZWVlm/alTp2TYsGGmv/X19ZXGjRvL5MmTTV9fWNo3aykXpX9H+2vtw+1Wrlwp7du3N1eT/eEPf5DIyEj56quveI0AAOXWe++9Jy1btpTKlSub42hNCjt06FCedgsXLjRXXWsffccdd8iyZctk8ODBDse9hZGYmCgPPPCA6c/1b/7pT38y/fsvv/yS79Vjd955p/mbt956q7z99tvW8X/uq8Lnzp0rd911l1SpUkUCAwPl4Ycflh9//PEGnhGgfCITHYBFS6k8+eSTpgPWjlmD5dnZ2bJx40YZPny4jB071nT0GuTWjrpfv36SmpoqnTt3NgfL8+bNM0Hu5cuXy8iRI4v0zP71r381Pzz0MUeNGmUC5vbLw7/99lvp0KGDCQa89dZbUqdOHVm/fr3JgNMfCpMmTeJVBAB4tK+//lo6deokNWvWlFdffVWaNGli+mgtj3b58mVJS0uTtm3bSsWKFeWVV16R2267TXbs2CGvvfaaqeu6ePHiQv2dVatWmZPYixYtknXr1pkT7/aT4FOmTDEB9qeeesr8r3/3jTfekD//+c+ya9cuEzAAAKA8mTp1qrz44ovmJLbePnfunAlS6wnl3bt3m/5YLViwwBxnP/TQQzJr1iy5cOGCOZFtP9FdFD/88IN5fD2G1n5Y+/GZM2ea3wEHDhwQHx8f0077aT2+vvfee81Jbj1p/uabb8rp06fzPKZum9Z712Ps6dOnmyvE9feEHofrbww9cQ54PBsAjzdp0iSbnnx+6623HJ6Lu+66yyz/+OOPrWXZ2dm2W265xdavXz8z//zzz9sqVKhgO3jwoMN9IyMjzX03b95c6Oc3JSXF3OeNN97I81j16tWzXbhwwWH5yJEjbZUrV7b9+uuvHv8aAgA823333WerXr267cyZM/muHzZsmO0Pf/iD7dixYw7L33zzTdP35u7HC/O74ezZs9ay48eP27y9vW2jRo1yaHvx4kVbnTp1bP379y/yPgEA4GoWL15s+kA9dk1LS7NVqVLFdv/99zu00T6xUqVKtgEDBpj5q1evmr4wLCzMoZ32yT4+PraGDRve8PZcu3bNHKPrY+l2ffLJJ9a6Nm3a2OrXr2/Lyspy6JeDgoJMW7sdO3bkGw84ceKE2b9x48YVenv0+F8f68MPP7zhfQJcFeVcAFiioqIcng0tmaKXeemgYXbe3t4mC/3YsWNmfuvWrRISEpInu0zPxBeH33//XT7//HNzSZyWi9Gz5/bp/vvvN+t37tzJqwgA8Fi//fab6Y/1ijItq5aff//739K1a1cJDg526Evtfbze/2boFWL6eFo+Jufj62XmesVaYcu7AQDgLvSKrszMTFOSJaf69evLfffdZ45jlZZH1ZJq2k/npFdad+zYsch/98yZM/K3v/3N/B09PtfM84YNG5p19jIyly5dkj179kjfvn1NCTc7LbXWu3fvPL8R9Lhfr0rP2YfrFeB6tTh9OPD/oZwLAIvWb8tJO1sNXOsBcO7l6enp5rZerqY1VXMrrsu99PG1A4+LizNTfvKr/QYAgKfQUi1Xr161yqrkRy/d/vTTT61LvIu7L7VfGt6mTZt812sZGQAAyhM9VlV169bNs05PWmuJ1Jzt8jtG1mUpKSmF/pvXrl0z45H9/PPP8vLLL0toaKgprarL27VrZ4L69t8GWufc2d/M3Yc7a6u0ljoAgugAblJQUFC+NdX0THtx0AFNvLy8ZODAgWbA0vzkF8QHAMCTToJrX3ny5EmnbbRWug4s9vrrr+e7Xg/2b4Y+vvroo4+sbDgAAMr7sbDSMUhy0yC3vW+0tyuO4+bk5GRTo1zrlw8aNMha/v333+c5jtbs8sL8Td1ObfvFF19Y45LllN8ywBORiQ7gpugl2jo4iQ7+mbOky4oVK4rlmdVMeL38/KuvvjIH/zkvRQMAACJVqlQx/fGHH35oguT2g/actGTbZ599ZgYU1QPr4hYZGWkuKdfBznTQNAAAyjsd3FP74ISEBHnkkUes5XpSe9OmTfLwww+b+WbNmpnSKP/7v/8rY8aMsdodP35ckpKSinQiW4Pd+QW258+f7zCv2emtW7eW1atXm+N1+3F0RkaGKd+S+zfCtGnT5KeffspTcgbA/0MQHcBNiY6Olvfee8/UVNXRu/USsGXLlsl///vfYrt8++233zYjjf/5z3+Wv//979KoUSO5ePGiOduul6brDxQAADzZzJkzTV8ZFhYm48ePN+OXaPbZmjVrzIG19tF6WXmHDh1k9OjR5oBexxU5evSoCa6/++67BZaDuR7tm/VvTJw4UX788Ufp0aOHCdbrNuzatcsczE+ePLlY9xkAgLJUvXp1U1LlxRdfNGOC6LhgWrpF+zstiTpp0iTrmFiXDRs2zATWn376aTl//rxZpqVginLMfPvtt5sT4trXawkWvRpNj4ntpWNy0n65V69e5kT3s88+a0q/vfHGG6Yu+q+//mq107rszzzzjDz11FOmjvq9995r+m3NsN++fbspGaPH4UXhbNwyPenvbPwWwNURRAdwU/SsuQ5GpsF0HdxEM8d1EFDtsPXyMv1hcbM0w33fvn3yP//zP/LSSy+ZgVT0cZs0aWIGFwUAwNPpwF8arNYD9gkTJpiTzZr1pgObafaZHqTrgbH2pXoArVly/v7+piSaPeB9s/Tvap+tJ7+XL18uWVlZZhu0Trr+RgAAoLzRvq9WrVryj3/8Q1auXGky07t06SJTpkwxx6t2GqTWLPIZM2aY42U9+ayB8E8++cRkpBeWjm2iQXMNimtQXq8CCw8Pl40bN5qBSnPS/v1f//qXvPLKK/Loo4+aPnn48OGm1MzSpUsd2uoJd62prv/PnTvX1FjXY30NsLdt27bIz8tbb72V7/LNmzeb5wdwRxVseuoKAIqZ/kjQA2g9E08JFgAAAAAA/h/NRm/atKn07dtXFixYUCpPTXZ2ttx1113yxz/+UTZs2MDLARQBmegAbppmnetZah21215j7Z///KfJGieADgAAAADwZDqYp45bouN96UCjx44dk1mzZpkrxzSrvKQMGTJEunfvbq5I023Q8m2HDh0yV40BKBqC6ABuml5SZr80/MqVK+ayNa3Nav8xoBe8aP21gnh5eVmDpAAAgNKnfXVBF6lqP639NQAAKBodCFTHIdFyKlqPXMugavkUDWq3aNGixPphDdKPHTtWzp49a47b77nnHjMWipaAKQqO6QHKuQAoBfHx8WaQkoJQGw0AgLKlNUp1nBNnGjZsaAIAAADAs/phjukBgugASoHWRU9JSSmwTbNmzcwAZwAAoGwcPnzYZKwVlEUXGhpaqtsEAICncOV+mGN6gCA6AAAAAAAAAABOVXS+CgAAAAAAAAAAz8bAooV07do1+fnnn025CQY/BACUBh3ARy/pDA4OlooVOe9dVPTdAIDSRt99c+i7AQCu2ncTRC8kDaDXr1+/uF4fAAAK7cSJE1KvXj2esSKi7wYAlBX67htD3w0AcNW+myB6IdkHPNQntFq1asXz6gAAUID09HRzApdBd28MfTcAoLTRd98c+m4AgKv23QTRC8lewkUD6ATRAQCliTJiN/e80XcDAEobfffNPW/03QAAV+u7KbAKAAAAAAAAAIATBNEBAAAAAAAAAHCCIDoAAAAAAAAAAE4QRAcAAAAAAAAAwAmC6AAAAAAAAAAAOEEQHQAAAAAAAAAAJwiiAwAAAAAAAADgBEF0AAAAAAAAAACcIIgOAAAAAAAAAIATBNEBAEChTZ06Vdq0aSP+/v5Sq1Yt6du3rxw+fNihzeDBg6VChQoOU7t27RzaZGVlyahRo6RmzZri5+cnffr0kZMnTzq0SUtLk4EDB0pAQICZ9Pb58+d5tQAAAAAApYogOgAAKLStW7fKiBEjZOfOnZKYmChXrlyRiIgIuXTpkkO7Hj16SGpqqjV99tlnDuujo6Nl1apVsmLFCtm+fbtkZGRIVFSUXL161WozYMAA2b9/v6xbt85MelsD6QAAAAAAlCbvUv1rAADArWkwO6fFixebjPS9e/fKvffeay2vVKmS1KlTJ9/HuHDhgixatEiWLl0q4eHhZllCQoLUr19fNm7cKJGRkXLo0CHztzRYHxYWZtosXLhQ2rdvbzLfmzVrVqL7CQAAAACAHUF0lEuNxq8tUvuj03qV2LYAQHmmAXFVo0YNh+VbtmwxwfXq1atL586d5fXXXzfzSgPu2dnZJoPdLjg4WEJCQiQpKckE0Xfs2GFKuNgD6EpLwugybUMQHXCt31KK31MAAAD8niqvCKIDAIAbYrPZZMyYMdKpUycTALfr2bOnPPLII9KwYUNJSUmRl19+We677z4TPNcM9VOnTomvr68EBgY6PF7t2rXNOqX/24PuOekye5vctM66Tnbp6em8sgAAAACAm0YQHQAA3JCRI0fKN998Y2qa5/Too49atzW43rp1axNQX7t2rfTr16/AoLwOQmqX87azNrkHPZ08eTKvJgAAAACgWDGwKAAAKLJRo0bJmjVrZPPmzVKvXr0C29atW9cE0Y8cOWLmtVb65cuXJS0tzaHdmTNnTDa6vc3p06fzPNbZs2etNrlNmDDBlJexTydOnOCVBQAAAADcNILoAACg0DQTXDPQP/74Y9m0aZM0btz4uvc5d+6cCWhrMF21atVKfHx8JDEx0WqTmpoqycnJ0qFDBzOvA4hqIHzXrl1Wmy+//NIss7fJTUvFVKtWzWECAAAAAOBmUc4FAAAU2ogRI2TZsmXyySefiL+/v1WfXAf8rFKlimRkZEhsbKw89NBDJmh+9OhRefHFF6VmzZry4IMPWm2HDBkiMTExEhQUZAYlHTt2rISGhkp4eLhp07x5c+nRo4cMHTpU5s+fb5Y988wzEhUVxaCiAAAAAIBSRRAdAAAU2rx588z/Xbp0cVi+ePFiGTx4sHh5ecmBAwfk/fffl/Pnz5tAeteuXWXlypUm6G43a9Ys8fb2lv79+0tmZqZ069ZN4uPjzf3tPvjgAxk9erRERESY+T59+sicOXN4tQAAAAAApYogOgAAKFI5l4JoNvr69euv+ziVK1eWuLg4MzmjGeoJCQm8OgAAAACAMkVNdAAAAAAAAAAAnCCIDgAAAAAAAACAKwbRt23bJr1795bg4GCpUKGCrF692mnbYcOGmTazZ892WJ6VlSWjRo0yA5b5+fmZeqknT550aJOWliYDBw40A5nppLe1TisAAAAAAAAAAC4bRL906ZK0bNnyuoOEaXD9yy+/NMH23KKjo2XVqlWyYsUK2b59u2RkZEhUVJRcvXrVajNgwADZv3+/rFu3zkx6WwPpAAAAAAAAAAC47MCiPXv2NFNBfvrpJxk5cqQZpKxXr14O6y5cuCCLFi2SpUuXSnh4uFmmA5DVr19fNm7cKJGRkXLo0CETON+5c6eEhYWZNgsXLpT27dvL4cOHpVmzZiW4hwAAAAAAAABwfY3Gry3S03R0mmOsFB5aE/3atWsmY/z555+XFi1a5Fm/d+9eyc7OloiICGuZZquHhIRIUlKSmd+xY4cp4WIPoKt27dqZZfY2+dEyMenp6Q4TAAAAAAAAAMCzuHQQffr06eLt7S2jR4/Od/2pU6fE19dXAgMDHZbXrl3brLO3qVWrVp776jJ7m/xMnTrVqqGuk2a3AwAAAAAAAAA8i8sG0TXL/O2335b4+HgzoGhR2Gw2h/vkd//cbXKbMGGCKRdjn06cOFHEPQAAAAAAAAAAuDuXDaJ/8cUXcubMGWnQoIHJRtfp2LFjEhMTI40aNTJt6tSpI5cvX5a0tDSH++r9NBvd3ub06dN5Hv/s2bNWm/xUqlRJqlWr5jABAAAAAAAAADyLywbRtRb6N998I/v377cmrXeu9dF1kFHVqlUr8fHxkcTEROt+qampkpycLB06dDDzOoCoZpLv2rXLavPll1+aZfY2AAAAAAAAAADkx1vKUEZGhnz//ffWfEpKigmW16hRw2SgBwUFObTXgLlmljdr1szMa63yIUOGmOx0bav3Gzt2rISGhkp4eLhp07x5c+nRo4cMHTpU5s+fb5Y988wzEhUVZT0OAAAAAAAAAAAuF0Tfs2ePdO3a1ZofM2aM+X/QoEGmFnphzJo1y5R66d+/v2RmZkq3bt3Mfb28vKw2H3zwgRmcNCIiwsz36dNH5syZU+z7AwAAAAAAAAAoX8o0iN6lSxczwGdhHT16NM+yypUrS1xcnJmc0Qz1hISEG95OAAAAAAAAAIBnctma6AAAAAAAAAAAlDWC6AAAAAAAAAAAOEEQHQAAAAAAAAAAJwiiAwAAAAAAAADgBEF0AAAAAAAAAACcIIgOAAAAAAAAAIATBNEBAAAAAAAAAHCCIDoAAAAAAAAAAE4QRAcAAAAAAAAAwAlvZysAAAAAlA+Nxq8t600AAAAA3BaZ6AAAAAAAAAAAOEEQHQAAAAAAAAAAJwiiAwAAAAAAAADgBEF0AAAAAAAAAACcYGBR4AYH2zo6rRfPHQAAAAAAAFDOkYkOAAAAAAAAAIATBNEBAAAAAAAAAHCCIDoAAAAAAAAAAE4QRAcAAAAAAAAAwAmC6AAAAAAAeKgrV67ISy+9JI0bN5YqVarIrbfeKq+++qpcu3bNamOz2SQ2NlaCg4NNmy5dusjBgwcdHicrK0tGjRolNWvWFD8/P+nTp4+cPHmyDPYIAIDi510CjwkAAAAAANzA9OnT5d1335UlS5ZIixYtZM+ePfLUU09JQECAPPvss6bNjBkzZObMmRIfHy9NmzaV1157Tbp37y6HDx8Wf39/0yY6Olo+/fRTWbFihQQFBUlMTIxERUXJ3r17xcvLq4z3EgDKp0bj1xb5Pken9SqRbSnvCKIDAAAAAOChduzYIQ888ID06vX/BVUaNWoky5cvN8F0exb67NmzZeLEidKvXz+zTAPutWvXlmXLlsmwYcPkwoULsmjRIlm6dKmEh4ebNgkJCVK/fn3ZuHGjREZGluEeAgBw8yjnAgAAAACAh+rUqZN8/vnn8t1335n5r7/+WrZv3y7333+/mU9JSZFTp05JRESEdZ9KlSpJ586dJSkpycxrtnl2drZDGy39EhISYrUBAMCdkYkOAAAAAICHeuGFF0wm+e23327Krly9elVef/11efzxx816DaArzTzPSeePHTtmtfH19ZXAwMA8bez3z4/WUdfJLj09vVj3DQCA4kImOgAAAAAAHmrlypWm9IqWZtm3b58p1fLmm2+a/3OqUKGCw7yWecm9LLfrtZk6daqpvW6ftPwLAACuiCA6AAAAAAAe6vnnn5fx48fLY489JqGhoTJw4EB57rnnTIBb1alTx/yfO6P8zJkzVna6trl8+bKkpaU5bZOfCRMmmCx4+3TixIkS2EMAAG4eQXQAAAAAADzUb7/9JhUrOoYGtKzLtWvXzO3GjRubIHliYqK1XgPmW7dulQ4dOpj5Vq1aiY+Pj0Ob1NRUSU5OttrkR2urV6tWzWECAMAVURMdAAAAAAAP1bt3b1MDvUGDBtKiRQv56quvZObMmfL000+b9VqOJTo6WqZMmSJNmjQxk96uWrWqDBgwwLTRUixDhgyRmJgYCQoKkho1asjYsWNNZnt4eHgZ7yEAADePIDoAAAAAAB4qLi5OXn75ZRk+fLgpvxIcHCzDhg2TV155xWozbtw4yczMNG20ZEtYWJhs2LBB/P39rTazZs0Sb29v6d+/v2nbrVs3iY+PN1ntAAC4uzIt57Jt2zZz1ls7aT27vXr1amtddna2GSVcz1z7+fmZNn/5y1/k559/dngMHcl71KhRUrNmTdOuT58+cvLkSYc22slrXTf7YCV6+/z586W2nwAAAAAAuCINhM+ePVuOHTtmgt8//PCDvPbaa+Lr62u10eP12NhYU6Ll999/N6VcQkJCHB6ncuXKJiB/7tw5UyLm008/ZaBQAEC5UaaZ6JcuXZKWLVvKU089JQ899JDDOu10dWRwPSOubTQQrpeQaZB8z549Vjtdpp3zihUrzGVjevlYVFSU7N271zrjrZeYaWB93bp1Zv6ZZ54xgXS9HwAAAAAAAAA402j8Wp4cD1emQfSePXuaKT+aMZ5zUBKlZ7Xbtm0rx48fN/XadPTuRYsWydKlS606awkJCeZs98aNGyUyMlIOHTpkguc7d+40l5yphQsXSvv27eXw4cPSrFmzUthT3Ay+qAAAAAAAAAB4ZDmXotKguV5GVr16dTOv2eZa9iUiIsJqo2Vf9LKypKQkM79jxw4TkLcH0FW7du3MMnsbAAAAAAAAAADcemBRrbs2fvx4U5qlWrVqZtmpU6dMnbbAwECHtrVr1zbr7G1q1aqV5/F0mb1NfrTWuk526enpxbg3AAAAAAAAAAB34BaZ6Jpt/thjj8m1a9dk7ty5121vs9lMxrpdztvO2uQ2depUayBSnbREDAAAAAAAAADAs1R0hwB6//79JSUlxdRIt2ehqzp16sjly5fNoKM5nTlzxmSj29ucPn06z+OePXvWapOfCRMmmPIx9unEiRPFul8AAAAAAAAAANfn0kF0ewD9yJEjZqDQoKAgh/WtWrUSHx8fhwFIU1NTJTk5WTp06GDmdQBRDYLv2rXLavPll1+aZfY2+alUqZIJ2OecAADwdHqlVps2bcTf39+URuvbt68ZqDv31V6xsbFmnJIqVapIly5d5ODBgw5ttGTaqFGjpGbNmuLn5yd9+vSRkydPOrTRk+QDBw60rgrT2+fPny+V/QQAAAAAwCWC6BkZGbJ//34zKc0219vHjx+XK1euyMMPPyx79uyRDz74QK5evWpqmOuk2edKD6iHDBkiMTEx8vnnn8tXX30lTz75pISGhkp4eLhp07x5c+nRo4cMHTpUdu7caSa9HRUVJc2aNSvL3QcAwO1s3bpVRowYYfpTPYmt/bUO8H3p0iWrzYwZM2TmzJkyZ84c2b17t7kqrHv37nLx4kWrTXR0tKxatUpWrFgh27dvN78JtG/W/t5Ox0HR3wXr1q0zk97WQDoAAAAAAB4zsKgGyLt27WrNjxkzxvw/aNAgk8G2Zs0aM3/XXXc53G/z5s0mq03NmjVLvL29TcZ6ZmamdOvWTeLj48XLy8tqr0H40aNHm4N8pdluemAPAACKRoPZOS1evNhkpO/du1fuvfdek4U+e/ZsmThxovTr18+0WbJkiSmhtmzZMhk2bJi5GmzRokWydOlS66R3QkKCGX9ErzyLjIyUQ4cOmb+lwfqwsDDTZuHCheYKM81850Q4AAAAAMAjgugaCNeDbWcKWmdXuXJliYuLM5MzNWrUMAfnAACgeGlA3N7X2q8q06vG7Ceu7SXSOnfuLElJSSaIrgF3LdmWs42WfgkJCTFtNIi+Y8cOc8WZPYCu2rVrZ5Zpm/yC6FoiRie79PR0Xm4AAAAAQPmuiQ4AAFyXnuzWq8g6depkAuBKA+gq9+DdOm9fp//7+vpKYGBggW00wz03XWZvk1+9dnv9dJ00sx0AAAAAgJtFEB0AANyQkSNHyjfffCPLly/Ps65ChQp5Au65l+WWu01+7Qt6nAkTJpjMePt04sSJIuwNAAAAAAD5I4gOAACKbNSoUWbsEh2npF69etZyHURU5c4WP3PmjJWdrm10kPC0tLQC25w+fTrP3z179myeLPecZWOqVavmMAEAAAAAcLMIogMAgELTTHDNQP/4449l06ZN0rhxY4f1Oq8B8MTERGuZBsy3bt0qHTp0MPOtWrUSHx8fhzapqamSnJxstdEBRDWbfNeuXVabL7/80iyztwEAAAAAoNwPLAoAANzLiBEjZNmyZfLJJ5+Iv7+/lXGuNcirVKliSq1ER0fLlClTpEmTJmbS21WrVpUBAwZYbYcMGSIxMTESFBRkBiUdO3ashIaGSnh4uGnTvHlz6dGjhwwdOlTmz59vlj3zzDMSFRWV76CiAAAAAACUFILoAACg0ObNm2f+79Kli8PyxYsXy+DBg83tcePGSWZmpgwfPtyUbAkLC5MNGzaYoLvdrFmzxNvbW/r372/aduvWTeLj48XLy8tq88EHH8jo0aMlIiLCzPfp00fmzJnDqwUAAAAAKFUE0QEAQJHKuVyPZqPHxsaayZnKlStLXFycmZzRDPWEhAReHQAAAABAmaImOgAAAAAAAAAAThBEBwAAAAAAAADACYLoAAAAAAAAAAA4QRAdAAAAAAAAAAAnGFgUAAAAAAAAgMdoNH5tWW8C3AyZ6AAAAAAAAAAAOEEQHQAAAAAAAAAAJwiiAwAAAAAAAADgBEF0AAAAAAAAAACcIIgOAAAAAAAAAIATBNEBAAAAAAAAAHCCIDoAAAAAAAAAAE4QRAcAAAAAAAAAwAmC6AAAAAAAAAAAOOHtbAUAAAAAAAAAoPxoNH5tkdofndarxLbFnZCJDgAAAAAAAACAE2SiAwAAALhpRc1qUmQ2AQAAwB2QiQ4AAAAAAAAAgBME0QEAAAAAAAAAcIIgOgAAAAAAAAAAThBEBwAAAAAAAADACYLoAAAAAAAAAAA4QRAdAAAAAAAAAABXDKJv27ZNevfuLcHBwVKhQgVZvXq1w3qbzSaxsbFmfZUqVaRLly5y8OBBhzZZWVkyatQoqVmzpvj5+UmfPn3k5MmTDm3S0tJk4MCBEhAQYCa9ff78+VLZRwAAAAAAAACA+yrTIPqlS5ekZcuWMmfOnHzXz5gxQ2bOnGnW7969W+rUqSPdu3eXixcvWm2io6Nl1apVsmLFCtm+fbtkZGRIVFSUXL161WozYMAA2b9/v6xbt85MelsD6QAAAAAAAAAAFMRbylDPnj3NlB/NQp89e7ZMnDhR+vXrZ5YtWbJEateuLcuWLZNhw4bJhQsXZNGiRbJ06VIJDw83bRISEqR+/fqyceNGiYyMlEOHDpnA+c6dOyUsLMy0WbhwobRv314OHz4szZo1K8U9BgAAAAAAAAC4E5etiZ6SkiKnTp2SiIgIa1mlSpWkc+fOkpSUZOb37t0r2dnZDm209EtISIjVZseOHaaEiz2Artq1a2eW2dvkR8vEpKenO0wAAAAAAAAAAM/iskF0DaArzTzPSeft6/R/X19fCQwMLLBNrVq18jy+LrO3yc/UqVOtGuo6aXY7AAAAAAAAAMCzuGwQ3U4HHM1d5iX3stxyt8mv/fUeZ8KECaZcjH06ceLEDW0/AAAAAAAAAMB9uWwQXQcRVbmzxc+cOWNlp2uby5cvS1paWoFtTp8+nefxz549myfLPSctHVOtWjWHCQAAAAAAAADgWcp0YNGCNG7c2ATAExMT5e677zbLNGC+detWmT59uplv1aqV+Pj4mDb9+/c3y1JTUyU5OVlmzJhh5nUA0f9fe3cCZ1PdP3D8O2bGmpkYe5ZRIVsSmlChGYMMSU8UiRLK1mAekXoaLbayPBHRXzPK1vMUpb8SypKGLJElkUJopGUYMsZ2/q/v7/+69zV3zGWWe+/c5fN+vY6555zfvc793XPv75zv+Z3vT3uSb968We644w6z7JtvvjHLWrRoUWjvDwAAAAAAAEDBRI5aThXCv4PoZ86ckQMHDjgMJrpjxw4pW7asVK9eXeLj42XcuHFSq1YtM+njkiVLSo8ePUx5zVXet29fGTFihERERJjnJSQkSMOGDSUmJsaUqVu3rrRv31769esns2fPNsv69+8vcXFxUqdOnUJ65wAAAAAAAAAAX1CoQfStW7dKmzZt7PPDhw83f3v37i3JyckycuRIycjIkIEDB5qULVFRUbJy5UopXbq0/TlTp06VkJAQ0xNdy0ZHR5vnBgcH28ssWLBAhg4dKrGxsWa+c+fOMmPGDI++VwAAAAAAAACA7ynUIHrr1q3NAJ/O6MCfiYmJZnKmePHiMn36dDM5oz3U58+fX+DtBQAAAAAAAAAEFq8dWBQAAAAAAAAAgMJGEB0AAAAAAAAAACcIogMAAAAAEMCOHTsmjz76qEREREjJkiXltttuk23bttnXaxpWTbNapUoVKVGihEnNumfPHofXyMzMlCFDhki5cuWkVKlSZiyyo0ePFsK7AQDA9QiiAwAAAAAQoNLS0qRly5YSGhoqn332mXz//fcyefJkuf766+1lJk2aJFOmTJEZM2bIli1bpFKlStK2bVs5ffq0vUx8fLwsXbpUFi9eLBs2bJAzZ85IXFycXLp0qZDeGQAAfjKwKAAAAAAAKDwTJ06UatWqSVJSkn1ZZGSkQy/0adOmyZgxY6Rr165m2bx586RixYqycOFCGTBggJw6dUrmzp0r7733nsTExJgy8+fPN6+7evVqadeuXSG8MwAAXIee6AAAAAAABKhly5ZJ06ZN5aGHHpIKFSpI48aN5e2337avP3jwoBw/flxiY2Pty4oVKyatWrWSlJQUM6+pXy5cuOBQRlO/NGjQwF4mJ5oCJj093WECAMAbEUQHAAAAACBA/fzzzzJr1iypVauWfP755/LUU0/J0KFD5d133zXrNYCutOd5VjpvW6d/ixYtKmXKlHFaJifjx4+X8PBw+6Q91wEA8EYE0QEAAAAACFCXL1+W22+/XcaNG2d6oWt6ln79+pnAelZBQUEO85rmJfuy7K5VZvTo0SYVjG06cuRIAd8NAADuQRAdAAAAAIAAVblyZalXr57Dsrp168ovv/xiHusgoip7j/ITJ07Ye6drmfPnz5tBSp2VyYmmhQkLC3OYAADwRgwsCuRT5KjleSp/aEJH6hoAAACAV2nZsqXs27fPYdn+/fulRo0a5nHNmjVNkHzVqlWmp7rSgPm6devMoKSqSZMmEhoaasp069bNLEtNTZXdu3fLpEmTPP6eAABwNYLoAAAAAAAEqGHDhkmLFi1MOhcNgG/evFnmzJljJqXpWOLj4816zZuukz4uWbKk9OjRw5TRfOZ9+/aVESNGSEREhJQtW1YSEhKkYcOGEhMTU8jvEACAgiOIDgAAAABAgGrWrJksXbrU5Cd/6aWXTM/zadOmSc+ePe1lRo4cKRkZGTJw4ECTsiUqKkpWrlwppUuXtpeZOnWqhISEmEC8lo2Ojpbk5GQJDg4upHcGAIDrEEQHAAAAACCAxcXFmckZ7Y2emJhoJmeKFy8u06dPNxMAeDJ9LuAJDCwKAAAAAAAAAIATBNEBAECurV+/Xjp16iRVqlQxvdI++ugjh/V9+vQxy7NOd955p0OZzMxMGTJkiJQrV05KlSolnTt3lqNHjzqU0VvFe/XqZXKs6qSPT548yScFAAAAAPA40rnA47gtBwB8199//y2NGjWSxx9/XB588MEcy7Rv316SkpLs80WLFnVYr4OTffLJJ7J48WIz+JgOQqa3kG/bts2eN1UHKtPA+ooVK8x8//79TSBdnwcAAAAAgCcRRAcAALnWoUMHM11NsWLFpFKlSjmuO3XqlMydO1fee+89iYmJMcvmz58v1apVk9WrV0u7du1k7969Jni+adMmM3CZevvtt6V58+ayb98+qVOnDp8YAAAAAMBjSOcCAABcau3atVKhQgWpXbu29OvXT06cOGFfp73NL1y4ILGxsfZlmhqmQYMGkpKSYuY3btxoUrjYAuhKU8LoMlsZAAAAAAA8hZ7oAADAZbSX+kMPPSQ1atSQgwcPygsvvCD33nuvCZ5rD/Xjx4+b9C5lypRxeF7FihXNOqV/NQifnS6zlcmJ5lrXySY9PZ1PFgAAAABQYATRAQCAy3Tv3t3+WHuXN23a1ATUly9fLl27dnX6PMuyzCCkNlkfOyuT3fjx42Xs2LEF2n4AAAAAALIjnQsAAHCbypUrmyD6jz/+aOY1V/r58+clLS3NoZymfNHe6LYyv/322xWv9fvvv9vL5GT06NEm57ptOnLkiMvfDwAAAAAg8OQriH7jjTfKn3/+ecXykydPmnUAAMC7FFbbrf+nBrM1mK6aNGkioaGhsmrVKnuZ1NRU2b17t7Ro0cLM6wCiGgTfvHmzvcw333xjltnK5ETTxYSFhTlMAAD4M87NAQDw4nQuhw4dkkuXLl2xXPOQHjt2zBXbBQAAXMhVbfeZM2fkwIED9nnNe75jxw4pW7asmRITE+XBBx80QXP9P5977jkpV66cPPDAA6a8Dg7at29fGTFihERERJjnJCQkSMOGDSUmJsaUqVu3rrRv394MSjp79myzrH///hIXFyd16tRxQW0AAOAfODcHAMALg+jLli2zP/7888/NibCNnph/8cUXEhkZ6dotBAAA+ebqtnvr1q3Spk0b+/zw4cPN3969e8usWbNk165d8u6775oe7hpI17Lvv/++lC5d2v6cqVOnSkhIiHTr1k0yMjIkOjpakpOTJTg42F5mwYIFMnToUImNjTXznTt3lhkzZrAnAADAuTkAAN4dRO/SpYv5q4N66clyVnprtp6ET5482bVbCAAA8s3VbXfr1q3NAJ/OaKD+WooXLy7Tp083kzPaQ33+/Pm53i4AAAIJ5+YAAHhxEP3y5cvmb82aNWXLli3m9mwAAOC9aLsBAPA/tO8AAPhATnTNfwoAAHwHbTcAAP6H9h0AAC8OoivNoarTiRMn7FfBbd555x1XbBsAAHAh2m4AAPwP7TsAAF4aRB87dqy89NJL0rRpUzNomOZZBQAA3ou2GwAA/0P7DgCAFwfR33rrLUlOTpZevXqJO128eFESExNlwYIFcvz4cROw79Onjzz//PNSpEgRU0YHN9MDhzlz5khaWppERUXJm2++KfXr17e/TmZmpiQkJMiiRYskIyNDoqOjZebMmVK1alW3bj8AAN7CU203AADwHNp3AAA84/8j0Xl0/vx5adGihbjbxIkTzUHBjBkzZO/evTJp0iR57bXXZPr06fYyumzKlCmmjA52WqlSJWnbtq2cPn3aXiY+Pl6WLl0qixcvlg0bNsiZM2ckLi5OLl265Pb3AACAN/BU2w0AADyH9h0AAC8Ooj/55JOycOFCcbeNGzfK/fffLx07dpTIyEj5xz/+IbGxsbJ161Z7L/Rp06bJmDFjpGvXrtKgQQOZN2+enD171r59p06dkrlz58rkyZMlJiZGGjduLPPnz5ddu3bJ6tWr3f4eAADwBp5quwEAgOfQvgMA4MXpXM6dO2fSp2gQ+tZbb5XQ0FCH9doz3BXuuusu0xN9//79Urt2bfnuu+9MT3INnNtGItc0LxpYtylWrJi0atVKUlJSZMCAAbJt2za5cOGCQ5kqVaqYgLuWadeunUu2FQAAb+apthsAAHgO7TsAAF4cRN+5c6fcdttt5vHu3bsd1rlykNFnn33W9CS/5ZZbJDg42KRfefXVV+WRRx4x6zWAripWrOjwPJ0/fPiwvUzRokWlTJkyV5SxPT8nmkddJ5v09HSXvS8AADzNU203AADwHNp3AIC7RY5anufnHJrQUfxNvoLoa9asEU94//33TeoVvf1cBwrdsWOHyW+uPcl79+7t9ORf07xcKyBwrTLjx483A5YCAOAPPNV2AwAAz6F9BwDAi3Oie8o///lPGTVqlDz88MPSsGFD6dWrlwwbNswEuJUOIqqy9yg/ceKEvXe6ltHBVtLS0pyWycno0aNNL3jbdOTIETe8QwAAAAAAAACA3/VEb9OmzVV7cX/55ZfiCjpAaJEijnF+Tety+fJl87hmzZomSL5q1SozYKjSgPm6detk4sSJZr5JkyYm76uW6datm1mWmppqbmWfNGmS0/9bc6vrBACAP/BU2w0AADyH9h0AAC8OottyqtrowJ2aakUD01nTrBRUp06dTA706tWrm3Qu27dvNwOfPfHEE2a9BgM0vcu4ceOkVq1aZtLHJUuWlB49epgy4eHh0rdvXxkxYoRERERI2bJlJSEhwfRsj4mJcdm2AgDgzTzVdgMAAM+hfQcAwIuD6FOnTs1xeWJiopw5c0ZcZfr06fLCCy/IwIEDTfoVzYU+YMAA+de//mUvM3LkSMnIyDBlNGVLVFSUrFy5UkqXLu2wvSEhIaYnupaNjo6W5ORk06sdAIBA4Km2GwAAeA7tOwAAnhFk6QibLnLgwAG544475K+//hJ/k56ebnq1a370sLCwwt6cgBvV1x/448jEAHy/7aHtBgKDtx5/cXwEf+Mt542+2r57S/0BKFzeetwC/zzGy23b49KBRTdu3CjFixd35UsCAAA3ou0GAMD/0L4DAOAF6Vy6du3qMK+d2XWwzq1bt5r0KwAAwLvQdgMA4H9o3wEA8OIgunZxz6pIkSJSp04deemllyQ2NtZV2wYAAFyEthsAAP9D+w4AgBcH0ZOSkly/JQAAwG1ouwEA8D+07wAAeHEQ3Wbbtm2yd+9eCQoKknr16knjxo1dt2UAAMDlaLsBAPA/tO8AAHhhEP3EiRPy8MMPy9q1a+X66683OdF1BNM2bdrI4sWLpXz58q7fUgAAkG+03QAA+B/adwAAPKNIfp40ZMgQSU9Plz179shff/0laWlpsnv3brNs6NChrt9KAABQILTdAAD4H9p3AAC8uCf6ihUrZPXq1VK3bl37Mk3n8uabbzKwKAAAXoi2GwAA/0P7DsDbRY5aXtibABReT/TLly9LaGjoFct1ma4DAADehbYbAAD/Q/sOAIAXB9HvvfdeeeaZZ+TXX3+1Lzt27JgMGzZMoqOjXbl9AADABWi7AQDwP7TvAAB4cRB9xowZcvr0aYmMjJSbbrpJbr75ZqlZs6ZZNn36dNdvJQAAKBDabgAA/A/tOwAAXpwTvVq1avLtt9/KqlWr5IcffhDLskxO9JiYGNdvIQAAKDDabgAA/A/tOwAAXtgT/csvvzTB8vT0dDPftm1bMxr40KFDpVmzZlK/fn356quv3LWtAAAgj2i7AQDwP7TvAAB4cRB92rRp0q9fPwkLC7tiXXh4uAwYMECmTJniyu0DAAAFQNsNAID/oX0HAMCL07l89913MnHiRKfrY2Nj5fXXX3fFdgEAABeg7QYAwP/QvgMAvFnkqOV5fs6hCR3Fb4Lov/32m4SGhjp/sZAQ+f33312xXYDf8ccfEADej7YbgD8dH3FsBPw/2ncAALw4ncsNN9wgu3btcrp+586dUrlyZVdsFwAAcAHabgAA/A/tOwAAXhxEv+++++Rf//qXnDt37op1GRkZ8uKLL0pcXJwrtw8AABQAbTcAAP6H9h0AAC9O5/L888/LkiVLpHbt2jJ48GCpU6eOBAUFyd69e+XNN9+US5cuyZgxY9y3tQAAIE9ouwEA8D+07wAAeHEQvWLFipKSkiJPP/20jB49WizLMss1kN6uXTuZOXOmKQMAALwDbTcAAP6H9h0AAC8OoqsaNWrIp59+KmlpaXLgwAETSK9Vq5aUKVPGPVsIAAAKhLYbAAD/Q/sOAIAXB9FtNGjerFkz124NAABwG9puAAD8D+07AABeNrAoAAAAAAAAAACBhCA6AAAAAAAAAACuTucCAAAAAAAAIHBEjlpe2JsAFAp6ogMAAAAAAGP8+PESFBQk8fHx9hqxLEsSExOlSpUqUqJECWndurXs2bPHocYyMzNlyJAhUq5cOSlVqpR07txZjh49Sq0CAPwCQXQAAAAAACBbtmyROXPmyK233upQG5MmTZIpU6bIjBkzTJlKlSpJ27Zt5fTp0/YyGnRfunSpLF68WDZs2CBnzpyRuLg4uXTpEjULAPB5BNEBAAAAAAhwGvTu2bOnvP3221KmTBmHXujTpk2TMWPGSNeuXaVBgwYyb948OXv2rCxcuNCUOXXqlMydO1cmT54sMTEx0rhxY5k/f77s2rVLVq9eXYjvCgAA1yCIDgAAAABAgBs0aJB07NjRBMGzOnjwoBw/flxiY2Pty4oVKyatWrWSlJQUM79t2za5cOGCQxlN/aIBd1sZAAB8GQOLAgAAAAAQwDQFy7fffmtStWSnAXRVsWJFh+U6f/jwYXuZokWLOvRgt5WxPT8nmkddJ5v09PQCvxcAAAIyiH7s2DF59tln5bPPPpOMjAypXbu2uU2sSZMm9lvLxo4da/K2paWlSVRUlLz55ptSv359+2too5yQkCCLFi0yrxEdHS0zZ86UqlWrFuI7AwAAAACgcB05ckSeeeYZWblypRQvXtxpOR1sNCs9F8++LLtrldFBTPV8HgCAyFHL81QJhyZ09GileXU6Fw2Kt2zZUkJDQ00Q/fvvvzc51q6//np7GQY4AQAAAAAgfzQVy4kTJ0xHtZCQEDOtW7dO3njjDfPY1gM9e49yfY5tnQ40ev78eXMO76xMTkaPHm3yqdsmDegDAOCNvDqIPnHiRKlWrZokJSXJHXfcIZGRkaYX+U033WTWM8AJAAAAAAD5p+fYOgDojh077FPTpk3NIKP6+MYbbzRB8lWrVtmfowFzDbS3aNHCzGsAXju/ZS2Tmpoqu3fvtpfJieZWDwsLc5gAAPBGXh1EX7ZsmWm8H3roIalQoYIZ4VtHCrdhgBMAADxr/fr10qlTJzNYmN6e/dFHHzms1wvciYmJZn2JEiWkdevWsmfPHocymmZtyJAhUq5cOSlVqpR07txZjh496lBGe7L16tVLwsPDzaSPT5486ZH3CABAICldurQZADTrpO1zRESEeaztfXx8vIwbN06WLl1qAuN9+vSRkiVLSo8ePcxraFvdt29fGTFihHzxxReyfft2efTRR6Vhw4ZXDFQKAIAv8uqc6D///LPMmjVLhg8fLs8995xs3rxZhg4daq5WP/bYYwxwAgCAh/3999/SqFEjefzxx+XBBx+8Yr0tzVpycrIZx+SVV16Rtm3byr59+8xJutIT8U8++cQMYqYn6HrCHRcXZ24nDw4ONmX0pFwD6ytWrDDz/fv3N4F0fR4Q6PKaLxIACmrkyJFmfLGBAwfaxyLTHOq2tl1NnTrVpH/p1q2bfSwyPR6wte0AAPgyrw6iX7582fRE1yveSnuia282DaxrEN2GAU4AAPCMDh06mCkn2dOsqXnz5plcqAsXLpQBAwaYfKc6QPh7771n75k2f/58k75t9erV0q5dO9m7d68Jnm/atMmcpCu9E6158+YmGF+nTh0+bgAA3Gjt2rUO83rOrXea6eSMDko6ffp0MwEA4G+8Op1L5cqVpV69eg7L6tatK7/88ot5rHnZFAOcAABQ+FyVZm3jxo3mtnBbAF3deeedZpmtDAAAAAAAnuLVPdFbtmxpepxltX//fqlRo4Z5XLNmTfsAJ9pLPesAJzooafYBTvS2sqwDnOgt587oSb9OgK/drn1oQke3bAsAXIvtorb2PM9K5w8fPmwvU7RoUSlTpswVZWzP1786Fkp2uiz7hfPsudZ1sklPT+dDAwAAAAD4dxB92LBhZiRvTeeiAXDNiT5nzhwzqawDnNSqVctM+tjZACead7Vs2bKSkJDAACcAALiJK9Ks5VT+Wq8zfvx4GTt2bL62GQAAAAAAn0zn0qxZMzP696JFi8xt3i+//LLJtdqzZ0+HAU40kK4DnGj+9GPHjuU4wEmXLl1MIF57t2uQXQcmY4ATAABcx1Vp1rTMb7/9dsXr//7771f0cs9q9OjRJue6bTpy5IhL3hcAAAAAILB5dRBdxcXFya5du+TcuXNmoLF+/frlOMCJpmjRMprKRQPuOQ1w8ueff8rZs2dNAF0HMAMAAK6TNc2ajS3Nmt5Zlj3Nmo0tzZqtjA4gqkFwvQPN5ptvvjHLbGVyomnYwsLCHCYAAAAAAPw6nQsAAPAuZ86ckQMHDjgMJrpjxw6TLq169eouSbOmg4i3b9/eXDifPXu2Wda/f39zYb1OnTqF9M4BAAAAAIGKIDoAAMi1rVu3Sps2bezzw4cPN3979+4tycnJJs1aRkaGSbOmKVuioqJyTLMWEhJi0qxp2ejoaPPcrGnWFixYIEOHDpXY2Fgz37lzZ5kxYwafFAAAAADA4wiiAwCAXGvdurUZ4NMZW5o1nZyxpVnTyRntoT5//nw+GQAAAABAofP6nOgAAAAAAAAAABQWgugAAAAAAAAAADhBEB0AAAAAAAAAACcIogMAAAAAAAAA4ARBdAAAAAAAAAAAnCCIDgAAAAAAAACAEwTRAQAAAAAAAABwgiA6AAAAAAAAAABOhDhbAQAAAAAAAMA/RY5aXtibAPgMeqIDAAAAAAAAAOAEQXQAAAAAAAAAAJwgiA4AAAAAAAAAgBME0QEAAAAAAAAAcIIgOgAAAAAAAAAAThBEBwAAAAAAAADACYLoAAAAAAAAAAA4QRAdAAAAAAAAAAAnQpytAOCbIkctz1P5QxM6um1bAAAAAAAAAF9HT3QAAAAAAAAAAJwgiA4AAAAAAAAAgBME0QEAAAAAAAAAcIIgOgAAAAAAAAAAThBEBwAAAAAAAADAiRBnK4DciBy1nIoCAAAAAAAA4LfoiQ4AAAAAAAAAgBME0QEAAAAAAAAAcIIgOgAAAAAAAAAA/hBEHz9+vAQFBUl8fLx9mWVZkpiYKFWqVJESJUpI69atZc+ePQ7Py8zMlCFDhki5cuWkVKlS0rlzZzl69GghvAMAAAAAAAAAgC/xmSD6li1bZM6cOXLrrbc6LJ80aZJMmTJFZsyYYcpUqlRJ2rZtK6dPn7aX0aD70qVLZfHixbJhwwY5c+aMxMXFyaVLlwrhnQAAAAAAAAAAfIVPBNE16N2zZ095++23pUyZMg690KdNmyZjxoyRrl27SoMGDWTevHly9uxZWbhwoSlz6tQpmTt3rkyePFliYmKkcePGMn/+fNm1a5esXr26EN8VAAAAAAAAAMDb+UQQfdCgQdKxY0cTBM/q4MGDcvz4cYmNjbUvK1asmLRq1UpSUlLM/LZt2+TChQsOZTT1iwbcbWVyoilg0tPTHSYAAAAAAAAAQGAJES+nKVi+/fZbk6olOw2gq4oVKzos1/nDhw/byxQtWtShB7utjO35zvKvjx071kXvAgAAAAAAAADgi7y6J/qRI0fkmWeeMelXihcv7rScDjaalaZ5yb4su2uVGT16tEkFY5t0WwAAAAAAAAAAgcWre6JrKpYTJ05IkyZN7Mt0MND169ebgUT37dtnlmmP8sqVK9vL6HNsvdN1oNHz589LWlqaQ290LdOiRQun/7emhdEJAAAAAAAA8GaRo5YX9iYAfs2re6JHR0ebAUB37Nhhn5o2bWoGGdXHN954owmSr1q1yv4cDZivW7fOHiDXAHxoaKhDmdTUVNm9e/dVg+gAAAAAAAAAAHh1T/TSpUubAUCzKlWqlERERNiXx8fHy7hx46RWrVpm0sclS5aUHj16mPXh4eHSt29fGTFihHle2bJlJSEhQRo2bHjFQKUAAAAAAAAAAPhMED03Ro4cKRkZGTJw4ECTsiUqKkpWrlxpAvA2U6dOlZCQEOnWrZspqz3ck5OTJTg4uFC3HQAAAAAAAADg3XwuiL527VqHeR0cNDEx0UzO6KCk06dPNxMAAAAAAAAAAH4bRAcAAAAAAAD8GQOFAt7FqwcWBQAAAAAAAACgMBFEBwAAAAAAAADACYLoAAAAAAAAAAA4QRAdAAAAAAAAAAAnCKIDAAAAAAAAAOAEQXQAAAAAAAAAAJwgiA4AAAAAQIAaP368NGvWTEqXLi0VKlSQLl26yL59+xzKWJYliYmJUqVKFSlRooS0bt1a9uzZ41AmMzNThgwZIuXKlZNSpUpJ586d5ejRox5+NwAAuEeIm14XAAAAAAB4uXXr1smgQYNMIP3ixYsyZswYiY2Nle+//94Ew9WkSZNkypQpkpycLLVr15ZXXnlF2rZta4LtGnxX8fHx8sknn8jixYslIiJCRowYIXFxcbJt2zYJDg4u5HcJFK7IUcv5CAAfRxAdAAAAAIAAtWLFCof5pKQk0yNdg9/33HOP6YU+bdo0E1zv2rWrKTNv3jypWLGiLFy4UAYMGCCnTp2SuXPnynvvvScxMTGmzPz586VatWqyevVqadeuXaG8NwAAXIV0LgAAAAAAwNCAuCpbtqz5e/DgQTl+/LjpnW5TrFgxadWqlaSkpJh5DbhfuHDBoYymfmnQoIG9TE40BUx6errDBACANyKIDgAAXEbzpQYFBTlMlSpVsq8npyoAAN5L2+nhw4fLXXfdZQLgSgPoSnueZ6XztnX6t2jRolKmTBmnZZzlYw8PD7dP2nMdAABvRBAdAAC4VP369SU1NdU+7dq1y77OllN1xowZsmXLFhNg15yqp0+ftpfRnKpLly41OVU3bNggZ86cMTlVL126xCcFAIAbDR48WHbu3CmLFi26Yp1eGM8ecM++LLtrlRk9erTp+W6bjhw5UoCtBwDAfQiiAwAAlwoJCTHBcdtUvnx5szx7TlXt4aY5Vc+ePWtyqipbTtXJkyebnKqNGzc2OVU1EK85VQEAgHsMGTJEli1bJmvWrJGqVaval9vuKMveo/zEiRP23ula5vz585KWlua0TE40LUxYWJjDBACAN2JgUSDA5WeU8EMTOrplWwD4hx9//NHkQdUT46ioKBk3bpzceOON18ypqgOTXSun6tUGJtO8qjrZkFcV8D8ctwCupxe5NYCud4GtXbtWatas6bBe5zVIvmrVKnNxW2nAfN26dTJx4kQz36RJEwkNDTVlunXrZpbp3Wi7d+82d6EBAODrCKIDAACX0aD5u+++K7Vr15bffvtNXnnlFWnRooXs2bPnqjlVDx8+XKCcqra8qmPHjuXTBAAgDwYNGmTuCPv444+ldOnS9vZWc5SXKFHCpGPRVGt6UbxWrVpm0sclS5aUHj162Mv27dtXRowYIREREWZQ0oSEBGnYsKG5swwAAF9HEB0AALhMhw4d7I/1xLl58+Zy0003mbQtd955p9tyqtryqupgaFl7ojNAGQAAVzdr1izzt3Xr1g7Lk5KSpE+fPubxyJEjJSMjQwYOHGhStuhF85UrV5qgu83UqVNNSjftia5lo6OjJTk5WYKDg/kIAAA+jyA6AABwm1KlSplguqZ46dKli1mmPdwqV658zZyqWXujaxnt0X41mhpGJwAAkHt6ofpa9EJ2YmKimZwpXry4TJ8+3UwAAPgbBhYFAABuoznK9+7da4LmWXOq2thyqtoC5FlzqtrYcqpeK4gOAAAAAIA70BMdAAC4jOY/7dSpk1SvXt30Htec6JpWpXfv3uRUBQAAAAD4JILoAADAZY4ePSqPPPKI/PHHH1K+fHmTB33Tpk1So0YNs56cqgAAAAAAX0MQHXaRo5ZTGwCAAlm8ePFV15NTFQAAAADga8iJDgAAAAAAAACAE/REBwAAAAAAQEDeYX9oQke3bAsA/0JPdAAAAAAAAAAAnKAnOgAAAFCIGJcGAAAA8G70RAcAAAAAAAAAwAmC6AAAAAAAAAAAOEEQHQAAAAAAAAAAX8yJPn78eFmyZIn88MMPUqJECWnRooVMnDhR6tSpYy9jWZaMHTtW5syZI2lpaRIVFSVvvvmm1K9f314mMzNTEhISZNGiRZKRkSHR0dEyc+ZMqVq1aiG9MwAAAAAAAPji2CSHJnR0y7YA8F5e3RN93bp1MmjQINm0aZOsWrVKLl68KLGxsfL333/by0yaNEmmTJkiM2bMkC1btkilSpWkbdu2cvr0aXuZ+Ph4Wbp0qSxevFg2bNggZ86ckbi4OLl06VIhvTMAAAAAAAAAgC/w6p7oK1ascJhPSkqSChUqyLZt2+See+4xvdCnTZsmY8aMka5du5oy8+bNk4oVK8rChQtlwIABcurUKZk7d6689957EhMTY8rMnz9fqlWrJqtXr5Z27doVynsDAAAAAABAYPReB+DbvDqInp0GxFXZsmXN34MHD8rx48dN73SbYsWKSatWrSQlJcUE0TXgfuHCBYcyVapUkQYNGpgyBNEB+PoBGbcSAgAAAAAAuI/PBNG11/nw4cPlrrvuMgFwpQF0pT3Ps9L5w4cP28sULVpUypQpc0UZ2/NzonnUdbJJT0936fsBAAAAAAAAAHg/nwmiDx48WHbu3GlymmcXFBR0RcA9+7LsrlVGBzXVAUsBoCC4zQ8AAAAAAMC3efXAojZDhgyRZcuWyZo1a6Rq1ar25TqIqMreo/zEiRP23ula5vz585KWlua0TE5Gjx5t0sfYpiNHjrj4XQEAAAAAAAAAvJ1XB9G1t7j2QF+yZIl8+eWXUrNmTYf1Oq9B8lWrVtmXacB83bp10qJFCzPfpEkTCQ0NdSiTmpoqu3fvtpfJieZWDwsLc5gAAAAAAAAAAIHFq9O5DBo0SBYuXCgff/yxlC5d2t7jPDw8XEqUKGHSscTHx8u4ceOkVq1aZtLHJUuWlB49etjL9u3bV0aMGCERERFmUNKEhARp2LChxMTEFPI7BAAAAAAAAAB4M68Oos+aNcv8bd26tcPypKQk6dOnj3k8cuRIycjIkIEDB5qULVFRUbJy5UoTdLeZOnWqhISESLdu3UzZ6OhoSU5OluDgYA+/IwDwjrzrhyZ05KMAAAAAAADw9SC6pnO5Fu2NnpiYaCZnihcvLtOnTzcTAAAAAAAAAAB+kRMdAAAAAAAAAIDCRBAdAAAAAAAAAABfTOcCAAAAAACAwJSf8Z8AwB0IogNw+4EMg1h6HwYjBQAAAAAAyB2C6ACQB/SEAAAAAAAACCzkRAcAAAAAAAAAwAmC6AAAAAAAAAAAOEEQHQAAAAAAAAAAJ8iJDsBvBrFkwFMAAAAAAAC4Gj3RAQAAAAAAAABwgp7oAAJWfnrIBzJ6+gMAfBHtFwAAAAqKnugAAAAAAAAAADhBT3QAAADARbjLCQAAAPA/9EQHAAAAAAAAAMAJeqID8Er05AMAAAAA78WYEwACCUF0AAAAAAAAuBUdpQD4MtK5AAAAAAAAAADgBEF0AAAAAAAAAACcIJ2LH+NWKQAAAAAAAAAoGHqiAwAAAAAAAADgBD3RAQBeczfMoQkd3bItAAAAAJzjTnYAuDp6ogMAAAAAAAAA4AQ90QEAAAAAAPwIPcsBwLXoiQ4AAAAAAAAAgBME0QEAAAAAAAAAcIJ0LgAAAEAOuBUeAAAAgKInOgAAAAAAAAAAThBEBwAAAAAAAADACdK5AAAAAAAAeCnSiwFA4SOIDgDw2ROEQxM6um1bAACBKT/BKtojAAAA/xZQQfSZM2fKa6+9JqmpqVK/fn2ZNm2a3H333eILuPIMAAhEvtx2AwAQiGi7AQD+KGCC6O+//77Ex8ebBr1ly5Yye/Zs6dChg3z//fdSvXr1wt48AACQDW03XI1OCQDgXrTd10ZbBAC+KciyLEsCQFRUlNx+++0ya9Ys+7K6detKly5dZPz48dd8fnp6uoSHh8upU6ckLCxMPI2GFgAC7/b5wm57Cpuvt93wvnROHE/BW/h7+xXIAr3toe2+NtoiAPCu46nctt0B0RP9/Pnzsm3bNhk1apTD8tjYWElJSSmUbaLhBIDC+S0lcOEbvLHthncd53AsBQDehbYbAODPAiKI/scff8ilS5ekYsWKDst1/vjx4zk+JzMz00w2ejXCdnXCFS5nnnXJ6wAA8qb6sP+6vcp2j23nktextTkBctOY17fdDV783CWvAwCeaIuQN7TdBUfbDQDwJFed5+X2vDsggug2QUFBDvNaOdmX2eht4mPHjr1iebVq1dy2fQAA/xA+zbWvd/r0aXN7WSCi7QYAeAJtt+vQdgMA/LHtDoggerly5SQ4OPiKnmsnTpy4ooebzejRo2X48OH2+cuXL8tff/0lERERTgPvhU2vnGiQ/8iRIwGZf88Z6oW6Yb/h++SrvzV6sVcb8ipVqkigcVfbTZuQO9QT9eRK7E/UVSDtU7Td7jnv9ubPPDfYfuqffYfvLr893vu7mdu2OyCC6EWLFpUmTZrIqlWr5IEHHrAv1/n7778/x+cUK1bMTFldf/314gt05/LFAwt3o16oG/Ybvk+++FsTqD3Q3d120ybkDvVEPbkS+xN1FSj7FG23+867vfUzzy22n/pn3/FNfHf9v+5z03YHRBBd6dXtXr16SdOmTaV58+YyZ84c+eWXX+Spp54q7E0DAAA5oO0GAMC30HYDAPxVwATRu3fvLn/++ae89NJLkpqaKg0aNJBPP/1UatSoUdibBgAAckDbDQCAb6HtBgD4q4AJoquBAweayV/pbXAvvvjiFbfDBTrqhbphv+H7xG+N73J1202bQD25EvsT9eRq7FPUkz9wx3m3r3832H7qn32H7y6/Pb7/uxlkafZ0AAAAAAAAAABwhSJXLgIAAAAAAAAAAATRAQAAAAAAAAC4CnqiAwAAAAAAAADgBEF0H5aWlia9evWS8PBwM+njkydPXvU5S5YskXbt2km5cuUkKChIduzYIf5g5syZUrNmTSlevLg0adJEvvrqq6uWX7dunSmn5W+88UZ56623xF/lpW5SU1OlR48eUqdOHSlSpIjEx8eLv8pLvej3pm3btlK+fHkJCwuT5s2by+effy7+Ki91s2HDBmnZsqVERERIiRIl5JZbbpGpU6eKv8rrb43N119/LSEhIXLbbbe5fRtxpVdffVVatGghJUuWlOuvvz5XVdSnTx/TTmad7rzzTr+u3vzUkw6tk5iYKFWqVDG/Aa1bt5Y9e/aIP8vP8Vcg7E8ci7m+ntauXXvFfqPTDz/8IP5s/fr10qlTJ/O7ou/3o48+uuZzAunY3l/l57c1p++HTq+99pq9jLZL2dc//PDDPtM2ZGZmypAhQ8z5e6lSpaRz585y9OjRQt/+CxcuyLPPPisNGzY026Xf18cee0x+/fVXh3Luqn93tDkffvih1KtXzwxiqH+XLl1a4O301LlocnJyjt+Fc+fO+Uxb5qn6z8u25/Qd1al+/fqFUvfuah89Vffr87j93rjvE0T3YRrs1CD4ihUrzKSPtbG7mr///tsEvCZMmCD+4v333zfB3jFjxsj27dvl7rvvlg4dOsgvv/ySY/mDBw/KfffdZ8pp+eeee06GDh1qfjj8TV7rRg/S9AdKyzdq1Ej8VV7rRX/s9cf7008/lW3btkmbNm3Mj78+N9DrRg+aBw8ebOpo79698vzzz5tpzpw5Euh1Y3Pq1ClzUhEdHe2xbYWj8+fPy0MPPSRPP/10nqqmffv25uKibdLfAH+Wn3qaNGmSTJkyRWbMmCFbtmyRSpUqmd/L06dPi7/Kz/GXv+9PHIu5p55s9u3b57Dv1KpVS/yZnq/ocaj+ruRGIB3b+7P8/LZm/V7o9M4775jgyYMPPuhQrl+/fg7lZs+e7RXbn5u2QX8zNKC1ePFi03nlzJkzEhcXJ5cuXSrU7T979qx8++238sILL5i/Guzav3+/CfJn5+r6d0ebs3HjRunevbt5z9999535261bN/nmm28KtK2ePBfVIGP274QGTgt7+3PTlnmq/vO67f/+978dtvnIkSNStmxZc7xcGHXvjvbRk/v+33ncfm/b9w0LPun777+39OPbtGmTfdnGjRvNsh9++OGazz948KApu337dsvX3XHHHdZTTz3lsOyWW26xRo0alWP5kSNHmvVZDRgwwLrzzjstf5PXusmqVatW1jPPPGP5o4LUi029evWssWPHWv7GFXXzwAMPWI8++qjlb/JbN927d7eef/5568UXX7QaNWrk5q3E1SQlJVnh4eG5qqTevXtb999/f0BWaG7r6fLly1alSpWsCRMm2JedO3fOPPett96y/FF+j7/8fX/iWMw99bRmzRqzb6WlpVmBSt//0qVLr1omkI7t/VVBz21t9Hf23nvv9fg5jbvahpMnT1qhoaHW4sWL7cuOHTtmFSlSxFqxYkWhb392mzdvNs85fPiwW+vfHW1Ot27drPbt2zuUadeunfXwww9bvnAumpdjXG9syzxV/wWte22PgoKCrEOHDhVK3bujffTkvp/X7fe2fV/RE91H6dUivc0qKirKvkxv/dJlKSkpEii015xekYqNjXVYrvPO6kHrLnt5TXGzdetWc1taINdNIHBFvVy+fNn0stSr0P7EFXWjV4W1bKtWrcSf5LdukpKS5KeffpIXX3zRA1sJV9PbTytUqCC1a9c2vahOnDhBJWfr3XL8+HGH74XeBqrff39tZwpy/OWv+xPHYu6rJ5vGjRtL5cqVzR1Na9asKcCn5Z8C5djen7ni3Pa3336T5cuXS9++fa9Yt2DBApMORVMwJCQkuPxuKXe1Dfqboftw1v1b0yA0aNDApe2sq2ILevel3gmQPSWcK+vfXW2OszKuPp5x57mo3qVQo0YNqVq1qrlbwR13TburLfNE/bui7ufOnSsxMTGmnj1d9/nhTfu+KxTmvm9DEN1H6UmrNrbZ6TJdFyj++OMPcytbxYoVHZbrvLN60OU5lb948aJ5vUCum0DginqZPHmyuRVJb3PyJwWpG22wNHjWtGlTGTRokDz55JMS6HXz448/yqhRo8yJg+ZDh2/RWzv1s/vyyy/Nd15Tldx7770m7RX+n23fD6R2Jr/HX/68P3Es5r560mCDpkfT2641VYKOWaPBB729GYF3bO/PXHFuO2/ePCldurR07drVYXnPnj1l0aJFJlitqUf0+5S9jLe2DfrcokWLSpkyZdzazrqi/jX/sB73aloYTa3grvp3V5vjrIyrj2fcdS6q41Jpbuhly5aZ+tZUFprGV89HfKEt80T9F7TuNUXIZ599dsV5rqfqPj+8ad93hcLc9204q/cyOjjX2LFjr1pGG1elV3mz07siclru77K/52vVQ07lc1oeiHUTKPJbL/rDrN/Tjz/+OMeDzUCtGx2QRa8Ab9q0yRxA33zzzfLII49IoNaNHqDpSYT+nmvPIhRee6kXdvJDcwPaaI8vfR3t4aC93Fx98u3L9eQv7Yy7j7/8ZX+6Go7FXF9PGmjQyUYH1NJ8rK+//rrcc889Bfq8/E0gHdv7Ek+e22o+dA3YZs+Dq727s/7+ah5m/Q3WPN633367T7YNua0XT9W/9mjVwUK1l6gO2uiq+vd0m+PJ4xlXn4vqnQNZB6XVIKLW7/Tp0+WNN97wibbMU/Wf3/9HA7V6l0WXLl0clnu67vPK2/b9/PKWfZ8gupfRAfquNVp1ZGSk7Ny509yylt3vv/9+xVUkf6a3hQUHB19xlUxvg3NWDzroWU7ltadoRESEBHLdBIKC1IsORKK3iP73v/81t3H5m4LUjY5wrho2bGh+m7SB86cgel7rRm8z09vk9FYy/V1XemKhByT6W7Ny5UrT0wjuby9dRXvR6ImtN/Qq8ZZ60vZU6fdC68eX2xlPH3/56v6UE47F3FdPOdETxfnz5+fxU/JvgXJs74s89duqnTl00EI9Vr8WDa6Ehoaa399rBXELu23QfVtTUKSlpTn0Rtf9u0WLFtd8PU9svwbQtVeopnjTHvVZe6EXtP492eY4K+Pq4xlPnYsWKVJEmjVr5vLjDHe1ZZ6o/4Jsu57D6YU6HXRT7w4pjLrPD2/a9wvCG/Z9G4LoXka/2Dpdi16905xjmzdvljvuuMMs09FzdVluGlR/oT9gTZo0kVWrVskDDzxgX67z999/v9O6++STTxyWaUBLr4hrgx7IdRMI8lsveuXziSeeMH87duwo/shV+4weZPhDioKC1I2eQOzatcthmfbM0ZOLDz74wH7RAe5vL13lzz//NL1msgaLA72edD/WA2/9HmieS6Un++vWrZOJEyeKL/H08Zev7k854VjMffWUE7046w/7jSsFyrG9L/LUb6vmKdbvV6NGja5Zds+ePSbwm5vvUWG3DfqedB/W3wlb+gJNKbF7926ZNGlSoW+/LYCuwSrNcZ2bi1Z5qX9PtjlaRl9j2LBhDmVcHVvx1Lmono/t2LHDdHDyhbbME/VfkG3XY9sDBw7kOOaCp+o+P7xp388vb9n3s/4H8FE6gu6tt95qRs7WqWHDhlZcXJxDmTp16lhLliyxz//555/W9u3breXLl5vRcHWkb51PTU21fJW+Bx21fO7cuWZk8fj4eKtUqVL2EZN1pOVevXrZy//8889WyZIlrWHDhpny+jx9/gcffGD5m7zWjdL9QacmTZpYPXr0MI/37NljBXK9LFy40AoJCbHefPNN812xTSdPnrT8TV7rZsaMGdayZcus/fv3m+mdd96xwsLCrDFjxlj+Jj/fp6xefPFFq1GjRh7cYtgcPnzY/JbpSO7XXXed/Xfu9OnTObaXunzEiBFWSkqKdfDgQWvNmjVW8+bNrRtuuMFKT0/324rNaz2pCRMmWOHh4WbZrl27rEceecSqXLmyX9dTXo+/AmF/4ljMPfU0depUa+nSpaZ93b17t1mvx+8ffvih5c/0O2P7/dH3O2XKFPNYf6MC/djen+Xn3FadOnXKfP6zZs264jUPHDhg2rQtW7aY3189B77lllusxo0bWxcvXvSJtuGpp56yqlataq1evdr69ttvrXvvvdccTxb29l+4cMHq3Lmz2bYdO3Y4nCNlZma6tf7d0eZ8/fXXVnBwsDmu2bt3r/mr53+bNm3K93Z68lw0MTHRWrFihfXTTz+Z38vHH3/cPOebb77xibbMU/Wf3/O5Rx991IqKisrxNT1Z9+5oHz2575/O4/Z7276vCKL7MA2I9+zZ0ypdurSZ9HFaWppDGd0xk5KS7PP6WJdlnzS448v0S1WjRg2raNGi1u23326tW7fOvq53795Wq1atHMqvXbvWNN5aPjIyMseDLn+R17rJaf/Q5wdyvejjnOpFy/mjvNTNG2+8YdWvX980zho81+/VzJkzrUuXLln+KK/fp6wIohce/Wxy+g7rCWtO7eXZs2et2NhYq3z58uZAs3r16uY1fvnlF8uf5bWe1OXLl82+XalSJatYsWLWPffcY4Lp/iyvx1+Bsj9xLOb6epo4caJ10003WcWLF7fKlClj3XXXXSYI5e/0N+dqx12Bfmzvr/Jzbqtmz55tlShRIsfOLfo7q+1S2bJlzb6h36ehQ4ea/8tX2oaMjAxr8ODB5j3o+9TAtjvaj7xuvwbFc/qeZj1ucGf9u6PN+e9//2suFOjnocF+d16wdPW5qAaDdR/S19N9SvctvUDjS22Zp+o/r/uO/rbod2/OnDk5vp4n695d7aOn6n5NHrffG/f9IP3HPX3cAQAAAAAAAADwbUUKewMAAAAAAAAAAPBWBNEBAAAAAAAAAHCCIDoAAAAAAAAAAE4QRAcAAAAAAAAAwAmC6AAAAAAAAAAAOEEQHQAAAAAAAAAAJwiiAwAAAAAAAADgBEF0AAAAAAAAAACcIIgOwCt89NFHcvPNN0twcLDEx8cX9uYAAAAnEhMTpWLFihIUFGTabwAAAl3r1q05jwX8HEF0AF5hwIAB8o9//EOOHDkiL7/8cmFvDgAAyMHevXtl7NixMnv2bElNTZUOHTpQTwAABJjk5GS5/vrrC3szAI8K8ex/BwBXOnPmjJw4cULatWsnVapUoYoAAPBSP/30k/l7//33m57oAAAAQCCgJzqAHG9FGzJkiLkdrUyZMuaW7Tlz5sjff/8tjz/+uJQuXVpuuukm+eyzz+zPWbZsmdSqVUtKlCghbdq0kXnz5pmT65MnT161hteuXWteT917773mObpMpaSkyD333GNes1q1ajJ06FCzDQAA4OouX74sEydONKnSihUrJtWrV5dXX33VrDt27Jh0797dtPEREREmIH7o0KFcpXHp1KmTeVykSBGHIHpSUpLUrVtXihcvLrfccovMnDmTjwgAEJDS0tLkscceM+1syZIlzV1bP/74o0OZt99+25zj6voHHnhApkyZkuue3XpBW9tuPU+/7rrrpFmzZrJ69WqHMnq3WMeOHc25dM2aNWXhwoUSGRkp06ZNs5c5deqU9O/fXypUqCBhYWHmfPy7775zUS0A/ocgOoAcaRC8XLlysnnzZhNQf/rpp+Whhx6SFi1ayLfffmt6jffq1UvOnj1rTrw1FUuXLl1kx44dJjXLmDFjclWz+nr79u0zjz/88EPT2OuyXbt2mf+ja9eusnPnTnn//fdlw4YNMnjwYD4xAACuYfTo0SaI/sILL8j3339vTp71ZFvbbb3YrSfd69evN22rPm7fvr2cP3/+qq+ZkJBgguVK22udbIEAbfc1SK/pXsaNG2f+Xz2WAAAg0PTp00e2bt1qOppt3LhRLMuS++67Ty5cuGDWf/311/LUU0/JM888Y86f27Zta7/Qnds7ufX1NHC+fft2c96sF7l/+eUXexkN4v/666+mg5qeZ2unOL3720a3SYPsx48fl08//VS2bdsmt99+u0RHR8tff/3l4hoB/IQFANm0atXKuuuuu+zzFy9etEqVKmX16tXLviw1NdXSn5CNGzdazz77rNWgQQOH1xgzZoxZn5aWds361TJads2aNfZl+n/179/fodxXX31lFSlSxMrIyOAzAwDAifT0dKtYsWLW22+/fcW6uXPnWnXq1LEuX75sX5aZmWmVKFHC+vzzz69Zp0uXLjVtdlbVqlWzFi5c6LDs5Zdftpo3b85nBAAImHPoZ555xtq/f79pJ7/++mv7uj/++MO0s//5z3/MfPfu3a2OHTs6PL9nz55WeHh4vv//evXqWdOnTzeP9+7da7Zhy5Yt9vU//vijWTZ16lQz/8UXX1hhYWHWuXPnHF7npptusmbPnn3N/y8pKalA2wv4InKiA8jRrbfean8cHBxsbvdu2LChfZn2ZlN6NVt7kustZFndcccdBapZvRJ+4MABWbBgQdaLfub29IMHD5pbxgEAwJW0N3hmZqbpTeasfbWlUrM5d+6cPd95Xvz+++9mUPC+fftKv3797MsvXrwo4eHhfDwAgIBrg0NCQiQqKsq+TM+l69SpY9YpPX/WFC7Zz5//93//N1f/h6Y41UG+tbz2Ntc2NyMjw94TXV9ft0F7lttoejdNL5P1eEB7tOu2ZaWvk5/jASAQEEQHkKPQ0FCHec17mnWZLQ+qBrU1uJ19cDFdVhD6upoWRvOgZ6d5XQEAQM40/+nV2tcmTZo4XKS2KV++fJ6rVF/PltIla8DAdhEeAIBA4uw8OOs5c0HPn//5z3/K559/Lq+//roJjmu7r+lVbWnZrrYNWdvvypUr28cjyyq3udmBQEMQHUCB6QBimkctK80BVxB61XzPnj3moAAAAOSebaDvL774Qp588skr2lcdZ8Q2iFhB6Z1pN9xwg/z888/Ss2dPPiYAQECrV6+e6Rn+zTffmLG+1J9//in79++3302t58869lh+z5+/+uork3fd1ptde5RnHSBcX1+3QfOl64VzpXehnTx50uF4QPOha491HXAUwLUxsCiAAtMe4z/88IM8++yz5uDgP//5jyQnJ5t12a+w55a+lg7CMmjQIDPYio5mrgOz6CCnAADAueLFi5t2dOTIkfLuu++a27I3bdokc+fONYFuHTj8/vvvNyfhmiJt3bp1ZnCzo0eP5qtaExMTZfz48fLvf//bHAfo4OA6AOmUKVP4mAAAAXchW9tYTXGmg3d/99138uijj5oLzrpc6TmtdkLTdlLPc2fPni2fffZZrs+dtaPZkiVLzHmyvn6PHj3sd4bZgugxMTHSv39/E6zXYLo+1gvstv9D1zdv3ly6dOlierVrED4lJUWef/75XAf0L126ZLYh66SDmQP+iiA6gAKrWbOmfPDBB6Yh11zqs2bNkjFjxph1xYoVy9dr6uvoSb0eVNx9993SuHFjeeGFF8wtZwAA4Oq0zRwxYoT861//Mj3funfvbsYxKVmypKxfv96kRuvatatZ98QTT5gcqPntma693f/nf/7HXEDX8VNatWplHuvxAQAAgUYvJGsP8Li4OBOo1jQqGjS3pUdt2bKlvPXWWyaI3qhRI1mxYoUMGzbMXATPjalTp5r85trTvVOnTtKuXTuH/OdKL6Lr3WL33HOP6bGuQX0dD8X2f2gwXbdJ1+txQO3ateXhhx82wXTb+GfXoj3g9Tw963Tfffflub4AXxGko4sW9kYA8D+vvvqqOTDQwcYAAAAAAEDONMitd3frXWLuoHebVatWTVavXp3jwOMAro2c6ABcYubMmdKsWTMzuvfXX38tr732mgwePJjaBQAAAAAgCx0UtG3btlKqVCmTymXevHnmnNpVvvzyS9NTXO8QS01NNSneNPe59jwHkD+kcwHgEpp2RXO86UAqL7/8srmFXHOkqg4dOsh1112X4zRu3Dg+AQAAvISz9lond/WOAwAg0Giucg2ia5Bb7+B+44037IOB169f32lbvGDBgly9/oULF+S5554zr6XpXMqXLy9r1661p5S5FldsA+BvSOcCwO2OHTtmcq3mpGzZsmYCAACF78CBA07X6aBoOigZAABwn8OHD5sgeE40X7nmNg+EbQC8DUF0AAAAAAAAAACcIJ0LAAAAAAAAAABOEEQHAAAAAAAAAMAJgugAAAAAAAAAADhBEB0AAAAAAAAAACcIogMAAAAAAAAA4ARBdAAAAAAAAAAAnCCIDgAAAAAAAACAEwTRAQAAAAAAAACQnP0fyU8ptG5vtNIAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Automatically plot all label columns as histograms in subplots\n", + "\n", + "num_labels = len(labels.columns)\n", + "ncols = 3\n", + "nrows = (num_labels + ncols - 1) // ncols\n", + "\n", + "fig, axes = plt.subplots(nrows, ncols, figsize=(5 * ncols, 4 * nrows))\n", + "axes = axes.flatten()\n", + "\n", + "for i, col in enumerate(labels.columns):\n", + " axes[i].hist(labels[col].dropna(), bins=30)\n", + " axes[i].set_title(col)\n", + " axes[i].set_xlabel(col)\n", + " axes[i].set_ylabel('Count')\n", + "\n", + "# Hide any unused subplots\n", + "for j in range(i + 1, len(axes)):\n", + " axes[j].set_visible(False)\n", + "\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "5c2ff835", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-03T06:40:33.775143Z", + "iopub.status.busy": "2026-06-03T06:40:33.774985Z", + "iopub.status.idle": "2026-06-03T06:40:33.782905Z", + "shell.execute_reply": "2026-06-03T06:40:33.782562Z" + } + }, + "outputs": [], + "source": [ + "# Integer continuum-pixel indices. Keep only those valid for this grid.\n", + "continuum_pixels = np.loadtxt(SRC / \"bulge-ages-and-orbits/data/continuum.list\", dtype=int, comments=\"#\")\n", + "continuum_pixels = continuum_pixels[continuum_pixels < dispersion.size]\n", + "APOGEE_REGIONS = ([15090, 15822], [15823, 16451], [16452, 16971])" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "d1b1ac2a", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "E0609 22:31:56.428383 1266583 cuda_executor.cc:1206] [0] Failed to allocate device memory: INTERNAL: [0] Failed to allocate 29.76GiB (31950225408 bytes) of device memory: : CUDA_ERROR_OUT_OF_MEMORY: out of memory\n", + "E0609 22:31:56.428897 1266583 cuda_executor.cc:1206] [0] Failed to allocate device memory: INTERNAL: [0] Failed to allocate 26.78GiB (28755202048 bytes) of device memory: : CUDA_ERROR_OUT_OF_MEMORY: out of memory\n", + "E0609 22:31:56.429350 1266583 cuda_executor.cc:1206] [0] Failed to allocate device memory: INTERNAL: [0] Failed to allocate 24.10GiB (25879681024 bytes) of device memory: : CUDA_ERROR_OUT_OF_MEMORY: out of memory\n", + "E0609 22:31:56.429799 1266583 cuda_executor.cc:1206] [0] Failed to allocate device memory: INTERNAL: [0] Failed to allocate 21.69GiB (23291711488 bytes) of device memory: : CUDA_ERROR_OUT_OF_MEMORY: out of memory\n", + "E0609 22:31:56.430240 1266583 cuda_executor.cc:1206] [0] Failed to allocate device memory: INTERNAL: [0] Failed to allocate 19.52GiB (20962539520 bytes) of device memory: : CUDA_ERROR_OUT_OF_MEMORY: out of memory\n", + "E0609 22:31:56.430675 1266583 cuda_executor.cc:1206] [0] Failed to allocate device memory: INTERNAL: [0] Failed to allocate 17.57GiB (18866284544 bytes) of device memory: : CUDA_ERROR_OUT_OF_MEMORY: out of memory\n", + "E0609 22:31:56.431118 1266583 cuda_executor.cc:1206] [0] Failed to allocate device memory: INTERNAL: [0] Failed to allocate 15.81GiB (16979655680 bytes) of device memory: : CUDA_ERROR_OUT_OF_MEMORY: out of memory\n", + "E0609 22:31:56.431550 1266583 cuda_executor.cc:1206] [0] Failed to allocate device memory: INTERNAL: [0] Failed to allocate 14.23GiB (15281689600 bytes) of device memory: : CUDA_ERROR_OUT_OF_MEMORY: out of memory\n", + "E0609 22:31:56.431976 1266583 cuda_executor.cc:1206] [0] Failed to allocate device memory: INTERNAL: [0] Failed to allocate 12.81GiB (13753520128 bytes) of device memory: : CUDA_ERROR_OUT_OF_MEMORY: out of memory\n", + "E0609 22:31:56.432403 1266583 cuda_executor.cc:1206] [0] Failed to allocate device memory: INTERNAL: [0] Failed to allocate 11.53GiB (12378167296 bytes) of device memory: : CUDA_ERROR_OUT_OF_MEMORY: out of memory\n", + "E0609 22:31:56.432828 1266583 cuda_executor.cc:1206] [0] Failed to allocate device memory: INTERNAL: [0] Failed to allocate 10.38GiB (11140349952 bytes) of device memory: : CUDA_ERROR_OUT_OF_MEMORY: out of memory\n", + "E0609 22:31:56.433252 1266583 cuda_executor.cc:1206] [0] Failed to allocate device memory: INTERNAL: [0] Failed to allocate 9.34GiB (10026314752 bytes) of device memory: : CUDA_ERROR_OUT_OF_MEMORY: out of memory\n" + ] + } + ], + "source": [ + "import jax.numpy as jnp\n", + "dispersion = jnp.array(dispersion)\n", + "flux = jnp.array(flux)\n", + "ivar = jnp.array(ivar)\n", + "continuum_pixels = jnp.array(continuum_pixels)\n", + "APOGEE_REGIONS = [jnp.array(r) for r in APOGEE_REGIONS]" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "140ea664", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-03T06:40:33.784657Z", + "iopub.status.busy": "2026-06-03T06:40:33.784521Z", + "iopub.status.idle": "2026-06-03T06:40:38.214899Z", + "shell.execute_reply": "2026-06-03T06:40:38.213697Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:Some pixels have measured flux values (e.g., ivar > 0) but are not included in any specified continuum region. These pixels won't be continuum-normalised (1 spectra affected).\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "77e1476d51d047e19a04eb3233bbc829", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Normalizing: 0%| | 0/3 [00:00 0\n", + "print(\"median raw flux on good pixels: \", np.median(flux[good]))\n", + "print(\"median normalized flux (should ~1):\", np.median(normalized_flux[good]))" + ] + }, + { + "cell_type": "markdown", + "id": "1c045dbd", + "metadata": {}, + "source": [ + "## Build, train, and test a CannonModel\n", + "\n", + "Split into training / validation sets (dropping rows with non-finite labels),\n", + "fit a quadratic model on the training set, then predict labels for the\n", + "validation set and compare against the ASPCAP values." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "90c1d14e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['raw_teff', 'raw_logg', 'raw_fe_h', 'mg_fe', 'ce_fe', 'log_age_L']" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(labels.columns)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "e1d7a25e", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-03T06:40:38.227071Z", + "iopub.status.busy": "2026-06-03T06:40:38.226484Z", + "iopub.status.idle": "2026-06-03T06:40:38.245186Z", + "shell.execute_reply": "2026-06-03T06:40:38.244704Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "967 training, 1006 validation stars (690 dropped for non-finite labels)\n" + ] + } + ], + "source": [ + "LABEL_NAMES = list(labels.columns)\n", + "\n", + "# Drop stars with any non-finite label -- NaNs would break the training fit.\n", + "finite = np.isfinite(labels[LABEL_NAMES].values).all(axis=1)\n", + "\n", + "# Reproducible split: ~10% train, ~10% validate (as in the old getting_started.py).\n", + "rng = np.random.RandomState(888)\n", + "q = rng.randint(0, 10, len(spectra))\n", + "train_set = finite & (q == 1)\n", + "validate_set = finite & (q == 0)\n", + "print(f\"{train_set.sum()} training, {validate_set.sum()} validation stars \"\n", + " f\"({(~finite).sum()} dropped for non-finite labels)\")" + ] + }, + { + "cell_type": "markdown", + "id": "le-cell-20", + "metadata": {}, + "source": [ + "### [label errors] Assemble the per-label uncertainties\n", + "\n", + "Build a `(n_stars, n_labels)` array of 1-sigma label errors, aligned to\n", + "`LABEL_NAMES`, that we will hand to the model below." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "le-cell-21", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [fallback] no error column for 'raw_teff'; using 30.0\n", + " [fallback] no error column for 'raw_logg'; using 0.05\n", + " [fallback] no error column for 'raw_fe_h'; using 0.02\n", + " [fallback] no error column for 'mg_fe'; using 0.03\n", + " [fallback] no error column for 'ce_fe'; using 0.05\n", + " [fallback] no error column for 'log_age_L'; using 0.1\n", + "label_err shape: (10633, 6)\n", + "median 1-sigma per label: {'raw_teff': np.float64(30.0), 'raw_logg': np.float64(0.05), 'raw_fe_h': np.float64(0.02), 'mg_fe': np.float64(0.03), 'ce_fe': np.float64(0.05), 'log_age_L': np.float64(0.1)}\n" + ] + } + ], + "source": [ + "# [label errors] Build per-label 1-sigma uncertainties aligned to LABEL_NAMES.\n", + "#\n", + "# Plug in your real uncertainty columns here. ASPCAP-style tables usually carry\n", + "# per-label error columns; derived labels propagate from their components, e.g.\n", + "# mg_fe = raw_mg_h - raw_fe_h -> sigma = sqrt(sigma_mg_h^2 + sigma_fe_h^2)\n", + "#\n", + "# We try a few common column-name conventions and fall back to representative\n", + "# 1-sigma floors when a column is missing, so the notebook runs end-to-end\n", + "# either way. Replace the fallbacks / mappings with whatever your table uses.\n", + "\n", + "LABEL_NAMES = list(labels.columns)\n", + "\n", + "# Representative 1-sigma floors (in each label's native units) used only when no\n", + "# error column is found. Tune these to your survey.\n", + "FALLBACK_SIGMA = {\n", + " \"raw_teff\": 30.0, # K\n", + " \"raw_logg\": 0.05, # dex\n", + " \"raw_fe_h\": 0.02, # dex\n", + " \"mg_fe\": 0.03, # dex\n", + " \"ce_fe\": 0.05, # dex\n", + " \"log_age_L\": 0.10, # dex (log10 age)\n", + "}\n", + "\n", + "def _find_err_column(df, label):\n", + " for cand in (f\"{label}_err\", f\"e_{label}\", f\"{label}_e\",\n", + " f\"err_{label}\", f\"sigma_{label}\"):\n", + " if cand in df.columns:\n", + " return cand\n", + " return None\n", + "\n", + "def label_sigma(df, label):\n", + " col = _find_err_column(df, label)\n", + " if col is not None:\n", + " return np.asarray(df[col], dtype=float)\n", + " # Error propagation for the derived labels, when the components are present.\n", + " if label == \"mg_fe\" and {\"raw_mg_h_err\", \"raw_fe_h_err\"} <= set(df.columns):\n", + " return np.hypot(df[\"raw_mg_h_err\"], df[\"raw_fe_h_err\"]).to_numpy()\n", + " if label == \"ce_fe\" and {\"raw_ce_h_err\", \"raw_fe_h_err\"} <= set(df.columns):\n", + " return np.hypot(df[\"raw_ce_h_err\"], df[\"raw_fe_h_err\"]).to_numpy()\n", + " if label == \"log_age_L\" and \"age_L_err\" in df.columns:\n", + " # d log10(age) = sigma_age / (age * ln 10)\n", + " return (df[\"age_L_err\"] / (df[\"age_L\"] * np.log(10.0))).to_numpy()\n", + " print(f\" [fallback] no error column for {label!r}; \"\n", + " f\"using {FALLBACK_SIGMA[label]}\")\n", + " return np.full(len(df), FALLBACK_SIGMA[label], dtype=float)\n", + "\n", + "label_err_full = np.column_stack([label_sigma(spectra, ln) for ln in LABEL_NAMES])\n", + "\n", + "# Errors must be finite and strictly positive; clamp anything else to the floor.\n", + "floor = np.array([FALLBACK_SIGMA[ln] for ln in LABEL_NAMES])\n", + "bad = ~np.isfinite(label_err_full) | (label_err_full <= 0)\n", + "label_err_full = np.where(bad, floor, label_err_full)\n", + "\n", + "print(\"label_err shape:\", label_err_full.shape)\n", + "print(\"median 1-sigma per label:\",\n", + " dict(zip(LABEL_NAMES, np.round(np.median(label_err_full, axis=0), 4))))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "le-cell-22", + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "CannonModel._verify_training_data() got an unexpected keyword argument 'training_set_label_err'", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mTypeError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[19]\u001b[39m\u001b[32m, line 9\u001b[39m\n\u001b[32m 2\u001b[39m vectorizer = tc.vectorizer.PolynomialVectorizer(label_names=LABEL_NAMES, order=\u001b[32m2\u001b[39m)\n\u001b[32m 4\u001b[39m \u001b[38;5;66;03m# [label errors] Pass `training_set_label_err`. With it set, `train()` runs the\u001b[39;00m\n\u001b[32m 5\u001b[39m \u001b[38;5;66;03m# errors-in-variables path: it propagates the label uncertainties into the\u001b[39;00m\n\u001b[32m 6\u001b[39m \u001b[38;5;66;03m# per-pixel weights and refines them with `n_irls` reweighting passes, so stars\u001b[39;00m\n\u001b[32m 7\u001b[39m \u001b[38;5;66;03m# with uncertain labels are down-weighted. Drop this argument (or pass None) for\u001b[39;00m\n\u001b[32m 8\u001b[39m \u001b[38;5;66;03m# the standard exact-label Cannon.\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m9\u001b[39m model = \u001b[43mtc\u001b[49m\u001b[43m.\u001b[49m\u001b[43mCannonModel\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 10\u001b[39m \u001b[43m \u001b[49m\u001b[43mlabels\u001b[49m\u001b[43m[\u001b[49m\u001b[43mtrain_set\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 11\u001b[39m \u001b[43m \u001b[49m\u001b[43mnormalized_flux\u001b[49m\u001b[43m[\u001b[49m\u001b[43mtrain_set\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 12\u001b[39m \u001b[43m \u001b[49m\u001b[43mnormalized_ivar\u001b[49m\u001b[43m[\u001b[49m\u001b[43mtrain_set\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 13\u001b[39m \u001b[43m \u001b[49m\u001b[43mvectorizer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 14\u001b[39m \u001b[43m \u001b[49m\u001b[43mdispersion\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdispersion\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 15\u001b[39m \u001b[43m \u001b[49m\u001b[43mregularization\u001b[49m\u001b[43m=\u001b[49m\u001b[32;43m1e-6\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 16\u001b[39m \u001b[43m \u001b[49m\u001b[43mtraining_set_label_err\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlabel_err_full\u001b[49m\u001b[43m[\u001b[49m\u001b[43mtrain_set\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# <-- the new bit\u001b[39;00m\n\u001b[32m 18\u001b[39m \u001b[38;5;28mprint\u001b[39m(model)\n\u001b[32m 19\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33mlabel_err set:\u001b[39m\u001b[33m\"\u001b[39m, model.training_set_label_err \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m 20\u001b[39m \u001b[33m\"\u001b[39m\u001b[33m| shape:\u001b[39m\u001b[33m\"\u001b[39m, np.asarray(model.training_set_label_err).shape)\n", + "\u001b[36mFile \u001b[39m\u001b[32m/scratch/y89/mj8805/miniforge/envs/astro/lib/python3.11/site-packages/thecannon/model.py:126\u001b[39m, in \u001b[36mCannonModel.__init__\u001b[39m\u001b[34m(self, training_set_labels, training_set_flux, training_set_ivar, vectorizer, dispersion, regularization, censors, **kwargs)\u001b[39m\n\u001b[32m 122\u001b[39m \u001b[38;5;28mself\u001b[39m._training_set_labels = np.array(\n\u001b[32m 123\u001b[39m [training_set_labels[ln] \u001b[38;5;28;01mfor\u001b[39;00m ln \u001b[38;5;129;01min\u001b[39;00m vectorizer.label_names]).T\n\u001b[32m 125\u001b[39m \u001b[38;5;66;03m# Check that the flux and ivar are valid.\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m126\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_verify_training_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 128\u001b[39m \u001b[38;5;66;03m# Set regularization, censoring, dispersion.\u001b[39;00m\n\u001b[32m 129\u001b[39m \u001b[38;5;28mself\u001b[39m.regularization = regularization\n", + "\u001b[31mTypeError\u001b[39m: CannonModel._verify_training_data() got an unexpected keyword argument 'training_set_label_err'" + ] + } + ], + "source": [ + "# Quadratic (order=2) polynomial vectorizer in the labels.\n", + "vectorizer = tc.vectorizer.PolynomialVectorizer(label_names=LABEL_NAMES, order=2)\n", + "\n", + "# [label errors] Pass `training_set_label_err`. With it set, `train()` runs the\n", + "# errors-in-variables path: it propagates the label uncertainties into the\n", + "# per-pixel weights and refines them with `n_irls` reweighting passes, so stars\n", + "# with uncertain labels are down-weighted. Drop this argument (or pass None) for\n", + "# the standard exact-label Cannon.\n", + "model = tc.CannonModel(\n", + " labels[train_set],\n", + " normalized_flux[train_set],\n", + " normalized_ivar[train_set],\n", + " vectorizer,\n", + " dispersion=dispersion,\n", + " regularization=1e-6,\n", + " training_set_label_err=label_err_full[train_set]) # <-- the new bit\n", + "\n", + "print(model)\n", + "print(\"label_err set:\", model.training_set_label_err is not None,\n", + " \"| shape:\", np.asarray(model.training_set_label_err).shape)\n", + "\n", + "# Train (fits theta + scatter s2 at every pixel). `n_irls` (default 5) controls\n", + "# how many label-error reweighting passes are used; it is ignored when no label\n", + "# errors are set.\n", + "theta, s2, train_meta = model.train(n_irls=5)\n", + "print(\"trained:\", model.is_trained, \"| theta shape:\", np.asarray(theta).shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f700c182", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-03T06:47:24.785834Z", + "iopub.status.busy": "2026-06-03T06:47:24.785565Z", + "iopub.status.idle": "2026-06-03T06:47:37.915679Z", + "shell.execute_reply": "2026-06-03T06:47:37.914773Z" + } + }, + "outputs": [], + "source": [ + "# Predict labels for the validation set (the \"test\" step). Returns\n", + "# (labels, covariance, metadata). Note: it's .test(), not .fit().\n", + "val_labels, val_cov, val_meta = model.test(\n", + " normalized_flux[validate_set], normalized_ivar[validate_set])\n", + "val_labels = np.asarray(val_labels)\n", + "truth = labels[validate_set].values\n", + "print(\"predicted label array shape:\", val_labels.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1df0f71", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-03T06:47:37.919256Z", + "iopub.status.busy": "2026-06-03T06:47:37.918967Z", + "iopub.status.idle": "2026-06-03T06:47:38.369846Z", + "shell.execute_reply": "2026-06-03T06:47:38.369431Z" + } + }, + "outputs": [], + "source": [ + "# One-to-one comparison of Cannon-predicted vs ASPCAP labels on the validation set.\n", + "fig, axes = plt.subplots(3, 2, figsize=(10, 10))\n", + "for i, (name, ax) in enumerate(zip(LABEL_NAMES, axes.ravel())):\n", + " x, y = truth[:, i], val_labels[:, i]\n", + " ax.scatter(x, y, facecolor=\"k\", s=10)\n", + " lims = [min(x.min(), y.min()), max(x.max(), y.max())]\n", + " ax.plot(lims, lims, \"-\", color=\"r\", lw=1) # 1:1 line\n", + " ax.set_xlim(lims); ax.set_ylim(lims)\n", + " d = y - x\n", + " ax.set_xlabel(f\"{name} (ASPCAP)\")\n", + " ax.set_ylabel(f\"{name} (Cannon)\")\n", + " ax.set_title(f\"{name}: bias={np.nanmean(d):+.3f}, scatter={np.nanstd(d):.3f}\")\n", + "fig.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b60134e", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "le-cell-26", + "metadata": {}, + "source": [ + "### [label errors] How much do the label errors change the fit?\n", + "\n", + "Train an exact-label model on the same data (no `training_set_label_err`) and\n", + "compare the validation scatter per label. This retrains from scratch, so it\n", + "roughly doubles the runtime -- skip it if you only wanted the API." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "le-cell-27", + "metadata": {}, + "outputs": [], + "source": [ + "# Exact-label baseline (same data, no label_err) for comparison.\n", + "model_noerr = tc.CannonModel(\n", + " labels[train_set],\n", + " normalized_flux[train_set],\n", + " normalized_ivar[train_set],\n", + " vectorizer,\n", + " dispersion=dispersion,\n", + " regularization=1e-6)\n", + "model_noerr.train()\n", + "\n", + "val_noerr, _, _ = model_noerr.test(\n", + " normalized_flux[validate_set], normalized_ivar[validate_set])\n", + "val_noerr = np.asarray(val_noerr)\n", + "\n", + "print(f\"{'label':<12} {'scatter (exact)':>16} {'scatter (EIV)':>16}\")\n", + "for i, name in enumerate(LABEL_NAMES):\n", + " s_exact = np.nanstd(val_noerr[:, i] - truth[:, i])\n", + " s_eiv = np.nanstd(val_labels[:, i] - truth[:, i])\n", + " print(f\"{name:<12} {s_exact:>16.4f} {s_eiv:>16.4f}\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "astro", + "language": "python", + "name": "astro" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/thecannon/fitting.py b/thecannon/fitting.py index 826ded6..226afd0 100644 --- a/thecannon/fitting.py +++ b/thecannon/fitting.py @@ -397,6 +397,185 @@ def fit(flux, ivar, design_matrix, column_mask): return fit +# --------------------------------------------------------------------------- # +# Errors-in-variables (uncertain labels) per-pixel fits # +# --------------------------------------------------------------------------- # +# +# The standard Cannon treats the training-set labels (the features that build +# the design matrix) as exact. When the labels carry uncertainty, propagate it +# to first order through the model: for star ``s`` and pixel coefficients +# ``theta``, the model prediction ``m = theta . v(x_s)`` has variance +# +# Var(m_s) ~= (theta . J_v[s])^T Sigma_s (theta . J_v[s]) +# +# where ``J_v[s] = d v / d x`` is the vectorizer Jacobian at star ``s`` and +# ``Sigma_s`` is the (scaled) label covariance. With diagonal label errors this +# is ``sum_l (theta . J_v[s])_l^2 * var_label[s, l]``. The propagated variance +# is folded into the per-star weight exactly as the scatter term is, via +# ``ivar_eff = ivar / (1 + ivar * var_label)``. Because the weight then depends +# on ``theta``, the weighted least-squares solve is iterated to a fixed point +# (iteratively reweighted least squares); a handful of iterations suffices. + + +def _label_variance_term(theta, label_jac, label_var): + """ + First-order propagated model variance from diagonal label errors. + + :param theta: + The pixel coefficients, shape ``(T,)``. + + :param label_jac: + The vectorizer Jacobian ``d v / d x`` at every star, shape + ``(S, T, L)`` (in the scaled label space the design matrix uses). + + :param label_var: + The per-star variance of the scaled labels, shape ``(S, L)``. + + :returns: + The propagated model variance for every star, shape ``(S,)``. + """ + g = jnp.einsum("t,stl->sl", theta, label_jac) # d(theta . v)/dx per star + return jnp.sum(g * g * label_var, axis=1) + + +def make_pixel_closed_form_eiv(label_jac, label_var, n_irls=5): + """ + Closed-form per-pixel fit that accounts for diagonal label uncertainties via + iteratively reweighted least squares. Same conventions and return signature + as :func:`make_pixel_closed_form`; ``label_jac`` (S, T, L) and ``label_var`` + (S, L) are shared across pixels and closed over. + """ + + def fit(flux, ivar, design_matrix, column_mask): + + T = design_matrix.shape[1] + mask = column_mask.astype(design_matrix.dtype) + no_info = jnp.sum(ivar) < ivar.size + dm = design_matrix * mask[None, :] + fiducial = jnp.concatenate([jnp.ones(1), jnp.zeros(T - 1)]) + + def solve(w): + # Weighted normal equations with per-star weights ``w``. The unit on + # the censored diagonal keeps the Gram matrix non-singular and pins + # those coefficients to exactly zero (see make_pixel_closed_form). + CiA = dm * w[:, None] + gram = jnp.dot(dm.T, CiA) + jnp.diag(1.0 - mask) + rhs = jnp.dot(dm.T, flux * w) + return jnp.linalg.solve(gram, rhs) * mask + + def reweight(theta): + v_label = _label_variance_term(theta, label_jac, label_var) + return ivar / (1.0 + ivar * v_label) + + def step(_, theta): + return solve(reweight(theta)) + + # Start from the exact-label solution, then reweight to a fixed point. + theta = lax.fori_loop(0, n_irls, step, solve(ivar)) + + ok = jnp.all(jnp.isfinite(theta)) + theta = jnp.where(ok, theta, fiducial) + + w = reweight(theta) + residuals_squared = (flux - jnp.dot(theta, dm.T)) ** 2 + s2 = _fit_scatter(residuals_squared, w) + fopt = _chi_sq_only(theta, dm, flux, w) + + theta = jnp.where(no_info, fiducial, theta) + s2 = jnp.where(no_info, jnp.inf, s2) + fopt = jnp.where(no_info, jnp.nan, fopt) + + return (theta, s2, fopt) + + return fit + + +def make_pixel_fitter_eiv(label_jac, label_var, n_irls=5, op_method="l_bfgs_b", + maxiter=_TRAIN_MAXITER, tol=_TRAIN_TOL, bounds=None): + """ + Regularized/bounded per-pixel fit that accounts for diagonal label + uncertainties. Same conventions and return signature as the optimizer built + by :func:`make_pixel_fitter`, wrapped in an outer iteratively-reweighted + loop: each iteration fixes the per-star weights from the current ``theta`` + and re-solves the (regularized) weighted problem. ``label_jac`` (S, T, L) + and ``label_var`` (S, L) are shared across pixels and closed over. + """ + + op_method = (op_method or "l_bfgs_b").lower() + if op_method == "powell": + op_method = "l_bfgs_b" + if op_method not in ("l_bfgs_b", "proximal"): + raise ValueError("unknown optimization method '{}' -- 'l_bfgs_b' or " + "'proximal' are available".format(op_method)) + if bounds is not None: + lower, upper = (jnp.asarray(bounds[0]), jnp.asarray(bounds[1])) + + def fit(flux, ivar, init_stack, design_matrix, regularization, column_mask): + + T = design_matrix.shape[1] + mask = column_mask.astype(design_matrix.dtype) + no_info = jnp.sum(ivar) < ivar.size + dm = design_matrix * mask[None, :] + fiducial = jnp.concatenate([jnp.ones(1), jnp.zeros(T - 1)]) + + def objective(theta, w): + return _chi_sq_only(theta, dm, flux, w) \ + + regularization * jnp.sum(jnp.abs(theta[1:])) + + # Weighted solve, warm-started from ``start``, with per-star weights ``w``. + if bounds is not None: + lower_eff = jnp.where(column_mask, lower, -jnp.inf) + upper_eff = jnp.where(column_mask, upper, jnp.inf) + + def solve(start, w): + s = jnp.clip(start, lower_eff, upper_eff) + solver = LBFGSB(fun=lambda th: objective(th, w), + maxiter=maxiter, tol=tol) + return solver.run(s, bounds=(lower_eff, upper_eff)).params + elif op_method == "proximal": + def solve(start, w): + l1reg = jnp.full((T,), regularization).at[0].set(0.0) * mask + solver = ProximalGradient( + fun=lambda th: _chi_sq_only(th, dm, flux, w), + prox=prox_lasso, maxiter=maxiter, tol=tol) + return solver.run(start, l1reg).params + else: + def solve(start, w): + solver = LBFGS(fun=lambda th: objective(th, w), + maxiter=maxiter, tol=tol) + return solver.run(start).params + + def reweight(theta): + v_label = _label_variance_term(theta, label_jac, label_var) + return ivar / (1.0 + ivar * v_label) + + # Choose the best starting point against the raw-ivar objective. + feval = jax.vmap(lambda th: objective(th, ivar))(init_stack) + feval = jnp.where(jnp.isnan(feval), jnp.inf, feval) + best_init = init_stack[jnp.argmin(feval)] + + # First solve with raw weights, then reweight to a fixed point. + theta0 = solve(best_init, ivar) + + def step(_, theta): + return solve(theta, reweight(theta)) + + theta = lax.fori_loop(0, n_irls, step, theta0) * mask + + w = reweight(theta) + residuals_squared = (flux - jnp.dot(theta, dm.T)) ** 2 + s2 = _fit_scatter(residuals_squared, w) + fopt = objective(theta, w) + + theta = jnp.where(no_info, fiducial, theta) + s2 = jnp.where(no_info, jnp.inf, s2) + fopt = jnp.where(no_info, jnp.nan, fopt) + + return (theta, s2, fopt) + + return fit + + def fit_pixel_fixed_scatter(flux, ivar, initial_thetas, design_matrix, regularization, censoring_mask, **kwargs): """ diff --git a/thecannon/model.py b/thecannon/model.py index 33fae1b..d1d28cb 100644 --- a/thecannon/model.py +++ b/thecannon/model.py @@ -30,6 +30,16 @@ logger = logging.getLogger(__name__) +# Target memory budget (bytes) for the per-batch working set when ``train`` is +# left to pick its own ``batch_size``. The default batch size is shrunk so that +# the order-sensitive per-pixel temporaries (the ``(N, T)`` design-matrix +# products and the ``(T, T)`` Gram/Hessian) stay within roughly this budget, +# which keeps higher polynomial orders from becoming memory-hungry without the +# caller having to hand-tune ``batch_size``. Override on the class/module if a +# machine has more or less headroom. +_TRAIN_BATCH_MEM_BYTES = 1 << 30 # 1 GiB + + def requires_training(method): """ A decorator for model methods that require training before being run. @@ -82,7 +92,8 @@ class CannonModel(object): """ _data_attributes = \ - ("training_set_labels", "training_set_flux", "training_set_ivar") + ("training_set_labels", "training_set_flux", "training_set_ivar", + "training_set_label_err") # Descriptive attributes are needed to train *and* test the model. _descriptive_attributes = \ @@ -92,7 +103,8 @@ class CannonModel(object): _trained_attributes = ("theta", "s2") def __init__(self, training_set_labels, training_set_flux, training_set_ivar, - vectorizer, dispersion=None, regularization=None, censors=None, **kwargs): + vectorizer, dispersion=None, regularization=None, censors=None, + training_set_label_err=None, **kwargs): # Save the vectorizer. if not isinstance(vectorizer, BaseVectorizer): @@ -125,6 +137,26 @@ def __init__(self, training_set_labels, training_set_flux, training_set_ivar, # Check that the flux and ivar are valid. self._verify_training_data(**kwargs) + # Optional 1-sigma uncertainties on the training-set labels (the + # "feature" uncertainties). Stored in the same (num_stars, num_labels) + # layout as the resolved label array; ``None`` recovers the standard + # exact-label Cannon. Used by ``train`` to propagate label errors into + # the per-pixel weights (errors-in-variables). + if training_set_label_err is None or self._training_set_labels is None: + self._training_set_label_err = None + else: + label_err = np.atleast_2d( + np.asarray(training_set_label_err, dtype=float)) + if label_err.shape != self._training_set_labels.shape: + raise ValueError( + "training_set_label_err shape {0} does not match the " + "training set labels shape {1}".format( + label_err.shape, self._training_set_labels.shape)) + if not np.all(np.isfinite(label_err)) or np.any(label_err < 0): + raise ValueError( + "training_set_label_err must be finite and non-negative") + self._training_set_label_err = label_err + # Set regularization, censoring, dispersion. self.regularization = regularization self.censors = censors @@ -186,6 +218,12 @@ def training_set_ivar(self): return self._training_set_ivar + @property + def training_set_label_err(self): + """ Return the 1-sigma uncertainties on the training set labels. """ + return self._training_set_label_err + + @property def vectorizer(self): """ Return the vectorizer for this model. """ @@ -595,7 +633,7 @@ def read(cls, path, **kwargs): def train(self, threads=None, op_method=None, op_strict=True, op_kwds=None, - progressbar=True, batch_size=None, **kwargs): + progressbar=True, batch_size=None, n_irls=5, **kwargs): """ Train the model. @@ -623,6 +661,13 @@ def train(self, threads=None, op_method=None, op_strict=True, op_kwds=None, compilation cost of the vmapped optimizer. Defaults to a value that yields ~50 progress updates with reasonably large batches. + :param n_irls: [optional] + The number of iteratively-reweighted least-squares iterations used + when the model carries label uncertainties + (``training_set_label_err``). Each iteration refines the per-star + weights from the propagated label variance. Ignored when no label + uncertainties are set. + :returns: A three-length tuple containing the spectral coefficients `theta`, the squared scatter term at each pixel `s2`, and metadata related to @@ -688,8 +733,32 @@ def train(self, threads=None, op_method=None, op_strict=True, op_kwds=None, or (np.ndim(self.regularization) == 0 and float(self.regularization) == 0.0)) + # Optional errors-in-variables: when the training labels carry + # uncertainties, propagate them into the per-pixel weights. The label + # Jacobian d v / d x and the (scaled) label variances depend only on the + # labels -- not on the pixel -- so they are computed once and shared + # across all pixels (closed over by the fitter, not vmapped). + eiv = self.training_set_label_err is not None + if eiv: + scaled_labels = jnp.asarray( + (np.asarray(self.training_set_labels, dtype=float) + - self._fiducials) / self._scales) # (S, L) + # d(design-matrix row)/d(scaled label) at every star: (S, T, L). + label_jac = jax.vmap( + self.vectorizer.get_label_vector_derivative)(scaled_labels) + # Variance of the scaled labels (sigma in scaled space = sigma/scale). + label_var = jnp.asarray( + (np.asarray(self.training_set_label_err, dtype=float) + / self._scales) ** 2) # (S, L) + logger.info("Propagating label uncertainties via {0} IRLS " + "iteration(s)".format(n_irls)) + if closed_form: - fitter = fitting.make_pixel_closed_form() + if eiv: + fitter = fitting.make_pixel_closed_form_eiv( + label_jac, label_var, n_irls=n_irls) + else: + fitter = fitting.make_pixel_closed_form() else: # Initial theta guesses for every pixel: a linear-algebra estimate # and the fiducial value. The regularized objective is convex, so @@ -709,15 +778,41 @@ def _initial_stack(flux_PN, ivar_PN): init_stack = _initial_stack(flux_PN, ivar_PN) # (P, 2, T) - fitter = fitting.make_pixel_fitter( - op_method=op_method, maxiter=maxiter, tol=tol, bounds=bounds) + if eiv: + fitter = fitting.make_pixel_fitter_eiv( + label_jac, label_var, n_irls=n_irls, op_method=op_method, + maxiter=maxiter, tol=tol, bounds=bounds) + else: + fitter = fitting.make_pixel_fitter( + op_method=op_method, maxiter=maxiter, tol=tol, bounds=bounds) # Pixels are fit independently, so we run them in fixed-size batches # (vmap within a batch, lax.scan across batches) rather than one giant # vmap. The scan drives a per-batch jax-tqdm progress bar, bounds peak # memory, and keeps the compiled body small (compiled once, reused). if batch_size is None: + # Granularity default: ~50 progress updates with reasonably large + # batches. batch_size = max(512, int(np.ceil(P / 50))) + # ...but cap it so the order-sensitive per-pixel temporaries fit the + # memory budget. Under the vmap each pixel allocates roughly the + # (N, T) design-matrix products and the (T, T) Gram/Hessian, so the + # per-batch working set scales as batch_size * (N*T + T**2). Both N*T + # and T**2 grow with the polynomial order (via T), so this is what + # makes higher orders shrink the batch automatically. + per_pixel_bytes = 8 * (2 * S * T + 2 * T * T) + if eiv: + # Each pixel also forms the (S, L) label-gradient term per IRLS + # pass; account for it so label errors at high order still fit. + Ln = self.training_set_labels.shape[1] + per_pixel_bytes += 8 * (S * Ln + S * T) + cap = max(1, int(_TRAIN_BATCH_MEM_BYTES // per_pixel_bytes)) + if cap < batch_size: + logger.info("Capping batch_size {0} -> {1} to keep the per-batch " + "working set within ~{2:.0f} MiB (N={3}, T={4})"\ + .format(batch_size, cap, _TRAIN_BATCH_MEM_BYTES / (1 << 20), + S, T)) + batch_size = cap batch_size = int(min(max(1, batch_size), P)) # Pad the pixel axis up to a whole number of equally-sized batches so diff --git a/thecannon/tests/test_label_errors.py b/thecannon/tests/test_label_errors.py new file mode 100644 index 0000000..a8ba2be --- /dev/null +++ b/thecannon/tests/test_label_errors.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Tests for training with label (feature) uncertainties. + +The standard Cannon treats the training-set labels as exact. When +``training_set_label_err`` is supplied, ``train`` propagates the label +uncertainties into the per-pixel weights and refines them with iteratively +reweighted least squares (errors-in-variables). These tests check that: + + * supplying ``None`` reproduces the exact-label fit bit-for-bit (no regression); + * large per-star label errors down-weight the stars whose labels we do not + trust, recovering coefficients a naive exact-label fit gets wrong; + * the regularized path also runs and stays finite; + * label errors survive a save/read round-trip; + * construction validates the error array's shape and values. +""" + +from __future__ import (division, print_function, absolute_import, + unicode_literals) + +import os +import tempfile + +import numpy as np +import pytest + +import thecannon as tc +from thecannon.vectorizer.polynomial import PolynomialVectorizer + + +def _vec(order=2): + return PolynomialVectorizer(label_names=["a", "b", "c"], order=order) + + +def _toy(seed=1, S=60, P=40, L=3): + rng = np.random.RandomState(seed) + labels = rng.normal(size=(S, L)) + flux = 1.0 + 0.02 * rng.normal(size=(S, P)) + ivar = np.full((S, P), 1e4) + return labels, flux, ivar + + +# --------------------------------------------------------------------------- # +# No regression: label_err=None is identical to the exact-label fit # +# --------------------------------------------------------------------------- # + +def test_none_matches_exact_label_fit(): + labels, flux, ivar = _toy() + + base = tc.CannonModel(labels, flux, ivar, _vec()) + t0, s0, _ = base.train(progressbar=False) + + same = tc.CannonModel(labels, flux, ivar, _vec(), + training_set_label_err=None) + t1, s1, _ = same.train(progressbar=False) + + assert np.allclose(np.asarray(t0), np.asarray(t1)) + assert np.allclose(np.asarray(s0), np.asarray(s1)) + + +# --------------------------------------------------------------------------- # +# EIV runs, stays finite, and changes the answer # +# --------------------------------------------------------------------------- # + +def test_eiv_closed_form_runs_and_differs(): + labels, flux, ivar = _toy() + lerr = np.full_like(labels, 0.1) + + exact = tc.CannonModel(labels, flux, ivar, _vec()) + t_exact = np.asarray(exact.train(progressbar=False)[0]) + + eiv = tc.CannonModel(labels, flux, ivar, _vec(), + training_set_label_err=lerr) + theta, s2, _ = eiv.train(progressbar=False) + theta, s2 = np.asarray(theta), np.asarray(s2) + + assert np.all(np.isfinite(theta)) + assert np.all(s2 >= 0.0) + assert not np.allclose(theta, t_exact) + + +def test_eiv_regularized_runs(): + labels, flux, ivar = _toy() + lerr = np.full_like(labels, 0.1) + eiv = tc.CannonModel(labels, flux, ivar, _vec(), regularization=10.0, + training_set_label_err=lerr) + theta, s2, _ = eiv.train(progressbar=False, n_irls=3) + assert np.all(np.isfinite(np.asarray(theta))) + assert np.all(np.asarray(s2) >= 0.0) + + +# --------------------------------------------------------------------------- # +# Mechanism: untrusted labels are down-weighted # +# --------------------------------------------------------------------------- # + +def test_large_label_errors_downweight_untrusted_stars(): + rng = np.random.RandomState(3) + S, L = 200, 2 + x_true = rng.normal(size=(S, L)) + theta_true = np.array([0.5, -0.3, 0.8]) # [pivot, a, b], order-1 + flux = (theta_true[0] + x_true @ theta_true[1:]).reshape(S, 1) + ivar = np.full((S, 1), 1e6) + + # Corrupt the recorded labels of some stars and flag them as untrusted. + x_obs = x_true.copy() + bad = rng.choice(S, 30, replace=False) + x_obs[bad] += rng.normal(scale=5.0, size=(30, L)) + lerr = np.full((S, L), 1e-3) + lerr[bad] = 50.0 + + def predict(model, theta): + scaled = (x_true - model._fiducials) / model._scales + dm = np.asarray(model.vectorizer(scaled).T) + return dm @ np.asarray(theta) + + vec = lambda: PolynomialVectorizer(label_names=["a", "b"], order=1) + naive = tc.CannonModel(x_obs, flux, ivar, vec()) + t_naive = np.asarray(naive.train(progressbar=False)[0])[0] + + eiv = tc.CannonModel(x_obs, flux, ivar, vec(), + training_set_label_err=lerr) + t_eiv = np.asarray(eiv.train(progressbar=False, n_irls=8)[0])[0] + + rms_naive = np.sqrt(np.mean((predict(naive, t_naive) - flux[:, 0]) ** 2)) + rms_eiv = np.sqrt(np.mean((predict(eiv, t_eiv) - flux[:, 0]) ** 2)) + + # The naive fit trusts the corrupted labels and is biased; EIV all but + # eliminates the error by down-weighting the flagged stars. + assert rms_eiv < rms_naive / 5.0 + + +# --------------------------------------------------------------------------- # +# Serialization and validation # +# --------------------------------------------------------------------------- # + +def test_label_err_survives_round_trip(): + labels, flux, ivar = _toy() + lerr = np.abs(np.random.RandomState(7).normal(size=labels.shape)) + 0.01 + model = tc.CannonModel(labels, flux, ivar, _vec(), + training_set_label_err=lerr) + model.train(progressbar=False) + + path = os.path.join(tempfile.mkdtemp(), "model.model") + model.write(path, overwrite=True) + reloaded = tc.CannonModel.read(path) + + assert reloaded.training_set_label_err is not None + assert np.allclose(np.asarray(reloaded.training_set_label_err), lerr) + + +def test_label_err_validates_shape(): + labels, flux, ivar = _toy() + with pytest.raises(ValueError): + tc.CannonModel(labels, flux, ivar, _vec(), + training_set_label_err=np.ones((labels.shape[0], 99))) + + +def test_label_err_rejects_negative(): + labels, flux, ivar = _toy() + bad = np.ones_like(labels) + bad[0, 0] = -1.0 + with pytest.raises(ValueError): + tc.CannonModel(labels, flux, ivar, _vec(), + training_set_label_err=bad) From 6117e7af06281d9cac2a3a21cf4c8a239523b353 Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Tue, 9 Jun 2026 22:55:57 +1000 Subject: [PATCH 11/17] Add experiment_cannon: sweep cutoff/filter/order/reg, reconstruct + report chi2 A driver that adds the data knobs (S/N cutoff and arbitrary pandas-query row filters) on top of the existing label-set/order/regularization sweep, and for each grid point trains, validates, then: - reports the recovered spread (per-label bias/scatter/RMSE + mean_scatter) and writes a one-to-one spread figure; - reconstructs the validation spectra by forward-modelling the recovered (and reference) labels through the trained model; - plots a few observed-vs-reconstructed spectra spanning the chi2 range; - reports reconstruction reduced chi2 at both recovered and reference labels alongside the test-step chi2. Results go to experiment_results.csv plus per-config figures. Reuses the load/normalize/split helpers and spread_figure; optional --label-err trains with per-label uncertainties. Includes a self-contained --demo. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/experiment_cannon.py | 560 +++++++++++++++++++++++++++++++++++ 1 file changed, 560 insertions(+) create mode 100644 scripts/experiment_cannon.py diff --git a/scripts/experiment_cannon.py b/scripts/experiment_cannon.py new file mode 100644 index 0000000..fd750d6 --- /dev/null +++ b/scripts/experiment_cannon.py @@ -0,0 +1,560 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Run a grid of Cannon "experiments" and report what was recovered. + +Where :mod:`scripts.sweep_cannon` cross-validates over the *model* knobs (label +set, polynomial order, regularization), this driver adds the two *data* knobs -- +an S/N cutoff and arbitrary row filters on the table columns -- and, for every +grid point, also: + + * reports the recovered spread (per-label bias / scatter / RMSE) on a held-out + validation set, + * reconstructs the validation spectra by forward-modelling the recovered + labels through the trained model, + * plots a few observed-vs-reconstructed spectra, and + * reports the reconstruction chi-squared (reduced, at both the recovered and + the reference labels). + +Each grid point is therefore one experiment over + + (filter) x (snr_cutoff) x (label_set) x (order) x (regularization) + +and writes a row to a tidy CSV plus a one-to-one spread figure and a +reconstruction figure. + +JAX device selection is via the ``JAX_PLATFORMS`` environment variable. + +Usage +----- +On the real data:: + + python -m scripts.experiment_cannon \\ + --spectra /path/cleaned_ages.parquet \\ + --continuum-list /path/continuum.list \\ + --labels raw_teff,raw_logg,raw_fe_h,mg_fe,ce_fe,log_age_L \\ + --snr-cutoffs 100,150,200 \\ + --filter "EvoState == 2" \\ + --filter "EvoState in [1, 2]" \\ + --orders 1,2 --regularizations 0,1e-6,1e2 \\ + --output-dir results/ + +Quick self-contained smoke test (no data files, runs in seconds):: + + JAX_PLATFORMS=cpu python -m scripts.experiment_cannon --demo +""" + +from __future__ import (division, print_function, absolute_import, + unicode_literals) + +import argparse +import logging +import os +import re + +import numpy as np + +import matplotlib +matplotlib.use("Agg") # headless: only saves figures +import matplotlib.pyplot as plt + +import thecannon as tc + +# Work both as a package module (`python -m scripts.experiment_cannon`) and when +# run directly from the scripts/ directory. +try: + from scripts.train_cannon import (load_spectra, normalize_spectra, + make_split, DEFAULT_DATA_DIR, + DEFAULT_LABELS) + from scripts.sweep_cannon import spread_figure +except ImportError: + from train_cannon import (load_spectra, normalize_spectra, make_split, + DEFAULT_DATA_DIR, DEFAULT_LABELS) + from sweep_cannon import spread_figure + +logger = logging.getLogger("thecannon.experiment") + + +# --------------------------------------------------------------------------- # +# Row filtering (the data knobs: S/N cutoff + arbitrary column queries) # +# --------------------------------------------------------------------------- # + +def filter_mask(table, snr_column=None, snr_cutoff=None, query=None): + """ + Boolean mask over the rows of ``table`` (a pandas DataFrame) selecting stars + that pass an optional ``snr_column > snr_cutoff`` cut *and* an optional + pandas ``query`` expression (e.g. ``"EvoState == 2"``). Either may be + ``None`` to skip it. + """ + n = len(table) + mask = np.ones(n, dtype=bool) + + if snr_cutoff is not None: + if snr_column not in getattr(table, "columns", []): + raise KeyError("snr column {0!r} not in table; pass --snr-column" + .format(snr_column)) + mask &= np.asarray(table[snr_column], dtype=float) > float(snr_cutoff) + + if query: + # table.query returns the surviving rows; map back to a positional mask. + kept_index = table.query(query).index + mask &= np.asarray(table.index.isin(kept_index)) + + return mask + + +def _tag(s): + """ Filesystem-safe short tag from an arbitrary string. """ + return re.sub(r"[^0-9A-Za-z]+", "_", str(s)).strip("_") or "none" + + +# --------------------------------------------------------------------------- # +# Reconstruction + chi-squared # +# --------------------------------------------------------------------------- # + +def reconstruct(model, label_array): + """ Forward-model labels -> (N, P) flux through the trained model. """ + return np.atleast_2d(np.asarray(model(label_array))) + + +def reconstruction_chisq(flux, ivar, model_flux, n_labels): + """ + Per-star chi-squared between observed and reconstructed flux, using only the + measured pixels (``ivar > 0``). Returns ``(chi2, reduced_chi2, n_good)``; + the reduced value uses ``dof = n_good - n_labels - 1`` (matching the test + step's convention). + """ + flux = np.atleast_2d(flux) + ivar = np.atleast_2d(ivar) + model_flux = np.atleast_2d(model_flux) + + good = ivar > 0 + resid2 = np.where(good, (flux - model_flux) ** 2 * ivar, 0.0) + chi2 = resid2.sum(axis=1) + n_good = good.sum(axis=1) + dof = np.maximum(1, n_good - n_labels - 1) + return chi2, chi2 / dof, n_good + + +# --------------------------------------------------------------------------- # +# Figures # +# --------------------------------------------------------------------------- # + +def _default_window(dispersion): + """ A ~120-pixel window near the middle of the grid, for legible plots. """ + P = len(dispersion) + lo = int(0.45 * P) + hi = min(P - 1, lo + 120) + return float(dispersion[lo]), float(dispersion[hi]) + + +def spectra_figure(dispersion, flux, ivar, recovered_flux, reference_flux, + star_indices, red_chi_sq, wmin, wmax, title=None): + """ + Observed vs reconstructed spectra for a handful of validation stars, one row + per star, zoomed to ``[wmin, wmax]``. The observed flux is drawn with its + 1-sigma band; the recovered-label reconstruction is overplotted (and the + reference-label reconstruction, if given, as a dotted line). + """ + dispersion = np.asarray(dispersion) + win = (dispersion >= wmin) & (dispersion <= wmax) + if not win.any(): + win = np.ones(len(dispersion), dtype=bool) + + k = len(star_indices) + fig, axes = plt.subplots(k, 1, figsize=(10, 2.1 * k), squeeze=False, + sharex=True) + axes = axes.ravel() + + for ax, s in zip(axes, star_indices): + f = np.asarray(flux[s])[win] + iv = np.asarray(ivar[s])[win] + sigma = np.where(iv > 0, 1.0 / np.sqrt(np.where(iv > 0, iv, 1.0)), np.nan) + wl = dispersion[win] + + ax.fill_between(wl, f - sigma, f + sigma, color="0.8", step="mid", + label="observed 1$\\sigma$") + ax.step(wl, f, where="mid", color="k", lw=0.8, label="observed") + ax.plot(wl, np.asarray(recovered_flux[s])[win], color="C3", lw=1.0, + label="model (recovered labels)") + if reference_flux is not None: + ax.plot(wl, np.asarray(reference_flux[s])[win], color="C0", lw=1.0, + ls=":", label="model (reference labels)") + ax.set_ylabel("flux") + ax.text(0.01, 0.04, "star {0}: reduced $\\chi^2$={1:.2f}".format( + s, red_chi_sq[s]), transform=ax.transAxes, fontsize=8, + va="bottom", ha="left") + + axes[0].legend(fontsize=7, ncol=4, loc="upper right") + axes[-1].set_xlabel("wavelength") + if title: + fig.suptitle(title, fontsize=10) + fig.tight_layout() + return fig + + +# --------------------------------------------------------------------------- # +# One experiment (one grid point) # +# --------------------------------------------------------------------------- # + +def run_one(table, dispersion, norm_flux, norm_ivar, label_names, order, reg, + base_mask, train_frac, validate_frac, seed, output_dir, tag, + n_plot=3, wmin=None, wmax=None, label_err=None): + """ + Train + validate one (filter, cutoff, label_set, order, reg) combination, + then reconstruct the validation spectra and write the spread + reconstruction + figures. Returns a tidy metric row. + """ + label_array = np.vstack( + [np.asarray(table[n], dtype=float) for n in label_names]).T + + # Combine the data filter with per-label finiteness, then split. + finite = np.isfinite(label_array).all(axis=1) + use = base_mask & finite + n_use = int(use.sum()) + if n_use < 5: + raise ValueError("only {0} usable stars after filtering".format(n_use)) + + idx = np.where(use)[0] + sub_labels = label_array[idx] + sub_flux = np.asarray(norm_flux)[idx] + sub_ivar = np.asarray(norm_ivar)[idx] + sub_err = None if label_err is None else np.asarray(label_err)[idx] + + train_set, validate_set = make_split( + sub_labels, train_frac, validate_frac, seed) + if train_set.sum() == 0 or validate_set.sum() == 0: + raise ValueError("empty train/validation fold; adjust fractions/seed") + + vectorizer = tc.vectorizer.PolynomialVectorizer( + label_names=label_names, order=order) + model = tc.CannonModel( + sub_labels[train_set], sub_flux[train_set], sub_ivar[train_set], + vectorizer, dispersion=dispersion, regularization=reg, + training_set_label_err=(None if sub_err is None else sub_err[train_set])) + model.train(progressbar=False) + + # Test step: recover labels for the held-out spectra. + val_flux = sub_flux[validate_set] + val_ivar = sub_ivar[validate_set] + truth = sub_labels[validate_set] + recovered, _, meta = model.test(val_flux, val_ivar, progressbar=False) + recovered = np.asarray(recovered) + test_r_chi_sq = np.array([m["r_chi_sq"] for m in meta], dtype=float) + + # Reconstruct the validation spectra from the recovered (and reference) + # labels and score the reconstruction. + L = len(label_names) + recon_recovered = reconstruct(model, recovered) + recon_reference = reconstruct(model, truth) + _, red_chi_recovered, _ = reconstruction_chisq( + val_flux, val_ivar, recon_recovered, L) + _, red_chi_reference, _ = reconstruction_chisq( + val_flux, val_ivar, recon_reference, L) + + # --- metric row --- + residual = recovered - truth + row = dict(tag=tag, label_set="+".join(label_names), n_labels=L, + order=order, regularization=reg, + n_train=int(train_set.sum()), n_val=int(validate_set.sum()), + n_used=n_use) + scatters = [] + for i, name in enumerate(label_names): + r = residual[:, i] + r = r[np.isfinite(r)] + row["bias_{0}".format(name)] = float(np.mean(r)) if r.size else np.nan + row["scatter_{0}".format(name)] = float(np.std(r)) if r.size else np.nan + row["rmse_{0}".format(name)] = \ + float(np.sqrt(np.mean(r ** 2))) if r.size else np.nan + scatters.append(row["scatter_{0}".format(name)]) + row["mean_scatter"] = float(np.nanmean(scatters)) + row["median_test_r_chi_sq"] = float(np.nanmedian(test_r_chi_sq)) + row["median_recon_r_chi_sq_recovered"] = float(np.nanmedian(red_chi_recovered)) + row["median_recon_r_chi_sq_reference"] = float(np.nanmedian(red_chi_reference)) + try: + row["frac_in_hull"] = float(np.mean(model.in_convex_hull(truth))) + except Exception: + row["frac_in_hull"] = np.nan + + # --- figures --- + os.makedirs(output_dir, exist_ok=True) + title = "{0} | order={1} | reg={2:g}".format(row["label_set"], order, reg) + + fig = spread_figure(recovered, truth, label_names, title=title) + if fig is not None: + fig.savefig(os.path.join(output_dir, "spread__{0}.png".format(tag)), + dpi=150) + plt.close(fig) + + if wmin is None or wmax is None: + wmin, wmax = _default_window(dispersion) + # Plot the n_plot validation stars spanning the reconstruction-chi2 range + # (best, worst, and evenly spaced in between) so the figure is representative. + finite_chi = np.where(np.isfinite(red_chi_recovered))[0] + if finite_chi.size: + ordered = finite_chi[np.argsort(red_chi_recovered[finite_chi])] + pick = np.unique(np.linspace( + 0, len(ordered) - 1, min(n_plot, len(ordered))).astype(int)) + star_indices = ordered[pick].tolist() + fig = spectra_figure( + dispersion, val_flux, val_ivar, recon_recovered, recon_reference, + star_indices, red_chi_recovered, wmin, wmax, + title="reconstruction | " + title) + if fig is not None: + fig.savefig( + os.path.join(output_dir, "reconstruction__{0}.png".format(tag)), + dpi=150) + plt.close(fig) + + logger.info("%s -> mean_scatter=%.4f recon_chi2(recovered)=%.2f " + "recon_chi2(reference)=%.2f", tag, row["mean_scatter"], + row["median_recon_r_chi_sq_recovered"], + row["median_recon_r_chi_sq_reference"]) + return row + + +# --------------------------------------------------------------------------- # +# The experiment grid # +# --------------------------------------------------------------------------- # + +def experiment(table, dispersion, norm_flux, norm_ivar, label_sets, orders, + regularizations, snr_cutoffs=(None,), filters=(None,), snr_column="snr", + train_frac=0.1, validate_frac=0.1, seed=888, output_dir=".", + n_plot=3, wmin=None, wmax=None, label_err=None): + """ + Loop the full (filter x snr_cutoff x label_set x order x reg) grid, calling + :func:`run_one` for each, and collect a tidy results table. Failures are + recorded (status column) without stopping the grid. + """ + rows = [] + n_combo = (len(filters) * len(snr_cutoffs) * len(label_sets) + * len(orders) * len(regularizations)) + logger.info("Running %d experiments", n_combo) + + i = 0 + for query in filters: + for cutoff in snr_cutoffs: + base_mask = filter_mask(table, snr_column, cutoff, query) + logger.info("filter=%r snr>%s -> %d stars", query, cutoff, + int(base_mask.sum())) + for label_set in label_sets: + for order in orders: + for reg in regularizations: + i += 1 + tag = "{0}__snr{1}__{2}__o{3}__reg{4:g}".format( + _tag(query), "none" if cutoff is None else cutoff, + _tag("+".join(label_set)), order, reg) + meta = dict(filter=("" if query is None else query), + snr_cutoff=("" if cutoff is None else cutoff)) + try: + row = run_one( + table, dispersion, norm_flux, norm_ivar, + list(label_set), order, reg, base_mask, + train_frac, validate_frac, seed, output_dir, tag, + n_plot=n_plot, wmin=wmin, wmax=wmax, + label_err=label_err) + row.update(meta) + row["status"] = "ok" + except Exception as exc: # keep the grid alive + logger.warning("[%d/%d] %s FAILED: %s", + i, n_combo, tag, exc) + row = dict(tag=tag, label_set="+".join(label_set), + order=order, regularization=reg, + status="error: {0}".format(exc)) + row.update(meta) + rows.append(row) + + csv_path = os.path.join(output_dir, "experiment_results.csv") + os.makedirs(output_dir, exist_ok=True) + _write_csv(rows, csv_path) + logger.info("Wrote %d rows to %s", len(rows), csv_path) + + # Console summary of the recovered spread + reconstruction chi-squared. + print("\n=== experiment summary ===") + cols = ("tag", "n_train", "n_val", "mean_scatter", + "median_recon_r_chi_sq_recovered", + "median_recon_r_chi_sq_reference", "status") + print(" ".join("{0:>10s}".format(c[:18]) for c in cols)) + for r in rows: + print(" ".join("{0:>10}".format( + ("{0:.4g}".format(r[c]) if isinstance(r.get(c), float) + else str(r.get(c, ""))[:18])) for c in cols)) + + try: + import pandas as pd + return pd.DataFrame(rows) + except ImportError: + return rows + + +def _write_csv(rows, path): + """ CSV with a stable union-of-keys header. """ + import csv + fieldnames = [] + for row in rows: + for key in row: + if key not in fieldnames: + fieldnames.append(key) + with open(path, "w", newline="") as fp: + writer = csv.DictWriter(fp, fieldnames=fieldnames) + writer.writeheader() + for row in rows: + writer.writerow(row) + + +# --------------------------------------------------------------------------- # +# Demo (self-contained synthetic data with snr + a population column) # +# --------------------------------------------------------------------------- # + +def _demo(output_dir): + """ + A tiny synthetic data set with a known quadratic label dependence, plus + ``snr`` and ``pop`` columns, so the S/N cutoff and filter knobs are + exercised end-to-end without any data files. The flux is already normalized. + """ + import pandas as pd + + rng = np.random.default_rng(0) + names = ["teff", "logg", "feh"] + N, P = 300, 200 + x = rng.uniform(-1, 1, size=(N, len(names))) + + dispersion = np.linspace(15100.0, 16900.0, P) + A = rng.normal(0, 0.05, size=(P, len(names))) + B = rng.normal(0, 0.01, size=(P, len(names))) + flux = 1.0 + x @ A.T + (x ** 2) @ B.T + + snr = rng.uniform(50, 400, size=N) + noise = rng.normal(0, 1.0 / snr[:, None], size=(N, P)) + flux = flux + noise + ivar = np.broadcast_to((snr[:, None]) ** 2, (N, P)).astype(float) + + table = pd.DataFrame({names[0]: 4800 + 600 * x[:, 0], + names[1]: 2.5 + 1.0 * x[:, 1], + names[2]: -0.2 + 0.4 * x[:, 2], + "snr": snr, + "pop": rng.integers(0, 3, size=N)}) + + print("Demo: {0} synthetic stars x {1} pixels (snr in [50, 400], " + "pop in {{0,1,2}})".format(N, P)) + return experiment( + table, dispersion, flux, ivar, + label_sets=[tuple(names)], + orders=[1, 2], + regularizations=[0.0, 1e-4], + snr_cutoffs=[None, 150], + filters=[None, "pop == 2"], + snr_column="snr", + train_frac=0.5, validate_frac=0.5, seed=888, + output_dir=output_dir, n_plot=3) + + +# --------------------------------------------------------------------------- # +# CLI # +# --------------------------------------------------------------------------- # + +def _floats(s): + return [float(x) for x in s.split(",")] + + +def _ints(s): + return [int(x) for x in s.split(",")] + + +def _cutoffs(s): + # "none" or empty -> no cutoff; otherwise a list of floats. + out = [] + for tok in s.split(","): + tok = tok.strip() + out.append(None if tok.lower() in ("", "none") else float(tok)) + return out + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--spectra", + default=os.path.join(DEFAULT_DATA_DIR, + "cleaned_ages.parquet"), + help="parquet table of spectra + labels") + parser.add_argument("--continuum-list", + default=os.path.join(DEFAULT_DATA_DIR, "continuum.list"), + help="text file of continuum pixel indices") + parser.add_argument("--labels", type=lambda s: s.split(","), + default=DEFAULT_LABELS, + help="comma-separated label column names (one label set)") + parser.add_argument("--label-set", dest="label_sets", action="append", + type=lambda s: tuple(s.split(",")), default=None, + help="explicit label set (repeatable); overrides --labels") + parser.add_argument("--orders", type=_ints, default=[1, 2], + help="comma-separated polynomial orders") + parser.add_argument("--regularizations", type=_floats, + default=[0.0], help="comma-separated L1 strengths") + parser.add_argument("--snr-cutoffs", type=_cutoffs, default=[None], + help="comma-separated S/N cutoffs (keep snr > cutoff); " + "use 'none' for no cut, e.g. none,100,150,200") + parser.add_argument("--snr-column", default="snr", + help="name of the S/N column in the table") + parser.add_argument("--filter", dest="filters", action="append", + default=None, + help="pandas query string to keep (repeatable), e.g. " + "\"EvoState == 2\"; omit for no filter") + parser.add_argument("--label-err", dest="label_err_cols", + type=lambda s: s.split(","), default=None, + help="comma-separated per-label 1-sigma error columns " + "(aligned to the label set) to train with label " + "uncertainties") + parser.add_argument("--train-frac", type=float, default=0.1) + parser.add_argument("--validate-frac", type=float, default=0.1) + parser.add_argument("--seed", type=int, default=888) + parser.add_argument("--n-plot", type=int, default=3, + help="number of example reconstructed spectra to plot") + parser.add_argument("--plot-wmin", type=float, default=None, + help="min wavelength for the reconstruction plots") + parser.add_argument("--plot-wmax", type=float, default=None, + help="max wavelength for the reconstruction plots") + parser.add_argument("--output-dir", default="experiment_results") + parser.add_argument("--demo", action="store_true", + help="run a self-contained synthetic experiment") + parser.add_argument("-v", "--verbose", action="store_true") + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO if args.verbose else logging.WARNING, + format="%(asctime)s [%(levelname)s] %(message)s") + + if args.demo: + _demo(args.output_dir) + return + + table, dispersion, flux, ivar = load_spectra(args.spectra) + norm_flux, norm_ivar = normalize_spectra( + dispersion, flux, ivar, args.continuum_list) + + label_sets = args.label_sets or [tuple(args.labels)] + filters = args.filters if args.filters else [None] + + # Optional per-label uncertainties: pull the named error columns into an + # (N, K) array aligned with the (single) label set. + label_err = None + if args.label_err_cols: + label_set0 = list(label_sets[0]) + if len(args.label_err_cols) != len(label_set0): + raise ValueError("--label-err must list one error column per label " + "in the (first) label set") + label_err = np.vstack( + [np.asarray(table[c], dtype=float) for c in args.label_err_cols]).T + + experiment( + table, dispersion, norm_flux, norm_ivar, + label_sets=label_sets, orders=args.orders, + regularizations=args.regularizations, snr_cutoffs=args.snr_cutoffs, + filters=filters, snr_column=args.snr_column, + train_frac=args.train_frac, validate_frac=args.validate_frac, + seed=args.seed, output_dir=args.output_dir, n_plot=args.n_plot, + wmin=args.plot_wmin, wmax=args.plot_wmax, label_err=label_err) + + +if __name__ == "__main__": + main() From 6903a9f7b874addf196aad58a01de63cce259337 Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Tue, 9 Jun 2026 23:55:30 +1000 Subject: [PATCH 12/17] Make the test step memory-aware and stop accumulating unused model flux The test step was already batched over stars, but two things made it OOM the device on large models: - the default batch_size ignored the model size, so a model with many pixels / labels used a batch far larger than the device could hold. Cap it to a memory budget (batch_size * per_star, per_star ~ (P, L) Jacobian + (P, T) design), shrinking it automatically; an explicit batch_size still wins. - the per-star model flux (P,) returned by the spectrum fitter is not part of test()'s output, yet lax.scan accumulated it across batches -- wasting device memory of order the whole validation set (S, P). Drop it inside the scan body. Results are unchanged (stars are fit independently; verified batch-size independent). Full suite green. Co-Authored-By: Claude Opus 4.8 (1M context) --- thecannon/model.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/thecannon/model.py b/thecannon/model.py index d1d28cb..9383d8a 100644 --- a/thecannon/model.py +++ b/thecannon/model.py @@ -39,6 +39,14 @@ # machine has more or less headroom. _TRAIN_BATCH_MEM_BYTES = 1 << 30 # 1 GiB +# Same idea for the test step: the per-star working set is dominated by the +# (P, L) residual Jacobian and the (P, T) design products the Levenberg-Marquardt +# solve forms under the vmap, so the default test batch size is shrunk to keep +# batch_size * per_star within roughly this budget. Kept conservative because +# the LM workspace holds several such copies and the compiled scan body also +# drives a device-side CUDA graph whose size grows with the batch. +_TEST_BATCH_MEM_BYTES = 1 << 29 # 512 MiB + def requires_training(method): """ @@ -980,7 +988,22 @@ def test(self, flux, ivar, initial_labels=None, threads=None, # progress bar. Stars are fit independently (vmap within each batch); # the batching is only loop granularity and does not change results. if batch_size is None: + # Granularity default: ~50 progress updates. batch_size = max(1, int(np.ceil(S / 50))) + # ...but cap it to the memory budget. Each star forms ~ (P, L) + # residual Jacobians and (P, T) design products under the vmap, and + # the LM solve holds several such copies, so bound + # batch_size * per_star. This shrinks the batch automatically for + # models with many pixels / labels instead of OOMing the device. + T = self.design_matrix.shape[1] + per_star_bytes = 8 * P * (L + T + 8) + cap = max(1, int(_TEST_BATCH_MEM_BYTES // per_star_bytes)) + if cap < batch_size: + logger.info("Capping test batch_size %d -> %d to keep the " + "per-batch working set within ~%.0f MiB " + "(P=%d, L=%d, T=%d)", batch_size, cap, + _TEST_BATCH_MEM_BYTES / (1 << 20), P, L, T) + batch_size = cap batch_size = int(min(max(1, batch_size), S)) n_batches = int(np.ceil(S / batch_size)) @@ -1002,7 +1025,11 @@ def test(self, flux, ivar, initial_labels=None, threads=None, def fit_batch(carry, x): _i, fb, ib, lb = x - return carry, spectra(fb, ib, lb) + op_labels, cov, chi_sq, _model_flux, n_use = spectra(fb, ib, lb) + # Drop the per-star model flux (P,): ``test`` does not return it, so + # accumulating it across the scan would waste device memory of order + # the whole validation set (S, P) for nothing. + return carry, (op_labels, cov, chi_sq, n_use) if progressbar: fit_batch = scan_tqdm(n_batches, desc="Testing")(fit_batch) @@ -1014,7 +1041,7 @@ def run_test_scan(xs): xs = (jnp.arange(n_batches), reshape_batches(flux_j), reshape_batches(ivar_j), reshape_batches(init_j)) - op_labels, cov, chi_sq, model_flux, n_use = run_test_scan(xs) + op_labels, cov, chi_sq, n_use = run_test_scan(xs) # Flatten the batch axis back onto the star axis and drop padding. unbatch = lambda a: a.reshape((n_batches * batch_size,) + a.shape[2:])[:S] From c995385915cc2e5fc07eb0c60b8cecf57bfc2233 Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Wed, 10 Jun 2026 08:57:58 +1000 Subject: [PATCH 13/17] Lower default test memory budget and add --test-batch-size to the scripts - model.py: drop the default test working-set budget from 512 to 256 MiB so the memory-aware test batch_size is more conservative out of the box. - train_cannon.py / experiment_cannon.py: add a --test-batch-size knob threaded into model.test(...), so the test batch can be hand-lowered when the device OOMs (overrides the auto default). Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/experiment_cannon.py | 16 +++++++++++----- scripts/train_cannon.py | 14 ++++++++++---- thecannon/model.py | 2 +- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/scripts/experiment_cannon.py b/scripts/experiment_cannon.py index fd750d6..ef2bc4a 100644 --- a/scripts/experiment_cannon.py +++ b/scripts/experiment_cannon.py @@ -200,7 +200,7 @@ def spectra_figure(dispersion, flux, ivar, recovered_flux, reference_flux, def run_one(table, dispersion, norm_flux, norm_ivar, label_names, order, reg, base_mask, train_frac, validate_frac, seed, output_dir, tag, - n_plot=3, wmin=None, wmax=None, label_err=None): + n_plot=3, wmin=None, wmax=None, label_err=None, test_batch_size=None): """ Train + validate one (filter, cutoff, label_set, order, reg) combination, then reconstruct the validation spectra and write the spread + reconstruction @@ -239,7 +239,8 @@ def run_one(table, dispersion, norm_flux, norm_ivar, label_names, order, reg, val_flux = sub_flux[validate_set] val_ivar = sub_ivar[validate_set] truth = sub_labels[validate_set] - recovered, _, meta = model.test(val_flux, val_ivar, progressbar=False) + recovered, _, meta = model.test(val_flux, val_ivar, progressbar=False, + batch_size=test_batch_size) recovered = np.asarray(recovered) test_r_chi_sq = np.array([m["r_chi_sq"] for m in meta], dtype=float) @@ -321,7 +322,7 @@ def run_one(table, dispersion, norm_flux, norm_ivar, label_names, order, reg, def experiment(table, dispersion, norm_flux, norm_ivar, label_sets, orders, regularizations, snr_cutoffs=(None,), filters=(None,), snr_column="snr", train_frac=0.1, validate_frac=0.1, seed=888, output_dir=".", - n_plot=3, wmin=None, wmax=None, label_err=None): + n_plot=3, wmin=None, wmax=None, label_err=None, test_batch_size=None): """ Loop the full (filter x snr_cutoff x label_set x order x reg) grid, calling :func:`run_one` for each, and collect a tidy results table. Failures are @@ -353,7 +354,8 @@ def experiment(table, dispersion, norm_flux, norm_ivar, label_sets, orders, list(label_set), order, reg, base_mask, train_frac, validate_frac, seed, output_dir, tag, n_plot=n_plot, wmin=wmin, wmax=wmax, - label_err=label_err) + label_err=label_err, + test_batch_size=test_batch_size) row.update(meta) row["status"] = "ok" except Exception as exc: # keep the grid alive @@ -508,6 +510,9 @@ def main(): parser.add_argument("--train-frac", type=float, default=0.1) parser.add_argument("--validate-frac", type=float, default=0.1) parser.add_argument("--seed", type=int, default=888) + parser.add_argument("--test-batch-size", type=int, default=None, + help="spectra fit per batch in the test step; lower it " + "if the device OOMs (default: memory-aware auto)") parser.add_argument("--n-plot", type=int, default=3, help="number of example reconstructed spectra to plot") parser.add_argument("--plot-wmin", type=float, default=None, @@ -553,7 +558,8 @@ def main(): filters=filters, snr_column=args.snr_column, train_frac=args.train_frac, validate_frac=args.validate_frac, seed=args.seed, output_dir=args.output_dir, n_plot=args.n_plot, - wmin=args.plot_wmin, wmax=args.plot_wmax, label_err=label_err) + wmin=args.plot_wmin, wmax=args.plot_wmax, label_err=label_err, + test_batch_size=args.test_batch_size) if __name__ == "__main__": diff --git a/scripts/train_cannon.py b/scripts/train_cannon.py index 7c305d9..5006413 100644 --- a/scripts/train_cannon.py +++ b/scripts/train_cannon.py @@ -180,7 +180,7 @@ def one_to_one_figure(truth, predicted, label_names, title=None): def run(labels, normalized_flux, normalized_ivar, dispersion, label_names, order=2, regularization=0, train_frac=0.1, validate_frac=0.1, seed=888, - output_dir=".", save_model=None): + output_dir=".", save_model=None, test_batch_size=None): """ Train on the training split, test on the validation split, and write the one-to-one figure + predictions CSV. ``labels`` is anything CannonModel @@ -211,7 +211,8 @@ def run(labels, normalized_flux, normalized_ivar, dispersion, label_names, np.asarray(model.theta).shape) predicted, _, _ = model.test( - normalized_flux[validate_set], normalized_ivar[validate_set]) + normalized_flux[validate_set], normalized_ivar[validate_set], + batch_size=test_batch_size) predicted = np.asarray(predicted) truth = label_array[validate_set] @@ -257,7 +258,8 @@ def _run_real(args): spectra, normalized_flux, normalized_ivar, dispersion, args.labels, order=args.order, regularization=args.regularization, train_frac=args.train_frac, validate_frac=args.validate_frac, - seed=args.seed, output_dir=args.output_dir, save_model=args.save_model) + seed=args.seed, output_dir=args.output_dir, save_model=args.save_model, + test_batch_size=args.test_batch_size) def _run_demo(args): @@ -281,7 +283,8 @@ def _run_demo(args): labels, meta["flux"], meta["ivar"], meta["dispersion"], names, order=args.order, regularization=args.regularization, train_frac=0.5, validate_frac=0.5, seed=args.seed, - output_dir=args.output_dir, save_model=args.save_model) + output_dir=args.output_dir, save_model=args.save_model, + test_batch_size=args.test_batch_size) def main(): @@ -311,6 +314,9 @@ def main(): help="directory for the figure and predictions CSV") parser.add_argument("--save-model", default=None, help="optional path to write the trained .model file") + parser.add_argument("--test-batch-size", type=int, default=None, + help="spectra fit per batch in the test step; lower it " + "if the device OOMs (default: memory-aware auto)") parser.add_argument("--demo", action="store_true", help="run on the bundled golden data instead") parser.add_argument("-v", "--verbose", action="store_true", diff --git a/thecannon/model.py b/thecannon/model.py index 9383d8a..0dcbf0f 100644 --- a/thecannon/model.py +++ b/thecannon/model.py @@ -45,7 +45,7 @@ # batch_size * per_star within roughly this budget. Kept conservative because # the LM workspace holds several such copies and the compiled scan body also # drives a device-side CUDA graph whose size grows with the batch. -_TEST_BATCH_MEM_BYTES = 1 << 29 # 512 MiB +_TEST_BATCH_MEM_BYTES = 1 << 28 # 256 MiB def requires_training(method): From 39c7de91c6338deefa7787324438a399137706a5 Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Wed, 10 Jun 2026 16:44:09 +1000 Subject: [PATCH 14/17] Reuse compiled train/test programs so the sweep runs fast sequentially on GPU Previously every train()/test() call built fresh @jax.jit closures with the design matrix and trained theta baked in as compile-time constants, so a hyper-parameter sweep recompiled the XLA programs for every fold of every grid point -- on a GPU the compiles dominate wall-clock for a sequential sweep. - Hoist the train/test lax.scan runners to module level, memoize them per (fitter, progress bar), and pass everything that varies between calls (design matrix, theta/s2/fiducials/scales, data batches) as arguments, so same-shape calls hit JAX's jit cache: all folds and all regularization strengths of a grid point now share one compiled program. - Memoize the per-pixel fitter factories and key the per-spectrum core on the vectorizer's terms so repeated factory calls return the same function object. - Move the initial-theta stack to a module-level jitted function. - cross_validate: one vectorizer for all folds and no per-fold tqdm (the host callbacks would be baked into the scan and force recompiles). - run_sweep: enable JAX's persistent on-disk compilation cache (programs now contain no array constants, so identical shapes hit it across processes and restarted jobs); add --jax-cache-dir. - Add scripts/sweep_gpu.pbs: single-GPU gpuvolta job that runs the sweep sequentially (each grid point already saturates the device via vmap), with a backend assertion, no-prealloc, and the shared compilation cache. Demo sweep: scan compiles 48 -> 24 (regs now share programs), identical results; cross-process warm start halves a small train step on CPU. Co-Authored-By: Claude Fable 5 --- scripts/run_sweep.py | 28 +++++++ scripts/sweep_cannon.py | 12 ++- scripts/sweep_gpu.pbs | 85 +++++++++++++++++++ thecannon/fitting.py | 63 ++++++++++++++ thecannon/model.py | 177 +++++++++++++++++++++++++++------------- 5 files changed, 307 insertions(+), 58 deletions(-) create mode 100644 scripts/sweep_gpu.pbs diff --git a/scripts/run_sweep.py b/scripts/run_sweep.py index 80a06bd..b61c258 100644 --- a/scripts/run_sweep.py +++ b/scripts/run_sweep.py @@ -54,6 +54,25 @@ logger = logging.getLogger("thecannon.run_sweep") +def enable_jax_compilation_cache(cache_dir): + """ + Turn on JAX's persistent (on-disk) compilation cache. The sweep's jitted + train/test programs take every model-specific array as an argument, so + grid points that share shapes produce byte-identical HLO -- the first run + of a sweep pays the XLA compilation cost once per shape, and every later + fold, regularization strength, restarted job, or re-run hits this cache + instead of recompiling. + """ + import jax + cache_dir = os.path.abspath(os.path.expanduser(cache_dir)) + os.makedirs(cache_dir, exist_ok=True) + jax.config.update("jax_compilation_cache_dir", cache_dir) + # Persist every program that takes >= 1 s to compile, regardless of size. + jax.config.update("jax_persistent_cache_min_compile_time_secs", 1.0) + jax.config.update("jax_persistent_cache_min_entry_size_bytes", -1) + logger.info("JAX persistent compilation cache: %s", cache_dir) + + def build_label_sets(base_core, age_cols, mass_col, abundances, mode): """ Build the grid of label sets from a fixed core, one or more age columns @@ -188,6 +207,12 @@ def main(): parser.add_argument("--wandb-mode", default="offline", choices=["offline", "online", "disabled"], help="W&B run mode (default: offline)") + parser.add_argument("--jax-cache-dir", + default=os.environ.get("JAX_COMPILATION_CACHE_DIR", + "~/.cache/thecannon-jax"), + help="persistent XLA compilation cache directory " + "(reused across folds/grid points/jobs); pass an " + "empty string to disable") parser.add_argument("--demo", action="store_true", help="run on the bundled golden data instead") parser.add_argument("-v", "--verbose", action="store_true", @@ -198,6 +223,9 @@ def main(): level=logging.INFO if args.verbose else logging.WARNING, format="%(asctime)s [%(levelname)s] %(message)s") + if args.jax_cache_dir: + enable_jax_compilation_cache(args.jax_cache_dir) + if args.demo: import pickle golden_path = os.path.join( diff --git a/scripts/sweep_cannon.py b/scripts/sweep_cannon.py index f0bfc49..ca7c058 100644 --- a/scripts/sweep_cannon.py +++ b/scripts/sweep_cannon.py @@ -146,17 +146,23 @@ def cross_validate(label_array, flux, ivar, dispersion, label_names, order, r_chi_sq = np.full(N, np.nan, dtype=float) in_hull = np.zeros(N, dtype=bool) + # One vectorizer for every fold (it is stateless), and no per-fold progress + # bars: tqdm host callbacks get baked into the compiled scan, which would + # force a recompile per fold instead of reusing one program on the device. + vectorizer = PolynomialVectorizer(label_names=label_names, order=order) + train_kwds = {"progressbar": False, **(train_kwds or {})} + test_kwds = {"progressbar": False, **(test_kwds or {})} + for train_idx, test_idx in _kfold_indices(N, n_splits, seed): - vectorizer = PolynomialVectorizer(label_names=label_names, order=order) model = CannonModel( label_array[train_idx], flux[train_idx], ivar[train_idx], vectorizer, dispersion=dispersion, regularization=regularization, censors=censors) - model.train(progressbar=False, **(train_kwds or {})) + model.train(**train_kwds) op_labels, _, meta = model.test( - flux[test_idx], ivar[test_idx], **(test_kwds or {})) + flux[test_idx], ivar[test_idx], **test_kwds) recovered[test_idx] = op_labels r_chi_sq[test_idx] = [m["r_chi_sq"] for m in meta] diff --git a/scripts/sweep_gpu.pbs b/scripts/sweep_gpu.pbs new file mode 100644 index 0000000..4edeea6 --- /dev/null +++ b/scripts/sweep_gpu.pbs @@ -0,0 +1,85 @@ +#!/bin/bash +# =========================================================================== +# The Cannon hyper-parameter sweep -- single-GPU PBS job. +# +# The sweep runs the grid points *sequentially* on one GPU: each train/test +# step is internally vectorized (jax.vmap over pixels/stars), so a single +# grid point already saturates the device and running combinations in +# parallel would only contend for it. The compiled train/test programs are +# reused across folds, regularization strengths and same-shape grid points, +# and the persistent XLA cache (JAX_COMPILATION_CACHE_DIR) carries the +# compilations across restarted/repeated jobs, so only the first encounter +# of each new shape pays a compile. +# +# Queue/resource notes (NCI Gadi): gpuvolta charges 12 CPUs per GPU, so +# ncpus must be 12 * ngpus; memory up to ~95GB per GPU. Adjust for other +# PBS sites. The Python environment must have a CUDA-enabled jaxlib +# (e.g. `pip install "jax[cuda12]"`). +# +# Submit with: qsub scripts/sweep_gpu.pbs +# =========================================================================== +#PBS -N cannon-sweep-gpu +#PBS -P REPLACE_WITH_PROJECT +#PBS -q gpuvolta +#PBS -l ngpus=1 +#PBS -l ncpus=12 +#PBS -l mem=95GB +#PBS -l walltime=04:00:00 +#PBS -l storage=scratch/REPLACE_WITH_PROJECT+gdata/REPLACE_WITH_PROJECT +#PBS -l wd +#PBS -j oe + +set -uo pipefail + +# ---- user configuration --------------------------------------------------- +PROJECT_DIR="${PBS_O_WORKDIR:-$PWD}" +# Absolute path to the Python interpreter (conda/venv) that has thecannon with +# a CUDA jaxlib + wandb installed. +PYTHON_BIN="python" +WANDB_PROJECT="cannon-sweep" +SWEEP_CMD="${PYTHON_BIN} -m scripts.run_sweep --wandb-project ${WANDB_PROJECT} --wandb-mode offline -v" +# --------------------------------------------------------------------------- + +cd "${PROJECT_DIR}" + +# Persistent XLA compilation cache, shared across jobs. Keep it on /scratch +# (not $HOME, which has a small quota) so repeated sweeps skip compilation. +export JAX_COMPILATION_CACHE_DIR="${PROJECT_DIR}/.jax-cache" +mkdir -p "${JAX_COMPILATION_CACHE_DIR}" + +# Let JAX pick the GPU; fall back loudly (rather than silently training on +# CPU for hours) if the CUDA backend is unavailable. +unset JAX_PLATFORMS +${PYTHON_BIN} - <<'EOF' +import jax +devices = jax.devices() +print("JAX backend:", jax.default_backend(), devices) +assert jax.default_backend() == "gpu", \ + "no GPU backend -- is a CUDA jaxlib installed in this environment?" +EOF + +# The default is to grab 75% of GPU memory up front; the sweep's batch sizes +# are already memory-budgeted, so let allocations grow as needed instead and +# leave headroom for the largest grid points. +export XLA_PYTHON_CLIENT_PREALLOCATE=false + +# Keep W&B fully offline on the compute node; runs are uploaded later on copyq. +export WANDB_MODE=offline +export WANDB_DIR="${PROJECT_DIR}/wandb" +mkdir -p "${WANDB_DIR}" + +echo "[$(date)] starting GPU sweep on $(hostname)" +${SWEEP_CMD} +sweep_rc=$? +echo "[$(date)] sweep finished with exit code ${sweep_rc}" + +# Spawn the copyq job to upload the offline runs (it has external network +# access). Whatever offline runs exist are synced, even if the sweep partially +# failed, so we submit regardless of sweep_rc. +echo "[$(date)] submitting copyq sync job" +sync_job=$(qsub \ + -v "PROJECT_DIR=${PROJECT_DIR},PYTHON_BIN=${PYTHON_BIN},WANDB_DIR=${WANDB_DIR}" \ + "${PROJECT_DIR}/scripts/sync_wandb.pbs") +echo "[$(date)] submitted sync job: ${sync_job}" + +exit ${sweep_rc} diff --git a/thecannon/fitting.py b/thecannon/fitting.py index 226afd0..8cfdc1c 100644 --- a/thecannon/fitting.py +++ b/thecannon/fitting.py @@ -27,6 +27,7 @@ import numpy as np import jax import jax.numpy as jnp +from functools import lru_cache from jax import lax from time import time @@ -255,6 +256,11 @@ def make_pixel_fitter(op_method="l_bfgs_b", maxiter=_TRAIN_MAXITER, Build a pure, ``jax.vmap``-able function that fits the theta coefficients and noise residual for a single pixel. + Unbounded fitters are memoized on ``(op_method, maxiter, tol)`` so that + repeated calls (e.g. one per cross-validation fold or sweep grid point) + return the *same* function object and JAX's jit cache can reuse the + compiled program instead of recompiling. + :param op_method: [optional] The optimization method. ``"l_bfgs_b"`` (default) minimizes the combined ``chi_sq + lambda * ||theta[1:]||_1`` objective with L-BFGS @@ -284,6 +290,21 @@ def make_pixel_fitter(op_method="l_bfgs_b", maxiter=_TRAIN_MAXITER, raise ValueError("unknown optimization method '{}' -- 'l_bfgs_b' or " "'proximal' are available".format(op_method)) + if bounds is None: + # Bounds are arrays (unhashable), so only the unbounded fitters are + # memoized; bounded models (RestrictedCannonModel) rebuild each time. + return _make_unbounded_pixel_fitter(op_method, maxiter, tol) + + return _build_pixel_fitter(op_method, maxiter, tol, bounds) + + +@lru_cache(maxsize=None) +def _make_unbounded_pixel_fitter(op_method, maxiter, tol): + return _build_pixel_fitter(op_method, maxiter, tol, None) + + +def _build_pixel_fitter(op_method, maxiter, tol, bounds): + if bounds is not None: lower, upper = (jnp.asarray(bounds[0]), jnp.asarray(bounds[1])) if op_method == "proximal": @@ -346,12 +367,15 @@ def smooth_objective(theta): return fit +@lru_cache(maxsize=None) def make_pixel_closed_form(): """ Build a pure, ``jax.vmap``-able function that fits a single pixel in closed form. With no regularization and no box constraints the pixel objective is a convex quadratic (weighted least squares), so its minimum is the normal-equations solution and the iterative optimizer is unnecessary. + Memoized: every call returns the same function object so JAX's jit cache + can reuse the compiled program across folds and sweep grid points. :returns: A function ``fit(flux, ivar, design_matrix, column_mask)`` returning @@ -658,6 +682,36 @@ def fit_pixel_fixed_scatter(flux, ivar, initial_thetas, design_matrix, # Per-spectrum label inference (test step) # # --------------------------------------------------------------------------- # +def _freeze_terms(terms): + """ A hashable (nested-tuple) copy of a vectorizer's ``terms`` structure. """ + return tuple(tuple(tuple(t) for t in term) for term in terms) + + +def spectrum_fitter_core(vectorizer, maxiter=_TEST_MAXITER, tol=_TEST_TOL): + """ + Return the pure, ``jax.vmap``-able single-spectrum fitter + + ``core(flux, ivar, initial_labels, theta, s2, fiducials, scales)`` + + for the given vectorizer. The trained-model arrays are call-time arguments + (not closed-over constants), and the core is memoized on the vectorizer's + ``terms``, so models that share a vectorizer structure -- e.g. every + cross-validation fold of a sweep grid point -- reuse one compiled program + instead of recompiling per trained model. + """ + key = (type(vectorizer).__name__, _freeze_terms(vectorizer.terms), + maxiter, tol) + try: + return _SPECTRUM_FITTER_CACHE[key] + except KeyError: + core = _build_spectrum_fitter(vectorizer, maxiter, tol) + _SPECTRUM_FITTER_CACHE[key] = core + return core + + +_SPECTRUM_FITTER_CACHE = {} + + def make_spectrum_fitter(vectorizer, theta, s2, fiducials, scales, maxiter=_TEST_MAXITER, tol=_TEST_TOL): """ @@ -670,12 +724,21 @@ def make_spectrum_fitter(vectorizer, theta, s2, fiducials, scales, is a ``(n_init, L)`` array of starting points (the best is selected). """ + fitter = spectrum_fitter_core(vectorizer, maxiter=maxiter, tol=tol) theta = jnp.asarray(theta) s2 = jnp.asarray(s2) fiducials = jnp.asarray(fiducials) scales = jnp.asarray(scales) def core(flux, ivar, initial_labels): + return fitter(flux, ivar, initial_labels, theta, s2, fiducials, scales) + + return core + + +def _build_spectrum_fitter(vectorizer, maxiter, tol): + + def core(flux, ivar, initial_labels, theta, s2, fiducials, scales): adjusted_ivar = ivar / (1. + ivar * s2) diff --git a/thecannon/model.py b/thecannon/model.py index 0dcbf0f..edfc461 100644 --- a/thecannon/model.py +++ b/thecannon/model.py @@ -18,7 +18,7 @@ import os import pickle from datetime import datetime -from functools import wraps +from functools import lru_cache, wraps from sys import version_info from scipy.spatial import Delaunay from jax_tqdm import scan_tqdm @@ -48,6 +48,106 @@ _TEST_BATCH_MEM_BYTES = 1 << 28 # 256 MiB +# --------------------------------------------------------------------------- # +# Cached scan runners # +# --------------------------------------------------------------------------- # +# +# The train/test steps run a jitted lax.scan whose body vmaps a per-pixel (or +# per-spectrum) fitter. Building that jitted function inside train()/test() +# would hand XLA a *new* Python callable on every call -- so a hyper-parameter +# sweep or k-fold cross-validation would recompile the same program for every +# fold of every grid point, which on a GPU can cost more than the actual +# compute. Instead the runners are built once per (fitter, progress-bar) +# combination and memoized, and everything that varies between calls (the +# design matrix, the trained theta/s2, the data batches) is passed in as an +# argument. Calls that share shapes then hit JAX's jit cache, and -- because +# no large arrays are baked into the HLO as constants -- identical programs +# also hit the persistent compilation cache across processes when +# ``jax.config.jax_compilation_cache_dir`` is set. +# +# ``maxsize`` bounds the number of live jitted programs (relevant for +# errors-in-variables / bounded fitters, which are rebuilt per call and would +# otherwise grow the cache without limit). + +@lru_cache(maxsize=32) +def _training_scan_runner(fitter, closed_form, tqdm_n_batches): + """ + Build the jitted ``run(design_matrix, xs) -> (theta, s2, fopt)`` scan for a + per-pixel ``fitter``. ``tqdm_n_batches`` attaches a jax-tqdm progress bar + when non-zero (it must equal the number of scan steps). + """ + if closed_form: + pixels = jax.vmap(fitter, in_axes=(0, 0, None, 0)) + + def fit_batch(design_matrix, x): + _i, fb, ib, cm = x + return design_matrix, pixels(fb, ib, design_matrix, cm) + else: + pixels = jax.vmap(fitter, in_axes=(0, 0, 0, None, 0, 0)) + + def fit_batch(design_matrix, x): + _i, fb, ib, st, rg, cm = x + return design_matrix, pixels(fb, ib, st, design_matrix, rg, cm) + + if tqdm_n_batches: + fit_batch = scan_tqdm(tqdm_n_batches, desc="Training")(fit_batch) + + @jax.jit + def run(design_matrix, xs): + _, out = jax.lax.scan(fit_batch, design_matrix, xs) + return out + + return run + + +@lru_cache(maxsize=32) +def _test_scan_runner(core, tqdm_n_batches): + """ + Build the jitted ``run(model_state, xs) -> (op_labels, cov, chi_sq, n_use)`` + scan for a per-spectrum ``core``, where ``model_state`` is the + ``(theta, s2, fiducials, scales)`` tuple of the trained model. + """ + spectra = jax.vmap(core, in_axes=(0, 0, 0, None, None, None, None)) + + def fit_batch(model_state, x): + _i, fb, ib, lb = x + op_labels, cov, chi_sq, _model_flux, n_use = spectra( + fb, ib, lb, *model_state) + # Drop the per-star model flux (P,): ``test`` does not return it, so + # accumulating it across the scan would waste device memory of order + # the whole validation set (S, P) for nothing. + return model_state, (op_labels, cov, chi_sq, n_use) + + if tqdm_n_batches: + fit_batch = scan_tqdm(tqdm_n_batches, desc="Testing")(fit_batch) + + @jax.jit + def run(model_state, xs): + _, out = jax.lax.scan(fit_batch, model_state, xs) + return out + + return run + + +@jax.jit +def _initial_theta_stack(flux_PN, ivar_PN, design_matrix): + """ + The ``(P, 2, T)`` stack of initial theta guesses for every pixel: the + linear-algebra estimate and the fiducial value. Module-level and jitted so + repeated same-shape calls reuse the compiled program. + """ + P = flux_PN.shape[0] + T = design_matrix.shape[1] + fiducial = jnp.concatenate([jnp.ones(1), jnp.zeros(T - 1)]) + linalg_theta = jax.vmap( + lambda f, i: fitting.fit_theta_by_linalg( + f, i, 0.0, design_matrix)[0])(flux_PN, ivar_PN) + finite = jnp.all(jnp.isfinite(linalg_theta), axis=1, keepdims=True) + linalg_theta = jnp.where(finite, linalg_theta, fiducial[None, :]) + return jnp.stack( + [linalg_theta, jnp.broadcast_to(fiducial, (P, T))], axis=1) + + def requires_training(method): """ A decorator for model methods that require training before being run. @@ -771,20 +871,9 @@ def train(self, threads=None, op_method=None, op_strict=True, op_kwds=None, # Initial theta guesses for every pixel: a linear-algebra estimate # and the fiducial value. The regularized objective is convex, so # the optimum is independent of the starting point; these only set - # where the optimizer begins. Computed in a single jitted (hence - # fused) call rather than op-by-op. - @jax.jit - def _initial_stack(flux_PN, ivar_PN): - linalg_theta = jax.vmap( - lambda f, i: fitting.fit_theta_by_linalg( - f, i, 0.0, design_matrix)[0])(flux_PN, ivar_PN) - finite = jnp.all( - jnp.isfinite(linalg_theta), axis=1, keepdims=True) - linalg_theta = jnp.where(finite, linalg_theta, fiducial[None, :]) - return jnp.stack( - [linalg_theta, jnp.broadcast_to(fiducial, (P, T))], axis=1) - - init_stack = _initial_stack(flux_PN, ivar_PN) # (P, 2, T) + # where the optimizer begins. + init_stack = _initial_theta_stack( + flux_PN, ivar_PN, design_matrix) # (P, 2, T) if eiv: fitter = fitting.make_pixel_fitter_eiv( @@ -851,33 +940,20 @@ def _initial_stack(flux_PN, ivar_PN): mask_b = reshape_batches(column_mask) if closed_form: - pixels = jax.vmap(fitter, in_axes=(0, 0, None, 0)) - - def fit_batch(carry, x): - _i, fb, ib, cm = x - return carry, pixels(fb, ib, design_matrix, cm) - xs = (jnp.arange(n_batches), flux_b, ivar_b, mask_b) else: init_b = reshape_batches(init_stack) reg_b = reshape_batches(reg_P) - pixels = jax.vmap(fitter, in_axes=(0, 0, 0, None, 0, 0)) - - def fit_batch(carry, x): - _i, fb, ib, st, rg, cm = x - return carry, pixels(fb, ib, st, design_matrix, rg, cm) - xs = (jnp.arange(n_batches), flux_b, ivar_b, init_b, reg_b, mask_b) - if progressbar: - fit_batch = scan_tqdm(n_batches, desc="Training")(fit_batch) + # The runner is memoized on (fitter, closed_form, progress bar) and the + # design matrix is an argument, so same-shape calls -- e.g. every fold + # and regularization strength of a sweep grid point -- reuse one + # compiled program instead of recompiling. + run_training_scan = _training_scan_runner( + fitter, closed_form, n_batches if progressbar else 0) - @jax.jit - def run_training_scan(xs): - _, out = jax.lax.scan(fit_batch, None, xs) - return out - - theta_b, s2_b, fopt_b = run_training_scan(xs) + theta_b, s2_b, fopt_b = run_training_scan(design_matrix, xs) # Flatten the batch axis back onto the pixel axis and drop any padding. # The trained quantities are kept as JAX arrays end-to-end. @@ -980,9 +1056,14 @@ def test(self, flux, ivar, initial_labels=None, threads=None, maxiter = op_kwds.get("maxiter", fitting._TEST_MAXITER) tol = op_kwds.get("tol", fitting._TEST_TOL) - core = fitting.make_spectrum_fitter( - self.vectorizer, self.theta, self.s2, self._fiducials, - self._scales, maxiter=maxiter, tol=tol) + # The core is memoized on the vectorizer's terms and takes the trained + # model arrays as arguments, so every same-shape test call (e.g. each + # cross-validation fold of a sweep grid point) reuses one compiled + # program rather than recompiling with theta baked in as a constant. + core = fitting.spectrum_fitter_core( + self.vectorizer, maxiter=maxiter, tol=tol) + model_state = (jnp.asarray(self.theta), jnp.asarray(self.s2), + jnp.asarray(self._fiducials), jnp.asarray(self._scales)) # Batch over stars so a device-side lax.scan can drive a jax-tqdm # progress bar. Stars are fit independently (vmap within each batch); @@ -1021,27 +1102,13 @@ def test(self, flux, ivar, initial_labels=None, threads=None, init_j = jnp.concatenate([init_j, edge(init_j)], axis=0) reshape_batches = lambda a: a.reshape((n_batches, batch_size) + a.shape[1:]) - spectra = jax.vmap(core, in_axes=(0, 0, 0)) - - def fit_batch(carry, x): - _i, fb, ib, lb = x - op_labels, cov, chi_sq, _model_flux, n_use = spectra(fb, ib, lb) - # Drop the per-star model flux (P,): ``test`` does not return it, so - # accumulating it across the scan would waste device memory of order - # the whole validation set (S, P) for nothing. - return carry, (op_labels, cov, chi_sq, n_use) - - if progressbar: - fit_batch = scan_tqdm(n_batches, desc="Testing")(fit_batch) - @jax.jit - def run_test_scan(xs): - _, out = jax.lax.scan(fit_batch, None, xs) - return out + run_test_scan = _test_scan_runner( + core, n_batches if progressbar else 0) xs = (jnp.arange(n_batches), reshape_batches(flux_j), reshape_batches(ivar_j), reshape_batches(init_j)) - op_labels, cov, chi_sq, n_use = run_test_scan(xs) + op_labels, cov, chi_sq, n_use = run_test_scan(model_state, xs) # Flatten the batch axis back onto the star axis and drop padding. unbatch = lambda a: a.reshape((n_batches * batch_size,) + a.shape[2:])[:S] From eef3ac8b30770ed0e5fcf60f1710589d76ea444b Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Wed, 10 Jun 2026 20:04:18 +1000 Subject: [PATCH 15/17] Drop the mass label from the PBS sweep commands Pass --mass-col= (empty) so build_label_sets produces no base+mass variants; the swept label sets are base(+age) plus abundance additions only. Co-Authored-By: Claude Fable 5 --- scripts/sweep.pbs | 3 ++- scripts/sweep_gpu.pbs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/sweep.pbs b/scripts/sweep.pbs index 2f3dd86..62429a7 100755 --- a/scripts/sweep.pbs +++ b/scripts/sweep.pbs @@ -35,7 +35,8 @@ WANDB_PROJECT="cannon-sweep" # spectra (defaults point at the bulge-ages-and-orbits data) and cross-validates # the grid. Override --spectra/--continuum-list/--labels/--orders/etc. as needed, # or pass --demo to dry-run on the bundled golden data. -SWEEP_CMD="${PYTHON_BIN} -m scripts.run_sweep --wandb-project ${WANDB_PROJECT} --wandb-mode offline -v" +# --mass-col= (empty) disables the base+mass label-set variants. +SWEEP_CMD="${PYTHON_BIN} -m scripts.run_sweep --mass-col= --wandb-project ${WANDB_PROJECT} --wandb-mode offline -v" # --------------------------------------------------------------------------- cd "${PROJECT_DIR}" diff --git a/scripts/sweep_gpu.pbs b/scripts/sweep_gpu.pbs index 4edeea6..f6d67a7 100644 --- a/scripts/sweep_gpu.pbs +++ b/scripts/sweep_gpu.pbs @@ -37,7 +37,8 @@ PROJECT_DIR="${PBS_O_WORKDIR:-$PWD}" # a CUDA jaxlib + wandb installed. PYTHON_BIN="python" WANDB_PROJECT="cannon-sweep" -SWEEP_CMD="${PYTHON_BIN} -m scripts.run_sweep --wandb-project ${WANDB_PROJECT} --wandb-mode offline -v" +# --mass-col= (empty) disables the base+mass label-set variants. +SWEEP_CMD="${PYTHON_BIN} -m scripts.run_sweep --mass-col= --wandb-project ${WANDB_PROJECT} --wandb-mode offline -v" # --------------------------------------------------------------------------- cd "${PROJECT_DIR}" From b3604bdd0f7c2e7901340885a697d11600c27e4b Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Wed, 10 Jun 2026 20:07:48 +1000 Subject: [PATCH 16/17] Sweep abundances as [X/Fe] instead of [X/H] load_spectra now derives a _fe = raw__h - raw_fe_h column for every raw [X/H] abundance except iron itself (never overwriting existing columns), and the sweep defaults use them: the base set carries mg_fe and the swept abundances are ce/ca/si/ni/mn/al/c/n over Fe. raw_fe_h itself stays [Fe/H]. Co-Authored-By: Claude Fable 5 --- scripts/run_sweep.py | 13 +++++++------ scripts/train_cannon.py | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/scripts/run_sweep.py b/scripts/run_sweep.py index b61c258..d150a9a 100644 --- a/scripts/run_sweep.py +++ b/scripts/run_sweep.py @@ -20,9 +20,8 @@ python -m scripts.run_sweep \\ --spectra /path/cleaned_ages.parquet \\ --continuum-list /path/continuum.list \\ - --labels raw_teff,raw_logg,raw_fe_h,raw_mg_h,raw_ce_h,age_L,mass_L \\ --label-set raw_teff,raw_logg,raw_fe_h \\ - --label-set raw_teff,raw_logg,raw_fe_h,raw_mg_h,raw_ce_h \\ + --label-set raw_teff,raw_logg,raw_fe_h,mg_fe,ce_fe \\ --orders 1,2 --regularizations 0,1e2,1e3,1e4 \\ --n-splits 5 --wandb-project cannon-sweep @@ -169,7 +168,7 @@ def main(): default=os.path.join(DEFAULT_DATA_DIR, "continuum.list"), help="text file of continuum pixel indices") parser.add_argument("--base", type=lambda s: s.split(","), - default=["raw_teff", "raw_logg", "raw_fe_h", "raw_mg_h"], + default=["raw_teff", "raw_logg", "raw_fe_h", "mg_fe"], help="comma-separated core labels in every set (the age " "column from --age-cols is appended to this)") parser.add_argument("--age-cols", type=lambda s: s.split(","), @@ -180,9 +179,11 @@ def main(): help="mass column added by the 'age and mass' sets " "(empty string to disable)") parser.add_argument("--abundances", type=lambda s: s.split(","), - default=["raw_ce_h", "raw_ca_h", "raw_si_h", "raw_ni_h", - "raw_mn_h", "raw_al_h", "raw_c_h", "raw_n_h"], - help="comma-separated abundances to test on top of base") + default=["ce_fe", "ca_fe", "si_fe", "ni_fe", + "mn_fe", "al_fe", "c_fe", "n_fe"], + help="comma-separated abundances to test on top of base " + "(the _fe columns are derived from raw__h - " + "raw_fe_h at load time)") parser.add_argument("--label-set-mode", default="one-at-a-time", choices=["one-at-a-time", "cumulative", "minimal"], help="how extras are combined with each base") diff --git a/scripts/train_cannon.py b/scripts/train_cannon.py index 5006413..bd50f2c 100644 --- a/scripts/train_cannon.py +++ b/scripts/train_cannon.py @@ -74,14 +74,47 @@ def _to_array(x): return np.asarray(x, dtype=float) +def add_x_fe_columns(table): + """ + Derive [X/Fe] abundance columns from the raw [X/H] ones: every + ``raw__h`` column except iron itself gains a ``_fe`` counterpart + equal to ``raw__h - raw_fe_h``. Existing columns are never overwritten, + and the table is returned (modified in place) for convenience. + """ + import re + + if "raw_fe_h" not in table.columns: + logger.warning("no raw_fe_h column; cannot derive any [X/Fe] columns") + return table + + fe_h = np.asarray(table["raw_fe_h"], dtype=float) + derived = [] + for column in list(table.columns): + match = re.fullmatch(r"raw_([a-z0-9]+)_h", str(column)) + if match is None or match.group(1) == "fe": + continue + name = "{0}_fe".format(match.group(1)) + if name in table.columns: + continue + table[name] = np.asarray(table[column], dtype=float) - fe_h + derived.append(name) + + if derived: + logger.info("derived [X/Fe] columns: %s", ", ".join(derived)) + return table + + def load_spectra(spectra_path): """ Read the parquet table and assemble ``(spectra, dispersion, flux, ivar)``. - Each of the ``wavelength``/``flux``/``ivar`` columns holds one array per row. + Each of the ``wavelength``/``flux``/``ivar`` columns holds one array per + row. Derived ``_fe`` abundance columns are added alongside the raw + ``raw__h`` ones (see :func:`add_x_fe_columns`). """ import pandas as pd spectra = pd.read_parquet(spectra_path) + add_x_fe_columns(spectra) dispersion = _to_array(spectra["wavelength"].iloc[0]) # (n_pixels,) flux = np.vstack([_to_array(x) for x in spectra["flux"]]) # (n_stars, P) From eb15590f84db94588583ad7be409519632655cf4 Mon Sep 17 00:00:00 2001 From: Maja Jablonska Date: Wed, 10 Jun 2026 20:10:20 +1000 Subject: [PATCH 17/17] Reject flagged spectra before training in the sweep Add quality_mask: stars with spectrum_flags != 0 or any warn_* column set to True are dropped. run_sweep applies it right after loading -- before continuum normalization and training -- and aborts if the cuts reject every star. Missing columns skip the corresponding cut with a warning; NaN warn values count as not-set, NaN spectrum_flags as not-clean. Co-Authored-By: Claude Fable 5 --- scripts/run_sweep.py | 17 +++++++++++++++-- scripts/train_cannon.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/scripts/run_sweep.py b/scripts/run_sweep.py index d150a9a..0b6d42f 100644 --- a/scripts/run_sweep.py +++ b/scripts/run_sweep.py @@ -43,10 +43,11 @@ # directly from the scripts/ directory. try: from scripts.train_cannon import (load_spectra, normalize_spectra, - DEFAULT_DATA_DIR, DEFAULT_LABELS) + quality_mask, DEFAULT_DATA_DIR, + DEFAULT_LABELS) from scripts.sweep_cannon import sweep except ImportError: - from train_cannon import (load_spectra, normalize_spectra, + from train_cannon import (load_spectra, normalize_spectra, quality_mask, DEFAULT_DATA_DIR, DEFAULT_LABELS) from sweep_cannon import sweep @@ -246,6 +247,18 @@ def main(): norm_flux.shape[0], norm_flux.shape[1], names)) else: label_source, dispersion, flux, ivar = load_spectra(args.spectra) + + # Quality cuts before anything is normalized or trained on: drop stars + # with flagged spectra (spectrum_flags != 0) or any warn_* label set. + good = quality_mask(label_source) + if not good.any(): + raise ValueError("quality cuts rejected every star; check the " + "spectrum_flags / warn_* columns") + logger.info("quality cuts: keeping %d/%d stars", + int(good.sum()), good.size) + label_source = label_source[good] + flux, ivar = flux[good], ivar[good] + norm_flux, norm_ivar = normalize_spectra( dispersion, flux, ivar, args.continuum_list) label_sets = args.label_sets or build_label_sets( diff --git a/scripts/train_cannon.py b/scripts/train_cannon.py index bd50f2c..6c137cf 100644 --- a/scripts/train_cannon.py +++ b/scripts/train_cannon.py @@ -104,6 +104,36 @@ def add_x_fe_columns(table): return table +def quality_mask(table): + """ + Boolean mask over the rows of ``table`` selecting stars that pass the + quality cuts: ``spectrum_flags == 0`` (no flagged reduction/calibration + issues) and no ``warn_*`` column set to True. Missing columns skip the + corresponding cut with a warning rather than rejecting everything. + """ + n = len(table) + mask = np.ones(n, dtype=bool) + + if "spectrum_flags" in table.columns: + clean = np.asarray(table["spectrum_flags"].fillna(-1) == 0) + logger.info("quality cut: %d/%d stars rejected by spectrum_flags != 0", + int((~clean).sum()), n) + mask &= clean + else: + logger.warning("no spectrum_flags column; skipping that quality cut") + + warn_columns = [c for c in table.columns if str(c).startswith("warn_")] + if warn_columns: + for column in warn_columns: + mask &= ~np.asarray(table[column].fillna(False), dtype=bool) + logger.info("quality cut: %d/%d stars left after rejecting any of " + "%s set", int(mask.sum()), n, ", ".join(warn_columns)) + else: + logger.warning("no warn_* columns; skipping that quality cut") + + return mask + + def load_spectra(spectra_path): """ Read the parquet table and assemble ``(spectra, dispersion, flux, ivar)``.